How to stream a video or a file considering request and response range headers? - asp.net-core-mvc

I am now using FileStreamResult and it works to stream a video, but can't seek it. It always starts again from the beginning.
I was using ByteRangeStreamContent but it seems that it is not available anymore with dnxcore50.
So how to proceed ?
Do i need to manually parse the request range headers and write a custom FileResult that sets the response Content-Range and the rest of the headers and writes the buffer range to the response body or is there something already implemented and i'm missing it ?

Here is a naive implementation of a VideoStreamResult. I am using at the moment (the multipart content part is not tested):
public class VideoStreamResult : FileStreamResult
{
// default buffer size as defined in BufferedStream type
private const int BufferSize = 0x1000;
private string MultipartBoundary = "<qwe123>";
public VideoStreamResult(Stream fileStream, string contentType)
: base(fileStream, contentType)
{
}
public VideoStreamResult(Stream fileStream, MediaTypeHeaderValue contentType)
: base(fileStream, contentType)
{
}
private bool IsMultipartRequest(RangeHeaderValue range)
{
return range != null && range.Ranges != null && range.Ranges.Count > 1;
}
private bool IsRangeRequest(RangeHeaderValue range)
{
return range != null && range.Ranges != null && range.Ranges.Count > 0;
}
protected async Task WriteVideoAsync(HttpResponse response)
{
var bufferingFeature = response.HttpContext.Features.Get<IHttpBufferingFeature>();
bufferingFeature?.DisableResponseBuffering();
var length = FileStream.Length;
var range = response.HttpContext.GetRanges(length);
if (IsMultipartRequest(range))
{
response.ContentType = $"multipart/byteranges; boundary={MultipartBoundary}";
}
else
{
response.ContentType = ContentType.ToString();
}
response.Headers.Add("Accept-Ranges", "bytes");
if (IsRangeRequest(range))
{
response.StatusCode = (int)HttpStatusCode.PartialContent;
if (!IsMultipartRequest(range))
{
response.Headers.Add("Content-Range", $"bytes {range.Ranges.First().From}-{range.Ranges.First().To}/{length}");
}
foreach (var rangeValue in range.Ranges)
{
if (IsMultipartRequest(range)) // I don't know if multipart works
{
await response.WriteAsync($"--{MultipartBoundary}");
await response.WriteAsync(Environment.NewLine);
await response.WriteAsync($"Content-type: {ContentType}");
await response.WriteAsync(Environment.NewLine);
await response.WriteAsync($"Content-Range: bytes {range.Ranges.First().From}-{range.Ranges.First().To}/{length}");
await response.WriteAsync(Environment.NewLine);
}
await WriteDataToResponseBody(rangeValue, response);
if (IsMultipartRequest(range))
{
await response.WriteAsync(Environment.NewLine);
}
}
if (IsMultipartRequest(range))
{
await response.WriteAsync($"--{MultipartBoundary}--");
await response.WriteAsync(Environment.NewLine);
}
}
else
{
await FileStream.CopyToAsync(response.Body);
}
}
private async Task WriteDataToResponseBody(RangeItemHeaderValue rangeValue, HttpResponse response)
{
var startIndex = rangeValue.From ?? 0;
var endIndex = rangeValue.To ?? 0;
byte[] buffer = new byte[BufferSize];
long totalToSend = endIndex - startIndex;
int count = 0;
long bytesRemaining = totalToSend + 1;
response.ContentLength = bytesRemaining;
FileStream.Seek(startIndex, SeekOrigin.Begin);
while (bytesRemaining > 0)
{
try
{
if (bytesRemaining <= buffer.Length)
count = FileStream.Read(buffer, 0, (int)bytesRemaining);
else
count = FileStream.Read(buffer, 0, buffer.Length);
if (count == 0)
return;
await response.Body.WriteAsync(buffer, 0, count);
bytesRemaining -= count;
}
catch (IndexOutOfRangeException)
{
await response.Body.FlushAsync();
return;
}
finally
{
await response.Body.FlushAsync();
}
}
}
public override async Task ExecuteResultAsync(ActionContext context)
{
await WriteVideoAsync(context.HttpContext.Response);
}
}
And parse request headers range:
public static RangeHeaderValue GetRanges(this HttpContext context, long contentSize)
{
RangeHeaderValue rangesResult = null;
string rangeHeader = context.Request.Headers["Range"];
if (!string.IsNullOrEmpty(rangeHeader))
{
// rangeHeader contains the value of the Range HTTP Header and can have values like:
// Range: bytes=0-1 * Get bytes 0 and 1, inclusive
// Range: bytes=0-500 * Get bytes 0 to 500 (the first 501 bytes), inclusive
// Range: bytes=400-1000 * Get bytes 500 to 1000 (501 bytes in total), inclusive
// Range: bytes=-200 * Get the last 200 bytes
// Range: bytes=500- * Get all bytes from byte 500 to the end
//
// Can also have multiple ranges delimited by commas, as in:
// Range: bytes=0-500,600-1000 * Get bytes 0-500 (the first 501 bytes), inclusive plus bytes 600-1000 (401 bytes) inclusive
// Remove "Ranges" and break up the ranges
string[] ranges = rangeHeader.Replace("bytes=", string.Empty).Split(",".ToCharArray());
rangesResult = new RangeHeaderValue();
for (int i = 0; i < ranges.Length; i++)
{
const int START = 0, END = 1;
long endByte, startByte;
long parsedValue;
string[] currentRange = ranges[i].Split("-".ToCharArray());
if (long.TryParse(currentRange[END], out parsedValue))
endByte = parsedValue;
else
endByte = contentSize - 1;
if (long.TryParse(currentRange[START], out parsedValue))
startByte = parsedValue;
else
{
// No beginning specified, get last n bytes of file
// We already parsed end, so subtract from total and
// make end the actual size of the file
startByte = contentSize - endByte;
endByte = contentSize - 1;
}
rangesResult.Ranges.Add(new RangeItemHeaderValue(startByte, endByte));
}
}
return rangesResult;
}

FYI, built-in support for range requests will be present in .NET Core 2.1
https://github.com/aspnet/Mvc/pull/6895

Related

How to sort List in dart based on value returned by a function

I have to show a list of stores in a sorted order by nearest location
and I have a unsorted list of stores and a function to calculate
distance of each store from my current location but I don't know how
to sort List in dart based on value returned by a function.
I am getting the unsorted List of stores data from an api.
I need logic for this question for sorting the _kikkleStores List
class KikkleStoresBloc extends BlocBase {
List<KikkleStoreInfo> _kikkleStores = [];
//for distance sorting
List<KikkleStoreInfo> _kikkleStoresSorted = [];
List<double> distanceFromCurrentLocation;//already got
value in it
bool hasReachedEndOfList = false;
Coordinate _currentLocation;
KikkleStoresBloc();
final _kikkleStoreSubject =
BehaviorSubject<List<KikkleStoreInfo>>
();
// Getter Stream
Stream<List<KikkleStoreInfo>> get kikkleStores =>
_kikkleStoreSubject.stream;`enter code here`
// Getter Sink
Function(List<KikkleStoreInfo>) get _fetchedkikkleStores =>
_kikkleStoreSubject.sink.add;
getKikkleStores() async {
try {
if (_currentPage <= _totalPages) {
final response = await
ApiController.getKikkleStores(_currentPage);
_kikkleStores.addAll(response.item1);
//here how to sort _kikkleStores by using
getStoreDistance function
_totalPages = response.item3;
_fetchedkikkleStores(_kikkleStores);
if (_currentPage == _totalPages) {
hasReachedEndOfList = true;
} else if (_currentPage < _totalPages &&
_kikkleStores.length
< 10) {
}
}
}
}
// this function returns distance
getStoreDistance(Coordinate currentLocation, KikkleStoreInfo
store)
async {
if (currentLocation == null) return 0.0;
try {
double distanceInMeter = await
LocationUtils.getDistanceInMeters(
currentLocation, Coordinate(store.latitude,
store.longitude));
// final miles = (distanceInMeter / 1609.344).round();
return distanceInMeter;
} catch (e) {
return 0.0;
}
}
void getCurrentLocation() async {
try {
final isAllowed = await
PermissionsUtils.isLocationAccessAllowed();
if (isAllowed) {
final coordinates = await
LocationUtils.getCurrentLocation();
if (coordinates != null) {
_currentLocation = coordinates;
}
}
}
catch (e) {
print(e);
}
}
}
Take the reference of below code
void main(){
List<POJO> pojo = [POJO(5), POJO(3),POJO(7),POJO(1)];
// fill list
pojo..sort((a, b) => a.id.compareTo(b.id));
for(var i in pojo){
print(i.id); // prints list in sorted order i.e 1 3 5 7
}
}
class POJO {
int id;
POJO(this.id);
}
void sortfun() async {
for (int c = 0; c < (_kikkleStores.length - 1); c++) {
for (int d = 0; d < _kikkleStores.length - c - 1; d++) {
if (await getStoreDistance(_currentLocation, _kikkleStores[d]) >
await getStoreDistance(_currentLocation,
_kikkleStores[d + 1])) /* For descending order use < */
{
swap = _kikkleStores[d];
_kikkleStores[d] = _kikkleStores[d + 1];
_kikkleStores[d + 1] = swap;
}
}
}
}

Spring GCP - Datastore performance: Batch processing, iteration through all entity list is very slow

Following code is work really slow, almost 30 second to process 400 entities:
int page = 0;
org.springframework.data.domain.Page<MyEntity> slice = null;
while (true) {
if (slice == null) {
slice = repo.findAll(PageRequest.of(page, 400, Sort.by("date")));
} else {
slice = repo.findAll(slice.nextPageable());
}
if (!slice.hasNext()) {
break;
}
slice.getContent().forEach(v -> v.setApp(SApplication.NAME_XXX));
repo.saveAll(slice.getContent());
LOGGER.info("processed: " + page);
page++;
}
I use following instead, 4-6 sec per 400 entities (gcp lib to work with datastore)
Datastore service = DatastoreOptions.getDefaultInstance().getService();
StructuredQuery.Builder<?> query = Query.newEntityQueryBuilder();
int limit = 400;
query.setKind("ENTITY_KIND").setLimit(limit);
int count = 0;
Cursor cursor = null;
while (true) {
if (cursor != null) {
query.setStartCursor(cursor);
}
QueryResults<?> queryResult = service.run(query.build());
List<Entity> entityList = new ArrayList<>();
while (queryResult.hasNext()) {
Entity loadEntity = (Entity) queryResult.next();
Entity.Builder newEntity = Entity.newBuilder(loadEntity).set("app", SApplication.NAME_XXX.name());
entityList.add(newEntity.build());
}
service.put(entityList.toArray(new Entity[0]));
count += entityList.size();
if (entityList.size() == limit) {
cursor = queryResult.getCursorAfter();
} else {
break;
}
LOGGER.info("Processed: {}", count);
}
Why I can't use spring to do that batch processing?
Full discussion here: https://github.com/spring-cloud/spring-cloud-gcp/issues/1824
First:
you need to use correct lib version: at least 1.2.0.M2
Second:
you need to implement new method in repository interface:
#Query("select * from your_kind")
Slice<TestEntity> findAllSlice(Pageable pageable);
Final code looks like:
LOGGER.info("start");
int page = 0;
Slice<TestEntity> slice = null;
while (true) {
if (slice == null) {
slice = repo.findAllSlice(DatastorePageable.of(page, 400, Sort.by("date")));
} else {
slice = repo.findAllSlice(slice.nextPageable());
}
if (!slice.hasNext()) {
break;
}
slice.getContent().forEach(v -> v.setApp("xx"));
repo.saveAll(slice.getContent());
LOGGER.info("processed: " + page);
page++;
}
LOGGER.info("end");

How to get File in Content-Disposition attachment using WebRequest and/or HTML Agility Pack

I'm developting a web crawler that will download a PDF file from a website.
I checked the source code of the website before and I discovery that the button which download the PDF is actually a submit input to a form. That form retrives the file in Content-Disposition header in response.
Here's a picture of it:
My question is, how to get this file using web request (or HTML Agility pack). I tried in this way, but the hearder returns null.
HttpWebResponse response = (HttpWebResponse)req.GetResponse();
string file = response.Headers["Content-Disposition"];
Thanks in advance
I already have my answers, here is what I've done to get the file
response = (HttpWebResponse)request.GetResponse();
stream = response.GetResponseStream();
byte[] retorno = ReadToEnd(stream);
response.Close();
stream.Close();
public static byte[] ReadToEnd(System.IO.Stream stream)
{
long originalPosition = 0;
if (stream.CanSeek)
{
originalPosition = stream.Position;
stream.Position = 0;
}
try
{
byte[] readBuffer = new byte[4096];
int totalBytesRead = 0;
int bytesRead;
while ((bytesRead = stream.Read(readBuffer, totalBytesRead, readBuffer.Length - totalBytesRead)) > 0)
{
totalBytesRead += bytesRead;
if (totalBytesRead == readBuffer.Length)
{
int nextByte = stream.ReadByte();
if (nextByte != -1)
{
byte[] temp = new byte[readBuffer.Length * 2];
Buffer.BlockCopy(readBuffer, 0, temp, 0, readBuffer.Length);
Buffer.SetByte(temp, totalBytesRead, (byte)nextByte);
readBuffer = temp;
totalBytesRead++;
}
}
}
byte[] buffer = readBuffer;
if (readBuffer.Length != totalBytesRead)
{
buffer = new byte[totalBytesRead];
Buffer.BlockCopy(readBuffer, 0, buffer, 0, totalBytesRead);
}
return buffer;
}
finally
{
if (stream.CanSeek)
{
stream.Position = originalPosition;
}
}
}
thanks

Attempt at Circular Buffer - Javascript

Right! Here is my attempt at a circular buffer (For use in a graphing program, using the canvas element). Have yet to get round to testing it out.
Question is - Can anyone see any flaws in my logic? Or bottlenecks?
/**
* A circular buffer class.
* #To add value -> bufferObject.addValue(xValue, yValue);
* #To get the First-in value use -> bufferObject.getValue(0);
* #To get the Last-in value use -> bufferObject.getValue(bufferObject.length);
**/
var circularBuffer = function (bufferSize) {
this.bufferSize = bufferSize;
this.buffer = new Array(this.bufferSize); // After testing on jPerf -> 2 x 1D array seems fastest solution.
this.end = 0;
this.start = 0;
// Adds values to array in circular.
this.addValue = function(xValue, yValue) {
this.buffer[this.end] = {x : xValue, y: yValue};
if (this.end != this.bufferSize) this.end++;
else this.end = 0;
if(this.end == this.start) this.start ++;
};
// Returns a value from the buffer
this.getValue = function(index) {
var i = index+this.start;
if(i >= this.bufferSize) i -= this.bufferSize; //Check here.
return this.buffer[i]
};
// Returns the length of the buffer
this.getLength = function() {
if(this.start > this.end || this.start == this.bufferSize) {
return this.xBuffer.length;
} else {
return this.end - this.start;
}
};
// Returns true if the buffer has been initialized.
this.isInitialized = function() {
if(this.end != this.start) return true;
else return false;
};
}
Please feel free to reuse this code.
Updated twice (and tested!).
Update: Found another implementation Circular buffer in JavaScript
Made class variables private, corrected old xBuffer reference. Will do more edits tonight.
/**
* A circular buffer class.
* #To add value -> bufferObject.addValue(xValue, yValue);
* #To get the First-in value use -> bufferObject.getValue(0);
* #To get the Last-in value use -> bufferObject.getValue(bufferObject.length);
**/
var circularBuffer = function (buffer_size) {
var bufferSize = buffer_size;
var buffer = new Array(bufferSize); // After testing on jPerf -> 2 x 1D array seems fastest solution.
var end = 0;
var start = 0;
// Adds values to array in circular.
this.addValue = function(xValue, yValue) {
buffer[end] = {x : xValue, y: yValue};
if (end != bufferSize) end++;
else end = 0;
if(end == start) start++;
};
// Returns a value from the buffer
this.getValue = function(index) {
var i = index+start;
if(i >= bufferSize) i -= bufferSize; //Check here.
return buffer[i];
};
// Returns the length of the buffer
this.getLength = function() {
if(start > end || start == bufferSize) {
return buffer.length;
} else {
return end - start;
}
};
// Returns true if the buffer has been initialized.
this.isInitialized = function() {
return (end != start) ? true : false;
};
}
I implemented Vogomatix's code above, and got a few bugs. The code writes off the end of the buffer, expanding the buffer size automatically, and the addValue function is bound to a particular type. I've adjusted the code to work with any object type, added some private subroutines to simplify, and added a function to dump the contents out to a string, with an optional delimiter. Also used a namespace.
What's missing is a removeValue() but it would be just a check of count to be greater than zero, then a call to _pop().
This was done because I needed a rolling, scrolling text buffer for inbound messages, that did not grow indefinitely. I use the object with a textarea, so I get behaviour like a console window, a scrolling text box that does not chew up memory indefinitely.
This has been tested with expediency in mind, in that I am coding quickly, posted here in the hope that fellow OverFlow-ers use and anneal the code.
///////////////////////////////////////////////////////////////////////////////
// STYLE DECLARATION
// Use double quotes in JavaScript
///////////////////////////////////////////////////////////////////////////////
// Global Namespace for this application
//
var nz = nz || {};
nz.cbuffer = new Object();
///////////////////////////////////////////////////////////////////////////////
// CIRCULAR BUFFER
//
// CREDIT:
// Based on...
// Vogomatix http://stackoverflow.com/questions/20119513/attempt-at-circular-buffer-javascript
// But re-written after finding some undocumented features...
/**
* A circular buffer class, storing any type of Javascript object.
* To add value -> bufferObject.addValue(obj);
* To get the First-in value use -> bufferObject.getValue(0);
* To get the Last-in value use -> bufferObject.getValue(bufferObject.length);
* To dump to string use -> bufferObject.streamToString(sOptionalDelimiter); // Defaults to "\r\n"
**/
nz.cbuffer.circularBuffer = function (buffer_size) {
var bufferSize = buffer_size > 0 ? buffer_size : 1; // At worst, make an array of size 1
var buffer = new Array(bufferSize);
var end = 0; // Index of last element.
var start = 0; // Index of first element.
var count = 0; // Count of elements
// 'Private' function to push object onto buffer.
this._push = function (obj) {
buffer[end] = obj; // Write
end++; // Advance
if (end == bufferSize) {
end = 0; // Wrap if illegal
}
count++;
}
// 'Private' function to pop object from buffer.
this._pop = function () {
var obj = buffer[start];
start++;
if (start == bufferSize) {
start = 0; // Wrap
}
count--;
return obj;
}
// Adds values to buffer.
this.addValue = function (obj) {
if (count < bufferSize) {
// Just push
this._push(obj);
}
else {
// Pop, then push
this._pop();
this._push(obj);
}
}
// Returns a value from the buffer. Index is relative to current notional start.
this.getValue = function (index) {
if (index >= count || index < 0) return; // Catch attempt to access illegal index
var i = index + start;
if (i >= bufferSize) {
i -= bufferSize;
}
return buffer[i];
}
// Returns the length of the buffer.
this.getLength = function () {
return count;
}
// Returns all items as strings, separated by optional delimiter.
this.streamToString = function (delim) {
delim = (typeof delim === "undefined") ? "\r\n" : delim; // Default syntax; Default to CRLF
var strReturn = "";
var once = 0;
var index = 0;
var read = index + start;
for (; index < count; ++index) {
if (once == 1) strReturn += delim.toString();
strReturn += buffer[read].toString();
read++;
if (read >= bufferSize) read = 0;
once = 1;
}
return strReturn;
}
}

j2me midlet chinese character display message garbled

My J2ME midlet could retrieves message in Chinese character from a PHP server but it's garbled. The server basically returns the response string and by detecting the first 2 characters. AA = good, anything else indicates error of which the message is to be passed to the calling function for display
InputStream is = null;
StringBuffer sb = null;
String str = "";
HttpConnection http = null;
DataOutputStream dos = null;
try
{
URL = login.getURL();
URL += ctlFunction + "/" + uriStr;
URL = EncodeURL(URL);
//System.out.println(URL);
if(!ctlFunction.equals("login"))
{
msg += "&user=" + login.getUsername();
msg += "&token=" + login.getToken();
}
msg += "&lang=" + System.getProperty("microedition.locale");
// establish the connection
http = (HttpConnection) Connector.open(URL);
http.setRequestMethod(HttpConnection.POST);
http.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
http.setRequestProperty("Content-length", ""+EncodeURL(msg).getBytes().length);
dos = http.openDataOutputStream();
byte[] request_body = EncodeURL(msg).getBytes();
for (int i = 0; i < request_body.length; i++)
{
dos.writeByte(request_body[i]);
}
// server response
if (http.getResponseCode() == HttpConnection.HTTP_OK)
{
is = http.openDataInputStream();
int length = (int) http.getLength();
if (length != -1)
{
// Read data in one chunk
byte serverData[] = new byte[length];
is.read(serverData);
str = new String(serverData);
}
else // Length not available...
{
ByteArrayOutputStream bStrm = new ByteArrayOutputStream();
int ch;
while ((ch = is.read()) != -1)
bStrm.write(ch);
str = new String(bStrm.toByteArray());
bStrm.close();
}
}
else
{
networkError();
}
}
catch (Exception e)
{
System.err.println("Error3: " + e.toString());
networkError(e.toString());
}
finally
{
if (is != null)
is.close();
if (!str.equals(""))
post = str;
else
networkError();
if (http != null)
http.close();
}
if (post != null)
{
String fate = post.substring(0, 2);
if(fate.equals("AA"))
{
if(ctlFunction.equals("login"))
{
String rawPost = post.substring(2);
Vector v = new Vector();
int index = 0;
//find the first occurrence of the SPLITTER
int endIndex = rawPost.indexOf(SPLITTER, index);
String item = "";
//extract the items until the end of the last SPLITTER found in the rawPost string
while(endIndex != -1)
{
item = rawPost.substring(index, endIndex);
index = endIndex + 1;
endIndex = rawPost.indexOf(SPLITTER, index);
v.addElement(item);
}
//extract the rest of the rawPost (the text item)
item = rawPost.substring(index);
v.addElement(item);
String[] ret = new String[v.size()];
v.copyInto(ret);
login.setToken(ret[0]);
login.setToday(ret[1]);
login.setNextDrawDay(ret[2]);
}
midlet.returnResults(post.substring(2), getCurrentDisplay(), ctlFunction);
}
else
{
String errmessage = post.substring(2);
System.out.println(post);
midlet.showInfo(post, getCurrentDisplay());
}
}
else
{
networkError();
}
On the PHP server, I have set the header to UTF-8 encoding
<?php header("Content-Type:text/plain; charset=utf-8"); ?>
What could possibly be wrong?
I found that this user has the same problem and it's been answered
Reading UTF8 strings from a server through http using MIDP. Kudos to the answer.
I basically edited my MIDP code from
// is = http.openDataInputStream();
// int length = (int) http.getLength();
// if (length != -1)
// {
// // Read data in one chunk
// byte serverData[] = new byte[length];
// is.read(serverData);
// str = new String(serverData);
// }
// else // Length not available...
// {
// ByteArrayOutputStream bStrm = new ByteArrayOutputStream();
// int ch;
// while ((ch = is.read()) != -1)
// bStrm.write(ch);
//
// str = new String(bStrm.toByteArray());
// bStrm.close();
// }
to
Reader r = new InputStreamReader(http.openInputStream(), "UTF-8");
int ch;
while((ch = r.read()) != -1)
str = str + (char)ch;
just wondering though why does reading bytes messes up the UTF-8 characters?

Resources