I am trying to implement fast IO under Windows, and working my way up to Overlapped IO. In my research, Unbuffered IO requires page aligned buffers. Ive attempted to implement this in my code below. However, I occasionally have Readfiles last error report no access (error 998, ERROR_NOACCESS) - prior to completing the read, and after a few reads of a page aligned buffer. Sometimes 16. Sometimes 4, etc.
I cant for the life of me figure out why i am occasionally throwing an error. Any insight would be helpful.
ci::BufferRef CinderSequenceRendererApp::CreateFileLoadWinNoBufferSequential(fs::path path) {
HANDLE file = CreateFile(path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, FILE_FLAG_NO_BUFFERING | FILE_FLAG_SEQUENTIAL_SCAN, 0);
if (file == INVALID_HANDLE_VALUE)
{
console() << "Could not open file for reading" << std::endl;
}
ci::BufferRef latestAvailableBufferRef = nullptr;
LARGE_INTEGER nLargeInteger = { 0 };
GetFileSizeEx(file, &nLargeInteger);
// how many reads do we need to fill our buffer with a buffer size of x and a read size of y
// Our buffer needs to hold 'n' sector sizes that wil fit the size of the file
SYSTEM_INFO si;
GetSystemInfo(&si);
long readAmount = si.dwPageSize;
int numReads = 0;
ULONG bufferSize = 0;
// calculate sector aligned buffer size that holds our file size
while (bufferSize < nLargeInteger.QuadPart)
{
numReads++;
bufferSize = (numReads) * readAmount;
}
// need one page extra for null if we need it
latestAvailableBufferRef = ci::Buffer::create(bufferSize + readAmount);
if (latestAvailableBufferRef != nullptr)
{
DWORD outputBytes = 1;
// output bytes = 0 when OEF
void* address = latestAvailableBufferRef->getData();
DWORD bytesRead = 0;
while (outputBytes != 0)
{
bool result = ReadFile(file, address, readAmount, &outputBytes, 0);
if (!result )//&& (outputBytes == 0))
{
getLastReadError();
}
address = (void*)((long)address + readAmount);
bytesRead += outputBytes;
}
}
CloseHandle(file);
// resize our buffer to expected file size?
latestAvailableBufferRef->resize(nLargeInteger.QuadPart);
return latestAvailableBufferRef;
}
Cast to long long - I was truncating my pointer address. Duh. Thanks to #jonathan-potter
Related
I am using CreateFile, ReadFile and WriteFile to access a disk's sectors directly. It looks like I can read any sector I want, but when it comes to writing, I get ERROR_ACCESS_DENIED for sectors 16 and above. I am at a loss to explain why I can write to the first 15 sectors but not the others.
This is on Windows 10.
Note that I have not tried every single sector above 16, just a random sampling of them and they all seem to fail.
int wmain(int argc, WCHAR *argv[])
{
HANDLE hDisk = NULL;
hDisk = CreateFile(
L"\\\\.\\Q:",
GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_WRITE,
NULL,
OPEN_EXISTING,
0,
NULL);
char *rgb = (char *) malloc(512);
BOOL b = FALSE;
DWORD dw = 0;
LONG lo = 0;
LONG hi = 0;
for(int i=0; i<20; i++)
{
hi = 0;
lo = i * 512;
dw = SetFilePointer(hDisk, lo, &hi, FILE_BEGIN);
b = ReadFile(hDisk, rgb, 512, &dw, NULL);
if (b == FALSE)
printf("Cannot read sector %d\r\n", i);
hi = 0;
lo = i * 512;
dw = SetFilePointer(hDisk, lo, &hi, FILE_BEGIN);
b = WriteFile(hDisk, rgb, 512, &dw, NULL);
if (b == FALSE)
printf("Cannot write sector %d\r\n", i);
}
return 0;
}
The code above outputs:
Cannot write sector 16
Cannot write sector 17
Cannot write sector 18
Cannot write sector 19
I've omitted error handling code to keep things short.
I found my problem.
Because I opened the drive with FILE_SHARE_READ | FILE_SHARE_WRITE, I was denied access to the part of the disk that contained a volume that was in use.
At least, that's my educated guess.
Once I removed the SHARE flags and made sure I had sole access to the drive, I could read and write any sector.
In linux there is the fstat system call which gives the inode number of a filedescriptor.
Is there any system call or winapi function which would give MFT Record Number of a given file, from its HANDLE or file path?
If there isn't any function or system call so how should I reach to the MFT Record of a file in MFT Table?
for got MFT Record Number for given file need use FileInternalInformation - here returned FILE_INTERNAL_INFORMATION. really this is 48 low bit MftRecordIndex and 16 high bit SequenceNumber
struct
{
LONGLONG MftRecordIndex : 48;
LONGLONG SequenceNumber : 16;
};
look also MFT_SEGMENT_REFERENCE - this is same struct
then for got MFT Record use FSCTL_GET_NTFS_FILE_RECORD as input data - FileReferenceNumber - this is FILE_INTERNAL_INFORMATION.IndexNumber but(!) only low 48 bits(MftRecordIndex) so you need zero high 16 bits(SequenceNumber) and then use FILE_INTERNAL_INFORMATION in place NTFS_FILE_RECORD_INPUT_BUFFER for know NTFS_FILE_RECORD_OUTPUT_BUFFER size - you need first get NTFS_VOLUME_DATA_BUFFER with help FSCTL_GET_NTFS_VOLUME_DATA and use NTFS_VOLUME_DATA_BUFFER.BytesPerFileRecordSegment
NTSTATUS Test(POBJECT_ATTRIBUTES poa)
{
HANDLE hFile, hVolume = 0;
IO_STATUS_BLOCK iosb;
NTSTATUS status = NtOpenFile(&hFile, SYNCHRONIZE, poa, &iosb, FILE_SHARE_VALID_FLAGS, FILE_SYNCHRONOUS_IO_NONALERT);
if (0 <= status)
{
union
{
FILE_INTERNAL_INFORMATION fii;
NTFS_FILE_RECORD_INPUT_BUFFER nfrib;
struct
{
LONGLONG MftRecordIndex : 48;
LONGLONG SequenceNumber : 16;
};
};
if (0 <= (status = NtQueryInformationFile(hFile, &iosb, &fii, sizeof(fii), FileInternalInformation)))
{
//need open '\Device\HarddiskVolume<N>' or '<X>:'
status = OpenVolume(hFile, &hVolume);
}
NtClose(hFile);
if (0 <= status)
{
NTFS_VOLUME_DATA_BUFFER nvdb;
if (0 <= (status = NtFsControlFile(hVolume, 0, 0, 0, &iosb, FSCTL_GET_NTFS_VOLUME_DATA, 0, 0, &nvdb, sizeof(nvdb))))
{
DWORD cb = FIELD_OFFSET(NTFS_FILE_RECORD_OUTPUT_BUFFER,
FileRecordBuffer[nvdb.BytesPerFileRecordSegment]);
PNTFS_FILE_RECORD_OUTPUT_BUFFER pnfrob = (PNTFS_FILE_RECORD_OUTPUT_BUFFER)alloca(cb);
SequenceNumber = 0;
if (0 <= (status = NtFsControlFile(hVolume, 0, 0, 0, &iosb,
FSCTL_GET_NTFS_FILE_RECORD, &nfrib, sizeof nfrib, pnfrob, cb)))
{
NTFS_FILE_RECORD_HEADER* pnfrh = (NTFS_FILE_RECORD_HEADER*)pnfrob->FileRecordBuffer;;
}
}
NtClose(hVolume);
}
}
return status;
}
NTFS_FILE_RECORD_HEADER - this is FILE_RECORD_SEGMENT_HEADER (i take self structs name from here)
I'm using CoreAudio low level API for audio capturing. The app target is MAC OSX, not iOS.
During testing it, from time to time we got very annoying noise modulate with real audio. the phenomena develops with time, started from barely noticeable and become more and more dominant.
Analyze the captured audio under Audacity indicate that the end part of the audio packet is wrong.
Here are sample picture:
the intrusion repeat every 40 ms which is the configured packetization time (in terms of buffer samples)
Update:
Over time the gap became larger, here is another snapshot from the same captured file 10 minutes later. the gap now contains 1460 samples which is 33ms from the total 40ms of the packet!!
CODE SNIPPESTS:
capture callback
OSStatus MacOS_AudioDevice::captureCallback(void *inRefCon,
AudioUnitRenderActionFlags *ioActionFlags,
const AudioTimeStamp *inTimeStamp,
UInt32 inBusNumber,
UInt32 inNumberFrames,
AudioBufferList *ioData)
{
MacOS_AudioDevice* _this = static_cast<MacOS_AudioDevice*>(inRefCon);
// Get the new audio data
OSStatus err = AudioUnitRender(_this->m_AUHAL, ioActionFlags, inTimeStamp, inBusNumber, inNumberFrames, _this->m_InputBuffer);
if (err != noErr)
{
...
return err;
}
// ignore callback on unexpected buffer size
if (_this->m_params.bufferSizeSamples != inNumberFrames)
{
...
return noErr;
}
// Deliver audio data
DeviceIOMessage message;
message.bufferSizeBytes = _this->m_deviceBufferSizeBytes;
message.buffer = _this->m_InputBuffer->mBuffers[0].mData;
if (_this->m_callbackFunc)
{
_this->m_callbackFunc(_this, message);
}
}
Open and start capture device:
void MacOS_AudioDevice::openAUHALCapture()
{
UInt32 enableIO;
AudioStreamBasicDescription streamFormat;
UInt32 size;
SInt32 *channelArr;
std::stringstream ss;
AudioObjectPropertyAddress deviceBufSizeProperty =
{
kAudioDevicePropertyBufferFrameSize,
kAudioDevicePropertyScopeInput,
kAudioObjectPropertyElementMaster
};
// AUHAL
AudioComponentDescription cd = {kAudioUnitType_Output, kAudioUnitSubType_HALOutput, kAudioUnitManufacturer_Apple, 0, 0};
AudioComponent HALOutput = AudioComponentFindNext(NULL, &cd);
verify_macosapi(AudioComponentInstanceNew(HALOutput, &m_AUHAL));
verify_macosapi(AudioUnitInitialize(m_AUHAL));
// enable input IO
enableIO = 1;
verify_macosapi(AudioUnitSetProperty(m_AUHAL, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Input, 1, &enableIO, sizeof(enableIO)));
// disable output IO
enableIO = 0;
verify_macosapi(AudioUnitSetProperty(m_AUHAL, kAudioOutputUnitProperty_EnableIO, kAudioUnitScope_Output, 0, &enableIO, sizeof(enableIO)));
// Setup current device
size = sizeof(AudioDeviceID);
verify_macosapi(AudioUnitSetProperty(m_AUHAL, kAudioOutputUnitProperty_CurrentDevice, kAudioUnitScope_Global, 0, &m_MacDeviceID, sizeof(AudioDeviceID)));
// Set device native buffer length before setting AUHAL stream
size = sizeof(m_originalDeviceBufferTimeFrames);
verify_macosapi(AudioObjectSetPropertyData(m_MacDeviceID, &deviceBufSizeProperty, 0, NULL, size, &m_originalDeviceBufferTimeFrames));
// Get device format
size = sizeof(AudioStreamBasicDescription);
verify_macosapi(AudioUnitGetProperty(m_AUHAL, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Input, 1, &streamFormat, &size));
// Setup channel map
assert(m_params.numOfChannels <= streamFormat.mChannelsPerFrame);
channelArr = new SInt32[streamFormat.mChannelsPerFrame];
for (int i = 0; i < streamFormat.mChannelsPerFrame; i++)
channelArr[i] = -1;
for (int i = 0; i < m_params.numOfChannels; i++)
channelArr[i] = i;
verify_macosapi(AudioUnitSetProperty(m_AUHAL, kAudioOutputUnitProperty_ChannelMap, kAudioUnitScope_Input, 1, channelArr, sizeof(SInt32) * streamFormat.mChannelsPerFrame));
delete [] channelArr;
// Setup stream converters
streamFormat.mFormatID = kAudioFormatLinearPCM;
streamFormat.mFormatFlags = kAudioFormatFlagIsSignedInteger;
streamFormat.mFramesPerPacket = m_SamplesPerPacket;
streamFormat.mBitsPerChannel = m_params.sampleDepthBits;
streamFormat.mSampleRate = m_deviceSampleRate;
streamFormat.mChannelsPerFrame = 1;
streamFormat.mBytesPerFrame = 2;
streamFormat.mBytesPerPacket = streamFormat.mFramesPerPacket * streamFormat.mBytesPerFrame;
verify_macosapi(AudioUnitSetProperty(m_AUHAL, kAudioUnitProperty_StreamFormat, kAudioUnitScope_Output, 1, &streamFormat, size));
// Setup callbacks
AURenderCallbackStruct input;
input.inputProc = captureCallback;
input.inputProcRefCon = this;
verify_macosapi(AudioUnitSetProperty(m_AUHAL, kAudioOutputUnitProperty_SetInputCallback, kAudioUnitScope_Global, 0, &input, sizeof(input)));
// Calculate the size of the IO buffer (in samples)
if (m_params.bufferSizeMS != -1)
{
unsigned int desiredSignalsInBuffer = (m_params.bufferSizeMS / (double)1000) * m_deviceSampleRate;
// making sure the value stay in the device's supported range
desiredSignalsInBuffer = std::min<unsigned int>(desiredSignalsInBuffer, m_deviceBufferFramesRange.mMaximum);
desiredSignalsInBuffer = std::max<unsigned int>(m_deviceBufferFramesRange.mMinimum, desiredSignalsInBuffer);
m_deviceBufferFrames = desiredSignalsInBuffer;
}
// Set device buffer length
size = sizeof(m_deviceBufferFrames);
verify_macosapi(AudioObjectSetPropertyData(m_MacDeviceID, &deviceBufSizeProperty, 0, NULL, size, &m_deviceBufferFrames));
m_deviceBufferSizeBytes = m_deviceBufferFrames * streamFormat.mBytesPerFrame;
m_deviceBufferTimeMS = 1000 * m_deviceBufferFrames/m_deviceSampleRate;
// Calculate number of buffers from channels
size = offsetof(AudioBufferList, mBuffers[0]) + (sizeof(AudioBuffer) * m_params.numOfChannels);
// Allocate input buffer
m_InputBuffer = (AudioBufferList *)malloc(size);
m_InputBuffer->mNumberBuffers = m_params.numOfChannels;
// Pre-malloc buffers for AudioBufferLists
for(UInt32 i = 0; i< m_InputBuffer->mNumberBuffers ; i++)
{
m_InputBuffer->mBuffers[i].mNumberChannels = 1;
m_InputBuffer->mBuffers[i].mDataByteSize = m_deviceBufferSizeBytes;
m_InputBuffer->mBuffers[i].mData = malloc(m_deviceBufferSizeBytes);
}
// Update class properties
m_params.sampleRateHz = streamFormat.mSampleRate;
m_params.bufferSizeSamples = m_deviceBufferFrames;
m_params.bufferSizeBytes = m_params.bufferSizeSamples * streamFormat.mBytesPerFrame;
}
eADMReturnCode MacOS_AudioDevice::start()
{
eADMReturnCode ret = OK;
LOGAPI(ret);
if (!m_isStarted && m_isOpen)
{
OSStatus err = AudioOutputUnitStart(m_AUHAL);
if (err == noErr)
m_isStarted = true;
else
ret = ERROR;
}
return ret;
}
Any idea what cause it and how to solve?
Thanks in advance!
Periodic glitches or dropouts can be caused by not paying attention to or by not fully processing the number of frames sent to each audio callback. Valid buffers don't always contain the expected or same number of samples (inNumberFrames might not equal bufferSizeSamples or the previous inNumberFrames in a perfectly valid audio buffer).
It is possible that these types of glitches might be caused by attempting to record at 44.1k on some models of iOS devices that only support 48k audio in hardware.
Some types of glitch might also be caused by any non-hard-real-time code within your m_callbackFunc function (such as any synchronous file reads/writes, OS calls, Objective C message dispatch, GC, or memory allocation/deallocation).
I want to map file into memory with chunk size equal system granularity. First chunk read without error and all others fails with error 5 (ERROR_ACCESS_DENIED). I tried run program with administrator privileges.
My code:
#include <windows.h>
#include <stdio.h>
int main() {
HANDLE hFile = CreateFile( TEXT("db.txt"),
GENERIC_READ,
FILE_SHARE_READ,
NULL,
OPEN_EXISTING,
FILE_ATTRIBUTE_NORMAL,
NULL);
if (hFile == INVALID_HANDLE_VALUE) {
printf("[ERROR] File opening error %d\n", GetLastError());
return 1;
}
printf("[DONE] File opened successfully.\n");
HANDLE hMap = CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
if (hMap == NULL) {
printf("[ERROR] Create mapping error %d\n", GetLastError());
return 2;
}
printf("[DONE] Create mapping successfully.\n");
LARGE_INTEGER file_size = { };
if (!GetFileSizeEx(hFile, &file_size)) {
printf("[ERROR] Getiing filesize error %d\n", GetLastError());
return 3;
}
printf("[DONE] Getting file size.\n");
SYSTEM_INFO info = { };
GetSystemInfo(&info);
printf("[DONE] Getting system memory granularity %d.\n", info.dwAllocationGranularity);
DWORD offset = 0;
int size = 0;
do {
char* ENTRY = (char*)MapViewOfFile(hMap, FILE_MAP_READ, HIWORD(offset), LOWORD(offset), info.dwAllocationGranularity);
if (ENTRY == NULL) {
printf("[ERROR] Map entry error %d\n", GetLastError());
} else {
printf("[DONE] MAPPING PART WITH OFFSET %d\n", offset);
//printf("%s\n", ENTRY);
}
if (offset + info.dwAllocationGranularity < file_size.QuadPart) {
offset += info.dwAllocationGranularity;
} else {
offset = file_size.QuadPart;
}
//offset += size;
UnmapViewOfFile(ENTRY);
} while (offset < file_size.QuadPart);
CloseHandle(hMap);
CloseHandle(hFile);
system("pause");
return 0;
}
How I fix it?
You're using HIWORD and LOWORD for the offset in the call to MapViewOfFile, but these only take a 32-bit value and split it into two 16-bit halves - what you want is a 64-bit value split into two 32-bit halves.
Instead you need HIDWORD and LODWORD, which are defined in <intsafe.h>:
#define LODWORD(_qw) ((DWORD)(_qw))
#define HIDWORD(_qw) ((DWORD)(((_qw) >> 32) & 0xffffffff))
Like so:
char* ENTRY = (char*)MapViewOfFile(hMap, FILE_MAP_READ, HIDWORD(offset), LODWORD(offset), info.dwAllocationGranularity);
You need this even though your offset variable is 32 bit (in which case, HIDWORD will just return 0 and the full value of offset is passed as the low-order DWORD).
I want to convert a PNG image found in a path to base64 for a html page in Windows phone7.1.How can it be done?
Stream imgStream;
imgStream = Assembly.GetExecutingAssembly().GetManifestResourceStream("NewUIChanges.Htmlfile.round1.png");
byte[] data = new byte[(int)imgStream.Length];
int offset = 0;
while (offset < data.Length)
{
int bytesRead = imgStream.Read(data, offset, data.Length - offset);
if (bytesRead <= 0)
{
throw new EndOfStreamException("Stream wasn't as long as it claimed");
}
offset += bytesRead;
}
The fact that it's a PNG image is actually irrelevant - all you need to know is that you've got some bytes that you need to convert into base64.
Read the data from a stream into a byte array, and then use Convert.ToBase64String. Reading a byte array from a stream can be slightly fiddly, depending on whether the stream advertises its length or not. If it does, you can use:
byte[] data = new byte[(int) stream.Length];
int offset = 0;
while (offset < data.Length)
{
int bytesRead = stream.Read(data, offset, data.Length - offset);
if (bytesRead <= 0)
{
throw new EndOfStreamException("Stream wasn't as long as it claimed");
}
offset += bytesRead;
}
If it doesn't, the simplest approach is probably to copy it to a MemoryStream:
using (MemoryStream ms = new MemoryStream())
{
byte[] buffer = new byte[8 * 1024];
int bytesRead;
while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0)
{
ms.Write(buffer, 0, bytesRead);
}
return ms.ToByteArray();
}
So once you've used either of those bits of code (or anything else suitable) to get a byte array, just use Convert.ToBase64String and you're away.
There are probably streaming solutions which will avoid ever having the whole byte array in memory - e.g. building up a StringBuilder of base64 data as it goes - but they would be more complicated. Unless you're going to deal with very large files, I'd stick with the above.