Asynchronous NtQueryDirectoryFile? - winapi

Question

Has anyone managed to figure out how asynchronous calls to NtQueryDirectoryFile work?
By an asynchronous call, I mean calling NtQueryDirectoryFile on directories not opened with FILE_SYNCHRONOUS_IO_ALERT or with FILE_SYNCHRONOUS_IO_NONALERT.
For me, it seems to return STATUS_PENDING just like a normal NtReadFile request does on a normal file, but when I tried using NtWaitForSingleObject on the directory, it didn't end properly, and I still don't get all the data... why does this happen?

Answer 1

As far as I know, none of the Windows filesystems support asynchronous query directory calls.
The Win32 APIs never call NtQueryDirectoryFile asnchronously, so support for it is hit-or-miss.
NTFS theoretically supports asynchronous NtQueryDirectoryFile but (as I mentioned) it is not extensively tested so it may not work.
You response indicated that you called WaitForSingleObject on the directory - that's not how the async pattern works in NT - you need to call WaitForSingleObject on the event handle provided as a parameter to NtQueryDirectoryFile.
This update is a result of asking the NTFS developer for more information, he tested this scenario on his machine and it worked for him (on Windows 7).

Answer 2

NtQueryDirectoryFile works well in asynchronous!
pass callback in ApcRoutine, and callback data in ApcContext
asynchronous procedure calls only call when the thread is in alertable state(for example: calling SleepEx(INFINITE, TRUE), WSAaccept)
this program shows how asynchronous NtQueryDirectoryFile work.
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#include <stdio.h>
#include <winternl.h>
#include <winnt.h>
#define LIST_DIR_SIZE 2000
#define STATUS_NO_MORE_FILES ((NTSTATUS)80000006)
typedef struct _FILE_NAMES_INFORMATION {
ULONG NextEntryOffset;
ULONG FileIndex;
ULONG FileNameLength;
WCHAR FileName[1];
} FILE_NAMES_INFORMATION, * PFILE_NAMES_INFORMATION;
typedef struct {
HANDLE hFile;
OVERLAPPED ol;
DECLSPEC_ALIGN(4) FILE_NAMES_INFORMATION buf[LIST_DIR_SIZE];
IO_STATUS_BLOCK iob;
bool finished;
} LIST_DIR_DATA, * PLIST_DIR_DATA; // my private data
__kernel_entry NTSYSCALLAPI
NTSTATUS
NTAPI
NtQueryDirectoryFile(
_In_ HANDLE FileHandle,
_In_opt_ HANDLE Event,
_In_opt_ PIO_APC_ROUTINE ApcRoutine,
_In_opt_ PVOID ApcContext,
_Out_ PIO_STATUS_BLOCK IoStatusBlock,
_Out_writes_bytes_(Length) PVOID FileInformation,
_In_ ULONG Length,
_In_ FILE_INFORMATION_CLASS FileInformationClass,
_In_ BOOLEAN ReturnSingleEntry,
_In_opt_ PUNICODE_STRING FileName,
_In_ BOOLEAN RestartScan
);
#define NTDLL_extern(s) typedef decltype(&s) s##T;s##T s##F;
#define NTDLL_import(s) s##F = (s##T)GetProcAddress(ntdll, #s);
NTDLL_extern(NtOpenFile);
NTDLL_extern(NtQueryDirectoryFile);
NTDLL_extern(NtClose);
NTDLL_extern(RtlInitUnicodeString);
HMODULE ntdll;
VOID NTAPI callback(
IN PVOID ApcContext,
IN PIO_STATUS_BLOCK IoStatusBlock,
IN ULONG Reserved) {
UNREFERENCED_PARAMETER(Reserved);
PFILE_NAMES_INFORMATION file_info = ((PLIST_DIR_DATA)ApcContext)->buf;
do {
fputws(file_info->FileName, stdout);
putwchar(L'\t');
file_info = (PFILE_NAMES_INFORMATION)((char*)file_info + file_info->NextEntryOffset);
} while (file_info->NextEntryOffset);
fputws(file_info->FileName, stdout);
putwchar(L'\t');
PLIST_DIR_DATA c = (PLIST_DIR_DATA)ApcContext;
if (IoStatusBlock->Information != 0) {
NTSTATUS status = NtQueryDirectoryFileF(
c->hFile,
NULL,
callback,
ApcContext,
&c->iob,
c->buf,
sizeof(c->buf),
FILE_INFORMATION_CLASS(12),
FALSE, NULL, FALSE);
switch (status) {
case STATUS_PENDING:
break;
default:
fputs("warning: status != STATUS_PENDING", stderr);
}
}
else {
c->finished = true;
}
}
BOOL init() {
ntdll = LoadLibraryW(L"NtDLL.dll");
if (ntdll == NULL) {
fputs("LoadLibraryW", stderr);
return FALSE;
}
NTDLL_import(NtQueryDirectoryFile);
NTDLL_import(NtOpenFile);
NTDLL_import(NtClose);
NTDLL_import(RtlInitUnicodeString);
if (NtCloseF != NULL && NtOpenFileF != NULL && NtCloseF != NULL) {
return TRUE;
}
else {
fputs("GetProcAddress", stderr);
return FALSE;
}
}
int main() {
if (init() == FALSE) {
fputs("error: init() failed!", stderr);
return -1;
}
NTSTATUS status;
PLIST_DIR_DATA data = new LIST_DIR_DATA{};
{
OBJECT_ATTRIBUTES ObjectAttributes;
UNICODE_STRING s;
RtlInitUnicodeStringF(&s, L"\\??\\c:\\Windows\\System32");
InitializeObjectAttributes(
&ObjectAttributes,
&s,
OBJ_CASE_INSENSITIVE,
NULL,
NULL);
status = NtOpenFileF(
&data->hFile,
FILE_READ_DATA | FILE_LIST_DIRECTORY, // | FILE_TRAVERSE | SYNCHRONIZE
&ObjectAttributes,
&data->iob,
FILE_SHARE_READ,
FILE_DIRECTORY_FILE); // | FILE_SYNCHRONOUS_IO_NONALERT
}
if (status < 0 || data->hFile == NULL) {
fputs("error: NtOpenFile failed", stderr);
return -2;
}
status = NtQueryDirectoryFileF(
data->hFile,
NULL,
callback,
data,
&data->iob,
data->buf,
sizeof(data->buf),
FILE_INFORMATION_CLASS(12),
FALSE, NULL, FALSE);
switch (status) {
case STATUS_PENDING:
break;
default:
fputs("warning: status != STATUS_PENDING", stderr);
}
for (;data->finished==false;) SleepEx(INFINITE, TRUE); // put main thread into alertable wait
NtCloseF(data->hFile);
FreeLibrary(ntdll);
return 0;
}
if you want UTF-8 output, try this (note: recommand use support UTF-8 terminal)
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#include <winternl.h>
#include <winnt.h>
#include <crtdbg.h>
#include <cstdio>
#define LIST_DIR_SIZE 200
#define STATUS_NO_MORE_FILES ((NTSTATUS)80000006)
typedef struct _FILE_NAMES_INFORMATION {
ULONG NextEntryOffset;
ULONG FileIndex;
ULONG FileNameLength;
WCHAR FileName[1];
} FILE_NAMES_INFORMATION, * PFILE_NAMES_INFORMATION;
typedef struct {
HANDLE hFile;
OVERLAPPED ol;
DECLSPEC_ALIGN(4) FILE_NAMES_INFORMATION buf[LIST_DIR_SIZE];
IO_STATUS_BLOCK iob;
bool finished;
} LIST_DIR_DATA, * PLIST_DIR_DATA; // my private data
__kernel_entry NTSYSCALLAPI
NTSTATUS
NTAPI
NtQueryDirectoryFile(
_In_ HANDLE FileHandle,
_In_opt_ HANDLE Event,
_In_opt_ PIO_APC_ROUTINE ApcRoutine,
_In_opt_ PVOID ApcContext,
_Out_ PIO_STATUS_BLOCK IoStatusBlock,
_Out_writes_bytes_(Length) PVOID FileInformation,
_In_ ULONG Length,
_In_ FILE_INFORMATION_CLASS FileInformationClass,
_In_ BOOLEAN ReturnSingleEntry,
_In_opt_ PUNICODE_STRING FileName,
_In_ BOOLEAN RestartScan
);
#define NTDLL_extern(s) typedef decltype(&s) s##T;s##T s##F;
#define NTDLL_init(s) s##F = (s##T)GetProcAddress(ntdll, #s);
NTDLL_extern(NtOpenFile);
NTDLL_extern(NtQueryDirectoryFile);
NTDLL_extern(NtClose);
NTDLL_extern(RtlInitUnicodeString);
HMODULE ntdll;
HANDLE heap;
VOID NTAPI callback(
IN PVOID ApcContext,
IN PIO_STATUS_BLOCK IoStatusBlock,
IN ULONG Reserved) {
UNREFERENCED_PARAMETER(Reserved);
PLIST_DIR_DATA c = (PLIST_DIR_DATA)ApcContext;
if (IoStatusBlock->Information){
PFILE_NAMES_INFORMATION file_info = c->buf;
ULONG_PTR length = 0;
ULONG last;
do {
last = file_info->NextEntryOffset;
file_info->FileNameLength /= 2; // wide char length always base of 2 in bytes
length += (
file_info->FileIndex=WideCharToMultiByte(
CP_UTF8, WC_ERR_INVALID_CHARS,
file_info->FileName, file_info->FileNameLength,
NULL, 0,
NULL, NULL)
)+1;
if (file_info->FileIndex == 0) { // FileIndex is how many byte is the UTF-8 string
_RPTF0(_CRT_WARN, "WideCharToMultiByte failed!");
}
file_info = (PFILE_NAMES_INFORMATION)((char*)file_info + file_info->NextEntryOffset);
} while (last);
LPSTR pData = (LPSTR)HeapAlloc(heap, HEAP_NO_SERIALIZE, length), ptr=pData;
if (ptr == NULL) {
_RPTF0(_CRT_ERROR, "HeapAlloc failed!");
return;
}
file_info = c->buf;
do {
last = file_info->NextEntryOffset;
if (WideCharToMultiByte(
CP_UTF8, WC_ERR_INVALID_CHARS,
file_info->FileName, file_info->FileNameLength,
pData, file_info->FileIndex,
NULL, NULL)==0) {
_RPTF0(_CRT_WARN, "WideCharToMultiByte failed!");
}
pData += file_info->FileIndex;
*pData++ = '\n';
file_info = (PFILE_NAMES_INFORMATION)((char*)file_info + file_info->NextEntryOffset);
} while (last);
// use data here
fwrite(ptr, length, 1, stdout);
// use data here
HeapFree(heap, HEAP_NO_SERIALIZE, ptr);
NTSTATUS status = NtQueryDirectoryFileF(
c->hFile,
NULL,
callback,
ApcContext,
&c->iob,
c->buf,
sizeof(c->buf),
FILE_INFORMATION_CLASS(12),
FALSE, NULL, FALSE);
switch (status) {
case STATUS_PENDING:
break;
default:
_RPTF0(_CRT_WARN, "status != STATUS_PENDING");
}
}else{
c->finished = true;
}
}
BOOL init() {
ntdll = LoadLibraryW(L"NtDLL.dll");
if (ntdll == NULL) {
_RPTF0(_CRT_ERROR, "fail to load NtDLL.dll");
return FALSE;
}
NTDLL_init(NtQueryDirectoryFile);
NTDLL_init(NtOpenFile);
NTDLL_init(NtClose);
NTDLL_init(RtlInitUnicodeString);
if (NtCloseF != NULL &&
NtOpenFileF != NULL &&
NtCloseF != NULL &&
(heap = HeapCreate(HEAP_NO_SERIALIZE, 4096,0))!=NULL
){
return TRUE;
}
else {
_RPTF0(_CRT_ERROR, "failed to load function and create heap");
return FALSE;
}
}
int main() {
if (init() == FALSE) {
_RPTF0(_CRT_ERROR, "init failed");
return -1;
}
SetConsoleCP(CP_UTF8);
NTSTATUS status;
PLIST_DIR_DATA data = new LIST_DIR_DATA{};
{
OBJECT_ATTRIBUTES ObjectAttributes;
UNICODE_STRING s;
RtlInitUnicodeStringF(&s, L"\\??\\c:\\Users");
InitializeObjectAttributes(
&ObjectAttributes,
&s,
OBJ_CASE_INSENSITIVE,
NULL,
NULL);
status = NtOpenFileF(
&data->hFile,
FILE_READ_DATA | FILE_LIST_DIRECTORY, // | FILE_TRAVERSE | SYNCHRONIZE
&ObjectAttributes,
&data->iob,
FILE_SHARE_READ,
FILE_DIRECTORY_FILE); // | FILE_SYNCHRONOUS_IO_NONALERT
}
if (status < 0 || data->hFile == NULL) {
_RPTF0(_CRT_ERROR, "NtOpenFile failed!");
return -2;
}
status = NtQueryDirectoryFileF(
data->hFile,
NULL,
callback,
data,
&data->iob,
data->buf,
sizeof(data->buf),
FILE_INFORMATION_CLASS(12),
FALSE, NULL, FALSE);
switch (status) {
case STATUS_PENDING:
break;
default:
_RPTF0(_CRT_WARN, "status != STATUS_PENDING");
}
for (;data->finished==false;) SleepEx(INFINITE, TRUE); // put main thread into alertable wait
if (NtCloseF(data->hFile)<0) {
_RPTF0(_CRT_ERROR, "NtClose failed!");
}
if (FreeLibrary(ntdll) == FALSE) {
_RPTF0(_CRT_WARN, "failed to Free libary");
}
if (HeapDestroy(heap) == FALSE) {
_RPTF0(_CRT_WARN, "fail to destroy heap");
}
}