Force a DLL to be loaded above 2GB (0x80000000) in a 32-bit process on Windows - winapi

Question

To test a corner case in our debugger, I need to come up with a program which has a DLL loaded above 2GB (0x80000000). Current test case is a multi-GB game which loads >700 DLLs, and I'd like to have something simpler and smaller. Is there a way to achieve it reliably without too much fiddling? I assume I need to use /LARGEADDRESSAWARE and somehow consume enough of the VA space to bump the new DLLs above 2GB but I'm fuzzy on the details...

Answer 1

Okay, it took me a few tries but I managed to come up with something working.
// cl /MT /Ox test.cpp /link /LARGEADDRESSAWARE
// occupy the 2 gigabytes!
#define ALLOCSIZE (64*1024)
#define TWOGB (2*1024ull*1024*1024)
#include <windows.h>
#include <stdio.h>
int main()
{
int nallocs = TWOGB/ALLOCSIZE;
for ( int i = 0; i < nallocs+200; i++ )
{
void * p = VirtualAlloc(NULL, ALLOCSIZE, MEM_RESERVE, PAGE_NOACCESS);
if ( i%100 == 0)
{
if ( p != NULL )
printf("%d: %p\n", i, p);
else
{
printf("%d: failed!\n", i);
break;
}
}
}
printf("finished VirtualAlloc. Loading a DLL.\n");
//getchar();
HMODULE hDll = LoadLibrary("winhttp");
printf("DLL base: %p.\n", hDll);
//getchar();
FreeLibrary(hDll);
}
On Win10 x64 produces:
0: 00D80000
100: 03950000
200: 03F90000
[...]
31800: 7FBC0000
31900: 00220000
32000: 00860000
32100: 80140000
32200: 80780000
32300: 80DC0000
32400: 81400000
32500: 81A40000
32600: 82080000
32700: 826C0000
32800: 82D00000
32900: 83340000
finished VirtualAlloc. Loading a DLL.
DLL base: 83780000.

Answer 2

for your own DLL you need set 3 linker option:
/LARGEADDRESSAWARE
/DYNAMICBASE:NO
/BASE:"0x********"
note that link.exe allow only image full located bellow 3GB (0xC0000000) for 32-bit image. in other word, he want that ImageBase + ImageSize <= 0xC0000000
so say /BASE:0xB0000000 will be ok, /BASE:0xBFFF0000 only if your image size <= 0x10000 and for /BASE:0xC0000000 and higher we always got error LNK1249 - image exceeds maximum extent with base address address and size size
also EXE mandatory must have /LARGEADDRESSAWARE too, because are all 4GB space available for wow64 process based only on EXE options.
if we want do this for external DLL - here question more hard. first of all - are this DLL can correct handle this situation (load base > 0x80000000) ? ok. let test this. any api (including most low level LdrLoadDll) not let specify base address, for DLL load. here only hook solution exist.
when library loaded, internal always called ZwMapViewOfSection and it 3-rd parameter BaseAddress - Pointer to a variable that receives the base address of the view. if we set this variable to 0 - system yourself select loaded address. if we set this to specific address - system map view (DLL image in our case) only at this address, or return error STATUS_CONFLICTING_ADDRESSES.
working solution - hook call to ZwMapViewOfSection and replace value of variable, to which point BaseAddress. for find address > 0x80000000 we can use VirtualAlloc with MEM_TOP_DOWN option. note - despite ZwMapViewOfSection also allow use MEM_TOP_DOWN in AllocationType parameter, here it will be have not needed effect - section anyway will be loaded by preferred address or top-down from 0x7FFFFFFF not from 0xFFFFFFFF. but with VirtualAlloc the MEM_TOP_DOWN begin search from 0xFFFFFFFF if process used 4Gb user space. for know - how many memory need for section - we can call ZwQuerySection with SectionBasicInformation - despite this is undocumented - for debug and test - this is ok.
for hook of course can be used some detour lib, but possible do hook with DRx breakpoint - set some Drx register to NtMapViewOfSection address. and set AddVectoredExceptionHandler - which handle exception. this will be perfect work, if process not under debugger. but under debugger it break - most debuggers alwas stop under single step exception and usually no option not handle it but pass to application. of course we can start program not under debugger, and attach it later - after dll load. or possible do this task in separate thread and hide this thread from debugger. disadvantage here - that debugger not got notify about dll load in this case and not load symbols for this. however for external (system dll) for which you have not src code - this in most case can be not a big problem. so solution exit, if we can implement it ). possible code:
PVOID pvNtMapViewOfSection;
LONG NTAPI OnVex(::PEXCEPTION_POINTERS ExceptionInfo)
{
if (ExceptionInfo->ExceptionRecord->ExceptionCode == STATUS_SINGLE_STEP &&
ExceptionInfo->ExceptionRecord->ExceptionAddress == pvNtMapViewOfSection)
{
struct MapViewOfSection_stack
{
PVOID ReturnAddress;
HANDLE SectionHandle;
HANDLE ProcessHandle;
PVOID *BaseAddress;
ULONG_PTR ZeroBits;
SIZE_T CommitSize;
PLARGE_INTEGER SectionOffset;
PSIZE_T ViewSize;
SECTION_INHERIT InheritDisposition;
ULONG AllocationType;
ULONG Win32Protect;
} * stack = (MapViewOfSection_stack*)(ULONG_PTR)ExceptionInfo->ContextRecord->Esp;
if (stack->ProcessHandle == NtCurrentProcess())
{
SECTION_BASIC_INFORMATION sbi;
if (0 <= ZwQuerySection(stack->SectionHandle, SectionBasicInformation, &sbi, sizeof(sbi), 0))
{
if (PVOID pv = VirtualAlloc(0, (SIZE_T)sbi.Size.QuadPart, MEM_RESERVE|MEM_TOP_DOWN, PAGE_NOACCESS))
{
if (VirtualFree(pv, 0, MEM_RELEASE))
{
*stack->BaseAddress = pv;
}
}
}
}
// RESUME_FLAG ( 0x10000) not supported by xp, but anyway not exist 64bit xp
ExceptionInfo->ContextRecord->EFlags |= RESUME_FLAG;
return EXCEPTION_CONTINUE_EXECUTION;
}
return EXCEPTION_CONTINUE_SEARCH;
}
struct LOAD_DATA {
PCWSTR lpLibFileName;
HMODULE hmod;
ULONG dwError;
};
ULONG WINAPI HideFromDebuggerThread(LOAD_DATA* pld)
{
NtSetInformationThread(NtCurrentThread(), ThreadHideFromDebugger, 0, 0);
ULONG dwError = 0;
HMODULE hmod = 0;
if (PVOID pv = AddVectoredExceptionHandler(TRUE, OnVex))
{
::CONTEXT ctx = {};
ctx.ContextFlags = CONTEXT_DEBUG_REGISTERS;
ctx.Dr7 = 0x404;
ctx.Dr1 = (ULONG_PTR)pvNtMapViewOfSection;
if (SetThreadContext(GetCurrentThread(), &ctx))
{
if (hmod = LoadLibraryW(pld->lpLibFileName))
{
pld->hmod = hmod;
}
else
{
dwError = GetLastError();
}
ctx.Dr7 = 0x400;
ctx.Dr1 = 0;
SetThreadContext(GetCurrentThread(), &ctx);
}
else
{
dwError = GetLastError();
}
RemoveVectoredExceptionHandler(pv);
}
else
{
dwError = GetLastError();
}
pld->dwError = dwError;
return dwError;
}
HMODULE LoadLibHigh(PCWSTR lpLibFileName)
{
BOOL bWow;
HMODULE hmod = 0;
if (IsWow64Process(GetCurrentProcess(), &bWow) && bWow)
{
if (pvNtMapViewOfSection = GetProcAddress(GetModuleHandle(L"ntdll"), "NtMapViewOfSection"))
{
LOAD_DATA ld = { lpLibFileName };
if (IsDebuggerPresent())
{
if (HANDLE hThread = CreateThread(0, 0, (PTHREAD_START_ROUTINE)HideFromDebuggerThread, &ld, 0, 0))
{
WaitForSingleObject(hThread, INFINITE);
CloseHandle(hThread);
}
}
else
{
HideFromDebuggerThread(&ld);
}
if (!(hmod = ld.hmod))
{
SetLastError(ld.dwError);
}
}
}
else
{
hmod = LoadLibrary(lpLibFileName);
}
return hmod;
}