Why WriteConsoleW breaks After call CoInitialize using ml64 - windows

Question

I am trying to do some Office automation via 64-bit assembly using ml64.exe from Visual Studio 2019. Before I can call the Office COM interfaces I need to call CoInitialize. I am currently just testing initializing COM and writing to the console (I normally don't write assembly code) If I comment out line
call CoInitialize
The WriteConsoleW api call works as expected and outputs message to the screen "COM Failed to Initialize" However as soon as I add call CoInitialize back there is nothing output to console screen, and crash either.
; *************************************************************************
; Proto types for API functions and structures
; *************************************************************************
EXTRN GetStdHandle:PROC
EXTRN WriteConsoleW:PROC
EXTRN CoCreateInstance:PROC
EXTRN CoInitialize:PROC
EXTRN SysFreeString:PROC
EXTRN SysStringByteLen:PROC
EXTRN SysAllocStringByteLen:PROC
EXTRN OleRun:PROC
EXTRN ExitProcess:PROC
.const
STD_OUTPUT_HANDLE = -11
STD_ERROR_HANDLE = -12
; *************************************************************************
; Object libraries
; *************************************************************************
includelib user32.lib
includelib kernel32.lib
includelib ole32.lib
includelib oleaut32.lib
; *************************************************************************
; Our data section.
; *************************************************************************
.data
strErrComFailed dw 'C','O','M',' ','F','a','i','l','e','d',' ','t','o',' ','i','n','i','t','i','a','l','i','z','e',0,0
strErrOutlookFailed dw 'F','a','i','l','e','d',' ','t','o',' ','i','n','i','t','i','a','l','i','z','e',' ','O','u','t','l','o','o','k',0,0
; {0006F03A-0000-0000-C000-000000000046}
CLSID_OutlookApplication dd 0006f03ah
dw 0000h
dw 0000h
dw 0C000h
db 00h
db 00h
db 00h
db 00h
db 00h
db 46h
; {00063001-0000-0000-C000-000000000046}
IID_OutlookApplication dd 00063001h
dw 0000h
dw 0000h
dw 0C000h
db 00h
db 00h
db 00h
db 00h
db 00h
db 46h
; {00000000-0000-0000-C000-000000000046}
IID_IUnknown dd 00000000h
dw 0000h
dw 0000h
dw 0C000h
db 00h
db 00h
db 00h
db 00h
db 00h
db 46h
; *************************************************************************
; Our executable assembly code starts here in the .code section
; *************************************************************************
.code
wcslen PROC inputString:QWORD
LOCAL stringLength:QWORD
mov QWORD PTR inputString, rcx
mov QWORD PTR stringLength, 0h
continue:
mov rax, QWORD PTR inputString
mov rcx, QWORD PTR stringLength
movzx eax, word ptr [rax+rcx*2]
test eax, eax
je finished
mov rax, QWORD PTR stringLength
inc rax
mov QWORD PTR stringLength, rax
jmp continue
finished:
mov rax, QWORD PTR stringLength
ret
wcslen ENDP
main PROC
LOCAL hStdOutput:QWORD
LOCAL hErrOutput:QWORD
LOCAL hResult:DWORD
xor ecx,ecx
call CoInitialize
mov DWORD PTR hResult, eax
mov ecx, STD_OUTPUT_HANDLE
call GetStdHandle
mov QWORD PTR hStdOutput, rax
mov ecx, STD_ERROR_HANDLE
call GetStdHandle
mov QWORD PTR hErrOutput, rax
lea rcx,strErrComFailed
call wcslen
mov QWORD PTR [rsp+32], 0
xor r9d, r9d ; lpNumberOfCharsWritten
mov r8d, eax ; nNumberOfCharsToWrite
lea rdx,QWORD PTR strErrComFailed
mov rcx,QWORD PTR hStdOutput
call WriteConsoleW
; When the message box has been closed, exit the app with exit code eax
mov ecx, eax
call ExitProcess
ret
main ENDP
End
Prior to CoInitialize being called WinDbg shows the following register state:
00007ff7`563e1041 e865000000 call Win64App+0x10ab (00007ff7`563e10ab)
0:000> r
rax=00007ff7563e1037 rbx=0000000000000000 rcx=0000000000000000
rdx=00007ff7563e1037 rsi=0000000000000000 rdi=0000000000000000
rip=00007ff7563e1041 rsp=000000a905affa58 rbp=000000a905affa70
r8=000000a9058d6000 r9=00007ff7563e1037 r10=0000000000000000
r11=0000000000000000 r12=0000000000000000 r13=0000000000000000
r14=0000000000000000 r15=0000000000000000
iopl=0 nv up ei pl zr na po nc
cs=0033 ss=002b ds=002b es=002b fs=0053 gs=002b efl=00000246
Win64App+0x1041:
00007ff7`563e1041 e865000000 call Win64App+0x10ab (00007ff7`563e10ab)
0:000> r ecx
ecx=0
Following CoInitialize being called there is the following register state:
0:000> r
rax=0000000000000000 rbx=0000000000000000 rcx=8aa77f80a0990000
rdx=0000000000000015 rsi=0000000000000000 rdi=0000000000000000
rip=00007ff7563e1046 rsp=000000a905affa58 rbp=000000a905affa70
r8=0000029af97e2620 r9=0000029af97e1440 r10=0000000000000005
r11=000000a905aff9d8 r12=0000000000000000 r13=0000000000000000
r14=0000000000000000 r15=0000000000000000
iopl=0 nv up ei pl nz na po nc
cs=0033 ss=002b ds=002b es=002b fs=0053 gs=002b efl=00000206
Win64App+0x1046:
00007ff7`563e1046 8945ec mov dword ptr [rbp-14h],eax ss:000000a9`05affa5c=00000000
0:000> r eax
eax=0
After calling GetStdHandle :
0:000> r
rax=0000000000000074 rbx=0000000000000000 rcx=0000029af97d2840
rdx=0000000000000015 rsi=0000000000000000 rdi=0000000000000000
rip=00007ff7563e1053 rsp=000000a905affa58 rbp=000000a905affa70
r8=0000029af97e2620 r9=0000029af97e1440 r10=0000000000000005
r11=000000a905aff9d8 r12=0000000000000000 r13=0000000000000000
r14=0000000000000000 r15=0000000000000000
iopl=0 nv up ei pl nz na po nc
cs=0033 ss=002b ds=002b es=002b fs=0053 gs=002b efl=00000206
On call to WriteConsoleW, it seems like parameters are still correct but nothing output to screen:
KERNEL32!WriteConsoleW:
00007ffb`a97028f0 ff258a4c0500 jmp qword ptr [KERNEL32!_imp_WriteConsoleW (00007ffb`a9757580)] ds:00007ffb`a9757580={KERNELBASE!WriteConsoleW (00007ffb`a697b750)}
0:000> du rdx
00007ff7`563e3000 "COM Failed to initialize"
0:000> r
rax=0000000000000018 rbx=0000000000000000 rcx=0000000000000074
rdx=00007ff7563e3000 rsi=0000000000000000 rdi=0000000000000000
rip=00007ffba97028f0 rsp=000000a905affa50 rbp=000000a905affa70
r8=0000000000000018 r9=0000000000000000 r10=0000000000000005
r11=000000a905aff9d8 r12=0000000000000000 r13=0000000000000000
r14=0000000000000000 r15=0000000000000000
iopl=0 nv up ei pl zr na po nc
cs=0033 ss=002b ds=002b es=002b fs=0053 gs=002b efl=00000246
KERNEL32!WriteConsoleW:
00007ffb`a97028f0 ff258a4c0500 jmp qword ptr [KERNEL32!_imp_WriteConsoleW (00007ffb`a9757580)] ds:00007ffb`a9757580={KERNELBASE!WriteConsoleW (00007ffb`a697b750)}
I tried using CoInitializeEx instead and had same problem:
mov edx, COINIT_APARTMENTTHREADED ; dwCoInit (COINIT_APARTMENTTHREADED = 2)
xor ecx, ecx ; pvReserved
call CoInitializeEx

Answer 1

x64 ABI require The stack is always 16-byte aligned when a call instruction is executed also 32 byte reserved space. so at every function entry point we will be have:
RSP == 16*N + 8
so we in general must do SUB RSP,40 + N*16 in function body, if we will call another functions.
but when we declare LOCAL variables in function - compiler (masm64) do some stack allocation but not carry about stack align and 32 byte reserved space. so need do this yourself. also when you using LOCAL variables - masm64 use RBP register for save old RSP value and restore it at the end (use leave instruction). and access locals via RBP so you can not change RBP in function yourself.
so code can be next
STD_OUTPUT_HANDLE = -11
STD_ERROR_HANDLE = -12
EXTRN __imp_GetStdHandle:QWORD
EXTRN __imp_WriteConsoleW:QWORD
EXTRN __imp_CoInitialize:QWORD
EXTRN __imp_ExitProcess:QWORD
EXTRN __imp__getch:QWORD
EXTRN __imp_wcslen:QWORD
WSTRING macro text:VARARG
FOR arg, <text>
if #InStr( , arg, # )
f = 0
FORC c, <arg>
IF f
DW '&c'
ENDIF
f = 1
ENDM
else
DW &arg
endif
ENDM
DW 0
ENDM
.const
strErrComFailed: WSTRING #----, 13, 10, #Press any key:, 13, 10
.code
maina PROC
LOCAL hStdOutput:QWORD
LOCAL hErrOutput:QWORD
LOCAL hResult:DWORD
sub rsp,32
and rsp,not 15
xor ecx,ecx
call __imp_CoInitialize
mov hResult, eax
mov ecx, STD_OUTPUT_HANDLE
call __imp_GetStdHandle
mov hStdOutput, rax
mov ecx, STD_ERROR_HANDLE
call __imp_GetStdHandle
mov hErrOutput, rax
lea rcx,strErrComFailed
call __imp_wcslen
mov QWORD PTR [rsp+32], 0
xor r9d, r9d ; lpNumberOfCharsWritten
mov r8d, eax ; nNumberOfCharsToWrite
lea rdx,strErrComFailed
mov rcx,hStdOutput
call __imp_WriteConsoleW
call __imp__getch
mov ecx, eax
call __imp_ExitProcess
ret
maina ENDP
END
also some notes:
import functions always called via pointer. all this pointer names begin from __imp_ prefix. so we need declare imported api Xxx as EXTRN __imp_Xxx:QWORD - this more efficient compare PROC declaration - in this case linker need to build stub proc with single jmp instruction:
Xxx proc
jmp __imp_Xxx
Xxx endp
of course better do direct call __imp_Xxx and not have Xxx stub, instead call Xxx - code will be smaller and faster
you not need implement wcslen yourself - you can import it implementation from ntdllp.lib (always) or msvcrt.lib (very depend from concrete lib implementation, but msvcrt.dll of course export wcslen - you can build msvcrt.lib yourself) and use call __imp_wcslen
use declaration like
strErrComFailed dw 'C','O','M',' '...
very uncomfortable. you can use for example such a macro
WSTRING macro text:VARARG
FOR arg, <text>
if #InStr( , arg, # )
f = 0
FORC c, <arg>
IF f
DW '&c'
ENDIF
f = 1
ENDM
else
DW &arg
endif
ENDM
DW 0
ENDM
;strErrComFailed: WSTRING #----, 13, 10, #Press any key:, 13, 10
finally the lpNumberOfCharsWritten parameter in function WriteConsoleW is optional. if you look for declaration is sdk - it declared with __out_opt or _Out_opt_. so you can pass 0 here if not need such info