frame pointer register 'ebx' modified by inline assembly code - visual-studio-2010

Unfortunately, I had to re-image my laptop to install Visual Studio 2012. My project build but with above warning. Previously I had Visual Studio 2010 and I never got the above warning. The code is as follows:
__asm
{
//Initialize pointers on matrices
mov eax, dword ptr [this]
mov ebx, dword ptr [eax+UPkk]
mov dword ptr [UPkk_ptr],ebx
mov ebx, dword ptr [eax+UPk1k]
mov dword ptr [UPk1k_ptr],ebx
mov ebx, dword ptr [eax+DPk1k]
mov dword ptr [DPk1k_ptr],ebx
mov ebx, dword ptr [eax+DPkk]
mov dword ptr [DPkk_ptr],ebx
mov ebx, dword ptr [eax+mat_A]
mov dword ptr [mat_A_ptr],ebx
mov ebx, dword ptr [eax+vec_a]
mov dword ptr [vec_a_ptr],ebx
mov ebx, dword ptr [eax+vec_b]
mov dword ptr [vec_b_ptr],ebx
}
Do I need to change any settings in the project?
Best Regards
Chintan
Edit: In the above code when I replace ebx with ecx, the warnings go away and the code works fine. However, there is another piece of code where I have used ebx and ecx and in that case my program crashes. Here is the code:
__asm
{
//Initialize UPk1k[idx_4] pointer
mov eax, dword ptr [UPk1k_ptr]
mov ebx, dword ptr [idx_4]
imul ebx,8
add eax,ebx
mov dword ptr [UPk1k_id4_ptr],eax
//Initialize UPkk[idx_4] pointer
mov eax, dword ptr [UPkk_ptr]
mov ebx, dword ptr [idx_4]
imul ebx,8
add eax,ebx
mov dword ptr [UPkk_id4_ptr],eax
//Initialize UPk1k[idx_4] pointer
mov eax, dword ptr [vec_b_ptr]
mov ebx, dword ptr [idx_1]
imul ebx,8
add eax,ebx
mov dword ptr [vec_b_id1_ptr],eax
mov edi, dword ptr [idx_1] //Load idx_1 in edi
mov esi, 0 //initialize loop counter
jmp start_proc11
start_for11:inc esi //idx_2++
start_proc11:cmp esi, edi //idx_2<idx_1 ?
jge end_for11 //If yes so end of the loop
mov eax, UPk1k_id4_ptr //load UPk1k[idx_4] adress
mov ebx, vec_b_ptr //load vec_b adress
mov ecx, esi
imul ecx,8
add eax, ecx //UPk1k[idx_4+idx_2] in eax
add ebx, ecx //vec_b[idx_2] in eax
fld qword ptr [eax]//push UPk1k[idx_4+idx_2]
fld qword ptr [ebx] //push vec_b[idx_2]
mov edx,dword ptr [Sd_ptr]
fmul qword ptr [edx] //vec_b[idx_2]*Sd
fadd //pop UPk1k[idx_4+idx_2]+vec_b[idx_2]*Sd
mov edx,dword ptr [UPkk_id4_ptr]
fstp qword ptr [edx+esi*8] //pop UPkk[idx_4+idx_2]=UPk1k[idx_4+idx_2]+vec_b[idx_2]*Sd
fld qword ptr [ebx] //push vec_b[idx_2]
mov edx,dword ptr [vec_b_id1_ptr]
fld qword ptr [edx] //push vec_b[idx_2]
fmul qword ptr [eax]
fadd
fstp qword ptr [ebx]
jmp start_for11 //end of the loop
end_for11:
}
Many Thanks
Best Regards
CS

See MSDN about registers and that warning. They explain why the warning is produced: it forces the compiler to preserve value of EBX, which might be counter-productive to performance, the usual reason inline asm is used. Relevant quote:
In addition, by using EBX, ESI or EDI in inline assembly code, you
force the compiler to save and restore those registers in the function
prologue and epilogue.
To disable the warning, I think the syntax is
#pragma warning( disable : 4731 )
However, I'd try to use some other register instead, because the warning is there for a good reason, really, like most warnings.
In fact, Looking at your asm code, simply replace ebx With ecx, that should solve the problem.

Related

Binary create a shell as another user

I have a specific question,
I have a binary that launch a shell with execv but the shell change the user and with gdb i can't seem to find where the user change is happening.
level0#RainFall:~$ whoami
level0
level0#RainFall:~$ ls -la
-rwsr-x---+ 1 level1 users 747441 Mar 6 2016 level0
level0#RainFall:~$ gdb
(gdb) file level0
Reading symbols from /home/user/level0/level0...(no debugging symbols found)...done.
(gdb) run 423
Starting program: /home/user/level0/level0 423
process 3718 is executing new program: /bin/dash
$ whoami
level0
But when I don't use gdb:
level0#RainFall:~$ ./level0 423
$ whoami
level1
$
Here is the disas of main
0x08048ec0 <+0>: push ebp
0x08048ec1 <+1>: mov ebp,esp
0x08048ec3 <+3>: and esp,0xfffffff0
0x08048ec6 <+6>: sub esp,0x20
0x08048ec9 <+9>: mov eax,DWORD PTR [ebp+0xc]
0x08048ecc <+12>: add eax,0x4
0x08048ecf <+15>: mov eax,DWORD PTR [eax]
0x08048ed1 <+17>: mov DWORD PTR [esp],eax
0x08048ed4 <+20>: call 0x8049710 <atoi>
0x08048ed9 <+25>: cmp eax,0x1a7
// it compare argv[1] with 423 if it is unequal it print No!
0x08048ede <+30>: jne 0x8048f58 <main+152>
0x08048ee0 <+32>: mov DWORD PTR [esp],0x80c5348
0x08048ee7 <+39>: call 0x8050bf0 <strdup>
0x08048eec <+44>: mov DWORD PTR [esp+0x10],eax
0x08048ef0 <+48>: mov DWORD PTR [esp+0x14],0x0
0x08048ef8 <+56>: call 0x8054680 <getegid>
0x08048efd <+61>: mov DWORD PTR [esp+0x1c],eax
0x08048f01 <+65>: call 0x8054670 <geteuid>
0x08048f06 <+70>: mov DWORD PTR [esp+0x18],eax
0x08048f0a <+74>: mov eax,DWORD PTR [esp+0x1c]
0x08048f0e <+78>: mov DWORD PTR [esp+0x8],eax
0x08048f12 <+82>: mov eax,DWORD PTR [esp+0x1c]
0x08048f16 <+86>: mov DWORD PTR [esp+0x4],eax
0x08048f1a <+90>: mov eax,DWORD PTR [esp+0x1c]
0x08048f1e <+94>: mov DWORD PTR [esp],eax
0x08048f21 <+97>: call 0x8054700 <setresgid>
0x08048f26 <+102>: mov eax,DWORD PTR [esp+0x18]
0x08048f2a <+106>: mov DWORD PTR [esp+0x8],eax
0x08048f2e <+110>: mov eax,DWORD PTR [esp+0x18]
0x08048f32 <+114>: mov DWORD PTR [esp+0x4],eax
0x08048f36 <+118>: mov eax,DWORD PTR [esp+0x18]
0x08048f3a <+122>: mov DWORD PTR [esp],eax
0x08048f3d <+125>: call 0x8054690 <setresuid>
0x08048f42 <+130>: lea eax,[esp+0x10]
0x08048f46 <+134>: mov DWORD PTR [esp+0x4],eax
0x08048f4a <+138>: mov DWORD PTR [esp],0x80c5348
//at this point euid and egid are the one of the user that launched gdb`
0x08048f51 <+145>: call 0x8054640 <execv>
// we never go there since execv opens a shell
0x08048f56 <+150>: jmp 0x8048f80 <main+192>
0x08048f58 <+152>: mov eax,ds:0x80ee170
0x08048f5d <+157>: mov edx,eax
0x08048f5f <+159>: mov eax,0x80c5350
0x08048f64 <+164>: mov DWORD PTR [esp+0xc],edx
0x08048f68 <+168>: mov DWORD PTR [esp+0x8],0x5
0x08048f70 <+176>: mov DWORD PTR [esp+0x4],0x1
0x08048f78 <+184>: mov DWORD PTR [esp],eax
0x08048f7b <+187>: call 0x804a230 <fwrite>
0x08048f80 <+192>: mov eax,0x0
0x08048f85 <+197>: leave
0x08048f86 <+198>: ret
End of assembler dump.
I don't understand how the binary changes behaviour if I execute it by gdb or in the shell, maybe its because the binary's proprietary is level1?
If someone has the time to explain to me how it works I'll be really greatful
Thanks a lot
I don't understand how the binary changes behaviour
The binary doesn't -- the kernel creates a new process with different UID when presented a set-uid binary (that's what s in -rwsr-x---+ means).
For obvious security reasons the kernel doesn't do that when the binary is being debugged.

What does the "C" used as an offset in assembly mean?

I have the following piece of code:
RtlGetElementGenericTable:
7C9624E0 PUSH EBP
7C9624E1 MOV EBP,ESP
7C9624E3 MOV ECX,DWORD PTR [EBP+8]
7C9624E6 MOV EDX,DWORD PTR [ECX+14]
7C9624E9 MOV EAX,DWORD PTR [ECX+C]
And I was wondering what does the "C" in MOV EAX,DWORD PTR [ECX+C] means.
Is a variable? Is it a register? Is it something else?
The C in MOV EAX,DWORD PTR [ECX+C] is hexadecimal C (decimal 12).
And note that 8 in MOV ECX,DWORD PTR [EBP+8] and 14 in MOV EDX,DWORD PTR [ECX+14] are also hexadecimal numbers.
It's probably an hexadecimal value. So it means 12.

What syntax does a disassembled .s file is not acceptable in nasm?

This code is a C program (bubble sort) disassembled into assembly. How can I make the following code run if I put it in a .asm file and use nasm to assemble? If you know what needs changing, please say what to change it to. For instance I understand that nasm won't accept DWORD PTR, but I haven't found out what to do instead. Thanks
.file "sort.c" .intel_syntax noprefix .text .globl
sort .type sort, #function
sort: .LFB0:
.cfi_startproc
push rbp
.cfi_def_cfa_offset 16
.cfi_offset 6, -16
mov rbp, rsp
.cfi_def_cfa_register 6
mov QWORD PTR [rbp-24], rdi
mov DWORD PTR [rbp-28], esi
mov DWORD PTR [rbp-12], 0
jmp .L2
.L6:
mov DWORD PTR [rbp-8], 0
jmp .L3
.L5:
mov eax, DWORD PTR [rbp-8]
cdqe
sal rax, 2
add rax, QWORD PTR [rbp-24]
mov edx, DWORD PTR [rax]
mov eax, DWORD PTR [rbp-8]
cdqe
add rax, 1
sal rax, 2
add rax, QWORD PTR [rbp-24]
mov eax, DWORD PTR [rax]
cmp edx, eax
jle .L4
mov eax, DWORD PTR [rbp-8]
cdqe
sal rax, 2
add rax, QWORD PTR [rbp-24]
mov eax, DWORD PTR [rax]
mov DWORD PTR [rbp-4], eax
mov eax, DWORD PTR [rbp-8]
cdqe
sal rax, 2
add rax, QWORD PTR [rbp-24]
mov edx, DWORD PTR [rbp-8]
movsx rdx, edx
add rdx, 1
sal rdx, 2
add rdx, QWORD PTR [rbp-24]
mov edx, DWORD PTR [rdx]
mov DWORD PTR [rax], edx
mov eax, DWORD PTR [rbp-8]
cdqe
add rax, 1
sal rax, 2
add rax, QWORD PTR [rbp-24]
mov edx, DWORD PTR [rbp-4]
mov DWORD PTR [rax], edx
.L4:
add DWORD PTR [rbp-8], 1
.L3:
mov eax, DWORD PTR [rbp-28]
sub eax, 1
sub eax, DWORD PTR [rbp-12]
cmp eax, DWORD PTR [rbp-8]
jg .L5
add DWORD PTR [rbp-12], 1
.L2:
mov eax, DWORD PTR [rbp-28]
sub eax, 1
cmp eax, DWORD PTR [rbp-12]
jg .L6
pop rbp
.cfi_def_cfa 7, 8
ret
.cfi_endproc
.LFE0:
.size sort, .-sort
.ident "GCC: (Ubuntu/Linaro 4.6.3-1ubuntu5) 4.6.3"
.section .note.GNU-stack,"",#progbits
Just remove PTR and all nonsensical .somethings.
This assembles just fine:
; file: gas-nasm-sort.asm
bits 64
sort:
push rbp
mov rbp, rsp
mov QWORD [rbp-24], rdi
mov DWORD [rbp-28], esi
mov DWORD [rbp-12], 0
jmp .L2
.L6:
mov DWORD [rbp-8], 0
jmp .L3
.L5:
mov eax, DWORD [rbp-8]
cdqe
sal rax, 2
add rax, QWORD [rbp-24]
mov edx, DWORD [rax]
mov eax, DWORD [rbp-8]
cdqe
add rax, 1
sal rax, 2
add rax, QWORD [rbp-24]
mov eax, DWORD [rax]
cmp edx, eax
jle .L4
mov eax, DWORD [rbp-8]
cdqe
sal rax, 2
add rax, QWORD [rbp-24]
mov eax, DWORD [rax]
mov DWORD [rbp-4], eax
mov eax, DWORD [rbp-8]
cdqe
sal rax, 2
add rax, QWORD [rbp-24]
mov edx, DWORD [rbp-8]
movsx rdx, edx
add rdx, 1
sal rdx, 2
add rdx, QWORD [rbp-24]
mov edx, DWORD [rdx]
mov DWORD [rax], edx
mov eax, DWORD [rbp-8]
cdqe
add rax, 1
sal rax, 2
add rax, QWORD [rbp-24]
mov edx, DWORD [rbp-4]
mov DWORD [rax], edx
.L4:
add DWORD [rbp-8], 1
.L3:
mov eax, DWORD [rbp-28]
sub eax, 1
sub eax, DWORD [rbp-12]
cmp eax, DWORD [rbp-8]
jg .L5
add DWORD [rbp-12], 1
.L2:
mov eax, DWORD [rbp-28]
sub eax, 1
cmp eax, DWORD [rbp-12]
jg .L6
pop rbp
ret
Command:
nasm gas-nasm-sort.asm -f bin -o gas-nasm-sort.bin
But again, there's NASM documentation. Read it. In particular these sections:
2.2.2 NASM Requires Square Brackets For Memory References
2.2.3 NASM Doesn't Store Variable Types

Disassembling CGContextReplacePathWithShapePath()?

I am beginner at dis-assembly. Below is the disassembly of CGContextReplacePathWithShapePath from IDA pro. I am trying to figure out the parameters to the routine.
Any help is highly appreciated. Also please post some pointers/hints on how to attack the dis-assembly.
__text:00024D1F ; =============== S U B R O U T I N E =======================================
__text:00024D1F
__text:00024D1F ; Attributes: bp-based frame
__text:00024D1F
__text:00024D1F public _CGContextReplacePathWithShapePath
__text:00024D1F _CGContextReplacePathWithShapePath proc near
__text:00024D1F
__text:00024D1F var_2C = dword ptr -2Ch
__text:00024D1F var_20 = dword ptr -20h
__text:00024D1F var_1C = dword ptr -1Ch
__text:00024D1F arg_0 = dword ptr 8
__text:00024D1F arg_4 = dword ptr 0Ch
__text:00024D1F
__text:00024D1F push ebp
__text:00024D20 mov ebp, esp
__text:00024D22 push edi
__text:00024D23 push esi
__text:00024D24 push ebx
__text:00024D25 sub esp, 3Ch
__text:00024D28 call $+5
__text:00024D2D pop ebx
__text:00024D2E mov esi, [ebp+arg_0]
__text:00024D31 test esi, esi
__text:00024D33 jz short loc_24D3E
__text:00024D35 cmp dword ptr [esi+8], 43545854h
__text:00024D3C jz short loc_24D5F
__text:00024D3E
__text:00024D3E loc_24D3E: ; CODE XREF: _CGContextReplacePathWithShapePath+14j
__text:00024D3E mov [esp+8], esi
__text:00024D42 lea eax, (___func___26084 - 24D2Dh)[ebx] ; "CGContextReplacePathWithShapePath"
__text:00024D48 mov [esp+4], eax
__text:00024D4C lea eax, (aSInvalidContex - 24D2Dh)[ebx] ; "%s: invalid context %p"
__text:00024D52 mov [esp], eax
__text:00024D55 call _CGPostError
__text:00024D5A jmp loc_24DF2
__text:00024D5F ; ---------------------------------------------------------------------------
__text:00024D5F
__text:00024D5F loc_24D5F: ; CODE XREF: _CGContextReplacePathWithShapePath+1Dj
__text:00024D5F mov eax, [esi+58h]
__text:00024D62 test eax, eax
__text:00024D64 jz short loc_24D75
__text:00024D66 mov [esp], eax
__text:00024D69 call _CGPathRelease
__text:00024D6E mov dword ptr [esi+58h], 0
__text:00024D75
__text:00024D75 loc_24D75: ; CODE XREF: _CGContextReplacePathWithShapePath+45j
__text:00024D75 mov eax, [ebp+arg_4]
__text:00024D78 mov [esp], eax
__text:00024D7B call _CGSRegionPathEnumerator
__text:00024D80 mov edi, eax
__text:00024D82 xor ebx, ebx
__text:00024D84 lea eax, [ebp+var_20]
__text:00024D87 mov [ebp+var_2C], eax
__text:00024D8A jmp short loc_24DCA
__text:00024D8C ; ---------------------------------------------------------------------------
__text:00024D8C
__text:00024D8C loc_24D8C: ; CODE XREF: _CGContextReplacePathWithShapePath+BDj
__text:00024D8C jge short loc_24DB4
__text:00024D8E inc ebx
__text:00024D8F cmp ebx, 1
__text:00024D92 jz short loc_24D9C
__text:00024D94 mov [esp], esi
__text:00024D97 call _CGContextClosePath
__text:00024D9C
__text:00024D9C loc_24D9C: ; CODE XREF: _CGContextReplacePathWithShapePath+73j
__text:00024D9C mov eax, [ebp+var_1C]
__text:00024D9F mov [esp+8], eax
__text:00024DA3 mov eax, [ebp+var_20]
__text:00024DA6 mov [esp+4], eax
__text:00024DAA mov [esp], esi
__text:00024DAD call _CGContextMoveToPoint
__text:00024DB2 jmp short loc_24DCA
__text:00024DB4 ; ---------------------------------------------------------------------------
__text:00024DB4
__text:00024DB4 loc_24DB4: ; CODE XREF: _CGContextReplacePathWithShapePath:loc_24D8Cj
__text:00024DB4 mov eax, [ebp+var_1C]
__text:00024DB7 mov [esp+8], eax
__text:00024DBB mov eax, [ebp+var_20]
__text:00024DBE mov [esp+4], eax
__text:00024DC2 mov [esp], esi
__text:00024DC5 call _CGContextAddLineToPoint
__text:00024DCA
__text:00024DCA loc_24DCA: ; CODE XREF: _CGContextReplacePathWithShapePath+6Bj
__text:00024DCA ; _CGContextReplacePathWithShapePath+93j
__text:00024DCA mov eax, [ebp+var_2C]
__text:00024DCD mov [esp+4], eax
__text:00024DD1 mov [esp], edi
__text:00024DD4 call _CGSNextPoint
__text:00024DD9 cmp eax, 0
__text:00024DDC jnz short loc_24D8C
__text:00024DDE test ebx, ebx
__text:00024DE0 jz short loc_24DEA
__text:00024DE2 mov [esp], esi
__text:00024DE5 call _CGContextClosePath
__text:00024DEA
__text:00024DEA loc_24DEA: ; CODE XREF: _CGContextReplacePathWithShapePath+C1j
__text:00024DEA mov [esp], edi
__text:00024DED call _CGSReleaseRegionEnumerator
__text:00024DF2
__text:00024DF2 loc_24DF2: ; CODE XREF: _CGContextReplacePathWithShapePath+3Bj
__text:00024DF2 add esp, 3Ch
__text:00024DF5 pop ebx
__text:00024DF6 pop esi
__text:00024DF7 pop edi
__text:00024DF8 leave
__text:00024DF9 retn
__text:00024DF9 _CGContextReplacePathWithShapePath endp
The first parameter is a context parameter of some sort.
The function checks to see if it's NULL and in that case jumps to loc_24D3E (where you can see it calls an error function _CGPostError with the format string "%s: invalid context %p"). BTW: Right after it checks a magic value in the context (at context+8) to see if it is valid otherwise it exits through the same error path. The magic value43545854h is CTXT as chars btw.
__text:00024D2E mov esi, [ebp+arg_0]
__text:00024D31 test esi, esi
__text:00024D33 jz short loc_24D3E
As for the second parameter, it looks to be some kind of path as it is used as the sole parameter to CGSRegionPathEnumerator.
__text:00024D75 mov eax, [ebp+arg_4]
__text:00024D78 mov [esp], eax
__text:00024D7B call _CGSRegionPathEnumerator
In C the function would look something like this:
void CGContextReplacePathWithShapePath(Context* context, Path* path) {
if(context == NULL || context->magic != 0x43545854) {
CGPostError("%s: invalid context %p", "CGContextReplacePathWithShapePath", context);
return;
}
// loc_24D5F
if (context->path != NULL) {
CGPathRelease(context->path);
context->path = NULL;
}
// loc_24D75
RegionPathEnumerator* rpe = CGSRegionPathEnumerator(path);
// ....
}

Reversing of _PrepareMenuWindow() subroutine

Can some one help me with reversing of _PrepareMenuWindow() subroutine?
I am trying to find the signature of the method.
__text:000639A7 _PrepareMenuWindow proc near ; CODE XREF: DrawTheMenu(MenuSelectData *,__CFArray **,uchar,uchar *)+274p
__text:000639A7 ; PopUpMenuSelectCore(MenuData *,Point,double,Point,ushort,uint,Rect const*,ushort,ulong,Rect const*,Rect const*,__CFString const*,OpaqueMenuRef **,ushort *)+528p
__text:000639A7
__text:000639A7 var_44 = dword ptr -44h
__text:000639A7 var_40 = dword ptr -40h
__text:000639A7 var_3C = dword ptr -3Ch
__text:000639A7 var_34 = dword ptr -34h
__text:000639A7 var_30 = dword ptr -30h
__text:000639A7 var_2C = dword ptr -2Ch
__text:000639A7 var_28 = dword ptr -28h
__text:000639A7 var_24 = word ptr -24h
__text:000639A7 var_20 = dword ptr -20h
__text:000639A7 var_1A = word ptr -1Ah
__text:000639A7 arg_0 = dword ptr 8
__text:000639A7 arg_4 = dword ptr 0Ch
__text:000639A7 arg_8 = dword ptr 10h
__text:000639A7
__text:000639A7 push ebp
__text:000639A8 mov ebp, esp
__text:000639AA push edi
__text:000639AB push esi
__text:000639AC push ebx
__text:000639AD sub esp, 5Ch
__text:000639B0 xor edi, edi
__text:000639B2 mov eax, [ebp+arg_0]
__text:000639B5 test eax, eax
__text:000639B7 jz short loc_639C6
__text:000639B9 mov eax, [ebp+arg_0]
__text:000639BC mov [esp], eax
__text:000639BF call __ZNK8HIObject13GetEncodedRefEv ; HIObject::GetEncodedRef(void)
__text:000639C4 mov edi, eax
__text:000639C6
__text:000639C6 loc_639C6: ; CODE XREF: _PrepareMenuWindow+10j
__text:000639C6 mov ecx, [ebp+arg_4]
__text:000639C9 mov eax, [ecx]
__text:000639CB mov edx, [ecx+4]
__text:000639CE mov [ebp+var_2C], eax
__text:000639D1 mov [ebp+var_28], edx
__text:000639D4 lea eax, [ebp+var_1A]
__text:000639D7 mov [ebp+var_40], eax
__text:000639DA mov [esp+4], eax
__text:000639DE mov [esp], edi
__text:000639E1 call _GetMenuType
__text:000639E6 mov dword ptr [esp+4], 0
__text:000639EE mov [esp], edi
__text:000639F1 call _IsMenuItemEnabled
__text:000639F6 movzx edx, [ebp+var_1A]
__text:000639FA or dh, 1
__text:000639FD test al, al
__text:000639FF movzx ebx, [ebp+var_1A]
__text:00063A03 cmovz ebx, edx
__text:00063A06 mov [ebp+var_1A], bx
__text:00063A0A mov eax, [ebp+arg_8]
__text:00063A0D mov [esp+0Ch], eax
__text:00063A11 lea ecx, [ebp+var_2C]
__text:00063A14 mov [ebp+var_44], ecx
__text:00063A17 mov [esp+8], ecx
__text:00063A1B mov eax, [ebp+arg_4]
__text:00063A1E mov [esp+4], eax
__text:00063A22 mov [esp], edi
__text:00063A25 call __AddOpenMenu
__text:00063A2A mov ecx, [ebp+var_44]
__text:00063A2D mov [esp], ecx
__text:00063A30 call _EmptyRect
__text:00063A35 test al, al
__text:00063A37 jnz loc_63B94
__text:00063A3D mov [esp], edi
__text:00063A40 call __Z11GetMenuDataP13OpaqueMenuRef ; GetMenuData(OpaqueMenuRef *)
__text:00063A45 mov [ebp+var_3C], eax
__text:00063A48 call _NewRgn
__text:00063A4D mov esi, eax
__text:00063A4F test eax, eax
__text:00063A51 jz loc_63BDD
__text:00063A57 movzx ebx, bx
__text:00063A5A mov eax, [ebp+var_3C]
__text:00063A5D mov eax, [eax+40h]
__text:00063A60 test eax, eax
__text:00063A62 jnz loc_63B23
__text:00063A68 mov [ebp+var_1A], 0
__text:00063A6E mov eax, [ebp+var_2C]
__text:00063A71 mov edx, [ebp+var_28]
__text:00063A74 mov [ebp+var_34], eax
__text:00063A77 mov [ebp+var_30], edx
__text:00063A7A mov ecx, [ebp+var_40]
__text:00063A7D mov [esp+10h], ecx
__text:00063A81 mov dword ptr [esp+0Ch], 0
__text:00063A89 lea eax, [ebp+var_34]
__text:00063A8C mov [esp+8], eax
__text:00063A90 mov dword ptr [esp+4], 7
__text:00063A98 mov eax, [ebp+var_3C]
__text:00063A9B mov [esp], eax
__text:00063A9E call __Z12_CallMenuDefP8MenuDatasP4Rect5PointPs ; _CallMenuDef(MenuData *,short,Rect *,Point,short *)
__text:00063AA3 cmp [ebp+var_1A], 7473h
__text:00063AA9 jz short loc_63ADC
__text:00063AAB add word ptr [ebp+var_2C], 3
__text:00063AB0 mov dword ptr [esp+8], 0FFFFFFFCh
__text:00063AB8 mov dword ptr [esp+4], 0FFFFFFFCh
__text:00063AC0 mov ecx, [ebp+var_44]
__text:00063AC3 mov [esp], ecx
__text:00063AC6 call _InsetRect
__text:00063ACB mov eax, [ebp+var_44]
__text:00063ACE mov [esp+4], eax
__text:00063AD2 mov [esp], esi
__text:00063AD5 call _RectRgn
__text:00063ADA jmp short loc_63B23
__text:00063ADC ; ---------------------------------------------------------------------------
__text:00063ADC
__text:00063ADC loc_63ADC: ; CODE XREF: _PrepareMenuWindow+102j
__text:00063ADC lea eax, [ebp+var_24]
__text:00063ADF mov [esp+8], eax
__text:00063AE3 lea eax, [ebp+var_20]
__text:00063AE6 mov [esp+4], eax
__text:00063AEA mov [esp], edi
__text:00063AED call __GetMenuCallout
__text:00063AF2 movsx eax, [ebp+var_24]
__text:00063AF6 mov [esp+10h], eax
__text:00063AFA mov eax, [ebp+var_20]
__text:00063AFD mov [esp+0Ch], eax
__text:00063B01 mov [esp+8], esi
__text:00063B05 mov [esp+4], ebx
__text:00063B09 mov ecx, [ebp+var_44]
__text:00063B0C mov [esp], ecx
__text:00063B0F call __GetThemeMenuBackgroundRegionWithCallout
__text:00063B14 mov eax, [ebp+var_44]
__text:00063B17 mov [esp+4], eax
__text:00063B1B mov [esp], esi
__text:00063B1E call _GetRegionBounds
__text:00063B23
__text:00063B23 loc_63B23: ; CODE XREF: _PrepareMenuWindow+BBj
__text:00063B23 ; _PrepareMenuWindow+133j
__text:00063B23 mov [esp+0Ch], esi
__text:00063B27 mov ecx, [ebp+var_44]
__text:00063B2A mov [esp+8], ecx
__text:00063B2E mov [esp+4], ebx
__text:00063B32 mov [esp], edi
__text:00063B35 call __ZL13GetMenuWindowP13OpaqueMenuReftPK4RectP15OpaqueRgnHandle ; GetMenuWindow(OpaqueMenuRef *,ushort,Rect const*,OpaqueRgnHandle *)
__text:00063B3A test eax, eax
__text:00063B3C jz short loc_63BA1
__text:00063B3E mov [esp], eax
__text:00063B41 call _GetWindowPort
__text:00063B46 mov [esp], eax
__text:00063B49 call _SetPortWrapper
__text:00063B4E mov [esp], esi
__text:00063B51 call _SetClipWrapper
__text:00063B56 mov [esp], esi
__text:00063B59 call _DisposeRgn
__text:00063B5E mov eax, [ebp+var_3C]
__text:00063B61 mov eax, [eax+40h]
__text:00063B64 test eax, eax
__text:00063B66 jnz short loc_63BDD
__text:00063B68 mov dword ptr [esp+14h], 0
__text:00063B70 mov dword ptr [esp+10h], 0
__text:00063B78 mov [esp+0Ch], ebx
__text:00063B7C mov ecx, [ebp+arg_4]
__text:00063B7F mov [esp+8], ecx
__text:00063B83 mov eax, [ebp+var_44]
__text:00063B86 mov [esp+4], eax
__text:00063B8A mov [esp], edi
__text:00063B8D call __Z18DrawMenuBackgroundP13OpaqueMenuRefRK4RectS3_thPv ; DrawMenuBackground(OpaqueMenuRef *,Rect const&,Rect const&,ushort,uchar,void *)
__text:00063B92 jmp short loc_63BDD
__text:00063B94 ; ---------------------------------------------------------------------------
__text:00063B94
__text:00063B94 loc_63B94: ; CODE XREF: _PrepareMenuWindow+90j
__text:00063B94 mov ecx, [ebp+arg_0]
__text:00063B97 mov [esp], ecx
__text:00063B9A call _DisposeMenuWindow
__text:00063B9F jmp short loc_63BDD
__text:00063BA1 ; ---------------------------------------------------------------------------
__text:00063BA1
__text:00063BA1 loc_63BA1: ; CODE XREF: _PrepareMenuWindow+195j
__text:00063BA1 mov eax, [ebp+arg_0]
__text:00063BA4 mov [esp], eax
__text:00063BA7 call __Z11FindMBEntryP8MenuData ; FindMBEntry(MenuData *)
__text:00063BAC mov ecx, eax
__text:00063BAE test eax, eax
__text:00063BB0 jz short loc_63BD5
__text:00063BB2 mov word ptr [eax+1Eh], 0
__text:00063BB8 mov word ptr [eax+1Ch], 0
__text:00063BBE mov word ptr [eax+1Ah], 0
__text:00063BC4 mov word ptr [eax+18h], 0
__text:00063BCA mov eax, [eax+18h]
__text:00063BCD mov edx, [ecx+1Ch]
__text:00063BD0 mov [ecx], eax
__text:00063BD2 mov [ecx+4], edx
__text:00063BD5
__text:00063BD5 loc_63BD5: ; CODE XREF: _PrepareMenuWindow+209j
__text:00063BD5 mov [esp], esi
__text:00063BD8 call _DisposeRgn
__text:00063BDD
__text:00063BDD loc_63BDD: ; CODE XREF: _PrepareMenuWindow+AAj
__text:00063BDD ; _PrepareMenuWindow+1BFj ...
__text:00063BDD xor eax, eax
__text:00063BDF add esp, 5Ch
__text:00063BE2 pop ebx
__text:00063BE3 pop esi
__text:00063BE4 pop edi
__text:00063BE5 leave
__text:00063BE6 retn
__text:00063BE6 _PrepareMenuWindow endp
What have you got so far that isn't generated by IDA? (ie: your analysis of the function).
From the looks of it its a __cdecl function that always returns NULL/false/0. It also seems to take 3 arguments(which can be confirmed by looking at what cleanup is by the caller, if there is any).
Arg 0 is a MenuData*, arg 4 seems to be a Rect&(which is secretly just Rect*), arg 8 would be whatever type __AddOpenMenu takes as its fourth argument.
So i'd assume something along the lines of typedef BOOL(__cdecl*)(MenuData*,Rect&,void*)

Resources