Related
I have a specific question,
I have a binary that launch a shell with execv but the shell change the user and with gdb i can't seem to find where the user change is happening.
level0#RainFall:~$ whoami
level0
level0#RainFall:~$ ls -la
-rwsr-x---+ 1 level1 users 747441 Mar 6 2016 level0
level0#RainFall:~$ gdb
(gdb) file level0
Reading symbols from /home/user/level0/level0...(no debugging symbols found)...done.
(gdb) run 423
Starting program: /home/user/level0/level0 423
process 3718 is executing new program: /bin/dash
$ whoami
level0
But when I don't use gdb:
level0#RainFall:~$ ./level0 423
$ whoami
level1
$
Here is the disas of main
0x08048ec0 <+0>: push ebp
0x08048ec1 <+1>: mov ebp,esp
0x08048ec3 <+3>: and esp,0xfffffff0
0x08048ec6 <+6>: sub esp,0x20
0x08048ec9 <+9>: mov eax,DWORD PTR [ebp+0xc]
0x08048ecc <+12>: add eax,0x4
0x08048ecf <+15>: mov eax,DWORD PTR [eax]
0x08048ed1 <+17>: mov DWORD PTR [esp],eax
0x08048ed4 <+20>: call 0x8049710 <atoi>
0x08048ed9 <+25>: cmp eax,0x1a7
// it compare argv[1] with 423 if it is unequal it print No!
0x08048ede <+30>: jne 0x8048f58 <main+152>
0x08048ee0 <+32>: mov DWORD PTR [esp],0x80c5348
0x08048ee7 <+39>: call 0x8050bf0 <strdup>
0x08048eec <+44>: mov DWORD PTR [esp+0x10],eax
0x08048ef0 <+48>: mov DWORD PTR [esp+0x14],0x0
0x08048ef8 <+56>: call 0x8054680 <getegid>
0x08048efd <+61>: mov DWORD PTR [esp+0x1c],eax
0x08048f01 <+65>: call 0x8054670 <geteuid>
0x08048f06 <+70>: mov DWORD PTR [esp+0x18],eax
0x08048f0a <+74>: mov eax,DWORD PTR [esp+0x1c]
0x08048f0e <+78>: mov DWORD PTR [esp+0x8],eax
0x08048f12 <+82>: mov eax,DWORD PTR [esp+0x1c]
0x08048f16 <+86>: mov DWORD PTR [esp+0x4],eax
0x08048f1a <+90>: mov eax,DWORD PTR [esp+0x1c]
0x08048f1e <+94>: mov DWORD PTR [esp],eax
0x08048f21 <+97>: call 0x8054700 <setresgid>
0x08048f26 <+102>: mov eax,DWORD PTR [esp+0x18]
0x08048f2a <+106>: mov DWORD PTR [esp+0x8],eax
0x08048f2e <+110>: mov eax,DWORD PTR [esp+0x18]
0x08048f32 <+114>: mov DWORD PTR [esp+0x4],eax
0x08048f36 <+118>: mov eax,DWORD PTR [esp+0x18]
0x08048f3a <+122>: mov DWORD PTR [esp],eax
0x08048f3d <+125>: call 0x8054690 <setresuid>
0x08048f42 <+130>: lea eax,[esp+0x10]
0x08048f46 <+134>: mov DWORD PTR [esp+0x4],eax
0x08048f4a <+138>: mov DWORD PTR [esp],0x80c5348
//at this point euid and egid are the one of the user that launched gdb`
0x08048f51 <+145>: call 0x8054640 <execv>
// we never go there since execv opens a shell
0x08048f56 <+150>: jmp 0x8048f80 <main+192>
0x08048f58 <+152>: mov eax,ds:0x80ee170
0x08048f5d <+157>: mov edx,eax
0x08048f5f <+159>: mov eax,0x80c5350
0x08048f64 <+164>: mov DWORD PTR [esp+0xc],edx
0x08048f68 <+168>: mov DWORD PTR [esp+0x8],0x5
0x08048f70 <+176>: mov DWORD PTR [esp+0x4],0x1
0x08048f78 <+184>: mov DWORD PTR [esp],eax
0x08048f7b <+187>: call 0x804a230 <fwrite>
0x08048f80 <+192>: mov eax,0x0
0x08048f85 <+197>: leave
0x08048f86 <+198>: ret
End of assembler dump.
I don't understand how the binary changes behaviour if I execute it by gdb or in the shell, maybe its because the binary's proprietary is level1?
If someone has the time to explain to me how it works I'll be really greatful
Thanks a lot
I don't understand how the binary changes behaviour
The binary doesn't -- the kernel creates a new process with different UID when presented a set-uid binary (that's what s in -rwsr-x---+ means).
For obvious security reasons the kernel doesn't do that when the binary is being debugged.
I have 2 functions written in assembly (masm) in visual studio that i use in my C++ project. They are an Unsigned 64-Bit multiply function that produces a 128-Bit result, and a Unsigned 128-Bit divide function that produces a 128-Bit Quotient and returns a 32-Bit Remainder.
What i need is a signed version of the functions but I'm not sure how to do it.
Below is the code of the .asm file with the Unsigned functions:
.MODEL flat, stdcall
.CODE
MUL64 PROC, A:QWORD, B:QWORD, pu128:DWORD
push EAX
push EDX
push EBX
push ECX
push EDI
mov EDI,pu128
; LO(A) * LO(B)
mov EAX,DWORD PTR A
mov EDX,DWORD PTR B
MUL EDX
mov [EDI],EAX ; Save the partial product.
mov ECX,EDX
; LO(A) * HI(B)
mov EAX,DWORD PTR A
mov EDX,DWORD PTR B+4
MUL EDX
ADD EAX,ECX
ADC EDX,0
mov EBX,EAX
mov ECX,EDX
; HI(A) * LO(B)
mov EAX,DWORD PTR A+4
mov EDX,DWORD PTR B
MUL EDX
ADD EAX,EBX
ADC ECX,EDX
PUSHFD ; Save carry.
mov [EDI+4],EAX ; Save the partial product.
; HI(A) * HI(B)
mov EAX,DWORD PTR A+4
mov EDX,DWORD PTR B+4
MUL EDX
POPFD ; Retrieve carry from above.
ADC EAX,ECX
ADC EDX,0
mov [EDI+8],EAX ; Save the partial product.
mov [EDI+12],EDX ; Save the partial product.
pop EDI
pop ECX
pop EBX
pop EDX
pop EAX
ret 20
MUL64 ENDP
IMUL64 PROC, A:SQWORD, B:SQWORD, pi128:DWORD
; How to make this work?
ret 20
IMUL64 ENDP
DIV128 PROC, pDividend128:DWORD, Divisor:DWORD, pQuotient128:DWORD
push EDX
push EBX
push ESI
push EDI
MOV ESI,pDividend128
MOV EDI,pQuotient128
MOV EBX,Divisor
XOR EDX,EDX
MOV EAX,[ESI+12]
DIV EBX
MOV [EDI+12],EAX
MOV EAX,[ESI+8]
DIV EBX
MOV [EDI+8],EAX
MOV EAX,[ESI+4]
DIV EBX
MOV [EDI+4],EAX
MOV EAX,[ESI]
DIV EBX
MOV [EDI],EAX
MOV EAX,EDX
pop EDI
pop ESI
pop EBX
pop EDX
ret 12
DIV128 ENDP
IDIV128 PROC, pDividend128:DWORD, Divisor:DWORD, pQuotient128:DWORD
; How to make this work?
ret 12
IDIV128 ENDP
END
If you found this helpful in anyway please help the project by helping code the Signed version of the functions.
First, the MUL64 function does not work 100%
If you try to do 0xFFFFFFFFFFFFFFFF x 0xFFFFFFFFFFFFFFFF, the Hi 64-bit result is 0xFFFFFFFeFFFFFFFF, it should be 0xFFFFFFFFFFFFFFFe
To fix this, the carry flag after the POPFD instruction should be added to EDX, the highest 32-bit part of the result. Now following Peter Cordes advice, remove the push and pops of EAX/ECX/EDX. Finally use setc BL and movzx EBX,BL to save the flag. Note: you cannot easily use xor EBX,EBX to zero it because xor effects the flags. We use movzx because its faster than add BL,0xFF and add is faster than adc based on Skylake specs.
The Result:
MUL64 PROC, A:QWORD, B:QWORD, pu128:DWORD
push EBX
push EDI
mov EDI,pu128
; LO(A) * LO(B)
mov EAX,DWORD PTR A
mov EDX,DWORD PTR B
mul EDX
mov [EDI],EAX ; Save the partial product.
mov ECX,EDX
; LO(A) * HI(B)
mov EAX,DWORD PTR A
mov EDX,DWORD PTR B+4
mul EDX
add EAX,ECX
adc EDX,0
mov EBX,EAX
mov ECX,EDX
; HI(A) * LO(B)
mov EAX,DWORD PTR A+4
mov EDX,DWORD PTR B
mul EDX
add EAX,EBX
adc ECX,EDX
setc BL ; Save carry.
movzx EBX,BL ; Zero-Extend carry.
mov [EDI+4],EAX ; Save the partial product.
; HI(A) * HI(B)
mov EAX,DWORD PTR A+4
mov EDX,DWORD PTR B+4
mul EDX
add EDX,EBX ; Add carry from above.
add EAX,ECX
adc EDX,0
mov [EDI+8],EAX ; Save the partial product.
mov [EDI+12],EDX ; Save the partial product.
pop EDI
pop EBX
ret 20
MUL64 ENDP
Now, to make a signed version of the function use this formula:
my128.Hi -= (((A < 0) ? B : 0) + ((B < 0) ? A : 0));
The Result:
IMUL64 PROC, A:SQWORD, B:SQWORD, pi128:DWORD
push EBX
push EDI
mov EDI,pi128
; LO(A) * LO(B)
mov EAX,DWORD PTR A
mov EDX,DWORD PTR B
mul EDX
mov [EDI],EAX ; Save the partial product.
mov ECX,EDX
; LO(A) * HI(B)
mov EAX,DWORD PTR A
mov EDX,DWORD PTR B+4
mul EDX
add EAX,ECX
adc EDX,0
mov EBX,EAX
mov ECX,EDX
; HI(A) * LO(B)
mov EAX,DWORD PTR A+4
mov EDX,DWORD PTR B
mul EDX
add EAX,EBX
adc ECX,EDX
setc BL ; Save carry.
movzx EBX,BL ; Zero-Extend carry.
mov [EDI+4],EAX ; Save the partial product.
; HI(A) * HI(B)
mov EAX,DWORD PTR A+4
mov EDX,DWORD PTR B+4
mul EDX
add EDX,EBX ; Add carry from above.
add EAX,ECX
adc EDX,0
mov [EDI+8],EAX ; Save the partial product.
mov [EDI+12],EDX ; Save the partial product.
; Signed version only:
cmp DWORD PTR A+4,0
jg zero_b
jl use_b
cmp DWORD PTR A,0
jae zero_b
use_b:
mov ECX,DWORD PTR B
mov EBX,DWORD PTR B+4
jmp test_b
zero_b:
xor ECX,ECX
mov EBX,ECX
test_b:
cmp DWORD PTR B+4,0
jg zero_a
jl use_a
cmp DWORD PTR B,0
jae zero_a
use_a:
mov EAX,DWORD PTR A
mov EDX,DWORD PTR A+4
jmp do_last_op
zero_a:
xor EAX,EAX
mov EDX,EAX
do_last_op:
add EAX,ECX
adc EDX,EBX
sub [EDI+8],EAX
sbb [EDI+12],EDX
; End of signed version!
pop EDI
pop EBX
ret 20
IMUL64 ENDP
The DIV128 function should be fine (also probably the fastest) for getting a 128-bit quotient from a 32-bit divisor, but if you need to use a 128-bit divisor then look at this code https://www.codeproject.com/Tips/785014/UInt-Division-Modulus that has an example of using the Binary Shift Algorithm for 128-bit division. It could probably be 3x faster if written in assembly.
To make a signed version of DIV128, first determine if the sign of the divisor and dividend are the same or different. If they are the same, then the result should be positive. If they are different, then the result should be negative. So... Make the dividend and divisor positive if they are negative and call DIV128, after that, negate the results if the signs were different.
Here is some example code written in C++
VOID IDIV128(PSDQWORD Dividend, PSDQWORD Divisor, PSDQWORD Quotient, PSDQWORD Remainder)
{
BOOL Negate;
DQWORD DD, DV;
Negate = TRUE;
// Use local DD and DV so Dividend and Divisor dont get currupted.
DD.Lo = Dividend->Lo;
DD.Hi = Dividend->Hi;
DV.Lo = Divisor->Lo;
DV.Hi = Divisor->Hi;
// if the signs are the same then: Negate = FALSE;
if ((DD.Hi & 0x8000000000000000) == (DV.Hi & 0x8000000000000000)) Negate = FALSE;
// Covert Dividend and Divisor to possitive if negative: (negate)
if (DD.Hi & 0x8000000000000000) NEG128((PSDQWORD)&DD);
if (DV.Hi & 0x8000000000000000) NEG128((PSDQWORD)&DV);
DIV128(&DD, &DV, (PDQWORD)Quotient, (PDQWORD)Remainder);
if (Negate == TRUE)
{
NEG128(Quotient);
NEG128(Remainder);
}
}
EDIT:
Following Peter Cordes advice, we can optimize MUL64/IMUL64 even more. Look at the comments for specific changes being made. I have also replaced MUL64 PROC, A:QWORD, B:QWORD, pu128:DWORD with MUL64#20: and IMUL64#20: to eliminate unnecessary use of EBP that masm adds. I also optimized the sign-fixing work for IMUL64.
The current .asm file for MUL64/IMUL64
.MODEL flat, stdcall
EXTERNDEF MUL64#20 :PROC
EXTERNDEF IMUL64#20 :PROC
.CODE
MUL64#20:
push EBX
push EDI
; -----------------
; | pu128 |
; |---------------|
; | B |
; |---------------|
; | A |
; |---------------|
; | ret address |
; |---------------|
; | EBX |
; |---------------|
; ESP---->| EDI |
; -----------------
A TEXTEQU <[ESP+12]>
B TEXTEQU <[ESP+20]>
pu128 TEXTEQU <[ESP+28]>
mov EDI,pu128
; LO(A) * LO(B)
mov EAX,DWORD PTR A
mul DWORD PTR B
mov [EDI],EAX ; Save the partial product.
mov ECX,EDX
; LO(A) * HI(B)
mov EAX,DWORD PTR A
mul DWORD PTR B+4
add EAX,ECX
adc EDX,0
mov EBX,EAX
mov ECX,EDX
; HI(A) * LO(B)
mov EAX,DWORD PTR A+4
mul DWORD PTR B
add EAX,EBX
adc ECX,EDX
setc BL ; Save carry.
mov [EDI+4],EAX ; Save the partial product.
; HI(A) * HI(B)
mov EAX,DWORD PTR A+4
mul DWORD PTR B+4
add EAX,ECX
movzx ECX,BL ; Zero-Extend saved carry from above.
adc EDX,ECX
mov [EDI+8],EAX ; Save the partial product.
mov [EDI+12],EDX ; Save the partial product.
pop EDI
pop EBX
ret 20
IMUL64#20:
push EBX
push EDI
; -----------------
; | pi128 |
; |---------------|
; | B |
; |---------------|
; | A |
; |---------------|
; | ret address |
; |---------------|
; | EBX |
; |---------------|
; ESP---->| EDI |
; -----------------
A TEXTEQU <[ESP+12]>
B TEXTEQU <[ESP+20]>
pi128 TEXTEQU <[ESP+28]>
mov EDI,pi128
; LO(A) * LO(B)
mov EAX,DWORD PTR A
mul DWORD PTR B
mov [EDI],EAX ; Save the partial product.
mov ECX,EDX
; LO(A) * HI(B)
mov EAX,DWORD PTR A
mul DWORD PTR B+4
add EAX,ECX
adc EDX,0
mov EBX,EAX
mov ECX,EDX
; HI(A) * LO(B)
mov EAX,DWORD PTR A+4
mul DWORD PTR B
add EAX,EBX
adc ECX,EDX
setc BL ; Save carry.
mov [EDI+4],EAX ; Save the partial product.
; HI(A) * HI(B)
mov EAX,DWORD PTR A+4
mul DWORD PTR B+4
add EAX,ECX
movzx ECX,BL ; Zero-Extend saved carry from above.
adc EDX,ECX
mov [EDI+8],EAX ; Save the partial product.
mov [EDI+12],EDX ; Save the partial product.
; Signed version only:
mov BL,BYTE PTR B+7
and BL,80H
jz zero_a
mov EAX,DWORD PTR A
mov EDX,DWORD PTR A+4
jmp test_a
zero_a:
xor EAX,EAX
mov EDX,EAX
test_a:
mov BL,BYTE PTR A+7
and BL,80H
jz do_last_op
add EAX,DWORD PTR B
adc EDX,DWORD PTR B+4
do_last_op:
sub [EDI+8],EAX
sbb [EDI+12],EDX
; End of signed version!
pop EDI
pop EBX
ret 20
END
I have the following piece of code:
RtlGetElementGenericTable:
7C9624E0 PUSH EBP
7C9624E1 MOV EBP,ESP
7C9624E3 MOV ECX,DWORD PTR [EBP+8]
7C9624E6 MOV EDX,DWORD PTR [ECX+14]
7C9624E9 MOV EAX,DWORD PTR [ECX+C]
And I was wondering what does the "C" in MOV EAX,DWORD PTR [ECX+C] means.
Is a variable? Is it a register? Is it something else?
The C in MOV EAX,DWORD PTR [ECX+C] is hexadecimal C (decimal 12).
And note that 8 in MOV ECX,DWORD PTR [EBP+8] and 14 in MOV EDX,DWORD PTR [ECX+14] are also hexadecimal numbers.
It's probably an hexadecimal value. So it means 12.
This code is a C program (bubble sort) disassembled into assembly. How can I make the following code run if I put it in a .asm file and use nasm to assemble? If you know what needs changing, please say what to change it to. For instance I understand that nasm won't accept DWORD PTR, but I haven't found out what to do instead. Thanks
.file "sort.c" .intel_syntax noprefix .text .globl
sort .type sort, #function
sort: .LFB0:
.cfi_startproc
push rbp
.cfi_def_cfa_offset 16
.cfi_offset 6, -16
mov rbp, rsp
.cfi_def_cfa_register 6
mov QWORD PTR [rbp-24], rdi
mov DWORD PTR [rbp-28], esi
mov DWORD PTR [rbp-12], 0
jmp .L2
.L6:
mov DWORD PTR [rbp-8], 0
jmp .L3
.L5:
mov eax, DWORD PTR [rbp-8]
cdqe
sal rax, 2
add rax, QWORD PTR [rbp-24]
mov edx, DWORD PTR [rax]
mov eax, DWORD PTR [rbp-8]
cdqe
add rax, 1
sal rax, 2
add rax, QWORD PTR [rbp-24]
mov eax, DWORD PTR [rax]
cmp edx, eax
jle .L4
mov eax, DWORD PTR [rbp-8]
cdqe
sal rax, 2
add rax, QWORD PTR [rbp-24]
mov eax, DWORD PTR [rax]
mov DWORD PTR [rbp-4], eax
mov eax, DWORD PTR [rbp-8]
cdqe
sal rax, 2
add rax, QWORD PTR [rbp-24]
mov edx, DWORD PTR [rbp-8]
movsx rdx, edx
add rdx, 1
sal rdx, 2
add rdx, QWORD PTR [rbp-24]
mov edx, DWORD PTR [rdx]
mov DWORD PTR [rax], edx
mov eax, DWORD PTR [rbp-8]
cdqe
add rax, 1
sal rax, 2
add rax, QWORD PTR [rbp-24]
mov edx, DWORD PTR [rbp-4]
mov DWORD PTR [rax], edx
.L4:
add DWORD PTR [rbp-8], 1
.L3:
mov eax, DWORD PTR [rbp-28]
sub eax, 1
sub eax, DWORD PTR [rbp-12]
cmp eax, DWORD PTR [rbp-8]
jg .L5
add DWORD PTR [rbp-12], 1
.L2:
mov eax, DWORD PTR [rbp-28]
sub eax, 1
cmp eax, DWORD PTR [rbp-12]
jg .L6
pop rbp
.cfi_def_cfa 7, 8
ret
.cfi_endproc
.LFE0:
.size sort, .-sort
.ident "GCC: (Ubuntu/Linaro 4.6.3-1ubuntu5) 4.6.3"
.section .note.GNU-stack,"",#progbits
Just remove PTR and all nonsensical .somethings.
This assembles just fine:
; file: gas-nasm-sort.asm
bits 64
sort:
push rbp
mov rbp, rsp
mov QWORD [rbp-24], rdi
mov DWORD [rbp-28], esi
mov DWORD [rbp-12], 0
jmp .L2
.L6:
mov DWORD [rbp-8], 0
jmp .L3
.L5:
mov eax, DWORD [rbp-8]
cdqe
sal rax, 2
add rax, QWORD [rbp-24]
mov edx, DWORD [rax]
mov eax, DWORD [rbp-8]
cdqe
add rax, 1
sal rax, 2
add rax, QWORD [rbp-24]
mov eax, DWORD [rax]
cmp edx, eax
jle .L4
mov eax, DWORD [rbp-8]
cdqe
sal rax, 2
add rax, QWORD [rbp-24]
mov eax, DWORD [rax]
mov DWORD [rbp-4], eax
mov eax, DWORD [rbp-8]
cdqe
sal rax, 2
add rax, QWORD [rbp-24]
mov edx, DWORD [rbp-8]
movsx rdx, edx
add rdx, 1
sal rdx, 2
add rdx, QWORD [rbp-24]
mov edx, DWORD [rdx]
mov DWORD [rax], edx
mov eax, DWORD [rbp-8]
cdqe
add rax, 1
sal rax, 2
add rax, QWORD [rbp-24]
mov edx, DWORD [rbp-4]
mov DWORD [rax], edx
.L4:
add DWORD [rbp-8], 1
.L3:
mov eax, DWORD [rbp-28]
sub eax, 1
sub eax, DWORD [rbp-12]
cmp eax, DWORD [rbp-8]
jg .L5
add DWORD [rbp-12], 1
.L2:
mov eax, DWORD [rbp-28]
sub eax, 1
cmp eax, DWORD [rbp-12]
jg .L6
pop rbp
ret
Command:
nasm gas-nasm-sort.asm -f bin -o gas-nasm-sort.bin
But again, there's NASM documentation. Read it. In particular these sections:
2.2.2 NASM Requires Square Brackets For Memory References
2.2.3 NASM Doesn't Store Variable Types
Can some one help me with reversing of _PrepareMenuWindow() subroutine?
I am trying to find the signature of the method.
__text:000639A7 _PrepareMenuWindow proc near ; CODE XREF: DrawTheMenu(MenuSelectData *,__CFArray **,uchar,uchar *)+274p
__text:000639A7 ; PopUpMenuSelectCore(MenuData *,Point,double,Point,ushort,uint,Rect const*,ushort,ulong,Rect const*,Rect const*,__CFString const*,OpaqueMenuRef **,ushort *)+528p
__text:000639A7
__text:000639A7 var_44 = dword ptr -44h
__text:000639A7 var_40 = dword ptr -40h
__text:000639A7 var_3C = dword ptr -3Ch
__text:000639A7 var_34 = dword ptr -34h
__text:000639A7 var_30 = dword ptr -30h
__text:000639A7 var_2C = dword ptr -2Ch
__text:000639A7 var_28 = dword ptr -28h
__text:000639A7 var_24 = word ptr -24h
__text:000639A7 var_20 = dword ptr -20h
__text:000639A7 var_1A = word ptr -1Ah
__text:000639A7 arg_0 = dword ptr 8
__text:000639A7 arg_4 = dword ptr 0Ch
__text:000639A7 arg_8 = dword ptr 10h
__text:000639A7
__text:000639A7 push ebp
__text:000639A8 mov ebp, esp
__text:000639AA push edi
__text:000639AB push esi
__text:000639AC push ebx
__text:000639AD sub esp, 5Ch
__text:000639B0 xor edi, edi
__text:000639B2 mov eax, [ebp+arg_0]
__text:000639B5 test eax, eax
__text:000639B7 jz short loc_639C6
__text:000639B9 mov eax, [ebp+arg_0]
__text:000639BC mov [esp], eax
__text:000639BF call __ZNK8HIObject13GetEncodedRefEv ; HIObject::GetEncodedRef(void)
__text:000639C4 mov edi, eax
__text:000639C6
__text:000639C6 loc_639C6: ; CODE XREF: _PrepareMenuWindow+10j
__text:000639C6 mov ecx, [ebp+arg_4]
__text:000639C9 mov eax, [ecx]
__text:000639CB mov edx, [ecx+4]
__text:000639CE mov [ebp+var_2C], eax
__text:000639D1 mov [ebp+var_28], edx
__text:000639D4 lea eax, [ebp+var_1A]
__text:000639D7 mov [ebp+var_40], eax
__text:000639DA mov [esp+4], eax
__text:000639DE mov [esp], edi
__text:000639E1 call _GetMenuType
__text:000639E6 mov dword ptr [esp+4], 0
__text:000639EE mov [esp], edi
__text:000639F1 call _IsMenuItemEnabled
__text:000639F6 movzx edx, [ebp+var_1A]
__text:000639FA or dh, 1
__text:000639FD test al, al
__text:000639FF movzx ebx, [ebp+var_1A]
__text:00063A03 cmovz ebx, edx
__text:00063A06 mov [ebp+var_1A], bx
__text:00063A0A mov eax, [ebp+arg_8]
__text:00063A0D mov [esp+0Ch], eax
__text:00063A11 lea ecx, [ebp+var_2C]
__text:00063A14 mov [ebp+var_44], ecx
__text:00063A17 mov [esp+8], ecx
__text:00063A1B mov eax, [ebp+arg_4]
__text:00063A1E mov [esp+4], eax
__text:00063A22 mov [esp], edi
__text:00063A25 call __AddOpenMenu
__text:00063A2A mov ecx, [ebp+var_44]
__text:00063A2D mov [esp], ecx
__text:00063A30 call _EmptyRect
__text:00063A35 test al, al
__text:00063A37 jnz loc_63B94
__text:00063A3D mov [esp], edi
__text:00063A40 call __Z11GetMenuDataP13OpaqueMenuRef ; GetMenuData(OpaqueMenuRef *)
__text:00063A45 mov [ebp+var_3C], eax
__text:00063A48 call _NewRgn
__text:00063A4D mov esi, eax
__text:00063A4F test eax, eax
__text:00063A51 jz loc_63BDD
__text:00063A57 movzx ebx, bx
__text:00063A5A mov eax, [ebp+var_3C]
__text:00063A5D mov eax, [eax+40h]
__text:00063A60 test eax, eax
__text:00063A62 jnz loc_63B23
__text:00063A68 mov [ebp+var_1A], 0
__text:00063A6E mov eax, [ebp+var_2C]
__text:00063A71 mov edx, [ebp+var_28]
__text:00063A74 mov [ebp+var_34], eax
__text:00063A77 mov [ebp+var_30], edx
__text:00063A7A mov ecx, [ebp+var_40]
__text:00063A7D mov [esp+10h], ecx
__text:00063A81 mov dword ptr [esp+0Ch], 0
__text:00063A89 lea eax, [ebp+var_34]
__text:00063A8C mov [esp+8], eax
__text:00063A90 mov dword ptr [esp+4], 7
__text:00063A98 mov eax, [ebp+var_3C]
__text:00063A9B mov [esp], eax
__text:00063A9E call __Z12_CallMenuDefP8MenuDatasP4Rect5PointPs ; _CallMenuDef(MenuData *,short,Rect *,Point,short *)
__text:00063AA3 cmp [ebp+var_1A], 7473h
__text:00063AA9 jz short loc_63ADC
__text:00063AAB add word ptr [ebp+var_2C], 3
__text:00063AB0 mov dword ptr [esp+8], 0FFFFFFFCh
__text:00063AB8 mov dword ptr [esp+4], 0FFFFFFFCh
__text:00063AC0 mov ecx, [ebp+var_44]
__text:00063AC3 mov [esp], ecx
__text:00063AC6 call _InsetRect
__text:00063ACB mov eax, [ebp+var_44]
__text:00063ACE mov [esp+4], eax
__text:00063AD2 mov [esp], esi
__text:00063AD5 call _RectRgn
__text:00063ADA jmp short loc_63B23
__text:00063ADC ; ---------------------------------------------------------------------------
__text:00063ADC
__text:00063ADC loc_63ADC: ; CODE XREF: _PrepareMenuWindow+102j
__text:00063ADC lea eax, [ebp+var_24]
__text:00063ADF mov [esp+8], eax
__text:00063AE3 lea eax, [ebp+var_20]
__text:00063AE6 mov [esp+4], eax
__text:00063AEA mov [esp], edi
__text:00063AED call __GetMenuCallout
__text:00063AF2 movsx eax, [ebp+var_24]
__text:00063AF6 mov [esp+10h], eax
__text:00063AFA mov eax, [ebp+var_20]
__text:00063AFD mov [esp+0Ch], eax
__text:00063B01 mov [esp+8], esi
__text:00063B05 mov [esp+4], ebx
__text:00063B09 mov ecx, [ebp+var_44]
__text:00063B0C mov [esp], ecx
__text:00063B0F call __GetThemeMenuBackgroundRegionWithCallout
__text:00063B14 mov eax, [ebp+var_44]
__text:00063B17 mov [esp+4], eax
__text:00063B1B mov [esp], esi
__text:00063B1E call _GetRegionBounds
__text:00063B23
__text:00063B23 loc_63B23: ; CODE XREF: _PrepareMenuWindow+BBj
__text:00063B23 ; _PrepareMenuWindow+133j
__text:00063B23 mov [esp+0Ch], esi
__text:00063B27 mov ecx, [ebp+var_44]
__text:00063B2A mov [esp+8], ecx
__text:00063B2E mov [esp+4], ebx
__text:00063B32 mov [esp], edi
__text:00063B35 call __ZL13GetMenuWindowP13OpaqueMenuReftPK4RectP15OpaqueRgnHandle ; GetMenuWindow(OpaqueMenuRef *,ushort,Rect const*,OpaqueRgnHandle *)
__text:00063B3A test eax, eax
__text:00063B3C jz short loc_63BA1
__text:00063B3E mov [esp], eax
__text:00063B41 call _GetWindowPort
__text:00063B46 mov [esp], eax
__text:00063B49 call _SetPortWrapper
__text:00063B4E mov [esp], esi
__text:00063B51 call _SetClipWrapper
__text:00063B56 mov [esp], esi
__text:00063B59 call _DisposeRgn
__text:00063B5E mov eax, [ebp+var_3C]
__text:00063B61 mov eax, [eax+40h]
__text:00063B64 test eax, eax
__text:00063B66 jnz short loc_63BDD
__text:00063B68 mov dword ptr [esp+14h], 0
__text:00063B70 mov dword ptr [esp+10h], 0
__text:00063B78 mov [esp+0Ch], ebx
__text:00063B7C mov ecx, [ebp+arg_4]
__text:00063B7F mov [esp+8], ecx
__text:00063B83 mov eax, [ebp+var_44]
__text:00063B86 mov [esp+4], eax
__text:00063B8A mov [esp], edi
__text:00063B8D call __Z18DrawMenuBackgroundP13OpaqueMenuRefRK4RectS3_thPv ; DrawMenuBackground(OpaqueMenuRef *,Rect const&,Rect const&,ushort,uchar,void *)
__text:00063B92 jmp short loc_63BDD
__text:00063B94 ; ---------------------------------------------------------------------------
__text:00063B94
__text:00063B94 loc_63B94: ; CODE XREF: _PrepareMenuWindow+90j
__text:00063B94 mov ecx, [ebp+arg_0]
__text:00063B97 mov [esp], ecx
__text:00063B9A call _DisposeMenuWindow
__text:00063B9F jmp short loc_63BDD
__text:00063BA1 ; ---------------------------------------------------------------------------
__text:00063BA1
__text:00063BA1 loc_63BA1: ; CODE XREF: _PrepareMenuWindow+195j
__text:00063BA1 mov eax, [ebp+arg_0]
__text:00063BA4 mov [esp], eax
__text:00063BA7 call __Z11FindMBEntryP8MenuData ; FindMBEntry(MenuData *)
__text:00063BAC mov ecx, eax
__text:00063BAE test eax, eax
__text:00063BB0 jz short loc_63BD5
__text:00063BB2 mov word ptr [eax+1Eh], 0
__text:00063BB8 mov word ptr [eax+1Ch], 0
__text:00063BBE mov word ptr [eax+1Ah], 0
__text:00063BC4 mov word ptr [eax+18h], 0
__text:00063BCA mov eax, [eax+18h]
__text:00063BCD mov edx, [ecx+1Ch]
__text:00063BD0 mov [ecx], eax
__text:00063BD2 mov [ecx+4], edx
__text:00063BD5
__text:00063BD5 loc_63BD5: ; CODE XREF: _PrepareMenuWindow+209j
__text:00063BD5 mov [esp], esi
__text:00063BD8 call _DisposeRgn
__text:00063BDD
__text:00063BDD loc_63BDD: ; CODE XREF: _PrepareMenuWindow+AAj
__text:00063BDD ; _PrepareMenuWindow+1BFj ...
__text:00063BDD xor eax, eax
__text:00063BDF add esp, 5Ch
__text:00063BE2 pop ebx
__text:00063BE3 pop esi
__text:00063BE4 pop edi
__text:00063BE5 leave
__text:00063BE6 retn
__text:00063BE6 _PrepareMenuWindow endp
What have you got so far that isn't generated by IDA? (ie: your analysis of the function).
From the looks of it its a __cdecl function that always returns NULL/false/0. It also seems to take 3 arguments(which can be confirmed by looking at what cleanup is by the caller, if there is any).
Arg 0 is a MenuData*, arg 4 seems to be a Rect&(which is secretly just Rect*), arg 8 would be whatever type __AddOpenMenu takes as its fourth argument.
So i'd assume something along the lines of typedef BOOL(__cdecl*)(MenuData*,Rect&,void*)