Segfault when trying to get a byte from memory address - algorithm

Question

Im trying to make Sieve of Eratosthenes work with big numbers
The problem I have is that it is giving me a segfault but idk why
It works up until about 100k
But the algorithm works if I replace cmp byte [rbx], 0 with cmp dword [rbx], 0
Im very confused as to why that is happening since all the values in the array are 0's and 1's so a byte should be enough
btw with cmp dword [rbx], 0 the results are incorrect so I cant use that
X86 64
%macro crossOut 4
xor rdi, rdi ;edi keeps track of how many numbers were crossed out
;if 0 end loop
mov rbx, %1 ;array
add rbx, %2 ;move position to starting index
mov rax, %3 ;every nth number to be crossed out
mov rbp, %4 ; array length
mov rcx, 0 ;counter
%%loop:
add rcx, rax
cmp rcx, rbp
jge %%exit
add rbx, rax
cmp byte [rbx], 0
je %%crossout
jmp %%loop
%%crossout:
mov byte [rbx], 1
inc rdi
jmp %%loop
%%exit:
cmp rdi, 0
%endmacro

Answer 1

See this CodeReview question for the OP's full program.
You are reading past the end of the numbers buffer because your code does not take into account the 2nd macro parameter (an offset into the array)!
The first time the offset in the array is 0, and the code will run ok. But later you add to the offset while keeping the array length the same, and so memory that does not belong to the array is addressed.
<---- RBP=10 ----->
0,0,0,0,0,0,0,0,0,0
First time RCX allows 4 iterations {2,4,6,8} less than 10:
<---- RBP=10 ----->
RBX
v
0,0,1,0,1,0,1,0,1,0
^ ^ ^ ^
1° 2° 3° 4°
Second time RCX allows 3 iterations {3,6,9} less than 10:
<---- RBP=10 ----->
RBX
v
0,0,1,0,1,0,1,1,1,0,?
^ ^ ^
1° 2° 3°
The 3° is past the buffer and, depending on total array length, at some point this buffer overrun will segfault!
The quick fix is to initialize RCX at the value for the starting index %2 instead of zeroing it.
%macro crossOut 4
xor edi, edi ;edi keeps track of how many numbers were crossed out
;if 0 end loop
mov rbx, %1 ;array
add rbx, %2 ;move position to starting index
mov rax, %3 ;every nth number to be crossed out
mov rbp, %4 ; array length
mov rcx, %2 ;counter
%%loop:
add rcx, rax
cmp rcx, rbp
jge %%exit
add rbx, rax
cmp byte [rbx], 0
je %%crossout
jmp %%loop
%%crossout:
mov byte [rbx], 1
inc rdi
jmp %%loop
%%exit:
cmp rdi, 0
%endmacro
A better fix is to establish an absolute last address that you have RBX compare against:
%macro crossOut 4
xor edi, edi ; EDI keeps track of how many numbers were crossed out
; if 0 end loop
mov rbx, %1 ; array
mov rcx, %4 ; array length
add rcx, rbx ; Last address
add rbx, %2 ; move position to starting index
mov rax, %3 ; every nth number to be crossed out
%%loop:
add rbx, rax
cmp rbx, rcx
jae %%exit
cmp byte [rbx], 0
jne %%loop
%%crossout:
mov byte [rbx], 1
inc rdi
jmp %%loop
%%exit:
cmp rdi, 0
%endmacro
Please notice that
je %%crossout
jmp %%loop
%%crossout:
is better written as
jne %%loop
%%crossout: