Floating Point Precision in x86-64 Assembly

README.md

Floating Point Precision in x86-64 Assembly

The lack of precison you get with floating point numbers isn't a language problem, it's a problem with floating point nubers themselves, so it even shows up as low as x86-64 assembly.

The assembly in the file bellow is the code I have written to showcase this.

To compile it you have to do this:

$ nasm floats.asm -o floats.o -f elf64
$ gcc floats.o -o floats -no-pie

Then, to run it, you simply have to run the generated binary:

$ ./floats
0.100 + 0.200 = 0.3000000119

floats.asm

;  Some example assembly to showcase the lack of precision in floating point operations.
;  Copyright (C) 2024  marvhus <[email protected]>
;
;  This program is free software: you can redistribute it and/or modify
;  it under the terms of the GNU General Public License as published by
;  the Free Software Foundation, either version 3 of the License, or
;  (at your option) any later version.
;
;  This program is distributed in the hope that it will be useful,
;  but WITHOUT ANY WARRANTY; without even the implied warranty of
;  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;  GNU General Public License for more details.
;
;  You should have received a copy of the GNU General Public License
;  along with this program.  If not, see <https://www.gnu.org/licenses/>.

section .text
    default rel   ; Relative pointers
    extern printf ; printf will be linked in from libc
    global main   ; we're compiling/linking with GCC (after assembling with nasm), so we're using main.

main:
    ; Create a stack-frame, re-aligning the tack to 16 byte alignment before calls
    push rbp
    mov rbp, rsp

    ; ===========================
    ; === Calculate the float ===
    ; ===========================

    ; load the single-precision floats
    movss xmm0, dword [float_1]
    movss xmm1, dword [float_2]
    ; add the single-precision floats
    addss xmm0, xmm1
    ; store the result for later
    movss dword [result], xmm0

    ; ========================
    ; === Print the floats ===
    ; ========================

    ; Use the format string
    mov rdi, fmt

    ; Use the floats
    mov rax, 3 ; Number of floats in the XMM registers.
    ; load the single-precision floats
    movss xmm0, dword [float_1]
    movss xmm1, dword [float_2]
    movss xmm2, dword [result]
    ; convert the single-precision floats to double-precision
    ; (printf can only handle double-precison)
    cvtss2sd xmm0, xmm0
    cvtss2sd xmm1, xmm1
    cvtss2sd xmm2, xmm2

    ; Print using printf
    call printf

    ; ===============
    ; === Cleanup ===
    ; ===============

    ; Pop stack
    pop rbp

    ; Exit with code 0
    mov rax, 0
    ret

section .data
    ; The format string for printf
    fmt db "%.3f + %.3f = %.10f", 10, 0
    ; The floats
    float_1 dd 0.1
    float_2 dd 0.2

section .bss
    result resd 1 ; Reserved space for double-precision float