Last active
August 12, 2017 14:52
-
-
Save codec-abc/f2946aa4803042f0bc29cf96c42aeb6f to your computer and use it in GitHub Desktop.
4 bits floating point number manipulation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| //F# Compiler for F# 4.0 (Open Source Edition), Mono 4.2.1 | |
| open System | |
| let u : byte = (byte) 0b11110000 | |
| let highMask : byte = (byte) 0b11110000 | |
| let lowMask : byte = (byte) 0b00001111 | |
| let uHigh : byte = u &&& highMask | |
| let uLow : byte = u &&& lowMask | |
| printfn "u %s" (((float) u).ToString()) | |
| printfn "u %s" (((float) u / 255.0).ToString()) | |
| printfn "u %s" (Convert.ToString(u, 2).PadLeft(8, '0')) | |
| printfn "" | |
| let lowerByteToFloat b = | |
| let zeroedHighBits = b &&& lowMask | |
| (float) zeroedHighBits / 15.0 | |
| let higherByteToFloat b = | |
| let zeroedHighits = b &&& highMask | |
| let offseted = zeroedHighits >>> 4 | |
| lowerByteToFloat offseted | |
| let byteToTwo4BitsFloats b = | |
| let high = higherByteToFloat b | |
| let low = lowerByteToFloat b | |
| (high, low) | |
| let (uHighFloat, uLowFloat) = byteToTwo4BitsFloats u | |
| let fixedHigh = uHighFloat / 1.0625 // 1.0625 is 1 + 1 - (1/2 + 1/4 + 1/8 + 1/16) or 17/16 | |
| let fixedLow = uLowFloat / 16.0 / 1.0625 | |
| let sum = fixedHigh + fixedLow | |
| let uHighFloatByte = (byte) (uHighFloat * 255.0) | |
| let uLowFloatByte = (byte) (uLowFloat * 255.0) | |
| let fixedHighByte = (byte) (fixedHigh * 255.0) | |
| let fixedLowByte = (byte) (fixedLow * 255.0) | |
| printfn "%s %s" (uHighFloat.ToString()) (uLowFloat.ToString()) | |
| printfn "%s + %s = %s" (fixedHigh.ToString()) (fixedLow.ToString()) (sum.ToString()) | |
| printfn "%s %s" ((Convert.ToString(uHighFloatByte, 2).PadLeft(8, '0'))) ((Convert.ToString(uLowFloatByte, 2).PadLeft(8, '0'))) | |
| printfn "%s %s" ((Convert.ToString(fixedHighByte, 2).PadLeft(8, '0'))) ((Convert.ToString(fixedLowByte, 2).PadLeft(8, '0'))) | |
| if sum = ((float) u / 255.0) then | |
| printfn "good" | |
| else |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment