Skip to content

lib: improve float parsing & rendering precision #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Fang- wants to merge 2 commits into
base: master
Choose a base branch
from
Open

lib: improve float parsing & rendering precision #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a703622
parse: fix float construction through rewrite
Fang- Aug 22, 2026
52c5833
render: don't reduce float precision
Fang- Aug 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 63 additions & 49 deletions src/r.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,57 +89,70 @@ function makeFloat(w: number, p: number, sign: boolean, intPart: string, floatPa
return construct(p, w, sign, BigInt(intPart), BigInt(floatPart.length), BigInt(floatPart));
}

//NOTE modified from an encodeFloat() written by by Jonas Raoni Soares Silva,
// made to operate on (big)integers, without using js's float logic.
// http://jsfromhell.com/classes/binary-parser
// (yes, this code is vaguely deranged. but it works!)
// encode IEEE 754 float with round-to-nearest, ties-to-even
function construct(precisionBits: number, exponentBits: number,
sign: boolean, intPart: bigint, floatDits: bigint, floatPart: bigint) {
//REVIEW when do we trigger this?
// inputs representing nrs too large for exponentBits?
// inputs with precision we can't match?
// add tests for those cases! should match stdlib result.
function exceed(x: string) {
console.warn(x);
return 1;
sign: boolean, intPart: bigint, floatDits: bigint, floatPart: bigint) {
const bias = 2 ** (exponentBits - 1) - 1;
const minExp = 1 - bias;
const maxExp = bias;
const hiddenBit = 1n << BigInt(precisionBits);
const exponentMask = bitMask(BigInt(exponentBits));
const denominator = 10n ** floatDits;
const numerator = intPart * denominator + floatPart;
const signBit = sign ? '0' : '1';
const pad = (n: bigint, bits: number) => n.toString(2).padStart(bits, '0');
const infinity = () => signBit + pad(exponentMask, exponentBits) + '0'.repeat(precisionBits);

if (numerator === 0n)
return signBit + '0'.repeat(exponentBits + precisionBits);

// floor(log2(numerator / denominator)) without js numbers/floats
let exponent = numerator.toString(2).length - denominator.toString(2).length;
if (exponent >= 0
? numerator < (denominator << BigInt(exponent))
: (numerator << BigInt(-exponent)) < denominator)
exponent--;

const roundScaled = (shift: number): bigint => {
const scaledNumerator = shift >= 0
? numerator << BigInt(shift)
: numerator;
const scaledDenominator = shift >= 0
? denominator
: denominator << BigInt(-shift);
const quotient = scaledNumerator / scaledDenominator;
const remainder = scaledNumerator % scaledDenominator;
const twiceRemainder = remainder << 1n;
return twiceRemainder > scaledDenominator
|| (twiceRemainder === scaledDenominator && (quotient & 1n) === 1n)
? quotient + 1n
: quotient;
};

let exponentField: bigint;
let mantissa: bigint;
if (exponent < minExp) {
mantissa = roundScaled(precisionBits - minExp);
if (mantissa === 0n)
return signBit + '0'.repeat(exponentBits + precisionBits);
if (mantissa >= hiddenBit) {
exponentField = 1n;
mantissa = 0n;
} else {
exponentField = 0n;
}
} else {
mantissa = roundScaled(precisionBits - exponent);
if (mantissa === (hiddenBit << 1n)) {
mantissa >>= 1n;
exponent++;
}
if (exponent > maxExp) return infinity();
exponentField = BigInt(exponent + bias);
mantissa -= hiddenBit;
}

const bias = 2**(exponentBits - 1) - 1,
minExp = -bias + 1,
maxExp = bias,
minUnnormExp = minExp - precisionBits,
len = 2 * bias + 1 + precisionBits + 3,
bin = new Array(len),
denom = 10n ** floatDits;
var exp = 0,
signal = !sign,
i, lastBit, rounded, j, result, n;
// zero-initialize the bit-array
for (i = len; i; bin[--i] = 0);
// integral into bits
for (i = bias + 2; intPart && i; bin[--i] = intPart & 1n, intPart = intPart >> 1n);
// fractional into bits
for (i = bias + 1; floatPart > 0n && (i < len); (bin[++i] = (((floatPart *= 2n) >= denom) ? 1 : 0)) && (floatPart = floatPart - denom));
// walk cursor (i) to first 1-bit.
for (i = -1; ++i < len && !bin[i];);
// round if needed
if (bin[(lastBit = precisionBits - 1 + (i = (exp = bias + 1 - i) >= minExp && exp <= maxExp ? i + 1 : bias + 1 - (exp = minExp - 1))) + 1]) {
if (!(rounded = bin[lastBit]))
for (j = lastBit + 2; !rounded && j < len; rounded = bin[j++]);
for (j = lastBit + 1; rounded && --j >= 0; (bin[j] = (!bin[j] ? 1 : 0) - 0) && (rounded = 0));
}
// walk cursor (i) to first/next(??) 1-bit
for (i = i - 2 < 0 ? -1 : i - 3; ++i < len && !bin[i];);

// set exponent, throwing on under- and overflows
(exp = bias + 1 - i) >= minExp && exp <= maxExp ? ++i : exp < minExp &&
(exp != bias + 1 - len && exp < minUnnormExp && exceed('r.construct underflow'), i = bias + 1 - (exp = minExp - 1));
intPart && (exceed(intPart ? 'r.construct overflow' : 'r.construct'),
exp = maxExp + 1, i = bias + 2);
// exponent into bits
for (n = Math.abs(exp + bias), j = exponentBits + 1, result = ''; --j; result = (n & 1) + result, n = n >>= 1);
// final serialization: sign + exponent + mantissa
return (signal ? '1' : '0') + result + bin.slice(i, i + precisionBits).join('');
return signBit + pad(exponentField, exponentBits) + pad(mantissa, precisionBits);
};

//
Expand Down Expand Up @@ -235,7 +248,8 @@ function deconstruct(float: bigint, exponentBits: bigint, precisionBits: bigint)
unequalMargins = false;
}

const buf = (2n**precisionBits).toString(10).length + 1;
// include implicit leading bit in precision
const buf = (2n**(precisionBits+1n)).toString(10).length + 1;
const res = dragon4(mantissa, Number(exponent), mantissaHighBitIdx, unequalMargins, 'unique', 0, buf);
return { t: 'd', s: sign, e: res.outExponent, a: res.digits };
}
Expand Down
19 changes: 17 additions & 2 deletions test/data/atoms.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -490,6 +490,9 @@ export const FLOAT_16_TESTS: {
{ 'n': 14390n, 'rh': '.~~0.5264' },
{ 'n': 22479n, 'rh': '.~~124.94' },
{ 'n': 53512n, 'rh': '.~~-40.25' },
// off-by-one parsing risk
{ 'n': 195n, 'rh': '.~~1.16e-5' },
{ 'n': 237n, 'rh': '.~~1.41e-5' },
// specials
{ 'n': 32256n, 'rh': '.~~nan' },
{ 'n': 31744n, 'rh': '.~~inf' },
Expand Down Expand Up @@ -520,6 +523,9 @@ export const FLOAT_32_TESTS: {
{ 'n': 374924072n, 'rs': '.1.7520019e-25' },
{ 'n': 2891424105n, 'rs': '.-3.0642938e-12' },
{ 'n': 1504861925n, 'rs': '.6.2758604e15' },
// off-by-one rendering risk
{ 'n': 1733597780n, 'rs': '.1.00398855e24' },
{ 'n': 677934996n, 'rs': '.1.29045165e-14' },
// specials
{ 'n': 2143289344n, 'rs': '.nan' },
{ 'n': 2139095040n, 'rs': '.inf' },
Expand Down Expand Up @@ -550,9 +556,12 @@ export const FLOAT_64_TESTS: {
{ 'n': 5258441678658489674n, 'rd': '.~3.5873091546780243e43' },
{ 'n': 13356339406634822690n, 'rd': '.~-2.0992411948796013e-32' },
{ 'n': 5682954223851386361n, 'rd': '.~8.25703042630886e71' },
// off-by-one parsing risk
{ 'n': 53n, 'rd': '.~2.6e-322' },
{ 'n': 3641848585n, 'rd': '.~1.799312273e-314' },
// specials
{ 'n': 9221120237041090560n, 'rd': '.~nan' },
{ 'n': 9218868437227405312n, 'rd': '.~inf' },
{ 'n': 9221120237041090560n, 'rd': '.~nan' },
{ 'n': 9218868437227405312n, 'rd': '.~inf' },
{ 'n': 18442240474082181120n, 'rd': '.~-inf' },
];
export const FLOAT_128_TESTS: {
Expand Down Expand Up @@ -580,6 +589,12 @@ export const FLOAT_128_TESTS: {
{ 'n': 9047595725142263795917295497258429378n, 'rq': '.~~~6.26545010819892388144698108939643e-4408' },
{ 'n': 262829608976719100907507105384827825295n, 'rq': '.~~~-9.321356263350323349343422220817277e441' },
{ 'n': 197646596458388884873804679655774313010n, 'rq': '.~~~-8.161207090132586143046770776173326e-3338' },
// off-by-one parsing risk
{ 'n': 19n, 'rq': '.~~~1.2e-4964' },
{ 'n': 1064046800308167282057504130689765n, 'rq': '.~~~6.88988936727308553500238519748607e-4933' },
// off-by-one rendering risk
{ 'n': 116837006551835033447022060175812259703n, 'rq': '.~~~1.00031782838305963188203670270958605e1842' },
{ 'n': 194768082700864451280926943509909653083n, 'rq': '.~~~-1.00903169066287340777448554006121625e-3504' },
// specials
{ 'n': 170138587312039964317873038467719495680n, 'rq': '.~~~nan' },
{ 'n': 170135991163610696904058773219554885632n, 'rq': '.~~~inf' },
Expand Down
9 changes: 4 additions & 5 deletions test/fuzz.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,7 @@ function safeFloat(size: bigint, w: bigint, p: bigint) {
const makemask = (n: bigint) => full ^ (1n << (p + (n % w)));
return (n: bigint) => n & makemask(n);
}
//TODO tests fail, off-by-one...
// fuzz(4, 16, ['rh'], safeFloat( 16n, 5n, 10n));
// fuzz(4, 32, ['rs'], safeFloat( 32n, 8n, 23n));
// fuzz(4, 64, ['rd'], safeFloat( 64n, 11n, 52n));
// fuzz(4, 128, ['rq'], safeFloat(128n, 15n, 112n));
fuzz(4, 16, ['rh'], safeFloat( 16n, 5n, 10n));
fuzz(4, 32, ['rs'], safeFloat( 32n, 8n, 23n));
fuzz(4, 64, ['rd'], safeFloat( 64n, 11n, 52n));
fuzz(4, 128, ['rq'], safeFloat(128n, 15n, 112n));
Loading