fix(vector-index): preserve byteOffset/byteLength in base64 round-trip (closes #587, #584)

src/state/vector-index.ts

@@ -1,9 +1,23 @@
+// Pass byteOffset + byteLength explicitly so the round-trip survives
+// Node's Buffer pool. Buffer.from(b64, "base64") returns a slice of a
+// shared 8KB pool (poolSize), and `new Float32Array(buf.buffer)` ignores
+// the slice metadata — it would mint a 2048-element view over the whole
+// pool. Same risk on the encode side if the input Float32Array is itself
+// a sliced view. Reported as a phantom "2048 dimensions on disk" crash
+// in #455 / #469 / #584 / #587.
 function float32ToBase64(arr: Float32Array): string {
-  return Buffer.from(arr.buffer).toString("base64");
+  return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength).toString(
+    "base64",
+  );
 }
 
 function base64ToFloat32(b64: string): Float32Array {
-  return new Float32Array(Buffer.from(b64, "base64").buffer);
+  const buf = Buffer.from(b64, "base64");
+  return new Float32Array(
+    buf.buffer,
+    buf.byteOffset,
+    buf.byteLength / Float32Array.BYTES_PER_ELEMENT,
+  );
 }
 
 function cosineSimilarity(a: Float32Array, b: Float32Array): number {

test/vector-index.test.ts

@@ -76,4 +76,43 @@ describe("VectorIndex", () => {
     const results = index.search(new Float32Array([1, 0, 0]));
     expect(results[0].score).toBe(0);
   });
+
+  it("round-trip preserves dim + identity for pooled-Buffer sizes (#587)", () => {
+    // 384-dim floats = 1536 bytes, comfortably inside Node's 8KB Buffer
+    // pool. Without explicit byteOffset/byteLength in the base64 round-trip,
+    // deserialise reads pool offset 0 and reports the entire pool as a
+    // 2048-element view, which the live index then rejects with
+    // "dimensions seen on disk: 2048".
+    const DIM = 384;
+    const vecs = Array.from({ length: 5 }, (_, n) => {
+      const v = new Float32Array(DIM);
+      for (let i = 0; i < DIM; i++) v[i] = n * 1000 + i;
+      return v;
+    });
+    vecs.forEach((v, n) => index.add(`obs_${n}`, "ses_1", v));
+
+    const restored = VectorIndex.deserialize(index.serialize());
+    expect(restored.size).toBe(5);
+    const { mismatches } = restored.validateDimensions(DIM);
+    expect(mismatches).toEqual([]);
+    for (let n = 0; n < 5; n++) {
+      const results = restored.search(vecs[n], 1);
+      expect(results[0].obsId).toBe(`obs_${n}`);
+      expect(results[0].score).toBeCloseTo(1.0, 4);
+    }
+  });
+
+  it("preserves bytes when source Float32Array is itself a sliced view (#587)", () => {
+    // The encode side has the same risk: passing arr.buffer drops the
+    // slice metadata if arr is a sub-view (subarray / typedArray.set).
+    const backing = new Float32Array(8);
+    for (let i = 0; i < 8; i++) backing[i] = i;
+    const slice = backing.subarray(2, 6); // values 2, 3, 4, 5
+
+    index.add("obs_slice", "ses_1", slice);
+    const restored = VectorIndex.deserialize(index.serialize());
+    const results = restored.search(new Float32Array([2, 3, 4, 5]), 1);
+    expect(results[0].obsId).toBe("obs_slice");
+    expect(results[0].score).toBeCloseTo(1.0, 4);
+  });
 });