fix: compare Uint8Arrays by content for proper binary ID equality

Fixes `eq` function and hash indexing to compare Uint8Arrays/Buffers by content
instead of reference, enabling proper ULID comparisons in WHERE clauses.

Changes:
- Hash small Uint8Arrays (≤128 bytes) by content in db-ivm for better indexing
- Compare Uint8Arrays by content in eq operator via areValuesEqual() function
- Add comprehensive tests for Uint8Array equality comparison

Author: claude

.changeset/fix-uint8array-comparison.md

@@ -0,0 +1,6 @@
+---
+"@tanstack/db": patch
+"@tanstack/db-ivm": patch
+---
+
+Fix Uint8Array/Buffer comparison to work by content instead of reference. This enables proper equality checks for binary IDs like ULIDs in WHERE clauses using the `eq` function.

packages/db-ivm/src/hashing/hash.ts

@@ -17,6 +17,12 @@ const OBJECT_MARKER = randomHash()
 const ARRAY_MARKER = randomHash()
 const MAP_MARKER = randomHash()
 const SET_MARKER = randomHash()
+const UINT8ARRAY_MARKER = randomHash()
+
+// Maximum byte length for Uint8Arrays to hash by content instead of reference
+// Arrays smaller than this will be hashed by content, allowing proper equality comparisons
+// for small arrays like ULIDs (16 bytes) while still avoiding performance costs for large arrays
+const UINT8ARRAY_CONTENT_HASH_THRESHOLD = 128
 
 const hashCache = new WeakMap<object, number>()
 
@@ -35,6 +41,24 @@ function hashObject(input: object): number {
   let valueHash: number | undefined
   if (input instanceof Date) {
     valueHash = hashDate(input)
+  } else if (
+    // Check if input is a Uint8Array or Buffer
+    (typeof Buffer !== `undefined` && input instanceof Buffer) ||
+    input instanceof Uint8Array
+  ) {
+    // For small Uint8Arrays/Buffers (e.g., ULIDs, UUIDs), hash by content
+    // to enable proper equality comparisons. For large arrays, hash by reference
+    // to avoid performance costs.
+    if (input.byteLength <= UINT8ARRAY_CONTENT_HASH_THRESHOLD) {
+      valueHash = hashUint8Array(input)
+    } else {
+      // Deeply hashing large arrays would be too costly
+      // so we track them by reference and cache them in a weak map
+      return cachedReferenceHash(input)
+    }
+  } else if (input instanceof File) {
+    // Files are always hashed by reference due to their potentially large size
+    return cachedReferenceHash(input)
   } else {
     let plainObjectInput = input
     let marker = OBJECT_MARKER
@@ -53,17 +77,6 @@ function hashObject(input: object): number {
       plainObjectInput = [...input.entries()]
     }
 
-    if (
-      (typeof Buffer !== `undefined` && input instanceof Buffer) ||
-      input instanceof Uint8Array ||
-      input instanceof File
-    ) {
-      // Deeply hashing these objects would be too costly
-      // but we also don't want to ignore them
-      // so we track them by reference and cache them in a weak map
-      return cachedReferenceHash(input)
-    }
-
     valueHash = hashPlainObject(plainObjectInput, marker)
   }
 
@@ -78,6 +91,20 @@ function hashDate(input: Date): number {
   return hasher.digest()
 }
 
+function hashUint8Array(input: Uint8Array): number {
+  const hasher = new MurmurHashStream()
+  hasher.update(UINT8ARRAY_MARKER)
+  // Hash the byte length first to differentiate arrays of different sizes
+  hasher.update(input.byteLength)
+  // Hash each byte in the array
+  for (let i = 0; i < input.byteLength; i++) {
+    // Use _writeByte to hash the actual byte values
+    // @ts-expect-error - _writeByte is private but we need to use it here
+    hasher._writeByte(input[i]!)
+  }
+  return hasher.digest()
+}
+
 function hashPlainObject(input: object, marker: number): number {
   const hasher = new MurmurHashStream()
 

packages/db-ivm/tests/utils.test.ts

@@ -299,7 +299,8 @@ describe(`hash`, () => {
       expect(hash4).not.toBe(hash6) // Different Symbol content should have different hash
     })
 
-    it(`should hash Buffers, Uint8Arrays and File objects by reference`, () => {
+    it(`should hash small Buffers and Uint8Arrays by content`, () => {
+      // Small buffers (≤128 bytes) are hashed by content for proper equality comparisons
       const buffer1 = Buffer.from([1, 2, 3])
       const buffer2 = Buffer.from([1, 2, 3])
       const buffer3 = Buffer.from([1, 2, 3, 4])
@@ -309,7 +310,7 @@ describe(`hash`, () => {
       const hash3 = hash(buffer3)
 
       expect(typeof hash1).toBe(hashType)
-      expect(hash1).not.toBe(hash2) // Same content but different buffer instances have a different hash because it would be too costly to deeply hash buffers
+      expect(hash1).toBe(hash2) // Same content = same hash for small buffers
       expect(hash1).not.toBe(hash3) // Different Buffer content should have different hash
       expect(hash1).toBe(hash(buffer1)) // Hashing same buffer should return same hash
 
@@ -322,10 +323,46 @@ describe(`hash`, () => {
       const hash6 = hash(uint8Array3)
 
       expect(typeof hash4).toBe(hashType)
-      expect(hash4).not.toBe(hash5) // Same content but different uint8Array instances have a different hash because it would be too costly to deeply hash uint8Arrays
+      expect(hash4).toBe(hash5) // Same content = same hash for small Uint8Arrays
       expect(hash4).not.toBe(hash6) // Different uint8Array content should have different hash
       expect(hash4).toBe(hash(uint8Array1)) // Hashing same uint8Array should return same hash
+    })
+
+    it(`should hash large Buffers, Uint8Arrays and File objects by reference`, () => {
+      // Large buffers (>128 bytes) are hashed by reference to avoid performance costs
+      const largeBuffer1 = Buffer.alloc(300)
+      const largeBuffer2 = Buffer.alloc(300)
+
+      // Fill with same content
+      for (let i = 0; i < 300; i++) {
+        largeBuffer1[i] = i % 256
+        largeBuffer2[i] = i % 256
+      }
+
+      const hash1 = hash(largeBuffer1)
+      const hash2 = hash(largeBuffer2)
+
+      expect(typeof hash1).toBe(hashType)
+      expect(hash1).not.toBe(hash2) // Same content but different instances = different hash for large buffers
+      expect(hash1).toBe(hash(largeBuffer1)) // Hashing same buffer should return same hash
+
+      const largeUint8Array1 = new Uint8Array(300)
+      const largeUint8Array2 = new Uint8Array(300)
+
+      // Fill with same content
+      for (let i = 0; i < 300; i++) {
+        largeUint8Array1[i] = i % 256
+        largeUint8Array2[i] = i % 256
+      }
+
+      const hash3 = hash(largeUint8Array1)
+      const hash4 = hash(largeUint8Array2)
+
+      expect(typeof hash3).toBe(hashType)
+      expect(hash3).not.toBe(hash4) // Same content but different instances = different hash for large Uint8Arrays
+      expect(hash3).toBe(hash(largeUint8Array1)) // Hashing same uint8Array should return same hash
 
+      // Files are always hashed by reference regardless of size
       const file1 = new File([`Hello, world!`], `test.txt`)
       const file2 = new File([`Hello, world!`], `test.txt`)
       const file3 = new File([`Hello, world!`], `test.txt`)

packages/db/src/query/compiler/evaluators.ts

@@ -3,7 +3,7 @@ import {
   UnknownExpressionTypeError,
   UnknownFunctionError,
 } from "../../errors.js"
-import { normalizeValue } from "../../utils/comparison.js"
+import { areValuesEqual, normalizeValue } from "../../utils/comparison.js"
 import type { BasicExpression, Func, PropRef } from "../ir.js"
 import type { NamespacedRow } from "../../types.js"
 
@@ -172,7 +172,8 @@ function compileFunction(func: Func, isSingleRow: boolean): (data: any) => any {
         if (isUnknown(a) || isUnknown(b)) {
           return null
         }
-        return a === b
+        // Use areValuesEqual for proper Uint8Array/Buffer comparison
+        return areValuesEqual(a, b)
       }
     }
     case `gt`: {

packages/db/src/utils/comparison.ts

@@ -111,6 +111,21 @@ export const defaultComparator = makeComparator({
   stringSort: `locale`,
 })
 
+/**
+ * Compare two Uint8Arrays for content equality
+ */
+function areUint8ArraysEqual(a: Uint8Array, b: Uint8Array): boolean {
+  if (a.byteLength !== b.byteLength) {
+    return false
+  }
+  for (let i = 0; i < a.byteLength; i++) {
+    if (a[i] !== b[i]) {
+      return false
+    }
+  }
+  return true
+}
+
 /**
  * Normalize a value for comparison
  */
@@ -120,3 +135,29 @@ export function normalizeValue(value: any): any {
   }
   return value
 }
+
+/**
+ * Compare two values for equality, with special handling for Uint8Arrays and Buffers
+ */
+export function areValuesEqual(a: any, b: any): boolean {
+  // Fast path for reference equality
+  if (a === b) {
+    return true
+  }
+
+  // Check for Uint8Array/Buffer comparison
+  const aIsUint8Array =
+    (typeof Buffer !== `undefined` && a instanceof Buffer) ||
+    a instanceof Uint8Array
+  const bIsUint8Array =
+    (typeof Buffer !== `undefined` && b instanceof Buffer) ||
+    b instanceof Uint8Array
+
+  // If both are Uint8Arrays, compare by content
+  if (aIsUint8Array && bIsUint8Array) {
+    return areUint8ArraysEqual(a, b)
+  }
+
+  // Different types or not Uint8Arrays
+  return false
+}

packages/db/tests/query/compiler/evaluators.test.ts

@@ -423,6 +423,88 @@ describe(`evaluators`, () => {
 
             expect(compiled({})).toBe(false)
           })
+
+          it(`handles eq with matching Uint8Arrays (content equality)`, () => {
+            const array1 = new Uint8Array([1, 2, 3, 4, 5])
+            const array2 = new Uint8Array([1, 2, 3, 4, 5])
+            const func = new Func(`eq`, [new Value(array1), new Value(array2)])
+            const compiled = compileExpression(func)
+
+            // Should return true because content is the same
+            expect(compiled({})).toBe(true)
+          })
+
+          it(`handles eq with non-matching Uint8Arrays (different content)`, () => {
+            const array1 = new Uint8Array([1, 2, 3, 4, 5])
+            const array2 = new Uint8Array([1, 2, 3, 4, 6])
+            const func = new Func(`eq`, [new Value(array1), new Value(array2)])
+            const compiled = compileExpression(func)
+
+            // Should return false because content is different
+            expect(compiled({})).toBe(false)
+          })
+
+          it(`handles eq with Uint8Arrays of different lengths`, () => {
+            const array1 = new Uint8Array([1, 2, 3, 4])
+            const array2 = new Uint8Array([1, 2, 3, 4, 5])
+            const func = new Func(`eq`, [new Value(array1), new Value(array2)])
+            const compiled = compileExpression(func)
+
+            // Should return false because lengths are different
+            expect(compiled({})).toBe(false)
+          })
+
+          it(`handles eq with same Uint8Array reference`, () => {
+            const array = new Uint8Array([1, 2, 3, 4, 5])
+            const func = new Func(`eq`, [new Value(array), new Value(array)])
+            const compiled = compileExpression(func)
+
+            // Should return true (fast path for reference equality)
+            expect(compiled({})).toBe(true)
+          })
+
+          it(`handles eq with Uint8Array and non-Uint8Array`, () => {
+            const array = new Uint8Array([1, 2, 3])
+            const value = [1, 2, 3]
+            const func = new Func(`eq`, [new Value(array), new Value(value)])
+            const compiled = compileExpression(func)
+
+            // Should return false because types are different
+            expect(compiled({})).toBe(false)
+          })
+
+          it(`handles eq with ULIDs (16-byte Uint8Arrays)`, () => {
+            // Simulate ULID comparison - 16 bytes
+            const ulid1 = new Uint8Array(16)
+            const ulid2 = new Uint8Array(16)
+
+            // Fill with same values
+            for (let i = 0; i < 16; i++) {
+              ulid1[i] = i
+              ulid2[i] = i
+            }
+
+            const func = new Func(`eq`, [new Value(ulid1), new Value(ulid2)])
+            const compiled = compileExpression(func)
+
+            // Should return true because content is identical
+            expect(compiled({})).toBe(true)
+          })
+
+          it(`handles eq with Buffers (if available)`, () => {
+            if (typeof Buffer !== `undefined`) {
+              const buffer1 = Buffer.from([1, 2, 3, 4, 5])
+              const buffer2 = Buffer.from([1, 2, 3, 4, 5])
+              const func = new Func(`eq`, [
+                new Value(buffer1),
+                new Value(buffer2),
+              ])
+              const compiled = compileExpression(func)
+
+              // Should return true because content is the same
+              expect(compiled({})).toBe(true)
+            }
+          })
         })
 
         describe(`gt (greater than)`, () => {