fix(db): ensure deterministic iteration order for collections and indexes (#958)

* fix(db): ensure deterministic iteration order for collections and indexes

- SortedMap: add key-based tie-breaking for deterministic ordering
- SortedMap: optimize to skip value comparison when no comparator provided
- BTreeIndex: sort keys within same indexed value for deterministic order
- BTreeIndex: add fast paths for empty/single-key sets
- CollectionStateManager: always use SortedMap for deterministic iteration
- Extract compareKeys utility to utils/comparison.ts
- Add comprehensive tests for deterministic ordering behavior

* ci: apply automated fixes

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

Author: samwillis

.changeset/deterministic-collection-ordering.md

@@ -0,0 +1,23 @@
+---
+'@tanstack/db': patch
+---
+
+Ensure deterministic iteration order for collections and indexes.
+
+**SortedMap improvements:**
+
+- Added key-based tie-breaking when values compare as equal, ensuring deterministic ordering
+- Optimized to skip value comparison entirely when no comparator is provided (key-only sorting)
+- Extracted `compareKeys` utility to `utils/comparison.ts` for reuse
+
+**BTreeIndex improvements:**
+
+- Keys within the same indexed value are now returned in deterministic sorted order
+- Optimized with fast paths for empty sets and single-key sets to avoid unnecessary allocations
+
+**CollectionStateManager changes:**
+
+- Collections now always use `SortedMap` for `syncedData`, ensuring deterministic iteration order
+- When no `compare` function is provided, entries are sorted by key only
+
+This ensures that live queries with `orderBy` and `limit` produce stable, deterministic results even when multiple rows have equal sort values.

packages/db/src/SortedMap.ts

@@ -1,62 +1,81 @@
+import { compareKeys } from '@tanstack/db-ivm'
+
 /**
  * A Map implementation that keeps its entries sorted based on a comparator function
- * @template TKey - The type of keys in the map
+ * @template TKey - The type of keys in the map (must be string | number)
  * @template TValue - The type of values in the map
  */
-export class SortedMap<TKey, TValue> {
+export class SortedMap<TKey extends string | number, TValue> {
   private map: Map<TKey, TValue>
   private sortedKeys: Array<TKey>
-  private comparator: (a: TValue, b: TValue) => number
+  private comparator: ((a: TValue, b: TValue) => number) | undefined
 
   /**
    * Creates a new SortedMap instance
    *
-   * @param comparator - Optional function to compare values for sorting
+   * @param comparator - Optional function to compare values for sorting.
+   *                     If not provided, entries are sorted by key only.
    */
   constructor(comparator?: (a: TValue, b: TValue) => number) {
     this.map = new Map<TKey, TValue>()
     this.sortedKeys = []
-    this.comparator = comparator || this.defaultComparator
-  }
-
-  /**
-   * Default comparator function used when none is provided
-   *
-   * @param a - First value to compare
-   * @param b - Second value to compare
-   * @returns -1 if a < b, 1 if a > b, 0 if equal
-   */
-  private defaultComparator(a: TValue, b: TValue): number {
-    if (a < b) return -1
-    if (a > b) return 1
-    return 0
+    this.comparator = comparator
   }
 
   /**
    * Finds the index where a key-value pair should be inserted to maintain sort order.
-   * Uses binary search to find the correct position based on the value.
-   * Hence, it is in O(log n) time.
+   * Uses binary search to find the correct position based on the value (if comparator provided),
+   * with key-based tie-breaking for deterministic ordering when values compare as equal.
+   * If no comparator is provided, sorts by key only.
+   * Runs in O(log n) time.
    *
-   * @param key - The key to find position for
-   * @param value - The value to compare against
+   * @param key - The key to find position for (used as tie-breaker or primary sort when no comparator)
+   * @param value - The value to compare against (only used if comparator is provided)
    * @returns The index where the key should be inserted
    */
-  private indexOf(value: TValue): number {
+  private indexOf(key: TKey, value: TValue): number {
     let left = 0
     let right = this.sortedKeys.length
 
+    // Fast path: no comparator means sort by key only
+    if (!this.comparator) {
+      while (left < right) {
+        const mid = Math.floor((left + right) / 2)
+        const midKey = this.sortedKeys[mid]!
+        const keyComparison = compareKeys(key, midKey)
+        if (keyComparison < 0) {
+          right = mid
+        } else if (keyComparison > 0) {
+          left = mid + 1
+        } else {
+          return mid
+        }
+      }
+      return left
+    }
+
+    // With comparator: sort by value first, then key as tie-breaker
     while (left < right) {
       const mid = Math.floor((left + right) / 2)
       const midKey = this.sortedKeys[mid]!
       const midValue = this.map.get(midKey)!
-      const comparison = this.comparator(value, midValue)
+      const valueComparison = this.comparator(value, midValue)
 
-      if (comparison < 0) {
+      if (valueComparison < 0) {
         right = mid
-      } else if (comparison > 0) {
+      } else if (valueComparison > 0) {
         left = mid + 1
       } else {
-        return mid
+        // Values are equal, use key as tie-breaker for deterministic ordering
+        const keyComparison = compareKeys(key, midKey)
+        if (keyComparison < 0) {
+          right = mid
+        } else if (keyComparison > 0) {
+          left = mid + 1
+        } else {
+          // Same key (shouldn't happen during insert, but handle for lookups)
+          return mid
+        }
       }
     }
 
@@ -74,12 +93,12 @@ export class SortedMap<TKey, TValue> {
     if (this.map.has(key)) {
       // Need to remove the old key from the sorted keys array
       const oldValue = this.map.get(key)!
-      const oldIndex = this.indexOf(oldValue)
+      const oldIndex = this.indexOf(key, oldValue)
       this.sortedKeys.splice(oldIndex, 1)
     }
 
     // Insert the new key at the correct position
-    const index = this.indexOf(value)
+    const index = this.indexOf(key, value)
     this.sortedKeys.splice(index, 0, key)
 
     this.map.set(key, value)
@@ -106,7 +125,7 @@ export class SortedMap<TKey, TValue> {
   delete(key: TKey): boolean {
     if (this.map.has(key)) {
       const oldValue = this.map.get(key)
-      const index = this.indexOf(oldValue!)
+      const index = this.indexOf(key, oldValue!)
       this.sortedKeys.splice(index, 1)
       return this.map.delete(key)
     }

packages/db/src/collection/state.ts

@@ -43,7 +43,7 @@ export class CollectionStateManager<
   public pendingSyncedTransactions: Array<
     PendingSyncedTransaction<TOutput, TKey>
   > = []
-  public syncedData: Map<TKey, TOutput> | SortedMap<TKey, TOutput>
+  public syncedData: SortedMap<TKey, TOutput>
   public syncedMetadata = new Map<TKey, unknown>()
 
   // Optimistic state tracking - make public for testing
@@ -69,12 +69,9 @@ export class CollectionStateManager<
       a.compareCreatedAt(b),
     )
 
-    // Set up data storage with optional comparison function
-    if (config.compare) {
-      this.syncedData = new SortedMap<TKey, TOutput>(config.compare)
-    } else {
-      this.syncedData = new Map<TKey, TOutput>()
-    }
+    // Set up data storage - always use SortedMap for deterministic iteration.
+    // If a custom compare function is provided, use it; otherwise entries are sorted by key only.
+    this.syncedData = new SortedMap<TKey, TOutput>(config.compare)
   }
 
   setDeps(deps: {

packages/db/src/indexes/btree-index.ts

@@ -1,3 +1,4 @@
+import { compareKeys } from '@tanstack/db-ivm'
 import { BTree } from '../utils/btree.js'
 import { defaultComparator, normalizeValue } from '../utils/comparison.js'
 import { BaseIndex } from './base-index.js'
@@ -262,6 +263,7 @@ export class BTreeIndex<
     nextPair: (k?: any) => [any, any] | undefined,
     from?: any,
     filterFn?: (key: TKey) => boolean,
+    reversed: boolean = false,
   ): Array<TKey> {
     const keysInResult: Set<TKey> = new Set()
     const result: Array<TKey> = []
@@ -271,10 +273,12 @@ export class BTreeIndex<
     while ((pair = nextPair(key)) !== undefined && result.length < n) {
       key = pair[0]
       const keys = this.valueMap.get(key)
-      if (keys) {
-        const it = keys.values()
-        let ks: TKey | undefined
-        while (result.length < n && (ks = it.next().value)) {
+      if (keys && keys.size > 0) {
+        // Sort keys for deterministic order, reverse if needed
+        const sorted = Array.from(keys).sort(compareKeys)
+        if (reversed) sorted.reverse()
+        for (const ks of sorted) {
+          if (result.length >= n) break
           if (!keysInResult.has(ks) && (filterFn?.(ks) ?? true)) {
             result.push(ks)
             keysInResult.add(ks)
@@ -309,7 +313,7 @@ export class BTreeIndex<
     filterFn?: (key: TKey) => boolean,
   ): Array<TKey> {
     const nextPair = (k?: any) => this.orderedEntries.nextLowerPair(k)
-    return this.takeInternal(n, nextPair, from, filterFn)
+    return this.takeInternal(n, nextPair, from, filterFn, true)
   }
 
   /**