Deduplicate media items by dedupKey (#126)

* Deduplicate media items by dedupKey

There can be photos with different mediaKeys but same dedupKey, likely caused by saving shared albums.

We should not solve for them, since
- Google Photos natively collapses them into one in timeline/albums
- API doesn't allow to trash one of them without trashing others

So keep only the first item we see for each dedupKey.

* Fix totalItems/log to report raw item count, not post-dedup count

mediaItems was reassigned to the deduped set, so ScanTiming.totalItems
and the "[GPD] detectDuplicates" log line silently started meaning
"items after dedup" instead of "items scanned". Use a separate
dedupedItems binding for candidate filtering so totalItems keeps its
original meaning while duplicate detection still only runs on the
deduped set.

Claude-Session: https://claude.ai/code/session_01EwfkVcjRcCwgv9oMRhnMqu

* Rename dropDuplicateDedupKeys to dedupeByDedupKey; tidy test formatting

Claude-Session: https://claude.ai/code/session_01EwfkVcjRcCwgv9oMRhnMqu

---------

Co-authored-by: Mack Talcott <mack@tal.co.tt>

Author: zhogov

lib/duplicate-detector.ts

@@ -29,6 +29,28 @@ export function selectDefaultKeep(items: GpdMediaItem[]): string {
   return best[0].mediaKey;
 }
 
+/**
+ * Deduplicate media items by dedupKey.
+ * There can be photos with different mediaKeys but same dedupKey,
+ * likely caused by saving shared albums.
+ * We should not solve for them, since
+ * - Google Photos natively collapses them into one in timeline/albums
+ * - API doesn't allow to trash one of them without trashing others
+ * So keep only the first item we see for each dedupKey.
+ */
+export function dedupeByDedupKey(items: GpdMediaItem[]): GpdMediaItem[] {
+  const seen = new Set<string>();
+  const uniqueItems: GpdMediaItem[] = [];
+  for (const item of items) {
+    if (item.dedupKey) {
+      if (seen.has(item.dedupKey)) continue;
+      seen.add(item.dedupKey);
+    }
+    uniqueItems.push(item);
+  }
+  return uniqueItems;
+}
+
 const MODEL_URL =
   "https://storage.googleapis.com/mediapipe-models/image_embedder/mobilenet_v3_large/float32/latest/mobilenet_v3_large.tflite";
 
@@ -160,11 +182,13 @@ export async function fullDetectDuplicates(
 ): Promise<{ groups: DuplicateGroup[]; timing: ScanTiming }> {
   const scanStart = performance.now();
 
+  const dedupedItems = dedupeByDedupKey(mediaItems);
+
   // Filter to items with thumbnails (photos only, skip videos)
   // Include items with thumbnails. Video posters work too — two copies of the
   // same clip have identical poster frames, which produce near-identical
   // embeddings.
-  const candidates = mediaItems.filter((item) => item.thumb);
+  const candidates = dedupedItems.filter((item) => item.thumb);
   console.log(
     `[GPD] detectDuplicates: ${mediaItems.length} items → ${candidates.length} candidates`,
   );
@@ -466,10 +490,13 @@ export async function smartDetectDuplicates(
   logger?: ScanLogger,
 ): Promise<DuplicateGroup[]> {
   const scanStart = performance.now();
+
+  const dedupedItems = dedupeByDedupKey(mediaItems);
+
   // Include items with thumbnails. Video posters work too — two copies of the
   // same clip have identical poster frames, which produce near-identical
   // embeddings.
-  const candidates = mediaItems.filter((item) => item.thumb);
+  const candidates = dedupedItems.filter((item) => item.thumb);
 
   // Step 1: Bucket by timestamp — no I/O, instant
   const buckets = groupByTimestamp(candidates, windowMs);

tests/lib/duplicate-detector.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from "vitest"
-import { communityDetection, matMul, topK, groupByTimestamp, withinGroupDuplicates, selectDefaultKeep, fullDetectDuplicates } from "../../lib/duplicate-detector"
+import { communityDetection, matMul, topK, groupByTimestamp, withinGroupDuplicates, selectDefaultKeep, fullDetectDuplicates, dedupeByDedupKey } from "../../lib/duplicate-detector"
 import type { GpdMediaItem } from "../../lib/types"
 
 // ============================================================
@@ -535,3 +535,39 @@ describe("selectDefaultKeep", () => {
     expect(["a", "b"]).toContain(result)
   })
 })
+
+// ============================================================
+// dedupeByDedupKey
+// ============================================================
+
+describe("dedupeByDedupKey", () => {
+  it("keeps items with unique dedupKeys", () => {
+    const a = makeItem("media1", 1000)
+    a.dedupKey = "dedup1"
+    const b = makeItem("media2", 1000)
+    b.dedupKey = "dedup2"
+
+    const result = dedupeByDedupKey([a, b])
+    expect(result).toHaveLength(2)
+    expect(result).toContain(a)
+    expect(result).toContain(b)
+  })
+
+  it("keeps only the first item with a duplicated dedupKey", () => {
+    const a = makeItem("media1", 1000)
+    a.dedupKey = "shared-dedup"
+    const b = makeItem("media2", 1000)
+    b.dedupKey = "shared-dedup"
+    const c = makeItem("media3", 1000)
+    c.dedupKey = "other-dedup"
+
+    const result = dedupeByDedupKey([a, b, c])
+    expect(result).toHaveLength(2)
+    expect(result[0]).toBe(a)
+    expect(result[1]).toBe(c)
+  })
+
+  it("handles empty arrays", () => {
+    expect(dedupeByDedupKey([])).toEqual([])
+  })
+})