Merge pull request 'feat: group catalog duplicates' (#1) from feat/catalog-group-duplicates into main

Grouping + skip torrent folders named like videos + index-backed streams.

library.js

@@ -19,6 +19,68 @@ const nfoFor = (mediaRoot, file) => {
   } catch { return {}; }
 };
 
+
+const slug = value => String(value || '').toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '');
+
+export function catalogGroupKey(item) {
+  const imdb = item.imdbId || item.suggestedImdbId || null;
+  const episode = item.type === 'episode' || (item.season != null && item.episode != null);
+  if (episode) {
+    return imdb && item.season != null && item.episode != null ? `${imdb}:${item.season}:${item.episode}` : null;
+  }
+  if (imdb) return imdb;
+  const title = slug(item.title);
+  return title ? `${title}${item.year ? `-${item.year}` : ''}` : null;
+}
+
+const uniqueFiles = files => {
+  const seen = new Set();
+  return files.filter(file => {
+    if (seen.has(file.id)) return false;
+    seen.add(file.id);
+    return true;
+  });
+};
+
+export function catalogTitleKey(item) {
+  return catalogGroupKey({ ...item, imdbId: null, suggestedImdbId: null });
+}
+
+export function catalogAliases(item) {
+  return new Set([item?.id, item?.imdbId, item?.suggestedImdbId, catalogTitleKey(item)].filter(Boolean));
+}
+
+export function groupCatalog(items) {
+  const groups = new Map();
+  for (const item of items) {
+    const key = catalogGroupKey(item) || item.id;
+    const members = groups.get(key);
+    if (members) members.push(item);
+    else groups.set(key, [item]);
+  }
+  return [...groups.entries()].flatMap(([id, members]) => {
+    const files = uniqueFiles(members
+      .filter(member => (member.size || 0) > 0)
+      .map(member => ({ id: member.id, file: member.file, size: member.size || 0 }))
+      .sort((a, b) => b.size - a.size || a.file.localeCompare(b.file)));
+    if (!files.length) return [];
+    const representative = members.find(member => member.id === files[0].id) || members[0];
+    return [{ ...representative, id, file: files[0].file, files, size: files[0].size }];
+  });
+}
+
+export function catalogMembers(item, requestedId) {
+  const files = (item?.files || (item ? [{ id: item.id, file: item.file, size: item.size || 0 }] : []))
+    .filter(file => (file.size || 0) > 0)
+    .slice()
+    .sort((a, b) => (b.size || 0) - (a.size || 0) || String(a.file).localeCompare(String(b.file)));
+  if (requestedId && item && !catalogAliases(item).has(requestedId)) {
+    const match = files.find(file => file.id === requestedId);
+    return match ? [match] : [];
+  }
+  return files;
+}
+
 export function createLibrary(mediaRoot, indexPath, tmdbApiKey) {
   const metadataCache = new Map();
   const load = () => {
@@ -87,16 +149,18 @@ export function createLibrary(mediaRoot, indexPath, tmdbApiKey) {
     metadataCache.set(key, pending);
     return pending;
   };
+  let cachedScan = null;
   const scan = async () => {
     const index = load();
-    const files = fs.existsSync(mediaRoot) ? fs.readdirSync(mediaRoot, { recursive: true }).filter(file => video.test(file)) : [];
+    const listed = fs.existsSync(mediaRoot) ? fs.readdirSync(mediaRoot, { recursive: true }).filter(file => video.test(file)) : [];
-    const present = new Set(files);
+    const present = new Set();
+    const items = [];
     let changed = false;
-    for (const file of Object.keys(index.mappings)) {
+    for (const file of listed) {
-      if (!present.has(file) && !index.mappings[file].missing) { index.mappings[file].missing = true; changed = true; }
+      let stat;
-    }
+      try { stat = fs.statSync(path.join(mediaRoot, file)); } catch { continue; }
-    const items = files.map(file => {
+      if (!stat.isFile()) continue;
-      const stat = fs.statSync(path.join(mediaRoot, file));
+      present.add(file);
       const fingerprint = `${stat.size}:${Math.round(stat.mtimeMs)}`;
       const parsed = guessit(path.basename(file));
       const previous = Object.entries(index.mappings).find(([oldPath, value]) => oldPath !== file && value.missing && value.fingerprint === fingerprint)?.[1];
@@ -106,9 +170,10 @@ export function createLibrary(mediaRoot, indexPath, tmdbApiKey) {
         index.mappings[file] = { ...mapping, fingerprint, missing: false, updatedAt: mapping.updatedAt || new Date().toISOString() };
         changed = true;
       }
-      return {
+      items.push({
         file,
         id: Buffer.from(file).toString('base64url'),
+        size: stat.size,
         title: mapping.title || nfo.title || parsed.title || path.basename(file, path.extname(file)),
         year: mapping.year || nfo.year || parsed.year || null,
         imdbId: mapping.imdbId || nfo.imdbId || parsed.imdb_id || file.match(/\btt\d{7,10}\b/i)?.[0]?.toLowerCase() || null,
@@ -116,8 +181,15 @@ export function createLibrary(mediaRoot, indexPath, tmdbApiKey) {
         season: mapping.season || parsed.season || null,
         episode: mapping.episode || parsed.episode || null,
         mapped: Boolean(mapping.imdbId),
-      };
+      });
-    });
+    }
+    for (const file of Object.keys(index.mappings)) {
+      if (present.has(file) || index.mappings[file].missing) continue;
+      const size = Number(String(index.mappings[file].fingerprint || '').split(':')[0]);
+      if (size > 0) continue;
+      index.mappings[file].missing = true;
+      changed = true;
+    }
     const discovered = await Promise.all(items.map(discover));
     for (const item of discovered) {
       if (!item.autoMapped || index.mappings[item.file]?.imdbId) continue;
@@ -134,8 +206,32 @@ export function createLibrary(mediaRoot, indexPath, tmdbApiKey) {
       changed = true;
     }
     if (changed) save(index);
+    cachedScan = discovered;
     return discovered;
   };
+  const itemsFromIndex = () => Object.entries(load().mappings).flatMap(([file, mapping]) => {
+    const size = Number(String(mapping.fingerprint || '').split(':')[0]);
+    if (!size || !video.test(file) || file.includes('..')) return [];
+    const parsed = guessit(path.basename(file));
+    return [{
+      file,
+      id: Buffer.from(file).toString('base64url'),
+      size,
+      title: mapping.title || parsed.title || path.basename(file, path.extname(file)),
+      year: mapping.year || parsed.year || null,
+      imdbId: mapping.imdbId || parsed.imdb_id || file.match(/\btt\d{7,10}\b/i)?.[0]?.toLowerCase() || null,
+      type: parsed.type === 'episode' ? 'episode' : 'video',
+      season: mapping.season || parsed.season || null,
+      episode: mapping.episode || parsed.episode || null,
+      mapped: Boolean(mapping.imdbId),
+    }];
+  });
+  const findItem = (items, id) => items.find(item => catalogAliases(item).has(id) || item.files?.some(file => file.id === id)) || null;
+  let indexGrouped = null;
+  const groupedFromIndex = () => {
+    if (!indexGrouped) indexGrouped = groupCatalog(itemsFromIndex());
+    return indexGrouped;
+  };
   return {
     scan,
     startReconciler(seconds = 900) {
@@ -155,10 +251,15 @@ export function createLibrary(mediaRoot, indexPath, tmdbApiKey) {
       save(index);
     },
     async catalog() {
-      return Promise.all((await scan()).map(metadataFor));
+      return groupCatalog(await Promise.all((await scan()).map(metadataFor)));
     },
     async metadata(id) {
-      return (await this.catalog()).find(item => item.id === id) || null;
+      return findItem(await this.catalog(), id);
+    },
+    async members(id) {
+      const grouped = cachedScan ? groupCatalog(cachedScan) : groupedFromIndex();
+      const item = findItem(grouped, id);
+      return item ? catalogMembers(item, id) : null;
     },
   };
 }

library.test.js

@@ -0,0 +1,95 @@
+import assert from 'node:assert/strict';
+import { test } from 'node:test';
+import { catalogAliases, catalogGroupKey, catalogMembers, groupCatalog } from './library.js';
+
+const pathId = file => Buffer.from(file).toString('base64url');
+
+test('two files with the same imdbId become one catalog item with two streams', () => {
+  const items = [
+    { id: pathId('Evolution.mp4'), file: 'Evolution.mp4', title: 'Evolution', year: 2026, imdbId: 'tt6150344', type: 'video', size: 100 },
+    { id: pathId('Evolution (2026)/Evolution.mp4'), file: 'Evolution (2026)/Evolution.mp4', title: 'Evolution', year: 2026, imdbId: 'tt6150344', type: 'video', size: 250 },
+  ];
+  const catalog = groupCatalog(items);
+  assert.equal(catalog.length, 1);
+  assert.equal(catalog[0].id, 'tt6150344');
+  assert.equal(catalog[0].title, 'Evolution');
+  assert.equal(catalog[0].year, 2026);
+  assert.equal(catalog[0].imdbId, 'tt6150344');
+  const streams = catalogMembers(catalog[0], catalog[0].id);
+  assert.equal(streams.length, 2);
+  assert.equal(streams[0].file, 'Evolution (2026)/Evolution.mp4');
+  assert.equal(streams[1].file, 'Evolution.mp4');
+  assert.equal(streams[0].id, pathId('Evolution (2026)/Evolution.mp4'));
+});
+
+test('metadata keeps path-id as an alias of the group', () => {
+  const path = pathId('copy-a.mp4');
+  const catalog = groupCatalog([
+    { id: path, file: 'copy-a.mp4', title: 'Evolution', year: 2026, imdbId: 'tt6150344', type: 'video', size: 10 },
+    { id: pathId('copy-b.mp4'), file: 'copy-b.mp4', title: 'Evolution', year: 2026, suggestedImdbId: 'tt6150344', type: 'video', size: 20 },
+  ]);
+  const item = catalog.find(entry => entry.id === 'tt6150344' || entry.files.some(file => file.id === path));
+  assert.equal(item.id, 'tt6150344');
+  assert.equal(catalogMembers(item, path).length, 1);
+  assert.equal(catalogMembers(item, path)[0].file, 'copy-a.mp4');
+});
+
+test('episodes group by imdbId:season:episode and never collapse a series', () => {
+  const items = [
+    { id: 'e1', file: 'Show.S01E01.mkv', title: 'Show', year: 2020, imdbId: 'tt1111111', type: 'episode', season: 1, episode: 1, size: 1 },
+    { id: 'e2', file: 'Show.S01E02.mkv', title: 'Show', year: 2020, imdbId: 'tt1111111', type: 'episode', season: 1, episode: 2, size: 1 },
+    { id: 'e1b', file: 'Show.S01E01.alt.mkv', title: 'Show', year: 2020, imdbId: 'tt1111111', type: 'episode', season: 1, episode: 1, size: 2 },
+  ];
+  const catalog = groupCatalog(items);
+  assert.equal(catalog.length, 2);
+  assert.deepEqual(new Set(catalog.map(item => item.id)), new Set(['tt1111111:1:1', 'tt1111111:1:2']));
+  assert.equal(catalog.find(item => item.id === 'tt1111111:1:1').files.length, 2);
+});
+
+test('unmapped titles group by lowercase title and year', () => {
+  const catalog = groupCatalog([
+    { id: 'a', file: 'a.mp4', title: 'Evolution', year: 2026, type: 'video', size: 1 },
+    { id: 'b', file: 'b.mp4', title: 'evolution', year: 2026, type: 'video', size: 2 },
+  ]);
+  assert.equal(catalog.length, 1);
+  assert.equal(catalog[0].id, 'evolution-2026');
+  assert.equal(catalogGroupKey({ title: 'Evolution', year: 2026, type: 'video' }), 'evolution-2026');
+});
+
+test('size-0 torrent folders are dropped when a real file exists', () => {
+  const catalog = groupCatalog([
+    { id: 'dir', file: 'Rons.Gone.Wrong.2021.mkv', title: 'Rons Gone Wrong', year: 2021, type: 'video', size: 0 },
+    { id: 'real', file: 'Rons.Gone.Wrong.2021.mkv/Rons.Gone.Wrong.2021.mkv', title: 'Rons Gone Wrong', year: 2021, type: 'video', size: 1506040768 },
+    { id: 'yts', file: 'Rons Gone Wrong (2021)/Rons.Gone.Wrong.2021.mp4', title: 'Rons Gone Wrong', year: 2021, type: 'video', size: 2112849010 },
+  ]);
+  assert.equal(catalog.length, 1);
+  assert.equal(catalog[0].id, 'rons-gone-wrong-2021');
+  assert.equal(catalog[0].size, 2112849010);
+  assert.deepEqual(catalog[0].files.map(file => file.id), ['yts', 'real']);
+  assert.equal(catalogMembers(catalog[0], catalog[0].id).length, 2);
+});
+
+test('duplicate file ids collapse to one stream', () => {
+  const catalog = groupCatalog([
+    { id: 'a', file: 'Movie.mp4', title: 'Movie', year: 2020, type: 'video', size: 10 },
+    { id: 'a', file: 'Movie.mp4', title: 'Movie', year: 2020, type: 'video', size: 10 },
+  ]);
+  assert.equal(catalog[0].files.length, 1);
+});
+
+test('a size-0-only group is omitted', () => {
+  const catalog = groupCatalog([
+    { id: 'dir', file: 'Rons.Gone.Wrong.2021.mkv', title: 'Rons Gone Wrong', year: 2021, type: 'video', size: 0 },
+  ]);
+  assert.equal(catalog.length, 0);
+});
+
+test('title-year id still resolves after imdb grouping', () => {
+  const catalog = groupCatalog([
+    { id: 'a', file: 'Rons.mkv', title: 'Rons Gone Wrong', year: 2021, imdbId: 'tt7504818', type: 'video', size: 10 },
+    { id: 'b', file: 'Rons.mp4', title: 'Rons Gone Wrong', year: 2021, imdbId: 'tt7504818', type: 'video', size: 20 },
+  ]);
+  assert.equal(catalog[0].id, 'tt7504818');
+  assert.equal(catalogMembers(catalog[0], 'rons-gone-wrong-2021').length, 2);
+  assert.ok(catalogAliases(catalog[0]).has('rons-gone-wrong-2021'));
+});

package.json

@@ -4,6 +4,7 @@
   "type": "module",
   "scripts": {
     "start": "node server.js",
+    "test": "node --test library.test.js",
     "package:macos": "sh scripts/build-desktop-package.sh macos",
     "package:linux": "sh scripts/build-desktop-package.sh linux",
     "package:windows": "sh scripts/build-desktop-package.sh windows"

server.js

@@ -59,6 +59,20 @@ const admin = req => {
 const html = value => String(value).replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;').replaceAll('"', '&quot;');
 const libraryPage = items => `<!doctype html><title>CleanStream Library</title><style>body{font:16px system-ui;max-width:1100px;margin:2rem auto}form{display:grid;grid-template-columns:2fr 2fr 1fr auto;gap:.5rem;margin:.5rem 0}input{padding:.5rem}small{grid-column:1/-1;color:#666}</style><h1>Library review</h1><p>Confirm titles and IMDb IDs. Original files are never renamed or moved.</p>${items.map(item => `<form method="post" action="/library/mapping"><input type="hidden" name="file" value="${encodeURIComponent(item.file)}"><input name="title" value="${html(item.title)}"><input name="imdbId" placeholder="tt1234567" value="${html(item.imdbId || item.suggestedImdbId || '')}"><input name="year" value="${item.year || ''}" placeholder="year"><button>Save</button><small>${html(item.file)}${item.candidate ? ` · TMDB: ${html(item.candidate.title)} (${html(item.candidate.year)})` : ''}</small></form>`).join('')}`;
 
+const resolveMediaTarget = file => {
+  if (!file || file.includes('..')) return null;
+  const target = path.resolve(mediaRoot, file);
+  if (!target.startsWith(`${mediaRoot}${path.sep}`)) return null;
+  try {
+    const stat = fs.statSync(target);
+    if (stat.isFile()) return target;
+    if (stat.isDirectory()) {
+      const nested = path.join(target, path.basename(target));
+      if (nested.startsWith(`${mediaRoot}${path.sep}`) && fs.statSync(nested).isFile()) return nested;
+    }
+  } catch {}
+  return null;
+};
 http.createServer(async (req, res) => {
   const url = new URL(req.url, publicBaseUrl);
   console.log(`${req.method} ${url.pathname}`);
@@ -89,7 +103,31 @@ http.createServer(async (req, res) => {
     const item = await library.metadata(url.searchParams.get('id') || '');
     return item ? json(res, 200, { item: { ...item, imdbId: item.imdbId || item.suggestedImdbId, description: item.description || item.file } }) : json(res, 404, { error: 'Not found' });
   }
-  if (req.method === 'POST' && url.pathname === '/v1/stream') { if (!authenticated(req)) return json(res,401,{error:'Unauthorized'}); const body=await readJson(req); if (body.id === 'demo-stream' && demoStreamUrl) return json(res,200,{streams:[{id:'demo-stream',url:demoStreamUrl,filename:'demo.m3u8',headers:{}}]}); const file=Buffer.from(body.id || '', 'base64url').toString(); if (!(await videos()).some(v => v.id === body.id) || file.includes('..')) return json(res,404,{error:'Not found'}); return json(res,200,{streams:[{id:body.id,url:`${publicBaseUrl}/v1/media/${body.id}?token=${bearer(req)}`,filename:path.basename(file),headers:{}}]}); }
+  if (req.method === 'POST' && url.pathname === '/v1/stream') { if (!authenticated(req)) return json(res,401,{error:'Unauthorized'}); const body=await readJson(req); if (body.id === 'demo-stream' && demoStreamUrl) return json(res,200,{streams:[{id:'demo-stream',url:demoStreamUrl,filename:'demo.m3u8',headers:{}}]}); const members=await library.members(body.id || ''); if (!members?.length || members.some(member => member.file.includes('..'))) return json(res,404,{error:'Not found'}); return json(res,200,{streams:members.map(member => ({id:member.id,url:`${publicBaseUrl}/v1/media/${member.id}?token=${bearer(req)}`,filename:path.basename(member.file),headers:{}}))}); }
-  if (req.method === 'GET' && url.pathname.startsWith('/v1/media/')) { const token=url.searchParams.get('token'); const file=Buffer.from(url.pathname.split('/').pop(), 'base64url').toString(); const target=path.resolve(mediaRoot,file); if (!tokens.has(token) || !target.startsWith(`${mediaRoot}${path.sep}`) || !fs.existsSync(target)) return json(res,404,{error:'Not found'}); res.writeHead(200,{'content-type':'application/octet-stream','content-length':fs.statSync(target).size}); return fs.createReadStream(target).pipe(res); }
+  if ((req.method === 'GET' || req.method === 'HEAD') && url.pathname.startsWith('/v1/media/')) {
+    const token = url.searchParams.get('token');
+    const file = Buffer.from(url.pathname.split('/').pop(), 'base64url').toString();
+    const target = resolveMediaTarget(file);
+    if (!tokens.has(token) || !target) return json(res, 404, { error: 'Not found' });
+    const size = fs.statSync(target).size;
+    const match = req.headers.range?.match(/^bytes=(\d*)-(\d*)$/);
+    const start = Number(match?.[1] || 0);
+    const end = Math.min(Number(match?.[2] || size - 1), size - 1);
+    if (match && (start > end || start >= size)) {
+      res.writeHead(416, { 'content-range': `bytes */${size}` });
+      return res.end();
+    }
+    const headers = { 'content-type': 'application/octet-stream', 'accept-ranges': 'bytes', 'content-length': end - start + 1 };
+    if (match) headers['content-range'] = `bytes ${start}-${end}/${size}`;
+    res.writeHead(match ? 206 : 200, headers);
+    if (req.method === 'HEAD') return res.end();
+    const stream = fs.createReadStream(target, { start, end });
+    stream.on('error', error => {
+      console.error(`Media read failed: ${error.message}`);
+      if (!res.headersSent) json(res, 502, { error: 'Media read failed' });
+      else res.destroy(error);
+    });
+    return stream.pipe(res);
+  }
   json(res, 404, { error: 'Not found' });
 }).listen(port, () => console.log(`Connector listening on ${publicBaseUrl}`));