Merge pull request 'feat: group catalog duplicates' (#1) from feat/catalog-group-duplicates into main

Grouping + skip torrent folders named like videos + index-backed streams.

library.js

@@ -33,6 +33,23 @@ export function catalogGroupKey(item) {
   return title ? `${title}${item.year ? `-${item.year}` : ''}` : null;
 }
 
+const uniqueFiles = files => {
+  const seen = new Set();
+  return files.filter(file => {
+    if (seen.has(file.id)) return false;
+    seen.add(file.id);
+    return true;
+  });
+};
+
+export function catalogTitleKey(item) {
+  return catalogGroupKey({ ...item, imdbId: null, suggestedImdbId: null });
+}
+
+export function catalogAliases(item) {
+  return new Set([item?.id, item?.imdbId, item?.suggestedImdbId, catalogTitleKey(item)].filter(Boolean));
+}
+
 export function groupCatalog(items) {
   const groups = new Map();
   for (const item of items) {
@@ -41,20 +58,23 @@ export function groupCatalog(items) {
     if (members) members.push(item);
     else groups.set(key, [item]);
   }
-  return [...groups.entries()].map(([id, members]) => {
-    const files = members
+  return [...groups.entries()].flatMap(([id, members]) => {
+    const files = uniqueFiles(members
+      .filter(member => (member.size || 0) > 0)
       .map(member => ({ id: member.id, file: member.file, size: member.size || 0 }))
-      .sort((a, b) => b.size - a.size || a.file.localeCompare(b.file));
+      .sort((a, b) => b.size - a.size || a.file.localeCompare(b.file)));
+    if (!files.length) return [];
     const representative = members.find(member => member.id === files[0].id) || members[0];
-    return { ...representative, id, files };
+    return [{ ...representative, id, file: files[0].file, files, size: files[0].size }];
   });
 }
 
 export function catalogMembers(item, requestedId) {
   const files = (item?.files || (item ? [{ id: item.id, file: item.file, size: item.size || 0 }] : []))
+    .filter(file => (file.size || 0) > 0)
     .slice()
     .sort((a, b) => (b.size || 0) - (a.size || 0) || String(a.file).localeCompare(String(b.file)));
-  if (requestedId && item && requestedId !== item.id) {
+  if (requestedId && item && !catalogAliases(item).has(requestedId)) {
     const match = files.find(file => file.id === requestedId);
     return match ? [match] : [];
   }
@@ -129,16 +149,18 @@ export function createLibrary(mediaRoot, indexPath, tmdbApiKey) {
     metadataCache.set(key, pending);
     return pending;
   };
+  let cachedScan = null;
   const scan = async () => {
     const index = load();
-    const files = fs.existsSync(mediaRoot) ? fs.readdirSync(mediaRoot, { recursive: true }).filter(file => video.test(file)) : [];
-    const present = new Set(files);
+    const listed = fs.existsSync(mediaRoot) ? fs.readdirSync(mediaRoot, { recursive: true }).filter(file => video.test(file)) : [];
+    const present = new Set();
+    const items = [];
     let changed = false;
-    for (const file of Object.keys(index.mappings)) {
-      if (!present.has(file) && !index.mappings[file].missing) { index.mappings[file].missing = true; changed = true; }
-    }
-    const items = files.map(file => {
-      const stat = fs.statSync(path.join(mediaRoot, file));
+    for (const file of listed) {
+      let stat;
+      try { stat = fs.statSync(path.join(mediaRoot, file)); } catch { continue; }
+      if (!stat.isFile()) continue;
+      present.add(file);
       const fingerprint = `${stat.size}:${Math.round(stat.mtimeMs)}`;
       const parsed = guessit(path.basename(file));
       const previous = Object.entries(index.mappings).find(([oldPath, value]) => oldPath !== file && value.missing && value.fingerprint === fingerprint)?.[1];
@@ -148,7 +170,7 @@ export function createLibrary(mediaRoot, indexPath, tmdbApiKey) {
         index.mappings[file] = { ...mapping, fingerprint, missing: false, updatedAt: mapping.updatedAt || new Date().toISOString() };
         changed = true;
       }
-      return {
+      items.push({
         file,
         id: Buffer.from(file).toString('base64url'),
         size: stat.size,
@@ -159,8 +181,15 @@ export function createLibrary(mediaRoot, indexPath, tmdbApiKey) {
         season: mapping.season || parsed.season || null,
         episode: mapping.episode || parsed.episode || null,
         mapped: Boolean(mapping.imdbId),
-      };
-    });
+      });
+    }
+    for (const file of Object.keys(index.mappings)) {
+      if (present.has(file) || index.mappings[file].missing) continue;
+      const size = Number(String(index.mappings[file].fingerprint || '').split(':')[0]);
+      if (size > 0) continue;
+      index.mappings[file].missing = true;
+      changed = true;
+    }
     const discovered = await Promise.all(items.map(discover));
     for (const item of discovered) {
       if (!item.autoMapped || index.mappings[item.file]?.imdbId) continue;
@@ -177,8 +206,32 @@ export function createLibrary(mediaRoot, indexPath, tmdbApiKey) {
       changed = true;
     }
     if (changed) save(index);
+    cachedScan = discovered;
     return discovered;
   };
+  const itemsFromIndex = () => Object.entries(load().mappings).flatMap(([file, mapping]) => {
+    const size = Number(String(mapping.fingerprint || '').split(':')[0]);
+    if (!size || !video.test(file) || file.includes('..')) return [];
+    const parsed = guessit(path.basename(file));
+    return [{
+      file,
+      id: Buffer.from(file).toString('base64url'),
+      size,
+      title: mapping.title || parsed.title || path.basename(file, path.extname(file)),
+      year: mapping.year || parsed.year || null,
+      imdbId: mapping.imdbId || parsed.imdb_id || file.match(/\btt\d{7,10}\b/i)?.[0]?.toLowerCase() || null,
+      type: parsed.type === 'episode' ? 'episode' : 'video',
+      season: mapping.season || parsed.season || null,
+      episode: mapping.episode || parsed.episode || null,
+      mapped: Boolean(mapping.imdbId),
+    }];
+  });
+  const findItem = (items, id) => items.find(item => catalogAliases(item).has(id) || item.files?.some(file => file.id === id)) || null;
+  let indexGrouped = null;
+  const groupedFromIndex = () => {
+    if (!indexGrouped) indexGrouped = groupCatalog(itemsFromIndex());
+    return indexGrouped;
+  };
   return {
     scan,
     startReconciler(seconds = 900) {
@@ -201,10 +254,11 @@ export function createLibrary(mediaRoot, indexPath, tmdbApiKey) {
       return groupCatalog(await Promise.all((await scan()).map(metadataFor)));
     },
     async metadata(id) {
-      return (await this.catalog()).find(item => item.id === id || item.files?.some(file => file.id === id)) || null;
+      return findItem(await this.catalog(), id);
     },
     async members(id) {
-      const item = await this.metadata(id);
+      const grouped = cachedScan ? groupCatalog(cachedScan) : groupedFromIndex();
+      const item = findItem(grouped, id);
       return item ? catalogMembers(item, id) : null;
     },
   };

library.test.js

@@ -1,6 +1,6 @@
 import assert from 'node:assert/strict';
 import { test } from 'node:test';
-import { catalogGroupKey, catalogMembers, groupCatalog } from './library.js';
+import { catalogAliases, catalogGroupKey, catalogMembers, groupCatalog } from './library.js';
 
 const pathId = file => Buffer.from(file).toString('base64url');
 
@@ -55,3 +55,41 @@ test('unmapped titles group by lowercase title and year', () => {
   assert.equal(catalog[0].id, 'evolution-2026');
   assert.equal(catalogGroupKey({ title: 'Evolution', year: 2026, type: 'video' }), 'evolution-2026');
 });
+
+test('size-0 torrent folders are dropped when a real file exists', () => {
+  const catalog = groupCatalog([
+    { id: 'dir', file: 'Rons.Gone.Wrong.2021.mkv', title: 'Rons Gone Wrong', year: 2021, type: 'video', size: 0 },
+    { id: 'real', file: 'Rons.Gone.Wrong.2021.mkv/Rons.Gone.Wrong.2021.mkv', title: 'Rons Gone Wrong', year: 2021, type: 'video', size: 1506040768 },
+    { id: 'yts', file: 'Rons Gone Wrong (2021)/Rons.Gone.Wrong.2021.mp4', title: 'Rons Gone Wrong', year: 2021, type: 'video', size: 2112849010 },
+  ]);
+  assert.equal(catalog.length, 1);
+  assert.equal(catalog[0].id, 'rons-gone-wrong-2021');
+  assert.equal(catalog[0].size, 2112849010);
+  assert.deepEqual(catalog[0].files.map(file => file.id), ['yts', 'real']);
+  assert.equal(catalogMembers(catalog[0], catalog[0].id).length, 2);
+});
+
+test('duplicate file ids collapse to one stream', () => {
+  const catalog = groupCatalog([
+    { id: 'a', file: 'Movie.mp4', title: 'Movie', year: 2020, type: 'video', size: 10 },
+    { id: 'a', file: 'Movie.mp4', title: 'Movie', year: 2020, type: 'video', size: 10 },
+  ]);
+  assert.equal(catalog[0].files.length, 1);
+});
+
+test('a size-0-only group is omitted', () => {
+  const catalog = groupCatalog([
+    { id: 'dir', file: 'Rons.Gone.Wrong.2021.mkv', title: 'Rons Gone Wrong', year: 2021, type: 'video', size: 0 },
+  ]);
+  assert.equal(catalog.length, 0);
+});
+
+test('title-year id still resolves after imdb grouping', () => {
+  const catalog = groupCatalog([
+    { id: 'a', file: 'Rons.mkv', title: 'Rons Gone Wrong', year: 2021, imdbId: 'tt7504818', type: 'video', size: 10 },
+    { id: 'b', file: 'Rons.mp4', title: 'Rons Gone Wrong', year: 2021, imdbId: 'tt7504818', type: 'video', size: 20 },
+  ]);
+  assert.equal(catalog[0].id, 'tt7504818');
+  assert.equal(catalogMembers(catalog[0], 'rons-gone-wrong-2021').length, 2);
+  assert.ok(catalogAliases(catalog[0]).has('rons-gone-wrong-2021'));
+});

server.js

@@ -59,6 +59,20 @@ const admin = req => {
 const html = value => String(value).replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;').replaceAll('"', '&quot;');
 const libraryPage = items => `<!doctype html><title>CleanStream Library</title><style>body{font:16px system-ui;max-width:1100px;margin:2rem auto}form{display:grid;grid-template-columns:2fr 2fr 1fr auto;gap:.5rem;margin:.5rem 0}input{padding:.5rem}small{grid-column:1/-1;color:#666}</style><h1>Library review</h1><p>Confirm titles and IMDb IDs. Original files are never renamed or moved.</p>${items.map(item => `<form method="post" action="/library/mapping"><input type="hidden" name="file" value="${encodeURIComponent(item.file)}"><input name="title" value="${html(item.title)}"><input name="imdbId" placeholder="tt1234567" value="${html(item.imdbId || item.suggestedImdbId || '')}"><input name="year" value="${item.year || ''}" placeholder="year"><button>Save</button><small>${html(item.file)}${item.candidate ? ` · TMDB: ${html(item.candidate.title)} (${html(item.candidate.year)})` : ''}</small></form>`).join('')}`;
 
+const resolveMediaTarget = file => {
+  if (!file || file.includes('..')) return null;
+  const target = path.resolve(mediaRoot, file);
+  if (!target.startsWith(`${mediaRoot}${path.sep}`)) return null;
+  try {
+    const stat = fs.statSync(target);
+    if (stat.isFile()) return target;
+    if (stat.isDirectory()) {
+      const nested = path.join(target, path.basename(target));
+      if (nested.startsWith(`${mediaRoot}${path.sep}`) && fs.statSync(nested).isFile()) return nested;
+    }
+  } catch {}
+  return null;
+};
 http.createServer(async (req, res) => {
   const url = new URL(req.url, publicBaseUrl);
   console.log(`${req.method} ${url.pathname}`);
@@ -90,6 +104,30 @@ http.createServer(async (req, res) => {
     return item ? json(res, 200, { item: { ...item, imdbId: item.imdbId || item.suggestedImdbId, description: item.description || item.file } }) : json(res, 404, { error: 'Not found' });
   }
   if (req.method === 'POST' && url.pathname === '/v1/stream') { if (!authenticated(req)) return json(res,401,{error:'Unauthorized'}); const body=await readJson(req); if (body.id === 'demo-stream' && demoStreamUrl) return json(res,200,{streams:[{id:'demo-stream',url:demoStreamUrl,filename:'demo.m3u8',headers:{}}]}); const members=await library.members(body.id || ''); if (!members?.length || members.some(member => member.file.includes('..'))) return json(res,404,{error:'Not found'}); return json(res,200,{streams:members.map(member => ({id:member.id,url:`${publicBaseUrl}/v1/media/${member.id}?token=${bearer(req)}`,filename:path.basename(member.file),headers:{}}))}); }
-  if (req.method === 'GET' && url.pathname.startsWith('/v1/media/')) { const token=url.searchParams.get('token'); const file=Buffer.from(url.pathname.split('/').pop(), 'base64url').toString(); const target=path.resolve(mediaRoot,file); if (!tokens.has(token) || !target.startsWith(`${mediaRoot}${path.sep}`) || !fs.existsSync(target)) return json(res,404,{error:'Not found'}); res.writeHead(200,{'content-type':'application/octet-stream','content-length':fs.statSync(target).size}); return fs.createReadStream(target).pipe(res); }
+  if ((req.method === 'GET' || req.method === 'HEAD') && url.pathname.startsWith('/v1/media/')) {
+    const token = url.searchParams.get('token');
+    const file = Buffer.from(url.pathname.split('/').pop(), 'base64url').toString();
+    const target = resolveMediaTarget(file);
+    if (!tokens.has(token) || !target) return json(res, 404, { error: 'Not found' });
+    const size = fs.statSync(target).size;
+    const match = req.headers.range?.match(/^bytes=(\d*)-(\d*)$/);
+    const start = Number(match?.[1] || 0);
+    const end = Math.min(Number(match?.[2] || size - 1), size - 1);
+    if (match && (start > end || start >= size)) {
+      res.writeHead(416, { 'content-range': `bytes */${size}` });
+      return res.end();
+    }
+    const headers = { 'content-type': 'application/octet-stream', 'accept-ranges': 'bytes', 'content-length': end - start + 1 };
+    if (match) headers['content-range'] = `bytes ${start}-${end}/${size}`;
+    res.writeHead(match ? 206 : 200, headers);
+    if (req.method === 'HEAD') return res.end();
+    const stream = fs.createReadStream(target, { start, end });
+    stream.on('error', error => {
+      console.error(`Media read failed: ${error.message}`);
+      if (!res.headersSent) json(res, 502, { error: 'Media read failed' });
+      else res.destroy(error);
+    });
+    return stream.pipe(res);
+  }
   json(res, 404, { error: 'Not found' });
 }).listen(port, () => console.log(`Connector listening on ${publicBaseUrl}`));