FrederikBaerentsen/ComicOPDS
1
1
Fork 1
Code Issues 11 Pull Requests 1 Packages Projects Releases Wiki Activity

Added distinct by oldest/newest and updated smartlist layout

Browse Source
This commit is contained in:
FrederikBaerentsen
2025-09-09 16:54:47 +02:00
parent 8e52a089ef
commit 6c45331359
3 changed files with 172 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
app/db.py
View File

@@ -452,6 +452,12 @@ def _order_by_for_sort(sort: str) -> str:
return "COALESCE(m.title, i.name) ASC"
if s == "title_desc":
return "COALESCE(m.title, i.name) DESC"
if s == "publisher":
return "COALESCE(m.publisher, '') COLLATE NOCASE ASC, m.series COLLATE NOCASE ASC, i.name ASC"
if s == "title":
return "COALESCE(m.title, i.name) COLLATE NOCASE ASC"
if s == "series_number":
return "COALESCE(m.series, i.name) COLLATE NOCASE ASC, CAST(COALESCE(NULLIF(m.number,''),'0') AS INTEGER) ASC, i.name ASC"
if s == "added_asc":
return "i.mtime ASC"
if s == "added_desc":
@@ -459,30 +465,95 @@ def _order_by_for_sort(sort: str) -> str:
return "COALESCE(m.series, i.name) ASC, " \
"CAST(COALESCE(NULLIF(m.number,''),'0') AS INTEGER) ASC, i.name ASC"
def smartlist_query(conn: sqlite3.Connection, groups: List[Dict[str, Any]], sort: str,
limit: int, offset: int, distinct_by_series: bool):
# Build WHERE from groups (OR supply a full spec if you prefer)
# ---- FTS prefilter for smartlists (speeds up 'contains' text rules) ----
_TEXT_FIELDS_FOR_FTS = {
"title","series","publisher","writer","summary","genre",
"tags","characters","teams","locations","name","filename"
}
def _extract_fts_terms_from_groups(groups: List[Dict[str, Any]]) -> List[str]:
terms: List[str] = []
for g in (groups or []):
for r in (g.get("rules") or []):
field = (r.get("field") or "").lower()
op = (r.get("op") or "").lower()
val = r.get("value")
if field in _TEXT_FIELDS_FOR_FTS and op in ("contains","~") and isinstance(val, str) and not r.get("not"):
tokens = re.findall(r"[0-9A-Za-z]{2,}", val)
terms.extend(t + "*" for t in tokens)
return terms
# ---- Smartlist runners --------------------------------------------------------
def smartlist_query(
conn: sqlite3.Connection,
groups: List[Dict[str, Any]],
sort: str,
limit: int,
offset: int,
distinct_by_series: bool
):
"""
Backward-compatible API (used by existing routes).
- Adds FTS prefilter when possible.
- If distinct_by_series is 'latest' or 'oldest' (string), uses that mode.
If True, defaults to 'latest'.
"""
where, params = build_smartlist_where(groups)
order_clause = _order_by_for_sort(sort)
if not distinct_by_series:
# Optional FTS prefilter
fts_sql = ""
fts_params: List[Any] = []
if HAS_FTS5:
tokens = _extract_fts_terms_from_groups(groups)
if tokens:
fts_sql = " AND i.rel IN (SELECT rel FROM fts WHERE fts MATCH ?)"
fts_params = [" AND ".join(tokens)]
# Distinct mode handling
mode = "latest"
if isinstance(distinct_by_series, str) and distinct_by_series in ("latest", "oldest"):
use_distinct = True
mode = distinct_by_series
else:
use_distinct = bool(distinct_by_series)
if not use_distinct:
sql = f"""
SELECT i.*, m.*
FROM items i
LEFT JOIN meta m ON m.rel = i.rel
WHERE i.is_dir=0 AND {where}
ORDER BY {order_clause}
LIMIT ? OFFSET ?
FROM items i
LEFT JOIN meta m ON m.rel = i.rel
WHERE i.is_dir=0 AND {where}{fts_sql}
ORDER BY {order_clause}
LIMIT ? OFFSET ?
"""
return conn.execute(sql, (*params, limit, offset)).fetchall()
return conn.execute(sql, (*params, *fts_params, limit, offset)).fetchall()
# DISTINCT by (series, volume), with latest/oldest mode
cmp_year = "CAST(COALESCE(NULLIF(m2.year,''),'0') AS INTEGER) {op} CAST(COALESCE(NULLIF(m.year,''),'0') AS INTEGER)"
cmp_number = "CAST(COALESCE(NULLIF(m2.number,''),'0') AS INTEGER) {op} CAST(COALESCE(NULLIF(m.number,''),'0') AS INTEGER)"
cmp_mtime = "i2.mtime {op} i.mtime"
if mode == "oldest":
op_main, op_eq, op_time = "<", "=", "<"
else:
op_main, op_eq, op_time = ">", "=", ">"
dominance = f"""
(
{cmp_year.format(op=op_main)} OR
({cmp_year.format(op=op_eq)} AND {cmp_number.format(op=op_main)}) OR
({cmp_year.format(op=op_eq)} AND {cmp_number.format(op=op_eq)} AND {cmp_mtime.format(op=op_time)})
)
"""
# DISTINCT by series (pick "newest" per series)
sql = f"""
SELECT i.*, m.*
FROM items i
LEFT JOIN meta m ON m.rel = i.rel
WHERE i.is_dir=0
AND {where}
WHERE i.is_dir=0 AND {where}{fts_sql}
AND (
m.series IS NULL OR m.series='' OR
NOT EXISTS (
@@ -490,34 +561,33 @@ def smartlist_query(conn: sqlite3.Connection, groups: List[Dict[str, Any]], sort
FROM items i2
LEFT JOIN meta m2 ON m2.rel = i2.rel
WHERE i2.is_dir=0
AND m2.series = m.series AND m2.volume = m.volume
AND (
CAST(COALESCE(NULLIF(m2.year,''),'0') AS INTEGER) > CAST(COALESCE(NULLIF(m.year,''),'0') AS INTEGER) OR
(
CAST(COALESCE(NULLIF(m2.year,''),'0') AS INTEGER) = CAST(COALESCE(NULLIF(m.year,''),'0') AS INTEGER)
AND CAST(COALESCE(NULLIF(m2.number,''),'0') AS INTEGER) > CAST(COALESCE(NULLIF(m.number,''),'0') AS INTEGER)
) OR
(
CAST(COALESCE(NULLIF(m2.year,''),'0') AS INTEGER) = CAST(COALESCE(NULLIF(m.year,''),'0') AS INTEGER)
AND CAST(COALESCE(NULLIF(m2.number,''),'0') AS INTEGER) = CAST(COALESCE(NULLIF(m.number,''),'0') AS INTEGER)
AND i2.mtime > i.mtime
)
)
AND m2.series = m.series
AND COALESCE(m2.volume,'') = COALESCE(m.volume,'')
AND {dominance}
)
)
ORDER BY {order_clause}
LIMIT ? OFFSET ?
"""
return conn.execute(sql, (*params, limit, offset)).fetchall()
return conn.execute(sql, (*params, *fts_params, limit, offset)).fetchall()
def smartlist_count(conn: sqlite3.Connection, groups: List[Dict[str, Any]]) -> int:
where, params = build_smartlist_where(groups)
fts_sql = ""
fts_params: List[Any] = []
if HAS_FTS5:
tokens = _extract_fts_terms_from_groups(groups)
if tokens:
fts_sql = " AND i.rel IN (SELECT rel FROM fts WHERE fts MATCH ?)"
fts_params = [" AND ".join(tokens)]
row = conn.execute(f"""
SELECT COUNT(*)
FROM items i
LEFT JOIN meta m ON m.rel = i.rel
WHERE i.is_dir=0 AND {where}
""", params).fetchone()
FROM items i
LEFT JOIN meta m ON m.rel = i.rel
WHERE i.is_dir=0 AND {where}{fts_sql}
""", (*params, *fts_params)).fetchone()
return int(row[0]) if row else 0
# ----------------------------- Stats ------------------------------------------
@@ -546,7 +616,7 @@ def stats(conn: sqlite3.Connection) -> Dict[str, Any]:
"SELECT MAX(mtime) FROM items"
).fetchone()[0]
# Publishers breakdown (top N) — used by doughnut chart
# Publishers breakdown (top N)
top_pubs = [
{"publisher": row[0], "count": row[1]}
for row in conn.execute("""

29
app/main.py
View File

@@ -804,20 +804,39 @@ def opds_smart_list(slug: str, page: int = 1, _=Depends(require_basic)):
groups = sl.get("groups") or []
sort = (sl.get("sort") or "issued_desc").lower()
distinct_by = (sl.get("distinct_by") or "") == "series"
start = (page - 1) * PAGE_SIZE
# Distinct handling (series+volume) + mode
distinct_by = (sl.get("distinct_by") or "").strip().lower()
distinct_mode = (sl.get("distinct_mode") or "latest").strip().lower()
distinct_flag = distinct_mode if distinct_by == "series_volume" else False # db.smartlist_query expects False | "latest" | "oldest"
# Hard cap per list
sl_limit = int(sl.get("limit") or 0)
# paging
page = max(1, int(page))
page_size = PAGE_SIZE
start = (page - 1) * page_size
# effective page size when a hard cap exists
effective_page_size = page_size if sl_limit == 0 else max(0, min(page_size, sl_limit - start))
conn = db.connect()
try:
rows = db.smartlist_query(conn, groups, sort, PAGE_SIZE, start, distinct_by)
rows = db.smartlist_query(conn, groups, sort, effective_page_size, start, distinct_flag)
total = db.smartlist_count(conn, groups)
finally:
conn.close()
# Total for navigation honors the hard cap
total_for_nav = min(total, sl_limit) if sl_limit > 0 else total
entries_xml = [_entry_xml_from_row(r) for r in rows]
self_href = f"/opds/smart/{quote(slug)}?page={page}"
next_href = f"/opds/smart/{quote(slug)}?page={page+1}" if (start + PAGE_SIZE) < total else None
next_href = None
if (start + len(rows)) < total_for_nav:
next_href = f"/opds/smart/{quote(slug)}?page={page+1}"
xml = _feed(entries_xml, title=sl["name"], self_href=self_href, next_href=next_href)
return Response(content=xml, media_type="application/atom+xml;profile=opds-catalog")
@@ -940,4 +959,4 @@ def thumbs_errors_log(_=Depends(require_basic)):
media_type="text/plain",
filename="thumbs_errors.log",
headers={"Cache-Control": "no-store"}
)
)

71
app/templates/smartlists.html
View File

@@ -39,7 +39,7 @@
<hr class="my-3">
<div class="row g-2">
<div class="col-12 col-md-5">
<div class="col-12 col-md-4">
<label class="form-label">Sort</label>
<select id="sort" class="form-select">
<option value="issued_desc">Issued (newest first)</option>
@@ -48,15 +48,29 @@
<option value="publisher">Publisher</option>
</select>
</div>
<div class="col-6 col-md-3">
<div class="col-6 col-md-2">
<label class="form-label">Limit</label>
<input id="limit" class="form-control" type="number" min="0" value="0" />
<div class="form-text">0 = unlimited</div>
</div>
<div class="col-6 col-md-4 d-flex align-items-end">
<div class="form-check mt-3">
<input class="form-check-input" type="checkbox" id="distinctSeries">
<label class="form-check-label" for="distinctSeries">Distinct by series and volume (latest)</label>
<div class="col-6 col-md-3">
<label class="form-label">Distinct</label>
<select id="distinctBy" class="form-select">
<option value="">None</option>
<option value="series_volume">Series + Volume</option>
</select>
</div>
<div class="col-12 col-md-3" id="distinctModeWrap" style="display:none;">
<label class="form-label">Pick</label>
<div class="d-flex gap-2">
<div class="form-check">
<input class="form-check-input" type="radio" name="distinctMode" id="dmLatest" value="latest" checked>
<label class="form-check-label" for="dmLatest">Latest</label>
</div>
<div class="form-check">
<input class="form-check-input" type="radio" name="distinctMode" id="dmOldest" value="oldest">
<label class="form-check-label" for="dmOldest">Oldest</label>
</div>
</div>
</div>
</div>
@@ -68,8 +82,7 @@
<div class="small text-secondary mt-3">
Rules within a group are <b>AND</b>’d. Groups are <b>OR</b>’d.<br>
Date formats: <span class="monosmall">YYYY</span>, <span class="monosmall">YYYY-MM</span>, <span class="monosmall">YYYY-MM-DD</span>.<br>
Fields: <span class="monosmall">series, title, number, volume, publisher, imprint, writer, characters, teams, tags, year, month, day, issued, languageiso, comicvineissue, rel, ext, size, mtime, has_thumb, has_meta</span>
Fields: <span class="monosmall">series, title, number, volume, publisher, writer, characters, teams, tags, year, month, day, languageiso, comicvineissue, rel, ext, size, mtime</span>
</div>
</div>
</div>
@@ -105,27 +118,22 @@
<span class="text-secondary small">NOT</span>
<select class="form-select form-select-sm field">
<option>series</option><option>title</option><option>number</option><option>volume</option>
<option>publisher</option><option>imprint</option><option>writer</option>
<option>publisher</option><option>writer</option>
<option>characters</option><option>teams</option><option>tags</option>
<option>year</option><option>month</option><option>day</option><option>issued</option>
<option>year</option><option>month</option><option>day</option>
<option>languageiso</option><option>comicvineissue</option>
<option>rel</option><option>ext</option><option>size</option><option>mtime</option>
<option>has_thumb</option><option>has_meta</option>
</select>
<select class="form-select form-select-sm op">
<option value="contains">contains</option>
<option value="equals">equals</option>
<option value="startswith">starts with</option>
<option value="endswith">ends with</option>
<option value="regex">regex</option>
<option value="=">= (numeric)</option><option value="!=">!= (numeric)</option>
<option value=">">&gt; (numeric)</option><option value=">=">&gt;= (numeric)</option>
<option value="<">&lt; (numeric)</option><option value="<=">&lt;= (numeric)</option>
<option value="on">on (date)</option><option value="before">before (date)</option><option value="after">after (date)</option>
<option value="between">between (date,date)</option>
<option value="exists">exists</option><option value="missing">missing</option>
</select>
<input class="form-control form-control-sm value" placeholder="value (leave blank for exists/missing)" />
<input class="form-control form-control-sm value" placeholder="value"/>
<button class="btn btn-sm btn-outline-danger remove-rule"><i class="bi bi-x-lg"></i></button>
</div>
</template>
@@ -136,6 +144,13 @@
const groupTpl = document.getElementById('groupTpl');
const ruleTpl = document.getElementById('ruleTpl');
const distinctBySel = document.getElementById('distinctBy');
const distinctModeWrap = document.getElementById('distinctModeWrap');
distinctBySel.addEventListener('change', () => {
distinctModeWrap.style.display = (distinctBySel.value === 'series_volume') ? '' : 'none';
});
function addGroup(data) {
const node = groupTpl.content.cloneNode(true);
const card = node.querySelector('.group-card');
@@ -155,7 +170,7 @@
row.querySelector('.not-flag').checked = !!data.not;
row.querySelector('.field').value = data.field || 'series';
row.querySelector('.op').value = data.op || 'contains';
row.querySelector('.value').value = data.value || '';
row.querySelector('.value').value = (data.value ?? '');
}
row.querySelector('.remove-rule').onclick = () => row.remove();
container.appendChild(node);
@@ -169,7 +184,7 @@
op: r.querySelector('.op').value,
value: r.querySelector('.value').value
}));
return { rules: rules.filter(x => x.op === 'exists' || x.op === 'missing' || (x.value && x.value.trim())) };
return { rules: rules.filter(x => (x.value && x.value.trim()) || ["exists","missing"].includes(x.op)) };
}).filter(g => g.rules.length);
}
@@ -177,7 +192,9 @@
document.getElementById('listName').value = '';
document.getElementById('sort').value = 'issued_desc';
document.getElementById('limit').value = '0';
document.getElementById('distinctSeries').checked = false;
distinctBySel.value = '';
distinctModeWrap.style.display = 'none';
document.getElementById('dmLatest').checked = true;
groupsEl.innerHTML = '';
addGroup({rules:[{field:'series',op:'contains',value:''}]});
}
@@ -189,15 +206,16 @@
data.forEach(l => {
const col = document.createElement('div');
col.className = 'col-12';
const distinctTxt = l.distinct_by ? `${l.distinct_by} (${l.distinct_mode || 'latest'})` : '—';
col.innerHTML = `
<div class="card h-100">
<div class="card-body d-flex justify-content-between align-items-start">
<div>
<div class="fw-semibold">${l.name}</div>
<div class="small text-secondary">
${(l.groups||[]).map((g,i)=>'Group '+(i+1)+': '+g.rules.map(r => (r.not?'NOT ':'')+r.field+' '+r.op+' "'+r.value.replace(/"/g,'&quot;')+'"').join(' AND ')).join(' <b>OR</b> ') || '—'}
${(l.groups||[]).map((g,i)=>'Group '+(i+1)+': '+g.rules.map(r => (r.not?'NOT ':'')+r.field+' '+r.op+' "'+String(r.value||'').replace(/"/g,'&quot;')+'"').join(' AND ')).join(' <b>OR</b> ') || '—'}
</div>
<div class="small text-secondary mt-1">Sort: ${l.sort || 'issued_desc'} • Limit: ${l.limit || 0} • Distinct: ${l.distinct_by || '—'}</div>
<div class="small text-secondary mt-1">Sort: ${l.sort || 'issued_desc'} • Limit: ${l.limit || 0} • Distinct: ${distinctTxt}</div>
<div class="mt-2"><a class="btn btn-sm btn-outline-primary" href="/opds/smart/${l.slug}">Open in OPDS</a></div>
</div>
<div class="btn-group btn-group-sm">
@@ -210,7 +228,9 @@
document.getElementById('listName').value = l.name;
document.getElementById('sort').value = l.sort || 'issued_desc';
document.getElementById('limit').value = l.limit || 0;
document.getElementById('distinctSeries').checked = (l.distinct_by === 'series');
distinctBySel.value = l.distinct_by || '';
distinctModeWrap.style.display = (distinctBySel.value === 'series_volume') ? '' : 'none';
(l.distinct_mode === 'oldest' ? document.getElementById('dmOldest') : document.getElementById('dmLatest')).checked = true;
groupsEl.innerHTML = '';
(l.groups || []).forEach(g => addGroup(g));
};
@@ -229,7 +249,7 @@
method: 'POST',
headers: { 'Content-Type': 'application/json' },
credentials: 'include',
body: JSON.stringify(lists) // <-- must be an array
body: JSON.stringify(lists)
});
if (!res.ok) {
alert("Failed to save smartlists: " + res.status);
@@ -245,12 +265,13 @@
if (!name || groups.length === 0) { alert('Please provide a name and at least one rule.'); return; }
const sort = document.getElementById('sort').value;
const limit = parseInt(document.getElementById('limit').value || '0', 10);
const distinct_by = document.getElementById('distinctSeries').checked ? 'series' : '';
const distinct_by = distinctBySel.value; // '' or 'series_volume'
const distinct_mode = document.querySelector('input[name="distinctMode"]:checked').value; // latest|oldest
const lists = await loadLists();
const slug = (name.toLowerCase().replace(/[^a-z0-9]+/g,'-').replace(/^-+|-+$/g,'') || 'list');
const existing = lists.find(l => l.slug === slug) || lists.find(l => l.name === name);
const record = { name, slug, groups, sort, limit, distinct_by };
const record = { name, slug, groups, sort, limit, distinct_by, distinct_mode };
if (existing) Object.assign(existing, record); else lists.push(record);
await saveLists(lists);
resetForm();