feat(statistics): paid/retail/market comparison and instructions stats

This commit is contained in:
2026-06-21 16:22:38 +02:00
parent a3122c52d2
commit a43ec39d8c
10 changed files with 410 additions and 0 deletions
+68
View File
@@ -36,6 +36,7 @@ MESSAGES: Final[dict[str, str]] = {
'PART_LOADED': 'part_loaded',
'PROGRESS': 'progress',
'SET_LOADED': 'set_loaded',
'VALUE_ALL_SETS': 'value_all_sets',
}
@@ -298,6 +299,73 @@ class BrickSocket(object):
from .individual_part import IndividualPart
IndividualPart().create_bulk(self, data)
@self.socket.on(MESSAGES['VALUE_ALL_SETS'], namespace=self.namespace)
@authenticated_socket(self)
def value_all_sets(data: dict[str, Any], /) -> None:
logger.debug('Socket: VALUE_ALL_SETS (from: {fr})'.format(
fr=request.sid, # type: ignore
))
from .sidecar import BrickSidecar
from .sidecar_cache import BrickSidecarCache
from .sql import BrickSQL
if not BrickSidecar.enabled():
self.fail(message='The sidecar is not configured')
return
# Distinct set numbers in the collection (reuses the instructions
# statistics query).
try:
rows = BrickSQL().fetchall('statistics/set_numbers')
except Exception:
rows = []
refs = [row['set'] for row in rows]
self.progress_count = 0
self.update_total(len(refs))
priced = 0 # sets that ended up with a value (cached or fetched)
fetched = 0 # sets that required a live fetch this run
missing = 0 # sets with no value available at all
for ref in refs:
self.auto_progress(
message='Valuing set {ref}'.format(ref=ref),
)
# Was this set's price already fresh in the cache? If so,
# get_price() returns it without a network call.
cached, fetched_at = BrickSidecar.cached_price(ref)
was_fresh = (
cached is not None
and BrickSidecarCache.price_is_fresh(fetched_at)
)
try:
price = BrickSidecar.get_price(ref)
except Exception:
price = None
if price is not None:
priced += 1
if not was_fresh:
fetched += 1
else:
missing += 1
# Only pause when we actually hit the network, so re-runs over an
# already-cached collection stay fast.
if not was_fresh:
self.socket.sleep(0.2)
self.complete(
message=(
'{priced} set(s) now have a value ({fetched} newly fetched); '
'{missing} had none available.'
).format(priced=priced, fetched=fetched, missing=missing),
)
# Update the progress auto-incrementing
def auto_progress(
self,
@@ -0,0 +1,5 @@
-- Distinct set numbers in the collection. Used to intersect with the
-- filesystem instructions list (which is not in the database) for the
-- instructions statistics (#154).
SELECT DISTINCT "bricktracker_sets"."set" AS "set"
FROM "bricktracker_sets"
@@ -0,0 +1,32 @@
-- Aggregate paid / retail (MSRP) / BrickLink market value across the collection.
-- MSRP and market value come from the cached sidecar payloads (JSON). Region is
-- whitelisted (US/UK/CA/DE) by the caller before being interpolated here.
SELECT
COUNT(*) AS "total_sets",
SUM(CASE WHEN "s"."purchase_price" IS NOT NULL THEN "s"."purchase_price" ELSE 0 END) AS "total_paid",
SUM(CASE WHEN "s"."purchase_price" IS NOT NULL THEN 1 ELSE 0 END) AS "sets_with_paid",
SUM("c"."msrp") AS "total_msrp",
SUM(CASE WHEN "c"."msrp" IS NOT NULL THEN 1 ELSE 0 END) AS "sets_with_msrp",
SUM("c"."market_new") AS "total_market_new",
SUM("c"."market_used") AS "total_market_used",
SUM(CASE WHEN "c"."market_new" IS NOT NULL THEN 1 ELSE 0 END) AS "sets_with_market",
SUM(CASE WHEN "c"."market_used" IS NOT NULL THEN 1 ELSE 0 END) AS "sets_with_market_used",
SUM(CASE WHEN "c"."msrp" IS NOT NULL AND "s"."purchase_price" IS NOT NULL THEN "s"."purchase_price" ELSE 0 END) AS "paid_where_msrp",
SUM(CASE WHEN "c"."msrp" IS NOT NULL AND "s"."purchase_price" IS NOT NULL THEN "c"."msrp" ELSE 0 END) AS "msrp_where_paid",
SUM(CASE WHEN "c"."market_new" IS NOT NULL AND "s"."purchase_price" IS NOT NULL THEN "s"."purchase_price" ELSE 0 END) AS "paid_where_market",
SUM(CASE WHEN "c"."market_new" IS NOT NULL AND "s"."purchase_price" IS NOT NULL THEN "c"."market_new" ELSE 0 END) AS "market_where_paid",
SUM(CASE WHEN "c"."market_used" IS NOT NULL AND "s"."purchase_price" IS NOT NULL THEN "s"."purchase_price" ELSE 0 END) AS "paid_where_market_used",
SUM(CASE WHEN "c"."market_used" IS NOT NULL AND "s"."purchase_price" IS NOT NULL THEN "c"."market_used" ELSE 0 END) AS "market_used_where_paid",
-- Currency the BrickLink market values are stored in (consistent across
-- sets as they are all requested in the same currency); MAX picks a non-null.
MAX("c"."market_currency") AS "market_currency"
FROM "bricktracker_sets" AS "s"
LEFT JOIN (
SELECT
"set_ref",
json_extract("payload", '$.legoCom{{ region }}.retailPrice') AS "msrp",
json_extract("price_payload", '$.new_avg') AS "market_new",
json_extract("price_payload", '$.used_avg') AS "market_used",
json_extract("price_payload", '$.currency_code') AS "market_currency"
FROM "sidecar_set_cache"
) AS "c" ON "c"."set_ref" = "s"."set"
+97
View File
@@ -87,6 +87,103 @@ class BrickStatistics:
'purchase_locations_used': overview.get('purchase_locations_used') or 0
}
def get_instructions_summary(self) -> dict[str, Any] | None:
"""Instruction coverage across the collection (#154).
Instructions live on the filesystem (INSTRUCTIONS_FOLDER), not in the
database, so this intersects the distinct collection set numbers with the
cached instructions file list. Returns None when instructions are hidden.
"""
from flask import current_app
if current_app.config.get('HIDE_SET_INSTRUCTIONS', False):
return None
from .instructions_list import BrickInstructionsList
instructions = BrickInstructionsList()
rows = self.sql.fetchall('statistics/set_numbers')
set_numbers = {row['set'] for row in rows}
with_instructions = sum(
1 for number in set_numbers if number in instructions.sets
)
unique_sets = len(set_numbers)
return {
'instruction_files': instructions.sets_total,
'sets_with_instructions': with_instructions,
'unique_sets': unique_sets,
'percentage_with_instructions': min(round(
(with_instructions / max(unique_sets, 1)) * 100, 1
), 100.0),
}
def get_sidecar_pricing_summary(self) -> dict[str, Any] | None:
"""Collection-wide paid / retail (MSRP) / BrickLink market comparison.
Reads only from the local sidecar cache (no network). Returns None when
the sidecar is disabled or the cache is unavailable.
"""
from .sidecar import BrickSidecar
if not BrickSidecar.enabled():
return None
try:
row = self.sql.fetchone(
'statistics/sidecar_pricing',
region=BrickSidecar.retail_region(),
)
except Exception as exception:
logger.debug('sidecar pricing summary failed: %s', exception)
return None
if row is None:
return None
data = dict(row)
def number(key: str) -> float:
value = data.get(key)
try:
return float(value) if value is not None else 0.0
except (TypeError, ValueError):
return 0.0
# Savings vs retail and value change vs paid, computed only across the
# sets where both sides of the comparison are known.
data['total_saved_vs_msrp'] = round(
number('msrp_where_paid') - number('paid_where_msrp'), 2
)
data['total_gain_vs_paid'] = round(
number('market_where_paid') - number('paid_where_market'), 2
)
data['total_gain_vs_paid_used'] = round(
number('market_used_where_paid') - number('paid_where_market_used'), 2
)
data['retail_currency'] = BrickSidecar.retail_currency()
# Currency the user records purchase prices in (may be a symbol such as
# '$' or 'kr'). Compared against the retail/market ISO codes through the
# symbol map so '$' vs 'USD' and 'kr' vs 'DKK' are NOT flagged.
from flask import current_app
paid_currency = str(
current_app.config.get('PURCHASE_CURRENCY', '') or ''
).strip()
data['paid_currency'] = paid_currency
mismatch = False
if data.get('sets_with_paid'):
if not BrickSidecar.same_currency(paid_currency, data.get('market_currency')):
mismatch = True
if not BrickSidecar.same_currency(paid_currency, data['retail_currency']):
mismatch = True
data['currency_mismatch'] = mismatch
return data
def get_sets_by_year_statistics(self) -> list[dict[str, Any]]:
"""Get statistics grouped by LEGO set release year"""
results = self.sql.fetchall('statistics/sets_by_year')
+6
View File
@@ -23,6 +23,7 @@ from ...set_tag import BrickSetTag
from ...set_tag_list import BrickSetTagList
from ...sql_counter import BrickCounter
from ...sql import BrickSQL
from ...socket import MESSAGES
from ...theme_list import BrickThemeList
logger = logging.getLogger(__name__)
@@ -211,6 +212,7 @@ def admin() -> str:
open_logout = should_expand('authentication', open_logout)
open_retired = should_expand('retired', open_retired)
open_theme = should_expand('theme', open_theme)
open_value = should_expand('value', request.args.get('open_value', None))
# Metadata sub-sections
open_owner = should_expand('owner', open_owner)
@@ -286,6 +288,10 @@ def admin() -> str:
open_storage=open_storage,
open_tag=open_tag,
open_theme=open_theme,
open_value=open_value,
messages=MESSAGES,
path=current_app.config['SOCKET_PATH'],
namespace=current_app.config['SOCKET_NAMESPACE'],
owner_error=request.args.get('owner_error'),
purchase_location_error=request.args.get('purchase_location_error'),
retired=BrickRetiredList(),
+4
View File
@@ -32,6 +32,8 @@ def overview() -> str:
purchase_location_stats = stats.get_purchase_location_statistics()
financial_summary = stats.get_financial_summary()
collection_summary = stats.get_collection_summary()
instructions_summary = stats.get_instructions_summary()
sidecar_pricing = stats.get_sidecar_pricing_summary()
sets_by_year_stats = stats.get_sets_by_year_statistics()
purchases_by_year_stats = stats.get_purchases_by_year_statistics()
year_summary = stats.get_year_summary()
@@ -57,6 +59,8 @@ def overview() -> str:
purchase_location_statistics=purchase_location_stats,
financial_summary=financial_summary,
collection_summary=collection_summary,
instructions_summary=instructions_summary,
sidecar_pricing=sidecar_pricing,
sets_by_year_statistics=sets_by_year_stats,
purchases_by_year_statistics=purchases_by_year_stats,
year_summary=year_summary,
+35
View File
@@ -0,0 +1,35 @@
// Socket client for the bulk "value all sets" admin action. Connects, emits a
// single VALUE_ALL_SETS message and lets the generic BrickSocket base render
// the progress bar / completion message while the server prices each set.
class BrickValueSocket extends BrickSocket {
constructor(id, path, namespace, messages) {
// Single-shot operation (not bulk): the base complete() then stops the
// spinner, shows the success message and re-enables the button.
super(id, path, namespace, messages, false);
this.html_button = document.getElementById(`${id}-button`);
if (this.html_button) {
this.html_button.addEventListener("click", ((bricksocket) => () => {
bricksocket.start();
})(this));
}
}
// Kick off the server-side valuation
start() {
if (this.disabled) {
return;
}
// Connect lazily on first use so the host page (e.g. the admin page)
// does not open a socket until the action is actually triggered.
this.setup();
this.clear();
this.spinner(true);
this.toggle(false);
this.socket.emit(this.messages.VALUE_ALL_SETS, {});
}
}
+1
View File
@@ -45,6 +45,7 @@
{% endif %}
{% include 'admin/theme.html' %}
{% include 'admin/retired.html' %}
{% include 'admin/value.html' %}
{{ accordion.header('Set metadata', 'metadata', 'admin', expanded=open_metadata, icon='profile-line', class='p-0') }}
{% include 'admin/owner.html' %}
{% include 'admin/purchase_location.html' %}
+73
View File
@@ -0,0 +1,73 @@
{% import 'macro/accordion.html' as accordion %}
{% if sidecar_enabled %}
{{ accordion.header('Value', 'value', 'admin', expanded=open_value, icon='funds-line') }}
<p>
Fetch and cache the BrickLink market value for every set in your collection through the
sidecar. Sets whose value is still fresh (within the cache TTL) are skipped, so this is
safe to run again. It is the slow live path, so it runs one set at a time, politely.
</p>
<div id="value-fail" class="alert alert-danger d-none" role="alert"></div>
<div id="value-complete"></div>
<div class="mb-3">
<p>
Progress <span id="value-count"></span>
<span id="value-spinner" class="d-none">
<span class="spinner-border spinner-border-sm" aria-hidden="true"></span>
<span class="visually-hidden" role="status">Loading...</span>
</span>
</p>
<div id="value-progress" class="progress" role="progressbar">
<div id="value-progress-bar" class="progress-bar" style="width: 0%"></div>
</div>
<p id="value-progress-message" class="text-center d-none"></p>
</div>
<div class="d-flex align-items-center justify-content-end">
<button type="button" class="btn btn-primary" data-bs-toggle="modal" data-bs-target="#valueConfirmModal">
<i class="ri-funds-line"></i> Update all sets
</button>
</div>
<!-- Confirmation modal -->
<div class="modal fade" id="valueConfirmModal" tabindex="-1" aria-labelledby="valueConfirmModalLabel" aria-hidden="true">
<div class="modal-dialog modal-dialog-centered">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="valueConfirmModalLabel">Update all set values?</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
This will fetch the BrickLink market value for every set that is not already cached.
Depending on how many sets need pricing this can take a while and makes live calls
through the sidecar. Sets with a fresh cached value are skipped.
</div>
<div class="modal-footer">
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
<button type="button" id="value-button" class="btn btn-primary" data-bs-dismiss="modal">
<i class="ri-funds-line"></i> Update all sets
</button>
</div>
</div>
</div>
</div>
<script type="text/javascript">
document.addEventListener("DOMContentLoaded", () => {
new BrickValueSocket(
'value',
'{{ path }}',
'{{ namespace }}',
{
COMPLETE: '{{ messages['COMPLETE'] }}',
FAIL: '{{ messages['FAIL'] }}',
PROGRESS: '{{ messages['PROGRESS'] }}',
VALUE_ALL_SETS: '{{ messages['VALUE_ALL_SETS'] }}',
}
);
});
</script>
{{ accordion.footer() }}
{% endif %}
+89
View File
@@ -155,12 +155,101 @@
<small class="text-dark">Purchase Locations</small>
</div>
</div>
{% if instructions_summary %}
<div class="col-6">
<div class="text-center">
<div class="h4 text-primary mb-0">{{ instructions_summary.sets_with_instructions }}<span class="text-muted fs-6">/{{ instructions_summary.unique_sets }}</span></div>
<small class="text-dark">Sets with Instructions <span class="text-muted">({{ instructions_summary.percentage_with_instructions }}%)</span></small>
</div>
</div>
<div class="col-6">
<div class="text-center">
<div class="h4 text-dark mb-0">{{ instructions_summary.instruction_files }}</div>
<small class="text-dark">Instruction Files</small>
</div>
</div>
{% endif %}
</div>
</div>
</div>
</div>
</div>
<!-- Sidecar pricing comparison: paid / retail / worth now -->
{% if sidecar_pricing %}
{% set sp = sidecar_pricing %}
<div class="row g-3 mb-4">
<div class="col-12">
<div class="card">
<div class="card-header bg-info text-dark d-flex justify-content-between align-items-center">
<h5 class="card-title mb-0"><i class="ri-scales-line"></i> Pricing comparison (sidecar)</h5>
{% if g.login.is_authenticated() %}
<a href="{{ url_for('admin.admin', open_value='true') }}#value" class="btn btn-sm btn-dark">
<i class="ri-funds-line"></i> Value all sets
</a>
{% endif %}
</div>
<div class="card-body">
<div class="row g-2 text-center">
<div class="col-md-6 col-xl-3">
<div class="border rounded p-3">
<div class="h3 mb-0">{{ config['PURCHASE_CURRENCY'] }}{{ "%.2f"|format(sp.total_paid or 0) }}</div>
<small class="text-dark">Total paid <span class="text-muted">({{ sp.sets_with_paid or 0 }} sets)</span></small>
</div>
</div>
<div class="col-md-6 col-xl-3">
<div class="border rounded p-3">
<div class="h3 mb-0">{{ "%.2f"|format(sp.total_msrp or 0) }} {{ sp.retail_currency }}</div>
<small class="text-dark">Total retail / MSRP <span class="text-muted">({{ sp.sets_with_msrp or 0 }} sets)</span></small>
</div>
</div>
<div class="col-md-6 col-xl-3">
<div class="border rounded p-3">
<div class="h3 mb-0">{{ "%.2f"|format(sp.total_market_new or 0) }} {{ sp.market_currency or '' }}</div>
<small class="text-dark">Total worth now (BrickLink new) <span class="text-muted">({{ sp.sets_with_market or 0 }} sets)</span></small>
</div>
</div>
<div class="col-md-6 col-xl-3">
<div class="border rounded p-3">
<div class="h3 mb-0">{{ "%.2f"|format(sp.total_market_used or 0) }} {{ sp.market_currency or '' }}</div>
<small class="text-dark">Total worth now (BrickLink used) <span class="text-muted">({{ sp.sets_with_market_used or 0 }} sets)</span></small>
</div>
</div>
</div>
<div class="row g-2 text-center mt-1">
<div class="col-md-4">
<div class="border rounded p-2">
<span class="h5 {% if (sp.total_saved_vs_msrp or 0) >= 0 %}text-success{% else %}text-danger{% endif %} mb-0">{{ "%+.2f"|format(sp.total_saved_vs_msrp or 0) }} {{ sp.retail_currency }}</span>
<small class="d-block text-dark">Saved vs retail <span class="text-muted">(where both known)</span></small>
</div>
</div>
<div class="col-md-4">
<div class="border rounded p-2">
<span class="h5 {% if (sp.total_gain_vs_paid or 0) >= 0 %}text-success{% else %}text-danger{% endif %} mb-0">{{ "%+.2f"|format(sp.total_gain_vs_paid or 0) }} {{ sp.market_currency or '' }}</span>
<small class="d-block text-dark">Value change vs paid <span class="text-muted">(BrickLink new)</span></small>
</div>
</div>
<div class="col-md-4">
<div class="border rounded p-2">
<span class="h5 {% if (sp.total_gain_vs_paid_used or 0) >= 0 %}text-success{% else %}text-danger{% endif %} mb-0">{{ "%+.2f"|format(sp.total_gain_vs_paid_used or 0) }} {{ sp.market_currency or '' }}</span>
<small class="d-block text-dark">Value change vs paid <span class="text-muted">(BrickLink used)</span></small>
</div>
</div>
</div>
{% if sp.currency_mismatch %}
<div class="alert alert-warning small mb-2 mt-2" role="alert">
<i class="ri-error-warning-line"></i>
Currencies differ: you paid in <strong>{{ sp.paid_currency or '?' }}</strong>{% if sp.retail_currency %}, retail is in <strong>{{ sp.retail_currency }}</strong>{% endif %}{% if sp.market_currency %}, market value is in <strong>{{ sp.market_currency }}</strong>{% endif %}.
The "saved vs retail" and "value change vs paid" figures mix currencies and are <strong>not converted</strong>.
</div>
{% endif %}
<p class="text-muted small mb-0 mt-2"><i class="ri-information-line"></i> Estimate. Only covers sets the sidecar has data for; market value only covers sets whose value has been fetched. Currencies are not converted.</p>
</div>
</div>
</div>
</div>
{% endif %}
<!-- Collection Growth Charts -->
{% if config['STATISTICS_SHOW_CHARTS'] %}
<div class="row g-3 mb-4">