List of sets to be refreshed

CHANGELOG.md

@@ -69,6 +69,7 @@
 - Admin
     - Grey out legacy tables in the database view
     - Checkboxes renamed to Set statuses
+    - List of sets that may need to be refreshed
 
 - Cards
     - Use macros for badge in the card header
@@ -102,6 +103,7 @@
     - Collapsible controls depending on screen size
     - Manually collapsible filters (with configuration variable for default state)
     - Manually collapsible sort (with configuration variable for default state)
+    - Clear search bar
 
 - Storage
     - Storage list

bricktracker/app.py

@@ -19,6 +19,7 @@ from bricktracker.views.admin.instructions import admin_instructions_page
 from bricktracker.views.admin.owner import admin_owner_page
 from bricktracker.views.admin.purchase_location import admin_purchase_location_page  # noqa: E501
 from bricktracker.views.admin.retired import admin_retired_page
+from bricktracker.views.admin.set import admin_set_page
 from bricktracker.views.admin.status import admin_status_page
 from bricktracker.views.admin.storage import admin_storage_page
 from bricktracker.views.admin.tag import admin_tag_page
@@ -90,6 +91,7 @@ def setup_app(app: Flask) -> None:
     app.register_blueprint(admin_retired_page)
     app.register_blueprint(admin_owner_page)
     app.register_blueprint(admin_purchase_location_page)
+    app.register_blueprint(admin_set_page)
     app.register_blueprint(admin_status_page)
     app.register_blueprint(admin_storage_page)
     app.register_blueprint(admin_tag_page)

bricktracker/rebrickable_set.py

@@ -1,9 +1,9 @@
 import logging
 from sqlite3 import Row
 import traceback
-from typing import Any, TYPE_CHECKING
+from typing import Any, Self, TYPE_CHECKING
 
-from flask import current_app
+from flask import current_app, url_for
 
 from .exceptions import ErrorException, NotFoundException
 from .instructions import BrickInstructions
@@ -138,6 +138,21 @@ class RebrickableSet(BrickRecord):
 
         return False
 
+    # Select a specific set (with a set)
+    def select_specific(self, set: str, /) -> Self:
+        # Save the parameters to the fields
+        self.fields.set = set
+
+        # Load from database
+        if not self.select():
+            raise NotFoundException(
+                'Set with set {set} was not found in the database'.format(
+                    set=self.fields.set,
+                ),
+            )
+
+        return self
+
     # Return a short form of the Rebrickable set
     def short(self, /, *, from_download: bool = False) -> dict[str, Any]:
         return {
@@ -164,6 +179,10 @@ class RebrickableSet(BrickRecord):
 
         return ''
 
+    # Compute the url for the refresh button
+    def url_for_refresh(self, /) -> str:
+        return url_for('set.refresh', set=self.fields.set)
+
     # Normalize from Rebrickable
     @staticmethod
     def from_rebrickable(data: dict[str, Any], /, **_) -> dict[str, Any]:

bricktracker/rebrickable_set_list.py

@@ -9,6 +9,7 @@ class RebrickableSetList(BrickRecordList[RebrickableSet]):
 
     # Queries
     select_query: str = 'rebrickable/set/list'
+    refresh_query: str = 'rebrickable/set/need_refresh'
 
     # All the sets
     def all(self, /) -> Self:
@@ -19,3 +20,15 @@ class RebrickableSetList(BrickRecordList[RebrickableSet]):
             self.records.append(rebrickable_set)
 
         return self
+
+    # Sets needing refresh
+    def need_refresh(self, /) -> Self:
+        # Load the sets from the database
+        for record in self.select(
+            override_query=self.refresh_query
+        ):
+            rebrickable_set = RebrickableSet(record=record)
+
+            self.records.append(rebrickable_set)
+
+        return self

bricktracker/sql/rebrickable/set/need_refresh.sql

@@ -0,0 +1,53 @@
+SELECT
+    "rebrickable_sets"."set",
+    "rebrickable_sets"."name",
+    "rebrickable_sets"."number_of_parts",
+    "rebrickable_sets"."image",
+    "rebrickable_sets"."url",
+    "null_join"."null_rgb",
+    "null_join"."null_transparent",
+    "null_join"."null_url"
+FROM "rebrickable_sets"
+
+INNER JOIN (
+    SELECT
+        "null_sums"."set",
+        "null_sums"."null_rgb",
+        "null_sums"."null_transparent",
+        "null_sums"."null_url"
+    FROM (
+        SELECT
+            "unique_set_parts"."set",
+            SUM(CASE WHEN "unique_set_parts"."color_rgb" IS NULL THEN 1 ELSE 0 END) AS "null_rgb",
+            SUM(CASE WHEN "unique_set_parts"."color_transparent" IS NULL THEN 1 ELSE 0 END) AS "null_transparent",
+            SUM(CASE WHEN "unique_set_parts"."url" IS NULL THEN 1 ELSE 0 END) AS "null_url"
+        FROM (
+            SELECT
+                "bricktracker_sets"."set",
+                "rebrickable_parts"."color_rgb",
+                "rebrickable_parts"."color_transparent",
+                "rebrickable_parts"."url"
+            FROM "bricktracker_sets"
+
+            INNER JOIN "bricktracker_parts"
+            ON "bricktracker_sets"."id" IS NOT DISTINCT FROM "bricktracker_parts"."id"
+
+            LEFT JOIN "rebrickable_parts"
+            ON "bricktracker_parts"."part" IS NOT DISTINCT FROM "rebrickable_parts"."part"
+            AND "bricktracker_parts"."color" IS NOT DISTINCT FROM "rebrickable_parts"."color_id"
+
+            GROUP BY
+                "bricktracker_sets"."set",
+                "bricktracker_parts"."part",
+                "bricktracker_parts"."color"
+        ) "unique_set_parts"
+
+        GROUP BY "unique_set_parts"."set"
+
+    ) "null_sums"
+
+    WHERE "null_rgb" > 0
+    OR "null_transparent" > 0
+    OR "null_url" > 0
+) "null_join"
+ON "rebrickable_sets"."set" IS NOT DISTINCT FROM "null_join"."set"

bricktracker/sql_counter.py

@@ -10,11 +10,12 @@ ALIASES: dict[str, Tuple[str, str]] = {
     'bricktracker_minifigures': ('Bricktracker minifigures', 'group-line'),
     'bricktracker_parts': ('Bricktracker parts', 'shapes-line'),
     'bricktracker_set_checkboxes': ('Bricktracker set checkboxes (legacy)', 'checkbox-line'),  # noqa: E501
-    'bricktracker_set_owners': ('Bricktracker set owners', 'checkbox-line'),  # noqa: E501
+    'bricktracker_set_owners': ('Bricktracker set owners', 'checkbox-line'),
-    'bricktracker_set_statuses': ('Bricktracker set statuses', 'user-line'),  # noqa: E501
+    'bricktracker_set_statuses': ('Bricktracker set statuses', 'user-line'),
-    'bricktracker_set_tags': ('Bricktracker set tags', 'price-tag-2-line'),  # noqa: E501
+    'bricktracker_set_tags': ('Bricktracker set tags', 'price-tag-2-line'),
     'bricktracker_sets': ('Bricktracker sets', 'hashtag'),
     'bricktracker_wishes': ('Bricktracker wishes', 'gift-line'),
+    'bricktracker_wish_owners': ('Bricktracker wish owners', 'checkbox-line'),
     'inventory': ('Parts', 'shapes-line'),
     'inventory_old': ('Parts (legacy)', 'shapes-line'),
     'minifigures': ('Minifigures', 'group-line'),

bricktracker/views/admin/set.py

@@ -0,0 +1,20 @@
+from flask import Blueprint, render_template, request
+from flask_login import login_required
+
+from ..exceptions import exception_handler
+from ...rebrickable_set_list import RebrickableSetList
+
+admin_set_page = Blueprint('admin_set',  __name__, url_prefix='/admin/set')
+
+
+# Sets that need o be refreshed
+@admin_set_page.route('/refresh', methods=['GET'])
+@login_required
+@exception_handler(__file__)
+def refresh() -> str:
+    return render_template(
+        'admin.html',
+        refresh_set=True,
+        table_collection=RebrickableSetList().need_refresh(),
+        set_error=request.args.get('set_error')
+    )

bricktracker/views/set.py

@@ -13,8 +13,10 @@ from flask_login import login_required
 from werkzeug.wrappers.response import Response
 
 from .exceptions import exception_handler
+from ..exceptions import ErrorException
 from ..minifigure import BrickMinifigure
 from ..part import BrickPart
+from ..rebrickable_set import RebrickableSet
 from ..set import BrickSet
 from ..set_list import BrickSetList, set_metadata_lists
 from ..set_owner_list import BrickSetOwnerList
@@ -241,13 +243,22 @@ def problem_part(
 
 
 # Refresh a set
+@set_page.route('/refresh/<set>/', methods=['GET'])
 @set_page.route('/<id>/refresh', methods=['GET'])
 @login_required
 @exception_handler(__file__)
-def refresh(*, id: str) -> str:
+def refresh(*, id: str | None = None, set: str | None = None) -> str:
+    if id is not None:
+        item = BrickSet().select_specific(id)
+    elif set is not None:
+        item = RebrickableSet().select_specific(set)
+    else:
+        raise ErrorException('Could not load any set to refresh')
+
     return render_template(
         'refresh.html',
-        item=BrickSet().select_specific(id),
+        id=id,
+        item=item,
         path=current_app.config['SOCKET_PATH'],
         namespace=current_app.config['SOCKET_NAMESPACE'],
         messages=MESSAGES

templates/admin.html

@@ -32,6 +32,8 @@
             {% include 'admin/database/import.html' %}
             {% elif upgrade_database %}
             {% include 'admin/database/upgrade.html' %}
+            {% elif refresh_set %}
+            {% include 'admin/set/refresh.html' %}
             {% else %}
               {% include 'admin/logout.html' %}
               {% include 'admin/instructions.html' %}
@@ -47,6 +49,7 @@
                 {% include 'admin/storage.html' %}
                 {% include 'admin/tag.html' %}
               {{ accordion.footer() }}
+              {% include 'admin/refresh.html' %}
               {% include 'admin/database.html' %}
               {% include 'admin/configuration.html' %}
             {% endif %}

templates/admin/refresh.html

@@ -0,0 +1,5 @@
+{% import 'macro/accordion.html' as accordion %}
+
+{{ accordion.header('Set refresh', 'refresh', 'admin', icon='refresh-line') }}
+<a href="{{ url_for('admin_set.refresh') }}" class="btn btn-primary" role="button"><i class="ri-refresh-line"></i> Check for sets that may need a refresh</a>
+{{ accordion.footer() }}

templates/admin/set/refresh.html

@@ -0,0 +1,34 @@
+{% import 'macro/table.html' as table %}
+{% import 'macro/badge.html' as badge %}
+
+<div class="alert alert-info m-2" role="alert">This page lists the sets that may need a refresh because they have some of their newer fields containing empty values.</div>
+<div class="table-responsive-sm">
+  <table data-table="true" class="table table-striped align-middle" id="wish">
+    <thead>
+      <tr>
+        <th data-table-no-sort-and-search="true" class="no-sort" scope="col"><i class="ri-image-line fw-normal"></i> Image</th>
+        <th scope="col"><i class="ri-hashtag fw-normal"></i> Set</th>
+        <th scope="col"><i class="ri-pencil-line fw-normal"></i> Name</th>
+        <th scope="col"><i class="ri-shapes-line fw-normal"></i> Parts</th>
+        <th data-table-number="true" scope="col"><i class="ri-error-warning-line fw-normal"></i> Empty RGB</th>
+        <th data-table-number="true" scope="col"><i class="ri-error-warning-line fw-normal"></i> Empty transparent</th>
+        <th data-table-number="true" scope="col"><i class="ri-error-warning-line fw-normal"></i> Empty URL</th>
+        <th data-table-no-sort-and-search="true" class="no-sort" scope="col"><i class="ri-settings-4-line fw-normal"></i> Actions</th>
+      </tr>
+    </thead>
+    <tbody>
+      {% for item in table_collection %}
+        <tr>
+          {{ table.image(item.url_for_image(), caption=item.fields.name, alt=item.fields.set) }}
+          <td>{{ item.fields.set }} {{ table.rebrickable(item) }}</td>
+          <td>{{ item.fields.name }}</td>
+          <td>{{ item.fields.number_of_parts }}</td>
+          <td>{{ item.fields.null_rgb }}</td>
+          <td>{{ item.fields.null_transparent }}</td>
+          <td>{{ item.fields.null_url }}</td>
+          <td><a href="{{ item.url_for_refresh() }}" class="btn btn-primary" role="button"><i class="ri-refresh-line"></i> Refresh</a></td>
+        </tr>
+      {% endfor %}
+    </tbody>
+  </table>
+</div>

templates/refresh.html

@@ -51,7 +51,11 @@
         </div>
         <div class="card-footer text-end">
           <span id="refresh-status-icon" class="me-1"></span><span id="refresh-status" class="me-1"></span>
-          <a href="{{ url_for('set.details', id=item.fields.id) }}" class="btn btn-primary" role="button"><i class="ri-hashtag"></i> Back to the set details</a>
+          {% if id %}
+            <a href="{{ url_for('set.details', id=item.fields.id) }}" class="btn btn-primary" role="button"><i class="ri-hashtag"></i> Back to the set details</a>
+            {% else %}
+            <a href="{{ url_for('admin_set.refresh') }}" class="btn btn-danger" role="button"><i class="ri-hashtag"></i> List of sets to be refreshed</a>
+          {% endif %}
           <button id="refresh" type="button" class="btn btn-primary"><i class="ri-refresh-line"></i> Refresh</button>
         </div>
       </div>