Add a refresh mode for sets

bricktracker/minifigure.py

@@ -20,7 +20,7 @@ class BrickMinifigure(RebrickableMinifigure):
     select_query: str = 'minifigure/select/specific'
 
     # Import a minifigure into the database
-    def download(self, socket: 'BrickSocket') -> bool:
+    def download(self, socket: 'BrickSocket', refresh: bool = False) -> bool:
         if self.brickset is None:
             raise ErrorException('Importing a minifigure from Rebrickable outside of a set is not supported')  # noqa: E501
 
@@ -33,6 +33,7 @@ class BrickMinifigure(RebrickableMinifigure):
                 )
             )
 
+            if not refresh:
                 # Insert into database
                 self.insert(commit=False)
 
@@ -43,7 +44,8 @@ class BrickMinifigure(RebrickableMinifigure):
             if not BrickPartList.download(
                 socket,
                 self.brickset,
-                minifigure=self
+                minifigure=self,
+                refresh=refresh
             ):
                 return False
 

bricktracker/minifigure_list.py

@@ -134,7 +134,13 @@ class BrickMinifigureList(BrickRecordList[BrickMinifigure]):
 
     # Import the minifigures from Rebrickable
     @staticmethod
-    def download(socket: 'BrickSocket', brickset: 'BrickSet', /) -> bool:
+    def download(
+        socket: 'BrickSocket',
+        brickset: 'BrickSet',
+        /,
+        *,
+        refresh: bool = False
+    ) -> bool:
         try:
             socket.auto_progress(
                 message='Set {set}: loading minifigures from Rebrickable'.format(  # noqa: E501
@@ -157,7 +163,7 @@ class BrickMinifigureList(BrickRecordList[BrickMinifigure]):
 
             # Process each minifigure
             for minifigure in minifigures:
-                if not minifigure.download(socket):
+                if not minifigure.download(socket, refresh=refresh):
                     return False
 
             return True

bricktracker/part.py

@@ -34,7 +34,7 @@ class BrickPart(RebrickablePart):
             self.kind = 'Set'
 
     # Import a part into the database
-    def download(self, socket: 'BrickSocket') -> bool:
+    def download(self, socket: 'BrickSocket', refresh: bool = False) -> bool:
         if self.brickset is None:
             raise ErrorException('Importing a part from Rebrickable outside of a set is not supported')  # noqa: E501
 
@@ -48,6 +48,7 @@ class BrickPart(RebrickablePart):
                 )
             )
 
+            if not refresh:
                 # Insert into database
                 self.insert(commit=False)
 

bricktracker/part_list.py

@@ -139,6 +139,7 @@ class BrickPartList(BrickRecordList[BrickPart]):
         /,
         *,
         minifigure: 'BrickMinifigure | None' = None,
+        refresh: bool = False
     ) -> bool:
         if minifigure is not None:
             identifier = minifigure.fields.figure
@@ -174,7 +175,7 @@ class BrickPartList(BrickRecordList[BrickPart]):
 
             # Process each part
             for part in inventory:
-                if not part.download(socket):
+                if not part.download(socket, refresh=refresh):
                     return False
 
         except Exception as e:

bricktracker/rebrickable_minifigure.py

@@ -38,28 +38,23 @@ class RebrickableMinifigure(BrickRecord):
             self.ingest(record)
 
     # Insert the minifigure from Rebrickable
-    def insert_rebrickable(self, /) -> bool:
+    def insert_rebrickable(self, /) -> None:
         if self.brickset is None:
             raise ErrorException('Importing a minifigure from Rebrickable outside of a set is not supported')  # noqa: E501
 
         # Insert the Rebrickable minifigure to the database
-        rows, _ = self.insert(
+        self.insert(
             commit=False,
             no_defer=True,
             override_query=RebrickableMinifigure.insert_query
         )
 
-        inserted = rows > 0
-
-        if inserted:
         if not current_app.config['USE_REMOTE_IMAGES']:
             RebrickableImage(
                 self.brickset,
                 minifigure=self,
             ).download()
 
-        return inserted
-
     # Return a dict with common SQL parameters for a minifigure
     def sql_parameters(self, /) -> dict[str, Any]:
         parameters = super().sql_parameters()

bricktracker/rebrickable_part.py

@@ -48,20 +48,17 @@ class RebrickablePart(BrickRecord):
             self.ingest(record)
 
     # Insert the part from Rebrickable
-    def insert_rebrickable(self, /) -> bool:
+    def insert_rebrickable(self, /) -> None:
         if self.brickset is None:
             raise ErrorException('Importing a part from Rebrickable outside of a set is not supported')  # noqa: E501
 
         # Insert the Rebrickable part to the database
-        rows, _ = self.insert(
+        self.insert(
             commit=False,
             no_defer=True,
             override_query=RebrickablePart.insert_query
         )
 
-        inserted = rows > 0
-
-        if inserted:
         if not current_app.config['USE_REMOTE_IMAGES']:
             RebrickableImage(
                 self.brickset,
@@ -69,8 +66,6 @@ class RebrickablePart(BrickRecord):
                 part=self,
             ).download()
 
-        return inserted
-
     # Return a dict with common SQL parameters for a part
     def sql_parameters(self, /) -> dict[str, Any]:
         parameters = super().sql_parameters()

bricktracker/rebrickable_set.py

@@ -47,22 +47,17 @@ class RebrickableSet(BrickRecord):
             self.ingest(record)
 
     # Insert the set from Rebrickable
-    def insert_rebrickable(self, /) -> bool:
+    def insert_rebrickable(self, /) -> None:
         # Insert the Rebrickable set to the database
-        rows, _ = self.insert(
+        self.insert(
             commit=False,
             no_defer=True,
             override_query=RebrickableSet.insert_query
         )
 
-        inserted = rows > 0
-
-        if inserted:
         if not current_app.config['USE_REMOTE_IMAGES']:
             RebrickableImage(self).download()
 
-        return inserted
-
     # Ingest a set
     def ingest(self, record: Row | dict[str, Any], /):
         super().ingest(record)

bricktracker/record.py

@@ -1,5 +1,5 @@
 from sqlite3 import Row
-from typing import Any, ItemsView, Tuple
+from typing import Any, ItemsView
 
 from .fields import BrickRecordFields
 from .sql import BrickSQL
@@ -31,14 +31,14 @@ class BrickRecord(object):
         commit=True,
         no_defer=False,
         override_query: str | None = None
-    ) -> Tuple[int, str]:
+    ) -> None:
         if override_query:
             query = override_query
         else:
             query = self.insert_query
 
         database = BrickSQL()
-        rows, q = database.execute(
+        database.execute(
             query,
             parameters=self.sql_parameters(),
             defer=not commit and not no_defer,
@@ -47,8 +47,6 @@ class BrickRecord(object):
         if commit:
             database.commit()
 
-        return rows, q
-
     # Shorthand to field items
     def items(self, /) -> ItemsView[str, Any]:
         return self.fields.__dict__.items()

bricktracker/set.py

@@ -47,9 +47,13 @@ class BrickSet(RebrickableSet):
                 increment_total=True,
             )
 
+            # Grabbing the refresh flag
+            refresh: bool = bool(data.get('refresh', False))
+
             # Generate an UUID for self
             self.fields.id = str(uuid4())
 
+            if not refresh:
                 # Insert into database
                 self.insert(commit=False)
 
@@ -57,11 +61,11 @@ class BrickSet(RebrickableSet):
             self.insert_rebrickable()
 
             # Load the inventory
-            if not BrickPartList.download(socket, self):
+            if not BrickPartList.download(socket, self, refresh=refresh):
                 return False
 
             # Load the minifigures
-            if not BrickMinifigureList.download(socket, self):
+            if not BrickMinifigureList.download(socket, self, refresh=refresh):
                 return False
 
             # Commit the transaction to the database
@@ -74,12 +78,26 @@ class BrickSet(RebrickableSet):
 
             BrickSQL().commit()
 
+            if refresh:
                 # Info
                 logger.info('Set {set}: imported (id: {id})'.format(
                     set=self.fields.set,
                     id=self.fields.id,
                 ))
 
+                # Complete
+                socket.complete(
+                    message='Set {set}: refreshed'.format(  # noqa: E501
+                        set=self.fields.set,
+                    ),
+                    download=True
+                )
+            else:
+                # Info
+                logger.info('Set {set}: refreshed'.format(
+                    set=self.fields.set,
+                ))
+
                 # Complete
                 socket.complete(
                     message='Set {set}: imported (<a href="{url}">Go to the set</a>)'.format(  # noqa: E501
@@ -192,3 +210,10 @@ class BrickSet(RebrickableSet):
             )
         else:
             return ''
+
+    # Compute the url for the refresh button
+    def url_for_refresh(self, /) -> str:
+        return url_for(
+            'set.refresh',
+            id=self.fields.id,
+        )

bricktracker/set_checkbox.py

@@ -1,5 +1,5 @@
 from sqlite3 import Row
-from typing import Any, Self, Tuple
+from typing import Any, Self
 from uuid import uuid4
 
 from flask import url_for
@@ -60,7 +60,7 @@ class BrickSetCheckbox(BrickRecord):
         return self
 
     # Insert into database
-    def insert(self, **_) -> Tuple[int, str]:
+    def insert(self, **_) -> None:
         # Generate an ID for the checkbox (with underscores to make it
         # column name friendly)
         self.fields.id = str(uuid4()).replace('-', '_')
@@ -72,9 +72,6 @@ class BrickSetCheckbox(BrickRecord):
             displayed_on_grid=self.fields.displayed_on_grid
         )
 
-        # To accomodate the parent().insert we have overriden
-        return 0, ''
-
     # Rename the checkbox
     def rename(self, /) -> None:
         # Update the name

bricktracker/sql/rebrickable/minifigure/insert.sql

@@ -9,3 +9,9 @@ INSERT OR IGNORE INTO "rebrickable_minifigures" (
     :name,
     :image
 )
+ON CONFLICT("figure")
+DO UPDATE SET
+"number" = :number,
+"name" = :name,
+"image" = :image
+WHERE "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM :figure

bricktracker/sql/rebrickable/part/insert.sql

@@ -23,3 +23,16 @@ INSERT OR IGNORE INTO "rebrickable_parts" (
     :url,
     :print
 )
+ON CONFLICT("part", "color_id")
+DO UPDATE SET
+"color_name" = :color_name,
+"color_rgb" = :color_rgb,
+"color_transparent" = :color_transparent,
+"name" = :name,
+"category" = :category,
+"image" = :image,
+"image_id" = :image_id,
+"url" = :url,
+"print" = :print
+WHERE "rebrickable_parts"."part" IS NOT DISTINCT FROM :part
+AND "rebrickable_parts"."color_id" IS NOT DISTINCT FROM :color_id

bricktracker/sql/rebrickable/set/insert.sql

@@ -21,3 +21,15 @@ INSERT OR IGNORE INTO "rebrickable_sets" (
     :url,
     :last_modified
 )
+ON CONFLICT("set")
+DO UPDATE SET
+    "number" = :number,
+    "version" = :version,
+    "name" = :name,
+    "year" = :year,
+    "theme_id" = :theme_id,
+    "number_of_parts" = :number_of_parts,
+    "image" = :image,
+    "url" = :url,
+    "last_modified" = :last_modified
+WHERE "rebrickable_sets"."set" IS NOT DISTINCT FROM :set

bricktracker/views/set.py

@@ -2,6 +2,7 @@ import logging
 
 from flask import (
     Blueprint,
+    current_app,
     jsonify,
     render_template,
     redirect,
@@ -17,6 +18,7 @@ from ..part import BrickPart
 from ..set import BrickSet
 from ..set_checkbox_list import BrickSetCheckboxList
 from ..set_list import BrickSetList
+from ..socket import MESSAGES
 
 logger = logging.getLogger(__name__)
 
@@ -154,3 +156,16 @@ def missing_part(
     ))
 
     return jsonify({'missing': missing})
+
+
+# Refresh a set
+@set_page.route('/<id>/refresh', methods=['GET'])
+@exception_handler(__file__)
+def refresh(*, id: str) -> str:
+    return render_template(
+        'refresh.html',
+        item=BrickSet().select_specific(id),
+        path=current_app.config['SOCKET_PATH'],
+        namespace=current_app.config['SOCKET_NAMESPACE'],
+        messages=MESSAGES
+    )

static/scripts/socket/set.js

@@ -1,8 +1,11 @@
 // Set Socket class
 class BrickSetSocket extends BrickSocket {
-    constructor(id, path, namespace, messages, bulk=false) {
+    constructor(id, path, namespace, messages, bulk=false, refresh=false) {
         super(id, path, namespace, messages, bulk);
 
+        // Refresh mode
+        this.refresh = true
+
         // Listeners
         this.add_listener = undefined;
         this.input_listener = undefined;
@@ -82,7 +85,7 @@ class BrickSetSocket extends BrickSocket {
                 this.read_set_list();
             }
 
-            if (this.bulk || (this.html_no_confim && this.html_no_confim.checked)) {
+            if (this.bulk || this.refresh || (this.html_no_confim && this.html_no_confim.checked)) {
                 this.import_set(true);
             } else {
                 this.load_set();
@@ -140,6 +143,7 @@ class BrickSetSocket extends BrickSocket {
 
             this.socket.emit(this.messages.IMPORT_SET, {
                 set: (set !== undefined) ? set : this.html_input.value,
+                refresh: this.refresh
             });
         } else {
             this.fail("Could not find the input field for the set number");

templates/add.html

@@ -68,5 +68,7 @@
     </div>
   </div>
 </div>
-{% include 'set/socket.html' %}
+{% with id='add' %}
+  {% include 'set/socket.html' %}
+{% endwith %}
 {% endblock %}

templates/bulk.html

@@ -58,7 +58,7 @@
     </div>
   </div>
 </div>
-{% with bulk=true %}
+{% with id='add', bulk=true %}
   {% include 'set/socket.html' %}
 {% endwith %}
 {% endblock %}

templates/refresh.html

@@ -0,0 +1,64 @@
+{% extends 'base.html' %}
+
+{% block title %} - Refresh set {{ item.fields.set }}{% endblock %}
+
+{% block main %}
+<div class="container">
+  <div class="alert alert-primary" role="alert">
+    <h4 class="alert-heading">Refreshing from Rebrickable</h4>
+    <p class="mb-0">This will refresh all the Rebrickable data (set, minifigures, parts) associated with this set.</p>
+  </div>
+  <div class="row">
+    <div class="col-12">
+      <div class="card mb-3">
+        <div class="card-header">
+          <h5 class="mb-0"><i class="ri-refresh-line"></i> Refresh a set</h5>
+        </div>
+        <div class="card-body">
+          <div id="refresh-fail" class="alert alert-danger d-none" role="alert"></div>
+          <div id="refresh-complete" class="alert alert-success d-none" role="alert"></div>
+          <div class="mb-3">
+            <label for="refresh-set" class="form-label">Set number</label>
+            <input type="text" class="form-control" id="refresh-set" value="{{ item.fields.set }}">
+          </div>
+          <hr>
+          <div class="mb-3">
+            <p>
+              Progress <span id="refresh-count"></span>
+              <span id="refresh-spinner" class="d-none">
+                <span class="spinner-border spinner-border-sm" aria-hidden="true"></span>
+                <span class="visually-hidden" role="status">Loading...</span>
+              </span>
+            </p>
+            <div id="refresh-progress" class="progress" role="progressbar" aria-label="Refresh a set progress" aria-valuenow="0" aria-valuemin="0" aria-valuemax="100">
+              <div id="refresh-progress-bar" class="progress-bar" style="width: 0%"></div>
+            </div>
+            <p id="refresh-progress-message" class="text-center d-none"></p>
+          </div>
+          <div id="refresh-card" class="d-flex justify-content-center">
+            <div class="card mb-3 col-6">
+              <div class="card-header">
+                <h5 class="mb-0">
+                  <span class="badge text-bg-secondary fw-normal"><i class="ri-hashtag"></i> <span id="refresh-card-set">{{ item.fields.set }}</span></span>
+                  <span id="refresh-card-name">{{ item.fields.name }}</span>
+                </h5>
+              </div>
+              <div id="refresh-card-image-container" class="card-img" style="background-image: url({{ item.url_for_image() }})">
+                <img id="refresh-card-image" src="{{ item.url_for_image() }}">
+              </div>
+            </div>
+          </div>
+        </div>
+        <div class="card-footer text-end">
+          <span id="refresh-status-icon" class="me-1"></span><span id="refresh-status" class="me-1"></span>
+          <a href="{{ url_for('set.details', id=item.fields.id) }}" class="btn btn-primary" role="button"><i class="ri-hashtag"></i> Back to the set details</a>
+          <button id="refresh" type="button" class="btn btn-primary"><i class="ri-refresh-line"></i> Refresh</button>
+        </div>
+      </div>
+    </div>
+  </div>
+</div>
+{% with id='refresh', refresh=true %}
+  {% include 'set/socket.html' %}
+{% endwith %}
+{% endblock %}

templates/set/card.html

@@ -63,6 +63,9 @@
         {% endfor %}
       {% endif %}
       {% if g.login.is_authenticated() %}
+        {{ accordion.header('Management', 'management', 'set-details', icon='settings-4-line', class='text-end') }}
+            <a href="{{ item.url_for_refresh() }}" class="btn btn-primary" role="button"><i class="ri-refresh-line"></i> Refresh the set</a>
+        {{ accordion.footer() }}
         {{ accordion.header('Danger zone', 'danger-zone', 'set-details', expanded=delete, danger=true, class='text-end') }}
           {% if delete %}
             <form action="{{ item.url_for_do_delete() }}" method="post">

templates/set/socket.html

@@ -1,12 +1,19 @@
 <script type="text/javascript">
   document.addEventListener("DOMContentLoaded", () => {
-    new BrickSetSocket('add', '{{ path }}', '{{ namespace }}', {
+    new BrickSetSocket(
+      '{{ id }}',
+      '{{ path }}',
+      '{{ namespace }}',
+      {
         COMPLETE: '{{ messages['COMPLETE'] }}',
         FAIL: '{{ messages['FAIL'] }}',
         IMPORT_SET: '{{ messages['IMPORT_SET'] }}',
         LOAD_SET: '{{ messages['LOAD_SET'] }}',
         PROGRESS: '{{ messages['PROGRESS'] }}',
         SET_LOADED: '{{ messages['SET_LOADED'] }}',
-    }{% if bulk %}, true{% endif %});
+      },
+      {% if bulk %}true{% else %}false{% endif %},
+      {% if refresh %}true{% else %}false{% endif %}
+    );
   });
 </script>