Merge pull request 'Added migration to get new bricklink data fields, fixed bricklink links, added set refresh based on missing bricklink data' (#88) from feature/bricklink-data into master

Reviewed-on: #88

Fixes #87 and #55

.env.sample

@@ -28,7 +28,8 @@
 # BK_AUTHENTICATION_KEY=change-this-to-something-random
 
 # Optional: Pattern of the link to Bricklink for a part. Will be passed to Python .format()
-# Default: https://www.bricklink.com/v2/catalog/catalogitem.page?P={part}
+# Supports {part} and {color} parameters. BrickLink part numbers and color IDs are used when available.
+# Default: https://www.bricklink.com/v2/catalog/catalogitem.page?P={part}&C={color}
 # BK_BRICKLINK_LINK_PART_PATTERN=
 
 # Optional: Display Bricklink links wherever applicable

CHANGELOG.md

@@ -2,7 +2,26 @@
 
 ## Unreleased
 
-- Fix legibility of "Damaged" and "Missing" fields for tiny screen by reducing horizontal padding
+### Current PR
+
+- Added search/filter/sort options to `parts` and `minifigures`.
+
+### Next PR
+
+> **Warning**
+> To use the new BrickLink color parameter in URLs, update your `.env` file:
+> `BK_BRICKLINK_LINK_PART_PATTERN=https://www.bricklink.com/v2/catalog/catalogitem.page?P={part}&C={color}`
+
+- Add BrickLink color and part number support for accurate BrickLink URLs
+  - Database migrations to store BrickLink color ID, color name, and part number
+  - Updated Rebrickable API integration to extract BrickLink data from external_ids
+  - Enhanced BrickLink URL generation with proper part number fallback
+  - Extended admin set refresh to detect and track missing BrickLink data
+
+## 1.2.2
+
+Fix legibility of "Damaged" and "Missing" fields for tiny screen by reducing horizontal padding
+Fixed instructions download from Rebrickable
 
 ## 1.2.2:
 

bricktracker/config.py

@@ -10,7 +10,7 @@ from typing import Any, Final
 CONFIG: Final[list[dict[str, Any]]] = [
     {'n': 'AUTHENTICATION_PASSWORD', 'd': ''},
     {'n': 'AUTHENTICATION_KEY', 'd': ''},
-    {'n': 'BRICKLINK_LINK_PART_PATTERN', 'd': 'https://www.bricklink.com/v2/catalog/catalogitem.page?P={part}'},  # noqa: E501
+    {'n': 'BRICKLINK_LINK_PART_PATTERN', 'd': 'https://www.bricklink.com/v2/catalog/catalogitem.page?P={part}&C={color}'},  # noqa: E501
     {'n': 'BRICKLINK_LINKS', 'c': bool},
     {'n': 'DATABASE_PATH', 'd': './app.db'},
     {'n': 'DATABASE_TIMESTAMP_FORMAT', 'd': '%Y-%m-%d-%H-%M-%S'},

bricktracker/rebrickable_part.py

@@ -91,8 +91,17 @@ class RebrickablePart(BrickRecord):
     def url_for_bricklink(self, /) -> str:
         if current_app.config['BRICKLINK_LINKS']:
             try:
+                # Use BrickLink part number if available and not None/empty, otherwise fall back to Rebrickable part
+                bricklink_part = getattr(self.fields, 'bricklink_part_num', None)
+                part_param = bricklink_part if bricklink_part else self.fields.part
+
+                # Use BrickLink color ID if available and not None, otherwise fall back to Rebrickable color
+                bricklink_color = getattr(self.fields, 'bricklink_color_id', None)
+                color_param = bricklink_color if bricklink_color is not None else self.fields.color
+                print(f'BrickLink URL parameters: part={part_param}, color={color_param}')  # Debugging line, can be removed later
                 return current_app.config['BRICKLINK_LINK_PART_PATTERN'].format(  # noqa: E501
-                    part=self.fields.part,
+                    part=part_param,
+                    color=color_param,
                 )
             except Exception:
                 pass
@@ -168,6 +177,9 @@ class RebrickablePart(BrickRecord):
             'color_name': data['color']['name'],
             'color_rgb': data['color']['rgb'],
             'color_transparent': data['color']['is_trans'],
+            'bricklink_color_id': None,
+            'bricklink_color_name': None,
+            'bricklink_part_num': None,
             'name': data['part']['name'],
             'category': data['part']['part_cat_id'],
             'image': data['part']['part_img_url'],
@@ -176,6 +188,30 @@ class RebrickablePart(BrickRecord):
             'print': data['part']['print_of']
         }
 
+        # Extract BrickLink color info if available in external_ids
+        if 'color' in data and 'external_ids' in data['color']:
+            external_ids = data['color']['external_ids']
+            if 'BrickLink' in external_ids and external_ids['BrickLink']:
+                bricklink_data = external_ids['BrickLink']
+
+                # Extract BrickLink color ID and name from the nested structure
+                if isinstance(bricklink_data, dict):
+                    if 'ext_ids' in bricklink_data and bricklink_data['ext_ids']:
+                        record['bricklink_color_id'] = bricklink_data['ext_ids'][0]
+
+                    if 'ext_descrs' in bricklink_data and bricklink_data['ext_descrs']:
+                        # ext_descrs is a list of lists, get the first description from the first list
+                        if len(bricklink_data['ext_descrs']) > 0 and len(bricklink_data['ext_descrs'][0]) > 0:
+                            record['bricklink_color_name'] = bricklink_data['ext_descrs'][0][0]
+
+        # Extract BrickLink part number if available
+        if 'part' in data and 'external_ids' in data['part']:
+            part_external_ids = data['part']['external_ids']
+            if 'BrickLink' in part_external_ids and part_external_ids['BrickLink']:
+                bricklink_parts = part_external_ids['BrickLink']
+                if isinstance(bricklink_parts, list) and len(bricklink_parts) > 0:
+                    record['bricklink_part_num'] = bricklink_parts[0]
+
         if brickset is not None:
             record['id'] = brickset.fields.id
 

bricktracker/sql/migrations/0016.sql

@@ -0,0 +1,9 @@
+-- description: Add BrickLink color fields to rebrickable_parts table
+
+BEGIN TRANSACTION;
+
+-- Add BrickLink color fields to the rebrickable_parts table
+ALTER TABLE "rebrickable_parts" ADD COLUMN "bricklink_color_id" INTEGER;
+ALTER TABLE "rebrickable_parts" ADD COLUMN "bricklink_color_name" TEXT;
+
+COMMIT;

bricktracker/sql/migrations/0017.sql

@@ -0,0 +1,8 @@
+-- description: Add BrickLink part number field to rebrickable_parts table
+
+BEGIN TRANSACTION;
+
+-- Add BrickLink part number field to the rebrickable_parts table
+ALTER TABLE "rebrickable_parts" ADD COLUMN "bricklink_part_num" TEXT;
+
+COMMIT;

bricktracker/sql/part/base/base.sql

@@ -14,6 +14,9 @@ SELECT
     "rebrickable_parts"."color_name",
     "rebrickable_parts"."color_rgb",
     "rebrickable_parts"."color_transparent",
+    "rebrickable_parts"."bricklink_color_id",
+    "rebrickable_parts"."bricklink_color_name",
+    "rebrickable_parts"."bricklink_part_num",
     "rebrickable_parts"."name",
     --"rebrickable_parts"."category",
     "rebrickable_parts"."image",

bricktracker/sql/rebrickable/part/insert.sql

@@ -4,6 +4,9 @@ INSERT OR IGNORE INTO "rebrickable_parts" (
     "color_name",
     "color_rgb",
     "color_transparent",
+    "bricklink_color_id",
+    "bricklink_color_name",
+    "bricklink_part_num",
     "name",
     "category",
     "image",
@@ -16,6 +19,9 @@ INSERT OR IGNORE INTO "rebrickable_parts" (
     :color_name,
     :color_rgb,
     :color_transparent,
+    :bricklink_color_id,
+    :bricklink_color_name,
+    :bricklink_part_num,
     :name,
     :category,
     :image,
@@ -28,6 +34,9 @@ DO UPDATE SET
 "color_name" = :color_name,
 "color_rgb" = :color_rgb,
 "color_transparent" = :color_transparent,
+"bricklink_color_id" = :bricklink_color_id,
+"bricklink_color_name" = :bricklink_color_name,
+"bricklink_part_num" = :bricklink_part_num,
 "name" = :name,
 "category" = :category,
 "image" = :image,

bricktracker/sql/rebrickable/part/list.sql

@@ -4,6 +4,9 @@ SELECT
     "rebrickable_parts"."color_name",
     "rebrickable_parts"."color_rgb",
     "rebrickable_parts"."color_transparent",
+    "rebrickable_parts"."bricklink_color_id",
+    "rebrickable_parts"."bricklink_color_name",
+    "rebrickable_parts"."bricklink_part_num",
     "rebrickable_parts"."name",
     "rebrickable_parts"."category",
     "rebrickable_parts"."image",

bricktracker/sql/rebrickable/part/select.sql

@@ -4,6 +4,9 @@ SELECT
     "rebrickable_parts"."color_name",
     "rebrickable_parts"."color_rgb",
     "rebrickable_parts"."color_transparent",
+    "rebrickable_parts"."bricklink_color_id",
+    "rebrickable_parts"."bricklink_color_name",
+    "rebrickable_parts"."bricklink_part_num",
     "rebrickable_parts"."name",
     "rebrickable_parts"."category",
     "rebrickable_parts"."image",

bricktracker/sql/rebrickable/set/need_refresh.sql

@@ -6,7 +6,10 @@ SELECT
     "rebrickable_sets"."url",
     "null_join"."null_rgb",
     "null_join"."null_transparent",
-    "null_join"."null_url"
+    "null_join"."null_url",
+    "null_join"."null_bricklink_color_id",
+    "null_join"."null_bricklink_color_name",
+    "null_join"."null_bricklink_part_num"
 FROM "rebrickable_sets"
 
 INNER JOIN (
@@ -14,19 +17,28 @@ INNER JOIN (
         "null_sums"."set",
         "null_sums"."null_rgb",
         "null_sums"."null_transparent",
-        "null_sums"."null_url"
+        "null_sums"."null_url",
+        "null_sums"."null_bricklink_color_id",
+        "null_sums"."null_bricklink_color_name",
+        "null_sums"."null_bricklink_part_num"
     FROM (
         SELECT
             "unique_set_parts"."set",
             SUM(CASE WHEN "unique_set_parts"."color_rgb" IS NULL THEN 1 ELSE 0 END) AS "null_rgb",
             SUM(CASE WHEN "unique_set_parts"."color_transparent" IS NULL THEN 1 ELSE 0 END) AS "null_transparent",
-            SUM(CASE WHEN "unique_set_parts"."url" IS NULL THEN 1 ELSE 0 END) AS "null_url"
+            SUM(CASE WHEN "unique_set_parts"."url" IS NULL THEN 1 ELSE 0 END) AS "null_url",
+            SUM(CASE WHEN "unique_set_parts"."bricklink_color_id" IS NULL THEN 1 ELSE 0 END) AS "null_bricklink_color_id",
+            SUM(CASE WHEN "unique_set_parts"."bricklink_color_name" IS NULL THEN 1 ELSE 0 END) AS "null_bricklink_color_name",
+            SUM(CASE WHEN "unique_set_parts"."bricklink_part_num" IS NULL THEN 1 ELSE 0 END) AS "null_bricklink_part_num"
         FROM (
             SELECT
                 "bricktracker_sets"."set",
                 "rebrickable_parts"."color_rgb",
                 "rebrickable_parts"."color_transparent",
-                "rebrickable_parts"."url"
+                "rebrickable_parts"."url",
+                "rebrickable_parts"."bricklink_color_id",
+                "rebrickable_parts"."bricklink_color_name",
+                "rebrickable_parts"."bricklink_part_num"
             FROM "bricktracker_sets"
 
             INNER JOIN "bricktracker_parts"
@@ -49,5 +61,8 @@ INNER JOIN (
     WHERE "null_rgb" > 0
     OR "null_transparent" > 0
     OR "null_url" > 0
+    OR "null_bricklink_color_id" > 0
+    OR "null_bricklink_color_name" > 0
+    OR "null_bricklink_part_num" > 0
 ) "null_join"
 ON "rebrickable_sets"."set" IS NOT DISTINCT FROM "null_join"."set"

bricktracker/version.py

@@ -1,4 +1,4 @@
 from typing import Final
 
 __version__: Final[str] = '1.2.2'
-__database_version__: Final[int] = 15
+__database_version__: Final[int] = 17

bricktracker/views/admin/set.py

@@ -7,7 +7,7 @@ from ...rebrickable_set_list import RebrickableSetList
 admin_set_page = Blueprint('admin_set',  __name__, url_prefix='/admin/set')
 
 
-# Sets that need o be refreshed
+# Sets that need to be refreshed
 @admin_set_page.route('/refresh', methods=['GET'])
 @login_required
 @exception_handler(__file__)

templates/admin/set/refresh.html

@@ -1,5 +1,6 @@
 {% import 'macro/table.html' as table %}
 {% import 'macro/badge.html' as badge %}
+{% import 'macro/form.html' as form %}
 
 <div class="alert alert-info m-2" role="alert">This page lists the sets that may need a refresh because they have some of their newer fields containing empty values.</div>
 <div class="table-responsive-sm">
@@ -13,6 +14,7 @@
         <th data-table-number="true" scope="col"><i class="ri-error-warning-line fw-normal"></i> Empty RGB</th>
         <th data-table-number="true" scope="col"><i class="ri-error-warning-line fw-normal"></i> Empty transparent</th>
         <th data-table-number="true" scope="col"><i class="ri-error-warning-line fw-normal"></i> Empty URL</th>
+        <th data-table-number="true" scope="col"><i class="ri-palette-line fw-normal"></i>BrickLink Issues</th>
         <th data-table-no-sort-and-search="true" class="no-sort" scope="col"><i class="ri-settings-4-line fw-normal"></i> Actions</th>
       </tr>
     </thead>
@@ -26,6 +28,7 @@
           <td>{{ item.fields.null_rgb }}</td>
           <td>{{ item.fields.null_transparent }}</td>
           <td>{{ item.fields.null_url }}</td>
+          <td>{{ item.fields.null_bricklink_color_id + item.fields.null_bricklink_color_name + item.fields.null_bricklink_part_num }}</td>
           <td><a href="{{ item.url_for_refresh() }}" class="btn btn-primary" role="button"><i class="ri-refresh-line"></i> Refresh</a></td>
         </tr>
       {% endfor %}