Update changelog

.env.sample

@@ -2,7 +2,7 @@
 # If set, it will append a direct ORDER BY <whatever you set> to the SQL query
 # while listing objects. You can look at the structure of the SQLite database to
 # see the schema and the column names. Some fields are compound and not visible
-# directly from the schema (joins). You can check the query in the */list.sql files
+# directly from the schema (joins). You can check the query in the */list.sql and */base/*.sql files
 # in the source to see all column names.
 # The usual syntax for those variables is "<table>"."<column>" [ASC|DESC].
 # For composite fields (CASE, SUM, COUNT) the syntax is <field>, there is no <table> name.
@@ -28,7 +28,7 @@
 # BK_AUTHENTICATION_KEY=change-this-to-something-random
 
 # Optional: Pattern of the link to Bricklink for a part. Will be passed to Python .format()
-# Default: https://www.bricklink.com/v2/catalog/catalogitem.page?P={number}
+# Default: https://www.bricklink.com/v2/catalog/catalogitem.page?P={part}
 # BK_BRICKLINK_LINK_PART_PATTERN=
 
 # Optional: Display Bricklink links wherever applicable
@@ -107,9 +107,14 @@
 # Default: false
 # BK_HIDE_ALL_SETS=true
 
-# Optional: Hide the 'Missing' entry from the menu. Does not disable the route.
+# Optional: Hide the 'Problems' entry from the menu. Does not disable the route.
 # Default: false
-# BK_HIDE_MISSING_PARTS=true
+# Legacy name: BK_HIDE_MISSING_PARTS
+# BK_HIDE_PROBLEMS_PARTS=true
+
+# Optional: Hide the 'Instructions' entry in a Set card
+# Default: false
+# BK_HIDE_SET_INSTRUCTIONS=true
 
 # Optional: Hide the 'Wishlist' entry from the menu. Does not disable the route.
 # Default: false
@@ -117,10 +122,11 @@
 
 # Optional: Change the default order of minifigures. By default ordered by insertion order.
 # Useful column names for this option are:
-# - "minifigures"."fig_num": minifigure ID (fig-xxxxx)
-# - "minifigures"."name": minifigure name
-# Default: "minifigures"."name" ASC
-# BK_MINIFIGURES_DEFAULT_ORDER="minifigures"."name" ASC
+# - "rebrickable_minifigures"."figure": minifigure ID (fig-xxxxx)
+# - "rebrickable_minifigures"."number": minifigure ID as an integer (xxxxx)
+# - "rebrickable_minifigures"."name": minifigure name
+# Default: "rebrickable_minifigures"."name" ASC
+# BK_MINIFIGURES_DEFAULT_ORDER="rebrickable_minifigures"."name" ASC
 
 # Optional: Folder where to store the minifigures images, relative to the '/app/static/' folder
 # Default: minifigs
@@ -134,13 +140,13 @@
 
 # Optional: Change the default order of parts. By default ordered by insertion order.
 # Useful column names for this option are:
-# - "inventory"."part_num": part number
-# - "inventory"."name": part name
-# - "inventory"."color_name": part color name
-# - "inventory"."is_spare": par is a spare part
+# - "bricktracker_parts"."part": part number
+# - "bricktracker_parts"."spare": part is a spare part
+# - "rebrickable_parts"."name": part name
+# - "rebrickable_parts"."color_name": part color name
 # - "total_missing": number of missing parts
-# Default: "inventory"."name" ASC, "inventory"."color_name" ASC, "inventory"."is_spare" ASC
-# BK_PARTS_DEFAULT_ORDER="total_missing" DESC, "inventory"."name" ASC
+# Default: "rebrickable_parts"."name" ASC, "rebrickable_parts"."color_name" ASC, "bricktracker_parts"."spare" ASC
+# BK_PARTS_DEFAULT_ORDER="total_missing" DESC, "rebrickable_parts"."name"."name" ASC
 
 # Optional: Folder where to store the parts images, relative to the '/app/static/' folder
 # Default: parts
@@ -171,11 +177,11 @@
 # BK_REBRICKABLE_IMAGE_NIL_MINIFIGURE=
 
 # Optional: Pattern of the link to Rebrickable for a minifigure. Will be passed to Python .format()
-# Default: https://rebrickable.com/minifigs/{number}
+# Default: https://rebrickable.com/minifigs/{figure}
 # BK_REBRICKABLE_LINK_MINIFIGURE_PATTERN=
 
 # Optional: Pattern of the link to Rebrickable for a part. Will be passed to Python .format()
-# Default: https://rebrickable.com/parts/{number}/_/{color}
+# Default: https://rebrickable.com/parts/{part}/_/{color}
 # BK_REBRICKABLE_LINK_PART_PATTERN=
 
 # Optional: Pattern of the link to Rebrickable for instructions. Will be passed to Python .format()
@@ -221,6 +227,10 @@
 # Default: sets
 # BK_SETS_FOLDER=sets
 
+# Optional: Make the grid filters displayed by default, rather than collapsed
+# Default: false
+# BK_SHOW_GRID_FILTERS=true
+
 # Optional: Skip saving or displaying spare parts
 # Default: false
 # BK_SKIP_SPARE_PARTS=true

CHANGELOG.md

@@ -1,5 +1,75 @@
 # Changelog
 
+## Unreleased
+
+> **Warning**
+> "Missing" part has been renamed to "Problems" to accomodate for missing and damaged parts.
+> The associated environment variables have changed named (the old names are still valid)
+
+## Code
+
+- Form
+    - Migrate missing input fields to BrickChanger
+
+- General cleanup
+
+- Metadata
+    - Underlying class to implement more metadata-like features
+
+- Minifigure
+    - Deduplicate
+
+Parts
+    - Damaged parts
+
+- Sets
+    - Refresh data from Rebrickable
+    - Fix missing @login_required for set deletion
+    - Ownership
+    - Tags
+
+- Socket
+    - Add decorator for rebrickable, authenticated and threaded socket actions
+
+- SQL
+    - Allow for advanced migration scenarios through companion python files
+    - Add a bunch of the requested fields into the database for future implementation
+
+### UI
+
+- Add
+    - Allow adding or bulk adding by pressing Enter in the input field
+
+- Admin
+    - Grey out legacy tables in the database view
+    - Checkboxes renamed to Set statuses
+
+- Cards
+    - Use macros for badge in the card header
+
+- Form
+    - Add a clear button for dynamic text inputs
+    - Add error message in a tooltip for dynamic inputs
+
+- Parts
+    - Use Rebrickable URL if stored (+ color code)
+    - Display color and transparency
+    - Display if print of another part
+    - Display prints using the same base
+    - Damaged parts
+
+- Sets
+    - Add a flag to hide instructions in a set
+    - Make checkbox clickable on the whole width of the card
+    - Management
+        - Ownership
+        - Tags
+        - Refresh
+
+- Sets grid
+    - Collapsible controls depending on screen size
+    - Manually collapsible filters (with configuration variable for default state)
+
 ## 1.1.1: PDF Instructions Download
 
 ### Instructions

README.md

@@ -18,7 +18,9 @@ A web application for organizing and tracking LEGO sets, parts, and minifigures.
 
 Use the provided [compose.yaml](compose.yaml) file.
 
-See [setup](docs/setup.md).
+See [Quickstart](docs/quickstart.md) to get up and running right away.
+
+See [Setup](docs/setup.md) for a more setup guide.
 
 ## Usage
 

bricktracker/app.py

@@ -13,11 +13,13 @@ from bricktracker.sql import close
 from bricktracker.version import __version__
 from bricktracker.views.add import add_page
 from bricktracker.views.admin.admin import admin_page
-from bricktracker.views.admin.checkbox import admin_checkbox_page
 from bricktracker.views.admin.database import admin_database_page
 from bricktracker.views.admin.image import admin_image_page
 from bricktracker.views.admin.instructions import admin_instructions_page
+from bricktracker.views.admin.owner import admin_owner_page
 from bricktracker.views.admin.retired import admin_retired_page
+from bricktracker.views.admin.status import admin_status_page
+from bricktracker.views.admin.tag import admin_tag_page
 from bricktracker.views.admin.theme import admin_theme_page
 from bricktracker.views.error import error_404
 from bricktracker.views.index import index_page
@@ -78,11 +80,13 @@ def setup_app(app: Flask) -> None:
 
     # Register admin routes
     app.register_blueprint(admin_page)
-    app.register_blueprint(admin_checkbox_page)
     app.register_blueprint(admin_database_page)
     app.register_blueprint(admin_image_page)
     app.register_blueprint(admin_instructions_page)
     app.register_blueprint(admin_retired_page)
+    app.register_blueprint(admin_owner_page)
+    app.register_blueprint(admin_status_page)
+    app.register_blueprint(admin_tag_page)
     app.register_blueprint(admin_theme_page)
 
     # An helper to make global variables available to the

bricktracker/config.py

@@ -10,7 +10,7 @@ from typing import Any, Final
 CONFIG: Final[list[dict[str, Any]]] = [
     {'n': 'AUTHENTICATION_PASSWORD', 'd': ''},
     {'n': 'AUTHENTICATION_KEY', 'd': ''},
-    {'n': 'BRICKLINK_LINK_PART_PATTERN', 'd': 'https://www.bricklink.com/v2/catalog/catalogitem.page?P={number}'},  # noqa: E501
+    {'n': 'BRICKLINK_LINK_PART_PATTERN', 'd': 'https://www.bricklink.com/v2/catalog/catalogitem.page?P={part}'},  # noqa: E501
     {'n': 'BRICKLINK_LINKS', 'c': bool},
     {'n': 'DATABASE_PATH', 'd': './app.db'},
     {'n': 'DATABASE_TIMESTAMP_FORMAT', 'd': '%Y-%m-%d-%H-%M-%S'},
@@ -29,20 +29,21 @@ CONFIG: Final[list[dict[str, Any]]] = [
     {'n': 'HIDE_ALL_MINIFIGURES', 'c': bool},
     {'n': 'HIDE_ALL_PARTS', 'c': bool},
     {'n': 'HIDE_ALL_SETS', 'c': bool},
-    {'n': 'HIDE_MISSING_PARTS', 'c': bool},
+    {'n': 'HIDE_PROBLEMS_PARTS', 'e': 'BK_HIDE_MISSING_PARTS', 'c': bool},
+    {'n': 'HIDE_SET_INSTRUCTIONS', 'c': bool},
     {'n': 'HIDE_WISHES', 'c': bool},
-    {'n': 'MINIFIGURES_DEFAULT_ORDER', 'd': '"minifigures"."name" ASC'},
+    {'n': 'MINIFIGURES_DEFAULT_ORDER', 'd': '"rebrickable_minifigures"."name" ASC'},  # noqa: E501
     {'n': 'MINIFIGURES_FOLDER', 'd': 'minifigs', 's': True},
     {'n': 'NO_THREADED_SOCKET', 'c': bool},
-    {'n': 'PARTS_DEFAULT_ORDER', 'd': '"inventory"."name" ASC, "inventory"."color_name" ASC, "inventory"."is_spare" ASC'},  # noqa: E501
+    {'n': 'PARTS_DEFAULT_ORDER', 'd': '"rebrickable_parts"."name" ASC, "rebrickable_parts"."color_name" ASC, "bricktracker_parts"."spare" ASC'},  # noqa: E501
     {'n': 'PARTS_FOLDER', 'd': 'parts', 's': True},
     {'n': 'PORT', 'd': 3333, 'c': int},
     {'n': 'RANDOM', 'e': 'RANDOM', 'c': bool},
     {'n': 'REBRICKABLE_API_KEY', 'e': 'REBRICKABLE_API_KEY', 'd': ''},
     {'n': 'REBRICKABLE_IMAGE_NIL', 'd': 'https://rebrickable.com/static/img/nil.png'},  # noqa: E501
     {'n': 'REBRICKABLE_IMAGE_NIL_MINIFIGURE', 'd': 'https://rebrickable.com/static/img/nil_mf.jpg'},  # noqa: E501
-    {'n': 'REBRICKABLE_LINK_MINIFIGURE_PATTERN', 'd': 'https://rebrickable.com/minifigs/{number}'},  # noqa: E501
-    {'n': 'REBRICKABLE_LINK_PART_PATTERN', 'd': 'https://rebrickable.com/parts/{number}/_/{color}'},  # noqa: E501
+    {'n': 'REBRICKABLE_LINK_MINIFIGURE_PATTERN', 'd': 'https://rebrickable.com/minifigs/{figure}'},  # noqa: E501
+    {'n': 'REBRICKABLE_LINK_PART_PATTERN', 'd': 'https://rebrickable.com/parts/{part}/_/{color}'},  # noqa: E501
     {'n': 'REBRICKABLE_LINK_INSTRUCTIONS_PATTERN', 'd': 'https://rebrickable.com/instructions/{path}'},  # noqa: E501
     {'n': 'REBRICKABLE_USER_AGENT', 'd': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'},  # noqa: E501
     {'n': 'REBRICKABLE_LINKS', 'e': 'LINKS', 'c': bool},
@@ -51,6 +52,7 @@ CONFIG: Final[list[dict[str, Any]]] = [
     {'n': 'RETIRED_SETS_PATH', 'd': './retired_sets.csv'},
     {'n': 'SETS_DEFAULT_ORDER', 'd': '"rebrickable_sets"."number" DESC, "rebrickable_sets"."version" ASC'},  # noqa: E501
     {'n': 'SETS_FOLDER', 'd': 'sets', 's': True},
+    {'n': 'SHOW_GRID_FILTERS', 'c': bool},
     {'n': 'SKIP_SPARE_PARTS', 'c': bool},
     {'n': 'SOCKET_NAMESPACE', 'd': 'bricksocket'},
     {'n': 'SOCKET_PATH', 'd': '/bricksocket/'},

bricktracker/metadata.py

@@ -0,0 +1,218 @@
+import logging
+from sqlite3 import Row
+from typing import Any, Self, TYPE_CHECKING
+from uuid import uuid4
+
+from flask import url_for
+
+from .exceptions import DatabaseException, ErrorException, NotFoundException
+from .record import BrickRecord
+from .sql import BrickSQL
+if TYPE_CHECKING:
+    from .set import BrickSet
+
+logger = logging.getLogger(__name__)
+
+
+# Lego set metadata (customizable list of entries that can be checked)
+class BrickMetadata(BrickRecord):
+    kind: str
+
+    # Set state endpoint
+    set_state_endpoint: str
+
+    # Queries
+    delete_query: str
+    insert_query: str
+    select_query: str
+    update_field_query: str
+    update_set_state_query: str
+
+    def __init__(
+        self,
+        /,
+        *,
+        record: Row | dict[str, Any] | None = None,
+    ):
+        super().__init__()
+
+        # Ingest the record if it has one
+        if record is not None:
+            self.ingest(record)
+
+    # SQL column name
+    def as_column(self, /) -> str:
+        return '{kind}_{id}'.format(
+            id=self.fields.id,
+            kind=self.kind.lower()
+        )
+
+    # HTML dataset name
+    def as_dataset(self, /) -> str:
+        return '{id}'.format(
+            id=self.as_column().replace('_', '-')
+        )
+
+    # Delete from database
+    def delete(self, /) -> None:
+        BrickSQL().executescript(
+            self.delete_query,
+            id=self.fields.id,
+        )
+
+    # Grab data from a form
+    def from_form(self, form: dict[str, str], /) -> Self:
+        name = form.get('name', None)
+
+        if name is None or name == '':
+            raise ErrorException('Status name cannot be empty')
+
+        self.fields.name = name
+
+        return self
+
+    # Insert into database
+    def insert(self, /, **context) -> None:
+        self.safe()
+
+        # Generate an ID for the metadata (with underscores to make it
+        # column name friendly)
+        self.fields.id = str(uuid4()).replace('-', '_')
+
+        BrickSQL().executescript(
+            self.insert_query,
+            id=self.fields.id,
+            name=self.fields.safe_name,
+            **context
+        )
+
+    # Rename the entry
+    def rename(self, /) -> None:
+        self.safe()
+
+        self.update_field('name', value=self.fields.name)
+
+    # Make the name "safe"
+    # Security: eh.
+    def safe(self, /) -> None:
+        # Prevent self-ownage with accidental quote escape
+        self.fields.safe_name = self.fields.name.replace("'", "''")
+
+    # URL to change the selected state of this metadata item for a set
+    def url_for_set_state(self, id: str, /) -> str:
+        return url_for(
+            self.set_state_endpoint,
+            id=id,
+            metadata_id=self.fields.id
+        )
+
+    # Select a specific metadata (with an id)
+    def select_specific(self, id: str, /) -> Self:
+        # Save the parameters to the fields
+        self.fields.id = id
+
+        # Load from database
+        if not self.select():
+            raise NotFoundException(
+                '{kind} with ID {id} was not found in the database'.format(
+                    kind=self.kind.capitalize(),
+                    id=self.fields.id,
+                ),
+            )
+
+        return self
+
+    # Update a field
+    def update_field(
+        self,
+        field: str,
+        /,
+        *,
+        json: Any | None = None,
+        value: Any | None = None
+    ) -> Any:
+        if value is None:
+            value = json.get('value', None)  # type: ignore
+
+        if value is None:
+            raise ErrorException('"{field}" of a {kind} cannot be set to an empty value'.format(  # noqa: E501
+                field=field,
+                kind=self.kind
+            ))
+
+        if field == 'id' or not hasattr(self.fields, field):
+            raise NotFoundException('"{field}" is not a field of a {kind}'.format(  # noqa: E501
+                kind=self.kind,
+                field=field
+            ))
+
+        parameters = self.sql_parameters()
+        parameters['value'] = value
+
+        # Update the status
+        rows, _ = BrickSQL().execute_and_commit(
+            self.update_field_query,
+            parameters=parameters,
+            field=field,
+        )
+
+        if rows != 1:
+            raise DatabaseException('Could not update the field "{field}" for {kind} {name} ({id})'.format(  # noqa: E501
+                field=field,
+                kind=self.kind,
+                name=self.fields.name,
+                id=self.fields.id,
+            ))
+
+        # Info
+        logger.info('{kind} "{name}" ({id}): field "{field}" changed to "{value}"'.format(  # noqa: E501
+            kind=self.kind.capitalize(),
+            name=self.fields.name,
+            id=self.fields.id,
+            field=field,
+            value=value,
+        ))
+
+        return value
+
+    # Update the selected state of this metadata item for a set
+    def update_set_state(
+        self,
+        brickset: 'BrickSet',
+        /,
+        *,
+        json: Any | None = None,
+        state: bool | None = None,
+    ) -> Any:
+        if state is None:
+            state = json.get('value', False)  # type: ignore
+
+        parameters = self.sql_parameters()
+        parameters['set_id'] = brickset.fields.id
+        parameters['state'] = state
+
+        # Update the status
+        rows, _ = BrickSQL().execute_and_commit(
+            self.update_set_state_query,
+            parameters=parameters,
+            name=self.as_column(),
+        )
+
+        if rows != 1:
+            raise DatabaseException('Could not update the {kind} "{name}" state for set {set} ({id})'.format(  # noqa: E501
+                kind=self.kind,
+                name=self.fields.name,
+                set=brickset.fields.set,
+                id=brickset.fields.id,
+            ))
+
+        # Info
+        logger.info('{kind} "{name}" state change to "{state}" for set {set} ({id})'.format(  # noqa: E501
+            kind=self.kind,
+            name=self.fields.name,
+            state=state,
+            set=brickset.fields.set,
+            id=brickset.fields.id,
+        ))
+
+        return state

bricktracker/metadata_list.py

@@ -0,0 +1,79 @@
+import logging
+from typing import Type, TypeVar
+
+from .exceptions import NotFoundException
+from .fields import BrickRecordFields
+from .record_list import BrickRecordList
+from .set_owner import BrickSetOwner
+from .set_status import BrickSetStatus
+from .set_tag import BrickSetTag
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar('T', BrickSetStatus, BrickSetOwner, BrickSetTag)
+
+
+# Lego sets metadata list
+class BrickMetadataList(BrickRecordList[T]):
+    kind: str
+    mapping: dict[str, T]
+    model: Type[T]
+
+    # Database table
+    table: str
+
+    # Queries
+    select_query: str
+
+    def __init__(self, model: Type[T], /, *, force: bool = False):
+        # Load statuses only if there is none already loaded
+        records = getattr(self, 'records', None)
+
+        if records is None or force:
+            # Don't use super()__init__ as it would mask class variables
+            self.fields = BrickRecordFields()
+
+            logger.info('Loading {kind} list'.format(
+                kind=self.kind
+            ))
+
+            self.__class__.records = []
+            self.__class__.mapping = {}
+
+            # Load the statuses from the database
+            for record in self.select():
+                status = model(record=record)
+
+                self.__class__.records.append(status)
+                self.__class__.mapping[status.fields.id] = status
+
+    # Return the items as columns for a select
+    def as_columns(self, /, **kwargs) -> str:
+        return ', '.join([
+            '"{table}"."{column}"'.format(
+                table=self.table,
+                column=record.as_column(),
+            )
+            for record
+            in self.filter(**kwargs)
+        ])
+
+    # Filter the list of records (this one does nothing)
+    def filter(self) -> list[T]:
+        return self.records
+
+    # Grab a specific status
+    def get(self, id: str, /) -> T:
+        if id not in self.mapping:
+            raise NotFoundException(
+                '{kind} with ID {id} was not found in the database'.format(
+                    kind=self.kind.capitalize(),
+                    id=id,
+                ),
+            )
+
+        return self.mapping[id]
+
+    # Get the list of statuses depending on the context
+    def list(self, /, **kwargs) -> list[T]:
+        return self.filter(**kwargs)

bricktracker/migrations/0007.py

@@ -0,0 +1,29 @@
+from typing import Any, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..sql import BrickSQL
+
+
+# Grab the list of checkboxes to create a list of SQL columns
+def migration_0007(sql: 'BrickSQL', /) -> dict[str, Any]:
+    # Don't realy on sql files as they could be removed in the future
+    sql.cursor.execute('SELECT "bricktracker_set_checkboxes"."id" FROM "bricktracker_set_checkboxes"')  # noqa: E501
+    records = sql.cursor.fetchall()
+
+    return {
+        'sources': ', '.join([
+            '"bricktracker_set_statuses_old"."status_{id}"'.format(id=record['id'])  # noqa: E501
+            for record
+            in records
+        ]),
+        'targets': ', '.join([
+            '"status_{id}"'.format(id=record['id'])
+            for record
+            in records
+        ]),
+        'structure': ', '.join([
+            '"status_{id}" BOOLEAN NOT NULL DEFAULT 0'.format(id=record['id'])
+            for record
+            in records
+        ])
+    }

bricktracker/minifigure.py

@@ -1,48 +1,68 @@
-from sqlite3 import Row
-from typing import Any, Self, TYPE_CHECKING
-
-from flask import current_app, url_for
+import logging
+import traceback
+from typing import Self, TYPE_CHECKING
 
 from .exceptions import ErrorException, NotFoundException
 from .part_list import BrickPartList
-from .rebrickable_image import RebrickableImage
-from .record import BrickRecord
+from .rebrickable_minifigure import RebrickableMinifigure
 if TYPE_CHECKING:
     from .set import BrickSet
+    from .socket import BrickSocket
+
+logger = logging.getLogger(__name__)
 
 
 # Lego minifigure
-class BrickMinifigure(BrickRecord):
-    brickset: 'BrickSet | None'
-
+class BrickMinifigure(RebrickableMinifigure):
     # Queries
     insert_query: str = 'minifigure/insert'
     generic_query: str = 'minifigure/select/generic'
     select_query: str = 'minifigure/select/specific'
 
-    def __init__(
-        self,
-        /,
-        *,
-        brickset: 'BrickSet | None' = None,
-        record: Row | dict[str, Any] | None = None,
-    ):
-        super().__init__()
+    # Import a minifigure into the database
+    def download(self, socket: 'BrickSocket', refresh: bool = False) -> bool:
+        if self.brickset is None:
+            raise ErrorException('Importing a minifigure from Rebrickable outside of a set is not supported')  # noqa: E501
 
-        # Save the brickset
-        self.brickset = brickset
+        try:
+            # Insert into the database
+            socket.auto_progress(
+                message='Set {set}: inserting minifigure {figure} into database'.format(  # noqa: E501
+                    set=self.brickset.fields.set,
+                    figure=self.fields.figure
+                )
+            )
 
-        # Ingest the record if it has one
-        if record is not None:
-            self.ingest(record)
+            if not refresh:
+                # Insert into database
+                self.insert(commit=False)
 
-    # Return the number just in digits format
-    def clean_number(self, /) -> str:
-        number: str = self.fields.fig_num
-        number = number.removeprefix('fig-')
-        number = number.lstrip('0')
+            # Insert the rebrickable set into database
+            self.insert_rebrickable()
 
-        return number
+            # Load the inventory
+            if not BrickPartList.download(
+                socket,
+                self.brickset,
+                minifigure=self,
+                refresh=refresh
+            ):
+                return False
+
+        except Exception as e:
+            socket.fail(
+                message='Error while importing minifigure {figure} from {set}: {error}'.format(  # noqa: E501
+                    figure=self.fields.figure,
+                    set=self.brickset.fields.set,
+                    error=e,
+                )
+            )
+
+            logger.debug(traceback.format_exc())
+
+            return False
+
+        return True
 
     # Parts
     def generic_parts(self, /) -> BrickPartList:
@@ -51,108 +71,38 @@ class BrickMinifigure(BrickRecord):
     # Parts
     def parts(self, /) -> BrickPartList:
         if self.brickset is None:
-            raise ErrorException('Part list for minifigure {number} requires a brickset'.format(  # noqa: E501
-                number=self.fields.fig_num,
+            raise ErrorException('Part list for minifigure {figure} requires a brickset'.format(  # noqa: E501
+                figure=self.fields.figure,
             ))
 
-        return BrickPartList().load(self.brickset, minifigure=self)
+        return BrickPartList().list_specific(self.brickset, minifigure=self)
 
     # Select a generic minifigure
-    def select_generic(self, fig_num: str, /) -> Self:
+    def select_generic(self, figure: str, /) -> Self:
         # Save the parameters to the fields
-        self.fields.fig_num = fig_num
+        self.fields.figure = figure
 
         if not self.select(override_query=self.generic_query):
             raise NotFoundException(
-                'Minifigure with number {number} was not found in the database'.format(  # noqa: E501
-                    number=self.fields.fig_num,
+                'Minifigure with figure {figure} was not found in the database'.format(  # noqa: E501
+                    figure=self.fields.figure,
                 ),
             )
 
         return self
 
-    # Select a specific minifigure (with a set and an number)
-    def select_specific(self, brickset: 'BrickSet', fig_num: str, /) -> Self:
+    # Select a specific minifigure (with a set and a figure)
+    def select_specific(self, brickset: 'BrickSet', figure: str, /) -> Self:
         # Save the parameters to the fields
         self.brickset = brickset
-        self.fields.fig_num = fig_num
+        self.fields.figure = figure
 
         if not self.select():
             raise NotFoundException(
-                'Minifigure with number {number} from set {set} was not found in the database'.format(  # noqa: E501
-                    number=self.fields.fig_num,
+                'Minifigure with figure {figure} from set {set} was not found in the database'.format(  # noqa: E501
+                    figure=self.fields.figure,
                     set=self.brickset.fields.set,
                 ),
             )
 
         return self
-
-    # Return a dict with common SQL parameters for a minifigure
-    def sql_parameters(self, /) -> dict[str, Any]:
-        parameters = super().sql_parameters()
-
-        # Supplement from the brickset
-        if self.brickset is not None:
-            if 'u_id' not in parameters:
-                parameters['u_id'] = self.brickset.fields.id
-
-            if 'set_num' not in parameters:
-                parameters['set_num'] = self.brickset.fields.set
-
-        return parameters
-
-    # Self url
-    def url(self, /) -> str:
-        return url_for(
-            'minifigure.details',
-            number=self.fields.fig_num,
-        )
-
-    # Compute the url for minifigure part image
-    def url_for_image(self, /) -> str:
-        if not current_app.config['USE_REMOTE_IMAGES']:
-            if self.fields.set_img_url is None:
-                file = RebrickableImage.nil_minifigure_name()
-            else:
-                file = self.fields.fig_num
-
-            return RebrickableImage.static_url(file, 'MINIFIGURES_FOLDER')
-        else:
-            if self.fields.set_img_url is None:
-                return current_app.config['REBRICKABLE_IMAGE_NIL_MINIFIGURE']
-            else:
-                return self.fields.set_img_url
-
-    # Compute the url for the rebrickable page
-    def url_for_rebrickable(self, /) -> str:
-        if current_app.config['REBRICKABLE_LINKS']:
-            try:
-                return current_app.config['REBRICKABLE_LINK_MINIFIGURE_PATTERN'].format(  # noqa: E501
-                    number=self.fields.fig_num.lower(),
-                )
-            except Exception:
-                pass
-
-        return ''
-
-    # Normalize from Rebrickable
-    @staticmethod
-    def from_rebrickable(
-        data: dict[str, Any],
-        /,
-        *,
-        brickset: 'BrickSet | None' = None,
-        **_,
-    ) -> dict[str, Any]:
-        record = {
-            'fig_num': data['set_num'],
-            'name': data['set_name'],
-            'quantity': data['quantity'],
-            'set_img_url': data['set_img_url'],
-        }
-
-        if brickset is not None:
-            record['set_num'] = brickset.fields.set
-            record['u_id'] = brickset.fields.id
-
-        return record

bricktracker/minifigure_list.py

@@ -1,11 +1,17 @@
+import logging
+import traceback
 from typing import Any, Self, TYPE_CHECKING
 
 from flask import current_app
 
 from .minifigure import BrickMinifigure
+from .rebrickable import Rebrickable
 from .record_list import BrickRecordList
 if TYPE_CHECKING:
     from .set import BrickSet
+    from .socket import BrickSocket
+
+logger = logging.getLogger(__name__)
 
 
 # Lego minifigures
@@ -15,10 +21,11 @@ class BrickMinifigureList(BrickRecordList[BrickMinifigure]):
 
     # Queries
     all_query: str = 'minifigure/list/all'
+    damaged_part_query: str = 'minifigure/list/damaged_part'
     last_query: str = 'minifigure/list/last'
+    missing_part_query: str = 'minifigure/list/missing_part'
     select_query: str = 'minifigure/list/from_set'
     using_part_query: str = 'minifigure/list/using_part'
-    missing_part_query: str = 'minifigure/list/missing_part'
 
     def __init__(self, /):
         super().__init__()
@@ -41,13 +48,30 @@ class BrickMinifigureList(BrickRecordList[BrickMinifigure]):
 
         return self
 
+    # Minifigures with a part damaged part
+    def damaged_part(self, part: str, color: int, /) -> Self:
+        # Save the parameters to the fields
+        self.fields.part = part
+        self.fields.color = color
+
+        # Load the minifigures from the database
+        for record in self.select(
+            override_query=self.damaged_part_query,
+            order=self.order
+        ):
+            minifigure = BrickMinifigure(record=record)
+
+            self.records.append(minifigure)
+
+        return self
+
     # Last added minifigure
     def last(self, /, *, limit: int = 6) -> Self:
         # Randomize
         if current_app.config['RANDOM']:
             order = 'RANDOM()'
         else:
-            order = 'minifigures.rowid DESC'
+            order = '"bricktracker_minifigures"."rowid" DESC'
 
         for record in self.select(
             override_query=self.last_query,
@@ -61,7 +85,7 @@ class BrickMinifigureList(BrickRecordList[BrickMinifigure]):
         return self
 
     # Load minifigures from a brickset
-    def load(self, brickset: 'BrickSet', /) -> Self:
+    def from_set(self, brickset: 'BrickSet', /) -> Self:
         # Save the brickset
         self.brickset = brickset
 
@@ -73,29 +97,11 @@ class BrickMinifigureList(BrickRecordList[BrickMinifigure]):
 
         return self
 
-    # Return a dict with common SQL parameters for a minifigures list
-    def sql_parameters(self, /) -> dict[str, Any]:
-        parameters: dict[str, Any] = super().sql_parameters()
-
-        if self.brickset is not None:
-            parameters['u_id'] = self.brickset.fields.id
-            parameters['set_num'] = self.brickset.fields.set
-
-        return parameters
-
     # Minifigures missing a part
-    def missing_part(
-        self,
-        part_num: str,
-        color_id: int,
-        /,
-        *,
-        element_id: int | None = None,
-    ) -> Self:
+    def missing_part(self, part: str, color: int, /) -> Self:
         # Save the parameters to the fields
-        self.fields.part_num = part_num
-        self.fields.color_id = color_id
-        self.fields.element_id = element_id
+        self.fields.part = part
+        self.fields.color = color
 
         # Load the minifigures from the database
         for record in self.select(
@@ -109,18 +115,10 @@ class BrickMinifigureList(BrickRecordList[BrickMinifigure]):
         return self
 
     # Minifigure using a part
-    def using_part(
-        self,
-        part_num: str,
-        color_id: int,
-        /,
-        *,
-        element_id: int | None = None,
-    ) -> Self:
+    def using_part(self, part: str, color: int, /) -> Self:
         # Save the parameters to the fields
-        self.fields.part_num = part_num
-        self.fields.color_id = color_id
-        self.fields.element_id = element_id
+        self.fields.part = part
+        self.fields.color = color
 
         # Load the minifigures from the database
         for record in self.select(
@@ -132,3 +130,60 @@ class BrickMinifigureList(BrickRecordList[BrickMinifigure]):
             self.records.append(minifigure)
 
         return self
+
+    # Return a dict with common SQL parameters for a minifigures list
+    def sql_parameters(self, /) -> dict[str, Any]:
+        parameters: dict[str, Any] = super().sql_parameters()
+
+        if self.brickset is not None:
+            parameters['id'] = self.brickset.fields.id
+
+        return parameters
+
+    # Import the minifigures from Rebrickable
+    @staticmethod
+    def download(
+        socket: 'BrickSocket',
+        brickset: 'BrickSet',
+        /,
+        *,
+        refresh: bool = False
+    ) -> bool:
+        try:
+            socket.auto_progress(
+                message='Set {set}: loading minifigures from Rebrickable'.format(  # noqa: E501
+                    set=brickset.fields.set,
+                ),
+                increment_total=True,
+            )
+
+            logger.debug('rebrick.lego.get_set_minifigs("{set}")'.format(
+                set=brickset.fields.set,
+            ))
+
+            minifigures = Rebrickable[BrickMinifigure](
+                'get_set_minifigs',
+                brickset.fields.set,
+                BrickMinifigure,
+                socket=socket,
+                brickset=brickset,
+            ).list()
+
+            # Process each minifigure
+            for minifigure in minifigures:
+                if not minifigure.download(socket, refresh=refresh):
+                    return False
+
+            return True
+
+        except Exception as e:
+            socket.fail(
+                message='Error while importing set {set} minifigure list: {error}'.format(  # noqa: E501
+                    set=brickset.fields.set,
+                    error=e,
+                )
+            )
+
+            logger.debug(traceback.format_exc())
+
+            return False

bricktracker/navbar.py

@@ -11,7 +11,7 @@ NAVBAR: Final[list[dict[str, Any]]] = [
     {'e': 'set.list', 't': 'Sets', 'i': 'grid-line', 'f': 'HIDE_ALL_SETS'},  # noqa: E501
     {'e': 'add.add', 't': 'Add', 'i': 'add-circle-line', 'f': 'HIDE_ADD_SET'},  # noqa: E501
     {'e': 'part.list', 't': 'Parts', 'i': 'shapes-line', 'f': 'HIDE_ALL_PARTS'},  # noqa: E501
-    {'e': 'part.missing', 't': 'Missing', 'i': 'error-warning-line', 'f': 'HIDE_MISSING_PARTS'},  # noqa: E501
+    {'e': 'part.problem', 't': 'Problems', 'i': 'error-warning-line', 'f': 'HIDE_PROBLEMS_PARTS'},  # noqa: E501
     {'e': 'minifigure.list', 't': 'Minifigures', 'i': 'group-line', 'f': 'HIDE_ALL_MINIFIGURES'},  # noqa: E501
     {'e': 'instructions.list', 't': 'Instructions', 'i': 'file-line', 'f': 'HIDE_ALL_INSTRUCTIONS'},  # noqa: E501
     {'e': 'wish.list', 't': 'Wishlist', 'i': 'gift-line', 'f': 'HIDE_WISHES'},

bricktracker/part.py

@@ -1,103 +1,110 @@
-import os
-from sqlite3 import Row
+import logging
 from typing import Any, Self, TYPE_CHECKING
-from urllib.parse import urlparse
+import traceback
 
-from flask import current_app, url_for
+from flask import url_for
 
-from .exceptions import DatabaseException, ErrorException, NotFoundException
-from .rebrickable_image import RebrickableImage
-from .record import BrickRecord
+from .exceptions import ErrorException, NotFoundException
+from .rebrickable_part import RebrickablePart
 from .sql import BrickSQL
 if TYPE_CHECKING:
     from .minifigure import BrickMinifigure
     from .set import BrickSet
+    from .socket import BrickSocket
+
+logger = logging.getLogger(__name__)
 
 
 # Lego set or minifig part
-class BrickPart(BrickRecord):
-    brickset: 'BrickSet | None'
-    minifigure: 'BrickMinifigure | None'
+class BrickPart(RebrickablePart):
+    identifier: str
+    kind: str
 
     # Queries
     insert_query: str = 'part/insert'
     generic_query: str = 'part/select/generic'
     select_query: str = 'part/select/specific'
 
-    def __init__(
-        self,
-        /,
-        *,
-        brickset: 'BrickSet | None' = None,
-        minifigure: 'BrickMinifigure | None' = None,
-        record: Row | dict[str, Any] | None = None,
-    ):
-        super().__init__()
+    def __init__(self, /, **kwargs):
+        super().__init__(**kwargs)
 
-        # Save the brickset and minifigure
-        self.brickset = brickset
-        self.minifigure = minifigure
+        if self.minifigure is not None:
+            self.identifier = self.minifigure.fields.figure
+            self.kind = 'Minifigure'
+        elif self.brickset is not None:
+            self.identifier = self.brickset.fields.set
+            self.kind = 'Set'
 
-        # Ingest the record if it has one
-        if record is not None:
-            self.ingest(record)
+    # Import a part into the database
+    def download(self, socket: 'BrickSocket', refresh: bool = False) -> bool:
+        if self.brickset is None:
+            raise ErrorException('Importing a part from Rebrickable outside of a set is not supported')  # noqa: E501
 
-    # Delete missing part
-    def delete_missing(self, /) -> None:
-        BrickSQL().execute_and_commit(
-            'missing/delete/from_set',
-            parameters=self.sql_parameters()
-        )
-
-    # Set missing part
-    def set_missing(self, quantity: int, /) -> None:
-        parameters = self.sql_parameters()
-        parameters['quantity'] = quantity
-
-        # Can't use UPSERT because the database has no keys
-        # Try to update
-        database = BrickSQL()
-        rows, _ = database.execute(
-            'missing/update/from_set',
-            parameters=parameters,
-        )
-
-        # Insert if no row has been affected
-        if not rows:
-            rows, _ = database.execute(
-                'missing/insert',
-                parameters=parameters,
+        try:
+            # Insert into the database
+            socket.auto_progress(
+                message='{kind} {identifier}: inserting part {part} into database'.format(  # noqa: E501
+                    kind=self.kind,
+                    identifier=self.identifier,
+                    part=self.fields.part
+                )
             )
 
-            if rows != 1:
-                raise DatabaseException(
-                    'Could not update the missing quantity for part {id}'.format(  # noqa: E501
-                        id=self.fields.id
-                    )
-                )
+            if not refresh:
+                # Insert into database
+                self.insert(commit=False)
 
-        database.commit()
+            # Insert the rebrickable set into database
+            self.insert_rebrickable()
+
+        except Exception as e:
+            socket.fail(
+                message='Error while importing part {part} from {kind} {identifier}: {error}'.format(  # noqa: E501
+                    part=self.fields.part,
+                    kind=self.kind,
+                    identifier=self.identifier,
+                    error=e,
+                )
+            )
+
+            logger.debug(traceback.format_exc())
+
+            return False
+
+        return True
+
+    # A identifier for HTML component
+    def html_id(self, prefix: str | None = None, /) -> str:
+        components: list[str] = ['part']
+
+        if prefix is not None:
+            components.append(prefix)
+
+        if self.fields.figure is not None:
+            components.append(self.fields.figure)
+
+        components.append(self.fields.part)
+        components.append(str(self.fields.color))
+        components.append(str(self.fields.spare))
+
+        return '-'.join(components)
 
     # Select a generic part
     def select_generic(
         self,
-        part_num: str,
-        color_id: int,
+        part: str,
+        color: int,
         /,
-        *,
-        element_id: int | None = None
     ) -> Self:
         # Save the parameters to the fields
-        self.fields.part_num = part_num
-        self.fields.color_id = color_id
-        self.fields.element_id = element_id
+        self.fields.part = part
+        self.fields.color = color
 
         if not self.select(override_query=self.generic_query):
             raise NotFoundException(
-                'Part with number {number}, color ID {color} and element ID {element} was not found in the database'.format(  # noqa: E501
-                    number=self.fields.part_num,
-                    color=self.fields.color_id,
-                    element=self.fields.element_id,
+                'Part with number {number}, color ID {color} was not found in the database'.format(  # noqa: E501
+                    number=self.fields.part,
+                    color=self.fields.color,
                 ),
             )
 
@@ -107,7 +114,9 @@ class BrickPart(BrickRecord):
     def select_specific(
         self,
         brickset: 'BrickSet',
-        id: str,
+        part: str,
+        color: int,
+        spare: int,
         /,
         *,
         minifigure: 'BrickMinifigure | None' = None,
@@ -115,168 +124,73 @@ class BrickPart(BrickRecord):
         # Save the parameters to the fields
         self.brickset = brickset
         self.minifigure = minifigure
-        self.fields.id = id
+        self.fields.part = part
+        self.fields.color = color
+        self.fields.spare = spare
 
         if not self.select():
+            if self.minifigure is not None:
+                figure = self.minifigure.fields.figure
+            else:
+                figure = None
+
             raise NotFoundException(
-                'Part with ID {id} from set {set} was not found in the database'.format(  # noqa: E501
+                'Part {part} with color {color} (spare: {spare}) from set {set} ({id}) (minifigure: {figure}) was not found in the database'.format(  # noqa: E501
+                    part=self.fields.part,
+                    color=self.fields.color,
+                    spare=self.fields.spare,
                     id=self.fields.id,
                     set=self.brickset.fields.set,
+                    figure=figure,
                 ),
             )
 
         return self
 
-    # Return a dict with common SQL parameters for a part
-    def sql_parameters(self, /) -> dict[str, Any]:
-        parameters = super().sql_parameters()
+    # Update a problematic part
+    def update_problem(self, problem: str, json: Any | None, /) -> int:
+        amount: str | int = json.get('value', '')  # type: ignore
 
-        # Supplement from the brickset
-        if 'u_id' not in parameters and self.brickset is not None:
-            parameters['u_id'] = self.brickset.fields.id
+        # We need a positive integer
+        try:
+            if amount == '':
+                amount = 0
 
-        if 'set_num' not in parameters:
-            if self.minifigure is not None:
-                parameters['set_num'] = self.minifigure.fields.fig_num
+            amount = int(amount)
 
-            elif self.brickset is not None:
-                parameters['set_num'] = self.brickset.fields.set
+            if amount < 0:
+                amount = 0
+        except Exception:
+            raise ErrorException('"{amount}" is not a valid integer'.format(
+                amount=amount
+            ))
 
-        return parameters
+        if amount < 0:
+            raise ErrorException('Cannot set a negative amount')
 
-    # Update the missing part
-    def update_missing(self, missing: Any, /) -> None:
-        # If empty, delete it
-        if missing == '':
-            self.delete_missing()
+        setattr(self.fields, problem, amount)
 
-        else:
-            # Try to understand it as a number
-            try:
-                missing = int(missing)
-            except Exception:
-                raise ErrorException('"{missing}" is not a valid integer'.format(  # noqa: E501
-                    missing=missing
-                ))
-
-            # If 0, delete it
-            if missing == 0:
-                self.delete_missing()
-
-            else:
-                # If negative, it's an error
-                if missing < 0:
-                    raise ErrorException('Cannot set a negative missing value')
-
-                # Otherwise upsert it
-                # Not checking if it is too much, you do you
-                self.set_missing(missing)
-
-    # Self url
-    def url(self, /) -> str:
-        return url_for(
-            'part.details',
-            number=self.fields.part_num,
-            color=self.fields.color_id,
-            element=self.fields.element_id,
+        BrickSQL().execute_and_commit(
+            'part/update/{problem}'.format(problem=problem),
+            parameters=self.sql_parameters()
         )
 
-    # Compute the url for the bricklink page
-    def url_for_bricklink(self, /) -> str:
-        if current_app.config['BRICKLINK_LINKS']:
-            try:
-                return current_app.config['BRICKLINK_LINK_PART_PATTERN'].format(  # noqa: E501
-                    number=self.fields.part_num,
-                )
-            except Exception:
-                pass
+        return amount
 
-        return ''
-
-    # Compute the url for the part image
-    def url_for_image(self, /) -> str:
-        if not current_app.config['USE_REMOTE_IMAGES']:
-            if self.fields.part_img_url is None:
-                file = RebrickableImage.nil_name()
-            else:
-                file = self.fields.part_img_url_id
-
-            return RebrickableImage.static_url(file, 'PARTS_FOLDER')
-        else:
-            if self.fields.part_img_url is None:
-                return current_app.config['REBRICKABLE_IMAGE_NIL']
-            else:
-                return self.fields.part_img_url
-
-    # Compute the url for missing part
-    def url_for_missing(self, /) -> str:
+    # Compute the url for problematic part
+    def url_for_problem(self, problem: str, /) -> str:
         # Different URL for a minifigure part
         if self.minifigure is not None:
-            return url_for(
-                'set.missing_minifigure_part',
-                id=self.fields.u_id,
-                minifigure_id=self.minifigure.fields.fig_num,
-                part_id=self.fields.id,
-            )
+            figure = self.minifigure.fields.figure
+        else:
+            figure = None
 
         return url_for(
-            'set.missing_part',
-            id=self.fields.u_id,
-            part_id=self.fields.id
+            'set.problem_part',
+            id=self.fields.id,
+            figure=figure,
+            part=self.fields.part,
+            color=self.fields.color,
+            spare=self.fields.spare,
+            problem=problem,
         )
-
-    # Compute the url for the rebrickable page
-    def url_for_rebrickable(self, /) -> str:
-        if current_app.config['REBRICKABLE_LINKS']:
-            try:
-                return current_app.config['REBRICKABLE_LINK_PART_PATTERN'].format(  # noqa: E501
-                    number=self.fields.part_num,
-                    color=self.fields.color_id,
-                )
-            except Exception:
-                pass
-
-        return ''
-
-    # Normalize from Rebrickable
-    @staticmethod
-    def from_rebrickable(
-        data: dict[str, Any],
-        /,
-        *,
-        brickset: 'BrickSet | None' = None,
-        minifigure: 'BrickMinifigure | None' = None,
-        **_,
-    ) -> dict[str, Any]:
-        record = {
-            'set_num': data['set_num'],
-            'id': data['id'],
-            'part_num': data['part']['part_num'],
-            'name': data['part']['name'],
-            'part_img_url': data['part']['part_img_url'],
-            'part_img_url_id': None,
-            'color_id': data['color']['id'],
-            'color_name': data['color']['name'],
-            'quantity': data['quantity'],
-            'is_spare': data['is_spare'],
-            'element_id': data['element_id'],
-        }
-
-        if brickset is not None:
-            record['u_id'] = brickset.fields.id
-
-        if minifigure is not None:
-            record['set_num'] = data['fig_num']
-
-        # Extract the file name
-        if data['part']['part_img_url'] is not None:
-            part_img_url_file = os.path.basename(
-                urlparse(data['part']['part_img_url']).path
-            )
-
-            part_img_url_id, _ = os.path.splitext(part_img_url_file)
-
-            if part_img_url_id is not None or part_img_url_id != '':
-                record['part_img_url_id'] = part_img_url_id
-
-        return record

bricktracker/part_list.py

@@ -1,12 +1,18 @@
+import logging
 from typing import Any, Self, TYPE_CHECKING
+import traceback
 
 from flask import current_app
 
 from .part import BrickPart
+from .rebrickable import Rebrickable
 from .record_list import BrickRecordList
 if TYPE_CHECKING:
     from .minifigure import BrickMinifigure
     from .set import BrickSet
+    from .socket import BrickSocket
+
+logger = logging.getLogger(__name__)
 
 
 # Lego set or minifig parts
@@ -19,8 +25,9 @@ class BrickPartList(BrickRecordList[BrickPart]):
     all_query: str = 'part/list/all'
     last_query: str = 'part/list/last'
     minifigure_query: str = 'part/list/from_minifigure'
-    missing_query: str = 'part/list/missing'
-    select_query: str = 'part/list/from_set'
+    problem_query: str = 'part/list/problem'
+    print_query: str = 'part/list/from_print'
+    select_query: str = 'part/list/specific'
 
     def __init__(self, /):
         super().__init__()
@@ -44,8 +51,8 @@ class BrickPartList(BrickRecordList[BrickPart]):
 
         return self
 
-    # Load parts from a brickset or minifigure
-    def load(
+    # List specific parts from a brickset or minifigure
+    def list_specific(
         self,
         brickset: 'BrickSet',
         /,
@@ -64,7 +71,7 @@ class BrickPartList(BrickRecordList[BrickPart]):
                 record=record,
             )
 
-            if current_app.config['SKIP_SPARE_PARTS'] and part.fields.is_spare:
+            if current_app.config['SKIP_SPARE_PARTS'] and part.fields.spare:
                 continue
 
             self.records.append(part)
@@ -90,17 +97,51 @@ class BrickPartList(BrickRecordList[BrickPart]):
                 record=record,
             )
 
-            if current_app.config['SKIP_SPARE_PARTS'] and part.fields.is_spare:
+            if current_app.config['SKIP_SPARE_PARTS'] and part.fields.spare:
                 continue
 
             self.records.append(part)
 
         return self
 
-    # Load missing parts
-    def missing(self, /) -> Self:
+    # Load generic parts from a print
+    def from_print(
+        self,
+        brickpart: BrickPart,
+        /,
+    ) -> Self:
+        # Save the part and print
+        if brickpart.fields.print is not None:
+            self.fields.print = brickpart.fields.print
+        else:
+            self.fields.print = brickpart.fields.part
+
+        self.fields.part = brickpart.fields.part
+        self.fields.color = brickpart.fields.color
+
+        # Load the parts from the database
         for record in self.select(
-            override_query=self.missing_query,
+            override_query=self.print_query,
+            order=self.order
+        ):
+            part = BrickPart(
+                record=record,
+            )
+
+            if (
+                current_app.config['SKIP_SPARE_PARTS'] and
+                part.fields.spare
+            ):
+                continue
+
+            self.records.append(part)
+
+        return self
+
+    # Load problematic parts
+    def problem(self, /) -> Self:
+        for record in self.select(
+            override_query=self.problem_query,
             order=self.order
         ):
             part = BrickPart(record=record)
@@ -111,17 +152,78 @@ class BrickPartList(BrickRecordList[BrickPart]):
 
     # Return a dict with common SQL parameters for a parts list
     def sql_parameters(self, /) -> dict[str, Any]:
-        parameters: dict[str, Any] = {}
+        parameters: dict[str, Any] = super().sql_parameters()
 
         # Set id
         if self.brickset is not None:
-            parameters['u_id'] = self.brickset.fields.id
+            parameters['id'] = self.brickset.fields.id
 
         # Use the minifigure number if present,
-        # otherwise use the set number
         if self.minifigure is not None:
-            parameters['set_num'] = self.minifigure.fields.fig_num
-        elif self.brickset is not None:
-            parameters['set_num'] = self.brickset.fields.set
+            parameters['figure'] = self.minifigure.fields.figure
+        else:
+            parameters['figure'] = None
 
         return parameters
+
+    # Import the parts from Rebrickable
+    @staticmethod
+    def download(
+        socket: 'BrickSocket',
+        brickset: 'BrickSet',
+        /,
+        *,
+        minifigure: 'BrickMinifigure | None' = None,
+        refresh: bool = False
+    ) -> bool:
+        if minifigure is not None:
+            identifier = minifigure.fields.figure
+            kind = 'Minifigure'
+            method = 'get_minifig_elements'
+        else:
+            identifier = brickset.fields.set
+            kind = 'Set'
+            method = 'get_set_elements'
+
+        try:
+            socket.auto_progress(
+                message='{kind} {identifier}: loading parts inventory from Rebrickable'.format(  # noqa: E501
+                    kind=kind,
+                    identifier=identifier,
+                ),
+                increment_total=True,
+            )
+
+            logger.debug('rebrick.lego.{method}("{identifier}")'.format(
+                method=method,
+                identifier=identifier,
+            ))
+
+            inventory = Rebrickable[BrickPart](
+                method,
+                identifier,
+                BrickPart,
+                socket=socket,
+                brickset=brickset,
+                minifigure=minifigure,
+            ).list()
+
+            # Process each part
+            for part in inventory:
+                if not part.download(socket, refresh=refresh):
+                    return False
+
+        except Exception as e:
+            socket.fail(
+                message='Error while importing {kind} {identifier} parts list: {error}'.format(  # noqa: E501
+                    kind=kind,
+                    identifier=identifier,
+                    error=e,
+                )
+            )
+
+            logger.debug(traceback.format_exc())
+
+            return False
+
+        return True

bricktracker/rebrickable_image.py

@@ -8,16 +8,16 @@ from shutil import copyfileobj
 
 from .exceptions import DownloadException
 if TYPE_CHECKING:
-    from .minifigure import BrickMinifigure
-    from .part import BrickPart
+    from .rebrickable_minifigure import RebrickableMinifigure
+    from .rebrickable_part import RebrickablePart
     from .rebrickable_set import RebrickableSet
 
 
 # A set, part or minifigure image from Rebrickable
 class RebrickableImage(object):
     set: 'RebrickableSet'
-    minifigure: 'BrickMinifigure | None'
-    part: 'BrickPart | None'
+    minifigure: 'RebrickableMinifigure | None'
+    part: 'RebrickablePart | None'
 
     extension: str | None
 
@@ -26,8 +26,8 @@ class RebrickableImage(object):
         set: 'RebrickableSet',
         /,
         *,
-        minifigure: 'BrickMinifigure | None' = None,
-        part: 'BrickPart | None' = None,
+        minifigure: 'RebrickableMinifigure | None' = None,
+        part: 'RebrickablePart | None' = None,
     ):
         # Save all objects
         self.set = set
@@ -81,16 +81,16 @@ class RebrickableImage(object):
     # Return the id depending on the objects provided
     def id(self, /) -> str:
         if self.part is not None:
-            if self.part.fields.part_img_url_id is None:
+            if self.part.fields.image_id is None:
                 return RebrickableImage.nil_name()
             else:
-                return self.part.fields.part_img_url_id
+                return self.part.fields.image_id
 
         if self.minifigure is not None:
-            if self.minifigure.fields.set_img_url is None:
+            if self.minifigure.fields.image is None:
                 return RebrickableImage.nil_minifigure_name()
             else:
-                return self.minifigure.fields.fig_num
+                return self.minifigure.fields.figure
 
         return self.set.fields.set
 
@@ -105,16 +105,16 @@ class RebrickableImage(object):
     # Return the url depending on the objects provided
     def url(self, /) -> str:
         if self.part is not None:
-            if self.part.fields.part_img_url is None:
+            if self.part.fields.image is None:
                 return current_app.config['REBRICKABLE_IMAGE_NIL']
             else:
-                return self.part.fields.part_img_url
+                return self.part.fields.image
 
         if self.minifigure is not None:
-            if self.minifigure.fields.set_img_url is None:
+            if self.minifigure.fields.image is None:
                 return current_app.config['REBRICKABLE_IMAGE_NIL_MINIFIGURE']
             else:
-                return self.minifigure.fields.set_img_url
+                return self.minifigure.fields.image
 
         return self.set.fields.image
 

bricktracker/rebrickable_minifigure.py

@@ -0,0 +1,114 @@
+import logging
+from sqlite3 import Row
+from typing import Any, TYPE_CHECKING
+
+from flask import current_app, url_for
+
+from .exceptions import ErrorException
+from .rebrickable_image import RebrickableImage
+from .record import BrickRecord
+if TYPE_CHECKING:
+    from .set import BrickSet
+
+logger = logging.getLogger(__name__)
+
+
+# A minifigure from Rebrickable
+class RebrickableMinifigure(BrickRecord):
+    brickset: 'BrickSet | None'
+
+    # Queries
+    select_query: str = 'rebrickable/minifigure/select'
+    insert_query: str = 'rebrickable/minifigure/insert'
+
+    def __init__(
+        self,
+        /,
+        *,
+        brickset: 'BrickSet | None' = None,
+        record: Row | dict[str, Any] | None = None
+    ):
+        super().__init__()
+
+        # Save the brickset
+        self.brickset = brickset
+
+        # Ingest the record if it has one
+        if record is not None:
+            self.ingest(record)
+
+    # Insert the minifigure from Rebrickable
+    def insert_rebrickable(self, /) -> None:
+        if self.brickset is None:
+            raise ErrorException('Importing a minifigure from Rebrickable outside of a set is not supported')  # noqa: E501
+
+        # Insert the Rebrickable minifigure to the database
+        self.insert(
+            commit=False,
+            no_defer=True,
+            override_query=RebrickableMinifigure.insert_query
+        )
+
+        if not current_app.config['USE_REMOTE_IMAGES']:
+            RebrickableImage(
+                self.brickset,
+                minifigure=self,
+            ).download()
+
+    # Return a dict with common SQL parameters for a minifigure
+    def sql_parameters(self, /) -> dict[str, Any]:
+        parameters = super().sql_parameters()
+
+        # Supplement from the brickset
+        if self.brickset is not None and 'id' not in parameters:
+            parameters['id'] = self.brickset.fields.id
+
+        return parameters
+
+    # Self url
+    def url(self, /) -> str:
+        return url_for(
+            'minifigure.details',
+            figure=self.fields.figure,
+        )
+
+    # Compute the url for minifigure image
+    def url_for_image(self, /) -> str:
+        if not current_app.config['USE_REMOTE_IMAGES']:
+            if self.fields.image is None:
+                file = RebrickableImage.nil_minifigure_name()
+            else:
+                file = self.fields.figure
+
+            return RebrickableImage.static_url(file, 'MINIFIGURES_FOLDER')
+        else:
+            if self.fields.image is None:
+                return current_app.config['REBRICKABLE_IMAGE_NIL_MINIFIGURE']
+            else:
+                return self.fields.image
+
+    # Compute the url for the rebrickable page
+    def url_for_rebrickable(self, /) -> str:
+        if current_app.config['REBRICKABLE_LINKS']:
+            try:
+                return current_app.config['REBRICKABLE_LINK_MINIFIGURE_PATTERN'].format(  # noqa: E501
+                    number=self.fields.figure,
+                )
+            except Exception:
+                pass
+
+        return ''
+
+    # Normalize from Rebrickable
+    @staticmethod
+    def from_rebrickable(data: dict[str, Any], /, **_) -> dict[str, Any]:
+        # Extracting  number
+        number = int(str(data['set_num'])[5:])
+
+        return {
+            'figure': str(data['set_num']),
+            'number': int(number),
+            'name': str(data['set_name']),
+            'quantity': int(data['quantity']),
+            'image': data['set_img_url'],
+        }

bricktracker/rebrickable_minifigures.py

@@ -1,85 +0,0 @@
-import logging
-from typing import TYPE_CHECKING
-
-from flask import current_app
-
-from .minifigure import BrickMinifigure
-from .rebrickable import Rebrickable
-from .rebrickable_image import RebrickableImage
-from .rebrickable_parts import RebrickableParts
-if TYPE_CHECKING:
-    from .set import BrickSet
-    from .socket import BrickSocket
-
-logger = logging.getLogger(__name__)
-
-
-# Minifigures from Rebrickable
-class RebrickableMinifigures(object):
-    socket: 'BrickSocket'
-    brickset: 'BrickSet'
-
-    def __init__(self, socket: 'BrickSocket', brickset: 'BrickSet', /):
-        # Save the socket
-        self.socket = socket
-
-        # Save the objects
-        self.brickset = brickset
-
-    # Import the minifigures from Rebrickable
-    def download(self, /) -> None:
-        self.socket.auto_progress(
-            message='Set {number}: loading minifigures from Rebrickable'.format(  # noqa: E501
-                number=self.brickset.fields.set,
-            ),
-            increment_total=True,
-        )
-
-        logger.debug('rebrick.lego.get_set_minifigs("{set}")'.format(
-            set=self.brickset.fields.set,
-        ))
-
-        minifigures = Rebrickable[BrickMinifigure](
-            'get_set_minifigs',
-            self.brickset.fields.set,
-            BrickMinifigure,
-            socket=self.socket,
-            brickset=self.brickset,
-        ).list()
-
-        # Process each minifigure
-        total = len(minifigures)
-        for index, minifigure in enumerate(minifigures):
-            # Insert into the database
-            self.socket.auto_progress(
-                message='Set {number}: inserting minifigure {current}/{total} into database'.format(  # noqa: E501
-                    number=self.brickset.fields.set,
-                    current=index+1,
-                    total=total,
-                )
-            )
-
-            # Insert into database
-            minifigure.insert(commit=False)
-
-            # Grab the image
-            self.socket.progress(
-                message='Set {number}: downloading minifigure {current}/{total} image'.format(  # noqa: E501
-                    number=self.brickset.fields.set,
-                    current=index+1,
-                    total=total,
-                )
-            )
-
-            if not current_app.config['USE_REMOTE_IMAGES']:
-                RebrickableImage(
-                    self.brickset,
-                    minifigure=minifigure
-                ).download()
-
-            # Load the inventory
-            RebrickableParts(
-                self.socket,
-                self.brickset,
-                minifigure=minifigure,
-            ).download()

bricktracker/rebrickable_part.py

@@ -0,0 +1,199 @@
+import os
+import logging
+from sqlite3 import Row
+from typing import Any, TYPE_CHECKING
+from urllib.parse import urlparse
+
+from flask import current_app, url_for
+
+from .exceptions import ErrorException
+from .rebrickable_image import RebrickableImage
+from .record import BrickRecord
+if TYPE_CHECKING:
+    from .minifigure import BrickMinifigure
+    from .set import BrickSet
+    from .socket import BrickSocket
+
+logger = logging.getLogger(__name__)
+
+
+# A part from Rebrickable
+class RebrickablePart(BrickRecord):
+    socket: 'BrickSocket'
+    brickset: 'BrickSet | None'
+    minifigure: 'BrickMinifigure | None'
+
+    # Queries
+    select_query: str = 'rebrickable/part/select'
+    insert_query: str = 'rebrickable/part/insert'
+
+    def __init__(
+        self,
+        /,
+        *,
+        brickset: 'BrickSet | None' = None,
+        minifigure: 'BrickMinifigure | None' = None,
+        record: Row | dict[str, Any] | None = None
+    ):
+        super().__init__()
+
+        # Save the brickset
+        self.brickset = brickset
+
+        # Save the minifigure
+        self.minifigure = minifigure
+
+        # Ingest the record if it has one
+        if record is not None:
+            self.ingest(record)
+
+    # Insert the part from Rebrickable
+    def insert_rebrickable(self, /) -> None:
+        if self.brickset is None:
+            raise ErrorException('Importing a part from Rebrickable outside of a set is not supported')  # noqa: E501
+
+        # Insert the Rebrickable part to the database
+        self.insert(
+            commit=False,
+            no_defer=True,
+            override_query=RebrickablePart.insert_query
+        )
+
+        if not current_app.config['USE_REMOTE_IMAGES']:
+            RebrickableImage(
+                self.brickset,
+                minifigure=self.minifigure,
+                part=self,
+            ).download()
+
+    # Return a dict with common SQL parameters for a part
+    def sql_parameters(self, /) -> dict[str, Any]:
+        parameters = super().sql_parameters()
+
+        # Set id
+        if self.brickset is not None:
+            parameters['id'] = self.brickset.fields.id
+
+        # Use the minifigure number if present,
+        if self.minifigure is not None:
+            parameters['figure'] = self.minifigure.fields.figure
+        else:
+            parameters['figure'] = None
+
+        return parameters
+
+    # Self url
+    def url(self, /) -> str:
+        return url_for(
+            'part.details',
+            part=self.fields.part,
+            color=self.fields.color,
+        )
+
+    # Compute the url for the bricklink page
+    def url_for_bricklink(self, /) -> str:
+        if current_app.config['BRICKLINK_LINKS']:
+            try:
+                return current_app.config['BRICKLINK_LINK_PART_PATTERN'].format(  # noqa: E501
+                    part=self.fields.part,
+                )
+            except Exception:
+                pass
+
+        return ''
+
+    # Compute the url for the part image
+    def url_for_image(self, /) -> str:
+        if not current_app.config['USE_REMOTE_IMAGES']:
+            if self.fields.image is None:
+                file = RebrickableImage.nil_name()
+            else:
+                file = self.fields.image_id
+
+            return RebrickableImage.static_url(file, 'PARTS_FOLDER')
+        else:
+            if self.fields.image is None:
+                return current_app.config['REBRICKABLE_IMAGE_NIL']
+            else:
+                return self.fields.image
+
+    # Compute the url for the original of the printed part
+    def url_for_print(self, /) -> str:
+        if self.fields.print is not None:
+            return url_for(
+                'part.details',
+                part=self.fields.print,
+                color=self.fields.color,
+            )
+        else:
+            return ''
+
+    # Compute the url for the rebrickable page
+    def url_for_rebrickable(self, /) -> str:
+        if current_app.config['REBRICKABLE_LINKS']:
+            try:
+                if self.fields.url is not None:
+                    # The URL does not contain color info...
+                    return '{url}{color}'.format(
+                        url=self.fields.url,
+                        color=self.fields.color
+                    )
+                else:
+                    return current_app.config['REBRICKABLE_LINK_PART_PATTERN'].format(  # noqa: E501
+                        part=self.fields.part,
+                        color=self.fields.color,
+                    )
+            except Exception:
+                pass
+
+        return ''
+
+    # Normalize from Rebrickable
+    @staticmethod
+    def from_rebrickable(
+        data: dict[str, Any],
+        /,
+        *,
+        brickset: 'BrickSet | None' = None,
+        minifigure: 'BrickMinifigure | None' = None,
+        **_,
+    ) -> dict[str, Any]:
+        record = {
+            'id': None,
+            'figure': None,
+            'part': data['part']['part_num'],
+            'color': data['color']['id'],
+            'spare': data['is_spare'],
+            'quantity': data['quantity'],
+            'rebrickable_inventory': data['id'],
+            'element': data['element_id'],
+            'color_id': data['color']['id'],
+            'color_name': data['color']['name'],
+            'color_rgb': data['color']['rgb'],
+            'color_transparent': data['color']['is_trans'],
+            'name': data['part']['name'],
+            'category': data['part']['part_cat_id'],
+            'image': data['part']['part_img_url'],
+            'image_id': None,
+            'url': data['part']['part_url'],
+            'print': data['part']['print_of']
+        }
+
+        if brickset is not None:
+            record['id'] = brickset.fields.id
+
+        if minifigure is not None:
+            record['figure'] = minifigure.fields.figure
+
+        # Extract the file name
+        if record['image'] is not None:
+            image_id, _ = os.path.splitext(
+                os.path.basename(
+                    urlparse(record['image']).path
+                )
+            )
+
+            if image_id is not None or image_id != '':
+                record['image_id'] = image_id
+
+        return record

bricktracker/rebrickable_parts.py

@@ -1,113 +0,0 @@
-import logging
-from typing import TYPE_CHECKING
-
-from flask import current_app
-
-from .part import BrickPart
-from .rebrickable import Rebrickable
-from .rebrickable_image import RebrickableImage
-if TYPE_CHECKING:
-    from .minifigure import BrickMinifigure
-    from .set import BrickSet
-    from .socket import BrickSocket
-
-logger = logging.getLogger(__name__)
-
-
-# A list of parts from Rebrickable
-class RebrickableParts(object):
-    socket: 'BrickSocket'
-    brickset: 'BrickSet'
-    minifigure: 'BrickMinifigure | None'
-
-    number: str
-    kind: str
-    method: str
-
-    def __init__(
-        self,
-        socket: 'BrickSocket',
-        brickset: 'BrickSet',
-        /,
-        *,
-        minifigure: 'BrickMinifigure | None' = None,
-    ):
-        # Save the socket
-        self.socket = socket
-
-        # Save the objects
-        self.brickset = brickset
-        self.minifigure = minifigure
-
-        if self.minifigure is not None:
-            self.number = self.minifigure.fields.fig_num
-            self.kind = 'Minifigure'
-            self.method = 'get_minifig_elements'
-        else:
-            self.number = self.brickset.fields.set
-            self.kind = 'Set'
-            self.method = 'get_set_elements'
-
-    # Import the parts from Rebrickable
-    def download(self, /) -> None:
-        self.socket.auto_progress(
-            message='{kind} {number}: loading parts inventory from Rebrickable'.format(  # noqa: E501
-                kind=self.kind,
-                number=self.number,
-            ),
-            increment_total=True,
-        )
-
-        logger.debug('rebrick.lego.{method}("{number}")'.format(
-            method=self.method,
-            number=self.number,
-        ))
-
-        inventory = Rebrickable[BrickPart](
-            self.method,
-            self.number,
-            BrickPart,
-            socket=self.socket,
-            brickset=self.brickset,
-            minifigure=self.minifigure,
-        ).list()
-
-        # Process each part
-        total = len(inventory)
-        for index, part in enumerate(inventory):
-            # Skip spare parts
-            if (
-                current_app.config['SKIP_SPARE_PARTS'] and
-                part.fields.is_spare
-            ):
-                continue
-
-            # Insert into the database
-            self.socket.auto_progress(
-                message='{kind} {number}: inserting part {current}/{total} into database'.format(  # noqa: E501
-                    kind=self.kind,
-                    number=self.number,
-                    current=index+1,
-                    total=total,
-                )
-            )
-
-            # Insert into database
-            part.insert(commit=False)
-
-            # Grab the image
-            self.socket.progress(
-                message='{kind} {number}: downloading part {current}/{total} image'.format(  # noqa: E501
-                    kind=self.kind,
-                    number=self.number,
-                    current=index+1,
-                    total=total,
-                )
-            )
-
-            if not current_app.config['USE_REMOTE_IMAGES']:
-                RebrickableImage(
-                    self.brickset,
-                    minifigure=self.minifigure,
-                    part=part,
-                ).download()

bricktracker/rebrickable_set.py

@@ -21,7 +21,6 @@ logger = logging.getLogger(__name__)
 
 # A set from Rebrickable
 class RebrickableSet(BrickRecord):
-    socket: 'BrickSocket'
     theme: 'BrickTheme'
     instructions: list[BrickInstructions]
 
@@ -36,7 +35,6 @@ class RebrickableSet(BrickRecord):
         self,
         /,
         *,
-        socket: 'BrickSocket | None' = None,
         record: Row | dict[str, Any] | None = None
     ):
         super().__init__()
@@ -44,26 +42,21 @@ class RebrickableSet(BrickRecord):
         # Placeholders
         self.instructions = []
 
-        # Save the socket
-        if socket is not None:
-            self.socket = socket
-
         # Ingest the record if it has one
         if record is not None:
             self.ingest(record)
 
-    # Import the set from Rebrickable
-    def download_rebrickable(self, /) -> None:
+    # Insert the set from Rebrickable
+    def insert_rebrickable(self, /) -> None:
         # Insert the Rebrickable set to the database
-        rows, _ = self.insert(
+        self.insert(
             commit=False,
             no_defer=True,
             override_query=RebrickableSet.insert_query
         )
 
-        if rows > 0:
-            if not current_app.config['USE_REMOTE_IMAGES']:
-                RebrickableImage(self).download()
+        if not current_app.config['USE_REMOTE_IMAGES']:
+            RebrickableImage(self).download()
 
     # Ingest a set
     def ingest(self, record: Row | dict[str, Any], /):
@@ -88,20 +81,21 @@ class RebrickableSet(BrickRecord):
     # Load the set from Rebrickable
     def load(
         self,
+        socket: 'BrickSocket',
         data: dict[str, Any],
         /,
         *,
         from_download=False,
     ) -> bool:
         # Reset the progress
-        self.socket.progress_count = 0
-        self.socket.progress_total = 2
+        socket.progress_count = 0
+        socket.progress_total = 2
 
         try:
-            self.socket.auto_progress(message='Parsing set number')
+            socket.auto_progress(message='Parsing set number')
             set = parse_set(str(data['set']))
 
-            self.socket.auto_progress(
+            socket.auto_progress(
                 message='Set {set}: loading from Rebrickable'.format(
                     set=set,
                 ),
@@ -118,12 +112,12 @@ class RebrickableSet(BrickRecord):
                 instance=self,
             ).get()
 
-            self.socket.emit('SET_LOADED', self.short(
+            socket.emit('SET_LOADED', self.short(
                 from_download=from_download
             ))
 
             if not from_download:
-                self.socket.complete(
+                socket.complete(
                     message='Set {set}: loaded from Rebrickable'.format(
                         set=self.fields.set
                     )
@@ -132,7 +126,7 @@ class RebrickableSet(BrickRecord):
             return True
 
         except Exception as e:
-            self.socket.fail(
+            socket.fail(
                 message='Could not load the set from Rebrickable: {error}. Data: {data}'.format(  # noqa: E501
                     error=str(e),
                     data=data,

bricktracker/record.py

@@ -1,5 +1,5 @@
 from sqlite3 import Row
-from typing import Any, ItemsView, Tuple
+from typing import Any, ItemsView
 
 from .fields import BrickRecordFields
 from .sql import BrickSQL
@@ -31,14 +31,14 @@ class BrickRecord(object):
         commit=True,
         no_defer=False,
         override_query: str | None = None
-    ) -> Tuple[int, str]:
+    ) -> None:
         if override_query:
             query = override_query
         else:
             query = self.insert_query
 
         database = BrickSQL()
-        rows, q = database.execute(
+        database.execute(
             query,
             parameters=self.sql_parameters(),
             defer=not commit and not no_defer,
@@ -47,8 +47,6 @@ class BrickRecord(object):
         if commit:
             database.commit()
 
-        return rows, q
-
     # Shorthand to field items
     def items(self, /) -> ItemsView[str, Any]:
         return self.fields.__dict__.items()

bricktracker/record_list.py

@@ -8,13 +8,17 @@ if TYPE_CHECKING:
     from .part import BrickPart
     from .rebrickable_set import RebrickableSet
     from .set import BrickSet
-    from .set_checkbox import BrickSetCheckbox
+    from .set_owner import BrickSetOwner
+    from .set_status import BrickSetStatus
+    from .set_tag import BrickSetTag
     from .wish import BrickWish
 
 T = TypeVar(
     'T',
     'BrickSet',
-    'BrickSetCheckbox',
+    'BrickSetOwner',
+    'BrickSetStatus',
+    'BrickSetTag',
     'BrickPart',
     'BrickMinifigure',
     'BrickWish',

bricktracker/reload.py

@@ -1,6 +1,11 @@
 from .instructions_list import BrickInstructionsList
 from .retired_list import BrickRetiredList
-from .set_checkbox_list import BrickSetCheckboxList
+from .set_owner import BrickSetOwner
+from .set_owner_list import BrickSetOwnerList
+from .set_status import BrickSetStatus
+from .set_status_list import BrickSetStatusList
+from .set_tag import BrickSetTag
+from .set_tag_list import BrickSetTagList
 from .theme_list import BrickThemeList
 
 
@@ -11,8 +16,14 @@ def reload() -> None:
         # Reload the instructions
         BrickInstructionsList(force=True)
 
-        # Reload the checkboxes
-        BrickSetCheckboxList(force=True)
+        # Reload the set owners
+        BrickSetOwnerList(BrickSetOwner, force=True)
+
+        # Reload the set statuses
+        BrickSetStatusList(BrickSetStatus, force=True)
+
+        # Reload the set tags
+        BrickSetTagList(BrickSetTag, force=True)
 
         # Reload retired sets
         BrickRetiredList(force=True)

bricktracker/set.py

@@ -1,19 +1,23 @@
 import logging
 import traceback
-from typing import Any, Self
+from typing import Any, Self, TYPE_CHECKING
 from uuid import uuid4
 
-from flask import url_for
+from flask import current_app, url_for
 
-from .exceptions import DatabaseException, NotFoundException
+from .exceptions import NotFoundException
 from .minifigure_list import BrickMinifigureList
 from .part_list import BrickPartList
-from .rebrickable_minifigures import RebrickableMinifigures
-from .rebrickable_parts import RebrickableParts
 from .rebrickable_set import RebrickableSet
-from .set_checkbox import BrickSetCheckbox
-from .set_checkbox_list import BrickSetCheckboxList
+from .set_owner import BrickSetOwner
+from .set_owner_list import BrickSetOwnerList
+from .set_status import BrickSetStatus
+from .set_status_list import BrickSetStatusList
+from .set_tag import BrickSetTag
+from .set_tag_list import BrickSetTagList
 from .sql import BrickSQL
+if TYPE_CHECKING:
+    from .socket import BrickSocket
 
 logger = logging.getLogger(__name__)
 
@@ -33,81 +37,115 @@ class BrickSet(RebrickableSet):
         )
 
     # Import a set into the database
-    def download(self, data: dict[str, Any], /) -> None:
+    def download(self, socket: 'BrickSocket', data: dict[str, Any], /) -> bool:
         # Load the set
-        if not self.load(data, from_download=True):
-            return
+        if not self.load(socket, data, from_download=True):
+            return False
 
         try:
             # Insert into the database
-            self.socket.auto_progress(
-                message='Set {number}: inserting into database'.format(
-                    number=self.fields.set
+            socket.auto_progress(
+                message='Set {set}: inserting into database'.format(
+                    set=self.fields.set
                 ),
                 increment_total=True,
             )
 
+            # Grabbing the refresh flag
+            refresh: bool = bool(data.get('refresh', False))
+
             # Generate an UUID for self
             self.fields.id = str(uuid4())
 
-            # Insert into database
-            self.insert(commit=False)
+            if not refresh:
+                # Insert into database
+                self.insert(commit=False)
 
-            # Execute the parent download method
-            self.download_rebrickable()
+            # Insert the rebrickable set into database
+            self.insert_rebrickable()
 
             # Load the inventory
-            RebrickableParts(self.socket, self).download()
+            if not BrickPartList.download(socket, self, refresh=refresh):
+                return False
 
             # Load the minifigures
-            RebrickableMinifigures(self.socket, self).download()
+            if not BrickMinifigureList.download(socket, self, refresh=refresh):
+                return False
+
+            # Save the owners
+            owners: list[str] = list(data.get('owners', []))
+
+            for id in owners:
+                owner = BrickSetOwnerList(BrickSetOwner).get(id)
+                owner.update_set_state(self, state=True)
+
+            # Save the tags
+            tags: list[str] = list(data.get('tags', []))
+
+            for id in tags:
+                tag = BrickSetTagList(BrickSetTag).get(id)
+                tag.update_set_state(self, state=True)
 
             # Commit the transaction to the database
-            self.socket.auto_progress(
-                message='Set {number}: writing to the database'.format(
-                    number=self.fields.set
+            socket.auto_progress(
+                message='Set {set}: writing to the database'.format(
+                    set=self.fields.set
                 ),
                 increment_total=True,
             )
 
             BrickSQL().commit()
 
-            # Info
-            logger.info('Set {number}: imported (id: {id})'.format(
-                number=self.fields.set,
-                id=self.fields.id,
-            ))
+            if refresh:
+                # Info
+                logger.info('Set {set}: imported (id: {id})'.format(
+                    set=self.fields.set,
+                    id=self.fields.id,
+                ))
 
-            # Complete
-            self.socket.complete(
-                message='Set {number}: imported (<a href="{url}">Go to the set</a>)'.format(  # noqa: E501
-                    number=self.fields.set,
-                    url=self.url()
-                ),
-                download=True
-            )
+                # Complete
+                socket.complete(
+                    message='Set {set}: refreshed'.format(  # noqa: E501
+                        set=self.fields.set,
+                    ),
+                    download=True
+                )
+            else:
+                # Info
+                logger.info('Set {set}: refreshed'.format(
+                    set=self.fields.set,
+                ))
+
+                # Complete
+                socket.complete(
+                    message='Set {set}: imported (<a href="{url}">Go to the set</a>)'.format(  # noqa: E501
+                        set=self.fields.set,
+                        url=self.url()
+                    ),
+                    download=True
+                )
 
         except Exception as e:
-            self.socket.fail(
-                message='Error while importing set {number}: {error}'.format(
-                    number=self.fields.set,
+            socket.fail(
+                message='Error while importing set {set}: {error}'.format(
+                    set=self.fields.set,
                     error=e,
                 )
             )
 
             logger.debug(traceback.format_exc())
 
-    # Insert a Rebrickable set
-    def insert_rebrickable(self, /) -> None:
-        self.insert()
+            return False
+
+        return True
 
     # Minifigures
     def minifigures(self, /) -> BrickMinifigureList:
-        return BrickMinifigureList().load(self)
+        return BrickMinifigureList().from_set(self)
 
     # Parts
     def parts(self, /) -> BrickPartList:
-        return BrickPartList().load(self)
+        return BrickPartList().list_specific(self)
 
     # Select a light set (with an id)
     def select_light(self, id: str, /) -> Self:
@@ -131,7 +169,9 @@ class BrickSet(RebrickableSet):
 
         # Load from database
         if not self.select(
-            statuses=BrickSetCheckboxList().as_columns(solo=True)
+            owners=BrickSetOwnerList(BrickSetOwner).as_columns(),
+            statuses=BrickSetStatusList(BrickSetStatus).as_columns(all=True),
+            tags=BrickSetTagList(BrickSetTag).as_columns(),
         ):
             raise NotFoundException(
                 'Set with ID {id} was not found in the database'.format(
@@ -141,30 +181,6 @@ class BrickSet(RebrickableSet):
 
         return self
 
-    # Update a status
-    def update_status(
-        self,
-        checkbox: BrickSetCheckbox,
-        status: bool,
-        /
-    ) -> None:
-        parameters = self.sql_parameters()
-        parameters['status'] = status
-
-        # Update the status
-        rows, _ = BrickSQL().execute_and_commit(
-            'set/update/status',
-            parameters=parameters,
-            name=checkbox.as_column(),
-        )
-
-        if rows != 1:
-            raise DatabaseException('Could not update the status "{status}" for set {number} ({id})'.format(  # noqa: E501
-                status=checkbox.fields.name,
-                number=self.fields.set,
-                id=self.fields.id,
-            ))
-
     # Self url
     def url(self, /) -> str:
         return url_for('set.details', id=self.fields.id)
@@ -179,7 +195,10 @@ class BrickSet(RebrickableSet):
 
     # Compute the url for the set instructions
     def url_for_instructions(self, /) -> str:
-        if len(self.instructions):
+        if (
+            not current_app.config['HIDE_SET_INSTRUCTIONS'] and
+            len(self.instructions)
+        ):
             return url_for(
                 'set.details',
                 id=self.fields.id,
@@ -187,3 +206,10 @@ class BrickSet(RebrickableSet):
             )
         else:
             return ''
+
+    # Compute the url for the refresh button
+    def url_for_refresh(self, /) -> str:
+        return url_for(
+            'set.refresh',
+            id=self.fields.id,
+        )

bricktracker/set_checkbox.py

@@ -1,142 +0,0 @@
-from sqlite3 import Row
-from typing import Any, Self, Tuple
-from uuid import uuid4
-
-from flask import url_for
-
-from .exceptions import DatabaseException, ErrorException, NotFoundException
-from .record import BrickRecord
-from .sql import BrickSQL
-
-
-# Lego set checkbox
-class BrickSetCheckbox(BrickRecord):
-    # Queries
-    select_query: str = 'checkbox/select'
-
-    def __init__(
-        self,
-        /,
-        *,
-        record: Row | dict[str, Any] | None = None,
-    ):
-        super().__init__()
-
-        # Ingest the record if it has one
-        if record is not None:
-            self.ingest(record)
-
-    # SQL column name
-    def as_column(self) -> str:
-        return 'status_{id}'.format(id=self.fields.id)
-
-    # HTML dataset name
-    def as_dataset(self) -> str:
-        return '{id}'.format(
-            id=self.as_column().replace('_', '-')
-        )
-
-    # Delete from database
-    def delete(self) -> None:
-        BrickSQL().executescript(
-            'checkbox/delete',
-            id=self.fields.id,
-        )
-
-    # Grab data from a form
-    def from_form(self, form: dict[str, str]) -> Self:
-        name = form.get('name', None)
-        grid = form.get('grid', None)
-
-        if name is None or name == '':
-            raise ErrorException('Checkbox name cannot be empty')
-
-        # Security: eh.
-        # Prevent self-ownage with accidental quote escape
-        self.fields.name = name
-        self.fields.safe_name = self.fields.name.replace("'", "''")
-        self.fields.displayed_on_grid = grid == 'on'
-
-        return self
-
-    # Insert into database
-    def insert(self, **_) -> Tuple[int, str]:
-        # Generate an ID for the checkbox (with underscores to make it
-        # column name friendly)
-        self.fields.id = str(uuid4()).replace('-', '_')
-
-        BrickSQL().executescript(
-            'checkbox/add',
-            id=self.fields.id,
-            name=self.fields.safe_name,
-            displayed_on_grid=self.fields.displayed_on_grid
-        )
-
-        # To accomodate the parent().insert we have overriden
-        return 0, ''
-
-    # Rename the checkbox
-    def rename(self, /) -> None:
-        # Update the name
-        rows, _ = BrickSQL().execute_and_commit(
-            'checkbox/update/name',
-            parameters=self.sql_parameters(),
-        )
-
-        if rows != 1:
-            raise DatabaseException('Could not update the name for checkbox {name} ({id})'.format(  # noqa: E501
-                name=self.fields.name,
-                id=self.fields.id,
-            ))
-
-    # URL to change the status
-    def status_url(self, id: str) -> str:
-        return url_for(
-            'set.update_status',
-            id=id,
-            checkbox_id=self.fields.id
-        )
-
-    # Select a specific checkbox (with an id)
-    def select_specific(self, id: str, /) -> Self:
-        # Save the parameters to the fields
-        self.fields.id = id
-
-        # Load from database
-        if not self.select():
-            raise NotFoundException(
-                'Checkbox with ID {id} was not found in the database'.format(
-                    id=self.fields.id,
-                ),
-            )
-
-        return self
-
-    # Update a status
-    def update_status(
-        self,
-        name: str,
-        status: bool,
-        /
-    ) -> None:
-        if not hasattr(self.fields, name) or name in ['id', 'name']:
-            raise NotFoundException('{name} is not a field of a checkbox'.format(  # noqa: E501
-                name=name
-            ))
-
-        parameters = self.sql_parameters()
-        parameters['status'] = status
-
-        # Update the status
-        rows, _ = BrickSQL().execute_and_commit(
-            'checkbox/update/status',
-            parameters=parameters,
-            name=name,
-        )
-
-        if rows != 1:
-            raise DatabaseException('Could not update the status "{status}" for checkbox {name} ({id})'.format(  # noqa: E501
-                status=name,
-                name=self.fields.name,
-                id=self.fields.id,
-            ))

bricktracker/set_checkbox_list.py

@@ -1,74 +0,0 @@
-import logging
-
-from .exceptions import NotFoundException
-from .fields import BrickRecordFields
-from .record_list import BrickRecordList
-from .set_checkbox import BrickSetCheckbox
-
-logger = logging.getLogger(__name__)
-
-
-# Lego sets checkbox list
-class BrickSetCheckboxList(BrickRecordList[BrickSetCheckbox]):
-    checkboxes: dict[str, BrickSetCheckbox]
-
-    # Queries
-    select_query = 'checkbox/list'
-
-    def __init__(self, /, *, force: bool = False):
-        # Load checkboxes only if there is none already loaded
-        records = getattr(self, 'records', None)
-
-        if records is None or force:
-            # Don't use super()__init__ as it would mask class variables
-            self.fields = BrickRecordFields()
-
-            logger.info('Loading set checkboxes list')
-
-            BrickSetCheckboxList.records = []
-            BrickSetCheckboxList.checkboxes = {}
-
-            # Load the checkboxes from the database
-            for record in self.select():
-                checkbox = BrickSetCheckbox(record=record)
-
-                BrickSetCheckboxList.records.append(checkbox)
-                BrickSetCheckboxList.checkboxes[checkbox.fields.id] = checkbox
-
-    # Return the checkboxes as columns for a select
-    def as_columns(
-        self,
-        /,
-        *,
-        solo: bool = False,
-        table: str = 'bricktracker_set_statuses'
-    ) -> str:
-        return ', '.join([
-            '"{table}"."{column}"'.format(
-                table=table,
-                column=record.as_column(),
-            )
-            for record
-            in self.records
-            if solo or record.fields.displayed_on_grid
-        ])
-
-    # Grab a specific checkbox
-    def get(self, id: str, /) -> BrickSetCheckbox:
-        if id not in self.checkboxes:
-            raise NotFoundException(
-                'Checkbox with ID {id} was not found in the database'.format(
-                    id=self.fields.id,
-                ),
-            )
-
-        return self.checkboxes[id]
-
-    # Get the list of checkboxes depending on the context
-    def list(self, /, *, all: bool = False) -> list[BrickSetCheckbox]:
-        return [
-            record
-            for record
-            in self.records
-            if all or record.fields.displayed_on_grid
-        ]

bricktracker/set_list.py

@@ -3,7 +3,12 @@ from typing import Self
 from flask import current_app
 
 from .record_list import BrickRecordList
-from .set_checkbox_list import BrickSetCheckboxList
+from .set_owner import BrickSetOwner
+from .set_owner_list import BrickSetOwnerList
+from .set_status import BrickSetStatus
+from .set_status_list import BrickSetStatusList
+from .set_tag import BrickSetTag
+from .set_tag_list import BrickSetTagList
 from .set import BrickSet
 
 
@@ -13,6 +18,8 @@ class BrickSetList(BrickRecordList[BrickSet]):
     order: str
 
     # Queries
+    damaged_minifigure_query: str = 'set/list/damaged_minifigure'
+    damaged_part_query: str = 'set/list/damaged_part'
     generic_query: str = 'set/list/generic'
     light_query: str = 'set/list/light'
     missing_minifigure_query: str = 'set/list/missing_minifigure'
@@ -37,7 +44,9 @@ class BrickSetList(BrickRecordList[BrickSet]):
         # Load the sets from the database
         for record in self.select(
             order=self.order,
-            statuses=BrickSetCheckboxList().as_columns()
+            owners=BrickSetOwnerList(BrickSetOwner).as_columns(),
+            statuses=BrickSetStatusList(BrickSetStatus).as_columns(),
+            tags=BrickSetTagList(BrickSetTag).as_columns(),
         ):
             brickset = BrickSet(record=record)
 
@@ -50,6 +59,39 @@ class BrickSetList(BrickRecordList[BrickSet]):
 
         return self
 
+    # Sets with a minifigure part damaged
+    def damaged_minifigure(self, figure: str, /) -> Self:
+        # Save the parameters to the fields
+        self.fields.figure = figure
+
+        # Load the sets from the database
+        for record in self.select(
+            override_query=self.damaged_minifigure_query,
+            order=self.order
+        ):
+            brickset = BrickSet(record=record)
+
+            self.records.append(brickset)
+
+        return self
+
+    # Sets with a part damaged
+    def damaged_part(self, part: str, color: int, /) -> Self:
+        # Save the parameters to the fields
+        self.fields.part = part
+        self.fields.color = color
+
+        # Load the sets from the database
+        for record in self.select(
+            override_query=self.damaged_part_query,
+            order=self.order
+        ):
+            brickset = BrickSet(record=record)
+
+            self.records.append(brickset)
+
+        return self
+
     # A generic list of the different sets
     def generic(self, /) -> Self:
         for record in self.select(
@@ -73,7 +115,9 @@ class BrickSetList(BrickRecordList[BrickSet]):
         for record in self.select(
             order=order,
             limit=limit,
-            statuses=BrickSetCheckboxList().as_columns()
+            owners=BrickSetOwnerList(BrickSetOwner).as_columns(),
+            statuses=BrickSetStatusList(BrickSetStatus).as_columns(),
+            tags=BrickSetTagList(BrickSetTag).as_columns(),
         ):
             brickset = BrickSet(record=record)
 
@@ -81,14 +125,10 @@ class BrickSetList(BrickRecordList[BrickSet]):
 
         return self
 
-    # Sets missing a minifigure
-    def missing_minifigure(
-        self,
-        fig_num: str,
-        /
-    ) -> Self:
+    # Sets missing a minifigure part
+    def missing_minifigure(self, figure: str, /) -> Self:
         # Save the parameters to the fields
-        self.fields.fig_num = fig_num
+        self.fields.figure = figure
 
         # Load the sets from the database
         for record in self.select(
@@ -102,18 +142,10 @@ class BrickSetList(BrickRecordList[BrickSet]):
         return self
 
     # Sets missing a part
-    def missing_part(
-        self,
-        part_num: str,
-        color_id: int,
-        /,
-        *,
-        element_id: int | None = None,
-    ) -> Self:
+    def missing_part(self, part: str, color: int, /) -> Self:
         # Save the parameters to the fields
-        self.fields.part_num = part_num
-        self.fields.color_id = color_id
-        self.fields.element_id = element_id
+        self.fields.part = part
+        self.fields.color = color
 
         # Load the sets from the database
         for record in self.select(
@@ -127,13 +159,9 @@ class BrickSetList(BrickRecordList[BrickSet]):
         return self
 
     # Sets using a minifigure
-    def using_minifigure(
-        self,
-        fig_num: str,
-        /
-    ) -> Self:
+    def using_minifigure(self, figure: str, /) -> Self:
         # Save the parameters to the fields
-        self.fields.fig_num = fig_num
+        self.fields.figure = figure
 
         # Load the sets from the database
         for record in self.select(
@@ -147,18 +175,10 @@ class BrickSetList(BrickRecordList[BrickSet]):
         return self
 
     # Sets using a part
-    def using_part(
-        self,
-        part_num: str,
-        color_id: int,
-        /,
-        *,
-        element_id: int | None = None,
-    ) -> Self:
+    def using_part(self, part: str, color: int, /) -> Self:
         # Save the parameters to the fields
-        self.fields.part_num = part_num
-        self.fields.color_id = color_id
-        self.fields.element_id = element_id
+        self.fields.part = part
+        self.fields.color = color
 
         # Load the sets from the database
         for record in self.select(

bricktracker/set_owner.py

@@ -0,0 +1,16 @@
+from .metadata import BrickMetadata
+
+
+# Lego set owner metadata
+class BrickSetOwner(BrickMetadata):
+    kind: str = 'owner'
+
+    # Set state endpoint
+    set_state_endpoint: str = 'set.update_owner'
+
+    # Queries
+    delete_query: str = 'set/metadata/owner/delete'
+    insert_query: str = 'set/metadata/owner/insert'
+    select_query: str = 'set/metadata/owner/select'
+    update_field_query: str = 'set/metadata/owner/update/field'
+    update_set_state_query: str = 'set/metadata/owner/update/state'

bricktracker/set_owner_list.py

@@ -0,0 +1,17 @@
+import logging
+
+from .metadata_list import BrickMetadataList
+from .set_owner import BrickSetOwner
+
+logger = logging.getLogger(__name__)
+
+
+# Lego sets owner list
+class BrickSetOwnerList(BrickMetadataList[BrickSetOwner]):
+    kind: str = 'set owners'
+
+    # Database table
+    table: str = 'bricktracker_set_owners'
+
+    # Queries
+    select_query = 'set/metadata/owner/list'

bricktracker/set_status.py

@@ -0,0 +1,34 @@
+from typing import Self
+
+from .metadata import BrickMetadata
+
+
+# Lego set status metadata
+class BrickSetStatus(BrickMetadata):
+    kind: str = 'status'
+
+    # Set state endpoint
+    set_state_endpoint: str = 'set.update_status'
+
+    # Queries
+    delete_query: str = 'set/metadata/status/delete'
+    insert_query: str = 'set/metadata/status/insert'
+    select_query: str = 'set/metadata/status/select'
+    update_field_query: str = 'set/metadata/status/update/field'
+    update_set_state_query: str = 'set/metadata/status/update/state'
+
+    # Grab data from a form
+    def from_form(self, form: dict[str, str], /) -> Self:
+        super().from_form(form)
+
+        grid = form.get('grid', None)
+
+        self.fields.displayed_on_grid = grid == 'on'
+
+        return self
+
+    # Insert into database
+    def insert(self, /, **_) -> None:
+        super().insert(
+            displayed_on_grid=self.fields.displayed_on_grid
+        )

bricktracker/set_status_list.py

@@ -0,0 +1,26 @@
+import logging
+
+from .metadata_list import BrickMetadataList
+from .set_status import BrickSetStatus
+
+logger = logging.getLogger(__name__)
+
+
+# Lego sets status list
+class BrickSetStatusList(BrickMetadataList[BrickSetStatus]):
+    kind: str = 'set statuses'
+
+    # Database table
+    table: str = 'bricktracker_set_statuses'
+
+    # Queries
+    select_query = 'set/metadata/status/list'
+
+    # Filter the list of set status
+    def filter(self, all: bool = False) -> list[BrickSetStatus]:
+        return [
+            record
+            for record
+            in self.records
+            if all or record.fields.displayed_on_grid
+        ]

bricktracker/set_tag.py

@@ -0,0 +1,16 @@
+from .metadata import BrickMetadata
+
+
+# Lego set tag metadata
+class BrickSetTag(BrickMetadata):
+    kind: str = 'tag'
+
+    # Set state endpoint
+    set_state_endpoint: str = 'set.update_tag'
+
+    # Queries
+    delete_query: str = 'set/metadata/tag/delete'
+    insert_query: str = 'set/metadata/tag/insert'
+    select_query: str = 'set/metadata/tag/select'
+    update_field_query: str = 'set/metadata/tag/update/field'
+    update_set_state_query: str = 'set/metadata/tag/update/state'

bricktracker/set_tag_list.py

@@ -0,0 +1,17 @@
+import logging
+
+from .metadata_list import BrickMetadataList
+from .set_tag import BrickSetTag
+
+logger = logging.getLogger(__name__)
+
+
+# Lego sets tag list
+class BrickSetTagList(BrickMetadataList[BrickSetTag]):
+    kind: str = 'set tags'
+
+    # Database table
+    table: str = 'bricktracker_set_tags'
+
+    # Queries
+    select_query = 'set/metadata/tag/list'

bricktracker/socket.py

@@ -1,14 +1,13 @@
 import logging
 from typing import Any, Final, Tuple
 
-from flask import copy_current_request_context, Flask, request
+from flask import Flask, request
 from flask_socketio import SocketIO
 
-from .configuration_list import BrickConfigurationList
 from .instructions import BrickInstructions
 from .instructions_list import BrickInstructionsList
-from .login import LoginManager
 from .set import BrickSet
+from .socket_decorator import authenticated_socket, rebrickable_socket
 from .sql import close as sql_close
 
 logger = logging.getLogger(__name__)
@@ -87,12 +86,8 @@ class BrickSocket(object):
             self.disconnected()
 
         @self.socket.on(MESSAGES['DOWNLOAD_INSTRUCTIONS'], namespace=self.namespace)  # noqa: E501
+        @authenticated_socket(self)
         def download_instructions(data: dict[str, Any], /) -> None:
-            # Needs to be authenticated
-            if LoginManager.is_not_authenticated():
-                self.fail(message='You need to be authenticated')
-                return
-
             instructions = BrickInstructions(
                 '{name}.pdf'.format(name=data.get('alt', '')),
                 socket=self
@@ -107,71 +102,18 @@ class BrickSocket(object):
             except Exception:
                 pass
 
-            # Start it in a thread if requested
-            if self.threaded:
-                @copy_current_request_context
-                def do_download() -> None:
-                    instructions.download(path)
+            instructions.download(path)
 
-                    BrickInstructionsList(force=True)
-
-                self.socket.start_background_task(do_download)
-            else:
-                instructions.download(path)
-
-                BrickInstructionsList(force=True)
+            BrickInstructionsList(force=True)
 
         @self.socket.on(MESSAGES['IMPORT_SET'], namespace=self.namespace)
+        @rebrickable_socket(self)
         def import_set(data: dict[str, Any], /) -> None:
-            # Needs to be authenticated
-            if LoginManager.is_not_authenticated():
-                self.fail(message='You need to be authenticated')
-                return
-
-            # Needs the Rebrickable API key
-            try:
-                BrickConfigurationList.error_unless_is_set('REBRICKABLE_API_KEY')  # noqa: E501
-            except Exception as e:
-                self.fail(message=str(e))
-                return
-
-            brickset = BrickSet(socket=self)
-
-            # Start it in a thread if requested
-            if self.threaded:
-                @copy_current_request_context
-                def do_download() -> None:
-                    brickset.download(data)
-
-                self.socket.start_background_task(do_download)
-            else:
-                brickset.download(data)
+            BrickSet().download(self, data)
 
         @self.socket.on(MESSAGES['LOAD_SET'], namespace=self.namespace)
         def load_set(data: dict[str, Any], /) -> None:
-            # Needs to be authenticated
-            if LoginManager.is_not_authenticated():
-                self.fail(message='You need to be authenticated')
-                return
-
-            # Needs the Rebrickable API key
-            try:
-                BrickConfigurationList.error_unless_is_set('REBRICKABLE_API_KEY')  # noqa: E501
-            except Exception as e:
-                self.fail(message=str(e))
-                return
-
-            brickset = BrickSet(socket=self)
-
-            # Start it in a thread if requested
-            if self.threaded:
-                @copy_current_request_context
-                def do_load() -> None:
-                    brickset.load(data)
-
-                self.socket.start_background_task(do_load)
-            else:
-                brickset.load(data)
+            BrickSet().load(self, data)
 
     # Update the progress auto-incrementing
     def auto_progress(

bricktracker/socket_decorator.py

@@ -0,0 +1,93 @@
+from functools import wraps
+from threading import Thread
+from typing import Callable, ParamSpec, TYPE_CHECKING, Union
+
+from flask import copy_current_request_context
+
+from .configuration_list import BrickConfigurationList
+from .login import LoginManager
+if TYPE_CHECKING:
+    from .socket import BrickSocket
+
+# What a threaded function can return (None or Thread)
+SocketReturn = Union[None, Thread]
+
+# Threaded signature (*arg, **kwargs -> (None or Thread)
+P = ParamSpec('P')
+SocketCallable = Callable[P, SocketReturn]
+
+
+# Fail if not authenticated
+def authenticated_socket(
+    self: 'BrickSocket',
+    /,
+    *,
+    threaded: bool = True,
+) -> Callable[[SocketCallable], SocketCallable]:
+    def outer(function: SocketCallable, /) -> SocketCallable:
+        @wraps(function)
+        def wrapper(*args, **kwargs) -> SocketReturn:
+            # Needs to be authenticated
+            if LoginManager.is_not_authenticated():
+                self.fail(message='You need to be authenticated')
+                return
+
+            # Apply threading
+            if threaded:
+                return threaded_socket(self)(function)(*args, **kwargs)
+            else:
+                return function(*args, **kwargs)
+
+        return wrapper
+    return outer
+
+
+# Fail if not ready for Rebrickable (authenticated, API key)
+# Automatically makes it threaded
+def rebrickable_socket(
+    self: 'BrickSocket',
+    /,
+    *,
+    threaded: bool = True,
+) -> Callable[[SocketCallable], SocketCallable]:
+    def outer(function: SocketCallable, /) -> SocketCallable:
+        @wraps(function)
+        # Automatically authenticated
+        @authenticated_socket(self, threaded=False)
+        def wrapper(*args, **kwargs) -> SocketReturn:
+            # Needs the Rebrickable API key
+            try:
+                BrickConfigurationList.error_unless_is_set('REBRICKABLE_API_KEY')  # noqa: E501
+            except Exception as e:
+                self.fail(message=str(e))
+                return
+
+            # Apply threading
+            if threaded:
+                return threaded_socket(self)(function)(*args, **kwargs)
+            else:
+                return function(*args, **kwargs)
+
+        return wrapper
+    return outer
+
+
+# Start the function in a thread if the socket is threaded
+def threaded_socket(
+    self: 'BrickSocket',
+    /
+) -> Callable[[SocketCallable], SocketCallable]:
+    def outer(function: SocketCallable, /) -> SocketCallable:
+        @wraps(function)
+        def wrapper(*args, **kwargs) -> SocketReturn:
+            # Start it in a thread if requested
+            if self.threaded:
+                @copy_current_request_context
+                def do_function() -> None:
+                    function(*args, **kwargs)
+
+                return self.socket.start_background_task(do_function)
+            else:
+                return function(*args, **kwargs)
+        return wrapper
+    return outer

bricktracker/sql.py

@@ -1,3 +1,4 @@
+from importlib import import_module
 import logging
 import os
 import sqlite3
@@ -301,7 +302,37 @@ class BrickSQL(object):
                     version=pending.version)
                 )
 
-                self.executescript(pending.get_query())
+                # Load context from the migrations if it exists
+                # It looks for a file in migrations/ named after the SQL file
+                # and containing one function named migration_xxxx, also named
+                # after the SQL file, returning a context dict.
+                #
+                # For instance:
+                # - sql/migrations/0007.sql
+                # - migrations/0007.py
+                #    - def migration_0007(BrickSQL) -> dict[str, Any]
+                try:
+                    module = import_module(
+                        '.migrations.{name}'.format(
+                            name=pending.name
+                        ),
+                        package='bricktracker'
+                    )
+                except Exception:
+                    module = None
+
+                # If a module has been loaded, we need to fail if an error
+                # occured while executing the migration function
+                if module is not None:
+                    function = getattr(module, 'migration_{name}'.format(
+                        name=pending.name
+                    ))
+
+                    context: dict[str, Any] = function(self)
+                else:
+                    context: dict[str, Any] = {}
+
+                self.executescript(pending.get_query(), **context)
                 self.execute('schema/set_version', version=pending.version)
 
     # Tells whether the database needs upgrade

bricktracker/sql/checkbox/base.sql

@@ -1,7 +0,0 @@
-SELECT
-    "bricktracker_set_checkboxes"."id",
-    "bricktracker_set_checkboxes"."name",
-    "bricktracker_set_checkboxes"."displayed_on_grid"
-FROM "bricktracker_set_checkboxes"
-
-{% block where %}{% endblock %}

bricktracker/sql/checkbox/delete.sql

@@ -1,9 +0,0 @@
-BEGIN TRANSACTION;
-
-ALTER TABLE "bricktracker_set_statuses"
-DROP COLUMN "status_{{ id }}";
-
-DELETE FROM "bricktracker_set_checkboxes"
-WHERE "bricktracker_set_checkboxes"."id" IS NOT DISTINCT FROM '{{ id }}';
-
-COMMIT;

bricktracker/sql/checkbox/list.sql

@@ -1 +0,0 @@
-{% extends 'checkbox/base.sql' %}

bricktracker/sql/checkbox/select.sql

@@ -1,5 +0,0 @@
-{% extends 'checkbox/base.sql' %}
-
-{% block where %}
-WHERE "bricktracker_set_checkboxes"."id" IS NOT DISTINCT FROM :id
-{% endblock %}

bricktracker/sql/checkbox/update/name.sql

@@ -1,3 +0,0 @@
-UPDATE "bricktracker_set_checkboxes"
-SET "name" = :safe_name
-WHERE "bricktracker_set_checkboxes"."id" IS NOT DISTINCT FROM :id

bricktracker/sql/checkbox/update/status.sql

@@ -1,3 +0,0 @@
-UPDATE "bricktracker_set_checkboxes"
-SET "{{name}}" = :status
-WHERE "bricktracker_set_checkboxes"."id" IS NOT DISTINCT FROM :id

bricktracker/sql/migrations/0004.sql

@@ -4,7 +4,7 @@ PRAGMA foreign_keys = ON;
 
 BEGIN TRANSACTION;
 
--- Create a Bricktable set table: with their unique IDs, and a reference to the Rebrickable set
+-- Create a Bricktracker set table: with their unique IDs, and a reference to the Rebrickable set
 CREATE TABLE "bricktracker_sets" (
     "id" TEXT NOT NULL,
     "rebrickable_set" TEXT NOT NULL,

bricktracker/sql/migrations/0007.sql

@@ -0,0 +1,72 @@
+-- description: Renaming various complicated field names to something simpler, and add a bunch of extra fields for later
+
+PRAGMA foreign_keys = ON;
+
+BEGIN TRANSACTION;
+
+-- Rename sets table
+ALTER TABLE "bricktracker_sets" RENAME TO "bricktracker_sets_old";
+
+-- Create a Bricktracker set storage table for later
+CREATE TABLE "bricktracker_set_storages" (
+    "name" TEXT NOT NULL,
+    PRIMARY KEY("name")
+);
+
+-- Create a Bricktracker set storage table for later
+CREATE TABLE "bricktracker_set_purchase_locations" (
+    "name" TEXT NOT NULL,
+    PRIMARY KEY("name")
+);
+
+-- Re-Create a Bricktracker set table with the simplified name
+CREATE TABLE "bricktracker_sets" (
+    "id" TEXT NOT NULL,
+    "set" TEXT NOT NULL,
+    "description" TEXT,
+    "storage" TEXT, -- Storage bin location
+    "purchase_date" INTEGER, -- Purchase data
+    "purchase_location" TEXT, -- Purchase location
+    "purchase_price" REAL, -- Purchase price
+    PRIMARY KEY("id"),
+    FOREIGN KEY("set") REFERENCES "rebrickable_sets"("set"),
+    FOREIGN KEY("storage") REFERENCES "bricktracker_set_storages"("name"),
+    FOREIGN KEY("purchase_location") REFERENCES "bricktracker_set_purchase_locations"("name")
+);
+
+-- Insert existing sets into the new table
+INSERT INTO "bricktracker_sets" (
+    "id",
+    "set"
+)
+SELECT
+    "bricktracker_sets_old"."id",
+    "bricktracker_sets_old"."rebrickable_set"
+FROM "bricktracker_sets_old";
+
+-- Rename status table
+ALTER TABLE "bricktracker_set_statuses" RENAME TO "bricktracker_set_statuses_old";
+
+-- Re-create a table for the status of each checkbox
+CREATE TABLE "bricktracker_set_statuses" (
+    "id" TEXT NOT NULL,
+    {% if structure %}{{ structure }},{% endif %}
+    PRIMARY KEY("id"),
+    FOREIGN KEY("id") REFERENCES "bricktracker_sets"("id")
+);
+
+-- Insert existing status into the new table
+INSERT INTO "bricktracker_set_statuses" (
+    {% if targets %}{{ targets }},{% endif %}
+    "id"
+)
+SELECT
+    {% if sources %}{{ sources }},{% endif %}
+    "bricktracker_set_statuses_old"."bricktracker_set_id"
+FROM "bricktracker_set_statuses_old";
+
+-- Delete the original tables
+DROP TABLE "bricktracker_set_statuses_old";
+DROP TABLE "bricktracker_sets_old";
+
+COMMIT;

bricktracker/sql/migrations/0008.sql

@@ -0,0 +1,30 @@
+-- description: Creation of the deduplicated table of Rebrickable minifigures
+
+BEGIN TRANSACTION;
+
+-- Create a Rebrickable minifigures table: each unique minifigure imported from Rebrickable
+CREATE TABLE "rebrickable_minifigures" (
+    "figure" TEXT NOT NULL,
+    "number" INTEGER NOT NULL,
+    "name" TEXT NOT NULL,
+    "image" TEXT,
+    PRIMARY KEY("figure")
+);
+
+-- Insert existing sets into the new table
+INSERT INTO "rebrickable_minifigures" (
+    "figure",
+    "number",
+    "name",
+    "image"
+)
+SELECT
+    "minifigures"."fig_num",
+    CAST(SUBSTR("minifigures"."fig_num", 5) AS INTEGER),
+    "minifigures"."name",
+    "minifigures"."set_img_url"
+FROM "minifigures"
+GROUP BY
+    "minifigures"."fig_num";
+
+COMMIT;

bricktracker/sql/migrations/0009.sql

@@ -0,0 +1,32 @@
+-- description: Migrate the Bricktracker minifigures
+
+PRAGMA foreign_keys = ON;
+
+BEGIN TRANSACTION;
+
+-- Create a Bricktracker minifigures table: an amount of minifigures linked to a Bricktracker set
+CREATE TABLE "bricktracker_minifigures" (
+    "id" TEXT NOT NULL,
+    "figure" TEXT NOT NULL,
+    "quantity" INTEGER NOT NULL,
+    PRIMARY KEY("id", "figure"),
+    FOREIGN KEY("id") REFERENCES "bricktracker_sets"("id"),
+    FOREIGN KEY("figure") REFERENCES "rebrickable_minifigures"("figure")
+);
+
+-- Insert existing sets into the new table
+INSERT INTO "bricktracker_minifigures" (
+    "id",
+    "figure",
+    "quantity"
+)
+SELECT
+    "minifigures"."u_id",
+    "minifigures"."fig_num",
+    "minifigures"."quantity"
+FROM "minifigures";
+
+-- Rename the original table (don't delete it yet?)
+ALTER TABLE "minifigures" RENAME TO "minifigures_old";
+
+COMMIT;

bricktracker/sql/migrations/0010.sql

@@ -0,0 +1,42 @@
+-- description: Creation of the deduplicated table of Rebrickable parts, and add a bunch of extra fields for later
+
+BEGIN TRANSACTION;
+
+-- Create a Rebrickable parts table: each unique part imported from Rebrickable
+CREATE TABLE "rebrickable_parts" (
+    "part" TEXT NOT NULL,
+    "color_id" INTEGER NOT NULL,
+    "color_name" TEXT NOT NULL,
+    "color_rgb" TEXT, -- can be NULL because it was not saved before
+    "color_transparent" BOOLEAN, -- can be NULL because it was not saved before
+    "name" TEXT NOT NULL,
+    "category" INTEGER, -- can be NULL because it was not saved before
+    "image" TEXT,
+    "image_id" TEXT,
+    "url" TEXT, -- can be NULL because it was not saved before
+    "print" INTEGER, -- can be NULL, was not saved before
+    PRIMARY KEY("part", "color_id")
+);
+
+-- Insert existing parts into the new table
+INSERT INTO "rebrickable_parts" (
+    "part",
+    "color_id",
+    "color_name",
+    "name",
+    "image",
+    "image_id"
+)
+SELECT
+    "inventory"."part_num",
+    "inventory"."color_id",
+    "inventory"."color_name",
+    "inventory"."name",
+    "inventory"."part_img_url",
+    "inventory"."part_img_url_id"
+FROM "inventory"
+GROUP BY
+    "inventory"."part_num",
+    "inventory"."color_id";
+
+COMMIT;

bricktracker/sql/migrations/0011.sql

@@ -0,0 +1,60 @@
+-- description: Migrate the Bricktracker parts (and missing parts), and add a bunch of extra fields for later
+
+PRAGMA foreign_keys = ON;
+
+BEGIN TRANSACTION;
+
+-- Create a Bricktracker parts table: an amount of parts linked to a Bricktracker set
+CREATE TABLE "bricktracker_parts" (
+    "id" TEXT NOT NULL,
+    "figure" TEXT,
+    "part" TEXT NOT NULL,
+    "color" INTEGER NOT NULL,
+    "spare" BOOLEAN NOT NULL,
+    "quantity" INTEGER NOT NULL,
+    "element" INTEGER,
+    "rebrickable_inventory" INTEGER NOT NULL,
+    "missing" INTEGER NOT NULL DEFAULT 0,
+    "damaged" INTEGER NOT NULL DEFAULT 0,
+    PRIMARY KEY("id", "figure", "part", "color", "spare"),
+    FOREIGN KEY("id") REFERENCES "bricktracker_sets"("id"),
+    FOREIGN KEY("figure") REFERENCES "rebrickable_minifigures"("figure"),
+    FOREIGN KEY("part", "color") REFERENCES "rebrickable_parts"("part", "color_id")
+);
+
+-- Insert existing parts into the new table
+INSERT INTO "bricktracker_parts" (
+    "id",
+    "figure",
+    "part",
+    "color",
+    "spare",
+    "quantity",
+    "element",
+    "rebrickable_inventory",
+    "missing"
+)
+SELECT
+    "inventory"."u_id",
+    CASE WHEN SUBSTR("inventory"."set_num", 0, 5) = 'fig-' THEN "inventory"."set_num" ELSE NULL END,
+    "inventory"."part_num",
+    "inventory"."color_id",
+    "inventory"."is_spare",
+    "inventory"."quantity",
+    "inventory"."element_id",
+    "inventory"."id",
+    IFNULL("missing"."quantity", 0)
+FROM "inventory"
+LEFT JOIN "missing"
+ON "inventory"."set_num" IS NOT DISTINCT FROM "missing"."set_num"
+AND "inventory"."id" IS NOT DISTINCT FROM "missing"."id"
+AND "inventory"."part_num" IS NOT DISTINCT FROM "missing"."part_num"
+AND "inventory"."color_id" IS NOT DISTINCT FROM "missing"."color_id"
+AND "inventory"."element_id" IS NOT DISTINCT FROM "missing"."element_id"
+AND "inventory"."u_id" IS NOT DISTINCT FROM "missing"."u_id";
+
+-- Rename the original table (don't delete it yet?)
+ALTER TABLE "inventory" RENAME TO "inventory_old";
+ALTER TABLE "missing" RENAME TO "missing_old";
+
+COMMIT;

bricktracker/sql/migrations/0012.sql

@@ -0,0 +1,7 @@
+-- description: Rename checkboxes to status metadata
+
+BEGIN TRANSACTION;
+
+ALTER TABLE "bricktracker_set_checkboxes" RENAME TO "bricktracker_metadata_statuses";
+
+COMMIT;

bricktracker/sql/migrations/0013.sql

@@ -0,0 +1,19 @@
+-- description: Add set owners
+
+BEGIN TRANSACTION;
+
+-- Create a table to define each set owners: an id and a name
+CREATE TABLE "bricktracker_metadata_owners" (
+    "id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    PRIMARY KEY("id")
+);
+
+-- Create a table for the set owners
+CREATE TABLE "bricktracker_set_owners" (
+    "id" TEXT NOT NULL,
+    PRIMARY KEY("id"),
+    FOREIGN KEY("id") REFERENCES "bricktracker_sets"("id")
+);
+
+COMMIT;

bricktracker/sql/migrations/0014.sql

@@ -0,0 +1,19 @@
+-- description: Add set tags
+
+BEGIN TRANSACTION;
+
+-- Create a table to define each set tags: an id and a name
+CREATE TABLE "bricktracker_metadata_tags" (
+    "id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    PRIMARY KEY("id")
+);
+
+-- Create a table for the set tags
+CREATE TABLE "bricktracker_set_tags" (
+    "id" TEXT NOT NULL,
+    PRIMARY KEY("id"),
+    FOREIGN KEY("id") REFERENCES "bricktracker_sets"("id")
+);
+
+COMMIT;

bricktracker/sql/minifigure/base/base.sql

@@ -1,20 +1,25 @@
 SELECT
-    "minifigures"."fig_num",
-    "minifigures"."set_num",
-    "minifigures"."name",
-    "minifigures"."quantity",
-    "minifigures"."set_img_url",
-    "minifigures"."u_id",
+    "bricktracker_minifigures"."quantity",
+    "rebrickable_minifigures"."figure",
+    "rebrickable_minifigures"."number",
+    "rebrickable_minifigures"."name",
+    "rebrickable_minifigures"."image",
     {% block total_missing %}
     NULL AS "total_missing", -- dummy for order: total_missing
     {% endblock %}
+    {% block total_damaged %}
+    NULL AS "total_damaged", -- dummy for order: total_damaged
+    {% endblock %}
     {% block total_quantity %}
     NULL AS "total_quantity", -- dummy for order: total_quantity
     {% endblock %}
     {% block total_sets %}
     NULL AS "total_sets" -- dummy for order: total_sets
     {% endblock %}
-FROM "minifigures"
+FROM "bricktracker_minifigures"
+
+INNER JOIN "rebrickable_minifigures"
+ON "bricktracker_minifigures"."figure" IS NOT DISTINCT FROM "rebrickable_minifigures"."figure"
 
 {% block join %}{% endblock %}
 

bricktracker/sql/minifigure/insert.sql

@@ -1,15 +1,9 @@
-INSERT INTO "minifigures" (
-    "fig_num",
-    "set_num",
-    "name",
-    "quantity",
-    "set_img_url",
-    "u_id"
+INSERT INTO "bricktracker_minifigures" (
+    "id",
+    "figure",
+    "quantity"
 ) VALUES (
-    :fig_num,
-    :set_num,
-    :name,
-    :quantity,
-    :set_img_url,
-    :u_id
+    :id,
+    :figure,
+    :quantity
 )

bricktracker/sql/minifigure/list/all.sql

@@ -1,34 +1,40 @@
-{% extends 'minifigure/base/select.sql' %}
+{% extends 'minifigure/base/base.sql' %}
 
 {% block total_missing %}
-SUM(IFNULL("missing_join"."total", 0)) AS "total_missing",
+SUM(IFNULL("problem_join"."total_missing", 0)) AS "total_missing",
+{% endblock %}
+
+{% block total_damaged %}
+SUM(IFNULL("problem_join"."total_damaged", 0)) AS "total_damaged",
 {% endblock %}
 
 {% block total_quantity %}
-SUM(IFNULL("minifigures"."quantity", 0)) AS "total_quantity",
+SUM(IFNULL("bricktracker_minifigures"."quantity", 0)) AS "total_quantity",
 {% endblock %}
 
 {% block total_sets %}
-COUNT("minifigures"."set_num") AS "total_sets"
+COUNT("bricktracker_minifigures"."id") AS "total_sets"
 {% endblock %}
 
 {% block join %}
 -- LEFT JOIN + SELECT to avoid messing the total
 LEFT JOIN (
     SELECT
-        "missing"."set_num",
-        "missing"."u_id",
-        SUM("missing"."quantity") AS total
-    FROM "missing"
+        "bricktracker_parts"."id",
+        "bricktracker_parts"."figure",
+        SUM("bricktracker_parts"."missing") AS "total_missing",
+        SUM("bricktracker_parts"."damaged") AS "total_damaged"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."figure" IS NOT NULL
     GROUP BY
-        "missing"."set_num",
-        "missing"."u_id"
-) missing_join
-ON "minifigures"."u_id" IS NOT DISTINCT FROM "missing_join"."u_id"
-AND "minifigures"."fig_num" IS NOT DISTINCT FROM "missing_join"."set_num"
+        "bricktracker_parts"."id",
+        "bricktracker_parts"."figure"
+) "problem_join"
+ON "bricktracker_minifigures"."id" IS NOT DISTINCT FROM "problem_join"."id"
+AND "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM "problem_join"."figure"
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    "minifigures"."fig_num"
+    "rebrickable_minifigures"."figure"
 {% endblock %}

bricktracker/sql/minifigure/list/damaged_part.sql

@@ -0,0 +1,28 @@
+{% extends 'minifigure/base/base.sql' %}
+
+{% block total_damaged %}
+SUM("bricktracker_parts"."damaged") AS "total_damaged",
+{% endblock %}
+
+{% block join %}
+LEFT JOIN "bricktracker_parts"
+ON "bricktracker_minifigures"."id" IS NOT DISTINCT FROM "bricktracker_parts"."id"
+AND "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM "bricktracker_parts"."figure"
+{% endblock %}
+
+{% block where %}
+WHERE "rebrickable_minifigures"."figure" IN (
+    SELECT "bricktracker_parts"."figure"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."part" IS NOT DISTINCT FROM :part
+    AND "bricktracker_parts"."color" IS NOT DISTINCT FROM :color
+    AND "bricktracker_parts"."figure" IS NOT NULL
+    AND "bricktracker_parts"."damaged" > 0
+    GROUP BY "bricktracker_parts"."figure"
+)
+{% endblock %}
+
+{% block group %}
+GROUP BY
+    "rebrickable_minifigures"."figure"
+{% endblock %}

bricktracker/sql/minifigure/list/from_set.sql

@@ -1,6 +1,5 @@
-{% extends 'minifigure/base/select.sql' %}
+{% extends 'minifigure/base/base.sql' %}
 
 {% block where %}
-WHERE "minifigures"."u_id" IS NOT DISTINCT FROM :u_id
-AND "minifigures"."set_num" IS NOT DISTINCT FROM :set_num
+WHERE "bricktracker_minifigures"."id" IS NOT DISTINCT FROM :id
 {% endblock %}

bricktracker/sql/minifigure/list/last.sql

@@ -1,17 +1,21 @@
-{% extends 'minifigure/base/select.sql' %}
+{% extends 'minifigure/base/base.sql' %}
 
 {% block total_missing %}
-SUM(IFNULL("missing"."quantity", 0)) AS "total_missing",
+SUM("bricktracker_parts"."missing") AS "total_missing",
+{% endblock %}
+
+{% block total_damaged %}
+SUM("bricktracker_parts"."damaged") AS "total_damaged",
 {% endblock %}
 
 {% block join %}
-LEFT JOIN "missing"
-ON "minifigures"."fig_num" IS NOT DISTINCT FROM "missing"."set_num"
-AND "minifigures"."u_id" IS NOT DISTINCT FROM "missing"."u_id"
+LEFT JOIN "bricktracker_parts"
+ON "bricktracker_minifigures"."id" IS NOT DISTINCT FROM "bricktracker_parts"."id"
+AND "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM "bricktracker_parts"."figure"
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    "minifigures"."fig_num",
-    "minifigures"."u_id"
+    "rebrickable_minifigures"."figure",
+    "bricktracker_minifigures"."id"
 {% endblock %}

bricktracker/sql/minifigure/list/missing_part.sql

@@ -1,30 +1,28 @@
-{% extends 'minifigure/base/select.sql' %}
+{% extends 'minifigure/base/base.sql' %}
 
 {% block total_missing %}
-SUM(IFNULL("missing"."quantity", 0)) AS "total_missing",
+SUM("bricktracker_parts"."missing") AS "total_missing",
 {% endblock %}
 
 {% block join %}
-LEFT JOIN "missing"
-ON "minifigures"."fig_num" IS NOT DISTINCT FROM "missing"."set_num"
-AND "minifigures"."u_id" IS NOT DISTINCT FROM "missing"."u_id"
+LEFT JOIN "bricktracker_parts"
+ON "bricktracker_minifigures"."id" IS NOT DISTINCT FROM "bricktracker_parts"."id"
+AND "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM "bricktracker_parts"."figure"
 {% endblock %}
 
 {% block where %}
-WHERE "minifigures"."fig_num" IN (
-    SELECT
-        "missing"."set_num"
-    FROM "missing"
-
-    WHERE "missing"."color_id" IS NOT DISTINCT FROM :color_id
-    AND "missing"."element_id" IS NOT DISTINCT FROM :element_id
-    AND "missing"."part_num" IS NOT DISTINCT FROM :part_num
-
-    GROUP BY "missing"."set_num"
+WHERE "rebrickable_minifigures"."figure" IN (
+    SELECT "bricktracker_parts"."figure"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."part" IS NOT DISTINCT FROM :part
+    AND "bricktracker_parts"."color" IS NOT DISTINCT FROM :color
+    AND "bricktracker_parts"."figure" IS NOT NULL
+    AND "bricktracker_parts"."missing" > 0
+    GROUP BY "bricktracker_parts"."figure"
 )
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    "minifigures"."fig_num"
+    "rebrickable_minifigures"."figure"
 {% endblock %}

bricktracker/sql/minifigure/list/using_part.sql

@@ -1,24 +1,21 @@
-{% extends 'minifigure/base/select.sql' %}
+{% extends 'minifigure/base/base.sql' %}
 
 {% block total_quantity %}
-SUM("minifigures"."quantity") AS "total_quantity",
+SUM("bricktracker_minifigures"."quantity") AS "total_quantity",
 {% endblock %}
 
 {% block where %}
-WHERE "minifigures"."fig_num" IN (
-    SELECT
-        "inventory"."set_num"
-    FROM "inventory"
-
-    WHERE "inventory"."color_id" IS NOT DISTINCT FROM :color_id
-    AND "inventory"."element_id" IS NOT DISTINCT FROM :element_id
-    AND "inventory"."part_num" IS NOT DISTINCT FROM :part_num
-
-    GROUP BY "inventory"."set_num"
+WHERE "rebrickable_minifigures"."figure" IN (
+    SELECT "bricktracker_parts"."figure"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."part" IS NOT DISTINCT FROM :part
+    AND "bricktracker_parts"."color" IS NOT DISTINCT FROM :color
+    AND "bricktracker_parts"."figure" IS NOT NULL
+    GROUP BY "bricktracker_parts"."figure"
 )
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    "minifigures"."fig_num"
+    "rebrickable_minifigures"."figure"
 {% endblock %}

bricktracker/sql/minifigure/select/generic.sql

@@ -1,38 +1,40 @@
-{% extends 'minifigure/base/select.sql' %}
+{% extends 'minifigure/base/base.sql' %}
 
 {% block total_missing %}
-SUM(IFNULL("missing_join"."total", 0)) AS "total_missing",
+IFNULL("problem_join"."total_missing", 0) AS "total_missing",
+{% endblock %}
+
+{% block total_damaged %}
+IFNULL("problem_join"."total_damaged", 0) AS "total_damaged",
 {% endblock %}
 
 {% block total_quantity %}
-SUM(IFNULL("minifigures"."quantity", 0)) AS "total_quantity",
+SUM(IFNULL("bricktracker_minifigures"."quantity", 0)) AS "total_quantity",
 {% endblock %}
 
 {% block total_sets %}
-COUNT("minifigures"."set_num") AS "total_sets"
+COUNT(DISTINCT "bricktracker_minifigures"."id") AS "total_sets"
 {% endblock %}
 
 {% block join %}
 -- LEFT JOIN + SELECT to avoid messing the total
 LEFT JOIN (
     SELECT
-        "missing"."set_num",
-        "missing"."u_id",
-        SUM("missing"."quantity") AS "total"
-    FROM "missing"
-    GROUP BY
-        "missing"."set_num",
-        "missing"."u_id"
-) "missing_join"
-ON "minifigures"."u_id" IS NOT DISTINCT FROM "missing_join"."u_id"
-AND "minifigures"."fig_num" IS NOT DISTINCT FROM "missing_join"."set_num"
+        "bricktracker_parts"."figure",
+        SUM("bricktracker_parts"."missing") AS "total_missing",
+        SUM("bricktracker_parts"."damaged") AS "total_damaged"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."figure" IS NOT DISTINCT FROM :figure
+    GROUP BY "bricktracker_parts"."figure"
+) "problem_join"
+ON "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM "problem_join"."figure"
 {% endblock %}
 
 {% block where %}
-WHERE "minifigures"."fig_num" IS NOT DISTINCT FROM :fig_num
+WHERE "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM :figure
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    "minifigures"."fig_num"
+    "rebrickable_minifigures"."figure"
 {% endblock %}

bricktracker/sql/minifigure/select/specific.sql

@@ -1,7 +1,6 @@
-{% extends 'minifigure/base/select.sql' %}
+{% extends 'minifigure/base/base.sql' %}
 
 {% block where %}
-WHERE "minifigures"."fig_num" IS NOT DISTINCT FROM :fig_num
-AND "minifigures"."u_id" IS NOT DISTINCT FROM :u_id
-AND "minifigures"."set_num" IS NOT DISTINCT FROM :set_num
+WHERE "bricktracker_minifigures"."id" IS NOT DISTINCT FROM :id
+AND "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM :figure
 {% endblock %}

bricktracker/sql/missing/delete/from_set.sql

@@ -1,4 +0,0 @@
-DELETE FROM "missing"
-WHERE "missing"."set_num" IS NOT DISTINCT FROM :set_num
-AND "missing"."id" IS NOT DISTINCT FROM :id
-AND "missing"."u_id" IS NOT DISTINCT FROM :u_id

bricktracker/sql/missing/insert.sql

@@ -1,20 +0,0 @@
-INSERT INTO "missing" (
-    "set_num",
-    "id",
-    "part_num",
-    "part_img_url_id",
-    "color_id",
-    "quantity",
-    "element_id",
-    "u_id"
-)
-VALUES(
-    :set_num,
-    :id,
-    :part_num,
-    :part_img_url_id,
-    :color_id,
-    :quantity,
-    :element_id,
-    :u_id
-)

bricktracker/sql/missing/update/from_set.sql

@@ -1,5 +0,0 @@
-UPDATE "missing"
-SET "quantity" = :quantity
-WHERE "missing"."set_num" IS NOT DISTINCT FROM :set_num
-AND "missing"."id" IS NOT DISTINCT FROM :id
-AND "missing"."u_id" IS NOT DISTINCT FROM :u_id

bricktracker/sql/part/base/base.sql

@@ -0,0 +1,59 @@
+SELECT
+    "bricktracker_parts"."id",
+    "bricktracker_parts"."figure",
+    "bricktracker_parts"."part",
+    "bricktracker_parts"."color",
+    "bricktracker_parts"."spare",
+    "bricktracker_parts"."quantity",
+    "bricktracker_parts"."element",
+    --"bricktracker_parts"."rebrickable_inventory",
+    "bricktracker_parts"."missing",
+    "bricktracker_parts"."damaged",
+    --"rebrickable_parts"."part",
+    --"rebrickable_parts"."color_id",
+    "rebrickable_parts"."color_name",
+    "rebrickable_parts"."color_rgb",
+    "rebrickable_parts"."color_transparent",
+    "rebrickable_parts"."name",
+    --"rebrickable_parts"."category",
+    "rebrickable_parts"."image",
+    "rebrickable_parts"."image_id",
+    "rebrickable_parts"."url",
+    "rebrickable_parts"."print",
+    {% block total_missing %}
+    NULL AS "total_missing", -- dummy for order: total_missing
+    {% endblock %}
+    {% block total_damaged %}
+    NULL AS "total_damaged", -- dummy for order: total_damaged
+    {% endblock %}
+    {% block total_quantity %}
+    NULL AS "total_quantity", -- dummy for order: total_quantity
+    {% endblock %}
+    {% block total_spare %}
+    NULL AS "total_spare", -- dummy for order: total_spare
+    {% endblock %}
+    {% block total_sets %}
+    NULL AS "total_sets", -- dummy for order: total_sets
+    {% endblock %}
+    {% block total_minifigures %}
+    NULL AS "total_minifigures" -- dummy for order: total_minifigures
+    {% endblock %}
+FROM "bricktracker_parts"
+
+INNER JOIN "rebrickable_parts"
+ON "bricktracker_parts"."part" IS NOT DISTINCT FROM "rebrickable_parts"."part"
+AND "bricktracker_parts"."color" IS NOT DISTINCT FROM "rebrickable_parts"."color_id"
+
+{% block join %}{% endblock %}
+
+{% block where %}{% endblock %}
+
+{% block group %}{% endblock %}
+
+{% if order %}
+ORDER BY {{ order }}
+{% endif %}
+
+{% if limit %}
+LIMIT {{ limit }}
+{% endif %}

bricktracker/sql/part/base/select.sql

@@ -1,43 +0,0 @@
-SELECT
-    "inventory"."set_num",
-    "inventory"."id",
-    "inventory"."part_num",
-    "inventory"."name",
-    "inventory"."part_img_url",
-    "inventory"."part_img_url_id",
-    "inventory"."color_id",
-    "inventory"."color_name",
-    "inventory"."quantity",
-    "inventory"."is_spare",
-    "inventory"."element_id",
-    "inventory"."u_id",
-    {% block total_missing %}
-    NULL AS "total_missing", -- dummy for order: total_missing
-    {% endblock %}
-    {% block total_quantity %}
-    NULL AS "total_quantity", -- dummy for order: total_quantity
-    {% endblock %}
-    {% block total_spare %}
-    NULL AS "total_spare", -- dummy for order: total_spare
-    {% endblock %}
-    {% block total_sets %}
-    NULL AS "total_sets", -- dummy for order: total_sets
-    {% endblock %}
-    {% block total_minifigures %}
-    NULL AS "total_minifigures" -- dummy for order: total_minifigures
-    {% endblock %}
-FROM "inventory"
-
-{% block join %}{% endblock %}
-
-{% block where %}{% endblock %}
-
-{% block group %}{% endblock %}
-
-{% if order %}
-ORDER BY {{ order }}
-{% endif %}
-
-{% if limit %}
-LIMIT {{ limit }}
-{% endif %}

bricktracker/sql/part/insert.sql

@@ -1,27 +1,19 @@
-INSERT INTO inventory (
-    "set_num",
+INSERT INTO "bricktracker_parts" (
     "id",
-    "part_num",
-    "name",
-    "part_img_url",
-    "part_img_url_id",
-    "color_id",
-    "color_name",
+    "figure",
+    "part",
+    "color",
+    "spare",
     "quantity",
-    "is_spare",
-    "element_id",
-    "u_id"
+    "element",
+    "rebrickable_inventory"
 ) VALUES (
-    :set_num,
     :id,
-    :part_num,
-    :name,
-    :part_img_url,
-    :part_img_url_id,
-    :color_id,
-    :color_name,
+    :figure,
+    :part,
+    :color,
+    :spare,
     :quantity,
-    :is_spare,
-    :element_id,
-    :u_id
+    :element,
+    :rebrickable_inventory
 )

bricktracker/sql/part/list/all.sql

@@ -1,43 +1,34 @@
-{% extends 'part/base/select.sql' %}
+{% extends 'part/base/base.sql' %}
 
 {% block total_missing %}
-SUM(IFNULL("missing"."quantity", 0)) AS "total_missing",
+SUM("bricktracker_parts"."missing") AS "total_missing",
+{% endblock %}
+
+{% block total_damaged %}
+SUM("bricktracker_parts"."damaged") AS "total_damaged",
 {% endblock %}
 
 {% block total_quantity %}
-SUM("inventory"."quantity" * IFNULL("minifigures"."quantity", 1)) AS "total_quantity",
+SUM("bricktracker_parts"."quantity" * IFNULL("bricktracker_minifigures"."quantity", 1)) AS "total_quantity",
 {% endblock %}
 
 {% block total_sets %}
-COUNT(DISTINCT "bricktracker_sets"."id") AS "total_sets",
+COUNT(DISTINCT "bricktracker_parts"."id") AS "total_sets",
 {% endblock %}
 
 {% block total_minifigures %}
-SUM(IFNULL("minifigures"."quantity", 0)) AS "total_minifigures"
+SUM(IFNULL("bricktracker_minifigures"."quantity", 0)) AS "total_minifigures"
 {% endblock %}
 
 {% block join %}
-LEFT JOIN "missing"
-ON "inventory"."set_num" IS NOT DISTINCT FROM "missing"."set_num"
-AND "inventory"."id" IS NOT DISTINCT FROM "missing"."id"
-AND "inventory"."part_num" IS NOT DISTINCT FROM "missing"."part_num"
-AND "inventory"."color_id" IS NOT DISTINCT FROM "missing"."color_id"
-AND "inventory"."element_id" IS NOT DISTINCT FROM "missing"."element_id"
-AND "inventory"."u_id" IS NOT DISTINCT FROM "missing"."u_id"
-
-LEFT JOIN "minifigures"
-ON "inventory"."set_num" IS NOT DISTINCT FROM "minifigures"."fig_num"
-AND "inventory"."u_id" IS NOT DISTINCT FROM "minifigures"."u_id"
-
-LEFT JOIN "bricktracker_sets"
-ON "inventory"."u_id" IS NOT DISTINCT FROM "bricktracker_sets"."id"
+LEFT JOIN "bricktracker_minifigures"
+ON "bricktracker_parts"."id" IS NOT DISTINCT FROM "bricktracker_minifigures"."id"
+AND "bricktracker_parts"."figure" IS NOT DISTINCT FROM "bricktracker_minifigures"."figure"
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    "inventory"."part_num",
-    "inventory"."name",
-    "inventory"."color_id",
-    "inventory"."is_spare",
-    "inventory"."element_id"
+    "bricktracker_parts"."part",
+    "bricktracker_parts"."color",
+    "bricktracker_parts"."spare"
 {% endblock %}

bricktracker/sql/part/list/from_minifigure.sql

@@ -1,28 +1,21 @@
 
-{% extends 'part/base/select.sql' %}
+{% extends 'part/base/base.sql' %}
 
 {% block total_missing %}
-SUM(IFNULL("missing"."quantity", 0)) AS "total_missing",
+SUM("bricktracker_parts"."missing") AS "total_missing",
 {% endblock %}
 
-{% block join %}
-LEFT JOIN "missing"
-ON "missing"."set_num" IS NOT DISTINCT FROM "inventory"."set_num"
-AND "missing"."id" IS NOT DISTINCT FROM "inventory"."id"
-AND "missing"."part_num" IS NOT DISTINCT FROM "inventory"."part_num"
-AND "missing"."color_id" IS NOT DISTINCT FROM "inventory"."color_id"
-AND "missing"."element_id" IS NOT DISTINCT FROM "inventory"."element_id"
+{% block total_damaged %}
+SUM("bricktracker_parts"."damaged") AS "total_damaged",
 {% endblock %}
 
 {% block where %}
-WHERE "inventory"."set_num" IS NOT DISTINCT FROM :set_num
+WHERE "bricktracker_parts"."figure" IS NOT DISTINCT FROM :figure
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    "inventory"."part_num",
-    "inventory"."name",
-    "inventory"."color_id",
-    "inventory"."is_spare",
-    "inventory"."element_id"
+    "bricktracker_parts"."part",
+    "bricktracker_parts"."color",
+    "bricktracker_parts"."spare"
 {% endblock %}

bricktracker/sql/part/list/from_print.sql

@@ -0,0 +1,18 @@
+
+{% extends 'part/base/base.sql' %}
+
+{% block total_missing %}{% endblock %}
+
+{% block total_damaged %}{% endblock %}
+
+{% block where %}
+WHERE "rebrickable_parts"."print" IS NOT DISTINCT FROM :print
+AND "bricktracker_parts"."color" IS NOT DISTINCT FROM :color
+AND "bricktracker_parts"."part" IS DISTINCT FROM :part
+{% endblock %}
+
+{% block group %}
+GROUP BY
+    "bricktracker_parts"."part",
+    "bricktracker_parts"."color"
+{% endblock %}

bricktracker/sql/part/list/from_set.sql

@@ -1,21 +0,0 @@
-
-{% extends 'part/base/select.sql' %}
-
-{% block total_missing %}
-IFNULL("missing"."quantity", 0) AS "total_missing",
-{% endblock %}
-
-{% block join %}
-LEFT JOIN "missing"
-ON "inventory"."set_num" IS NOT DISTINCT FROM "missing"."set_num"
-AND "inventory"."id" IS NOT DISTINCT FROM "missing"."id"
-AND "inventory"."part_num" IS NOT DISTINCT FROM "missing"."part_num"
-AND "inventory"."color_id" IS NOT DISTINCT FROM "missing"."color_id"
-AND "inventory"."element_id" IS NOT DISTINCT FROM "missing"."element_id"
-AND "inventory"."u_id" IS NOT DISTINCT FROM "missing"."u_id"
-{% endblock %}
-
-{% block where %}
-WHERE "inventory"."u_id" IS NOT DISTINCT FROM :u_id
-AND "inventory"."set_num" IS NOT DISTINCT FROM :set_num
-{% endblock %}

bricktracker/sql/part/list/missing.sql

@@ -1,36 +0,0 @@
-{% extends 'part/base/select.sql' %}
-
-{% block total_missing %}
-SUM(IFNULL("missing"."quantity", 0)) AS "total_missing",
-{% endblock %}
-
-{% block total_sets %}
-COUNT("inventory"."u_id")  - COUNT("minifigures"."u_id") AS "total_sets",
-{% endblock %}
-
-{% block total_minifigures %}
-SUM(IFNULL("minifigures"."quantity", 0)) AS "total_minifigures"
-{% endblock %}
-
-{% block join %}
-INNER JOIN "missing"
-ON "missing"."set_num" IS NOT DISTINCT FROM "inventory"."set_num"
-AND "missing"."id" IS NOT DISTINCT FROM "inventory"."id"
-AND "missing"."part_num" IS NOT DISTINCT FROM "inventory"."part_num"
-AND "missing"."color_id" IS NOT DISTINCT FROM "inventory"."color_id"
-AND "missing"."element_id" IS NOT DISTINCT FROM "inventory"."element_id"
-AND "missing"."u_id" IS NOT DISTINCT FROM "inventory"."u_id"
-
-LEFT JOIN "minifigures"
-ON "missing"."set_num" IS NOT DISTINCT FROM "minifigures"."fig_num"
-AND "missing"."u_id" IS NOT DISTINCT FROM "minifigures"."u_id"
-{% endblock %}
-
-{% block group %}
-GROUP BY
-    "inventory"."part_num",
-    "inventory"."name",
-    "inventory"."color_id",
-    "inventory"."is_spare",
-    "inventory"."element_id"
-{% endblock %}

bricktracker/sql/part/list/problem.sql

@@ -0,0 +1,35 @@
+{% extends 'part/base/base.sql' %}
+
+{% block total_missing %}
+SUM("bricktracker_parts"."missing") AS "total_missing",
+{% endblock %}
+
+{% block total_damaged %}
+SUM("bricktracker_parts"."damaged") AS "total_damaged",
+{% endblock %}
+
+{% block total_sets %}
+COUNT("bricktracker_parts"."id")  - COUNT("bricktracker_parts"."figure") AS "total_sets",
+{% endblock %}
+
+{% block total_minifigures %}
+SUM(IFNULL("bricktracker_minifigures"."quantity", 0)) AS "total_minifigures"
+{% endblock %}
+
+{% block join %}
+LEFT JOIN "bricktracker_minifigures"
+ON "bricktracker_parts"."id" IS NOT DISTINCT FROM "bricktracker_minifigures"."id"
+AND "bricktracker_parts"."figure" IS NOT DISTINCT FROM "bricktracker_minifigures"."figure"
+{% endblock %}
+
+{% block where %}
+WHERE "bricktracker_parts"."missing" > 0
+OR "bricktracker_parts"."damaged" > 0
+{% endblock %}
+
+{% block group %}
+GROUP BY
+    "bricktracker_parts"."part",
+    "bricktracker_parts"."color",
+    "bricktracker_parts"."spare"
+{% endblock %}

bricktracker/sql/part/list/specific.sql

@@ -0,0 +1,15 @@
+
+{% extends 'part/base/base.sql' %}
+
+{% block total_missing %}
+IFNULL("bricktracker_parts"."missing", 0) AS "total_missing",
+{% endblock %}
+
+{% block total_damaged %}
+IFNULL("bricktracker_parts"."damaged", 0) AS "total_damaged",
+{% endblock %}
+
+{% block where %}
+WHERE "bricktracker_parts"."id" IS NOT DISTINCT FROM :id
+AND "bricktracker_parts"."figure" IS NOT DISTINCT FROM :figure
+{% endblock %}

bricktracker/sql/part/select/generic.sql

@@ -1,40 +1,34 @@
-{% extends 'part/base/select.sql' %}
+{% extends 'part/base/base.sql' %}
 
 {% block total_missing %}
-SUM(IFNULL("missing"."quantity", 0)) AS "total_missing",
+SUM("bricktracker_parts"."missing") AS "total_missing",
+{% endblock %}
+
+{% block total_damaged %}
+SUM("bricktracker_parts"."damaged") AS "total_damaged",
 {% endblock %}
 
 {% block total_quantity %}
-SUM((NOT "inventory"."is_spare") * "inventory"."quantity" * IFNULL("minifigures"."quantity", 1)) AS "total_quantity",
+SUM((NOT "bricktracker_parts"."spare") * "bricktracker_parts"."quantity" * IFNULL("bricktracker_minifigures"."quantity", 1)) AS "total_quantity",
 {% endblock %}
 
 {% block total_spare %}
-SUM("inventory"."is_spare" * "inventory"."quantity" * IFNULL("minifigures"."quantity", 1)) AS "total_spare",
+SUM("bricktracker_parts"."spare" * "bricktracker_parts"."quantity" * IFNULL("bricktracker_minifigures"."quantity", 1)) AS "total_spare",
 {% endblock %}
 
 {% block join %}
-LEFT JOIN "missing"
-ON "inventory"."set_num" IS NOT DISTINCT FROM "missing"."set_num"
-AND "inventory"."id" IS NOT DISTINCT FROM "missing"."id"
-AND "inventory"."part_num" IS NOT DISTINCT FROM "missing"."part_num"
-AND "inventory"."color_id" IS NOT DISTINCT FROM "missing"."color_id"
-AND "inventory"."element_id" IS NOT DISTINCT FROM "missing"."element_id"
-AND "inventory"."u_id" IS NOT DISTINCT FROM "missing"."u_id"
-
-LEFT JOIN "minifigures"
-ON "inventory"."set_num" IS NOT DISTINCT FROM "minifigures"."fig_num"
-AND "inventory"."u_id" IS NOT DISTINCT FROM "minifigures"."u_id"
+LEFT JOIN "bricktracker_minifigures"
+ON "bricktracker_parts"."id" IS NOT DISTINCT FROM "bricktracker_minifigures"."id"
+AND "bricktracker_parts"."figure" IS NOT DISTINCT FROM "bricktracker_minifigures"."figure"
 {% endblock %}
 
 {% block where %}
-WHERE "inventory"."part_num" IS NOT DISTINCT FROM :part_num
-AND "inventory"."color_id" IS NOT DISTINCT FROM :color_id
-AND "inventory"."element_id" IS NOT DISTINCT FROM :element_id
+WHERE "bricktracker_parts"."part" IS NOT DISTINCT FROM :part
+AND "bricktracker_parts"."color" IS NOT DISTINCT FROM :color
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    "inventory"."part_num",
-    "inventory"."color_id",
-    "inventory"."element_id"
+    "bricktracker_parts"."part",
+    "bricktracker_parts"."color"
 {% endblock %}

bricktracker/sql/part/select/specific.sql

@@ -1,24 +1,18 @@
-{% extends 'part/base/select.sql' %}
-
-{% block join %}
-LEFT JOIN "missing"
-ON "inventory"."set_num" IS NOT DISTINCT FROM "missing"."set_num"
-AND "inventory"."id" IS NOT DISTINCT FROM "missing"."id"
-AND "inventory"."u_id" IS NOT DISTINCT FROM "missing"."u_id"
-{% endblock %}
+{% extends 'part/base/base.sql' %}
 
 {% block where %}
-WHERE "inventory"."u_id" IS NOT DISTINCT FROM :u_id
-AND "inventory"."set_num" IS NOT DISTINCT FROM :set_num
-AND "inventory"."id" IS NOT DISTINCT FROM :id
+WHERE "bricktracker_parts"."id" IS NOT DISTINCT FROM :id
+AND "bricktracker_parts"."figure" IS NOT DISTINCT FROM :figure
+AND "bricktracker_parts"."part" IS NOT DISTINCT FROM :part
+AND "bricktracker_parts"."color" IS NOT DISTINCT FROM :color
+AND "bricktracker_parts"."spare" IS NOT DISTINCT FROM :spare
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    "inventory"."set_num",
-    "inventory"."id",
-    "inventory"."part_num",
-    "inventory"."color_id",
-    "inventory"."element_id",
-    "inventory"."u_id"
+    "bricktracker_parts"."id",
+    "bricktracker_parts"."figure",
+    "bricktracker_parts"."part",
+    "bricktracker_parts"."color",
+    "bricktracker_parts"."spare"
 {% endblock %}

bricktracker/sql/part/update/damaged.sql

@@ -0,0 +1,7 @@
+UPDATE "bricktracker_parts"
+SET "damaged" = :damaged
+WHERE "bricktracker_parts"."id" IS NOT DISTINCT FROM :id
+AND "bricktracker_parts"."figure" IS NOT DISTINCT FROM :figure
+AND "bricktracker_parts"."part" IS NOT DISTINCT FROM :part
+AND "bricktracker_parts"."color" IS NOT DISTINCT FROM :color
+AND "bricktracker_parts"."spare" IS NOT DISTINCT FROM :spare

bricktracker/sql/part/update/missing.sql

@@ -0,0 +1,7 @@
+UPDATE "bricktracker_parts"
+SET "missing" = :missing
+WHERE "bricktracker_parts"."id" IS NOT DISTINCT FROM :id
+AND "bricktracker_parts"."figure" IS NOT DISTINCT FROM :figure
+AND "bricktracker_parts"."part" IS NOT DISTINCT FROM :part
+AND "bricktracker_parts"."color" IS NOT DISTINCT FROM :color
+AND "bricktracker_parts"."spare" IS NOT DISTINCT FROM :spare

bricktracker/sql/rebrickable/minifigure/insert.sql

@@ -0,0 +1,17 @@
+INSERT OR IGNORE INTO "rebrickable_minifigures" (
+    "figure",
+    "number",
+    "name",
+    "image"
+) VALUES (
+    :figure,
+    :number,
+    :name,
+    :image
+)
+ON CONFLICT("figure")
+DO UPDATE SET
+"number" = :number,
+"name" = :name,
+"image" = :image
+WHERE "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM :figure

bricktracker/sql/rebrickable/minifigure/list.sql

@@ -0,0 +1,6 @@
+SELECT
+    "rebrickable_minifigures"."figure",
+    "rebrickable_minifigures"."number",
+    "rebrickable_minifigures"."name",
+    "rebrickable_minifigures"."image"
+FROM "rebrickable_minifigures"

bricktracker/sql/rebrickable/minifigure/select.sql

@@ -0,0 +1,8 @@
+SELECT
+    "rebrickable_minifigures"."figure",
+    "rebrickable_minifigures"."number",
+    "rebrickable_minifigures"."name",
+    "rebrickable_minifigures"."image"
+FROM "rebrickable_minifigures"
+
+WHERE "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM :figure

bricktracker/sql/rebrickable/part/insert.sql

@@ -0,0 +1,38 @@
+INSERT OR IGNORE INTO "rebrickable_parts" (
+    "part",
+    "color_id",
+    "color_name",
+    "color_rgb",
+    "color_transparent",
+    "name",
+    "category",
+    "image",
+    "image_id",
+    "url",
+    "print"
+) VALUES (
+    :part,
+    :color_id,
+    :color_name,
+    :color_rgb,
+    :color_transparent,
+    :name,
+    :category,
+    :image,
+    :image_id,
+    :url,
+    :print
+)
+ON CONFLICT("part", "color_id")
+DO UPDATE SET
+"color_name" = :color_name,
+"color_rgb" = :color_rgb,
+"color_transparent" = :color_transparent,
+"name" = :name,
+"category" = :category,
+"image" = :image,
+"image_id" = :image_id,
+"url" = :url,
+"print" = :print
+WHERE "rebrickable_parts"."part" IS NOT DISTINCT FROM :part
+AND "rebrickable_parts"."color_id" IS NOT DISTINCT FROM :color_id

bricktracker/sql/rebrickable/part/list.sql

@@ -0,0 +1,13 @@
+SELECT
+    "rebrickable_parts"."part",
+    "rebrickable_parts"."color_id",
+    "rebrickable_parts"."color_name",
+    "rebrickable_parts"."color_rgb",
+    "rebrickable_parts"."color_transparent",
+    "rebrickable_parts"."name",
+    "rebrickable_parts"."category",
+    "rebrickable_parts"."image",
+    "rebrickable_parts"."image_id",
+    "rebrickable_parts"."url",
+    "rebrickable_parts"."print"
+FROM "rebrickable_parts"

bricktracker/sql/rebrickable/part/select.sql

@@ -0,0 +1,16 @@
+SELECT
+    "rebrickable_parts"."part",
+    "rebrickable_parts"."color_id",
+    "rebrickable_parts"."color_name",
+    "rebrickable_parts"."color_rgb",
+    "rebrickable_parts"."color_transparent",
+    "rebrickable_parts"."name",
+    "rebrickable_parts"."category",
+    "rebrickable_parts"."image",
+    "rebrickable_parts"."image_id",
+    "rebrickable_parts"."url",
+    "rebrickable_parts"."print"
+FROM "rebrickable_parts"
+
+WHERE "rebrickable_minifigures"."part" IS NOT DISTINCT FROM :figure
+AND "rebrickable_minifigures"."color_id" IS NOT DISTINCT FROM :color

bricktracker/sql/rebrickable/set/insert.sql

@@ -21,3 +21,15 @@ INSERT OR IGNORE INTO "rebrickable_sets" (
     :url,
     :last_modified
 )
+ON CONFLICT("set")
+DO UPDATE SET
+    "number" = :number,
+    "version" = :version,
+    "name" = :name,
+    "year" = :year,
+    "theme_id" = :theme_id,
+    "number_of_parts" = :number_of_parts,
+    "image" = :image,
+    "url" = :url,
+    "last_modified" = :last_modified
+WHERE "rebrickable_sets"."set" IS NOT DISTINCT FROM :set

bricktracker/sql/schema/drop.sql

@@ -1,12 +1,25 @@
 BEGIN transaction;
 
+DROP TABLE IF EXISTS "bricktracker_metadata_owners";
+DROP TABLE IF EXISTS "bricktracker_metadata_statuses";
+DROP TABLE IF EXISTS "bricktracker_metadata_tags";
+DROP TABLE IF EXISTS "bricktracker_minifigures";
+DROP TABLE IF EXISTS "bricktracker_parts";
 DROP TABLE IF EXISTS "bricktracker_sets";
 DROP TABLE IF EXISTS "bricktracker_set_checkboxes";
+DROP TABLE IF EXISTS "bricktracker_set_owners";
 DROP TABLE IF EXISTS "bricktracker_set_statuses";
+DROP TABLE IF EXISTS "bricktracker_set_storages";
+DROP TABLE IF EXISTS "bricktracker_set_tags";
 DROP TABLE IF EXISTS "bricktracker_wishes";
 DROP TABLE IF EXISTS "inventory";
+DROP TABLE IF EXISTS "inventory_old";
 DROP TABLE IF EXISTS "minifigures";
+DROP TABLE IF EXISTS "minifigures_old";
 DROP TABLE IF EXISTS "missing";
+DROP TABLE IF EXISTS "missing_old";
+DROP TABLE IF EXISTS "rebrickable_minifigures";
+DROP TABLE IF EXISTS "rebrickable_parts";
 DROP TABLE IF EXISTS "rebrickable_sets";
 DROP TABLE IF EXISTS "sets";
 DROP TABLE IF EXISTS "sets_old";

bricktracker/sql/set/base/base.sql

@@ -9,19 +9,28 @@ SELECT
     "rebrickable_sets"."number_of_parts",
     "rebrickable_sets"."image",
     "rebrickable_sets"."url",
+    {% block owners %}
+        {% if owners %}{{ owners }},{% endif %}
+    {% endblock %}
+    {% block tags %}
+        {% if tags %}{{ tags }},{% endif %}
+    {% endblock %}
     {% block statuses %}
         {% if statuses %}{{ statuses }},{% endif %}
     {% endblock %}
     {% block total_missing %}
     NULL AS "total_missing", -- dummy for order: total_missing
     {% endblock %}
+    {% block total_damaged %}
+    NULL AS "total_damaged", -- dummy for order: total_damaged
+    {% endblock %}
     {% block total_quantity %}
     NULL AS "total_quantity", -- dummy for order: total_quantity
     {% endblock %}
 FROM "bricktracker_sets"
 
 INNER JOIN "rebrickable_sets"
-ON "bricktracker_sets"."rebrickable_set" IS NOT DISTINCT FROM "rebrickable_sets"."set"
+ON "bricktracker_sets"."set" IS NOT DISTINCT FROM "rebrickable_sets"."set"
 
 {% block join %}{% endblock %}
 

bricktracker/sql/set/base/full.sql

@@ -5,7 +5,11 @@
 {% endblock %}
 
 {% block total_missing %}
-IFNULL("missing_join"."total", 0) AS "total_missing",
+IFNULL("problem_join"."total_missing", 0) AS "total_missing",
+{% endblock %}
+
+{% block total_damaged %}
+IFNULL("problem_join"."total_damaged", 0) AS "total_damaged",
 {% endblock %}
 
 {% block total_quantity %}
@@ -13,30 +17,41 @@ IFNULL("minifigures_join"."total", 0) AS "total_minifigures"
 {% endblock %}
 
 {% block join %}
+{% if owners %}
+LEFT JOIN "bricktracker_set_owners"
+ON "bricktracker_sets"."id" IS NOT DISTINCT FROM "bricktracker_set_owners"."id"
+{% endif %}
+
 {% if statuses %}
 LEFT JOIN "bricktracker_set_statuses"
-ON "bricktracker_sets"."id" IS NOT DISTINCT FROM "bricktracker_set_statuses"."bricktracker_set_id"
+ON "bricktracker_sets"."id" IS NOT DISTINCT FROM "bricktracker_set_statuses"."id"
+{% endif %}
+
+{% if tags %}
+LEFT JOIN "bricktracker_set_tags"
+ON "bricktracker_sets"."id" IS NOT DISTINCT FROM "bricktracker_set_tags"."id"
 {% endif %}
 
 -- LEFT JOIN + SELECT to avoid messing the total
 LEFT JOIN (
     SELECT
-        "missing"."u_id",
-        SUM("missing"."quantity") AS "total"
-    FROM "missing"
+        "bricktracker_parts"."id",
+        SUM("bricktracker_parts"."missing") AS "total_missing",
+        SUM("bricktracker_parts"."damaged") AS "total_damaged"
+    FROM "bricktracker_parts"
     {% block where_missing %}{% endblock %}
-    GROUP BY "u_id"
-) "missing_join"
-ON "bricktracker_sets"."id" IS NOT DISTINCT FROM "missing_join"."u_id"
+    GROUP BY "bricktracker_parts"."id"
+) "problem_join"
+ON "bricktracker_sets"."id" IS NOT DISTINCT FROM "problem_join"."id"
 
 -- LEFT JOIN + SELECT to avoid messing the total
 LEFT JOIN (
     SELECT
-       "minifigures"."u_id",
-       SUM("minifigures"."quantity") AS "total"
-    FROM "minifigures"
+       "bricktracker_minifigures"."id",
+       SUM("bricktracker_minifigures"."quantity") AS "total"
+    FROM "bricktracker_minifigures"
     {% block where_minifigures %}{% endblock %}
-    GROUP BY "u_id"
+    GROUP BY "bricktracker_minifigures"."id"
 ) "minifigures_join"
-ON "bricktracker_sets"."id" IS NOT DISTINCT FROM "minifigures_join"."u_id"
+ON "bricktracker_sets"."id" IS NOT DISTINCT FROM "minifigures_join"."id"
 {% endblock %}

bricktracker/sql/set/base/light.sql

@@ -1,6 +1,6 @@
 SELECT
     "bricktracker_sets"."id",
-    "bricktracker_sets"."rebrickable_set" AS "set"
+    "bricktracker_sets"."set"
 FROM "bricktracker_sets"
 
 {% block join %}{% endblock %}

bricktracker/sql/set/delete/set.sql

@@ -6,16 +6,19 @@ BEGIN TRANSACTION;
 DELETE FROM "bricktracker_sets"
 WHERE "bricktracker_sets"."id" IS NOT DISTINCT FROM '{{ id }}';
 
+DELETE FROM "bricktracker_set_owners"
+WHERE "bricktracker_set_owners"."id" IS NOT DISTINCT FROM '{{ id }}';
+
 DELETE FROM "bricktracker_set_statuses"
-WHERE "bricktracker_set_statuses"."bricktracker_set_id" IS NOT DISTINCT FROM '{{ id }}';
+WHERE "bricktracker_set_statuses"."id" IS NOT DISTINCT FROM '{{ id }}';
 
-DELETE FROM "minifigures"
-WHERE "minifigures"."u_id" IS NOT DISTINCT FROM '{{ id }}';
+DELETE FROM "bricktracker_set_tags"
+WHERE "bricktracker_set_tags"."id" IS NOT DISTINCT FROM '{{ id }}';
 
-DELETE FROM "missing"
-WHERE "missing"."u_id" IS NOT DISTINCT FROM '{{ id }}';
+DELETE FROM "bricktracker_minifigures"
+WHERE "bricktracker_minifigures"."id" IS NOT DISTINCT FROM '{{ id }}';
 
-DELETE FROM "inventory"
-WHERE "inventory"."u_id" IS NOT DISTINCT FROM '{{ id }}';
+DELETE FROM "bricktracker_parts"
+WHERE "bricktracker_parts"."id" IS NOT DISTINCT FROM '{{ id }}';
 
 COMMIT;

bricktracker/sql/set/insert.sql

@@ -1,6 +1,6 @@
 INSERT OR IGNORE INTO "bricktracker_sets" (
     "id",
-    "rebrickable_set"
+    "set"
 ) VALUES (
     :id,
     :set

bricktracker/sql/set/list/damaged_minifigure.sql

@@ -0,0 +1,11 @@
+{% extends 'set/base/full.sql' %}
+
+{% block where %}
+WHERE "bricktracker_sets"."id" IN (
+    SELECT "bricktracker_parts"."id"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."figure" IS NOT DISTINCT FROM :figure
+    AND "bricktracker_parts"."missing" > 0
+    GROUP BY "bricktracker_parts"."id"
+)
+{% endblock %}

bricktracker/sql/set/list/damaged_part.sql

@@ -0,0 +1,12 @@
+{% extends 'set/base/full.sql' %}
+
+{% block where %}
+WHERE "bricktracker_sets"."id" IN (
+    SELECT "bricktracker_parts"."id"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."part" IS NOT DISTINCT FROM :part
+    AND "bricktracker_parts"."color" IS NOT DISTINCT FROM :color
+    AND "bricktracker_parts"."damaged" > 0
+    GROUP BY "bricktracker_parts"."id"
+)
+{% endblock %}

bricktracker/sql/set/list/generic.sql

@@ -2,5 +2,5 @@
 
 {% block group %}
 GROUP BY
-    "bricktracker_sets"."rebrickable_set"
+    "bricktracker_sets"."set"
 {% endblock %}

bricktracker/sql/set/list/missing_minifigure.sql

@@ -2,12 +2,10 @@
 
 {% block where %}
 WHERE "bricktracker_sets"."id" IN (
-    SELECT
-        "missing"."u_id"
-    FROM "missing"
-
-    WHERE "missing"."set_num" IS NOT DISTINCT FROM :fig_num
-
-    GROUP BY "missing"."u_id"
+    SELECT "bricktracker_parts"."id"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."figure" IS NOT DISTINCT FROM :figure
+    AND "bricktracker_parts"."missing" > 0
+    GROUP BY "bricktracker_parts"."id"
 )
 {% endblock %}

bricktracker/sql/set/list/missing_part.sql

@@ -2,14 +2,11 @@
 
 {% block where %}
 WHERE "bricktracker_sets"."id" IN (
-    SELECT
-        "missing"."u_id"
-    FROM "missing"
-
-    WHERE "missing"."color_id" IS NOT DISTINCT FROM :color_id
-    AND "missing"."element_id" IS NOT DISTINCT FROM :element_id
-    AND "missing"."part_num" IS NOT DISTINCT FROM :part_num
-
-    GROUP BY "missing"."u_id"
+    SELECT "bricktracker_parts"."id"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."part" IS NOT DISTINCT FROM :part
+    AND "bricktracker_parts"."color" IS NOT DISTINCT FROM :color
+    AND "bricktracker_parts"."missing" > 0
+    GROUP BY "bricktracker_parts"."id"
 )
 {% endblock %}

bricktracker/sql/set/list/using_minifigure.sql

@@ -2,12 +2,9 @@
 
 {% block where %}
 WHERE "bricktracker_sets"."id" IN (
-    SELECT
-        "inventory"."u_id"
-    FROM "inventory"
-
-    WHERE "inventory"."set_num" IS NOT DISTINCT FROM :fig_num
-
-    GROUP BY "inventory"."u_id"
+    SELECT "bricktracker_parts"."id"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."figure" IS NOT DISTINCT FROM :figure
+    GROUP BY "bricktracker_parts"."id"
 )
 {% endblock %}

bricktracker/sql/set/list/using_part.sql

@@ -2,14 +2,10 @@
 
 {% block where %}
 WHERE "bricktracker_sets"."id" IN (
-    SELECT
-        "inventory"."u_id"
-    FROM "inventory"
-
-    WHERE "inventory"."color_id" IS NOT DISTINCT FROM :color_id
-    AND "inventory"."element_id" IS NOT DISTINCT FROM :element_id
-    AND "inventory"."part_num" IS NOT DISTINCT FROM :part_num
-
-    GROUP BY "inventory"."u_id"
+    SELECT "bricktracker_parts"."id"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."part" IS NOT DISTINCT FROM :part
+    AND "bricktracker_parts"."color" IS NOT DISTINCT FROM :color
+    GROUP BY "bricktracker_parts"."id"
 )
 {% endblock %}

bricktracker/sql/set/metadata/owner/base.sql

@@ -0,0 +1,6 @@
+SELECT
+    "bricktracker_metadata_owners"."id",
+    "bricktracker_metadata_owners"."name"
+FROM "bricktracker_metadata_owners"
+
+{% block where %}{% endblock %}

bricktracker/sql/set/metadata/owner/delete.sql

@@ -0,0 +1,9 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE "bricktracker_set_owners"
+DROP COLUMN "owner_{{ id }}";
+
+DELETE FROM "bricktracker_metadata_owners"
+WHERE "bricktracker_metadata_owners"."id" IS NOT DISTINCT FROM '{{ id }}';
+
+COMMIT;