Added migration to get new bricklink data fields, fixed bricklink links, added set refresh based on missing bricklink data

Reviewed-on: FrederikBaerentsen/BrickTracker#81

.dockerignore

@@ -19,6 +19,8 @@ LICENSE
 
 # Database
 *.db
+*.db-shm
+*.db-wal
 
 # Python
 **/__pycache__

.env.sample

@@ -2,15 +2,15 @@
 # If set, it will append a direct ORDER BY <whatever you set> to the SQL query
 # while listing objects. You can look at the structure of the SQLite database to
 # see the schema and the column names. Some fields are compound and not visible
-# directly from the schema (joins). You can check the query in the */list.sql files
+# directly from the schema (joins). You can check the query in the */list.sql and */base/*.sql files
 # in the source to see all column names.
-# The usual syntax for those variables is <table>.<column> [ASC|DESC].
+# The usual syntax for those variables is "<table>"."<column>" [ASC|DESC].
 # For composite fields (CASE, SUM, COUNT) the syntax is <field>, there is no <table> name.
 # For instance:
-# - table.name (by table.name, default order)
-# - table.name ASC (by table.name, ascending)
-# - table.name DESC (by table.name, descending)
-# - field (by field, default order)
+# - "table"."name" (by "table"."name", default order)
+# - "table"."name" ASC (by "table"."name", ascending)
+# - "table"."name" DESC (by "table"."name", descending)
+# - "field" (by "field", default order)
 # - ...
 # You can combine the ordering options.
 # You can use the special column name 'rowid' to order by insertion order.
@@ -28,7 +28,8 @@
 # BK_AUTHENTICATION_KEY=change-this-to-something-random
 
 # Optional: Pattern of the link to Bricklink for a part. Will be passed to Python .format()
-# Default: https://www.bricklink.com/v2/catalog/catalogitem.page?P={number}
+# Supports {part} and {color} parameters. BrickLink part numbers and color IDs are used when available.
+# Default: https://www.bricklink.com/v2/catalog/catalogitem.page?P={part}&C={color}
 # BK_BRICKLINK_LINK_PART_PATTERN=
 
 # Optional: Display Bricklink links wherever applicable
@@ -60,6 +61,11 @@
 # Legacy name: DOMAIN_NAME
 # BK_DOMAIN_NAME=http://localhost:3333
 
+# Optional: Format of the timestamp for files on disk (instructions, themes)
+# Check https://docs.python.org/3/library/time.html#time.strftime for format details
+# Default: %d/%m/%Y, %H:%M:%S
+# BK_FILE_DATETIME_FORMAT=%m/%d/%Y, %H:%M
+
 # Optional: IP address the server will listen on.
 # Default: 0.0.0.0
 # BK_HOST=0.0.0.0
@@ -103,13 +109,30 @@
 # Default: false
 # BK_HIDE_ALL_PARTS=true
 
+# Optional: Hide the 'Problems' entry from the menu. Does not disable the route.
+# Default: false
+# Legacy name: BK_HIDE_MISSING_PARTS
+# BK_HIDE_ALL_PROBLEMS_PARTS=true
+
 # Optional: Hide the 'Sets' entry from the menu. Does not disable the route.
 # Default: false
 # BK_HIDE_ALL_SETS=true
 
-# Optional: Hide the 'Missing' entry from the menu. Does not disable the route.
+# Optional: Hide the 'Storages' entry from the menu. Does not disable the route.
 # Default: false
-# BK_HIDE_MISSING_PARTS=true
+# BK_HIDE_ALL_STORAGES=true
+
+# Optional: Hide the 'Instructions' entry in a Set card
+# Default: false
+# BK_HIDE_SET_INSTRUCTIONS=true
+
+# Optional: Hide the 'Damaged' column from the parts table.
+# Default: false
+# BK_HIDE_TABLE_DAMAGED_PARTS=true
+
+# Optional: Hide the 'Missing' column from the parts table.
+# Default: false
+# BK_HIDE_TABLE_MISSING_PARTS=true
 
 # Optional: Hide the 'Wishlist' entry from the menu. Does not disable the route.
 # Default: false
@@ -117,10 +140,11 @@
 
 # Optional: Change the default order of minifigures. By default ordered by insertion order.
 # Useful column names for this option are:
-# - minifigures.fig_num: minifigure ID (fig-xxxxx)
-# - minifigures.name: minifigure name
-# Default: minifigures.name ASC
-# BK_MINIFIGURES_DEFAULT_ORDER=minifigures.name ASC
+# - "rebrickable_minifigures"."figure": minifigure ID (fig-xxxxx)
+# - "rebrickable_minifigures"."number": minifigure ID as an integer (xxxxx)
+# - "rebrickable_minifigures"."name": minifigure name
+# Default: "rebrickable_minifigures"."name" ASC
+# BK_MINIFIGURES_DEFAULT_ORDER="rebrickable_minifigures"."name" ASC
 
 # Optional: Folder where to store the minifigures images, relative to the '/app/static/' folder
 # Default: minifigs
@@ -134,12 +158,13 @@
 
 # Optional: Change the default order of parts. By default ordered by insertion order.
 # Useful column names for this option are:
-# - inventory.part_num: part number
-# - inventory.name: part name
-# - inventory.color_name: par color name
-# - total_missing: number of missing parts
-# Default: inventory.name ASC, inventory.color_name ASC, is_spare ASC
-# BK_PARTS_DEFAULT_ORDER=total_missing DESC, inventory.name ASC
+# - "bricktracker_parts"."part": part number
+# - "bricktracker_parts"."spare": part is a spare part
+# - "rebrickable_parts"."name": part name
+# - "rebrickable_parts"."color_name": part color name
+# - "total_missing": number of missing parts
+# Default: "rebrickable_parts"."name" ASC, "rebrickable_parts"."color_name" ASC, "bricktracker_parts"."spare" ASC
+# BK_PARTS_DEFAULT_ORDER="total_missing" DESC, "rebrickable_parts"."name"."name" ASC
 
 # Optional: Folder where to store the parts images, relative to the '/app/static/' folder
 # Default: parts
@@ -149,6 +174,21 @@
 # Default: 3333
 # BK_PORT=3333
 
+# Optional: Format of the timestamp for purchase dates
+# Check https://docs.python.org/3/library/time.html#time.strftime for format details
+# Default: %d/%m/%Y
+# BK_PURCHASE_DATE_FORMAT=%m/%d/%Y
+
+# Optional: Currency to display for purchase prices.
+# Default: €
+# BK_PURCHASE_CURRENCY=£
+
+# Optional: Change the default order of purchase locations. By default ordered by insertion order.
+# Useful column names for this option are:
+# - "bricktracker_metadata_purchase_locations"."name" ASC: storage name
+# Default: "bricktracker_metadata_purchase_locations"."name" ASC
+# BK_PURCHASE_LOCATION_DEFAULT_ORDER="bricktracker_metadata_purchase_locations"."name" ASC
+
 # Optional: Shuffle the lists on the front page.
 # Default: false
 # Legacy name: RANDOM
@@ -170,16 +210,20 @@
 # BK_REBRICKABLE_IMAGE_NIL_MINIFIGURE=
 
 # Optional: Pattern of the link to Rebrickable for a minifigure. Will be passed to Python .format()
-# Default: https://rebrickable.com/minifigs/{number}
+# Default: https://rebrickable.com/minifigs/{figure}
 # BK_REBRICKABLE_LINK_MINIFIGURE_PATTERN=
 
 # Optional: Pattern of the link to Rebrickable for a part. Will be passed to Python .format()
-# Default: https://rebrickable.com/parts/{number}/_/{color}
+# Default: https://rebrickable.com/parts/{part}/_/{color}
 # BK_REBRICKABLE_LINK_PART_PATTERN=
 
-# Optional: Pattern of the link to Rebrickable for a set. Will be passed to Python .format()
-# Default: https://rebrickable.com/sets/{number}
-# BK_REBRICKABLE_LINK_SET_PATTERN=
+# Optional: Pattern of the link to Rebrickable for instructions. Will be passed to Python .format()
+# Default: https://rebrickable.com/instructions/{path}
+# BK_REBRICKABLE_LINK_INSTRUCTIONS_PATTERN=
+
+# Optional: User-Agent to use when querying Rebrickable outside of the Rebrick python library
+# Default: 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+# BK_REBRICKABLE_USER_AGENT=
 
 # Optional: Display Rebrickable links wherever applicable
 # Default: false
@@ -201,21 +245,29 @@
 
 # Optional: Change the default order of sets. By default ordered by insertion order.
 # Useful column names for this option are:
-# - sets.set_num: set number as a string
-# - sets.name: set name
-# - sets.year: set release year
-# - sets.num_parts: set number of parts
-# - set_number: the number part of set_num as an integer
-# - set_version: the version part of set_num as an integer
-# - total_missing: number of missing parts
-# - total_minifigures: number of minifigures
-# Default: set_number DESC, set_version ASC
-# BK_SETS_DEFAULT_ORDER=sets.year ASC
+# - "rebrickable_sets"."set": set number as a string
+# - "rebrickable_sets"."number": the number part of set as an integer
+# - "rebrickable_sets"."version": the version part of set as an integer
+# - "rebrickable_sets"."name": set name
+# - "rebrickable_sets"."year": set release year
+# - "rebrickable_sets"."number_of_parts": set number of parts
+# - "total_missing": number of missing parts
+# - "total_minifigures": number of minifigures
+# Default: "rebrickable_sets"."number" DESC, "rebrickable_sets"."version" ASC
+# BK_SETS_DEFAULT_ORDER="rebrickable_sets"."year" ASC
 
 # Optional: Folder where to store the sets images, relative to the '/app/static/' folder
 # Default: sets
 # BK_SETS_FOLDER=sets
 
+# Optional: Make the grid filters displayed by default, rather than collapsed
+# Default: false
+# BK_SHOW_GRID_FILTERS=true
+
+# Optional: Make the grid sort displayed by default, rather than collapsed
+# Default: false
+# BK_SHOW_GRID_SORT=true
+
 # Optional: Skip saving or displaying spare parts
 # Default: false
 # BK_SKIP_SPARE_PARTS=true
@@ -228,6 +280,12 @@
 # Default: /bricksocket/
 # BK_SOCKET_PATH=custompath
 
+# Optional: Change the default order of storages. By default ordered by insertion order.
+# Useful column names for this option are:
+# - "bricktracker_metadata_storages"."name" ASC: storage name
+# Default: "bricktracker_metadata_storages"."name" ASC
+# BK_STORAGE_DEFAULT_ORDER="bricktracker_metadata_storages"."name" ASC
+
 # Optional: URL to the themes.csv.gz on Rebrickable
 # Default: https://cdn.rebrickable.com/media/downloads/themes.csv.gz
 # BK_THEMES_FILE_URL=
@@ -249,9 +307,9 @@
 
 # Optional: Change the default order of sets. By default ordered by insertion order.
 # Useful column names for this option are:
-# - wishlist.set_num: set number as a string
-# - wishlist.name: set name
-# - wishlist.year: set release year
-# - wishlist.num_parts: set number of parts
-# Default: wishlist.rowid DESC
-# BK_WISHES_DEFAULT_ORDER=set_number DESC, set_version ASC
+# - "bricktracker_wishes"."set": set number as a string
+# - "bricktracker_wishes"."name": set name
+# - "bricktracker_wishes"."year": set release year
+# - "bricktracker_wishes"."number_of_parts": set number of parts
+# Default: "bricktracker_wishes"."rowid" DESC
+# BK_WISHES_DEFAULT_ORDER="bricktracker_wishes"."set" DESC

.gitignore

@@ -1,6 +1,8 @@
 # Application
 .env
 *.db
+*.db-shm
+*.db-wal
 
 # Python specifics
 __pycache__/
@@ -9,6 +11,7 @@ __pycache__/
 # Static folders
 static/instructions/
 static/minifigs/
+static/minifigures/
 static/parts/
 static/sets/
 

CHANGELOG.md

@@ -0,0 +1,346 @@
+# Changelog
+
+## Unreleased
+
+### Current PR
+
+- Added search/filter/sort options to `parts` and `minifigures`.
+
+### Next PR
+
+> **Warning**
+> To use the new BrickLink color parameter in URLs, update your `.env` file:
+> `BK_BRICKLINK_LINK_PART_PATTERN=https://www.bricklink.com/v2/catalog/catalogitem.page?P={part}&C={color}`
+
+- Add BrickLink color and part number support for accurate BrickLink URLs
+  - Database migrations to store BrickLink color ID, color name, and part number
+  - Updated Rebrickable API integration to extract BrickLink data from external_ids
+  - Enhanced BrickLink URL generation with proper part number fallback
+  - Extended admin set refresh to detect and track missing BrickLink data
+
+## 1.2.2
+
+Fix legibility of "Damaged" and "Missing" fields for tiny screen by reducing horizontal padding
+Fixed instructions download from Rebrickable
+
+## 1.2.2:
+
+This release fixes a bug where orphaned parts in the `inventory` table are blocking the database upgrade.
+
+## 1.2.1:
+
+This release fixes a bug where you could not add a set if no metadata was configured.
+
+## 1.2.0:
+
+> **Warning**
+> "Missing" part has been renamed to "Problems" to accomodate for missing and damaged parts.
+> The associated environment variables have changed named (the old names are still valid)
+
+### Environment
+
+- Renamed: `BK_HIDE_MISSING_PARTS` -> `BK_HIDE_ALL_PROBLEMS_PARTS`
+- Added: `BK_HIDE_TABLE_MISSING_PARTS`, hide the Missing column in all tables
+- Added: `BK_HIDE_TABLE_DAMAGED_PARTS`, hide the Damaged column in all tables
+- Added: `BK_SHOW_GRID_SORT`, show the sort options on the grid by default
+- Added: `BK_SHOW_GRID_FILTERS`, show the filter options on the grid by default
+- Added: `BK_HIDE_ALL_STORAGES`, hide the "Storages" menu entry
+- Added: `BK_STORAGE_DEFAULT_ORDER`, ordering of storages
+- Added: `BK_PURCHASE_LOCATION_DEFAULT_ORDER`, ordering of purchase locations
+- Added: `BK_PURCHASE_CURRENCY`, currency to display for purchase prices
+- Added: `BK_PURCHASE_DATE_FORMAT`, date format for purchase dates
+- Documented: `BK_FILE_DATETIME_FORMAT`, date format for files on disk (instructions, theme)
+
+### Code
+
+- Changer
+    - Revert the checked state of a checkbox if an error occured
+
+- Form
+    - Migrate missing input fields to BrickChanger
+
+- General cleanup
+
+- Metadata
+    - Underlying class to implement more metadata-like features
+
+- Minifigure
+    - Deduplicate
+    - Compute number of parts
+
+- Parts
+    - Damaged parts
+
+- Sets
+    - Refresh data from Rebrickable
+    - Fix missing @login_required for set deletion
+    - Ownership
+    - Tags
+    - Storage
+    - Purchase location, date, price
+
+- Storage
+    - Storage content and list
+
+- Socket
+    - Add decorator for rebrickable, authenticated and threaded socket actions
+
+- SQL
+    - Allow for advanced migration scenarios through companion python files
+    - Add a bunch of the requested fields into the database for future implementation
+
+- Wish
+    - Requester
+
+### UI
+
+- Add
+    - Allow adding or bulk adding by pressing Enter in the input field
+
+- Admin
+    - Grey out legacy tables in the database view
+    - Checkboxes renamed to Set statuses
+    - List of sets that may need to be refreshed
+
+- Cards
+    - Use macros for badge in the card header
+
+- Form
+    - Add a clear button for dynamic text inputs
+    - Add error message in a tooltip for dynamic inputs
+
+- Minifigure
+    - Display number of parts
+
+- Parts
+    - Use Rebrickable URL if stored (+ color code)
+    - Display color and transparency
+    - Display if print of another part
+    - Display prints using the same base
+    - Damaged parts
+    - Display same parts using a different color
+
+- Sets
+    - Add a flag to hide instructions in a set
+    - Make checkbox clickable on the whole width of the card
+    - Management
+        - Ownership
+        - Tags
+        - Refresh
+        - Storage
+        - Purchase location, date, price
+
+- Sets grid
+    - Collapsible controls depending on screen size
+    - Manually collapsible filters (with configuration variable for default state)
+    - Manually collapsible sort (with configuration variable for default state)
+    - Clear search bar
+
+- Storage
+    - Storage list
+    - Storage content
+
+- Wish
+    - Requester
+
+## 1.1.1: PDF Instructions Download
+
+### Instructions
+
+- Added buttons for instructions download from Rebrickable
+
+
+## 1.1.0: Deduped sets, custom checkboxes and database upgrade
+
+### Database
+
+- Sets
+    - Deduplicating rebrickable sets (unique) and bricktracker sets (can be n bricktracker sets for one rebrickable set)
+
+### Docs
+
+- Removed extra `<br>` to accomodate Gitea Markdown
+- Add an organized DOCS.md documentation page
+- Database upgrade/migration
+- Checkboxes
+
+### Code
+
+- Admin
+    - Split the views before admin because an unmanageable monster view
+
+- Checkboxes
+    - Customizable checkboxes for set (amount and names, displayed on the grid or not)
+    - Replaced the 3 original routes to update the status with a generic route to accomodate any custom status
+
+- Instructions
+    - Base instructions on RebrickableSet (the generic one) rather than BrickSet (the specific one)
+    - Refine set number detection in file name by making sure each first items is an integer
+
+- Python
+    - Make stricter function definition with no "arg_or_keyword" parameters
+
+- Records
+    - Consolidate the select() -> not None or Exception -> ingest() process duplicated in every child class
+
+- SQL
+    - Forward-only migration mechanism
+    - Check for database too far in version
+    - Inject the database version in the file when downloading it
+    - Quote all indentifiers as best practice
+    - Allow insert query to be overriden
+    - Allow insert query to force not being deferred even if not committed
+    - Allow select query to push context in BrickRecord and BrickRecordList
+    - Make SQL record counters failsafe as they are used in the admin and it should always work
+    - Remove BrickSQL.initialize() as it is replaced by upgrade()
+
+- Sets
+    - Now that it is deduplicated, adding the same set more than once will not pull it fully from the Rebrickable API (minifigures and parts)
+    - Make RebrickableSet extend BrickRecord since it is now an item in database
+    - Make BrickSet extend RebrickableSet now that RebrickableSet is a proper database item
+
+### UI
+
+- Checkboxes
+    - Possibility to hide the checkbox in the grid ("Sets") but sill have all them in the set details
+    - Management
+
+- Database
+    - Migration tool
+
+- Javascript
+    - Generic BrickChanger class to handle quick modification through a JSON request with a visual feedback indicator
+    - Simplify the way javascript scripts are loaded and instantiated
+
+- Set grid
+    - Filter by checkboxes and NOT checkboxes
+
+- Tables
+    - Fix table search looking inside links pills
+
+- Wishlist
+    - Add Rebrickable link badge for sets (@matthew)
+
+## 1.0.0: New Year revamp
+
+### Code
+
+- Authentication
+    - Basic authentication mechanism with ONE password to protect admin and writes
+- CSV
+    - Remove dependencies to numpy and panda for simpler built-in csv
+- Code
+    - Refactored the Python code
+    - Modularity (more functions, splitting files)
+    - Type hinting whenever possible
+    - Flake8 linter
+    - Retained most of the original behaviour (with its quirks)
+- Colors
+    - Remove dependency on color.csv
+- Configuration
+    - Moved all the hard-coded parameters into configuration variables
+    - Most of the variables are configuration through environment variables
+    - Force instruction, sets, etc path to be relative to static
+- Docker
+    - Added an entrypoint to grab PORT / HOST from the environment if set
+    - Remove the need to seed the container with files (*.csv, nil files)
+- Flask
+    - Fix improper socketio.run(app.run()) call which lead to hard crash on ^C
+    - Make use of url_for to create URLs
+    - Use blueprints to implement routes
+    - Move views into their own files
+    - Split GET and POST methods into two different routes for clarity
+- Images
+    - Add an option to use remote images from the Rebrickable CDN rather than downloading everything locally
+    - Handle nil.png and nil_mf.jpg as true images in /static/sets/ so that they are downloaded whenever necessary when importing a se with missing images
+- Instructions
+    - Scan the files once for the whole app, and re-use the data
+    - Refresh the instructions from the admin
+    - More lenient set number detection
+    - Update when uploading a new one
+    - Basic file management
+- Logs
+    - Added log lines for change actions (add, check, missing, delete) so that the server is not silent when DEBUG=false
+- Minifigures
+    - Added a variable to control default ordering
+- Part(s)
+    - Added a variable to control default ordering of listing
+- Retired sets
+    - Open the themes once for the whole app, and re-use the data
+    - Do not hard fail if themes.csv is missing, simply display the IDs
+    - Light management: resync, download
+- Set(s)
+    - Reworked the set checkboxes with a dedicated route per status
+    - Switch from homemade ID generator to proven UUID4 for sets ID
+        - Does not interfere with previously created sets
+    - Do not rely on sets.csv to check if the set exists
+    - When adding, commit the set to database only once everything has been processed
+    - Added a bulk add page
+    - Keep spare parts when importing
+    - Added a variable to control default ordering of listing
+- Socket
+    - Make use of socket.io rooms to avoid broadcasting messages to all clients
+- SQLite
+    - Do not hard fail if the database is not present or not initialized
+    - Open the database once for the context, and re-use the connection
+    - Move queries to .sql files and load them as Jinja templates
+    - Use named arguments rather than sets for SQLite queries
+    - Allow execute() to be deferred to the commit() call to avoid locking the database for long period while importing (locked while downloading images)
+- Themes
+    - Open the themes once for the whole app, and re-use the data
+    - Do not hard fail if themes.csv is missing, simply display the IDs
+    - Light management: resync, download
+
+### UI
+
+- Admin
+    - Initialize the database from the web interface
+    - Reset the database
+    - Delete the database
+    - Download the database
+    - Import the database
+    - Display the configuration variables
+    - Many things
+- Accordions
+    - Added a flag to make the accordion items independent
+- Branding:
+    - Add a brick as a logo (CC0 image from: https://iconduck.com/icons/71631/brick)
+- Global
+    - Redesign of the whole app
+    - Sticky menu bar on top of the page
+    - Execution time and SQL stats for fun
+- Libraries
+    - Switch from Bulma to Bootstrap, arbitrarily :D
+    - Use of baguettebox for images (https://github.com/feimosi/baguetteBox.js)
+    - Use of tinysort to sort and filter the grid (https://github.com/Sjeiti/TinySort)
+    - Use of sortable for set card tables (https://github.com/tofsjonas/sortable)
+    - Use of simple-datatables for big tables (https://github.com/fiduswriter/simple-datatables)
+- Minifigures
+    - Added a detail view for a minifigure
+    - Display which sets are using a minifigure
+    - Display which sets are missing a minifigure
+- Parts
+    - Added a detail view for a part
+    - Display which sets are using a part
+    - Display which sets are missing a part
+- Templates
+    - Use a common base template
+    - Use HTML fragments/macros for repeted or parametrics items
+    - a 404 page for wrong URLs
+    - an error page for expected error messages
+    - an exception page for unexpected error messages
+- Set add
+    - Two-tiered (with override) import where you see what you will import before importing it
+    - Add a visual indicator that the socket is connected
+- Set card
+    - Badges to display info like theme, year, parts, etc
+    - Set image on top of the card, filling the space
+    - Trick to have a blurry background image fill the void in the card
+    - Save missing parts on input change rather than by clicking
+        - Visual feedback of success
+    - Parts and minifigure in accordions
+    - Instructions file list
+- Set grid
+    - 4-2-1 card distribution depending on screen size
+    - Display the index with no set added, rather than redirecting
+    - Keep last sort in a cookie, and trigger it on page load (can be cleared)

README.md

@@ -18,7 +18,9 @@ A web application for organizing and tracking LEGO sets, parts, and minifigures.
 
 Use the provided [compose.yaml](compose.yaml) file.
 
-See [setup](docs/setup.md).
+See [Quickstart](docs/quickstart.md) to get up and running right away.
+
+See [Setup](docs/setup.md) for a more setup guide.
 
 ## Usage
 
@@ -27,6 +29,6 @@ See [first steps](docs/first-steps.md).
 ## Documentation
 
 Most of the pages should be self explanatory to use.
-However, you can find more specific documentation in the [docs](docs/) folder.
+However, you can find more specific documentation in the [documentation](docs/DOCS.md).
 
-You can find screenshots of the application in the [bricktracker](docs/bricktracker.md) documentation file.
+You can find screenshots of the application in the [overview](docs/overview.md) documentation file.

app.py

@@ -11,28 +11,44 @@ from bricktracker.socket import BrickSocket  # noqa: E402
 
 logger = logging.getLogger(__name__)
 
-# Create the Flask app
-app = Flask(__name__)
 
-# Setup the app
-setup_app(app)
+# Create the app
+# Using 'app' globally interferse with the teardown handlers of Flask
+def create_app(main: bool = False, /) -> Flask | BrickSocket:
+    # Create the Flask app
+    app = Flask(__name__)
 
-# Create the socket
-s = BrickSocket(
-    app,
-    threaded=not app.config['NO_THREADED_SOCKET'].value,
-)
+    # Setup the app
+    setup_app(app)
+
+    # Create the socket
+    s = BrickSocket(
+        app,
+        threaded=not app.config['NO_THREADED_SOCKET'],
+    )
+
+    if main:
+        return s
+    else:
+        return app
 
 
 if __name__ == '__main__':
+    s = create_app(True)
+
+    # This never happens, but makes the linter happy
+    if isinstance(s, Flask):
+        logger.critical('Cannot run locally with a Flask object, needs a BrickSocket. Use create_app(True) to return a BrickSocket')  # noqa: E501
+        exit(1)
+
     # Run the application
     logger.info('Starting BrickTracker on {host}:{port}'.format(
-        host=app.config['HOST'].value,
-        port=app.config['PORT'].value,
+        host=s.app.config['HOST'],
+        port=s.app.config['PORT'],
     ))
     s.socket.run(
-        app,
-        host=app.config['HOST'].value,
-        debug=app.config['DEBUG'].value,
-        port=app.config['PORT'].value,
+        s.app,
+        host=s.app.config['HOST'],
+        debug=s.app.config['DEBUG'],
+        port=s.app.config['PORT'],
     )

bricktracker/app.py

@@ -12,7 +12,18 @@ from bricktracker.navbar import Navbar
 from bricktracker.sql import close
 from bricktracker.version import __version__
 from bricktracker.views.add import add_page
-from bricktracker.views.admin import admin_page
+from bricktracker.views.admin.admin import admin_page
+from bricktracker.views.admin.database import admin_database_page
+from bricktracker.views.admin.image import admin_image_page
+from bricktracker.views.admin.instructions import admin_instructions_page
+from bricktracker.views.admin.owner import admin_owner_page
+from bricktracker.views.admin.purchase_location import admin_purchase_location_page  # noqa: E501
+from bricktracker.views.admin.retired import admin_retired_page
+from bricktracker.views.admin.set import admin_set_page
+from bricktracker.views.admin.status import admin_status_page
+from bricktracker.views.admin.storage import admin_storage_page
+from bricktracker.views.admin.tag import admin_tag_page
+from bricktracker.views.admin.theme import admin_theme_page
 from bricktracker.views.error import error_404
 from bricktracker.views.index import index_page
 from bricktracker.views.instructions import instructions_page
@@ -20,6 +31,7 @@ from bricktracker.views.login import login_page
 from bricktracker.views.minifigure import minifigure_page
 from bricktracker.views.part import part_page
 from bricktracker.views.set import set_page
+from bricktracker.views.storage import storage_page
 from bricktracker.views.wish import wish_page
 
 
@@ -28,7 +40,7 @@ def setup_app(app: Flask) -> None:
     BrickConfigurationList(app)
 
     # Set the logging level
-    if app.config['DEBUG'].value:
+    if app.config['DEBUG']:
         logging.basicConfig(
             stream=sys.stdout,
             level=logging.DEBUG,
@@ -60,17 +72,31 @@ def setup_app(app: Flask) -> None:
     # Register errors
     app.register_error_handler(404, error_404)
 
-    # Register routes
+    # Register app routes
     app.register_blueprint(add_page)
-    app.register_blueprint(admin_page)
     app.register_blueprint(index_page)
     app.register_blueprint(instructions_page)
     app.register_blueprint(login_page)
     app.register_blueprint(minifigure_page)
     app.register_blueprint(part_page)
     app.register_blueprint(set_page)
+    app.register_blueprint(storage_page)
     app.register_blueprint(wish_page)
 
+    # Register admin routes
+    app.register_blueprint(admin_page)
+    app.register_blueprint(admin_database_page)
+    app.register_blueprint(admin_image_page)
+    app.register_blueprint(admin_instructions_page)
+    app.register_blueprint(admin_retired_page)
+    app.register_blueprint(admin_owner_page)
+    app.register_blueprint(admin_purchase_location_page)
+    app.register_blueprint(admin_set_page)
+    app.register_blueprint(admin_status_page)
+    app.register_blueprint(admin_storage_page)
+    app.register_blueprint(admin_tag_page)
+    app.register_blueprint(admin_theme_page)
+
     # An helper to make global variables available to the
     # request
     @app.before_request
@@ -90,12 +116,12 @@ def setup_app(app: Flask) -> None:
         g.request_time = request_time
 
         # Register the timezone
-        g.timezone = ZoneInfo(current_app.config['TIMEZONE'].value)
+        g.timezone = ZoneInfo(current_app.config['TIMEZONE'])
 
         # Version
         g.version = __version__
 
     # Make sure all connections are closed at the end
-    @app.teardown_appcontext
-    def close_connections(exception, /) -> None:
+    @app.teardown_request
+    def teardown_request(_: BaseException | None) -> None:
         close()

bricktracker/config.py

@@ -10,7 +10,7 @@ from typing import Any, Final
 CONFIG: Final[list[dict[str, Any]]] = [
     {'n': 'AUTHENTICATION_PASSWORD', 'd': ''},
     {'n': 'AUTHENTICATION_KEY', 'd': ''},
-    {'n': 'BRICKLINK_LINK_PART_PATTERN', 'd': 'https://www.bricklink.com/v2/catalog/catalogitem.page?P={number}'},  # noqa: E501
+    {'n': 'BRICKLINK_LINK_PART_PATTERN', 'd': 'https://www.bricklink.com/v2/catalog/catalogitem.page?P={part}&C={color}'},  # noqa: E501
     {'n': 'BRICKLINK_LINKS', 'c': bool},
     {'n': 'DATABASE_PATH', 'd': './app.db'},
     {'n': 'DATABASE_TIMESTAMP_FORMAT', 'd': '%Y-%m-%d-%H-%M-%S'},
@@ -28,34 +28,45 @@ CONFIG: Final[list[dict[str, Any]]] = [
     {'n': 'HIDE_ALL_INSTRUCTIONS', 'c': bool},
     {'n': 'HIDE_ALL_MINIFIGURES', 'c': bool},
     {'n': 'HIDE_ALL_PARTS', 'c': bool},
+    {'n': 'HIDE_ALL_PROBLEMS_PARTS', 'e': 'BK_HIDE_MISSING_PARTS', 'c': bool},
     {'n': 'HIDE_ALL_SETS', 'c': bool},
-    {'n': 'HIDE_MISSING_PARTS', 'c': bool},
+    {'n': 'HIDE_ALL_STORAGES', 'c': bool},
+    {'n': 'HIDE_SET_INSTRUCTIONS', 'c': bool},
+    {'n': 'HIDE_TABLE_DAMAGED_PARTS', 'c': bool},
+    {'n': 'HIDE_TABLE_MISSING_PARTS', 'c': bool},
     {'n': 'HIDE_WISHES', 'c': bool},
-    {'n': 'MINIFIGURES_DEFAULT_ORDER', 'd': 'minifigures.name ASC'},
+    {'n': 'MINIFIGURES_DEFAULT_ORDER', 'd': '"rebrickable_minifigures"."name" ASC'},  # noqa: E501
     {'n': 'MINIFIGURES_FOLDER', 'd': 'minifigs', 's': True},
     {'n': 'NO_THREADED_SOCKET', 'c': bool},
-    {'n': 'PARTS_DEFAULT_ORDER', 'd': 'inventory.name ASC, inventory.color_name ASC, is_spare ASC'},  # noqa: E501
+    {'n': 'PARTS_DEFAULT_ORDER', 'd': '"rebrickable_parts"."name" ASC, "rebrickable_parts"."color_name" ASC, "bricktracker_parts"."spare" ASC'},  # noqa: E501
     {'n': 'PARTS_FOLDER', 'd': 'parts', 's': True},
     {'n': 'PORT', 'd': 3333, 'c': int},
+    {'n': 'PURCHASE_DATE_FORMAT', 'd': '%d/%m/%Y'},
+    {'n': 'PURCHASE_CURRENCY', 'd': '€'},
+    {'n': 'PURCHASE_LOCATION_DEFAULT_ORDER', 'd': '"bricktracker_metadata_purchase_locations"."name" ASC'},  # noqa: E501
     {'n': 'RANDOM', 'e': 'RANDOM', 'c': bool},
     {'n': 'REBRICKABLE_API_KEY', 'e': 'REBRICKABLE_API_KEY', 'd': ''},
     {'n': 'REBRICKABLE_IMAGE_NIL', 'd': 'https://rebrickable.com/static/img/nil.png'},  # noqa: E501
     {'n': 'REBRICKABLE_IMAGE_NIL_MINIFIGURE', 'd': 'https://rebrickable.com/static/img/nil_mf.jpg'},  # noqa: E501
-    {'n': 'REBRICKABLE_LINK_MINIFIGURE_PATTERN', 'd': 'https://rebrickable.com/minifigs/{number}'},  # noqa: E501
-    {'n': 'REBRICKABLE_LINK_PART_PATTERN', 'd': 'https://rebrickable.com/parts/{number}/_/{color}'},  # noqa: E501
-    {'n': 'REBRICKABLE_LINK_SET_PATTERN', 'd': 'https://rebrickable.com/sets/{number}'},  # noqa: E501
+    {'n': 'REBRICKABLE_LINK_MINIFIGURE_PATTERN', 'd': 'https://rebrickable.com/minifigs/{figure}'},  # noqa: E501
+    {'n': 'REBRICKABLE_LINK_PART_PATTERN', 'd': 'https://rebrickable.com/parts/{part}/_/{color}'},  # noqa: E501
+    {'n': 'REBRICKABLE_LINK_INSTRUCTIONS_PATTERN', 'd': 'https://rebrickable.com/instructions/{path}'},  # noqa: E501
+    {'n': 'REBRICKABLE_USER_AGENT', 'd': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'},  # noqa: E501
     {'n': 'REBRICKABLE_LINKS', 'e': 'LINKS', 'c': bool},
     {'n': 'REBRICKABLE_PAGE_SIZE', 'd': 100, 'c': int},
     {'n': 'RETIRED_SETS_FILE_URL', 'd': 'https://docs.google.com/spreadsheets/d/1rlYfEXtNKxUOZt2Mfv0H17DvK7bj6Pe0CuYwq6ay8WA/gviz/tq?tqx=out:csv&sheet=Sorted%20by%20Retirement%20Date'},  # noqa: E501
     {'n': 'RETIRED_SETS_PATH', 'd': './retired_sets.csv'},
-    {'n': 'SETS_DEFAULT_ORDER', 'd': 'set_number DESC, set_version ASC'},
+    {'n': 'SETS_DEFAULT_ORDER', 'd': '"rebrickable_sets"."number" DESC, "rebrickable_sets"."version" ASC'},  # noqa: E501
     {'n': 'SETS_FOLDER', 'd': 'sets', 's': True},
+    {'n': 'SHOW_GRID_FILTERS', 'c': bool},
+    {'n': 'SHOW_GRID_SORT', 'c': bool},
     {'n': 'SKIP_SPARE_PARTS', 'c': bool},
     {'n': 'SOCKET_NAMESPACE', 'd': 'bricksocket'},
     {'n': 'SOCKET_PATH', 'd': '/bricksocket/'},
+    {'n': 'STORAGE_DEFAULT_ORDER', 'd': '"bricktracker_metadata_storages"."name" ASC'},  # noqa: E501
     {'n': 'THEMES_FILE_URL', 'd': 'https://cdn.rebrickable.com/media/downloads/themes.csv.gz'},  # noqa: E501
     {'n': 'THEMES_PATH', 'd': './themes.csv'},
     {'n': 'TIMEZONE', 'd': 'Etc/UTC'},
     {'n': 'USE_REMOTE_IMAGES', 'c': bool},
-    {'n': 'WISHES_DEFAULT_ORDER', 'd': 'wishlist.rowid DESC'},
+    {'n': 'WISHES_DEFAULT_ORDER', 'd': '"bricktracker_wishes"."rowid" DESC'},
 ]

bricktracker/configuration.py

@@ -16,6 +16,7 @@ class BrickConfiguration(object):
     def __init__(
         self,
         /,
+        *,
         n: str,
         e: str | None = None,
         d: Any = None,
@@ -69,6 +70,7 @@ class BrickConfiguration(object):
                 # Remove static prefix
                 value = value.removeprefix('static/')
 
+        # Type casting
         if self.cast is not None:
             self.value = self.cast(value)
         else:

bricktracker/configuration_list.py

@@ -1,46 +1,60 @@
+import logging
 from typing import Generator
 
-from flask import current_app, Flask
+from flask import Flask
 
 from .config import CONFIG
 from .configuration import BrickConfiguration
 from .exceptions import ConfigurationMissingException
 
+logger = logging.getLogger(__name__)
+
 
 # Application configuration
 class BrickConfigurationList(object):
     app: Flask
+    configurations: dict[str, BrickConfiguration]
 
     # Load configuration
     def __init__(self, app: Flask, /):
         self.app = app
 
-        # Process all configuration items
-        for config in CONFIG:
-            item = BrickConfiguration(**config)
-            self.app.config[item.name] = item
+        # Load the configurations only there is none already loaded
+        configurations = getattr(self, 'configurations', None)
+
+        if configurations is None:
+            logger.info('Loading configuration variables')
+
+            BrickConfigurationList.configurations = {}
+
+            # Process all configuration items
+            for config in CONFIG:
+                item = BrickConfiguration(**config)
+
+                # Store in the list
+                BrickConfigurationList.configurations[item.name] = item
+
+                # Only store the value in the app to avoid breaking any
+                # existing variables
+                self.app.config[item.name] = item.value
 
     # Check whether a str configuration is set
     @staticmethod
     def error_unless_is_set(name: str):
-        config: BrickConfiguration = current_app.config[name]
+        configuration = BrickConfigurationList.configurations[name]
 
-        if config.value is None or config.value == '':
+        if configuration.value is None or configuration.value == '':
             raise ConfigurationMissingException(
                 '{name} must be defined (using the {environ} environment variable)'.format(  # noqa: E501
-                    name=config.name,
-                    environ=config.env_name
+                    name=name,
+                    environ=configuration.env_name
                 ),
             )
 
     # Get all the configuration items from the app config
     @staticmethod
     def list() -> Generator[BrickConfiguration, None, None]:
-        keys = list(current_app.config.keys())
-        keys.sort()
+        keys = sorted(BrickConfigurationList.configurations.keys())
 
         for name in keys:
-            config = current_app.config[name]
-
-            if isinstance(config, BrickConfiguration):
-                yield config
+            yield BrickConfigurationList.configurations[name]

bricktracker/fields.py

@@ -4,6 +4,9 @@ from typing import Any
 # SQLite record fields
 class BrickRecordFields(object):
     def __getattr__(self, name: str, /) -> Any:
+        if name not in self.__dict__:
+            raise AttributeError(name)
+
         return self.__dict__[name]
 
     def __setattr__(self, name: str, value: Any, /) -> None:

bricktracker/instructions.py

@@ -1,33 +1,56 @@
 from datetime import datetime, timezone
 import logging
 import os
-from typing import TYPE_CHECKING
+from urllib.parse import urljoin
+from shutil import copyfileobj
+import traceback
+from typing import Tuple, TYPE_CHECKING
 
+from bs4 import BeautifulSoup
 from flask import current_app, g, url_for
 import humanize
+import requests
 from werkzeug.datastructures import FileStorage
 from werkzeug.utils import secure_filename
+import re
+import cloudscraper
 
-from .exceptions import ErrorException
+from .exceptions import ErrorException, DownloadException
 if TYPE_CHECKING:
-    from .set import BrickSet
+    from .rebrickable_set import RebrickableSet
+    from .socket import BrickSocket
 
 logger = logging.getLogger(__name__)
 
 
 class BrickInstructions(object):
+    socket: 'BrickSocket'
+
     allowed: bool
-    brickset: 'BrickSet | None'
+    rebrickable: 'RebrickableSet | None'
     extension: str
     filename: str
     mtime: datetime
-    number: 'str | None'
+    set: 'str | None'
     name: str
     size: int
 
-    def __init__(self, file: os.DirEntry | str, /):
+    def __init__(
+        self,
+        file: os.DirEntry | str,
+        /,
+        *,
+        socket: 'BrickSocket | None' = None,
+    ):
+        # Save the socket
+        if socket is not None:
+            self.socket = socket
+
         if isinstance(file, str):
             self.filename = file
+
+            if self.filename == '':
+                raise ErrorException('An instruction filename cannot be empty')
         else:
             self.filename = file.name
 
@@ -39,11 +62,11 @@ class BrickInstructions(object):
         # Store the name and extension, check if extension is allowed
         self.name, self.extension = os.path.splitext(self.filename)
         self.extension = self.extension.lower()
-        self.allowed = self.extension in current_app.config['INSTRUCTIONS_ALLOWED_EXTENSIONS'].value  # noqa: E501
+        self.allowed = self.extension in current_app.config['INSTRUCTIONS_ALLOWED_EXTENSIONS']  # noqa: E501
 
         # Placeholder
-        self.brickset = None
-        self.number = None
+        self.rebrickable = None
+        self.set = None
 
         # Extract the set number
         if self.allowed:
@@ -54,30 +77,102 @@ class BrickInstructions(object):
             splits = normalized.split('-', 2)
 
             if len(splits) >= 2:
-                self.number = '-'.join(splits[:2])
+                try:
+                    # Trying to make sense of each part as integers
+                    int(splits[0])
+                    int(splits[1])
+
+                    self.set = '-'.join(splits[:2])
+                except Exception:
+                    pass
 
     # Delete an instruction file
     def delete(self, /) -> None:
         os.remove(self.path())
 
+    # Download an instruction file
+    def download(self, path: str, /) -> None:
+        """
+        Streams the PDF in chunks and uses self.socket.update_total
+        + self.socket.progress_count to drive a determinate bar.
+        """
+        try:
+            target = self.path(filename=secure_filename(self.filename))
+
+            # Skip if we already have it
+            if os.path.isfile(target):
+                return self.socket.complete(
+                    message=f"File {self.filename} already exists, skipped"
+                )
+
+            # Fetch PDF via cloudscraper (to bypass Cloudflare)
+            scraper = cloudscraper.create_scraper()
+            scraper.headers.update({
+                "User-Agent": current_app.config['REBRICKABLE_USER_AGENT']
+            })
+            resp = scraper.get(path, stream=True)
+            if not resp.ok:
+                raise DownloadException(f"Failed to download: HTTP {resp.status_code}")
+
+            # Tell the socket how many bytes in total
+            total = int(resp.headers.get("Content-Length", 0))
+            self.socket.update_total(total)
+
+            # Reset the counter and kick off at 0%
+            self.socket.progress_count = 0
+            self.socket.progress(message=f"Starting download {self.filename}")
+
+            # Write out in 8 KiB chunks and update the counter
+            with open(target, "wb") as f:
+                for chunk in resp.iter_content(chunk_size=8192):
+                    if not chunk:
+                        continue
+                    f.write(chunk)
+
+                    # Bump the internal counter and emit
+                    self.socket.progress_count += len(chunk)
+                    self.socket.progress(
+                        message=(
+                            f"Downloading {self.filename} "
+                            f"({humanize.naturalsize(self.socket.progress_count)}/"
+                            f"{humanize.naturalsize(self.socket.progress_total)})"
+                        )
+                    )
+
+            # Done!
+            logger.info(f"Downloaded {self.filename}")
+            self.socket.complete(
+                message=f"File {self.filename} downloaded ({self.human_size()})"
+            )
+
+        except Exception as e:
+            logger.debug(traceback.format_exc())
+            self.socket.fail(
+                message=f"Error downloading {self.filename}: {e}"
+            )
+
     # Display the size in a human format
     def human_size(self) -> str:
-        return humanize.naturalsize(self.size)
+        try:
+            size = self.size
+        except AttributeError:
+            size = os.path.getsize(self.path())
+        return humanize.naturalsize(size)
 
     # Display the time in a human format
     def human_time(self) -> str:
         return self.mtime.astimezone(g.timezone).strftime(
-            current_app.config['FILE_DATETIME_FORMAT'].value
+            current_app.config['FILE_DATETIME_FORMAT']
         )
 
     # Compute the path of an instruction file
-    def path(self, /, filename=None) -> str:
+    def path(self, /, *, filename=None) -> str:
         if filename is None:
             filename = self.filename
 
         return os.path.join(
             current_app.static_folder,  # type: ignore
-            current_app.config['INSTRUCTIONS_FOLDER'].value,
+            current_app.config['INSTRUCTIONS_FOLDER'],
             filename
         )
 
@@ -99,7 +194,7 @@ class BrickInstructions(object):
 
     # Upload a new instructions file
     def upload(self, file: FileStorage, /) -> None:
-        target = self.path(secure_filename(self.filename))
+        target = self.path(filename=secure_filename(self.filename))
 
         if os.path.isfile(target):
             raise ErrorException('Cannot upload {target} as it already exists'.format(  # noqa: E501
@@ -118,7 +213,7 @@ class BrickInstructions(object):
         if not self.allowed:
             return ''
 
-        folder: str = current_app.config['INSTRUCTIONS_FOLDER'].value
+        folder: str = current_app.config['INSTRUCTIONS_FOLDER']
 
         # Compute the path
         path = os.path.join(folder, self.filename)
@@ -135,3 +230,56 @@ class BrickInstructions(object):
             return 'file-image-line'
         else:
             return 'file-line'
+
+    # Find the instructions for a set
+    @staticmethod
+    def find_instructions(set: str, /) -> list[Tuple[str, str]]:
+        """
+        Scrape Rebrickable’s HTML and return a list of
+        (filename_slug, download_url). Duplicate slugs get _1, _2, …
+        """
+        page_url = f"https://rebrickable.com/instructions/{set}/"
+        logger.debug(f"[find_instructions] fetching HTML from {page_url!r}")
+
+        # Solve Cloudflare’s challenge
+        scraper = cloudscraper.create_scraper()
+        scraper.headers.update({'User-Agent': current_app.config['REBRICKABLE_USER_AGENT']})
+        resp = scraper.get(page_url)
+        if not resp.ok:
+            raise ErrorException(f'Failed to load instructions page for {set}. HTTP {resp.status_code}')
+
+        soup = BeautifulSoup(resp.content, 'html.parser')
+        link_re = re.compile(r'^/instructions/\d+/.+/download/')
+
+        raw: list[tuple[str, str]] = []
+        for a in soup.find_all('a', href=link_re):
+            img = a.find('img', alt=True)
+            if not img or set not in img['alt']:
+                continue
+
+            # Turn the alt text into a slug
+            alt_text = img['alt'].removeprefix('LEGO Building Instructions for ')
+            slug = re.sub(r'[^A-Za-z0-9]+', '-', alt_text).strip('-')
+
+            # Build the absolute download URL
+            download_url = urljoin('https://rebrickable.com', a['href'])
+            raw.append((slug, download_url))
+
+        if not raw:
+            raise ErrorException(f'No download links found on instructions page for {set}')
+
+        # Disambiguate duplicate slugs by appending _1, _2, …
+        from collections import Counter, defaultdict
+        counts = Counter(name for name, _ in raw)
+        seen: dict[str, int] = defaultdict(int)
+        unique: list[tuple[str, str]] = []
+        for name, url in raw:
+            idx = seen[name]
+            if counts[name] > 1 and idx > 0:
+                final_name = f"{name}_{idx}"
+            else:
+                final_name = name
+            seen[name] += 1
+            unique.append((final_name, url))
+
+        return unique

bricktracker/instructions_list.py

@@ -1,11 +1,14 @@
 import logging
 import os
-from typing import Generator
+from typing import Generator, TYPE_CHECKING
 
 from flask import current_app
 
 from .exceptions import NotFoundException
 from .instructions import BrickInstructions
+from .rebrickable_set_list import RebrickableSetList
+if TYPE_CHECKING:
+    from .rebrickable_set import RebrickableSet
 
 logger = logging.getLogger(__name__)
 
@@ -18,7 +21,7 @@ class BrickInstructionsList(object):
     sets_total: int
     unknown_total: int
 
-    def __init__(self, /, force=False):
+    def __init__(self, /, *, force=False):
         # Load instructions only if there is none already loaded
         all = getattr(self, 'all', None)
 
@@ -36,7 +39,7 @@ class BrickInstructionsList(object):
                 # Make a folder relative to static
                 folder: str = os.path.join(
                     current_app.static_folder,  # type: ignore
-                    current_app.config['INSTRUCTIONS_FOLDER'].value,
+                    current_app.config['INSTRUCTIONS_FOLDER'],
                 )
 
                 for file in os.scandir(folder):
@@ -46,47 +49,40 @@ class BrickInstructionsList(object):
                     BrickInstructionsList.all[instruction.filename] = instruction  # noqa: E501
 
                     if instruction.allowed:
-                        if instruction.number:
+                        if instruction.set:
                             # Instantiate the list if not existing yet
-                            if instruction.number not in BrickInstructionsList.sets:  # noqa: E501
-                                BrickInstructionsList.sets[instruction.number] = []  # noqa: E501
+                            if instruction.set not in BrickInstructionsList.sets:  # noqa: E501
+                                BrickInstructionsList.sets[instruction.set] = []  # noqa: E501
 
-                            BrickInstructionsList.sets[instruction.number].append(instruction)  # noqa: E501
+                            BrickInstructionsList.sets[instruction.set].append(instruction)  # noqa: E501
                             BrickInstructionsList.sets_total += 1
                         else:
                             BrickInstructionsList.unknown_total += 1
                     else:
                         BrickInstructionsList.rejected_total += 1
 
-                # Associate bricksets
-                # Not ideal, to avoid a circular import
-                from .set import BrickSet
-                from .set_list import BrickSetList
+                # List of Rebrickable sets
+                rebrickable_sets: dict[str, RebrickableSet] = {}
+                for rebrickable_set in RebrickableSetList().all():
+                    rebrickable_sets[rebrickable_set.fields.set] = rebrickable_set  # noqa: E501
 
-                # Grab the generic list of sets
-                bricksets: dict[str, BrickSet] = {}
-                for brickset in BrickSetList().generic().records:
-                    bricksets[brickset.fields.set_num] = brickset
-
-                # Return the files
+                # Inject the brickset if it exists
                 for instruction in self.all.values():
-                    # Inject the brickset if it exists
                     if (
                         instruction.allowed and
-                        instruction.number is not None and
-                        instruction.brickset is None and
-                        instruction.number in bricksets
+                        instruction.set is not None and
+                        instruction.rebrickable is None and
+                        instruction.set in rebrickable_sets
                     ):
-                        instruction.brickset = bricksets[instruction.number]
-
+                        instruction.rebrickable = rebrickable_sets[instruction.set]  # noqa: E501
             # Ignore errors
             except Exception:
                 pass
 
     # Grab instructions for a set
-    def get(self, number: str) -> list[BrickInstructions]:
-        if number in self.sets:
-            return self.sets[number]
+    def get(self, set: str) -> list[BrickInstructions]:
+        if set in self.sets:
+            return self.sets[set]
         else:
             return []
 

bricktracker/login.py

@@ -12,7 +12,7 @@ class LoginManager(object):
 
     def __init__(self, app: Flask, /):
         # Setup basic authentication
-        app.secret_key = app.config['AUTHENTICATION_KEY'].value
+        app.secret_key = app.config['AUTHENTICATION_KEY']
 
         manager = login_manager.LoginManager()
         manager.login_view = 'login.login'  # type: ignore
@@ -23,11 +23,11 @@ class LoginManager(object):
         def user_loader(*arg) -> LoginManager.User:
             return self.User(
                 'admin',
-                app.config['AUTHENTICATION_PASSWORD'].value
+                app.config['AUTHENTICATION_PASSWORD']
             )
 
         # If the password is unset, globally disable
-        app.config['LOGIN_DISABLED'] = app.config['AUTHENTICATION_PASSWORD'].value == ''  # noqa: E501
+        app.config['LOGIN_DISABLED'] = app.config['AUTHENTICATION_PASSWORD'] == ''  # noqa: E501
 
     # Tells whether the user is authenticated, meaning:
     # - Authentication disabled

bricktracker/metadata.py

@@ -0,0 +1,263 @@
+import logging
+from sqlite3 import Row
+from typing import Any, Self, TYPE_CHECKING
+from uuid import uuid4
+
+from flask import url_for
+
+from .exceptions import DatabaseException, ErrorException, NotFoundException
+from .record import BrickRecord
+from .sql import BrickSQL
+if TYPE_CHECKING:
+    from .set import BrickSet
+
+logger = logging.getLogger(__name__)
+
+
+# Lego set metadata (customizable list of entries that can be checked)
+class BrickMetadata(BrickRecord):
+    kind: str
+
+    # Set state endpoint
+    set_state_endpoint: str
+
+    # Queries
+    delete_query: str
+    insert_query: str
+    select_query: str
+    update_field_query: str
+    update_set_state_query: str
+    update_set_value_query: str
+
+    def __init__(
+        self,
+        /,
+        *,
+        record: Row | dict[str, Any] | None = None,
+    ):
+        super().__init__()
+
+        # Defined an empty ID
+        self.fields.id = None
+
+        # Ingest the record if it has one
+        if record is not None:
+            self.ingest(record)
+
+    # SQL column name
+    def as_column(self, /) -> str:
+        return '{kind}_{id}'.format(
+            id=self.fields.id,
+            kind=self.kind.lower().replace(' ', '-')
+        )
+
+    # HTML dataset name
+    def as_dataset(self, /) -> str:
+        return self.as_column().replace('_', '-')
+
+    # Delete from database
+    def delete(self, /) -> None:
+        BrickSQL().executescript(
+            self.delete_query,
+            id=self.fields.id,
+        )
+
+    # Grab data from a form
+    def from_form(self, form: dict[str, str], /) -> Self:
+        name = form.get('name', None)
+
+        if name is None or name == '':
+            raise ErrorException('Status name cannot be empty')
+
+        self.fields.name = name
+
+        return self
+
+    # Insert into database
+    def insert(self, /, **context) -> None:
+        self.safe()
+
+        # Generate an ID for the metadata (with underscores to make it
+        # column name friendly)
+        self.fields.id = str(uuid4()).replace('-', '_')
+
+        BrickSQL().executescript(
+            self.insert_query,
+            id=self.fields.id,
+            name=self.fields.safe_name,
+            **context
+        )
+
+    # Rename the entry
+    def rename(self, /) -> None:
+        self.update_field('name', value=self.fields.name)
+
+    # Make the name "safe"
+    # Security: eh.
+    def safe(self, /) -> None:
+        # Prevent self-ownage with accidental quote escape
+        self.fields.safe_name = self.fields.name.replace("'", "''")
+
+    # URL to change the selected state of this metadata item for a set
+    def url_for_set_state(self, id: str, /) -> str:
+        return url_for(
+            self.set_state_endpoint,
+            id=id,
+            metadata_id=self.fields.id
+        )
+
+    # Select a specific metadata (with an id)
+    def select_specific(self, id: str, /) -> Self:
+        # Save the parameters to the fields
+        self.fields.id = id
+
+        # Load from database
+        if not self.select():
+            raise NotFoundException(
+                '{kind} with ID {id} was not found in the database'.format(
+                    kind=self.kind.capitalize(),
+                    id=self.fields.id,
+                ),
+            )
+
+        return self
+
+    # Update a field
+    def update_field(
+        self,
+        field: str,
+        /,
+        *,
+        json: Any | None = None,
+        value: Any | None = None
+    ) -> Any:
+        if value is None and json is not None:
+            value = json.get('value', None)
+
+        if value is None:
+            raise ErrorException('"{field}" of a {kind} cannot be set to an empty value'.format(  # noqa: E501
+                field=field,
+                kind=self.kind
+            ))
+
+        if field == 'id' or not hasattr(self.fields, field):
+            raise NotFoundException('"{field}" is not a field of a {kind}'.format(  # noqa: E501
+                kind=self.kind,
+                field=field
+            ))
+
+        parameters = self.sql_parameters()
+        parameters['value'] = value
+
+        # Update the status
+        rows, _ = BrickSQL().execute_and_commit(
+            self.update_field_query,
+            parameters=parameters,
+            field=field,
+        )
+
+        if rows != 1:
+            raise DatabaseException('Could not update the field "{field}" for {kind} "{name}" ({id})'.format(  # noqa: E501
+                field=field,
+                kind=self.kind,
+                name=self.fields.name,
+                id=self.fields.id,
+            ))
+
+        # Info
+        logger.info('{kind} "{name}" ({id}): field "{field}" changed to "{value}"'.format(  # noqa: E501
+            kind=self.kind.capitalize(),
+            name=self.fields.name,
+            id=self.fields.id,
+            field=field,
+            value=value,
+        ))
+
+        return value
+
+    # Update the selected state of this metadata item for a set
+    def update_set_state(
+        self,
+        brickset: 'BrickSet',
+        /,
+        *,
+        json: Any | None = None,
+        state: Any | None = None
+    ) -> Any:
+        if state is None and json is not None:
+            state = json.get('value', False)
+
+        parameters = self.sql_parameters()
+        parameters['set_id'] = brickset.fields.id
+        parameters['state'] = state
+
+        rows, _ = BrickSQL().execute_and_commit(
+            self.update_set_state_query,
+            parameters=parameters,
+            name=self.as_column(),
+        )
+
+        if rows != 1:
+            raise DatabaseException('Could not update the {kind} "{name}" state for set {set} ({id})'.format(  # noqa: E501
+                kind=self.kind,
+                name=self.fields.name,
+                set=brickset.fields.set,
+                id=brickset.fields.id,
+            ))
+
+        # Info
+        logger.info('{kind} "{name}" state changed to "{state}" for set {set} ({id})'.format(  # noqa: E501
+            kind=self.kind,
+            name=self.fields.name,
+            state=state,
+            set=brickset.fields.set,
+            id=brickset.fields.id,
+        ))
+
+        return state
+
+    # Update the selected value of this metadata item for a set
+    def update_set_value(
+        self,
+        brickset: 'BrickSet',
+        /,
+        *,
+        json: Any | None = None,
+        value: Any | None = None,
+    ) -> Any:
+        if value is None and json is not None:
+            value = json.get('value', '')
+
+        if value == '':
+            value = None
+
+        parameters = self.sql_parameters()
+        parameters['set_id'] = brickset.fields.id
+        parameters['value'] = value
+
+        rows, _ = BrickSQL().execute_and_commit(
+            self.update_set_value_query,
+            parameters=parameters,
+        )
+
+        # Update the status
+        if value is None and not hasattr(self.fields, 'name'):
+            self.fields.name = 'None'
+
+        if rows != 1:
+            raise DatabaseException('Could not update the {kind} value for set {set} ({id})'.format(  # noqa: E501
+                kind=self.kind,
+                set=brickset.fields.set,
+                id=brickset.fields.id,
+            ))
+
+        # Info
+        logger.info('{kind} value changed to "{name}" ({value}) for set {set} ({id})'.format(  # noqa: E501
+            kind=self.kind,
+            name=self.fields.name,
+            value=value,
+            set=brickset.fields.set,
+            id=brickset.fields.id,
+        ))
+
+        return value

bricktracker/metadata_list.py

@@ -0,0 +1,176 @@
+import logging
+from typing import List, overload, Self, Type, TypeVar
+
+from flask import url_for
+
+from .exceptions import ErrorException, NotFoundException
+from .fields import BrickRecordFields
+from .record_list import BrickRecordList
+from .set_owner import BrickSetOwner
+from .set_purchase_location import BrickSetPurchaseLocation
+from .set_status import BrickSetStatus
+from .set_storage import BrickSetStorage
+from .set_tag import BrickSetTag
+from .wish_owner import BrickWishOwner
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar(
+    'T',
+    BrickSetOwner,
+    BrickSetPurchaseLocation,
+    BrickSetStatus,
+    BrickSetStorage,
+    BrickSetTag,
+    BrickWishOwner
+)
+
+
+# Lego sets metadata list
+class BrickMetadataList(BrickRecordList[T]):
+    kind: str
+    mapping: dict[str, T]
+    model: Type[T]
+
+    # Database
+    table: str
+    order: str
+
+    # Queries
+    select_query: str
+
+    # Set endpoints
+    set_state_endpoint: str
+    set_value_endpoint: str
+
+    def __init__(
+        self,
+        model: Type[T],
+        /,
+        *,
+        force: bool = False,
+        records: list[T] | None = None
+    ):
+        self.model = model
+
+        # Records override (masking the class variables with instance ones)
+        if records is not None:
+            self.override()
+
+            for metadata in records:
+                self.records.append(metadata)
+                self.mapping[metadata.fields.id] = metadata
+        else:
+            # Load metadata only if there is none already loaded
+            records = getattr(self, 'records', None)
+
+            if records is None or force:
+                # Don't use super()__init__ as it would mask class variables
+                self.fields = BrickRecordFields()
+
+                logger.info('Loading {kind} list'.format(
+                    kind=self.kind
+                ))
+
+                self.__class__.records = []
+                self.__class__.mapping = {}
+
+                # Load the metadata from the database
+                for record in self.select(order=self.order):
+                    metadata = model(record=record)
+
+                    self.__class__.records.append(metadata)
+                    self.__class__.mapping[metadata.fields.id] = metadata
+
+    # HTML prefix name
+    def as_prefix(self, /) -> str:
+        return self.kind.replace(' ', '-')
+
+    # Filter the list of records (this one does nothing)
+    def filter(self) -> list[T]:
+        return self.records
+
+    # Add a layer of override data
+    def override(self) -> None:
+        self.fields = BrickRecordFields()
+
+        self.records = []
+        self.mapping = {}
+
+    # Return the items as columns for a select
+    @classmethod
+    def as_columns(cls, /, **kwargs) -> str:
+        new = cls.new()
+
+        return ', '.join([
+            '"{table}"."{column}"'.format(
+                table=cls.table,
+                column=record.as_column(),
+            )
+            for record
+            in new.filter(**kwargs)
+        ])
+
+    # Grab a specific status
+    @classmethod
+    def get(cls, id: str | None, /, *, allow_none: bool = False) -> T:
+        new = cls.new()
+
+        if allow_none and (id == '' or id is None):
+            return new.model()
+
+        if id is None:
+            raise ErrorException('Cannot get {kind} with no ID'.format(
+                kind=new.kind.capitalize()
+            ))
+
+        if id not in new.mapping:
+            raise NotFoundException(
+                '{kind} with ID {id} was not found in the database'.format(
+                    kind=new.kind.capitalize(),
+                    id=id,
+                ),
+            )
+
+        return new.mapping[id]
+
+    # Get the list of statuses depending on the context
+    @overload
+    @classmethod
+    def list(cls, /, **kwargs) -> List[T]: ...
+
+    @overload
+    @classmethod
+    def list(cls, /, as_class: bool = False, **kwargs) -> Self: ...
+
+    @classmethod
+    def list(cls, /, as_class: bool = False, **kwargs) -> List[T] | Self:
+        new = cls.new()
+        list = new.filter(**kwargs)
+
+        if as_class:
+            # Return a copy of the metadata list with overriden records
+            return cls(new.model, records=list)
+        else:
+            return list
+
+    # Instantiate the list with the proper class
+    @classmethod
+    def new(cls, /, *, force: bool = False) -> Self:
+        raise Exception('new() is not implemented for BrickMetadataList')
+
+    # URL to change the selected state of this metadata item for a set
+    @classmethod
+    def url_for_set_state(cls, id: str, /) -> str:
+        return url_for(
+            cls.set_state_endpoint,
+            id=id,
+        )
+
+    # URL to change the selected value of this metadata item for a set
+    @classmethod
+    def url_for_set_value(cls, id: str, /) -> str:
+        return url_for(
+            cls.set_value_endpoint,
+            id=id,
+        )

bricktracker/migrations/0007.py

@@ -0,0 +1,29 @@
+from typing import Any, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from ..sql import BrickSQL
+
+
+# Grab the list of checkboxes to create a list of SQL columns
+def migration_0007(sql: 'BrickSQL', /) -> dict[str, Any]:
+    # Don't realy on sql files as they could be removed in the future
+    sql.cursor.execute('SELECT "bricktracker_set_checkboxes"."id" FROM "bricktracker_set_checkboxes"')  # noqa: E501
+    records = sql.cursor.fetchall()
+
+    return {
+        'sources': ', '.join([
+            '"bricktracker_set_statuses_old"."status_{id}"'.format(id=record['id'])  # noqa: E501
+            for record
+            in records
+        ]),
+        'targets': ', '.join([
+            '"status_{id}"'.format(id=record['id'])
+            for record
+            in records
+        ]),
+        'structure': ', '.join([
+            '"status_{id}" BOOLEAN NOT NULL DEFAULT 0'.format(id=record['id'])
+            for record
+            in records
+        ])
+    }

bricktracker/minifigure.py

@@ -1,47 +1,68 @@
-from sqlite3 import Row
-from typing import Any, Self, TYPE_CHECKING
-
-from flask import current_app, url_for
+import logging
+import traceback
+from typing import Self, TYPE_CHECKING
 
 from .exceptions import ErrorException, NotFoundException
 from .part_list import BrickPartList
-from .rebrickable_image import RebrickableImage
-from .record import BrickRecord
+from .rebrickable_minifigure import RebrickableMinifigure
 if TYPE_CHECKING:
     from .set import BrickSet
+    from .socket import BrickSocket
+
+logger = logging.getLogger(__name__)
 
 
 # Lego minifigure
-class BrickMinifigure(BrickRecord):
-    brickset: 'BrickSet | None'
-
+class BrickMinifigure(RebrickableMinifigure):
     # Queries
     insert_query: str = 'minifigure/insert'
     generic_query: str = 'minifigure/select/generic'
     select_query: str = 'minifigure/select/specific'
 
-    def __init__(
-        self,
-        /,
-        brickset: 'BrickSet | None' = None,
-        record: Row | dict[str, Any] | None = None,
-    ):
-        super().__init__()
+    # Import a minifigure into the database
+    def download(self, socket: 'BrickSocket', refresh: bool = False) -> bool:
+        if self.brickset is None:
+            raise ErrorException('Importing a minifigure from Rebrickable outside of a set is not supported')  # noqa: E501
 
-        # Save the brickset
-        self.brickset = brickset
+        try:
+            # Insert into the database
+            socket.auto_progress(
+                message='Set {set}: inserting minifigure {figure} into database'.format(  # noqa: E501
+                    set=self.brickset.fields.set,
+                    figure=self.fields.figure
+                )
+            )
 
-        # Ingest the record if it has one
-        if record is not None:
-            self.ingest(record)
+            if not refresh:
+                # Insert into database
+                self.insert(commit=False)
 
-    # Return the number just in digits format
-    def clean_number(self, /) -> str:
-        number: str = self.fields.fig_num
-        number = number.removeprefix('fig-')
-        number = number.lstrip('0')
+            # Load the inventory
+            if not BrickPartList.download(
+                socket,
+                self.brickset,
+                minifigure=self,
+                refresh=refresh
+            ):
+                return False
 
-        return number
+            # Insert the rebrickable set into database (after counting parts)
+            self.insert_rebrickable()
+
+        except Exception as e:
+            socket.fail(
+                message='Error while importing minifigure {figure} from {set}: {error}'.format(  # noqa: E501
+                    figure=self.fields.figure,
+                    set=self.brickset.fields.set,
+                    error=e,
+                )
+            )
+
+            logger.debug(traceback.format_exc())
+
+            return False
+
+        return True
 
     # Parts
     def generic_parts(self, /) -> BrickPartList:
@@ -50,117 +71,38 @@ class BrickMinifigure(BrickRecord):
     # Parts
     def parts(self, /) -> BrickPartList:
         if self.brickset is None:
-            raise ErrorException('Part list for minifigure {number} requires a brickset'.format(  # noqa: E501
-                number=self.fields.fig_num,
+            raise ErrorException('Part list for minifigure {figure} requires a brickset'.format(  # noqa: E501
+                figure=self.fields.figure,
             ))
 
-        return BrickPartList().load(self.brickset, minifigure=self)
+        return BrickPartList().list_specific(self.brickset, minifigure=self)
 
     # Select a generic minifigure
-    def select_generic(self, fig_num: str, /) -> Self:
+    def select_generic(self, figure: str, /) -> Self:
         # Save the parameters to the fields
-        self.fields.fig_num = fig_num
+        self.fields.figure = figure
 
-        record = self.select(override_query=self.generic_query)
-
-        if record is None:
+        if not self.select(override_query=self.generic_query):
             raise NotFoundException(
-                'Minifigure with number {number} was not found in the database'.format(  # noqa: E501
-                    number=self.fields.fig_num,
+                'Minifigure with figure {figure} was not found in the database'.format(  # noqa: E501
+                    figure=self.fields.figure,
                 ),
             )
 
-        # Ingest the record
-        self.ingest(record)
-
         return self
 
-    # Select a specific minifigure (with a set and an number)
-    def select_specific(self, brickset: 'BrickSet', fig_num: str, /) -> Self:
+    # Select a specific minifigure (with a set and a figure)
+    def select_specific(self, brickset: 'BrickSet', figure: str, /) -> Self:
         # Save the parameters to the fields
         self.brickset = brickset
-        self.fields.fig_num = fig_num
+        self.fields.figure = figure
 
-        record = self.select()
-
-        if record is None:
+        if not self.select():
             raise NotFoundException(
-                'Minifigure with number {number} from set {set} was not found in the database'.format(  # noqa: E501
-                    number=self.fields.fig_num,
-                    set=self.brickset.fields.set_num,
+                'Minifigure with figure {figure} from set {set} was not found in the database'.format(  # noqa: E501
+                    figure=self.fields.figure,
+                    set=self.brickset.fields.set,
                 ),
             )
 
-        # Ingest the record
-        self.ingest(record)
-
         return self
-
-    # Return a dict with common SQL parameters for a minifigure
-    def sql_parameters(self, /) -> dict[str, Any]:
-        parameters = super().sql_parameters()
-
-        # Supplement from the brickset
-        if self.brickset is not None:
-            if 'u_id' not in parameters:
-                parameters['u_id'] = self.brickset.fields.u_id
-
-            if 'set_num' not in parameters:
-                parameters['set_num'] = self.brickset.fields.set_num
-
-        return parameters
-
-    # Self url
-    def url(self, /) -> str:
-        return url_for(
-            'minifigure.details',
-            number=self.fields.fig_num,
-        )
-
-    # Compute the url for minifigure part image
-    def url_for_image(self, /) -> str:
-        if not current_app.config['USE_REMOTE_IMAGES'].value:
-            if self.fields.set_img_url is None:
-                file = RebrickableImage.nil_minifigure_name()
-            else:
-                file = self.fields.fig_num
-
-            return RebrickableImage.static_url(file, 'MINIFIGURES_FOLDER')
-        else:
-            if self.fields.set_img_url is None:
-                return current_app.config['REBRICKABLE_IMAGE_NIL_MINIFIGURE'].value  # noqa: E501
-            else:
-                return self.fields.set_img_url
-
-    # Compute the url for the rebrickable page
-    def url_for_rebrickable(self, /) -> str:
-        if current_app.config['REBRICKABLE_LINKS'].value:
-            try:
-                return current_app.config['REBRICKABLE_LINK_MINIFIGURE_PATTERN'].value.format(  # noqa: E501
-                    number=self.fields.fig_num.lower(),
-                )
-            except Exception:
-                pass
-
-        return ''
-
-    # Normalize from Rebrickable
-    @staticmethod
-    def from_rebrickable(
-        data: dict[str, Any],
-        /,
-        brickset: 'BrickSet | None' = None,
-        **_,
-    ) -> dict[str, Any]:
-        record = {
-            'fig_num': data['set_num'],
-            'name': data['set_name'],
-            'quantity': data['quantity'],
-            'set_img_url': data['set_img_url'],
-        }
-
-        if brickset is not None:
-            record['set_num'] = brickset.fields.set_num
-            record['u_id'] = brickset.fields.u_id
-
-        return record

bricktracker/minifigure_list.py

@@ -1,11 +1,17 @@
+import logging
+import traceback
 from typing import Any, Self, TYPE_CHECKING
 
 from flask import current_app
 
 from .minifigure import BrickMinifigure
+from .rebrickable import Rebrickable
 from .record_list import BrickRecordList
 if TYPE_CHECKING:
     from .set import BrickSet
+    from .socket import BrickSocket
+
+logger = logging.getLogger(__name__)
 
 
 # Lego minifigures
@@ -15,10 +21,12 @@ class BrickMinifigureList(BrickRecordList[BrickMinifigure]):
 
     # Queries
     all_query: str = 'minifigure/list/all'
+    all_by_owner_query: str = 'minifigure/list/all_by_owner'
+    damaged_part_query: str = 'minifigure/list/damaged_part'
     last_query: str = 'minifigure/list/last'
+    missing_part_query: str = 'minifigure/list/missing_part'
     select_query: str = 'minifigure/list/from_set'
     using_part_query: str = 'minifigure/list/using_part'
-    missing_part_query: str = 'minifigure/list/missing_part'
 
     def __init__(self, /):
         super().__init__()
@@ -27,49 +35,110 @@ class BrickMinifigureList(BrickRecordList[BrickMinifigure]):
         self.brickset = None
 
         # Store the order for this list
-        self.order = current_app.config['MINIFIGURES_DEFAULT_ORDER'].value
+        self.order = current_app.config['MINIFIGURES_DEFAULT_ORDER']
 
     # Load all minifigures
     def all(self, /) -> Self:
-        for record in self.select(
-            override_query=self.all_query,
-            order=self.order
-        ):
-            minifigure = BrickMinifigure(record=record)
+        self.list(override_query=self.all_query)
 
-            self.records.append(minifigure)
+        return self
+
+    # Load all minifigures by owner
+    def all_by_owner(self, owner_id: str | None = None, /) -> Self:
+        # Save the owner_id parameter
+        self.fields.owner_id = owner_id
+
+        # Load the minifigures from the database
+        self.list(override_query=self.all_by_owner_query)
+
+        return self
+
+    # Minifigures with a part damaged part
+    def damaged_part(self, part: str, color: int, /) -> Self:
+        # Save the parameters to the fields
+        self.fields.part = part
+        self.fields.color = color
+
+        # Load the minifigures from the database
+        self.list(override_query=self.damaged_part_query)
 
         return self
 
     # Last added minifigure
-    def last(self, /, limit: int = 6) -> Self:
+    def last(self, /, *, limit: int = 6) -> Self:
         # Randomize
-        if current_app.config['RANDOM'].value:
+        if current_app.config['RANDOM']:
             order = 'RANDOM()'
         else:
-            order = 'minifigures.rowid DESC'
+            order = '"bricktracker_minifigures"."rowid" DESC'
 
-        for record in self.select(
-            override_query=self.last_query,
-            order=order,
-            limit=limit
-        ):
-            minifigure = BrickMinifigure(record=record)
-
-            self.records.append(minifigure)
+        self.list(override_query=self.last_query, order=order, limit=limit)
 
         return self
 
+    # Base minifigure list
+    def list(
+        self,
+        /,
+        *,
+        override_query: str | None = None,
+        order: str | None = None,
+        limit: int | None = None,
+        **context: Any,
+    ) -> None:
+        if order is None:
+            order = self.order
+
+        if hasattr(self, 'brickset'):
+            brickset = self.brickset
+        else:
+            brickset = None
+
+        # Prepare template context for owner filtering
+        context = {}
+        if hasattr(self.fields, 'owner_id') and self.fields.owner_id is not None:
+            context['owner_id'] = self.fields.owner_id
+
+        # Load the sets from the database
+        for record in super().select(
+            override_query=override_query,
+            order=order,
+            limit=limit,
+            **context
+        ):
+            minifigure = BrickMinifigure(brickset=brickset, record=record)
+
+            self.records.append(minifigure)
+
     # Load minifigures from a brickset
-    def load(self, brickset: 'BrickSet', /) -> Self:
+    def from_set(self, brickset: 'BrickSet', /) -> Self:
         # Save the brickset
         self.brickset = brickset
 
         # Load the minifigures from the database
-        for record in self.select(order=self.order):
-            minifigure = BrickMinifigure(brickset=self.brickset, record=record)
+        self.list()
 
-            self.records.append(minifigure)
+        return self
+
+    # Minifigures missing a part
+    def missing_part(self, part: str, color: int, /) -> Self:
+        # Save the parameters to the fields
+        self.fields.part = part
+        self.fields.color = color
+
+        # Load the minifigures from the database
+        self.list(override_query=self.missing_part_query)
+
+        return self
+
+    # Minifigure using a part
+    def using_part(self, part: str, color: int, /) -> Self:
+        # Save the parameters to the fields
+        self.fields.part = part
+        self.fields.color = color
+
+        # Load the minifigures from the database
+        self.list(override_query=self.using_part_query)
 
         return self
 
@@ -78,55 +147,58 @@ class BrickMinifigureList(BrickRecordList[BrickMinifigure]):
         parameters: dict[str, Any] = super().sql_parameters()
 
         if self.brickset is not None:
-            parameters['u_id'] = self.brickset.fields.u_id
-            parameters['set_num'] = self.brickset.fields.set_num
+            parameters['id'] = self.brickset.fields.id
+
+        # Add owner_id parameter for owner filtering
+        if hasattr(self.fields, 'owner_id') and self.fields.owner_id is not None:
+            parameters['owner_id'] = self.fields.owner_id
 
         return parameters
 
-    # Minifigures missing a part
-    def missing_part(
-        self,
-        part_num: str,
-        color_id: int,
+    # Import the minifigures from Rebrickable
+    @staticmethod
+    def download(
+        socket: 'BrickSocket',
+        brickset: 'BrickSet',
         /,
-        element_id: int | None = None,
-    ) -> Self:
-        # Save the parameters to the fields
-        self.fields.part_num = part_num
-        self.fields.color_id = color_id
-        self.fields.element_id = element_id
+        *,
+        refresh: bool = False
+    ) -> bool:
+        try:
+            socket.auto_progress(
+                message='Set {set}: loading minifigures from Rebrickable'.format(  # noqa: E501
+                    set=brickset.fields.set,
+                ),
+                increment_total=True,
+            )
 
-        # Load the minifigures from the database
-        for record in self.select(
-            override_query=self.missing_part_query,
-            order=self.order
-        ):
-            minifigure = BrickMinifigure(record=record)
+            logger.debug('rebrick.lego.get_set_minifigs("{set}")'.format(
+                set=brickset.fields.set,
+            ))
 
-            self.records.append(minifigure)
+            minifigures = Rebrickable[BrickMinifigure](
+                'get_set_minifigs',
+                brickset.fields.set,
+                BrickMinifigure,
+                socket=socket,
+                brickset=brickset,
+            ).list()
 
-        return self
+            # Process each minifigure
+            for minifigure in minifigures:
+                if not minifigure.download(socket, refresh=refresh):
+                    return False
 
-    # Minifigure using a part
-    def using_part(
-        self,
-        part_num: str,
-        color_id: int,
-        /,
-        element_id: int | None = None,
-    ) -> Self:
-        # Save the parameters to the fields
-        self.fields.part_num = part_num
-        self.fields.color_id = color_id
-        self.fields.element_id = element_id
+            return True
 
-        # Load the minifigures from the database
-        for record in self.select(
-            override_query=self.using_part_query,
-            order=self.order
-        ):
-            minifigure = BrickMinifigure(record=record)
+        except Exception as e:
+            socket.fail(
+                message='Error while importing set {set} minifigure list: {error}'.format(  # noqa: E501
+                    set=brickset.fields.set,
+                    error=e,
+                )
+            )
 
-            self.records.append(minifigure)
+            logger.debug(traceback.format_exc())
 
-        return self
+            return False

bricktracker/navbar.py

@@ -11,9 +11,10 @@ NAVBAR: Final[list[dict[str, Any]]] = [
     {'e': 'set.list', 't': 'Sets', 'i': 'grid-line', 'f': 'HIDE_ALL_SETS'},  # noqa: E501
     {'e': 'add.add', 't': 'Add', 'i': 'add-circle-line', 'f': 'HIDE_ADD_SET'},  # noqa: E501
     {'e': 'part.list', 't': 'Parts', 'i': 'shapes-line', 'f': 'HIDE_ALL_PARTS'},  # noqa: E501
-    {'e': 'part.missing', 't': 'Missing', 'i': 'error-warning-line', 'f': 'HIDE_MISSING_PARTS'},  # noqa: E501
+    {'e': 'part.problem', 't': 'Problems', 'i': 'error-warning-line', 'f': 'HIDE_ALL_PROBLEMS_PARTS'},  # noqa: E501
     {'e': 'minifigure.list', 't': 'Minifigures', 'i': 'group-line', 'f': 'HIDE_ALL_MINIFIGURES'},  # noqa: E501
     {'e': 'instructions.list', 't': 'Instructions', 'i': 'file-line', 'f': 'HIDE_ALL_INSTRUCTIONS'},  # noqa: E501
+    {'e': 'storage.list', 't': 'Storages', 'i': 'archive-2-line', 'f': 'HIDE_ALL_STORAGES'},  # noqa: E501
     {'e': 'wish.list', 't': 'Wishlist', 'i': 'gift-line', 'f': 'HIDE_WISHES'},
     {'e': 'admin.admin', 't': 'Admin', 'i': 'settings-4-line', 'f': 'HIDE_ADMIN'},  # noqa: E501
 ]

bricktracker/parser.py

@@ -0,0 +1,37 @@
+from .exceptions import ErrorException
+
+
+# Make sense of string supposed to contain a set ID
+def parse_set(set: str, /) -> str:
+    number, _, version = set.partition('-')
+
+    # Making sure both are integers
+    if version == '':
+        version = 1
+
+    try:
+        number = int(number)
+    except Exception:
+        raise ErrorException('Number "{number}" is not a number'.format(
+            number=number,
+        ))
+
+    try:
+        version = int(version)
+    except Exception:
+        raise ErrorException('Version "{version}" is not a number'.format(
+            version=version,
+        ))
+
+    # Make sure both are positive
+    if number < 0:
+        raise ErrorException('Number "{number}" should be positive'.format(
+            number=number,
+        ))
+
+    if version < 0:
+        raise ErrorException('Version "{version}" should be positive'.format(  # noqa: E501
+            version=version,
+        ))
+
+    return '{number}-{version}'.format(number=number, version=version)

bricktracker/part.py

@@ -1,23 +1,25 @@
-import os
+import logging
 from sqlite3 import Row
 from typing import Any, Self, TYPE_CHECKING
-from urllib.parse import urlparse
+import traceback
 
-from flask import current_app, url_for
+from flask import url_for
 
-from .exceptions import DatabaseException, ErrorException, NotFoundException
-from .rebrickable_image import RebrickableImage
-from .record import BrickRecord
+from .exceptions import ErrorException, NotFoundException
+from .rebrickable_part import RebrickablePart
 from .sql import BrickSQL
 if TYPE_CHECKING:
     from .minifigure import BrickMinifigure
     from .set import BrickSet
+    from .socket import BrickSocket
+
+logger = logging.getLogger(__name__)
 
 
 # Lego set or minifig part
-class BrickPart(BrickRecord):
-    brickset: 'BrickSet | None'
-    minifigure: 'BrickMinifigure | None'
+class BrickPart(RebrickablePart):
+    identifier: str
+    kind: str
 
     # Queries
     insert_query: str = 'part/insert'
@@ -27,262 +29,180 @@ class BrickPart(BrickRecord):
     def __init__(
         self,
         /,
+        *,
         brickset: 'BrickSet | None' = None,
         minifigure: 'BrickMinifigure | None' = None,
-        record: Row | dict[str, Any] | None = None,
+        record: Row | dict[str, Any] | None = None
     ):
-        super().__init__()
-
-        # Save the brickset and minifigure
-        self.brickset = brickset
-        self.minifigure = minifigure
-
-        # Ingest the record if it has one
-        if record is not None:
-            self.ingest(record)
-
-    # Delete missing part
-    def delete_missing(self, /) -> None:
-        BrickSQL().execute_and_commit(
-            'missing/delete/from_set',
-            parameters=self.sql_parameters()
+        super().__init__(
+            brickset=brickset,
+            minifigure=minifigure,
+            record=record
         )
 
-    # Set missing part
-    def set_missing(self, quantity: int, /) -> None:
-        parameters = self.sql_parameters()
-        parameters['quantity'] = quantity
+        if self.minifigure is not None:
+            self.identifier = self.minifigure.fields.figure
+            self.kind = 'Minifigure'
+        elif self.brickset is not None:
+            self.identifier = self.brickset.fields.set
+            self.kind = 'Set'
 
-        # Can't use UPSERT because the database has no keys
-        # Try to update
-        database = BrickSQL()
-        rows, _ = database.execute(
-            'missing/update/from_set',
-            parameters=parameters,
-        )
+    # Import a part into the database
+    def download(self, socket: 'BrickSocket', refresh: bool = False) -> bool:
+        if self.brickset is None:
+            raise ErrorException('Importing a part from Rebrickable outside of a set is not supported')  # noqa: E501
 
-        # Insert if no row has been affected
-        if not rows:
-            rows, _ = database.execute(
-                'missing/insert',
-                parameters=parameters,
+        try:
+            # Insert into the database
+            socket.auto_progress(
+                message='{kind} {identifier}: inserting part {part} into database'.format(  # noqa: E501
+                    kind=self.kind,
+                    identifier=self.identifier,
+                    part=self.fields.part
+                )
             )
 
-            if rows != 1:
-                raise DatabaseException(
-                    'Could not update the missing quantity for part {id}'.format(  # noqa: E501
-                        id=self.fields.id
-                    )
-                )
+            if not refresh:
+                # Insert into database
+                self.insert(commit=False)
 
-        database.commit()
+            # Insert the rebrickable set into database
+            self.insert_rebrickable()
+
+        except Exception as e:
+            socket.fail(
+                message='Error while importing part {part} from {kind} {identifier}: {error}'.format(  # noqa: E501
+                    part=self.fields.part,
+                    kind=self.kind,
+                    identifier=self.identifier,
+                    error=e,
+                )
+            )
+
+            logger.debug(traceback.format_exc())
+
+            return False
+
+        return True
+
+    # A identifier for HTML component
+    def html_id(self, prefix: str | None = None, /) -> str:
+        components: list[str] = ['part']
+
+        if prefix is not None:
+            components.append(prefix)
+
+        if self.fields.figure is not None:
+            components.append(self.fields.figure)
+
+        components.append(self.fields.part)
+        components.append(str(self.fields.color))
+        components.append(str(self.fields.spare))
+
+        return '-'.join(components)
 
     # Select a generic part
     def select_generic(
         self,
-        part_num: str,
-        color_id: int,
+        part: str,
+        color: int,
         /,
-        element_id: int | None = None
     ) -> Self:
         # Save the parameters to the fields
-        self.fields.part_num = part_num
-        self.fields.color_id = color_id
-        self.fields.element_id = element_id
+        self.fields.part = part
+        self.fields.color = color
 
-        record = self.select(override_query=self.generic_query)
-
-        if record is None:
+        if not self.select(override_query=self.generic_query):
             raise NotFoundException(
-                'Part with number {number}, color ID {color} and element ID {element} was not found in the database'.format(  # noqa: E501
-                    number=self.fields.part_num,
-                    color=self.fields.color_id,
-                    element=self.fields.element_id,
+                'Part with number {number}, color ID {color} was not found in the database'.format(  # noqa: E501
+                    number=self.fields.part,
+                    color=self.fields.color,
                 ),
             )
 
-        # Ingest the record
-        self.ingest(record)
-
         return self
 
     # Select a specific part (with a set and an id, and option. a minifigure)
     def select_specific(
         self,
         brickset: 'BrickSet',
-        id: str,
+        part: str,
+        color: int,
+        spare: int,
         /,
+        *,
         minifigure: 'BrickMinifigure | None' = None,
     ) -> Self:
         # Save the parameters to the fields
         self.brickset = brickset
         self.minifigure = minifigure
-        self.fields.id = id
+        self.fields.part = part
+        self.fields.color = color
+        self.fields.spare = spare
 
-        record = self.select()
+        if not self.select():
+            if self.minifigure is not None:
+                figure = self.minifigure.fields.figure
+            else:
+                figure = None
 
-        if record is None:
             raise NotFoundException(
-                'Part with ID {id} from set {set} was not found in the database'.format(  # noqa: E501
+                'Part {part} with color {color} (spare: {spare}) from set {set} ({id}) (minifigure: {figure}) was not found in the database'.format(  # noqa: E501
+                    part=self.fields.part,
+                    color=self.fields.color,
+                    spare=self.fields.spare,
                     id=self.fields.id,
-                    set=self.brickset.fields.set_num,
+                    set=self.brickset.fields.set,
+                    figure=figure,
                 ),
             )
 
-        # Ingest the record
-        self.ingest(record)
-
         return self
 
-    # Return a dict with common SQL parameters for a part
-    def sql_parameters(self, /) -> dict[str, Any]:
-        parameters = super().sql_parameters()
+    # Update a problematic part
+    def update_problem(self, problem: str, json: Any | None, /) -> int:
+        amount: str | int = json.get('value', '')  # type: ignore
 
-        # Supplement from the brickset
-        if 'u_id' not in parameters and self.brickset is not None:
-            parameters['u_id'] = self.brickset.fields.u_id
+        # We need a positive integer
+        try:
+            if amount == '':
+                amount = 0
 
-        if 'set_num' not in parameters:
-            if self.minifigure is not None:
-                parameters['set_num'] = self.minifigure.fields.fig_num
+            amount = int(amount)
 
-            elif self.brickset is not None:
-                parameters['set_num'] = self.brickset.fields.set_num
+            if amount < 0:
+                amount = 0
+        except Exception:
+            raise ErrorException('"{amount}" is not a valid integer'.format(
+                amount=amount
+            ))
 
-        return parameters
+        if amount < 0:
+            raise ErrorException('Cannot set a negative amount')
 
-    # Update the missing part
-    def update_missing(self, missing: Any, /) -> None:
-        # If empty, delete it
-        if missing == '':
-            self.delete_missing()
+        setattr(self.fields, problem, amount)
 
-        else:
-            # Try to understand it as a number
-            try:
-                missing = int(missing)
-            except Exception:
-                raise ErrorException('"{missing}" is not a valid integer'.format(  # noqa: E501
-                    missing=missing
-                ))
-
-            # If 0, delete it
-            if missing == 0:
-                self.delete_missing()
-
-            else:
-                # If negative, it's an error
-                if missing < 0:
-                    raise ErrorException('Cannot set a negative missing value')
-
-                # Otherwise upsert it
-                # Not checking if it is too much, you do you
-                self.set_missing(missing)
-
-    # Self url
-    def url(self, /) -> str:
-        return url_for(
-            'part.details',
-            number=self.fields.part_num,
-            color=self.fields.color_id,
-            element=self.fields.element_id,
+        BrickSQL().execute_and_commit(
+            'part/update/{problem}'.format(problem=problem),
+            parameters=self.sql_parameters()
         )
 
-    # Compute the url for the bricklink page
-    def url_for_bricklink(self, /) -> str:
-        if current_app.config['BRICKLINK_LINKS'].value:
-            try:
-                return current_app.config['BRICKLINK_LINK_PART_PATTERN'].value.format(  # noqa: E501
-                    number=self.fields.part_num,
-                )
-            except Exception:
-                pass
+        return amount
 
-        return ''
-
-    # Compute the url for the part image
-    def url_for_image(self, /) -> str:
-        if not current_app.config['USE_REMOTE_IMAGES'].value:
-            if self.fields.part_img_url is None:
-                file = RebrickableImage.nil_name()
-            else:
-                file = self.fields.part_img_url_id
-
-            return RebrickableImage.static_url(file, 'PARTS_FOLDER')
-        else:
-            if self.fields.part_img_url is None:
-                return current_app.config['REBRICKABLE_IMAGE_NIL'].value
-            else:
-                return self.fields.part_img_url
-
-    # Compute the url for missing part
-    def url_for_missing(self, /) -> str:
+    # Compute the url for problematic part
+    def url_for_problem(self, problem: str, /) -> str:
         # Different URL for a minifigure part
         if self.minifigure is not None:
-            return url_for(
-                'set.missing_minifigure_part',
-                id=self.fields.u_id,
-                minifigure_id=self.minifigure.fields.fig_num,
-                part_id=self.fields.id,
-            )
+            figure = self.minifigure.fields.figure
+        else:
+            figure = None
 
         return url_for(
-            'set.missing_part',
-            id=self.fields.u_id,
-            part_id=self.fields.id
+            'set.problem_part',
+            id=self.fields.id,
+            figure=figure,
+            part=self.fields.part,
+            color=self.fields.color,
+            spare=self.fields.spare,
+            problem=problem,
         )
-
-    # Compute the url for the rebrickable page
-    def url_for_rebrickable(self, /) -> str:
-        if current_app.config['REBRICKABLE_LINKS'].value:
-            try:
-                return current_app.config['REBRICKABLE_LINK_PART_PATTERN'].value.format(  # noqa: E501
-                    number=self.fields.part_num,
-                    color=self.fields.color_id,
-                )
-            except Exception:
-                pass
-
-        return ''
-
-    # Normalize from Rebrickable
-    @staticmethod
-    def from_rebrickable(
-        data: dict[str, Any],
-        /,
-        brickset: 'BrickSet | None' = None,
-        minifigure: 'BrickMinifigure | None' = None,
-        **_,
-    ) -> dict[str, Any]:
-        record = {
-            'set_num': data['set_num'],
-            'id': data['id'],
-            'part_num': data['part']['part_num'],
-            'name': data['part']['name'],
-            'part_img_url': data['part']['part_img_url'],
-            'part_img_url_id': None,
-            'color_id': data['color']['id'],
-            'color_name': data['color']['name'],
-            'quantity': data['quantity'],
-            'is_spare': data['is_spare'],
-            'element_id': data['element_id'],
-        }
-
-        if brickset is not None:
-            record['u_id'] = brickset.fields.u_id
-
-        if minifigure is not None:
-            record['set_num'] = data['fig_num']
-
-        # Extract the file name
-        if data['part']['part_img_url'] is not None:
-            part_img_url_file = os.path.basename(
-                urlparse(data['part']['part_img_url']).path
-            )
-
-            part_img_url_id, _ = os.path.splitext(part_img_url_file)
-
-            if part_img_url_id is not None or part_img_url_id != '':
-                record['part_img_url_id'] = part_img_url_id
-
-        return record

bricktracker/part_list.py

@@ -1,12 +1,18 @@
+import logging
 from typing import Any, Self, TYPE_CHECKING
+import traceback
 
 from flask import current_app
 
 from .part import BrickPart
+from .rebrickable import Rebrickable
 from .record_list import BrickRecordList
 if TYPE_CHECKING:
     from .minifigure import BrickMinifigure
     from .set import BrickSet
+    from .socket import BrickSocket
+
+logger = logging.getLogger(__name__)
 
 
 # Lego set or minifig parts
@@ -17,10 +23,13 @@ class BrickPartList(BrickRecordList[BrickPart]):
 
     # Queries
     all_query: str = 'part/list/all'
+    all_by_owner_query: str = 'part/list/all_by_owner'
+    different_color_query = 'part/list/with_different_color'
     last_query: str = 'part/list/last'
     minifigure_query: str = 'part/list/from_minifigure'
-    missing_query: str = 'part/list/missing'
-    select_query: str = 'part/list/from_set'
+    problem_query: str = 'part/list/problem'
+    print_query: str = 'part/list/from_print'
+    select_query: str = 'part/list/specific'
 
     def __init__(self, /):
         super().__init__()
@@ -30,25 +39,97 @@ class BrickPartList(BrickRecordList[BrickPart]):
         self.minifigure = None
 
         # Store the order for this list
-        self.order = current_app.config['PARTS_DEFAULT_ORDER'].value
+        self.order = current_app.config['PARTS_DEFAULT_ORDER']
 
     # Load all parts
     def all(self, /) -> Self:
-        for record in self.select(
-            override_query=self.all_query,
-            order=self.order
-        ):
-            part = BrickPart(record=record)
-
-            self.records.append(part)
+        self.list(override_query=self.all_query)
 
         return self
 
-    # Load parts from a brickset or minifigure
-    def load(
+    # Load all parts by owner
+    def all_by_owner(self, owner_id: str | None = None, /) -> Self:
+        # Save the owner_id parameter
+        self.fields.owner_id = owner_id
+
+        # Load the parts from the database
+        self.list(override_query=self.all_by_owner_query)
+
+        return self
+
+    # Load all parts with filters (owner and/or color)
+    def all_filtered(self, owner_id: str | None = None, color_id: str | None = None, /) -> Self:
+        # Save the filter parameters
+        if owner_id is not None:
+            self.fields.owner_id = owner_id
+        if color_id is not None:
+            self.fields.color_id = color_id
+
+        # Choose query based on whether owner filtering is needed
+        if owner_id and owner_id != 'all':
+            query = self.all_by_owner_query
+        else:
+            query = self.all_query
+
+        # Load the parts from the database
+        self.list(override_query=query)
+
+        return self
+
+    # Base part list
+    def list(
+        self,
+        /,
+        *,
+        override_query: str | None = None,
+        order: str | None = None,
+        limit: int | None = None,
+        **context: Any,
+    ) -> None:
+        if order is None:
+            order = self.order
+
+        if hasattr(self, 'brickset'):
+            brickset = self.brickset
+        else:
+            brickset = None
+
+        if hasattr(self, 'minifigure'):
+            minifigure = self.minifigure
+        else:
+            minifigure = None
+
+        # Prepare template context for filtering
+        context_vars = {}
+        if hasattr(self.fields, 'owner_id') and self.fields.owner_id is not None:
+            context_vars['owner_id'] = self.fields.owner_id
+        if hasattr(self.fields, 'color_id') and self.fields.color_id is not None:
+            context_vars['color_id'] = self.fields.color_id
+
+        # Load the sets from the database
+        for record in super().select(
+            override_query=override_query,
+            order=order,
+            limit=limit,
+            **context_vars
+        ):
+            part = BrickPart(
+                brickset=brickset,
+                minifigure=minifigure,
+                record=record,
+            )
+
+            if current_app.config['SKIP_SPARE_PARTS'] and part.fields.spare:
+                continue
+
+            self.records.append(part)
+
+    # List specific parts from a brickset or minifigure
+    def list_specific(
         self,
         brickset: 'BrickSet',
         /,
+        *,
         minifigure: 'BrickMinifigure | None' = None,
     ) -> Self:
         # Save the brickset and minifigure
@@ -56,20 +137,7 @@ class BrickPartList(BrickRecordList[BrickPart]):
         self.minifigure = minifigure
 
         # Load the parts from the database
-        for record in self.select(order=self.order):
-            part = BrickPart(
-                brickset=self.brickset,
-                minifigure=minifigure,
-                record=record,
-            )
-
-            if (
-                current_app.config['SKIP_SPARE_PARTS'].value and
-                part.fields.is_spare
-            ):
-                continue
-
-            self.records.append(part)
+        self.list()
 
         return self
 
@@ -83,50 +151,133 @@ class BrickPartList(BrickRecordList[BrickPart]):
         self.minifigure = minifigure
 
         # Load the parts from the database
-        for record in self.select(
-            override_query=self.minifigure_query,
-            order=self.order
-        ):
-            part = BrickPart(
-                minifigure=minifigure,
-                record=record,
-            )
-
-            if (
-                current_app.config['SKIP_SPARE_PARTS'].value and
-                part.fields.is_spare
-            ):
-                continue
-
-            self.records.append(part)
+        self.list(override_query=self.minifigure_query)
 
         return self
 
-    # Load missing parts
-    def missing(self, /) -> Self:
-        for record in self.select(
-            override_query=self.missing_query,
-            order=self.order
-        ):
-            part = BrickPart(record=record)
+    # Load generic parts from a print
+    def from_print(
+        self,
+        brickpart: BrickPart,
+        /,
+    ) -> Self:
+        # Save the part and print
+        if brickpart.fields.print is not None:
+            self.fields.print = brickpart.fields.print
+        else:
+            self.fields.print = brickpart.fields.part
 
-            self.records.append(part)
+        self.fields.part = brickpart.fields.part
+        self.fields.color = brickpart.fields.color
+
+        # Load the parts from the database
+        self.list(override_query=self.print_query)
+
+        return self
+
+    # Load problematic parts
+    def problem(self, /) -> Self:
+        self.list(override_query=self.problem_query)
 
         return self
 
     # Return a dict with common SQL parameters for a parts list
     def sql_parameters(self, /) -> dict[str, Any]:
-        parameters: dict[str, Any] = {}
+        parameters: dict[str, Any] = super().sql_parameters()
 
         # Set id
         if self.brickset is not None:
-            parameters['u_id'] = self.brickset.fields.u_id
+            parameters['id'] = self.brickset.fields.id
 
         # Use the minifigure number if present,
-        # otherwise use the set number
         if self.minifigure is not None:
-            parameters['set_num'] = self.minifigure.fields.fig_num
-        elif self.brickset is not None:
-            parameters['set_num'] = self.brickset.fields.set_num
+            parameters['figure'] = self.minifigure.fields.figure
+        else:
+            parameters['figure'] = None
 
         return parameters
+
+    # Load generic parts with same base but different color
+    def with_different_color(
+        self,
+        brickpart: BrickPart,
+        /,
+    ) -> Self:
+        # Save the part
+        self.fields.part = brickpart.fields.part
+        self.fields.color = brickpart.fields.color
+
+        # Load the parts from the database
+        self.list(override_query=self.different_color_query)
+
+        return self
+
+    # Import the parts from Rebrickable
+    @staticmethod
+    def download(
+        socket: 'BrickSocket',
+        brickset: 'BrickSet',
+        /,
+        *,
+        minifigure: 'BrickMinifigure | None' = None,
+        refresh: bool = False
+    ) -> bool:
+        if minifigure is not None:
+            identifier = minifigure.fields.figure
+            kind = 'Minifigure'
+            method = 'get_minifig_elements'
+        else:
+            identifier = brickset.fields.set
+            kind = 'Set'
+            method = 'get_set_elements'
+
+        try:
+            socket.auto_progress(
+                message='{kind} {identifier}: loading parts inventory from Rebrickable'.format(  # noqa: E501
+                    kind=kind,
+                    identifier=identifier,
+                ),
+                increment_total=True,
+            )
+
+            logger.debug('rebrick.lego.{method}("{identifier}")'.format(
+                method=method,
+                identifier=identifier,
+            ))
+
+            inventory = Rebrickable[BrickPart](
+                method,
+                identifier,
+                BrickPart,
+                socket=socket,
+                brickset=brickset,
+                minifigure=minifigure,
+            ).list()
+
+            # Process each part
+            number_of_parts: int = 0
+            for part in inventory:
+                # Count the number of parts for minifigures
+                if minifigure is not None:
+                    number_of_parts += part.fields.quantity
+
+                if not part.download(socket, refresh=refresh):
+                    return False
+
+            if minifigure is not None:
+                minifigure.fields.number_of_parts = number_of_parts
+
+        except Exception as e:
+            socket.fail(
+                message='Error while importing {kind} {identifier} parts list: {error}'.format(  # noqa: E501
+                    kind=kind,
+                    identifier=identifier,
+                    error=e,
+                )
+            )
+
+            logger.debug(traceback.format_exc())
+
+            return False
+
+        return True

bricktracker/rebrickable.py

@@ -9,11 +9,12 @@ from .exceptions import NotFoundException, ErrorException
 if TYPE_CHECKING:
     from .minifigure import BrickMinifigure
     from .part import BrickPart
+    from .rebrickable_set import RebrickableSet
     from .set import BrickSet
     from .socket import BrickSocket
     from .wish import BrickWish
 
-T = TypeVar('T', 'BrickSet', 'BrickPart', 'BrickMinifigure', 'BrickWish')
+T = TypeVar('T', 'RebrickableSet', 'BrickPart', 'BrickMinifigure', 'BrickWish')
 
 
 # An helper around the rebrick library, autoconverting
@@ -23,10 +24,11 @@ class Rebrickable(Generic[T]):
     number: str
     model: Type[T]
 
-    socket: 'BrickSocket | None'
     brickset: 'BrickSet | None'
-    minifigure: 'BrickMinifigure | None'
+    instance: T | None
     kind: str
+    minifigure: 'BrickMinifigure | None'
+    socket: 'BrickSocket | None'
 
     def __init__(
         self,
@@ -34,9 +36,11 @@ class Rebrickable(Generic[T]):
         number: str,
         model: Type[T],
         /,
-        socket: 'BrickSocket | None' = None,
+        *,
         brickset: 'BrickSet | None' = None,
-        minifigure: 'BrickMinifigure | None' = None
+        instance: T | None = None,
+        minifigure: 'BrickMinifigure | None' = None,
+        socket: 'BrickSocket | None' = None,
     ):
         if not hasattr(lego, method):
             raise ErrorException('{method} is not a valid method for the rebrick.lego module'.format(  # noqa: E501
@@ -48,9 +52,10 @@ class Rebrickable(Generic[T]):
         self.number = number
         self.model = model
 
-        self.socket = socket
         self.brickset = brickset
+        self.instance = instance
         self.minifigure = minifigure
+        self.socket = socket
 
         if self.minifigure is not None:
             self.kind = 'Minifigure'
@@ -61,13 +66,15 @@ class Rebrickable(Generic[T]):
     def get(self, /) -> T:
         model_parameters = self.model_parameters()
 
-        return self.model(
-            **model_parameters,
-            record=self.model.from_rebrickable(
-                self.load(),
-                brickset=self.brickset,
-            ),
-        )
+        if self.instance is None:
+            self.instance = self.model(**model_parameters)
+
+        self.instance.ingest(self.model.from_rebrickable(
+            self.load(),
+            brickset=self.brickset,
+        ))
+
+        return self.instance
 
     # Get paginated elements from the Rebrickable API
     def list(self, /) -> list[T]:
@@ -77,7 +84,7 @@ class Rebrickable(Generic[T]):
 
         # Bootstrap a first set of parameters
         parameters: dict[str, Any] | None = {
-            'page_size': current_app.config['REBRICKABLE_PAGE_SIZE'].value,
+            'page_size': current_app.config['REBRICKABLE_PAGE_SIZE'],
         }
 
         # Read all pages
@@ -113,9 +120,9 @@ class Rebrickable(Generic[T]):
         return results
 
     # Load from the API
-    def load(self, /, parameters: dict[str, Any] = {}) -> dict[str, Any]:
+    def load(self, /, *, parameters: dict[str, Any] = {}) -> dict[str, Any]:
         # Inject the API key
-        parameters['api_key'] = current_app.config['REBRICKABLE_API_KEY'].value,  # noqa: E501
+        parameters['api_key'] = current_app.config['REBRICKABLE_API_KEY']
 
         try:
             return json.loads(

bricktracker/rebrickable_image.py

@@ -8,28 +8,29 @@ from shutil import copyfileobj
 
 from .exceptions import DownloadException
 if TYPE_CHECKING:
-    from .minifigure import BrickMinifigure
-    from .part import BrickPart
-    from .set import BrickSet
+    from .rebrickable_minifigure import RebrickableMinifigure
+    from .rebrickable_part import RebrickablePart
+    from .rebrickable_set import RebrickableSet
 
 
 # A set, part or minifigure image from Rebrickable
 class RebrickableImage(object):
-    brickset: 'BrickSet'
-    minifigure: 'BrickMinifigure | None'
-    part: 'BrickPart | None'
+    set: 'RebrickableSet'
+    minifigure: 'RebrickableMinifigure | None'
+    part: 'RebrickablePart | None'
 
     extension: str | None
 
     def __init__(
         self,
-        brickset: 'BrickSet',
+        set: 'RebrickableSet',
         /,
-        minifigure: 'BrickMinifigure | None' = None,
-        part: 'BrickPart | None' = None,
+        *,
+        minifigure: 'RebrickableMinifigure | None' = None,
+        part: 'RebrickablePart | None' = None,
     ):
         # Save all objects
-        self.brickset = brickset
+        self.set = set
         self.minifigure = minifigure
         self.part = part
 
@@ -70,28 +71,28 @@ class RebrickableImage(object):
     # Return the folder depending on the objects provided
     def folder(self, /) -> str:
         if self.part is not None:
-            return current_app.config['PARTS_FOLDER'].value
+            return current_app.config['PARTS_FOLDER']
 
         if self.minifigure is not None:
-            return current_app.config['MINIFIGURES_FOLDER'].value
+            return current_app.config['MINIFIGURES_FOLDER']
 
-        return current_app.config['SETS_FOLDER'].value
+        return current_app.config['SETS_FOLDER']
 
     # Return the id depending on the objects provided
     def id(self, /) -> str:
         if self.part is not None:
-            if self.part.fields.part_img_url_id is None:
+            if self.part.fields.image_id is None:
                 return RebrickableImage.nil_name()
             else:
-                return self.part.fields.part_img_url_id
+                return self.part.fields.image_id
 
         if self.minifigure is not None:
-            if self.minifigure.fields.set_img_url is None:
+            if self.minifigure.fields.image is None:
                 return RebrickableImage.nil_minifigure_name()
             else:
-                return self.minifigure.fields.fig_num
+                return self.minifigure.fields.figure
 
-        return self.brickset.fields.set_num
+        return self.set.fields.set
 
     # Return the path depending on the objects provided
     def path(self, /) -> str:
@@ -104,25 +105,25 @@ class RebrickableImage(object):
     # Return the url depending on the objects provided
     def url(self, /) -> str:
         if self.part is not None:
-            if self.part.fields.part_img_url is None:
-                return current_app.config['REBRICKABLE_IMAGE_NIL'].value
+            if self.part.fields.image is None:
+                return current_app.config['REBRICKABLE_IMAGE_NIL']
             else:
-                return self.part.fields.part_img_url
+                return self.part.fields.image
 
         if self.minifigure is not None:
-            if self.minifigure.fields.set_img_url is None:
-                return current_app.config['REBRICKABLE_IMAGE_NIL_MINIFIGURE'].value  # noqa: E501
+            if self.minifigure.fields.image is None:
+                return current_app.config['REBRICKABLE_IMAGE_NIL_MINIFIGURE']
             else:
-                return self.minifigure.fields.set_img_url
+                return self.minifigure.fields.image
 
-        return self.brickset.fields.set_img_url
+        return self.set.fields.image
 
     # Return the name of the nil image file
     @staticmethod
     def nil_name() -> str:
         filename, _ = os.path.splitext(
             os.path.basename(
-                urlparse(current_app.config['REBRICKABLE_IMAGE_NIL'].value).path  # noqa: E501
+                urlparse(current_app.config['REBRICKABLE_IMAGE_NIL']).path
             )
         )
 
@@ -133,7 +134,7 @@ class RebrickableImage(object):
     def nil_minifigure_name() -> str:
         filename, _ = os.path.splitext(
             os.path.basename(
-                urlparse(current_app.config['REBRICKABLE_IMAGE_NIL_MINIFIGURE'].value).path  # noqa: E501
+                urlparse(current_app.config['REBRICKABLE_IMAGE_NIL_MINIFIGURE']).path  # noqa: E501
             )
         )
 
@@ -142,7 +143,7 @@ class RebrickableImage(object):
     # Return the static URL for an image given a name and folder
     @staticmethod
     def static_url(name: str, folder_name: str) -> str:
-        folder: str = current_app.config[folder_name].value
+        folder: str = current_app.config[folder_name]
 
         # /!\ Everything is saved as .jpg, even if it came from a .png
         # not changing this behaviour.

bricktracker/rebrickable_minifigure.py

@@ -0,0 +1,111 @@
+from sqlite3 import Row
+from typing import Any, TYPE_CHECKING
+
+from flask import current_app, url_for
+
+from .exceptions import ErrorException
+from .rebrickable_image import RebrickableImage
+from .record import BrickRecord
+if TYPE_CHECKING:
+    from .set import BrickSet
+
+
+# A minifigure from Rebrickable
+class RebrickableMinifigure(BrickRecord):
+    brickset: 'BrickSet | None'
+
+    # Queries
+    select_query: str = 'rebrickable/minifigure/select'
+    insert_query: str = 'rebrickable/minifigure/insert'
+
+    def __init__(
+        self,
+        /,
+        *,
+        brickset: 'BrickSet | None' = None,
+        record: Row | dict[str, Any] | None = None
+    ):
+        super().__init__()
+
+        # Save the brickset
+        self.brickset = brickset
+
+        # Ingest the record if it has one
+        if record is not None:
+            self.ingest(record)
+
+    # Insert the minifigure from Rebrickable
+    def insert_rebrickable(self, /) -> None:
+        if self.brickset is None:
+            raise ErrorException('Importing a minifigure from Rebrickable outside of a set is not supported')  # noqa: E501
+
+        # Insert the Rebrickable minifigure to the database
+        self.insert(
+            commit=False,
+            no_defer=True,
+            override_query=RebrickableMinifigure.insert_query
+        )
+
+        if not current_app.config['USE_REMOTE_IMAGES']:
+            RebrickableImage(
+                self.brickset,
+                minifigure=self,
+            ).download()
+
+    # Return a dict with common SQL parameters for a minifigure
+    def sql_parameters(self, /) -> dict[str, Any]:
+        parameters = super().sql_parameters()
+
+        # Supplement from the brickset
+        if self.brickset is not None and 'id' not in parameters:
+            parameters['id'] = self.brickset.fields.id
+
+        return parameters
+
+    # Self url
+    def url(self, /) -> str:
+        return url_for(
+            'minifigure.details',
+            figure=self.fields.figure,
+        )
+
+    # Compute the url for minifigure image
+    def url_for_image(self, /) -> str:
+        if not current_app.config['USE_REMOTE_IMAGES']:
+            if self.fields.image is None:
+                file = RebrickableImage.nil_minifigure_name()
+            else:
+                file = self.fields.figure
+
+            return RebrickableImage.static_url(file, 'MINIFIGURES_FOLDER')
+        else:
+            if self.fields.image is None:
+                return current_app.config['REBRICKABLE_IMAGE_NIL_MINIFIGURE']
+            else:
+                return self.fields.image
+
+    # Compute the url for the rebrickable page
+    def url_for_rebrickable(self, /) -> str:
+        if current_app.config['REBRICKABLE_LINKS']:
+            try:
+                return current_app.config['REBRICKABLE_LINK_MINIFIGURE_PATTERN'].format(  # noqa: E501
+                    number=self.fields.figure,
+                )
+            except Exception:
+                pass
+
+        return ''
+
+    # Normalize from Rebrickable
+    @staticmethod
+    def from_rebrickable(data: dict[str, Any], /, **_) -> dict[str, Any]:
+        # Extracting  number
+        number = int(str(data['set_num'])[5:])
+
+        return {
+            'figure': str(data['set_num']),
+            'number': int(number),
+            'name': str(data['set_name']),
+            'quantity': int(data['quantity']),
+            'image': data['set_img_url'],
+        }

bricktracker/rebrickable_minifigures.py

@@ -1,85 +0,0 @@
-import logging
-from typing import TYPE_CHECKING
-
-from flask import current_app
-
-from .minifigure import BrickMinifigure
-from .rebrickable import Rebrickable
-from .rebrickable_image import RebrickableImage
-from .rebrickable_parts import RebrickableParts
-if TYPE_CHECKING:
-    from .set import BrickSet
-    from .socket import BrickSocket
-
-logger = logging.getLogger(__name__)
-
-
-# Minifigures from Rebrickable
-class RebrickableMinifigures(object):
-    socket: 'BrickSocket'
-    brickset: 'BrickSet'
-
-    def __init__(self, socket: 'BrickSocket', brickset: 'BrickSet', /):
-        # Save the socket
-        self.socket = socket
-
-        # Save the objects
-        self.brickset = brickset
-
-    # Import the minifigures from Rebrickable
-    def download(self, /) -> None:
-        self.socket.auto_progress(
-            message='Set {number}: loading minifigures from Rebrickable'.format(  # noqa: E501
-                number=self.brickset.fields.set_num,
-            ),
-            increment_total=True,
-        )
-
-        logger.debug('rebrick.lego.get_set_minifigs("{set_num}")'.format(
-            set_num=self.brickset.fields.set_num,
-        ))
-
-        minifigures = Rebrickable[BrickMinifigure](
-            'get_set_minifigs',
-            self.brickset.fields.set_num,
-            BrickMinifigure,
-            socket=self.socket,
-            brickset=self.brickset,
-        ).list()
-
-        # Process each minifigure
-        total = len(minifigures)
-        for index, minifigure in enumerate(minifigures):
-            # Insert into the database
-            self.socket.auto_progress(
-                message='Set {number}: inserting minifigure {current}/{total} into database'.format(  # noqa: E501
-                    number=self.brickset.fields.set_num,
-                    current=index+1,
-                    total=total,
-                )
-            )
-
-            # Insert into database
-            minifigure.insert(commit=False)
-
-            # Grab the image
-            self.socket.progress(
-                message='Set {number}: downloading minifigure {current}/{total} image'.format(  # noqa: E501
-                    number=self.brickset.fields.set_num,
-                    current=index+1,
-                    total=total,
-                )
-            )
-
-            if not current_app.config['USE_REMOTE_IMAGES'].value:
-                RebrickableImage(
-                    self.brickset,
-                    minifigure=minifigure
-                ).download()
-
-            # Load the inventory
-            RebrickableParts(
-                self.socket,
-                self.brickset,
-                minifigure=minifigure,
-            ).download()

bricktracker/rebrickable_part.py

@@ -0,0 +1,232 @@
+import os
+from sqlite3 import Row
+from typing import Any, TYPE_CHECKING
+from urllib.parse import urlparse
+
+from flask import current_app, url_for
+
+from .exceptions import ErrorException
+from .rebrickable_image import RebrickableImage
+from .record import BrickRecord
+if TYPE_CHECKING:
+    from .minifigure import BrickMinifigure
+    from .set import BrickSet
+    from .socket import BrickSocket
+
+
+# A part from Rebrickable
+class RebrickablePart(BrickRecord):
+    socket: 'BrickSocket'
+    brickset: 'BrickSet | None'
+    minifigure: 'BrickMinifigure | None'
+
+    # Queries
+    select_query: str = 'rebrickable/part/select'
+    insert_query: str = 'rebrickable/part/insert'
+
+    def __init__(
+        self,
+        /,
+        *,
+        brickset: 'BrickSet | None' = None,
+        minifigure: 'BrickMinifigure | None' = None,
+        record: Row | dict[str, Any] | None = None
+    ):
+        super().__init__()
+
+        # Save the brickset
+        self.brickset = brickset
+
+        # Save the minifigure
+        self.minifigure = minifigure
+
+        # Ingest the record if it has one
+        if record is not None:
+            self.ingest(record)
+
+    # Insert the part from Rebrickable
+    def insert_rebrickable(self, /) -> None:
+        if self.brickset is None:
+            raise ErrorException('Importing a part from Rebrickable outside of a set is not supported')  # noqa: E501
+
+        # Insert the Rebrickable part to the database
+        self.insert(
+            commit=False,
+            no_defer=True,
+            override_query=RebrickablePart.insert_query
+        )
+
+        if not current_app.config['USE_REMOTE_IMAGES']:
+            RebrickableImage(
+                self.brickset,
+                minifigure=self.minifigure,
+                part=self,
+            ).download()
+
+    # Return a dict with common SQL parameters for a part
+    def sql_parameters(self, /) -> dict[str, Any]:
+        parameters = super().sql_parameters()
+
+        # Set id
+        if self.brickset is not None:
+            parameters['id'] = self.brickset.fields.id
+
+        # Use the minifigure number if present,
+        if self.minifigure is not None:
+            parameters['figure'] = self.minifigure.fields.figure
+        else:
+            parameters['figure'] = None
+
+        return parameters
+
+    # Self url
+    def url(self, /) -> str:
+        return url_for(
+            'part.details',
+            part=self.fields.part,
+            color=self.fields.color,
+        )
+
+    # Compute the url for the bricklink page
+    def url_for_bricklink(self, /) -> str:
+        if current_app.config['BRICKLINK_LINKS']:
+            try:
+                # Use BrickLink part number if available and not None/empty, otherwise fall back to Rebrickable part
+                bricklink_part = getattr(self.fields, 'bricklink_part_num', None)
+                part_param = bricklink_part if bricklink_part else self.fields.part
+
+                # Use BrickLink color ID if available and not None, otherwise fall back to Rebrickable color
+                bricklink_color = getattr(self.fields, 'bricklink_color_id', None)
+                color_param = bricklink_color if bricklink_color is not None else self.fields.color
+                print(f'BrickLink URL parameters: part={part_param}, color={color_param}')  # Debugging line, can be removed later
+                return current_app.config['BRICKLINK_LINK_PART_PATTERN'].format(  # noqa: E501
+                    part=part_param,
+                    color=color_param,
+                )
+            except Exception:
+                pass
+
+        return ''
+
+    # Compute the url for the part image
+    def url_for_image(self, /) -> str:
+        if not current_app.config['USE_REMOTE_IMAGES']:
+            if self.fields.image is None:
+                file = RebrickableImage.nil_name()
+            else:
+                file = self.fields.image_id
+
+            return RebrickableImage.static_url(file, 'PARTS_FOLDER')
+        else:
+            if self.fields.image is None:
+                return current_app.config['REBRICKABLE_IMAGE_NIL']
+            else:
+                return self.fields.image
+
+    # Compute the url for the original of the printed part
+    def url_for_print(self, /) -> str:
+        if self.fields.print is not None:
+            return url_for(
+                'part.details',
+                part=self.fields.print,
+                color=self.fields.color,
+            )
+        else:
+            return ''
+
+    # Compute the url for the rebrickable page
+    def url_for_rebrickable(self, /) -> str:
+        if current_app.config['REBRICKABLE_LINKS']:
+            try:
+                if self.fields.url is not None:
+                    # The URL does not contain color info...
+                    return '{url}{color}'.format(
+                        url=self.fields.url,
+                        color=self.fields.color
+                    )
+                else:
+                    return current_app.config['REBRICKABLE_LINK_PART_PATTERN'].format(  # noqa: E501
+                        part=self.fields.part,
+                        color=self.fields.color,
+                    )
+            except Exception:
+                pass
+
+        return ''
+
+    # Normalize from Rebrickable
+    @staticmethod
+    def from_rebrickable(
+        data: dict[str, Any],
+        /,
+        *,
+        brickset: 'BrickSet | None' = None,
+        minifigure: 'BrickMinifigure | None' = None,
+        **_,
+    ) -> dict[str, Any]:
+        record = {
+            'id': None,
+            'figure': None,
+            'part': data['part']['part_num'],
+            'color': data['color']['id'],
+            'spare': data['is_spare'],
+            'quantity': data['quantity'],
+            'rebrickable_inventory': data['id'],
+            'element': data['element_id'],
+            'color_id': data['color']['id'],
+            'color_name': data['color']['name'],
+            'color_rgb': data['color']['rgb'],
+            'color_transparent': data['color']['is_trans'],
+            'bricklink_color_id': None,
+            'bricklink_color_name': None,
+            'bricklink_part_num': None,
+            'name': data['part']['name'],
+            'category': data['part']['part_cat_id'],
+            'image': data['part']['part_img_url'],
+            'image_id': None,
+            'url': data['part']['part_url'],
+            'print': data['part']['print_of']
+        }
+
+        # Extract BrickLink color info if available in external_ids
+        if 'color' in data and 'external_ids' in data['color']:
+            external_ids = data['color']['external_ids']
+            if 'BrickLink' in external_ids and external_ids['BrickLink']:
+                bricklink_data = external_ids['BrickLink']
+
+                # Extract BrickLink color ID and name from the nested structure
+                if isinstance(bricklink_data, dict):
+                    if 'ext_ids' in bricklink_data and bricklink_data['ext_ids']:
+                        record['bricklink_color_id'] = bricklink_data['ext_ids'][0]
+
+                    if 'ext_descrs' in bricklink_data and bricklink_data['ext_descrs']:
+                        # ext_descrs is a list of lists, get the first description from the first list
+                        if len(bricklink_data['ext_descrs']) > 0 and len(bricklink_data['ext_descrs'][0]) > 0:
+                            record['bricklink_color_name'] = bricklink_data['ext_descrs'][0][0]
+
+        # Extract BrickLink part number if available
+        if 'part' in data and 'external_ids' in data['part']:
+            part_external_ids = data['part']['external_ids']
+            if 'BrickLink' in part_external_ids and part_external_ids['BrickLink']:
+                bricklink_parts = part_external_ids['BrickLink']
+                if isinstance(bricklink_parts, list) and len(bricklink_parts) > 0:
+                    record['bricklink_part_num'] = bricklink_parts[0]
+
+        if brickset is not None:
+            record['id'] = brickset.fields.id
+
+        if minifigure is not None:
+            record['figure'] = minifigure.fields.figure
+
+        # Extract the file name
+        if record['image'] is not None:
+            image_id, _ = os.path.splitext(
+                os.path.basename(
+                    urlparse(record['image']).path
+                )
+            )
+
+            if image_id is not None or image_id != '':
+                record['image_id'] = image_id
+
+        return record

bricktracker/rebrickable_parts.py

@@ -1,112 +0,0 @@
-import logging
-from typing import TYPE_CHECKING
-
-from flask import current_app
-
-from .part import BrickPart
-from .rebrickable import Rebrickable
-from .rebrickable_image import RebrickableImage
-if TYPE_CHECKING:
-    from .minifigure import BrickMinifigure
-    from .set import BrickSet
-    from .socket import BrickSocket
-
-logger = logging.getLogger(__name__)
-
-
-# A list of parts from Rebrickable
-class RebrickableParts(object):
-    socket: 'BrickSocket'
-    brickset: 'BrickSet'
-    minifigure: 'BrickMinifigure | None'
-
-    number: str
-    kind: str
-    method: str
-
-    def __init__(
-        self,
-        socket: 'BrickSocket',
-        brickset: 'BrickSet',
-        /,
-        minifigure: 'BrickMinifigure | None' = None,
-    ):
-        # Save the socket
-        self.socket = socket
-
-        # Save the objects
-        self.brickset = brickset
-        self.minifigure = minifigure
-
-        if self.minifigure is not None:
-            self.number = self.minifigure.fields.fig_num
-            self.kind = 'Minifigure'
-            self.method = 'get_minifig_elements'
-        else:
-            self.number = self.brickset.fields.set_num
-            self.kind = 'Set'
-            self.method = 'get_set_elements'
-
-    # Import the parts from Rebrickable
-    def download(self, /) -> None:
-        self.socket.auto_progress(
-            message='{kind} {number}: loading parts inventory from Rebrickable'.format(  # noqa: E501
-                kind=self.kind,
-                number=self.number,
-            ),
-            increment_total=True,
-        )
-
-        logger.debug('rebrick.lego.{method}("{number}")'.format(
-            method=self.method,
-            number=self.number,
-        ))
-
-        inventory = Rebrickable[BrickPart](
-            self.method,
-            self.number,
-            BrickPart,
-            socket=self.socket,
-            brickset=self.brickset,
-            minifigure=self.minifigure,
-        ).list()
-
-        # Process each part
-        total = len(inventory)
-        for index, part in enumerate(inventory):
-            # Skip spare parts
-            if (
-                current_app.config['SKIP_SPARE_PARTS'].value and
-                part.fields.is_spare
-            ):
-                continue
-
-            # Insert into the database
-            self.socket.auto_progress(
-                message='{kind} {number}: inserting part {current}/{total} into database'.format(  # noqa: E501
-                    kind=self.kind,
-                    number=self.number,
-                    current=index+1,
-                    total=total,
-                )
-            )
-
-            # Insert into database
-            part.insert(commit=False)
-
-            # Grab the image
-            self.socket.progress(
-                message='{kind} {number}: downloading part {current}/{total} image'.format(  # noqa: E501
-                    kind=self.kind,
-                    number=self.number,
-                    current=index+1,
-                    total=total,
-                )
-            )
-
-            if not current_app.config['USE_REMOTE_IMAGES'].value:
-                RebrickableImage(
-                    self.brickset,
-                    minifigure=self.minifigure,
-                    part=part,
-                ).download()

bricktracker/rebrickable_set.py

@@ -1,150 +1,132 @@
 import logging
+from sqlite3 import Row
 import traceback
-from typing import Any, TYPE_CHECKING
-from uuid import uuid4
+from typing import Any, Self, TYPE_CHECKING
 
-from flask import current_app
+from flask import current_app, url_for
 
 from .exceptions import ErrorException, NotFoundException
+from .instructions import BrickInstructions
+from .parser import parse_set
 from .rebrickable import Rebrickable
 from .rebrickable_image import RebrickableImage
-from .rebrickable_minifigures import RebrickableMinifigures
-from .rebrickable_parts import RebrickableParts
-from .set import BrickSet
-from .sql import BrickSQL
-from .wish import BrickWish
+from .record import BrickRecord
+from .theme_list import BrickThemeList
 if TYPE_CHECKING:
     from .socket import BrickSocket
+    from .theme import BrickTheme
 
 logger = logging.getLogger(__name__)
 
 
 # A set from Rebrickable
-class RebrickableSet(object):
-    socket: 'BrickSocket'
+class RebrickableSet(BrickRecord):
+    theme: 'BrickTheme'
+    instructions: list[BrickInstructions]
 
-    def __init__(self, socket: 'BrickSocket', /):
-        # Save the socket
-        self.socket = socket
+    # Flags
+    resolve_instructions: bool = True
 
-    # Import the set from Rebrickable
-    def download(self, data: dict[str, Any], /) -> None:
-        # Reset the progress
-        self.socket.progress_count = 0
-        self.socket.progress_total = 0
+    # Queries
+    select_query: str = 'rebrickable/set/select'
+    insert_query: str = 'rebrickable/set/insert'
 
-        # Load the set
-        brickset = self.load(data, from_download=True)
+    def __init__(
+        self,
+        /,
+        *,
+        record: Row | dict[str, Any] | None = None
+    ):
+        super().__init__()
 
-        # None brickset means loading failed
-        if brickset is None:
-            return
+        # Placeholders
+        self.instructions = []
 
-        try:
-            # Insert into the database
-            self.socket.auto_progress(
-                message='Set {number}: inserting into database'.format(
-                    number=brickset.fields.set_num
-                ),
-                increment_total=True,
-            )
+        # Ingest the record if it has one
+        if record is not None:
+            self.ingest(record)
 
-            # Assign a unique ID to the set
-            brickset.fields.u_id = str(uuid4())
+    # Insert the set from Rebrickable
+    def insert_rebrickable(self, /) -> None:
+        # Insert the Rebrickable set to the database
+        self.insert(
+            commit=False,
+            no_defer=True,
+            override_query=RebrickableSet.insert_query
+        )
 
-            # Insert into database
-            brickset.insert(commit=False)
+        if not current_app.config['USE_REMOTE_IMAGES']:
+            RebrickableImage(self).download()
 
-            if not current_app.config['USE_REMOTE_IMAGES'].value:
-                RebrickableImage(brickset).download()
+    # Ingest a set
+    def ingest(self, record: Row | dict[str, Any], /):
+        super().ingest(record)
 
-            # Load the inventory
-            RebrickableParts(self.socket, brickset).download()
+        # Resolve theme
+        if not hasattr(self.fields, 'theme_id'):
+            self.fields.theme_id = 0
 
-            # Load the minifigures
-            RebrickableMinifigures(self.socket, brickset).download()
+        self.theme = BrickThemeList().get(self.fields.theme_id)
 
-            # Commit the transaction to the database
-            self.socket.auto_progress(
-                message='Set {number}: writing to the database'.format(
-                    number=brickset.fields.set_num
-                ),
-                increment_total=True,
-            )
+        # Resolve instructions
+        if self.resolve_instructions:
+            # Not idead, avoiding cyclic import
+            from .instructions_list import BrickInstructionsList
 
-            BrickSQL().commit()
-
-            # Info
-            logger.info('Set {number}: imported (id: {id})'.format(
-                number=brickset.fields.set_num,
-                id=brickset.fields.u_id,
-            ))
-
-            # Complete
-            self.socket.complete(
-                message='Set {number}: imported (<a href="{url}">Go to the set</a>)'.format(  # noqa: E501
-                    number=brickset.fields.set_num,
-                    url=brickset.url()
-                ),
-                download=True
-            )
-
-        except Exception as e:
-            self.socket.fail(
-                message='Error while importing set {number}: {error}'.format(
-                    number=brickset.fields.set_num,
-                    error=e,
+            if self.fields.set is not None:
+                self.instructions = BrickInstructionsList().get(
+                    self.fields.set
                 )
-            )
-
-            logger.debug(traceback.format_exc())
 
     # Load the set from Rebrickable
     def load(
         self,
+        socket: 'BrickSocket',
         data: dict[str, Any],
         /,
+        *,
         from_download=False,
-    ) -> BrickSet | None:
+    ) -> bool:
         # Reset the progress
-        self.socket.progress_count = 0
-        self.socket.progress_total = 2
+        socket.progress_count = 0
+        socket.progress_total = 2
 
         try:
-            self.socket.auto_progress(message='Parsing set number')
-            set_num = RebrickableSet.parse_number(str(data['set_num']))
+            socket.auto_progress(message='Parsing set number')
+            set = parse_set(str(data['set']))
 
-            self.socket.auto_progress(
-                message='Set {num}: loading from Rebrickable'.format(
-                    num=set_num,
+            socket.auto_progress(
+                message='Set {set}: loading from Rebrickable'.format(
+                    set=set,
                 ),
             )
 
-            logger.debug('rebrick.lego.get_set("{set_num}")'.format(
-                set_num=set_num,
+            logger.debug('rebrick.lego.get_set("{set}")'.format(
+                set=set,
             ))
 
-            brickset = Rebrickable[BrickSet](
+            Rebrickable[RebrickableSet](
                 'get_set',
-                set_num,
-                BrickSet,
+                set,
+                RebrickableSet,
+                instance=self,
             ).get()
 
-            short = brickset.short()
-            short['download'] = from_download
-
-            self.socket.emit('SET_LOADED', short)
+            socket.emit('SET_LOADED', self.short(
+                from_download=from_download
+            ))
 
             if not from_download:
-                self.socket.complete(
-                    message='Set {num}: loaded from Rebrickable'.format(
-                        num=brickset.fields.set_num
+                socket.complete(
+                    message='Set {set}: loaded from Rebrickable'.format(
+                        set=self.fields.set
                     )
                 )
 
-            return brickset
+            return True
+
         except Exception as e:
-            self.socket.fail(
+            socket.fail(
                 message='Could not load the set from Rebrickable: {error}. Data: {data}'.format(  # noqa: E501
                     error=str(e),
                     data=data,
@@ -154,61 +136,68 @@ class RebrickableSet(object):
             if not isinstance(e, (NotFoundException, ErrorException)):
                 logger.debug(traceback.format_exc())
 
-        return None
+        return False
 
-    # Make sense of the number from the data
+    # Select a specific set (with a set)
+    def select_specific(self, set: str, /) -> Self:
+        # Save the parameters to the fields
+        self.fields.set = set
+
+        # Load from database
+        if not self.select():
+            raise NotFoundException(
+                'Set with set {set} was not found in the database'.format(
+                    set=self.fields.set,
+                ),
+            )
+
+        return self
+
+    # Return a short form of the Rebrickable set
+    def short(self, /, *, from_download: bool = False) -> dict[str, Any]:
+        return {
+            'download': from_download,
+            'image': self.fields.image,
+            'name': self.fields.name,
+            'set': self.fields.set,
+        }
+
+    # Compute the url for the set image
+    def url_for_image(self, /) -> str:
+        if not current_app.config['USE_REMOTE_IMAGES']:
+            return RebrickableImage.static_url(
+                self.fields.set,
+                'SETS_FOLDER'
+            )
+        else:
+            return self.fields.image
+
+    # Compute the url for the rebrickable page
+    def url_for_rebrickable(self, /) -> str:
+        if current_app.config['REBRICKABLE_LINKS']:
+            return self.fields.url
+
+        return ''
+
+    # Compute the url for the refresh button
+    def url_for_refresh(self, /) -> str:
+        return url_for('set.refresh', set=self.fields.set)
+
+    # Normalize from Rebrickable
     @staticmethod
-    def parse_number(set_num: str, /) -> str:
-        number, _, version = set_num.partition('-')
+    def from_rebrickable(data: dict[str, Any], /, **_) -> dict[str, Any]:
+        # Extracting version and number
+        number, _, version = str(data['set_num']).partition('-')
 
-        # Making sure both are integers
-        if version == '':
-            version = 1
-
-        try:
-            number = int(number)
-        except Exception:
-            raise ErrorException('Number "{number}" is not a number'.format(
-                number=number,
-            ))
-
-        try:
-            version = int(version)
-        except Exception:
-            raise ErrorException('Version "{version}" is not a number'.format(
-                version=version,
-            ))
-
-        # Make sure both are positive
-        if number < 0:
-            raise ErrorException('Number "{number}" should be positive'.format(
-                number=number,
-            ))
-
-        if version < 0:
-            raise ErrorException('Version "{version}" should be positive'.format(  # noqa: E501
-                version=version,
-            ))
-
-        return '{number}-{version}'.format(number=number, version=version)
-
-    # Wish from Rebrickable
-    # Redefine this one outside of the socket logic
-    @staticmethod
-    def wish(set_num: str) -> None:
-        set_num = RebrickableSet.parse_number(set_num)
-        logger.debug('rebrick.lego.get_set("{set_num}")'.format(
-            set_num=set_num,
-        ))
-
-        brickwish = Rebrickable[BrickWish](
-            'get_set',
-            set_num,
-            BrickWish,
-        ).get()
-
-        # Insert into database
-        brickwish.insert()
-
-        if not current_app.config['USE_REMOTE_IMAGES'].value:
-            RebrickableImage(brickwish).download()
+        return {
+            'set': str(data['set_num']),
+            'number': int(number),
+            'version': int(version),
+            'name': str(data['name']),
+            'year': int(data['year']),
+            'theme_id': int(data['theme_id']),
+            'number_of_parts': int(data['num_parts']),
+            'image': str(data['set_img_url']),
+            'url': str(data['set_url']),
+            'last_modified': str(data['last_modified_dt']),
+        }

bricktracker/rebrickable_set_list.py

@@ -0,0 +1,34 @@
+from typing import Self
+
+from .rebrickable_set import RebrickableSet
+from .record_list import BrickRecordList
+
+
+# All the rebrickable sets from the database
+class RebrickableSetList(BrickRecordList[RebrickableSet]):
+
+    # Queries
+    select_query: str = 'rebrickable/set/list'
+    refresh_query: str = 'rebrickable/set/need_refresh'
+
+    # All the sets
+    def all(self, /) -> Self:
+        # Load the sets from the database
+        for record in self.select():
+            rebrickable_set = RebrickableSet(record=record)
+
+            self.records.append(rebrickable_set)
+
+        return self
+
+    # Sets needing refresh
+    def need_refresh(self, /) -> Self:
+        # Load the sets from the database
+        for record in self.select(
+            override_query=self.refresh_query
+        ):
+            rebrickable_set = RebrickableSet(record=record)
+
+            self.records.append(rebrickable_set)
+
+        return self

bricktracker/record.py

@@ -24,12 +24,24 @@ class BrickRecord(object):
 
     # Insert into the database
     # If we do not commit immediately, we defer the execute() call
-    def insert(self, /, commit=True) -> None:
+    def insert(
+        self,
+        /,
+        *,
+        commit=True,
+        no_defer=False,
+        override_query: str | None = None
+    ) -> None:
+        if override_query:
+            query = override_query
+        else:
+            query = self.insert_query
+
         database = BrickSQL()
-        rows, q = database.execute(
-            self.insert_query,
+        database.execute(
+            query,
             parameters=self.sql_parameters(),
-            defer=not commit,
+            defer=not commit and not no_defer,
         )
 
         if commit:
@@ -40,17 +52,33 @@ class BrickRecord(object):
         return self.fields.__dict__.items()
 
     # Get from the database using the query
-    def select(self, /, override_query: str | None = None) -> Row | None:
+    def select(
+        self,
+        /,
+        *,
+        override_query: str | None = None,
+        **context: Any
+    ) -> bool:
         if override_query:
             query = override_query
         else:
             query = self.select_query
 
-        return BrickSQL().fetchone(
+        record = BrickSQL().fetchone(
             query,
-            parameters=self.sql_parameters()
+            parameters=self.sql_parameters(),
+            **context
         )
 
+        # Ingest the record
+        if record is not None:
+            self.ingest(record)
+
+            return True
+
+        else:
+            return False
+
     # Generic SQL parameters from fields
     def sql_parameters(self, /) -> dict[str, Any]:
         parameters: dict[str, Any] = {}

bricktracker/record_list.py

@@ -6,10 +6,30 @@ from .sql import BrickSQL
 if TYPE_CHECKING:
     from .minifigure import BrickMinifigure
     from .part import BrickPart
+    from .rebrickable_set import RebrickableSet
     from .set import BrickSet
+    from .set_owner import BrickSetOwner
+    from .set_purchase_location import BrickSetPurchaseLocation
+    from .set_status import BrickSetStatus
+    from .set_storage import BrickSetStorage
+    from .set_tag import BrickSetTag
     from .wish import BrickWish
+    from .wish_owner import BrickWishOwner
 
-T = TypeVar('T', 'BrickSet', 'BrickPart', 'BrickMinifigure', 'BrickWish')
+T = TypeVar(
+    'T',
+    'BrickMinifigure',
+    'BrickPart',
+    'BrickSet',
+    'BrickSetOwner',
+    'BrickSetPurchaseLocation',
+    'BrickSetStatus',
+    'BrickSetStorage',
+    'BrickSetTag',
+    'BrickWish',
+    'BrickWishOwner',
+    'RebrickableSet'
+)
 
 
 # SQLite records
@@ -32,9 +52,11 @@ class BrickRecordList(Generic[T]):
     def select(
         self,
         /,
+        *,
         override_query: str | None = None,
         order: str | None = None,
         limit: int | None = None,
+        **context: Any,
     ) -> list[Row]:
         # Select the query
         if override_query:
@@ -47,6 +69,7 @@ class BrickRecordList(Generic[T]):
             parameters=self.sql_parameters(),
             order=order,
             limit=limit,
+            **context
         )
 
     # Generic SQL parameters from fields
@@ -62,6 +85,6 @@ class BrickRecordList(Generic[T]):
         for record in self.records:
             yield record
 
-    # Make the sets measurable
+    # Make the list measurable
     def __len__(self, /) -> int:
         return len(self.records)

bricktracker/reload.py

@@ -0,0 +1,43 @@
+from .instructions_list import BrickInstructionsList
+from .retired_list import BrickRetiredList
+from .set_owner_list import BrickSetOwnerList
+from .set_purchase_location_list import BrickSetPurchaseLocationList
+from .set_status_list import BrickSetStatusList
+from .set_storage_list import BrickSetStorageList
+from .set_tag_list import BrickSetTagList
+from .theme_list import BrickThemeList
+from .wish_owner_list import BrickWishOwnerList
+
+
+# Reload everything related to a database after an operation
+def reload() -> None:
+    # Failsafe
+    try:
+        # Reload the instructions
+        BrickInstructionsList(force=True)
+
+        # Reload the set owners
+        BrickSetOwnerList.new(force=True)
+
+        # Reload the set purchase locations
+        BrickSetPurchaseLocationList.new(force=True)
+
+        # Reload the set statuses
+        BrickSetStatusList.new(force=True)
+
+        # Reload the set storages
+        BrickSetStorageList.new(force=True)
+
+        # Reload the set tags
+        BrickSetTagList.new(force=True)
+
+        # Reload retired sets
+        BrickRetiredList(force=True)
+
+        # Reload themes
+        BrickThemeList(force=True)
+
+        # Reload the wish owners
+        BrickWishOwnerList.new(force=True)
+    except Exception:
+        pass

bricktracker/retired_list.py

@@ -22,7 +22,7 @@ class BrickRetiredList(object):
     size: int | None
     exception: Exception | None
 
-    def __init__(self, /, force: bool = False):
+    def __init__(self, /, *, force: bool = False):
         # Load sets only if there is none already loaded
         retired = getattr(self, 'retired', None)
 
@@ -33,7 +33,7 @@ class BrickRetiredList(object):
 
             # Try to read the themes from a CSV file
             try:
-                with open(current_app.config['RETIRED_SETS_PATH'].value, newline='') as themes_file:  # noqa: E501
+                with open(current_app.config['RETIRED_SETS_PATH'], newline='') as themes_file:  # noqa: E501
                     themes_reader = csv.reader(themes_file)
 
                     # Ignore the header
@@ -44,7 +44,7 @@ class BrickRetiredList(object):
                         BrickRetiredList.retired[retired.number] = retired
 
                 # File stats
-                stat = os.stat(current_app.config['RETIRED_SETS_PATH'].value)
+                stat = os.stat(current_app.config['RETIRED_SETS_PATH'])
                 BrickRetiredList.size = stat.st_size
                 BrickRetiredList.mtime = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc)  # noqa: E501
 
@@ -79,7 +79,7 @@ class BrickRetiredList(object):
     def human_time(self) -> str:
         if self.mtime is not None:
             return self.mtime.astimezone(g.timezone).strftime(
-                current_app.config['FILE_DATETIME_FORMAT'].value
+                current_app.config['FILE_DATETIME_FORMAT']
             )
         else:
             return ''
@@ -88,7 +88,7 @@ class BrickRetiredList(object):
     @staticmethod
     def update() -> None:
         response = requests.get(
-            current_app.config['RETIRED_SETS_FILE_URL'].value,
+            current_app.config['RETIRED_SETS_FILE_URL'],
             stream=True,
         )
 
@@ -99,7 +99,7 @@ class BrickRetiredList(object):
 
         content = gzip.GzipFile(fileobj=response.raw)
 
-        with open(current_app.config['RETIRED_SETS_PATH'].value, 'wb') as f:
+        with open(current_app.config['RETIRED_SETS_PATH'], 'wb') as f:
             copyfileobj(content, f)
 
     logger.info('Retired sets list updated')

bricktracker/set.py

@@ -1,218 +1,341 @@
-from sqlite3 import Row
-from typing import Any, Self
+from datetime import datetime
+import logging
+import traceback
+from typing import Any, Self, TYPE_CHECKING
+from uuid import uuid4
 
 from flask import current_app, url_for
 
-from .exceptions import DatabaseException, NotFoundException
-from .instructions import BrickInstructions
-from .instructions_list import BrickInstructionsList
+from .exceptions import NotFoundException, DatabaseException, ErrorException
 from .minifigure_list import BrickMinifigureList
 from .part_list import BrickPartList
-from .rebrickable_image import RebrickableImage
-from .record import BrickRecord
+from .rebrickable_set import RebrickableSet
+from .set_owner_list import BrickSetOwnerList
+from .set_purchase_location_list import BrickSetPurchaseLocationList
+from .set_status_list import BrickSetStatusList
+from .set_storage_list import BrickSetStorageList
+from .set_tag_list import BrickSetTagList
 from .sql import BrickSQL
-from .theme_list import BrickThemeList
+if TYPE_CHECKING:
+    from .socket import BrickSocket
+
+logger = logging.getLogger(__name__)
 
 
 # Lego brick set
-class BrickSet(BrickRecord):
-    instructions: list[BrickInstructions]
-    theme_name: str
-
+class BrickSet(RebrickableSet):
     # Queries
-    select_query: str = 'set/select'
+    select_query: str = 'set/select/full'
+    light_query: str = 'set/select/light'
     insert_query: str = 'set/insert'
-
-    def __init__(
-        self,
-        /,
-        record: Row | dict[str, Any] | None = None,
-    ):
-        super().__init__()
-
-        # Placeholders
-        self.theme_name = ''
-        self.instructions = []
-
-        # Ingest the record if it has one
-        if record is not None:
-            self.ingest(record)
-
-            # Resolve the theme
-            self.resolve_theme()
-
-            # Check for the instructions
-            self.resolve_instructions()
+    update_purchase_date_query: str = 'set/update/purchase_date'
+    update_purchase_price_query: str = 'set/update/purchase_price'
 
     # Delete a set
     def delete(self, /) -> None:
-        database = BrickSQL()
-        parameters = self.sql_parameters()
+        BrickSQL().executescript(
+            'set/delete/set',
+            id=self.fields.id
+        )
 
-        # Delete the set
-        database.execute('set/delete/set', parameters=parameters)
+    # Import a set into the database
+    def download(self, socket: 'BrickSocket', data: dict[str, Any], /) -> bool:
+        # Load the set
+        if not self.load(socket, data, from_download=True):
+            return False
 
-        # Delete the minifigures
-        database.execute(
-            'minifigure/delete/all_from_set', parameters=parameters)
+        try:
+            # Insert into the database
+            socket.auto_progress(
+                message='Set {set}: inserting into database'.format(
+                    set=self.fields.set
+                ),
+                increment_total=True,
+            )
 
-        # Delete the parts
-        database.execute(
-            'part/delete/all_from_set', parameters=parameters)
+            # Grabbing the refresh flag
+            refresh: bool = bool(data.get('refresh', False))
 
-        # Delete missing parts
-        database.execute('missing/delete/all_from_set', parameters=parameters)
+            # Generate an UUID for self
+            self.fields.id = str(uuid4())
 
-        # Commit to the database
-        database.commit()
+            if not refresh:
+                # Save the storage
+                storage = BrickSetStorageList.get(
+                    data.get('storage', ''),
+                    allow_none=True
+                )
+                self.fields.storage = storage.fields.id
+
+                # Save the purchase location
+                purchase_location = BrickSetPurchaseLocationList.get(
+                    data.get('purchase_location', ''),
+                    allow_none=True
+                )
+                self.fields.purchase_location = purchase_location.fields.id
+
+                # Insert into database
+                self.insert(commit=False)
+
+                # Save the owners
+                owners: list[str] = list(data.get('owners', []))
+
+                for id in owners:
+                    owner = BrickSetOwnerList.get(id)
+                    owner.update_set_state(self, state=True)
+
+                # Save the tags
+                tags: list[str] = list(data.get('tags', []))
+
+                for id in tags:
+                    tag = BrickSetTagList.get(id)
+                    tag.update_set_state(self, state=True)
+
+            # Insert the rebrickable set into database
+            self.insert_rebrickable()
+
+            # Load the inventory
+            if not BrickPartList.download(socket, self, refresh=refresh):
+                return False
+
+            # Load the minifigures
+            if not BrickMinifigureList.download(socket, self, refresh=refresh):
+                return False
+
+            # Commit the transaction to the database
+            socket.auto_progress(
+                message='Set {set}: writing to the database'.format(
+                    set=self.fields.set
+                ),
+                increment_total=True,
+            )
+
+            BrickSQL().commit()
+
+            if refresh:
+                # Info
+                logger.info('Set {set}: imported (id: {id})'.format(
+                    set=self.fields.set,
+                    id=self.fields.id,
+                ))
+
+                # Complete
+                socket.complete(
+                    message='Set {set}: refreshed'.format(  # noqa: E501
+                        set=self.fields.set,
+                    ),
+                    download=True
+                )
+            else:
+                # Info
+                logger.info('Set {set}: refreshed'.format(
+                    set=self.fields.set,
+                ))
+
+                # Complete
+                socket.complete(
+                    message='Set {set}: imported (<a href="{url}">Go to the set</a>)'.format(  # noqa: E501
+                        set=self.fields.set,
+                        url=self.url()
+                    ),
+                    download=True
+                )
+
+        except Exception as e:
+            socket.fail(
+                message='Error while importing set {set}: {error}'.format(
+                    set=self.fields.set,
+                    error=e,
+                )
+            )
+
+            logger.debug(traceback.format_exc())
+
+            return False
+
+        return True
+
+    # Purchase date
+    def purchase_date(self, /, *, standard: bool = False) -> str:
+        if self.fields.purchase_date is not None:
+            time = datetime.fromtimestamp(self.fields.purchase_date)
+
+            if standard:
+                return time.strftime('%Y/%m/%d')
+            else:
+                return time.strftime(
+                    current_app.config['PURCHASE_DATE_FORMAT']
+                )
+        else:
+            return ''
+
+    # Purchase price with currency
+    def purchase_price(self, /) -> str:
+        if self.fields.purchase_price is not None:
+            return '{price}{currency}'.format(
+                price=self.fields.purchase_price,
+                currency=current_app.config['PURCHASE_CURRENCY']
+            )
+        else:
+            return ''
 
     # Minifigures
     def minifigures(self, /) -> BrickMinifigureList:
-        return BrickMinifigureList().load(self)
+        return BrickMinifigureList().from_set(self)
 
     # Parts
     def parts(self, /) -> BrickPartList:
-        return BrickPartList().load(self)
+        return BrickPartList().list_specific(self)
 
-    # Add instructions to the set
-    def resolve_instructions(self, /) -> None:
-        if self.fields.set_num is not None:
-            self.instructions = BrickInstructionsList().get(
-                self.fields.set_num
-            )
-
-    # Add a theme to the set
-    def resolve_theme(self, /) -> None:
-        try:
-            id = self.fields.theme_id
-        except Exception:
-            id = 0
-
-        theme = BrickThemeList().get(id)
-        self.theme_name = theme.name
-
-    # Return a short form of the set
-    def short(self, /) -> dict[str, Any]:
-        return {
-            'name': self.fields.name,
-            'set_img_url': self.fields.set_img_url,
-            'set_num': self.fields.set_num,
-        }
-
-    # Select a specific part (with a set and an id)
-    def select_specific(self, u_id: str, /) -> Self:
+    # Select a light set (with an id)
+    def select_light(self, id: str, /) -> Self:
         # Save the parameters to the fields
-        self.fields.u_id = u_id
+        self.fields.id = id
 
         # Load from database
-        record = self.select()
-
-        if record is None:
+        if not self.select(override_query=self.light_query):
             raise NotFoundException(
                 'Set with ID {id} was not found in the database'.format(
-                    id=self.fields.u_id,
+                    id=self.fields.id,
                 ),
             )
 
-        # Ingest the record
-        self.ingest(record)
+        return self
 
-        # Resolve the theme
-        self.resolve_theme()
+    # Select a specific set (with an id)
+    def select_specific(self, id: str, /) -> Self:
+        # Save the parameters to the fields
+        self.fields.id = id
 
-        # Check for the instructions
-        self.resolve_instructions()
+        # Load from database
+        if not self.select(
+            owners=BrickSetOwnerList.as_columns(),
+            statuses=BrickSetStatusList.as_columns(all=True),
+            tags=BrickSetTagList.as_columns(),
+        ):
+            raise NotFoundException(
+                'Set with ID {id} was not found in the database'.format(
+                    id=self.fields.id,
+                ),
+            )
 
         return self
 
-    # Update a checked state
-    def update_checked(self, name: str, status: bool, /) -> None:
-        parameters = self.sql_parameters()
-        parameters['status'] = status
+    # Update the purchase date
+    def update_purchase_date(self, json: Any | None, /) -> Any:
+        value = json.get('value', None)  # type: ignore
+
+        try:
+            if value == '':
+                value = None
+
+            if value is not None:
+                value = datetime.strptime(value, '%Y/%m/%d').timestamp()
+        except Exception:
+            raise ErrorException('{value} is not a date'.format(
+                value=value,
+            ))
+
+        self.fields.purchase_date = value
 
-        # Update the checked status
         rows, _ = BrickSQL().execute_and_commit(
-            'set/update_checked',
-            parameters=parameters,
-            name=name,
+            self.update_purchase_date_query,
+            parameters=self.sql_parameters()
         )
 
         if rows != 1:
-            raise DatabaseException('Could not update the status {status} for set {number}'.format(  # noqa: E501
-                status=name,
-                number=self.fields.set_num,
+            raise DatabaseException('Could not update the purchase date for set {set} ({id})'.format(  # noqa: E501
+                set=self.fields.set,
+                id=self.fields.id,
             ))
 
+        # Info
+        logger.info('Purchase date changed to "{value}" for set {set} ({id})'.format(  # noqa: E501
+            value=value,
+            set=self.fields.set,
+            id=self.fields.id,
+        ))
+
+        return value
+
+    # Update the purchase price
+    def update_purchase_price(self, json: Any | None, /) -> Any:
+        value = json.get('value', None)  # type: ignore
+
+        try:
+            if value == '':
+                value = None
+
+            if value is not None:
+                value = float(value)
+        except Exception:
+            raise ErrorException('{value} is not a number or empty'.format(
+                value=value,
+            ))
+
+        self.fields.purchase_price = value
+
+        rows, _ = BrickSQL().execute_and_commit(
+            self.update_purchase_price_query,
+            parameters=self.sql_parameters()
+        )
+
+        if rows != 1:
+            raise DatabaseException('Could not update the purchase price for set {set} ({id})'.format(  # noqa: E501
+                set=self.fields.set,
+                id=self.fields.id,
+            ))
+
+        # Info
+        logger.info('Purchase price changed to "{value}" for set {set} ({id})'.format(  # noqa: E501
+            value=value,
+            set=self.fields.set,
+            id=self.fields.id,
+        ))
+
+        return value
+
     # Self url
     def url(self, /) -> str:
-        return url_for('set.details', id=self.fields.u_id)
+        return url_for('set.details', id=self.fields.id)
 
     # Deletion url
     def url_for_delete(self, /) -> str:
-        return url_for('set.delete', id=self.fields.u_id)
+        return url_for('set.delete', id=self.fields.id)
 
     # Actual deletion url
     def url_for_do_delete(self, /) -> str:
-        return url_for('set.do_delete', id=self.fields.u_id)
-
-    # Compute the url for the set image
-    def url_for_image(self, /) -> str:
-        if not current_app.config['USE_REMOTE_IMAGES'].value:
-            return RebrickableImage.static_url(
-                self.fields.set_num,
-                'SETS_FOLDER'
-            )
-        else:
-            return self.fields.set_img_url
+        return url_for('set.do_delete', id=self.fields.id)
 
     # Compute the url for the set instructions
     def url_for_instructions(self, /) -> str:
-        if len(self.instructions):
+        if (
+            not current_app.config['HIDE_SET_INSTRUCTIONS'] and
+            len(self.instructions)
+        ):
             return url_for(
                 'set.details',
-                id=self.fields.u_id,
+                id=self.fields.id,
                 open_instructions=True
             )
         else:
             return ''
 
-    # Check minifigure collected url
-    def url_for_minifigures_collected(self, /) -> str:
-        return url_for('set.minifigures_collected', id=self.fields.u_id)
+    # Compute the url for the refresh button
+    def url_for_refresh(self, /) -> str:
+        return url_for('set.refresh', id=self.fields.id)
 
-    # Compute the url for the rebrickable page
-    def url_for_rebrickable(self, /) -> str:
-        if current_app.config['REBRICKABLE_LINKS'].value:
-            try:
-                return current_app.config['REBRICKABLE_LINK_SET_PATTERN'].value.format(  # noqa: E501
-                    number=self.fields.set_num.lower(),
-                )
-            except Exception:
-                pass
+    # Compute the url for the set storage
+    def url_for_storage(self, /) -> str:
+        if self.fields.storage is not None:
+            return url_for('storage.details', id=self.fields.storage)
+        else:
+            return ''
 
-        return ''
+    # Update purchase date url
+    def url_for_purchase_date(self, /) -> str:
+        return url_for('set.update_purchase_date', id=self.fields.id)
 
-    # Check set checked url
-    def url_for_set_checked(self, /) -> str:
-        return url_for('set.set_checked', id=self.fields.u_id)
-
-    # Check set collected url
-    def url_for_set_collected(self, /) -> str:
-        return url_for('set.set_collected', id=self.fields.u_id)
-
-    # Normalize from Rebrickable
-    @staticmethod
-    def from_rebrickable(data: dict[str, Any], /, **_) -> dict[str, Any]:
-        return {
-            'set_num': data['set_num'],
-            'name': data['name'],
-            'year': data['year'],
-            'theme_id': data['theme_id'],
-            'num_parts': data['num_parts'],
-            'set_img_url': data['set_img_url'],
-            'set_url': data['set_url'],
-            'last_modified_dt': data['last_modified_dt'],
-            'mini_col': False,
-            'set_col': False,
-            'set_check': False,
-        }
+    # Update purchase price url
+    def url_for_purchase_price(self, /) -> str:
+        return url_for('set.update_purchase_price', id=self.fields.id)

bricktracker/set_list.py

@@ -1,8 +1,17 @@
-from typing import Self
+from typing import Any, Self, Union
 
 from flask import current_app
 
 from .record_list import BrickRecordList
+from .set_owner import BrickSetOwner
+from .set_owner_list import BrickSetOwnerList
+from .set_purchase_location import BrickSetPurchaseLocation
+from .set_purchase_location_list import BrickSetPurchaseLocationList
+from .set_status_list import BrickSetStatusList
+from .set_storage import BrickSetStorage
+from .set_storage_list import BrickSetStorageList
+from .set_tag import BrickSetTag
+from .set_tag_list import BrickSetTagList
 from .set import BrickSet
 
 
@@ -12,12 +21,16 @@ class BrickSetList(BrickRecordList[BrickSet]):
     order: str
 
     # Queries
+    damaged_minifigure_query: str = 'set/list/damaged_minifigure'
+    damaged_part_query: str = 'set/list/damaged_part'
     generic_query: str = 'set/list/generic'
+    light_query: str = 'set/list/light'
     missing_minifigure_query: str = 'set/list/missing_minifigure'
     missing_part_query: str = 'set/list/missing_part'
     select_query: str = 'set/list/all'
     using_minifigure_query: str = 'set/list/using_minifigure'
     using_part_query: str = 'set/list/using_part'
+    using_storage_query: str = 'set/list/using_storage'
 
     def __init__(self, /):
         super().__init__()
@@ -26,136 +39,154 @@ class BrickSetList(BrickRecordList[BrickSet]):
         self.themes = []
 
         # Store the order for this list
-        self.order = current_app.config['SETS_DEFAULT_ORDER'].value
+        self.order = current_app.config['SETS_DEFAULT_ORDER']
 
     # All the sets
     def all(self, /) -> Self:
-        themes = set()
-
         # Load the sets from the database
-        for record in self.select(order=self.order):
-            brickset = BrickSet(record=record)
-
-            self.records.append(brickset)
-            themes.add(brickset.theme_name)
-
-        # Convert the set into a list and sort it
-        self.themes = list(themes)
-        self.themes.sort()
+        self.list(do_theme=True)
 
         return self
 
-    # A generic list of the different sets
-    def generic(self, /) -> Self:
-        for record in self.select(
-            override_query=self.generic_query,
-            order=self.order
-        ):
-            brickset = BrickSet(record=record)
+    # Sets with a minifigure part damaged
+    def damaged_minifigure(self, figure: str, /) -> Self:
+        # Save the parameters to the fields
+        self.fields.figure = figure
 
-            self.records.append(brickset)
+        # Load the sets from the database
+        self.list(override_query=self.damaged_minifigure_query)
+
+        return self
+
+    # Sets with a part damaged
+    def damaged_part(self, part: str, color: int, /) -> Self:
+        # Save the parameters to the fields
+        self.fields.part = part
+        self.fields.color = color
+
+        # Load the sets from the database
+        self.list(override_query=self.damaged_part_query)
 
         return self
 
     # Last added sets
-    def last(self, /, limit: int = 6) -> Self:
+    def last(self, /, *, limit: int = 6) -> Self:
         # Randomize
-        if current_app.config['RANDOM'].value:
+        if current_app.config['RANDOM']:
             order = 'RANDOM()'
         else:
-            order = 'sets.rowid DESC'
+            order = '"bricktracker_sets"."rowid" DESC'
 
-        for record in self.select(order=order, limit=limit):
-            brickset = BrickSet(record=record)
-
-            self.records.append(brickset)
+        self.list(order=order, limit=limit)
 
         return self
 
-    # Sets missing a minifigure
-    def missing_minifigure(
+    # Base set list
+    def list(
         self,
-        fig_num: str,
         /,
-    ) -> Self:
-        # Save the parameters to the fields
-        self.fields.fig_num = fig_num
+        *,
+        override_query: str | None = None,
+        order: str | None = None,
+        limit: int | None = None,
+        do_theme: bool = False,
+        **context: Any,
+    ) -> None:
+        themes = set()
+
+        if order is None:
+            order = self.order
 
         # Load the sets from the database
-        for record in self.select(
-            override_query=self.missing_minifigure_query,
-            order=self.order
+        for record in super().select(
+            override_query=override_query,
+            order=order,
+            limit=limit,
+            owners=BrickSetOwnerList.as_columns(),
+            statuses=BrickSetStatusList.as_columns(),
+            tags=BrickSetTagList.as_columns(),
         ):
             brickset = BrickSet(record=record)
 
             self.records.append(brickset)
+            if do_theme:
+                themes.add(brickset.theme.name)
+
+        # Convert the set into a list and sort it
+        if do_theme:
+            self.themes = list(themes)
+            self.themes.sort()
+
+    # Sets missing a minifigure part
+    def missing_minifigure(self, figure: str, /) -> Self:
+        # Save the parameters to the fields
+        self.fields.figure = figure
+
+        # Load the sets from the database
+        self.list(override_query=self.missing_minifigure_query)
 
         return self
 
     # Sets missing a part
-    def missing_part(
-        self,
-        part_num: str,
-        color_id: int,
-        /,
-        element_id: int | None = None,
-    ) -> Self:
+    def missing_part(self, part: str, color: int, /) -> Self:
         # Save the parameters to the fields
-        self.fields.part_num = part_num
-        self.fields.color_id = color_id
-        self.fields.element_id = element_id
+        self.fields.part = part
+        self.fields.color = color
 
         # Load the sets from the database
-        for record in self.select(
-            override_query=self.missing_part_query,
-            order=self.order
-        ):
-            brickset = BrickSet(record=record)
-
-            self.records.append(brickset)
+        self.list(override_query=self.missing_part_query)
 
         return self
 
     # Sets using a minifigure
-    def using_minifigure(
-        self,
-        fig_num: str,
-        /,
-    ) -> Self:
+    def using_minifigure(self, figure: str, /) -> Self:
         # Save the parameters to the fields
-        self.fields.fig_num = fig_num
+        self.fields.figure = figure
 
         # Load the sets from the database
-        for record in self.select(
-            override_query=self.using_minifigure_query,
-            order=self.order
-        ):
-            brickset = BrickSet(record=record)
-
-            self.records.append(brickset)
+        self.list(override_query=self.using_minifigure_query)
 
         return self
 
     # Sets using a part
-    def using_part(
-        self,
-        part_num: str,
-        color_id: int,
-        /,
-        element_id: int | None = None,
-    ) -> Self:
+    def using_part(self, part: str, color: int, /) -> Self:
         # Save the parameters to the fields
-        self.fields.part_num = part_num
-        self.fields.color_id = color_id
-        self.fields.element_id = element_id
+        self.fields.part = part
+        self.fields.color = color
 
         # Load the sets from the database
-        for record in self.select(
-            override_query=self.using_part_query,
-            order=self.order
-        ):
-            brickset = BrickSet(record=record)
-
-            self.records.append(brickset)
+        self.list(override_query=self.using_part_query)
 
         return self
+
+    # Sets using a storage
+    def using_storage(self, storage: BrickSetStorage, /) -> Self:
+        # Save the parameters to the fields
+        self.fields.storage = storage.fields.id
+
+        # Load the sets from the database
+        self.list(override_query=self.using_storage_query)
+
+        return self
+
+
+# Helper to build the metadata lists
+def set_metadata_lists(
+    as_class: bool = False
+) -> dict[
+    str,
+    Union[
+        list[BrickSetOwner],
+        list[BrickSetPurchaseLocation],
+        BrickSetPurchaseLocation,
+        list[BrickSetStorage],
+        BrickSetStorageList,
+        list[BrickSetTag]
+    ]
+]:
+    return {
+        'brickset_owners': BrickSetOwnerList.list(),
+        'brickset_purchase_locations': BrickSetPurchaseLocationList.list(as_class=as_class),  # noqa: E501
+        'brickset_storages': BrickSetStorageList.list(as_class=as_class),
+        'brickset_tags': BrickSetTagList.list(),
+    }

bricktracker/set_owner.py

@@ -0,0 +1,16 @@
+from .metadata import BrickMetadata
+
+
+# Lego set owner metadata
+class BrickSetOwner(BrickMetadata):
+    kind: str = 'owner'
+
+    # Set state endpoint
+    set_state_endpoint: str = 'set.update_owner'
+
+    # Queries
+    delete_query: str = 'set/metadata/owner/delete'
+    insert_query: str = 'set/metadata/owner/insert'
+    select_query: str = 'set/metadata/owner/select'
+    update_field_query: str = 'set/metadata/owner/update/field'
+    update_set_state_query: str = 'set/metadata/owner/update/state'

bricktracker/set_owner_list.py

@@ -0,0 +1,21 @@
+from typing import Self
+
+from .metadata_list import BrickMetadataList
+from .set_owner import BrickSetOwner
+
+
+# Lego sets owner list
+class BrickSetOwnerList(BrickMetadataList[BrickSetOwner]):
+    kind: str = 'set owners'
+
+    # Database
+    table: str = 'bricktracker_set_owners'
+    order: str = '"bricktracker_metadata_owners"."name"'
+
+    # Queries
+    select_query = 'set/metadata/owner/list'
+
+    # Instantiate the list with the proper class
+    @classmethod
+    def new(cls, /, *, force: bool = False) -> Self:
+        return cls(BrickSetOwner, force=force)

bricktracker/set_purchase_location.py

@@ -0,0 +1,13 @@
+from .metadata import BrickMetadata
+
+
+# Lego set purchase location metadata
+class BrickSetPurchaseLocation(BrickMetadata):
+    kind: str = 'purchase location'
+
+    # Queries
+    delete_query: str = 'set/metadata/purchase_location/delete'
+    insert_query: str = 'set/metadata/purchase_location/insert'
+    select_query: str = 'set/metadata/purchase_location/select'
+    update_field_query: str = 'set/metadata/purchase_location/update/field'
+    update_set_value_query: str = 'set/metadata/purchase_location/update/value'

bricktracker/set_purchase_location_list.py

@@ -0,0 +1,42 @@
+from typing import Self
+
+from flask import current_app
+
+from .metadata_list import BrickMetadataList
+from .set_purchase_location import BrickSetPurchaseLocation
+
+
+# Lego sets purchase location list
+class BrickSetPurchaseLocationList(
+    BrickMetadataList[BrickSetPurchaseLocation]
+):
+    kind: str = 'set purchase locations'
+
+    # Order
+    order: str = '"bricktracker_metadata_purchase_locations"."name"'
+
+    # Queries
+    select_query: str = 'set/metadata/purchase_location/list'
+    all_query: str = 'set/metadata/purchase_location/all'
+
+    # Set value endpoint
+    set_value_endpoint: str = 'set.update_purchase_location'
+
+    # Load all purchase locations
+    @classmethod
+    def all(cls, /) -> Self:
+        new = cls.new()
+        new.override()
+
+        for record in new.select(
+            override_query=cls.all_query,
+            order=current_app.config['PURCHASE_LOCATION_DEFAULT_ORDER']
+        ):
+            new.records.append(new.model(record=record))
+
+        return new
+
+    # Instantiate the list with the proper class
+    @classmethod
+    def new(cls, /, *, force: bool = False) -> Self:
+        return cls(BrickSetPurchaseLocation, force=force)

bricktracker/set_status.py

@@ -0,0 +1,34 @@
+from typing import Self
+
+from .metadata import BrickMetadata
+
+
+# Lego set status metadata
+class BrickSetStatus(BrickMetadata):
+    kind: str = 'status'
+
+    # Set state endpoint
+    set_state_endpoint: str = 'set.update_status'
+
+    # Queries
+    delete_query: str = 'set/metadata/status/delete'
+    insert_query: str = 'set/metadata/status/insert'
+    select_query: str = 'set/metadata/status/select'
+    update_field_query: str = 'set/metadata/status/update/field'
+    update_set_state_query: str = 'set/metadata/status/update/state'
+
+    # Grab data from a form
+    def from_form(self, form: dict[str, str], /) -> Self:
+        super().from_form(form)
+
+        grid = form.get('grid', None)
+
+        self.fields.displayed_on_grid = grid == 'on'
+
+        return self
+
+    # Insert into database
+    def insert(self, /, **_) -> None:
+        super().insert(
+            displayed_on_grid=self.fields.displayed_on_grid
+        )

bricktracker/set_status_list.py

@@ -0,0 +1,30 @@
+from typing import Self
+
+from .metadata_list import BrickMetadataList
+from .set_status import BrickSetStatus
+
+
+# Lego sets status list
+class BrickSetStatusList(BrickMetadataList[BrickSetStatus]):
+    kind: str = 'set statuses'
+
+    # Database
+    table: str = 'bricktracker_set_statuses'
+    order: str = '"bricktracker_metadata_statuses"."name"'
+
+    # Queries
+    select_query = 'set/metadata/status/list'
+
+    # Filter the list of set status
+    def filter(self, all: bool = False) -> list[BrickSetStatus]:
+        return [
+            record
+            for record
+            in self.records
+            if all or record.fields.displayed_on_grid
+        ]
+
+    # Instantiate the list with the proper class
+    @classmethod
+    def new(cls, /, *, force: bool = False) -> Self:
+        return cls(BrickSetStatus, force=force)

bricktracker/set_storage.py

@@ -0,0 +1,22 @@
+from .metadata import BrickMetadata
+
+from flask import url_for
+
+
+# Lego set storage metadata
+class BrickSetStorage(BrickMetadata):
+    kind: str = 'storage'
+
+    # Queries
+    delete_query: str = 'set/metadata/storage/delete'
+    insert_query: str = 'set/metadata/storage/insert'
+    select_query: str = 'set/metadata/storage/select'
+    update_field_query: str = 'set/metadata/storage/update/field'
+    update_set_value_query: str = 'set/metadata/storage/update/value'
+
+    # Self url
+    def url(self, /) -> str:
+        return url_for(
+            'storage.details',
+            id=self.fields.id,
+        )

bricktracker/set_storage_list.py

@@ -0,0 +1,40 @@
+from typing import Self
+
+from flask import current_app
+
+from .metadata_list import BrickMetadataList
+from .set_storage import BrickSetStorage
+
+
+# Lego sets storage list
+class BrickSetStorageList(BrickMetadataList[BrickSetStorage]):
+    kind: str = 'set storages'
+
+    # Order
+    order: str = '"bricktracker_metadata_storages"."name"'
+
+    # Queries
+    select_query: str = 'set/metadata/storage/list'
+    all_query: str = 'set/metadata/storage/all'
+
+    # Set value endpoint
+    set_value_endpoint: str = 'set.update_storage'
+
+    # Load all storages
+    @classmethod
+    def all(cls, /) -> Self:
+        new = cls.new()
+        new.override()
+
+        for record in new.select(
+            override_query=cls.all_query,
+            order=current_app.config['STORAGE_DEFAULT_ORDER']
+        ):
+            new.records.append(new.model(record=record))
+
+        return new
+
+    # Instantiate the list with the proper class
+    @classmethod
+    def new(cls, /, *, force: bool = False) -> Self:
+        return cls(BrickSetStorage, force=force)

bricktracker/set_tag.py

@@ -0,0 +1,16 @@
+from .metadata import BrickMetadata
+
+
+# Lego set tag metadata
+class BrickSetTag(BrickMetadata):
+    kind: str = 'tag'
+
+    # Set state endpoint
+    set_state_endpoint: str = 'set.update_tag'
+
+    # Queries
+    delete_query: str = 'set/metadata/tag/delete'
+    insert_query: str = 'set/metadata/tag/insert'
+    select_query: str = 'set/metadata/tag/select'
+    update_field_query: str = 'set/metadata/tag/update/field'
+    update_set_state_query: str = 'set/metadata/tag/update/state'

bricktracker/set_tag_list.py

@@ -0,0 +1,21 @@
+from typing import Self
+
+from .metadata_list import BrickMetadataList
+from .set_tag import BrickSetTag
+
+
+# Lego sets tag list
+class BrickSetTagList(BrickMetadataList[BrickSetTag]):
+    kind: str = 'set tags'
+
+    # Database
+    table: str = 'bricktracker_set_tags'
+    order: str = '"bricktracker_metadata_tags"."name"'
+
+    # Queries
+    select_query: str = 'set/metadata/tag/list'
+
+    # Instantiate the list with the proper class
+    @classmethod
+    def new(cls, /, *, force: bool = False) -> Self:
+        return cls(BrickSetTag, force=force)

bricktracker/socket.py

@@ -1,22 +1,23 @@
 import logging
 from typing import Any, Final, Tuple
 
-from flask import copy_current_request_context, Flask, request
+from flask import Flask, request
 from flask_socketio import SocketIO
 
-from .configuration_list import BrickConfigurationList
-from .login import LoginManager
-from .rebrickable_set import RebrickableSet
+from .instructions import BrickInstructions
+from .instructions_list import BrickInstructionsList
+from .set import BrickSet
+from .socket_decorator import authenticated_socket, rebrickable_socket
 from .sql import close as sql_close
 
 logger = logging.getLogger(__name__)
 
 # Messages valid through the socket
 MESSAGES: Final[dict[str, str]] = {
-    'ADD_SET': 'add_set',
     'COMPLETE': 'complete',
     'CONNECT': 'connect',
     'DISCONNECT': 'disconnect',
+    'DOWNLOAD_INSTRUCTIONS': 'download_instructions',
     'FAIL': 'fail',
     'IMPORT_SET': 'import_set',
     'LOAD_SET': 'load_set',
@@ -56,19 +57,19 @@ class BrickSocket(object):
 
         # Compute the namespace
         self.namespace = '/{namespace}'.format(
-            namespace=app.config['SOCKET_NAMESPACE'].value
+            namespace=app.config['SOCKET_NAMESPACE']
         )
 
         # Inject CORS if a domain is defined
-        if app.config['DOMAIN_NAME'].value != '':
-            kwargs['cors_allowed_origins'] = app.config['DOMAIN_NAME'].value
+        if app.config['DOMAIN_NAME'] != '':
+            kwargs['cors_allowed_origins'] = app.config['DOMAIN_NAME']
 
         # Instantiate the socket
         self.socket = SocketIO(
             self.app,
             *args,
             **kwargs,
-            path=app.config['SOCKET_PATH'].value,
+            path=app.config['SOCKET_PATH'],
             async_mode='eventlet',
         )
 
@@ -84,62 +85,51 @@ class BrickSocket(object):
         def disconnect() -> None:
             self.disconnected()
 
-        @self.socket.on(MESSAGES['IMPORT_SET'], namespace=self.namespace)
-        def import_set(data: dict[str, Any], /) -> None:
-            # Needs to be authenticated
-            if LoginManager.is_not_authenticated():
-                self.fail(message='You need to be authenticated')
-                return
+        @self.socket.on(MESSAGES['DOWNLOAD_INSTRUCTIONS'], namespace=self.namespace)  # noqa: E501
+        @authenticated_socket(self)
+        def download_instructions(data: dict[str, Any], /) -> None:
+            instructions = BrickInstructions(
+                '{name}.pdf'.format(name=data.get('alt', '')),
+                socket=self
+            )
 
-            # Needs the Rebrickable API key
+            path = data.get('href', '').removeprefix('/instructions/')
+
+            # Update the progress
             try:
-                BrickConfigurationList.error_unless_is_set('REBRICKABLE_API_KEY')  # noqa: E501
-            except Exception as e:
-                self.fail(message=str(e))
-                return
+                self.progress_total = int(data.get('total', 0))
+                self.progress_count = int(data.get('current', 0))
+            except Exception:
+                pass
 
-            brickset = RebrickableSet(self)
+            instructions.download(path)
 
-            # Start it in a thread if requested
-            if self.threaded:
-                @copy_current_request_context
-                def do_download() -> None:
-                    brickset.download(data)
+            BrickInstructionsList(force=True)
 
-                self.socket.start_background_task(do_download)
-            else:
-                brickset.download(data)
+        @self.socket.on(MESSAGES['IMPORT_SET'], namespace=self.namespace)
+        @rebrickable_socket(self)
+        def import_set(data: dict[str, Any], /) -> None:
+            logger.debug('Socket: IMPORT_SET={data} (from: {fr})'.format(
+                data=data,
+                fr=request.sid,  # type: ignore
+            ))
+
+            BrickSet().download(self, data)
 
         @self.socket.on(MESSAGES['LOAD_SET'], namespace=self.namespace)
         def load_set(data: dict[str, Any], /) -> None:
-            # Needs to be authenticated
-            if LoginManager.is_not_authenticated():
-                self.fail(message='You need to be authenticated')
-                return
+            logger.debug('Socket: LOAD_SET={data} (from: {fr})'.format(
+                data=data,
+                fr=request.sid,  # type: ignore
+            ))
 
-            # Needs the Rebrickable API key
-            try:
-                BrickConfigurationList.error_unless_is_set('REBRICKABLE_API_KEY')  # noqa: E501
-            except Exception as e:
-                self.fail(message=str(e))
-                return
-
-            brickset = RebrickableSet(self)
-
-            # Start it in a thread if requested
-            if self.threaded:
-                @copy_current_request_context
-                def do_load() -> None:
-                    brickset.load(data)
-
-                self.socket.start_background_task(do_load)
-            else:
-                brickset.load(data)
+            BrickSet().load(self, data)
 
     # Update the progress auto-incrementing
     def auto_progress(
         self,
         /,
+        *,
         message: str | None = None,
         increment_total=False,
     ) -> None:
@@ -203,7 +193,7 @@ class BrickSocket(object):
         sql_close()
 
     # Update the progress
-    def progress(self, /, message: str | None = None) -> None:
+    def progress(self, /, *, message: str | None = None) -> None:
         # Save the las message
         if message is not None:
             self.progress_message = message
@@ -218,14 +208,14 @@ class BrickSocket(object):
         self.emit('PROGRESS', data)
 
     # Update the progress total only
-    def update_total(self, total: int, /, add: bool = False) -> None:
+    def update_total(self, total: int, /, *, add: bool = False) -> None:
         if add:
             self.progress_total += total
         else:
             self.progress_total = total
 
     # Update the total
-    def total_progress(self, total: int, /, add: bool = False) -> None:
+    def total_progress(self, total: int, /, *, add: bool = False) -> None:
         self.update_total(total, add=add)
 
         self.progress()

bricktracker/socket_decorator.py

@@ -0,0 +1,93 @@
+from functools import wraps
+from threading import Thread
+from typing import Callable, ParamSpec, TYPE_CHECKING, Union
+
+from flask import copy_current_request_context
+
+from .configuration_list import BrickConfigurationList
+from .login import LoginManager
+if TYPE_CHECKING:
+    from .socket import BrickSocket
+
+# What a threaded function can return (None or Thread)
+SocketReturn = Union[None, Thread]
+
+# Threaded signature (*arg, **kwargs -> (None or Thread)
+P = ParamSpec('P')
+SocketCallable = Callable[P, SocketReturn]
+
+
+# Fail if not authenticated
+def authenticated_socket(
+    self: 'BrickSocket',
+    /,
+    *,
+    threaded: bool = True,
+) -> Callable[[SocketCallable], SocketCallable]:
+    def outer(function: SocketCallable, /) -> SocketCallable:
+        @wraps(function)
+        def wrapper(*args, **kwargs) -> SocketReturn:
+            # Needs to be authenticated
+            if LoginManager.is_not_authenticated():
+                self.fail(message='You need to be authenticated')
+                return
+
+            # Apply threading
+            if threaded:
+                return threaded_socket(self)(function)(*args, **kwargs)
+            else:
+                return function(*args, **kwargs)
+
+        return wrapper
+    return outer
+
+
+# Fail if not ready for Rebrickable (authenticated, API key)
+# Automatically makes it threaded
+def rebrickable_socket(
+    self: 'BrickSocket',
+    /,
+    *,
+    threaded: bool = True,
+) -> Callable[[SocketCallable], SocketCallable]:
+    def outer(function: SocketCallable, /) -> SocketCallable:
+        @wraps(function)
+        # Automatically authenticated
+        @authenticated_socket(self, threaded=False)
+        def wrapper(*args, **kwargs) -> SocketReturn:
+            # Needs the Rebrickable API key
+            try:
+                BrickConfigurationList.error_unless_is_set('REBRICKABLE_API_KEY')  # noqa: E501
+            except Exception as e:
+                self.fail(message=str(e))
+                return
+
+            # Apply threading
+            if threaded:
+                return threaded_socket(self)(function)(*args, **kwargs)
+            else:
+                return function(*args, **kwargs)
+
+        return wrapper
+    return outer
+
+
+# Start the function in a thread if the socket is threaded
+def threaded_socket(
+    self: 'BrickSocket',
+    /
+) -> Callable[[SocketCallable], SocketCallable]:
+    def outer(function: SocketCallable, /) -> SocketCallable:
+        @wraps(function)
+        def wrapper(*args, **kwargs) -> SocketReturn:
+            # Start it in a thread if requested
+            if self.threaded:
+                @copy_current_request_context
+                def do_function() -> None:
+                    function(*args, **kwargs)
+
+                return self.socket.start_background_task(do_function)
+            else:
+                return function(*args, **kwargs)
+        return wrapper
+    return outer

bricktracker/sql.py

@@ -1,33 +1,47 @@
+from importlib import import_module
 import logging
 import os
 import sqlite3
-from typing import Any, Tuple
-
-from .sql_stats import BrickSQLStats
+from typing import Any, Final, Tuple
 
 from flask import current_app, g
 from jinja2 import Environment, FileSystemLoader
 from werkzeug.datastructures import FileStorage
 
+from .exceptions import DatabaseException
+from .sql_counter import BrickCounter
+from .sql_migration_list import BrickSQLMigrationList
+from .sql_stats import BrickSQLStats
+from .version import __database_version__
+
 logger = logging.getLogger(__name__)
 
+G_CONNECTION: Final[str] = 'database_connection'
+G_ENVIRONMENT: Final[str] = 'database_environment'
+G_DEFER: Final[str] = 'database_defer'
+G_STATS: Final[str] = 'database_stats'
+
 
 # SQLite3 client with our extra features
 class BrickSQL(object):
     connection: sqlite3.Connection
     cursor: sqlite3.Cursor
     stats: BrickSQLStats
+    version: int
 
-    def __init__(self, /):
+    def __init__(self, /, *, failsafe: bool = False):
         # Instantiate the database connection in the Flask
         # application context so that it can be used by all
         # requests without re-opening connections
-        database = getattr(g, 'database', None)
+        connection = getattr(g, G_CONNECTION, None)
 
         # Grab the existing connection if it exists
-        if database is not None:
-            self.connection = database
-            self.stats = getattr(g, 'database_stats', BrickSQLStats())
+        if connection is not None:
+            self.connection = connection
+            self.stats = getattr(g, G_STATS, BrickSQLStats())
+
+            # Grab a cursor
+            self.cursor = self.connection.cursor()
         else:
             # Instantiate the stats
             self.stats = BrickSQLStats()
@@ -37,26 +51,54 @@ class BrickSQL(object):
 
             logger.debug('SQLite3: connect')
             self.connection = sqlite3.connect(
-                current_app.config['DATABASE_PATH'].value
+                current_app.config['DATABASE_PATH']
             )
 
             # Setup the row factory to get pseudo-dicts rather than tuples
             self.connection.row_factory = sqlite3.Row
 
+            # Grab a cursor
+            self.cursor = self.connection.cursor()
+
+            # Grab the version and check
+            try:
+                version = self.fetchone('schema/get_version')
+
+                if version is None:
+                    raise Exception('version is None')
+
+                self.version = version[0]
+            except Exception as e:
+                self.version = 0
+
+                raise DatabaseException('Could not get the database version: {error}'.format(  # noqa: E501
+                    error=str(e)
+                ))
+
+            if self.upgrade_too_far():
+                raise DatabaseException('Your database version ({version}) is too far ahead for this version of the application. Expected at most {required}'.format(  # noqa: E501
+                    version=self.version,
+                    required=__database_version__,
+                ))
+
             # Debug: Attach the debugger
             # Uncomment manually because this is ultra verbose
             # self.connection.set_trace_callback(print)
 
             # Save the connection globally for later use
-            g.database = self.connection
-            g.database_stats = self.stats
+            setattr(g, G_CONNECTION, self.connection)
+            setattr(g, G_STATS, self.stats)
 
-        # Grab a cursor
-        self.cursor = self.connection.cursor()
+            if not failsafe:
+                if self.upgrade_needed():
+                    raise DatabaseException('Your database need to be upgraded from version {version} to version {required}'.format(  # noqa: E501
+                        version=self.version,
+                        required=__database_version__,
+                    ))
 
     # Clear the defer stack
     def clear_defer(self, /) -> None:
-        g.database_defer = []
+        setattr(g, G_DEFER, [])
 
     # Shorthand to commit
     def commit(self, /) -> None:
@@ -72,6 +114,27 @@ class BrickSQL(object):
         logger.debug('SQLite3: commit')
         return self.connection.commit()
 
+    # Count the database records
+    def count_records(self) -> list[BrickCounter]:
+        counters: list[BrickCounter] = []
+
+        # Get all tables
+        for table in self.fetchall('schema/tables'):
+            counter = BrickCounter(table['name'])
+
+            # Failsafe this one
+            try:
+                record = self.fetchone('schema/count', table=counter.table)
+
+                if record is not None:
+                    counter.count = record['count']
+            except Exception:
+                pass
+
+            counters.append(counter)
+
+        return counters
+
     # Defer a call to execute
     def defer(self, query: str, parameters: dict[str, Any], /):
         defer = self.get_defer()
@@ -82,16 +145,17 @@ class BrickSQL(object):
         defer.append((query, parameters))
 
         # Save the defer stack
-        g.database_defer = defer
+        setattr(g, G_DEFER, defer)
 
     # Shorthand to execute, returning number of affected rows
     def execute(
         self,
         query: str,
         /,
+        *,
         parameters: dict[str, Any] = {},
         defer: bool = False,
-        **context,
+        **context: Any,
     ) -> Tuple[int, str]:
         # Stats: execute
         self.stats.execute += 1
@@ -114,7 +178,7 @@ class BrickSQL(object):
             return result.rowcount, query
 
     # Shorthand to executescript
-    def executescript(self, query: str, /, **context) -> None:
+    def executescript(self, query: str, /, **context: Any) -> None:
         # Load the query
         query = self.load_query(query, **context)
 
@@ -129,8 +193,9 @@ class BrickSQL(object):
         self,
         query: str,
         /,
+        *,
         parameters: dict[str, Any] = {},
-        **context,
+        **context: Any,
     ) -> Tuple[int, str]:
         rows, query = self.execute(query, parameters=parameters, **context)
         self.commit()
@@ -142,8 +207,9 @@ class BrickSQL(object):
         self,
         query: str,
         /,
+        *,
         parameters: dict[str, Any] = {},
-        **context,
+        **context: Any,
     ) -> list[sqlite3.Row]:
         _, query = self.execute(query, parameters=parameters, **context)
 
@@ -163,8 +229,9 @@ class BrickSQL(object):
         self,
         query: str,
         /,
+        *,
         parameters: dict[str, Any] = {},
-        **context,
+        **context: Any,
     ) -> sqlite3.Row | None:
         _, query = self.execute(query, parameters=parameters, **context)
 
@@ -182,21 +249,18 @@ class BrickSQL(object):
 
     # Grab the defer stack
     def get_defer(self, /) -> list[Tuple[str, dict[str, Any]]]:
-        defer: list[Tuple[str, dict[str, Any]]] = getattr(
-            g,
-            'database_defer',
-            []
-        )
+        defer: list[Tuple[str, dict[str, Any]]] = getattr(g, G_DEFER, [])
 
         return defer
 
     # Load a query by name
-    def load_query(self, name: str, /, **context) -> str:
+    def load_query(self, name: str, /, **context: Any) -> str:
         # Grab the existing environment if it exists
-        environment = getattr(g, 'database_loader', None)
+        environment = getattr(g, G_ENVIRONMENT, None)
 
         # Instantiate Jinja environment for SQL files
         if environment is None:
+            logger.debug('SQLite3: instantiating the Jinja loader')
             environment = Environment(
                 loader=FileSystemLoader(
                     os.path.join(os.path.dirname(__file__), 'sql/')
@@ -204,10 +268,10 @@ class BrickSQL(object):
             )
 
             # Save the environment globally for later use
-            g.database_environment = environment
+            setattr(g, G_ENVIRONMENT, environment)
 
         # Grab the template
-        logger.debug('SQLite: loading {name} (context: {context})'.format(
+        logger.debug('SQLite3: loading {name} (context: {context})'.format(
             name=name,
             context=context,
         ))
@@ -221,7 +285,8 @@ class BrickSQL(object):
     def raw_execute(
         self,
         query: str,
-        parameters: dict[str, Any]
+        parameters: dict[str, Any],
+        /
     ) -> sqlite3.Cursor:
         logger.debug('SQLite3: execute: {query}'.format(
             query=BrickSQL.clean_query(query)
@@ -229,6 +294,55 @@ class BrickSQL(object):
 
         return self.cursor.execute(query, parameters)
 
+    # Upgrade the database
+    def upgrade(self) -> None:
+        if self.upgrade_needed():
+            for pending in BrickSQLMigrationList().pending(self.version):
+                logger.info('Applying migration {version}'.format(
+                    version=pending.version)
+                )
+
+                # Load context from the migrations if it exists
+                # It looks for a file in migrations/ named after the SQL file
+                # and containing one function named migration_xxxx, also named
+                # after the SQL file, returning a context dict.
+                #
+                # For instance:
+                # - sql/migrations/0007.sql
+                # - migrations/0007.py
+                #    - def migration_0007(BrickSQL) -> dict[str, Any]
+                try:
+                    module = import_module(
+                        '.migrations.{name}'.format(
+                            name=pending.name
+                        ),
+                        package='bricktracker'
+                    )
+                except Exception:
+                    module = None
+
+                # If a module has been loaded, we need to fail if an error
+                # occured while executing the migration function
+                if module is not None:
+                    function = getattr(module, 'migration_{name}'.format(
+                        name=pending.name
+                    ))
+
+                    context: dict[str, Any] = function(self)
+                else:
+                    context: dict[str, Any] = {}
+
+                self.executescript(pending.get_query(), **context)
+                self.execute('schema/set_version', version=pending.version)
+
+    # Tells whether the database needs upgrade
+    def upgrade_needed(self) -> bool:
+        return self.version < __database_version__
+
+    # Tells whether the database is too far
+    def upgrade_too_far(self) -> bool:
+        return self.version > __database_version__
+
     # Clean the query for debugging
     @staticmethod
     def clean_query(query: str, /) -> str:
@@ -249,7 +363,7 @@ class BrickSQL(object):
     # Delete the database
     @staticmethod
     def delete() -> None:
-        os.remove(current_app.config['DATABASE_PATH'].value)
+        os.remove(current_app.config['DATABASE_PATH'])
 
         # Info
         logger.info('The database has been deleted')
@@ -262,37 +376,10 @@ class BrickSQL(object):
         # Info
         logger.info('The database has been dropped')
 
-    # Count the database records
-    @staticmethod
-    def count_records() -> dict[str, int]:
-        database = BrickSQL()
-
-        counters: dict[str, int] = {}
-        for table in ['sets', 'minifigures', 'inventory', 'missing']:
-            record = database.fetchone('schema/count', table=table)
-
-            if record is not None:
-                counters[table] = record['count']
-
-        return counters
-
-    # Initialize the database
-    @staticmethod
-    def initialize() -> None:
-        BrickSQL().executescript('migrations/init')
-
-        # Info
-        logger.info('The database has been initialized')
-
-    # Check if the database is initialized
-    @staticmethod
-    def is_init() -> bool:
-        return BrickSQL().fetchone('schema/is_init') is not None
-
     # Replace the database with a new file
     @staticmethod
     def upload(file: FileStorage, /) -> None:
-        file.save(current_app.config['DATABASE_PATH'].value)
+        file.save(current_app.config['DATABASE_PATH'])
 
         # Info
         logger.info('The database has been imported using file {file}'.format(
@@ -302,11 +389,11 @@ class BrickSQL(object):
 
 # Close all existing SQLite3 connections
 def close() -> None:
-    database: sqlite3.Connection | None = getattr(g, 'database', None)
+    connection: sqlite3.Connection | None = getattr(g, G_CONNECTION, None)
 
-    if database is not None:
+    if connection is not None:
         logger.debug('SQLite3: close')
-        database.close()
+        connection.close()
 
         # Remove the database from the context
-        delattr(g, 'database')
+        delattr(g, G_CONNECTION)

bricktracker/sql/migrations/0001.sql

@@ -0,0 +1,66 @@
+-- description: Original database initialization
+-- FROM sqlite3 app.db .schema > init.sql with extra IF NOT EXISTS, transaction and quotes
+BEGIN TRANSACTION;
+
+CREATE TABLE IF NOT EXISTS "wishlist" (
+    "set_num" TEXT,
+    "name" TEXT,
+    "year" INTEGER,
+    "theme_id" INTEGER,
+    "num_parts" INTEGER,
+    "set_img_url" TEXT,
+    "set_url" TEXT,
+    "last_modified_dt" TEXT
+);
+
+CREATE TABLE IF NOT EXISTS "sets" (
+    "set_num" TEXT,
+    "name" TEXT,
+    "year" INTEGER,
+    "theme_id" INTEGER,
+    "num_parts" INTEGER,
+    "set_img_url" TEXT,
+    "set_url" TEXT,
+    "last_modified_dt" TEXT,
+    "mini_col" BOOLEAN,
+    "set_check" BOOLEAN,
+    "set_col" BOOLEAN,
+    "u_id" TEXT
+);
+
+CREATE TABLE IF NOT EXISTS "inventory" (
+    "set_num" TEXT,
+    "id" INTEGER,
+    "part_num" TEXT,
+    "name" TEXT,
+    "part_img_url" TEXT,
+    "part_img_url_id" TEXT,
+    "color_id" INTEGER,
+    "color_name" TEXT,
+    "quantity" INTEGER,
+    "is_spare" BOOLEAN,
+    "element_id" INTEGER,
+    "u_id" TEXT
+);
+
+CREATE TABLE IF NOT EXISTS "minifigures" (
+    "fig_num" TEXT,
+    "set_num" TEXT,
+    "name" TEXT,
+    "quantity" INTEGER,
+    "set_img_url" TEXT,
+    "u_id" TEXT
+);
+
+CREATE TABLE IF NOT EXISTS "missing" (
+    "set_num" TEXT,
+    "id" INTEGER,
+    "part_num" TEXT,
+    "part_img_url_id" TEXT,
+    "color_id" INTEGER,
+    "quantity" INTEGER,
+    "element_id" INTEGER,
+    "u_id" TEXT
+);
+
+COMMIT;

bricktracker/sql/migrations/0002.sql

@@ -0,0 +1,13 @@
+-- description: WAL journal, 'None' fix for missing table
+
+-- Set the journal mode to WAL
+PRAGMA journal_mode = WAL;
+
+BEGIN TRANSACTION;
+
+-- Fix a bug where 'None' was inserted in missing instead of NULL
+UPDATE "missing"
+SET "element_id" = NULL
+WHERE "missing"."element_id" = 'None';
+
+COMMIT;

bricktracker/sql/migrations/0003.sql

@@ -0,0 +1,48 @@
+-- description: Creation of the deduplicated table of Rebrickable sets
+
+BEGIN TRANSACTION;
+
+-- Create a Rebrickable set table: each unique set imported from Rebrickable
+CREATE TABLE "rebrickable_sets" (
+    "set" TEXT NOT NULL,
+    "number" INTEGER NOT NULL,
+    "version" INTEGER NOT NULL,
+    "name" TEXT NOT NULL,
+    "year" INTEGER NOT NULL,
+    "theme_id" INTEGER NOT NULL,
+    "number_of_parts" INTEGER NOT NULL,
+    "image" TEXT,
+    "url" TEXT,
+    "last_modified" TEXT,
+    PRIMARY KEY("set")
+);
+
+-- Insert existing sets into the new table
+INSERT INTO "rebrickable_sets" (
+    "set",
+    "number",
+    "version",
+    "name",
+    "year",
+    "theme_id",
+    "number_of_parts",
+    "image",
+    "url",
+    "last_modified"
+)
+SELECT
+    "sets"."set_num",
+    CAST(SUBSTR("sets"."set_num", 1, INSTR("sets"."set_num", '-') - 1) AS INTEGER),
+    CAST(SUBSTR("sets"."set_num", INSTR("sets"."set_num", '-') + 1) AS INTEGER),
+    "sets"."name",
+    "sets"."year",
+    "sets"."theme_id",
+    "sets"."num_parts",
+    "sets"."set_img_url",
+    "sets"."set_url",
+    "sets"."last_modified_dt"
+FROM "sets"
+GROUP BY
+    "sets"."set_num";
+
+COMMIT;

bricktracker/sql/migrations/0004.sql

@@ -0,0 +1,25 @@
+-- description: Migrate the Bricktracker sets
+
+PRAGMA foreign_keys = ON;
+
+BEGIN TRANSACTION;
+
+-- Create a Bricktracker set table: with their unique IDs, and a reference to the Rebrickable set
+CREATE TABLE "bricktracker_sets" (
+    "id" TEXT NOT NULL,
+    "rebrickable_set" TEXT NOT NULL,
+    PRIMARY KEY("id"),
+    FOREIGN KEY("rebrickable_set") REFERENCES "rebrickable_sets"("set")
+);
+
+-- Insert existing sets into the new table
+INSERT INTO "bricktracker_sets" (
+    "id",
+    "rebrickable_set"
+)
+SELECT
+    "sets"."u_id",
+    "sets"."set_num"
+FROM "sets";
+
+COMMIT;

bricktracker/sql/migrations/0005.sql

@@ -0,0 +1,72 @@
+-- description: Creation of the configurable set checkboxes
+
+PRAGMA foreign_keys = ON;
+
+BEGIN TRANSACTION;
+
+-- Create a table to define each set checkbox: with an ID, a name and if they should be displayed on the grid cards
+CREATE TABLE "bricktracker_set_checkboxes" (
+    "id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "displayed_on_grid" BOOLEAN NOT NULL DEFAULT 0,
+    PRIMARY KEY("id")
+);
+
+-- Seed our checkbox with the 3 original ones
+INSERT INTO "bricktracker_set_checkboxes" (
+    "id",
+    "name",
+    "displayed_on_grid"
+) VALUES (
+    "minifigures_collected",
+    "Minifigures are collected",
+    1
+);
+
+INSERT INTO "bricktracker_set_checkboxes" (
+    "id",
+    "name",
+    "displayed_on_grid"
+) VALUES (
+    "set_checked",
+    "Set is checked",
+    1
+);
+
+INSERT INTO "bricktracker_set_checkboxes" (
+    "id",
+    "name",
+    "displayed_on_grid"
+) VALUES (
+    "set_collected",
+    "Set is collected and boxed",
+    1
+);
+
+-- Create a table for the status of each checkbox: with the 3 first status
+CREATE TABLE "bricktracker_set_statuses" (
+    "bricktracker_set_id" TEXT NOT NULL,
+    "status_minifigures_collected" BOOLEAN NOT NULL DEFAULT 0,
+    "status_set_checked" BOOLEAN NOT NULL DEFAULT 0,
+    "status_set_collected" BOOLEAN NOT NULL DEFAULT 0,
+    PRIMARY KEY("bricktracker_set_id"),
+    FOREIGN KEY("bricktracker_set_id") REFERENCES "bricktracker_sets"("id")
+);
+
+INSERT INTO "bricktracker_set_statuses" (
+    "bricktracker_set_id",
+    "status_minifigures_collected",
+    "status_set_checked",
+    "status_set_collected"
+)
+SELECT
+    "sets"."u_id",
+    "sets"."mini_col",
+    "sets"."set_check",
+    "sets"."set_col"
+FROM "sets";
+
+-- Rename the original table (don't delete it yet?)
+ALTER TABLE "sets" RENAME TO "sets_old";
+
+COMMIT;

bricktracker/sql/migrations/0006.sql

@@ -0,0 +1,42 @@
+-- description: Migrate the whislist to have a Rebrickable sets structure
+
+BEGIN TRANSACTION;
+
+-- Create a Rebrickable wish table: each unique (light) set imported from Rebrickable
+CREATE TABLE "bricktracker_wishes" (
+    "set" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "year" INTEGER NOT NULL,
+    "theme_id" INTEGER NOT NULL,
+    "number_of_parts" INTEGER NOT NULL,
+    "image" TEXT,
+    "url" TEXT,
+    PRIMARY KEY("set")
+);
+
+-- Insert existing wishes into the new table
+INSERT INTO "bricktracker_wishes" (
+    "set",
+    "name",
+    "year",
+    "theme_id",
+    "number_of_parts",
+    "image",
+    "url"
+)
+SELECT
+    "wishlist"."set_num",
+    "wishlist"."name",
+    "wishlist"."year",
+    "wishlist"."theme_id",
+    "wishlist"."num_parts",
+    "wishlist"."set_img_url",
+    "wishlist"."set_url"
+FROM "wishlist"
+GROUP BY
+    "wishlist"."set_num";
+
+-- Rename the original table (don't delete it yet?)
+ALTER TABLE "wishlist" RENAME TO "wishlist_old";
+
+COMMIT;

bricktracker/sql/migrations/0007.sql

@@ -0,0 +1,74 @@
+-- description: Renaming various complicated field names to something simpler, and add a bunch of extra fields for later
+
+PRAGMA foreign_keys = ON;
+
+BEGIN TRANSACTION;
+
+-- Rename sets table
+ALTER TABLE "bricktracker_sets" RENAME TO "bricktracker_sets_old";
+
+-- Create a Bricktracker metadata storage table for later
+CREATE TABLE "bricktracker_metadata_storages" (
+    "id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    PRIMARY KEY("id")
+);
+
+-- Create a Bricktracker metadata purchase location table for later
+CREATE TABLE "bricktracker_metadata_purchase_locations" (
+    "id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    PRIMARY KEY("id")
+);
+
+-- Re-Create a Bricktracker set table with the simplified name
+CREATE TABLE "bricktracker_sets" (
+    "id" TEXT NOT NULL,
+    "set" TEXT NOT NULL,
+    "description" TEXT,
+    "storage" TEXT, -- Storage bin location
+    "purchase_date" REAL, -- Purchase data
+    "purchase_location" TEXT, -- Purchase location
+    "purchase_price" REAL, -- Purchase price
+    PRIMARY KEY("id"),
+    FOREIGN KEY("set") REFERENCES "rebrickable_sets"("set"),
+    FOREIGN KEY("storage") REFERENCES "bricktracker_metadata_storages"("id"),
+    FOREIGN KEY("purchase_location") REFERENCES "bricktracker_metadata_purchase_locations"("id")
+);
+
+-- Insert existing sets into the new table
+INSERT INTO "bricktracker_sets" (
+    "id",
+    "set"
+)
+SELECT
+    "bricktracker_sets_old"."id",
+    "bricktracker_sets_old"."rebrickable_set"
+FROM "bricktracker_sets_old";
+
+-- Rename status table
+ALTER TABLE "bricktracker_set_statuses" RENAME TO "bricktracker_set_statuses_old";
+
+-- Re-create a table for the status of each checkbox
+CREATE TABLE "bricktracker_set_statuses" (
+    "id" TEXT NOT NULL,
+    {% if structure %}{{ structure }},{% endif %}
+    PRIMARY KEY("id"),
+    FOREIGN KEY("id") REFERENCES "bricktracker_sets"("id")
+);
+
+-- Insert existing status into the new table
+INSERT INTO "bricktracker_set_statuses" (
+    {% if targets %}{{ targets }},{% endif %}
+    "id"
+)
+SELECT
+    {% if sources %}{{ sources }},{% endif %}
+    "bricktracker_set_statuses_old"."bricktracker_set_id"
+FROM "bricktracker_set_statuses_old";
+
+-- Delete the original tables
+DROP TABLE "bricktracker_set_statuses_old";
+DROP TABLE "bricktracker_sets_old";
+
+COMMIT;

bricktracker/sql/migrations/0008.sql

@@ -0,0 +1,30 @@
+-- description: Creation of the deduplicated table of Rebrickable minifigures
+
+BEGIN TRANSACTION;
+
+-- Create a Rebrickable minifigures table: each unique minifigure imported from Rebrickable
+CREATE TABLE "rebrickable_minifigures" (
+    "figure" TEXT NOT NULL,
+    "number" INTEGER NOT NULL,
+    "name" TEXT NOT NULL,
+    "image" TEXT,
+    PRIMARY KEY("figure")
+);
+
+-- Insert existing sets into the new table
+INSERT INTO "rebrickable_minifigures" (
+    "figure",
+    "number",
+    "name",
+    "image"
+)
+SELECT
+    "minifigures"."fig_num",
+    CAST(SUBSTR("minifigures"."fig_num", 5) AS INTEGER),
+    "minifigures"."name",
+    "minifigures"."set_img_url"
+FROM "minifigures"
+GROUP BY
+    "minifigures"."fig_num";
+
+COMMIT;

bricktracker/sql/migrations/0009.sql

@@ -0,0 +1,32 @@
+-- description: Migrate the Bricktracker minifigures
+
+PRAGMA foreign_keys = ON;
+
+BEGIN TRANSACTION;
+
+-- Create a Bricktracker minifigures table: an amount of minifigures linked to a Bricktracker set
+CREATE TABLE "bricktracker_minifigures" (
+    "id" TEXT NOT NULL,
+    "figure" TEXT NOT NULL,
+    "quantity" INTEGER NOT NULL,
+    PRIMARY KEY("id", "figure"),
+    FOREIGN KEY("id") REFERENCES "bricktracker_sets"("id"),
+    FOREIGN KEY("figure") REFERENCES "rebrickable_minifigures"("figure")
+);
+
+-- Insert existing sets into the new table
+INSERT INTO "bricktracker_minifigures" (
+    "id",
+    "figure",
+    "quantity"
+)
+SELECT
+    "minifigures"."u_id",
+    "minifigures"."fig_num",
+    "minifigures"."quantity"
+FROM "minifigures";
+
+-- Rename the original table (don't delete it yet?)
+ALTER TABLE "minifigures" RENAME TO "minifigures_old";
+
+COMMIT;

bricktracker/sql/migrations/0010.sql

@@ -0,0 +1,42 @@
+-- description: Creation of the deduplicated table of Rebrickable parts, and add a bunch of extra fields for later
+
+BEGIN TRANSACTION;
+
+-- Create a Rebrickable parts table: each unique part imported from Rebrickable
+CREATE TABLE "rebrickable_parts" (
+    "part" TEXT NOT NULL,
+    "color_id" INTEGER NOT NULL,
+    "color_name" TEXT NOT NULL,
+    "color_rgb" TEXT, -- can be NULL because it was not saved before
+    "color_transparent" BOOLEAN, -- can be NULL because it was not saved before
+    "name" TEXT NOT NULL,
+    "category" INTEGER, -- can be NULL because it was not saved before
+    "image" TEXT,
+    "image_id" TEXT,
+    "url" TEXT, -- can be NULL because it was not saved before
+    "print" INTEGER, -- can be NULL, was not saved before
+    PRIMARY KEY("part", "color_id")
+);
+
+-- Insert existing parts into the new table
+INSERT INTO "rebrickable_parts" (
+    "part",
+    "color_id",
+    "color_name",
+    "name",
+    "image",
+    "image_id"
+)
+SELECT
+    "inventory"."part_num",
+    "inventory"."color_id",
+    "inventory"."color_name",
+    "inventory"."name",
+    "inventory"."part_img_url",
+    "inventory"."part_img_url_id"
+FROM "inventory"
+GROUP BY
+    "inventory"."part_num",
+    "inventory"."color_id";
+
+COMMIT;

bricktracker/sql/migrations/0011.sql

@@ -0,0 +1,73 @@
+-- description: Migrate the Bricktracker parts (and missing parts), and add a bunch of extra fields for later
+
+PRAGMA foreign_keys = ON;
+
+BEGIN TRANSACTION;
+
+-- Fix: somehow a deletion bug was introduced in an older release?
+DELETE FROM "inventory"
+WHERE "inventory"."u_id" NOT IN (
+    SELECT "bricktracker_sets"."id"
+    FROM "bricktracker_sets"
+);
+
+DELETE FROM "missing"
+WHERE "missing"."u_id" NOT IN (
+    SELECT "bricktracker_sets"."id"
+    FROM "bricktracker_sets"
+);
+
+-- Create a Bricktracker parts table: an amount of parts linked to a Bricktracker set
+CREATE TABLE "bricktracker_parts" (
+    "id" TEXT NOT NULL,
+    "figure" TEXT,
+    "part" TEXT NOT NULL,
+    "color" INTEGER NOT NULL,
+    "spare" BOOLEAN NOT NULL,
+    "quantity" INTEGER NOT NULL,
+    "element" INTEGER,
+    "rebrickable_inventory" INTEGER NOT NULL,
+    "missing" INTEGER NOT NULL DEFAULT 0,
+    "damaged" INTEGER NOT NULL DEFAULT 0,
+    PRIMARY KEY("id", "figure", "part", "color", "spare"),
+    FOREIGN KEY("id") REFERENCES "bricktracker_sets"("id"),
+    FOREIGN KEY("figure") REFERENCES "rebrickable_minifigures"("figure"),
+    FOREIGN KEY("part", "color") REFERENCES "rebrickable_parts"("part", "color_id")
+);
+
+-- Insert existing parts into the new table
+INSERT INTO "bricktracker_parts" (
+    "id",
+    "figure",
+    "part",
+    "color",
+    "spare",
+    "quantity",
+    "element",
+    "rebrickable_inventory",
+    "missing"
+)
+SELECT
+    "inventory"."u_id",
+    CASE WHEN SUBSTR("inventory"."set_num", 0, 5) = 'fig-' THEN "inventory"."set_num" ELSE NULL END,
+    "inventory"."part_num",
+    "inventory"."color_id",
+    "inventory"."is_spare",
+    "inventory"."quantity",
+    "inventory"."element_id",
+    "inventory"."id",
+    IFNULL("missing"."quantity", 0)
+FROM "inventory"
+LEFT JOIN "missing"
+ON "inventory"."set_num" IS NOT DISTINCT FROM "missing"."set_num"
+AND "inventory"."id" IS NOT DISTINCT FROM "missing"."id"
+AND "inventory"."part_num" IS NOT DISTINCT FROM "missing"."part_num"
+AND "inventory"."color_id" IS NOT DISTINCT FROM "missing"."color_id"
+AND "inventory"."element_id" IS NOT DISTINCT FROM "missing"."element_id"
+AND "inventory"."u_id" IS NOT DISTINCT FROM "missing"."u_id";
+
+-- Rename the original table (don't delete it yet?)
+ALTER TABLE "inventory" RENAME TO "inventory_old";
+ALTER TABLE "missing" RENAME TO "missing_old";
+
+COMMIT;

bricktracker/sql/migrations/0012.sql

@@ -0,0 +1,7 @@
+-- description: Rename checkboxes to status metadata
+
+BEGIN TRANSACTION;
+
+ALTER TABLE "bricktracker_set_checkboxes" RENAME TO "bricktracker_metadata_statuses";
+
+COMMIT;

bricktracker/sql/migrations/0013.sql

@@ -0,0 +1,26 @@
+-- description: Add set owners
+
+BEGIN TRANSACTION;
+
+-- Create a table to define each set owners: an id and a name
+CREATE TABLE "bricktracker_metadata_owners" (
+    "id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    PRIMARY KEY("id")
+);
+
+-- Create a table for the set owners
+CREATE TABLE "bricktracker_set_owners" (
+    "id" TEXT NOT NULL,
+    PRIMARY KEY("id"),
+    FOREIGN KEY("id") REFERENCES "bricktracker_sets"("id")
+);
+
+-- Create a table for the wish owners
+CREATE TABLE "bricktracker_wish_owners" (
+    "set" TEXT NOT NULL,
+    PRIMARY KEY("set"),
+    FOREIGN KEY("set") REFERENCES "bricktracker_wishes"("set")
+);
+
+COMMIT;

bricktracker/sql/migrations/0014.sql

@@ -0,0 +1,19 @@
+-- description: Add set tags
+
+BEGIN TRANSACTION;
+
+-- Create a table to define each set tags: an id and a name
+CREATE TABLE "bricktracker_metadata_tags" (
+    "id" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    PRIMARY KEY("id")
+);
+
+-- Create a table for the set tags
+CREATE TABLE "bricktracker_set_tags" (
+    "id" TEXT NOT NULL,
+    PRIMARY KEY("id"),
+    FOREIGN KEY("id") REFERENCES "bricktracker_sets"("id")
+);
+
+COMMIT;

bricktracker/sql/migrations/0015.sql

@@ -0,0 +1,32 @@
+-- description: Add number of parts for minifigures
+
+BEGIN TRANSACTION;
+
+-- Add the number_of_parts column to the minifigures
+ALTER TABLE "rebrickable_minifigures"
+ADD COLUMN "number_of_parts" INTEGER NOT NULL DEFAULT 0;
+
+-- Update the number of parts for each minifigure
+UPDATE "rebrickable_minifigures"
+SET "number_of_parts" = "parts_sum"."number_of_parts"
+FROM (
+    SELECT
+        "parts"."figure",
+        SUM("parts"."quantity") as "number_of_parts"
+    FROM (
+        SELECT
+            "bricktracker_parts"."figure",
+            "bricktracker_parts"."quantity"
+        FROM "bricktracker_parts"
+        WHERE "bricktracker_parts"."figure" IS NOT NULL
+        GROUP BY
+            "bricktracker_parts"."figure",
+            "bricktracker_parts"."part",
+            "bricktracker_parts"."color",
+            "bricktracker_parts"."spare"
+    ) "parts"
+    GROUP BY "parts"."figure"
+) "parts_sum"
+WHERE "rebrickable_minifigures"."figure" = "parts_sum"."figure";
+
+COMMIT;

bricktracker/sql/migrations/0016.sql

@@ -0,0 +1,9 @@
+-- description: Add BrickLink color fields to rebrickable_parts table
+
+BEGIN TRANSACTION;
+
+-- Add BrickLink color fields to the rebrickable_parts table
+ALTER TABLE "rebrickable_parts" ADD COLUMN "bricklink_color_id" INTEGER;
+ALTER TABLE "rebrickable_parts" ADD COLUMN "bricklink_color_name" TEXT;
+
+COMMIT;

bricktracker/sql/migrations/0017.sql

@@ -0,0 +1,8 @@
+-- description: Add BrickLink part number field to rebrickable_parts table
+
+BEGIN TRANSACTION;
+
+-- Add BrickLink part number field to the rebrickable_parts table
+ALTER TABLE "rebrickable_parts" ADD COLUMN "bricklink_part_num" TEXT;
+
+COMMIT;

bricktracker/sql/migrations/init.sql

@@ -1,66 +0,0 @@
--- FROM sqlite3 app.db .schema > init.sql with extra IF NOT EXISTS and transaction
-BEGIN transaction;
-
-CREATE TABLE IF NOT EXISTS wishlist (
-        set_num TEXT,
-        name TEXT,
-        year INTEGER,
-        theme_id INTEGER,
-        num_parts INTEGER,
-        set_img_url TEXT,
-        set_url TEXT,
-        last_modified_dt TEXT
-    );
-CREATE TABLE IF NOT EXISTS sets (
-        set_num TEXT,
-        name TEXT,
-        year INTEGER,
-        theme_id INTEGER,
-        num_parts INTEGER,
-        set_img_url TEXT,
-        set_url TEXT,
-        last_modified_dt TEXT,
-        mini_col BOOLEAN,
-        set_check BOOLEAN,
-        set_col BOOLEAN,
-        u_id TEXT
-    );
-CREATE TABLE IF NOT EXISTS inventory (
-        set_num TEXT,
-        id INTEGER,
-        part_num TEXT,
-        name TEXT,
-        part_img_url TEXT,
-        part_img_url_id TEXT,
-        color_id INTEGER,
-        color_name TEXT,
-        quantity INTEGER,
-        is_spare BOOLEAN,
-        element_id INTEGER,
-        u_id TEXT
-    );
-CREATE TABLE IF NOT EXISTS minifigures (
-        fig_num TEXT,
-        set_num TEXT,
-        name TEXT,
-        quantity INTEGER,
-        set_img_url TEXT,
-        u_id TEXT
-    );
-CREATE TABLE IF NOT EXISTS missing (
-        set_num TEXT,
-        id INTEGER,
-        part_num TEXT,
-        part_img_url_id TEXT,
-        color_id INTEGER,
-        quantity INTEGER,
-        element_id INTEGER,
-        u_id TEXT
-    );
-
--- Fix a bug where 'None' was inserted in missing instead of NULL
-UPDATE missing
-SET element_id = NULL
-WHERE element_id = 'None';
-
-COMMIT;

bricktracker/sql/minifigure/base/base.sql

@@ -0,0 +1,37 @@
+SELECT
+    "bricktracker_minifigures"."quantity",
+    "rebrickable_minifigures"."figure",
+    "rebrickable_minifigures"."number",
+    "rebrickable_minifigures"."number_of_parts",
+    "rebrickable_minifigures"."name",
+    "rebrickable_minifigures"."image",
+    {% block total_missing %}
+    NULL AS "total_missing", -- dummy for order: total_missing
+    {% endblock %}
+    {% block total_damaged %}
+    NULL AS "total_damaged", -- dummy for order: total_damaged
+    {% endblock %}
+    {% block total_quantity %}
+    NULL AS "total_quantity", -- dummy for order: total_quantity
+    {% endblock %}
+    {% block total_sets %}
+    NULL AS "total_sets" -- dummy for order: total_sets
+    {% endblock %}
+FROM "bricktracker_minifigures"
+
+INNER JOIN "rebrickable_minifigures"
+ON "bricktracker_minifigures"."figure" IS NOT DISTINCT FROM "rebrickable_minifigures"."figure"
+
+{% block join %}{% endblock %}
+
+{% block where %}{% endblock %}
+
+{% block group %}{% endblock %}
+
+{% if order %}
+ORDER BY {{ order }}
+{% endif %}
+
+{% if limit %}
+LIMIT {{ limit }}
+{% endif %}

bricktracker/sql/minifigure/base/select.sql

@@ -1,31 +0,0 @@
-SELECT
-    minifigures.fig_num,
-    minifigures.set_num,
-    minifigures.name,
-    minifigures.quantity,
-    minifigures.set_img_url,
-    minifigures.u_id,
-    {% block total_missing %}
-    NULL AS total_missing, -- dummy for order: total_missing
-    {% endblock %}
-    {% block total_quantity %}
-    NULL AS total_quantity, -- dummy for order: total_quantity
-    {% endblock %}
-    {% block total_sets %}
-    NULL AS total_sets -- dummy for order: total_sets
-    {% endblock %}
-FROM minifigures
-
-{% block join %}{% endblock %}
-
-{% block where %}{% endblock %}
-
-{% block group %}{% endblock %}
-
-{% if order %}
-ORDER BY {{ order }}
-{% endif %}
-
-{% if limit %}
-LIMIT {{ limit }}
-{% endif %}

bricktracker/sql/minifigure/delete/all_from_set.sql

@@ -1,2 +0,0 @@
-DELETE FROM minifigures
-WHERE u_id IS NOT DISTINCT FROM :u_id

bricktracker/sql/minifigure/insert.sql

@@ -1,15 +1,9 @@
-INSERT INTO minifigures (
-    fig_num,
-    set_num,
-    name,
-    quantity,
-    set_img_url,
-    u_id
+INSERT INTO "bricktracker_minifigures" (
+    "id",
+    "figure",
+    "quantity"
 ) VALUES (
-    :fig_num,
-    :set_num,
-    :name,
-    :quantity,
-    :set_img_url,
-    :u_id
+    :id,
+    :figure,
+    :quantity
 )

bricktracker/sql/minifigure/list/all.sql

@@ -1,34 +1,40 @@
-{% extends 'minifigure/base/select.sql' %}
+{% extends 'minifigure/base/base.sql' %}
 
 {% block total_missing %}
-SUM(IFNULL(missing_join.total, 0)) AS total_missing,
+SUM(IFNULL("problem_join"."total_missing", 0)) AS "total_missing",
+{% endblock %}
+
+{% block total_damaged %}
+SUM(IFNULL("problem_join"."total_damaged", 0)) AS "total_damaged",
 {% endblock %}
 
 {% block total_quantity %}
-SUM(IFNULL(minifigures.quantity, 0)) AS total_quantity,
+SUM(IFNULL("bricktracker_minifigures"."quantity", 0)) AS "total_quantity",
 {% endblock %}
 
 {% block total_sets %}
-COUNT(minifigures.set_num) AS total_sets
+IFNULL(COUNT("bricktracker_minifigures"."id"), 0) AS "total_sets"
 {% endblock %}
 
 {% block join %}
 -- LEFT JOIN + SELECT to avoid messing the total
 LEFT JOIN (
     SELECT
-        set_num,
-        u_id,
-        SUM(quantity) AS total
-    FROM missing
+        "bricktracker_parts"."id",
+        "bricktracker_parts"."figure",
+        SUM("bricktracker_parts"."missing") AS "total_missing",
+        SUM("bricktracker_parts"."damaged") AS "total_damaged"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."figure" IS NOT NULL
     GROUP BY
-        set_num,
-        u_id
-) missing_join
-ON minifigures.u_id IS NOT DISTINCT FROM missing_join.u_id
-AND minifigures.fig_num IS NOT DISTINCT FROM missing_join.set_num
+        "bricktracker_parts"."id",
+        "bricktracker_parts"."figure"
+) "problem_join"
+ON "bricktracker_minifigures"."id" IS NOT DISTINCT FROM "problem_join"."id"
+AND "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM "problem_join"."figure"
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    minifigures.fig_num
+    "rebrickable_minifigures"."figure"
 {% endblock %}

bricktracker/sql/minifigure/list/all_by_owner.sql

@@ -0,0 +1,71 @@
+{% extends 'minifigure/base/base.sql' %}
+
+{% block total_missing %}
+SUM(IFNULL("problem_join"."total_missing", 0)) AS "total_missing",
+{% endblock %}
+
+{% block total_damaged %}
+SUM(IFNULL("problem_join"."total_damaged", 0)) AS "total_damaged",
+{% endblock %}
+
+{% block total_quantity %}
+{% if owner_id and owner_id != 'all' %}
+SUM(CASE WHEN "bricktracker_set_owners"."owner_{{ owner_id }}" = 1 THEN IFNULL("bricktracker_minifigures"."quantity", 0) ELSE 0 END) AS "total_quantity",
+{% else %}
+SUM(IFNULL("bricktracker_minifigures"."quantity", 0)) AS "total_quantity",
+{% endif %}
+{% endblock %}
+
+{% block total_sets %}
+{% if owner_id and owner_id != 'all' %}
+COUNT(CASE WHEN "bricktracker_set_owners"."owner_{{ owner_id }}" = 1 THEN "bricktracker_minifigures"."id" ELSE NULL END) AS "total_sets"
+{% else %}
+COUNT("bricktracker_minifigures"."id") AS "total_sets"
+{% endif %}
+{% endblock %}
+
+{% block join %}
+-- Join with sets to get owner information
+INNER JOIN "bricktracker_sets"
+ON "bricktracker_minifigures"."id" IS NOT DISTINCT FROM "bricktracker_sets"."id"
+
+-- Left join with set owners (using dynamic columns)
+LEFT JOIN "bricktracker_set_owners"
+ON "bricktracker_sets"."id" IS NOT DISTINCT FROM "bricktracker_set_owners"."id"
+
+-- LEFT JOIN + SELECT to avoid messing the total
+LEFT JOIN (
+    SELECT
+        "bricktracker_parts"."id",
+        "bricktracker_parts"."figure",
+        {% if owner_id and owner_id != 'all' %}
+        SUM(CASE WHEN "owner_parts"."owner_{{ owner_id }}" = 1 THEN "bricktracker_parts"."missing" ELSE 0 END) AS "total_missing",
+        SUM(CASE WHEN "owner_parts"."owner_{{ owner_id }}" = 1 THEN "bricktracker_parts"."damaged" ELSE 0 END) AS "total_damaged"
+        {% else %}
+        SUM("bricktracker_parts"."missing") AS "total_missing",
+        SUM("bricktracker_parts"."damaged") AS "total_damaged"
+        {% endif %}
+    FROM "bricktracker_parts"
+    INNER JOIN "bricktracker_sets" AS "parts_sets"
+    ON "bricktracker_parts"."id" IS NOT DISTINCT FROM "parts_sets"."id"
+    LEFT JOIN "bricktracker_set_owners" AS "owner_parts"
+    ON "parts_sets"."id" IS NOT DISTINCT FROM "owner_parts"."id"
+    WHERE "bricktracker_parts"."figure" IS NOT NULL
+    GROUP BY
+        "bricktracker_parts"."id",
+        "bricktracker_parts"."figure"
+) "problem_join"
+ON "bricktracker_minifigures"."id" IS NOT DISTINCT FROM "problem_join"."id"
+AND "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM "problem_join"."figure"
+{% endblock %}
+
+{% block where %}
+{% if owner_id and owner_id != 'all' %}
+WHERE "bricktracker_set_owners"."owner_{{ owner_id }}" = 1
+{% endif %}
+{% endblock %}
+
+{% block group %}
+GROUP BY
+    "rebrickable_minifigures"."figure"
+{% endblock %}

bricktracker/sql/minifigure/list/damaged_part.sql

@@ -0,0 +1,28 @@
+{% extends 'minifigure/base/base.sql' %}
+
+{% block total_damaged %}
+SUM("bricktracker_parts"."damaged") AS "total_damaged",
+{% endblock %}
+
+{% block join %}
+LEFT JOIN "bricktracker_parts"
+ON "bricktracker_minifigures"."id" IS NOT DISTINCT FROM "bricktracker_parts"."id"
+AND "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM "bricktracker_parts"."figure"
+{% endblock %}
+
+{% block where %}
+WHERE "rebrickable_minifigures"."figure" IN (
+    SELECT "bricktracker_parts"."figure"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."part" IS NOT DISTINCT FROM :part
+    AND "bricktracker_parts"."color" IS NOT DISTINCT FROM :color
+    AND "bricktracker_parts"."figure" IS NOT NULL
+    AND "bricktracker_parts"."damaged" > 0
+    GROUP BY "bricktracker_parts"."figure"
+)
+{% endblock %}
+
+{% block group %}
+GROUP BY
+    "rebrickable_minifigures"."figure"
+{% endblock %}

bricktracker/sql/minifigure/list/from_set.sql

@@ -1,6 +1,5 @@
-{% extends 'minifigure/base/select.sql' %}
+{% extends 'minifigure/base/base.sql' %}
 
 {% block where %}
-WHERE u_id IS NOT DISTINCT FROM :u_id
-AND set_num IS NOT DISTINCT FROM :set_num
+WHERE "bricktracker_minifigures"."id" IS NOT DISTINCT FROM :id
 {% endblock %}

bricktracker/sql/minifigure/list/last.sql

@@ -1,17 +1,21 @@
-{% extends 'minifigure/base/select.sql' %}
+{% extends 'minifigure/base/base.sql' %}
 
 {% block total_missing %}
-SUM(IFNULL(missing.quantity, 0)) AS total_missing,
+SUM("bricktracker_parts"."missing") AS "total_missing",
+{% endblock %}
+
+{% block total_damaged %}
+SUM("bricktracker_parts"."damaged") AS "total_damaged",
 {% endblock %}
 
 {% block join %}
-LEFT JOIN missing
-ON minifigures.fig_num IS NOT DISTINCT FROM missing.set_num
-AND minifigures.u_id IS NOT DISTINCT FROM missing.u_id
+LEFT JOIN "bricktracker_parts"
+ON "bricktracker_minifigures"."id" IS NOT DISTINCT FROM "bricktracker_parts"."id"
+AND "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM "bricktracker_parts"."figure"
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    minifigures.fig_num,
-    minifigures.u_id
+    "rebrickable_minifigures"."figure",
+    "bricktracker_minifigures"."id"
 {% endblock %}

bricktracker/sql/minifigure/list/missing_part.sql

@@ -1,30 +1,28 @@
-{% extends 'minifigure/base/select.sql' %}
+{% extends 'minifigure/base/base.sql' %}
 
 {% block total_missing %}
-SUM(IFNULL(missing.quantity, 0)) AS total_missing,
+SUM("bricktracker_parts"."missing") AS "total_missing",
 {% endblock %}
 
 {% block join %}
-LEFT JOIN missing
-ON minifigures.fig_num IS NOT DISTINCT FROM missing.set_num
-AND minifigures.u_id IS NOT DISTINCT FROM missing.u_id
+LEFT JOIN "bricktracker_parts"
+ON "bricktracker_minifigures"."id" IS NOT DISTINCT FROM "bricktracker_parts"."id"
+AND "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM "bricktracker_parts"."figure"
 {% endblock %}
 
 {% block where %}
-WHERE minifigures.fig_num IN (
-    SELECT
-        missing.set_num
-    FROM missing
-
-    WHERE missing.color_id IS NOT DISTINCT FROM :color_id
-    AND missing.element_id IS NOT DISTINCT FROM :element_id
-    AND missing.part_num IS NOT DISTINCT FROM :part_num
-
-    GROUP BY missing.set_num
+WHERE "rebrickable_minifigures"."figure" IN (
+    SELECT "bricktracker_parts"."figure"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."part" IS NOT DISTINCT FROM :part
+    AND "bricktracker_parts"."color" IS NOT DISTINCT FROM :color
+    AND "bricktracker_parts"."figure" IS NOT NULL
+    AND "bricktracker_parts"."missing" > 0
+    GROUP BY "bricktracker_parts"."figure"
 )
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    minifigures.fig_num
+    "rebrickable_minifigures"."figure"
 {% endblock %}

bricktracker/sql/minifigure/list/using_part.sql

@@ -1,24 +1,21 @@
-{% extends 'minifigure/base/select.sql' %}
+{% extends 'minifigure/base/base.sql' %}
 
 {% block total_quantity %}
-SUM(minifigures.quantity) AS total_quantity,
+SUM("bricktracker_minifigures"."quantity") AS "total_quantity",
 {% endblock %}
 
 {% block where %}
-WHERE minifigures.fig_num IN (
-    SELECT
-        inventory.set_num
-    FROM inventory
-
-    WHERE inventory.color_id IS NOT DISTINCT FROM :color_id
-    AND inventory.element_id IS NOT DISTINCT FROM :element_id
-    AND inventory.part_num IS NOT DISTINCT FROM :part_num
-
-    GROUP BY inventory.set_num
+WHERE "rebrickable_minifigures"."figure" IN (
+    SELECT "bricktracker_parts"."figure"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."part" IS NOT DISTINCT FROM :part
+    AND "bricktracker_parts"."color" IS NOT DISTINCT FROM :color
+    AND "bricktracker_parts"."figure" IS NOT NULL
+    GROUP BY "bricktracker_parts"."figure"
 )
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    minifigures.fig_num
+    "rebrickable_minifigures"."figure"
 {% endblock %}

bricktracker/sql/minifigure/select/generic.sql

@@ -1,38 +1,40 @@
-{% extends 'minifigure/base/select.sql' %}
+{% extends 'minifigure/base/base.sql' %}
 
 {% block total_missing %}
-SUM(IFNULL(missing_join.total, 0)) AS total_missing,
+IFNULL("problem_join"."total_missing", 0) AS "total_missing",
+{% endblock %}
+
+{% block total_damaged %}
+IFNULL("problem_join"."total_damaged", 0) AS "total_damaged",
 {% endblock %}
 
 {% block total_quantity %}
-SUM(IFNULL(minifigures.quantity, 0)) AS total_quantity,
+SUM(IFNULL("bricktracker_minifigures"."quantity", 0)) AS "total_quantity",
 {% endblock %}
 
 {% block total_sets %}
-COUNT(minifigures.set_num) AS total_sets
+IFNULL(COUNT(DISTINCT "bricktracker_minifigures"."id"), 0) AS "total_sets"
 {% endblock %}
 
 {% block join %}
 -- LEFT JOIN + SELECT to avoid messing the total
 LEFT JOIN (
     SELECT
-        set_num,
-        u_id,
-        SUM(quantity) AS total
-    FROM missing
-    GROUP BY
-        set_num,
-        u_id
-) missing_join
-ON minifigures.u_id IS NOT DISTINCT FROM missing_join.u_id
-AND minifigures.fig_num IS NOT DISTINCT FROM missing_join.set_num
+        "bricktracker_parts"."figure",
+        SUM("bricktracker_parts"."missing") AS "total_missing",
+        SUM("bricktracker_parts"."damaged") AS "total_damaged"
+    FROM "bricktracker_parts"
+    WHERE "bricktracker_parts"."figure" IS NOT DISTINCT FROM :figure
+    GROUP BY "bricktracker_parts"."figure"
+) "problem_join"
+ON "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM "problem_join"."figure"
 {% endblock %}
 
 {% block where %}
-WHERE fig_num IS NOT DISTINCT FROM :fig_num
+WHERE "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM :figure
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    minifigures.fig_num
+    "rebrickable_minifigures"."figure"
 {% endblock %}

bricktracker/sql/minifigure/select/specific.sql

@@ -1,7 +1,6 @@
-{% extends 'minifigure/base/select.sql' %}
+{% extends 'minifigure/base/base.sql' %}
 
 {% block where %}
-WHERE fig_num IS NOT DISTINCT FROM :fig_num
-AND u_id IS NOT DISTINCT FROM :u_id
-AND set_num IS NOT DISTINCT FROM :set_num
+WHERE "bricktracker_minifigures"."id" IS NOT DISTINCT FROM :id
+AND "rebrickable_minifigures"."figure" IS NOT DISTINCT FROM :figure
 {% endblock %}

bricktracker/sql/missing/count_none.sql

@@ -1,3 +0,0 @@
-SELECT count(*) AS count
-FROM missing
-WHERE element_id = 'None'

bricktracker/sql/missing/delete/all_from_set.sql

@@ -1,2 +0,0 @@
-DELETE FROM missing
-WHERE u_id IS NOT DISTINCT FROM :u_id

bricktracker/sql/missing/delete/from_set.sql

@@ -1,4 +0,0 @@
-DELETE FROM missing
-WHERE set_num IS NOT DISTINCT FROM :set_num
-AND id IS NOT DISTINCT FROM :id
-AND u_id IS NOT DISTINCT FROM :u_id

bricktracker/sql/missing/insert.sql

@@ -1,20 +0,0 @@
-INSERT INTO missing (
-    set_num,
-    id,
-    part_num,
-    part_img_url_id,
-    color_id,
-    quantity,
-    element_id,
-    u_id
-)
-VALUES(
-    :set_num,
-    :id,
-    :part_num,
-    :part_img_url_id,
-    :color_id,
-    :quantity,
-    :element_id,
-    :u_id
-)

bricktracker/sql/missing/update/from_set.sql

@@ -1,5 +0,0 @@
-UPDATE missing
-SET quantity = :quantity
-WHERE set_num IS NOT DISTINCT FROM :set_num
-AND id IS NOT DISTINCT FROM :id
-AND u_id IS NOT DISTINCT FROM :u_id

bricktracker/sql/part/base/base.sql

@@ -0,0 +1,62 @@
+SELECT
+    "bricktracker_parts"."id",
+    "bricktracker_parts"."figure",
+    "bricktracker_parts"."part",
+    "bricktracker_parts"."color",
+    "bricktracker_parts"."spare",
+    "bricktracker_parts"."quantity",
+    "bricktracker_parts"."element",
+    --"bricktracker_parts"."rebrickable_inventory",
+    "bricktracker_parts"."missing",
+    "bricktracker_parts"."damaged",
+    --"rebrickable_parts"."part",
+    --"rebrickable_parts"."color_id",
+    "rebrickable_parts"."color_name",
+    "rebrickable_parts"."color_rgb",
+    "rebrickable_parts"."color_transparent",
+    "rebrickable_parts"."bricklink_color_id",
+    "rebrickable_parts"."bricklink_color_name",
+    "rebrickable_parts"."bricklink_part_num",
+    "rebrickable_parts"."name",
+    --"rebrickable_parts"."category",
+    "rebrickable_parts"."image",
+    "rebrickable_parts"."image_id",
+    "rebrickable_parts"."url",
+    "rebrickable_parts"."print",
+    {% block total_missing %}
+    NULL AS "total_missing", -- dummy for order: total_missing
+    {% endblock %}
+    {% block total_damaged %}
+    NULL AS "total_damaged", -- dummy for order: total_damaged
+    {% endblock %}
+    {% block total_quantity %}
+    NULL AS "total_quantity", -- dummy for order: total_quantity
+    {% endblock %}
+    {% block total_spare %}
+    NULL AS "total_spare", -- dummy for order: total_spare
+    {% endblock %}
+    {% block total_sets %}
+    NULL AS "total_sets", -- dummy for order: total_sets
+    {% endblock %}
+    {% block total_minifigures %}
+    NULL AS "total_minifigures" -- dummy for order: total_minifigures
+    {% endblock %}
+FROM "bricktracker_parts"
+
+INNER JOIN "rebrickable_parts"
+ON "bricktracker_parts"."part" IS NOT DISTINCT FROM "rebrickable_parts"."part"
+AND "bricktracker_parts"."color" IS NOT DISTINCT FROM "rebrickable_parts"."color_id"
+
+{% block join %}{% endblock %}
+
+{% block where %}{% endblock %}
+
+{% block group %}{% endblock %}
+
+{% if order %}
+ORDER BY {{ order }}
+{% endif %}
+
+{% if limit %}
+LIMIT {{ limit }}
+{% endif %}

bricktracker/sql/part/base/select.sql

@@ -1,43 +0,0 @@
-SELECT
-    inventory.set_num,
-    inventory.id,
-    inventory.part_num,
-    inventory.name,
-    inventory.part_img_url,
-    inventory.part_img_url_id,
-    inventory.color_id,
-    inventory.color_name,
-    inventory.quantity,
-    inventory.is_spare,
-    inventory.element_id,
-    inventory.u_id,
-    {% block total_missing %}
-    NULL AS total_missing, -- dummy for order: total_missing
-    {% endblock %}
-    {% block total_quantity %}
-    NULL AS total_quantity, -- dummy for order: total_quantity
-    {% endblock %}
-    {% block total_spare %}
-    NULL AS total_spare, -- dummy for order: total_spare
-    {% endblock %}
-    {% block total_sets %}
-    NULL AS total_sets, -- dummy for order: total_sets
-    {% endblock %}
-    {% block total_minifigures %}
-    NULL AS total_minifigures -- dummy for order: total_minifigures
-    {% endblock %}
-FROM inventory
-
-{% block join %}{% endblock %}
-
-{% block where %}{% endblock %}
-
-{% block group %}{% endblock %}
-
-{% if order %}
-ORDER BY {{ order }}
-{% endif %}
-
-{% if limit %}
-LIMIT {{ limit }}
-{% endif %}

bricktracker/sql/part/colors/list.sql

@@ -0,0 +1,16 @@
+SELECT DISTINCT
+    "rebrickable_parts"."color_id" AS "color_id",
+    "rebrickable_parts"."color_name" AS "color_name",
+    "rebrickable_parts"."color_rgb" AS "color_rgb"
+FROM "rebrickable_parts"
+INNER JOIN "bricktracker_parts"
+ON "bricktracker_parts"."part" IS NOT DISTINCT FROM "rebrickable_parts"."part"
+AND "bricktracker_parts"."color" IS NOT DISTINCT FROM "rebrickable_parts"."color_id"
+{% if owner_id and owner_id != 'all' %}
+INNER JOIN "bricktracker_sets"
+ON "bricktracker_parts"."id" IS NOT DISTINCT FROM "bricktracker_sets"."id"
+INNER JOIN "bricktracker_set_owners"
+ON "bricktracker_sets"."id" IS NOT DISTINCT FROM "bricktracker_set_owners"."id"
+WHERE "bricktracker_set_owners"."owner_{{ owner_id }}" = 1
+{% endif %}
+ORDER BY "rebrickable_parts"."color_name" ASC

bricktracker/sql/part/delete/all_from_set.sql

@@ -1,2 +0,0 @@
-DELETE FROM inventory
-WHERE u_id IS NOT DISTINCT FROM :u_id

bricktracker/sql/part/insert.sql

@@ -1,27 +1,19 @@
-INSERT INTO inventory (
-    set_num,
-    id,
-    part_num,
-    name,
-    part_img_url,
-    part_img_url_id,
-    color_id,
-    color_name,
-    quantity,
-    is_spare,
-    element_id,
-    u_id
+INSERT INTO "bricktracker_parts" (
+    "id",
+    "figure",
+    "part",
+    "color",
+    "spare",
+    "quantity",
+    "element",
+    "rebrickable_inventory"
 ) VALUES (
-    :set_num,
     :id,
-    :part_num,
-    :name,
-    :part_img_url,
-    :part_img_url_id,
-    :color_id,
-    :color_name,
+    :figure,
+    :part,
+    :color,
+    :spare,
     :quantity,
-    :is_spare,
-    :element_id,
-    :u_id
+    :element,
+    :rebrickable_inventory
 )

bricktracker/sql/part/list/all.sql

@@ -1,43 +1,40 @@
-{% extends 'part/base/select.sql' %}
+{% extends 'part/base/base.sql' %}
 
 {% block total_missing %}
-SUM(IFNULL(missing.quantity, 0)) AS total_missing,
+SUM("bricktracker_parts"."missing") AS "total_missing",
+{% endblock %}
+
+{% block total_damaged %}
+SUM("bricktracker_parts"."damaged") AS "total_damaged",
 {% endblock %}
 
 {% block total_quantity %}
-SUM(inventory.quantity * IFNULL(minifigures.quantity, 1)) AS total_quantity,
+SUM("bricktracker_parts"."quantity" * IFNULL("bricktracker_minifigures"."quantity", 1)) AS "total_quantity",
 {% endblock %}
 
 {% block total_sets %}
-COUNT(DISTINCT sets.u_id) AS total_sets,
+IFNULL(COUNT(DISTINCT "bricktracker_parts"."id"), 0) AS "total_sets",
 {% endblock %}
 
 {% block total_minifigures %}
-SUM(IFNULL(minifigures.quantity, 0)) AS total_minifigures
+SUM(IFNULL("bricktracker_minifigures"."quantity", 0)) AS "total_minifigures"
 {% endblock %}
 
 {% block join %}
-LEFT JOIN missing
-ON inventory.set_num IS NOT DISTINCT FROM missing.set_num
-AND inventory.id IS NOT DISTINCT FROM missing.id
-AND inventory.part_num IS NOT DISTINCT FROM missing.part_num
-AND inventory.color_id IS NOT DISTINCT FROM missing.color_id
-AND inventory.element_id IS NOT DISTINCT FROM missing.element_id
-AND inventory.u_id IS NOT DISTINCT FROM missing.u_id
+LEFT JOIN "bricktracker_minifigures"
+ON "bricktracker_parts"."id" IS NOT DISTINCT FROM "bricktracker_minifigures"."id"
+AND "bricktracker_parts"."figure" IS NOT DISTINCT FROM "bricktracker_minifigures"."figure"
+{% endblock %}
 
-LEFT JOIN minifigures
-ON inventory.set_num IS NOT DISTINCT FROM minifigures.fig_num
-AND inventory.u_id IS NOT DISTINCT FROM minifigures.u_id
-
-LEFT JOIN sets
-ON inventory.u_id IS NOT DISTINCT FROM sets.u_id
+{% block where %}
+{% if color_id and color_id != 'all' %}
+WHERE "bricktracker_parts"."color" = {{ color_id }}
+{% endif %}
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    inventory.part_num,
-    inventory.name,
-    inventory.color_id,
-    inventory.is_spare,
-    inventory.element_id
+    "bricktracker_parts"."part",
+    "bricktracker_parts"."color",
+    "bricktracker_parts"."spare"
 {% endblock %}

bricktracker/sql/part/list/all_by_owner.sql

@@ -0,0 +1,78 @@
+{% extends 'part/base/base.sql' %}
+
+{% block total_missing %}
+{% if owner_id and owner_id != 'all' %}
+SUM(CASE WHEN "bricktracker_set_owners"."owner_{{ owner_id }}" = 1 THEN "bricktracker_parts"."missing" ELSE 0 END) AS "total_missing",
+{% else %}
+SUM("bricktracker_parts"."missing") AS "total_missing",
+{% endif %}
+{% endblock %}
+
+{% block total_damaged %}
+{% if owner_id and owner_id != 'all' %}
+SUM(CASE WHEN "bricktracker_set_owners"."owner_{{ owner_id }}" = 1 THEN "bricktracker_parts"."damaged" ELSE 0 END) AS "total_damaged",
+{% else %}
+SUM("bricktracker_parts"."damaged") AS "total_damaged",
+{% endif %}
+{% endblock %}
+
+{% block total_quantity %}
+{% if owner_id and owner_id != 'all' %}
+SUM(CASE WHEN "bricktracker_set_owners"."owner_{{ owner_id }}" = 1 THEN "bricktracker_parts"."quantity" * IFNULL("bricktracker_minifigures"."quantity", 1) ELSE 0 END) AS "total_quantity",
+{% else %}
+SUM("bricktracker_parts"."quantity" * IFNULL("bricktracker_minifigures"."quantity", 1)) AS "total_quantity",
+{% endif %}
+{% endblock %}
+
+{% block total_sets %}
+{% if owner_id and owner_id != 'all' %}
+COUNT(DISTINCT CASE WHEN "bricktracker_set_owners"."owner_{{ owner_id }}" = 1 THEN "bricktracker_parts"."id" ELSE NULL END) AS "total_sets",
+{% else %}
+COUNT(DISTINCT "bricktracker_parts"."id") AS "total_sets",
+{% endif %}
+{% endblock %}
+
+{% block total_minifigures %}
+{% if owner_id and owner_id != 'all' %}
+SUM(CASE WHEN "bricktracker_set_owners"."owner_{{ owner_id }}" = 1 THEN IFNULL("bricktracker_minifigures"."quantity", 0) ELSE 0 END) AS "total_minifigures"
+{% else %}
+SUM(IFNULL("bricktracker_minifigures"."quantity", 0)) AS "total_minifigures"
+{% endif %}
+{% endblock %}
+
+{% block join %}
+-- Join with sets to get owner information
+INNER JOIN "bricktracker_sets"
+ON "bricktracker_parts"."id" IS NOT DISTINCT FROM "bricktracker_sets"."id"
+
+-- Left join with set owners (using dynamic columns)
+LEFT JOIN "bricktracker_set_owners"
+ON "bricktracker_sets"."id" IS NOT DISTINCT FROM "bricktracker_set_owners"."id"
+
+-- Left join with minifigures
+LEFT JOIN "bricktracker_minifigures"
+ON "bricktracker_parts"."id" IS NOT DISTINCT FROM "bricktracker_minifigures"."id"
+AND "bricktracker_parts"."figure" IS NOT DISTINCT FROM "bricktracker_minifigures"."figure"
+{% endblock %}
+
+{% block where %}
+{% set has_where = false %}
+{% if owner_id and owner_id != 'all' %}
+WHERE "bricktracker_set_owners"."owner_{{ owner_id }}" = 1
+{% set has_where = true %}
+{% endif %}
+{% if color_id and color_id != 'all' %}
+{% if has_where %}
+AND "bricktracker_parts"."color" = {{ color_id }}
+{% else %}
+WHERE "bricktracker_parts"."color" = {{ color_id }}
+{% endif %}
+{% endif %}
+{% endblock %}
+
+{% block group %}
+GROUP BY
+    "bricktracker_parts"."part",
+    "bricktracker_parts"."color",
+    "bricktracker_parts"."spare"
+{% endblock %}

bricktracker/sql/part/list/from_minifigure.sql

@@ -1,28 +1,21 @@
 
-{% extends 'part/base/select.sql' %}
+{% extends 'part/base/base.sql' %}
 
 {% block total_missing %}
-SUM(IFNULL(missing.quantity, 0)) AS total_missing,
+SUM("bricktracker_parts"."missing") AS "total_missing",
 {% endblock %}
 
-{% block join %}
-LEFT JOIN missing
-ON missing.set_num IS NOT DISTINCT FROM inventory.set_num
-AND missing.id IS NOT DISTINCT FROM inventory.id
-AND missing.part_num IS NOT DISTINCT FROM inventory.part_num
-AND missing.color_id IS NOT DISTINCT FROM inventory.color_id
-AND missing.element_id IS NOT DISTINCT FROM inventory.element_id
+{% block total_damaged %}
+SUM("bricktracker_parts"."damaged") AS "total_damaged",
 {% endblock %}
 
 {% block where %}
-WHERE inventory.set_num IS NOT DISTINCT FROM :set_num
+WHERE "bricktracker_parts"."figure" IS NOT DISTINCT FROM :figure
 {% endblock %}
 
 {% block group %}
 GROUP BY
-    inventory.part_num,
-    inventory.name,
-    inventory.color_id,
-    inventory.is_spare,
-    inventory.element_id
+    "bricktracker_parts"."part",
+    "bricktracker_parts"."color",
+    "bricktracker_parts"."spare"
 {% endblock %}

bricktracker/sql/part/list/from_print.sql

@@ -0,0 +1,18 @@
+
+{% extends 'part/base/base.sql' %}
+
+{% block total_missing %}{% endblock %}
+
+{% block total_damaged %}{% endblock %}
+
+{% block where %}
+WHERE "rebrickable_parts"."print" IS NOT DISTINCT FROM :print
+AND "bricktracker_parts"."color" IS NOT DISTINCT FROM :color
+AND "bricktracker_parts"."part" IS DISTINCT FROM :part
+{% endblock %}
+
+{% block group %}
+GROUP BY
+    "bricktracker_parts"."part",
+    "bricktracker_parts"."color"
+{% endblock %}

bricktracker/sql/part/list/from_set.sql

@@ -1,21 +0,0 @@
-
-{% extends 'part/base/select.sql' %}
-
-{% block total_missing %}
-IFNULL(missing.quantity, 0) AS total_missing,
-{% endblock %}
-
-{% block join %}
-LEFT JOIN missing
-ON inventory.set_num IS NOT DISTINCT FROM missing.set_num
-AND inventory.id IS NOT DISTINCT FROM missing.id
-AND inventory.part_num IS NOT DISTINCT FROM missing.part_num
-AND inventory.color_id IS NOT DISTINCT FROM missing.color_id
-AND inventory.element_id IS NOT DISTINCT FROM missing.element_id
-AND inventory.u_id IS NOT DISTINCT FROM missing.u_id
-{% endblock %}
-
-{% block where %}
-WHERE inventory.u_id IS NOT DISTINCT FROM :u_id
-AND inventory.set_num IS NOT DISTINCT FROM :set_num
-{% endblock %}

bricktracker/sql/part/list/missing.sql

@@ -1,36 +0,0 @@
-{% extends 'part/base/select.sql' %}
-
-{% block total_missing %}
-SUM(IFNULL(missing.quantity, 0)) AS total_missing,
-{% endblock %}
-
-{% block total_sets %}
-COUNT(inventory.u_id)  - COUNT(minifigures.u_id) AS total_sets,
-{% endblock %}
-
-{% block total_minifigures %}
-SUM(IFNULL(minifigures.quantity, 0)) AS total_minifigures
-{% endblock %}
-
-{% block join %}
-INNER JOIN missing
-ON missing.set_num IS NOT DISTINCT FROM inventory.set_num
-AND missing.id IS NOT DISTINCT FROM inventory.id
-AND missing.part_num IS NOT DISTINCT FROM inventory.part_num
-AND missing.color_id IS NOT DISTINCT FROM inventory.color_id
-AND missing.element_id IS NOT DISTINCT FROM inventory.element_id
-AND missing.u_id IS NOT DISTINCT FROM inventory.u_id
-
-LEFT JOIN minifigures
-ON missing.set_num IS NOT DISTINCT FROM minifigures.fig_num
-AND missing.u_id IS NOT DISTINCT FROM minifigures.u_id
-{% endblock %}
-
-{% block group %}
-GROUP BY
-    inventory.part_num,
-    inventory.name,
-    inventory.color_id,
-    inventory.is_spare,
-    inventory.element_id
-{% endblock %}

bricktracker/sql/part/list/problem.sql

@@ -0,0 +1,35 @@
+{% extends 'part/base/base.sql' %}
+
+{% block total_missing %}
+SUM("bricktracker_parts"."missing") AS "total_missing",
+{% endblock %}
+
+{% block total_damaged %}
+SUM("bricktracker_parts"."damaged") AS "total_damaged",
+{% endblock %}
+
+{% block total_sets %}
+IFNULL(COUNT("bricktracker_parts"."id"), 0)  - IFNULL(COUNT("bricktracker_parts"."figure"), 0) AS "total_sets",
+{% endblock %}
+
+{% block total_minifigures %}
+SUM(IFNULL("bricktracker_minifigures"."quantity", 0)) AS "total_minifigures"
+{% endblock %}
+
+{% block join %}
+LEFT JOIN "bricktracker_minifigures"
+ON "bricktracker_parts"."id" IS NOT DISTINCT FROM "bricktracker_minifigures"."id"
+AND "bricktracker_parts"."figure" IS NOT DISTINCT FROM "bricktracker_minifigures"."figure"
+{% endblock %}
+
+{% block where %}
+WHERE "bricktracker_parts"."missing" > 0
+OR "bricktracker_parts"."damaged" > 0
+{% endblock %}
+
+{% block group %}
+GROUP BY
+    "bricktracker_parts"."part",
+    "bricktracker_parts"."color",
+    "bricktracker_parts"."spare"
+{% endblock %}

bricktracker/sql/part/list/specific.sql

@@ -0,0 +1,15 @@
+
+{% extends 'part/base/base.sql' %}
+
+{% block total_missing %}
+IFNULL("bricktracker_parts"."missing", 0) AS "total_missing",
+{% endblock %}
+
+{% block total_damaged %}
+IFNULL("bricktracker_parts"."damaged", 0) AS "total_damaged",
+{% endblock %}
+
+{% block where %}
+WHERE "bricktracker_parts"."id" IS NOT DISTINCT FROM :id
+AND "bricktracker_parts"."figure" IS NOT DISTINCT FROM :figure
+{% endblock %}