feat(database): add individual minifigures and parts schema with migrations
This commit is contained in:
@@ -0,0 +1,88 @@
|
||||
"""
|
||||
Migration 0027: Consolidate metadata tables - remove FK constraints from set metadata tables
|
||||
|
||||
This migration removes foreign key constraints from bricktracker_set_owners, _tags, and _statuses
|
||||
so they can accept any entity ID (sets, individual parts, individual minifigures, individual part lots).
|
||||
|
||||
Since these tables have dynamically added columns, we need to read the schema and recreate the tables
|
||||
with all existing columns but without the foreign key constraints.
|
||||
"""
|
||||
import logging
|
||||
from typing import Any, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from ..sql import BrickSQL
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def migration_0027(sql: 'BrickSQL') -> dict[str, Any]:
|
||||
"""
|
||||
Remove foreign key constraints from set metadata junction tables.
|
||||
|
||||
This allows the tables to store metadata for any entity type, not just sets.
|
||||
"""
|
||||
|
||||
tables_to_migrate = [
|
||||
'bricktracker_set_owners',
|
||||
'bricktracker_set_tags',
|
||||
'bricktracker_set_statuses'
|
||||
]
|
||||
|
||||
for table_name in tables_to_migrate:
|
||||
logger.info('Migrating {table_name} to remove foreign key constraint'.format(
|
||||
table_name=table_name
|
||||
))
|
||||
|
||||
# Get the current table schema
|
||||
cursor = sql.cursor.execute(f"PRAGMA table_info({table_name})")
|
||||
columns = cursor.fetchall()
|
||||
|
||||
# Build column definitions for new table (without FK constraint)
|
||||
column_defs = []
|
||||
column_names = []
|
||||
for col in columns:
|
||||
col_name = col[1]
|
||||
col_type = col[2]
|
||||
col_not_null = col[3]
|
||||
col_default = col[4]
|
||||
col_pk = col[5]
|
||||
|
||||
column_names.append(f'"{col_name}"')
|
||||
|
||||
col_def = f'"{col_name}" {col_type}'
|
||||
if col_pk:
|
||||
col_def += ' PRIMARY KEY'
|
||||
if col_not_null and not col_pk:
|
||||
if col_default is not None:
|
||||
col_def += f' NOT NULL DEFAULT {col_default}'
|
||||
else:
|
||||
col_def += ' NOT NULL'
|
||||
elif col_default is not None:
|
||||
col_def += f' DEFAULT {col_default}'
|
||||
|
||||
column_defs.append(col_def)
|
||||
|
||||
# Create new table without foreign key constraint
|
||||
new_table_name = f'{table_name}_new'
|
||||
create_sql = f'CREATE TABLE "{new_table_name}" ({", ".join(column_defs)})'
|
||||
logger.debug('Creating new table: {sql}'.format(sql=create_sql))
|
||||
sql.cursor.execute(create_sql)
|
||||
|
||||
# Copy all data
|
||||
column_list = ', '.join(column_names)
|
||||
copy_sql = f'INSERT INTO "{new_table_name}" ({column_list}) SELECT {column_list} FROM "{table_name}"'
|
||||
logger.debug('Copying data: {sql}'.format(sql=copy_sql))
|
||||
sql.cursor.execute(copy_sql)
|
||||
|
||||
# Drop old table
|
||||
sql.cursor.execute(f'DROP TABLE "{table_name}"')
|
||||
|
||||
# Rename new table to old name
|
||||
sql.cursor.execute(f'ALTER TABLE "{new_table_name}" RENAME TO "{table_name}"')
|
||||
|
||||
logger.info('Successfully migrated {table_name}'.format(table_name=table_name))
|
||||
|
||||
logger.info('Migration 0027 complete - all set metadata tables now accept any entity ID')
|
||||
|
||||
return {}
|
||||
@@ -0,0 +1,88 @@
|
||||
-- description: Add individual minifigures and individual parts tables
|
||||
|
||||
-- Individual minifigures table - tracks individual minifigures not associated with sets
|
||||
CREATE TABLE IF NOT EXISTS "bricktracker_individual_minifigures" (
|
||||
"id" TEXT NOT NULL,
|
||||
"figure" TEXT NOT NULL,
|
||||
"quantity" INTEGER NOT NULL DEFAULT 1,
|
||||
"description" TEXT,
|
||||
"storage" TEXT, -- Storage bin location
|
||||
"purchase_date" REAL, -- Purchase date
|
||||
"purchase_location" TEXT, -- Purchase location
|
||||
"purchase_price" REAL, -- Purchase price
|
||||
PRIMARY KEY("id"),
|
||||
FOREIGN KEY("figure") REFERENCES "rebrickable_minifigures"("figure"),
|
||||
FOREIGN KEY("storage") REFERENCES "bricktracker_metadata_storages"("id"),
|
||||
FOREIGN KEY("purchase_location") REFERENCES "bricktracker_metadata_purchase_locations"("id")
|
||||
);
|
||||
|
||||
-- Metadata for individual minifigures: use bricktracker_set_owners, bricktracker_set_tags, bricktracker_set_statuses tables
|
||||
|
||||
-- Parts table for individual minifigures - tracks constituent parts
|
||||
CREATE TABLE IF NOT EXISTS "bricktracker_individual_minifigure_parts" (
|
||||
"id" TEXT NOT NULL,
|
||||
"part" TEXT NOT NULL,
|
||||
"color" INTEGER NOT NULL,
|
||||
"spare" BOOLEAN NOT NULL,
|
||||
"quantity" INTEGER NOT NULL,
|
||||
"element" INTEGER,
|
||||
"rebrickable_inventory" INTEGER NOT NULL,
|
||||
"missing" INTEGER NOT NULL DEFAULT 0,
|
||||
"damaged" INTEGER NOT NULL DEFAULT 0,
|
||||
"checked" BOOLEAN DEFAULT 0,
|
||||
PRIMARY KEY("id", "part", "color", "spare"),
|
||||
FOREIGN KEY("id") REFERENCES "bricktracker_individual_minifigures"("id"),
|
||||
FOREIGN KEY("part", "color") REFERENCES "rebrickable_parts"("part", "color_id")
|
||||
);
|
||||
|
||||
-- Individual parts table - tracks individual parts not associated with sets
|
||||
CREATE TABLE IF NOT EXISTS "bricktracker_individual_parts" (
|
||||
"id" TEXT NOT NULL,
|
||||
"part" TEXT NOT NULL,
|
||||
"color" INTEGER NOT NULL,
|
||||
"quantity" INTEGER NOT NULL DEFAULT 1,
|
||||
"description" TEXT,
|
||||
"storage" TEXT, -- Storage bin location
|
||||
"purchase_date" REAL, -- Purchase date
|
||||
"purchase_location" TEXT, -- Purchase location
|
||||
"purchase_price" REAL, -- Purchase price
|
||||
PRIMARY KEY("id"),
|
||||
FOREIGN KEY("part", "color") REFERENCES "rebrickable_parts"("part", "color_id"),
|
||||
FOREIGN KEY("storage") REFERENCES "bricktracker_metadata_storages"("id"),
|
||||
FOREIGN KEY("purchase_location") REFERENCES "bricktracker_metadata_purchase_locations"("id")
|
||||
);
|
||||
|
||||
-- Metadata for individual parts: use bricktracker_set_owners, bricktracker_set_tags, bricktracker_set_statuses tables
|
||||
|
||||
-- Indexes for individual minifigures
|
||||
CREATE INDEX IF NOT EXISTS idx_bricktracker_individual_minifigures_figure
|
||||
ON bricktracker_individual_minifigures(figure);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bricktracker_individual_minifigures_storage
|
||||
ON bricktracker_individual_minifigures(storage);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bricktracker_individual_minifigures_purchase_location
|
||||
ON bricktracker_individual_minifigures(purchase_location);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bricktracker_individual_minifigures_purchase_date
|
||||
ON bricktracker_individual_minifigures(purchase_date);
|
||||
|
||||
-- Indexes for individual minifigure parts
|
||||
CREATE INDEX IF NOT EXISTS idx_bricktracker_individual_minifigure_parts_id_missing_damaged
|
||||
ON bricktracker_individual_minifigure_parts(id, missing, damaged);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bricktracker_individual_minifigure_parts_part_color
|
||||
ON bricktracker_individual_minifigure_parts(part, color);
|
||||
|
||||
-- Indexes for individual parts
|
||||
CREATE INDEX IF NOT EXISTS idx_bricktracker_individual_parts_part_color
|
||||
ON bricktracker_individual_parts(part, color);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bricktracker_individual_parts_storage
|
||||
ON bricktracker_individual_parts(storage);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bricktracker_individual_parts_purchase_location
|
||||
ON bricktracker_individual_parts(purchase_location);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bricktracker_individual_parts_purchase_date
|
||||
ON bricktracker_individual_parts(purchase_date);
|
||||
@@ -0,0 +1,91 @@
|
||||
-- description: Add individual part lots system for bulk/cart adding of parts
|
||||
|
||||
BEGIN TRANSACTION;
|
||||
|
||||
-- Create individual part lots table
|
||||
CREATE TABLE IF NOT EXISTS "bricktracker_individual_part_lots" (
|
||||
"id" TEXT NOT NULL PRIMARY KEY,
|
||||
"name" TEXT,
|
||||
"description" TEXT,
|
||||
"created_date" REAL NOT NULL,
|
||||
"storage" TEXT,
|
||||
"purchase_location" TEXT,
|
||||
"purchase_date" REAL,
|
||||
"purchase_price" REAL,
|
||||
FOREIGN KEY("storage") REFERENCES "bricktracker_metadata_storages"("id") ON DELETE SET NULL,
|
||||
FOREIGN KEY("purchase_location") REFERENCES "bricktracker_metadata_purchase_locations"("id") ON DELETE SET NULL
|
||||
);
|
||||
|
||||
-- Create index for faster lookups
|
||||
CREATE INDEX IF NOT EXISTS "idx_individual_part_lots_created_date"
|
||||
ON "bricktracker_individual_part_lots"("created_date");
|
||||
|
||||
-- Add missing/damaged/checked fields to individual parts table
|
||||
ALTER TABLE "bricktracker_individual_parts"
|
||||
ADD COLUMN "missing" INTEGER NOT NULL DEFAULT 0;
|
||||
|
||||
ALTER TABLE "bricktracker_individual_parts"
|
||||
ADD COLUMN "damaged" INTEGER NOT NULL DEFAULT 0;
|
||||
|
||||
ALTER TABLE "bricktracker_individual_parts"
|
||||
ADD COLUMN "checked" BOOLEAN NOT NULL DEFAULT 0;
|
||||
|
||||
-- Add lot_id column to individual parts table with foreign key constraint
|
||||
-- Note: SQLite doesn't support ALTER TABLE ADD CONSTRAINT for FK, so we need to recreate the table
|
||||
|
||||
-- Create new table with FK constraint
|
||||
CREATE TABLE "bricktracker_individual_parts_new" (
|
||||
"id" TEXT NOT NULL,
|
||||
"part" TEXT NOT NULL,
|
||||
"color" INTEGER NOT NULL,
|
||||
"quantity" INTEGER NOT NULL DEFAULT 1,
|
||||
"description" TEXT,
|
||||
"storage" TEXT,
|
||||
"purchase_date" REAL,
|
||||
"purchase_location" TEXT,
|
||||
"purchase_price" REAL,
|
||||
"missing" INTEGER NOT NULL DEFAULT 0,
|
||||
"damaged" INTEGER NOT NULL DEFAULT 0,
|
||||
"checked" BOOLEAN NOT NULL DEFAULT 0,
|
||||
"lot_id" TEXT,
|
||||
PRIMARY KEY("id"),
|
||||
FOREIGN KEY("part", "color") REFERENCES "rebrickable_parts"("part", "color_id"),
|
||||
FOREIGN KEY("storage") REFERENCES "bricktracker_metadata_storages"("id"),
|
||||
FOREIGN KEY("purchase_location") REFERENCES "bricktracker_metadata_purchase_locations"("id"),
|
||||
FOREIGN KEY("lot_id") REFERENCES "bricktracker_individual_part_lots"("id") ON DELETE SET NULL
|
||||
);
|
||||
|
||||
-- Copy existing data (set lot_id to NULL for all existing parts)
|
||||
INSERT INTO "bricktracker_individual_parts_new"
|
||||
(id, part, color, quantity, description, storage, purchase_date, purchase_location, purchase_price, missing, damaged, checked, lot_id)
|
||||
SELECT
|
||||
id, part, color, quantity, description, storage, purchase_date, purchase_location, purchase_price, missing, damaged, checked, NULL
|
||||
FROM "bricktracker_individual_parts";
|
||||
|
||||
-- Drop old table
|
||||
DROP TABLE "bricktracker_individual_parts";
|
||||
|
||||
-- Rename new table
|
||||
ALTER TABLE "bricktracker_individual_parts_new" RENAME TO "bricktracker_individual_parts";
|
||||
|
||||
-- Recreate existing indexes
|
||||
CREATE INDEX IF NOT EXISTS idx_bricktracker_individual_parts_part_color
|
||||
ON bricktracker_individual_parts(part, color);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bricktracker_individual_parts_storage
|
||||
ON bricktracker_individual_parts(storage);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bricktracker_individual_parts_purchase_location
|
||||
ON bricktracker_individual_parts(purchase_location);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_bricktracker_individual_parts_purchase_date
|
||||
ON bricktracker_individual_parts(purchase_date);
|
||||
|
||||
-- Create lot_id index
|
||||
CREATE INDEX IF NOT EXISTS "idx_individual_parts_lot_id"
|
||||
ON "bricktracker_individual_parts"("lot_id");
|
||||
|
||||
-- Metadata for individual part lots: use bricktracker_set_owners and bricktracker_set_tags tables
|
||||
-- Note: Part lots don't have statuses, only owners and tags
|
||||
|
||||
COMMIT;
|
||||
@@ -0,0 +1,13 @@
|
||||
-- description: Add missing indexes for individual part lots optimization
|
||||
|
||||
BEGIN TRANSACTION;
|
||||
|
||||
-- Add storage index for lots table (for filtering by storage)
|
||||
CREATE INDEX IF NOT EXISTS "idx_individual_part_lots_storage"
|
||||
ON "bricktracker_individual_part_lots"("storage");
|
||||
|
||||
-- Add purchase location index for lots table (for filtering by purchase location)
|
||||
CREATE INDEX IF NOT EXISTS "idx_individual_part_lots_purchase_location"
|
||||
ON "bricktracker_individual_part_lots"("purchase_location");
|
||||
|
||||
COMMIT;
|
||||
@@ -0,0 +1,16 @@
|
||||
-- description: Create rebrickable_colors translation table for BrickLink color ID mapping
|
||||
-- This table caches color information from Rebrickable API to avoid repeated API calls
|
||||
-- and provides mapping between Rebrickable and BrickLink color IDs
|
||||
|
||||
CREATE TABLE IF NOT EXISTS "rebrickable_colors" (
|
||||
"color_id" INTEGER PRIMARY KEY,
|
||||
"name" TEXT NOT NULL,
|
||||
"rgb" TEXT,
|
||||
"is_trans" BOOLEAN,
|
||||
"bricklink_color_id" INTEGER,
|
||||
"bricklink_color_name" TEXT
|
||||
);
|
||||
|
||||
-- Create index for faster lookups
|
||||
CREATE INDEX IF NOT EXISTS "idx_rebrickable_colors_bricklink"
|
||||
ON "rebrickable_colors"("bricklink_color_id");
|
||||
@@ -0,0 +1,17 @@
|
||||
-- description: Add performance indexes for individual parts and minifigure parts
|
||||
|
||||
BEGIN TRANSACTION;
|
||||
|
||||
-- Composite index for lot part listing (common query: list parts in a lot)
|
||||
CREATE INDEX IF NOT EXISTS idx_individual_parts_lot_id_part_color
|
||||
ON bricktracker_individual_parts(lot_id, part, color);
|
||||
|
||||
-- Problem tracking index for individual parts (common query: find parts with problems)
|
||||
CREATE INDEX IF NOT EXISTS idx_individual_parts_missing_damaged
|
||||
ON bricktracker_individual_parts(missing, damaged);
|
||||
|
||||
-- Checked state index for individual minifigure parts (common query: find unchecked parts)
|
||||
CREATE INDEX IF NOT EXISTS idx_individual_minifigure_parts_checked
|
||||
ON bricktracker_individual_minifigure_parts(id, checked);
|
||||
|
||||
COMMIT;
|
||||
@@ -0,0 +1,41 @@
|
||||
-- description: Standardize ON DELETE behavior for foreign keys (use RESTRICT everywhere)
|
||||
|
||||
BEGIN TRANSACTION;
|
||||
|
||||
-- Recreate bricktracker_individual_part_lots without ON DELETE SET NULL
|
||||
-- This makes FK behavior consistent: prevent deletion of metadata if referenced
|
||||
|
||||
CREATE TABLE "bricktracker_individual_part_lots_new" (
|
||||
"id" TEXT NOT NULL PRIMARY KEY,
|
||||
"name" TEXT,
|
||||
"description" TEXT,
|
||||
"created_date" REAL NOT NULL,
|
||||
"storage" TEXT,
|
||||
"purchase_location" TEXT,
|
||||
"purchase_date" REAL,
|
||||
"purchase_price" REAL,
|
||||
FOREIGN KEY("storage") REFERENCES "bricktracker_metadata_storages"("id"),
|
||||
FOREIGN KEY("purchase_location") REFERENCES "bricktracker_metadata_purchase_locations"("id")
|
||||
);
|
||||
|
||||
-- Copy existing data
|
||||
INSERT INTO "bricktracker_individual_part_lots_new"
|
||||
SELECT * FROM "bricktracker_individual_part_lots";
|
||||
|
||||
-- Drop old table
|
||||
DROP TABLE "bricktracker_individual_part_lots";
|
||||
|
||||
-- Rename new table
|
||||
ALTER TABLE "bricktracker_individual_part_lots_new" RENAME TO "bricktracker_individual_part_lots";
|
||||
|
||||
-- Recreate indexes
|
||||
CREATE INDEX IF NOT EXISTS "idx_individual_part_lots_created_date"
|
||||
ON "bricktracker_individual_part_lots"("created_date");
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "idx_individual_part_lots_storage"
|
||||
ON "bricktracker_individual_part_lots"("storage");
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "idx_individual_part_lots_purchase_location"
|
||||
ON "bricktracker_individual_part_lots"("purchase_location");
|
||||
|
||||
COMMIT;
|
||||
@@ -0,0 +1,6 @@
|
||||
-- description: Remove foreign key constraints from consolidated metadata tables
|
||||
|
||||
-- This migration is implemented entirely in Python (see migrations/0027.py)
|
||||
-- The Python code dynamically recreates bricktracker_set_owners, bricktracker_set_tags,
|
||||
-- and bricktracker_set_statuses without foreign key constraints so they can accept
|
||||
-- UUIDs from any entity type (sets, individual parts, individual minifigures, part lots)
|
||||
Reference in New Issue
Block a user