diff --git a/.dockerignore b/.dockerignore index 82afab6..fa88ec8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,6 +19,8 @@ LICENSE # Database *.db +*.db-shm +*.db-wal # Python **/__pycache__ diff --git a/.gitignore b/.gitignore index 0d192d5..cd54740 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # Application .env *.db +*.db-shm +*.db-wal # Python specifics __pycache__/ diff --git a/bricktracker/sql.py b/bricktracker/sql.py index e594191..eb368c9 100644 --- a/bricktracker/sql.py +++ b/bricktracker/sql.py @@ -3,13 +3,15 @@ import os import sqlite3 from typing import Any, Final, Tuple -from .sql_stats import BrickSQLStats - from flask import current_app, g from jinja2 import Environment, FileSystemLoader from werkzeug.datastructures import FileStorage +from .exceptions import DatabaseException from .sql_counter import BrickCounter +from .sql_migration_list import BrickSQLMigrationList +from .sql_stats import BrickSQLStats +from .version import __database_version__ logger = logging.getLogger(__name__) @@ -26,8 +28,9 @@ class BrickSQL(object): connection: sqlite3.Connection cursor: sqlite3.Cursor stats: BrickSQLStats + version: int - def __init__(self, /): + def __init__(self, /, failsafe: bool = False): # Instantiate the database connection in the Flask # application context so that it can be used by all # requests without re-opening connections @@ -37,6 +40,9 @@ class BrickSQL(object): if database is not None: self.connection = database self.stats = getattr(g, 'database_stats', BrickSQLStats()) + + # Grab a cursor + self.cursor = self.connection.cursor() else: # Instantiate the stats self.stats = BrickSQLStats() @@ -52,16 +58,38 @@ class BrickSQL(object): # Setup the row factory to get pseudo-dicts rather than tuples self.connection.row_factory = sqlite3.Row - # Debug: Attach the debugger - # Uncomment manually because this is ultra verbose - # self.connection.set_trace_callback(print) + # Grab a cursor + self.cursor = self.connection.cursor() - # Save the connection globally for later use - g.database = self.connection - g.database_stats = self.stats + # Grab the version and check + try: + version = self.fetchone('schema/get_version') - # Grab a cursor - self.cursor = self.connection.cursor() + if version is None: + raise Exception('version is None') + + self.version = version[0] + except Exception as e: + self.version = 0 + + raise DatabaseException('Could not get the database version: {error}'.format( # noqa: E501 + error=str(e) + )) + + if not failsafe: + if self.needs_upgrade(): + raise DatabaseException('Your database need to be upgraded from version {version} to version {required}'.format( # noqa: E501 + version=self.version, + required=__database_version__, + )) + + # Debug: Attach the debugger + # Uncomment manually because this is ultra verbose + # self.connection.set_trace_callback(print) + + # Save the connection globally for later use + g.database = self.connection + g.database_stats = self.stats # Clear the defer stack def clear_defer(self, /) -> None: @@ -236,11 +264,16 @@ class BrickSQL(object): return template.render(**context) + # Tells whether the database needs upgrade + def needs_upgrade(self) -> bool: + return self.version < __database_version__ + # Raw execute the query without any options def raw_execute( self, query: str, - parameters: dict[str, Any] + parameters: dict[str, Any], + /, ) -> sqlite3.Cursor: logger.debug('SQLite3: execute: {query}'.format( query=BrickSQL.clean_query(query) @@ -248,6 +281,17 @@ class BrickSQL(object): return self.cursor.execute(query, parameters) + # Upgrade the database + def upgrade(self) -> None: + if self.needs_upgrade(): + for pending in BrickSQLMigrationList().pending(self.version): + logger.info('Applying migration {version}'.format( + version=pending.version) + ) + + self.executescript(pending.get_query()) + self.execute('schema/set_version', version=pending.version) + # Clean the query for debugging @staticmethod def clean_query(query: str, /) -> str: @@ -289,11 +333,6 @@ class BrickSQL(object): # Info logger.info('The database has been initialized') - # Check if the database is initialized - @staticmethod - def is_init() -> bool: - return BrickSQL().fetchone('schema/is_init') is not None - # Replace the database with a new file @staticmethod def upload(file: FileStorage, /) -> None: diff --git a/bricktracker/sql/migrations/init.sql b/bricktracker/sql/migrations/0001.sql similarity index 91% rename from bricktracker/sql/migrations/init.sql rename to bricktracker/sql/migrations/0001.sql index 85defbb..846eddc 100644 --- a/bricktracker/sql/migrations/init.sql +++ b/bricktracker/sql/migrations/0001.sql @@ -1,3 +1,4 @@ +-- description: Database initialization -- FROM sqlite3 app.db .schema > init.sql with extra IF NOT EXISTS and transaction BEGIN transaction; @@ -58,9 +59,4 @@ CREATE TABLE IF NOT EXISTS missing ( u_id TEXT ); --- Fix a bug where 'None' was inserted in missing instead of NULL -UPDATE missing -SET element_id = NULL -WHERE element_id = 'None'; - COMMIT; \ No newline at end of file diff --git a/bricktracker/sql/migrations/0002.sql b/bricktracker/sql/migrations/0002.sql new file mode 100644 index 0000000..5dc9ed4 --- /dev/null +++ b/bricktracker/sql/migrations/0002.sql @@ -0,0 +1,13 @@ +-- description: WAL journal, 'None' fix for missing table + +-- Set the journal mode to WAL +PRAGMA journal_mode = WAL; + +BEGIN transaction; + +-- Fix a bug where 'None' was inserted in missing instead of NULL +UPDATE missing +SET element_id = NULL +WHERE element_id = 'None'; + +COMMIT; \ No newline at end of file diff --git a/bricktracker/sql/schema/get_version.sql b/bricktracker/sql/schema/get_version.sql new file mode 100644 index 0000000..693601a --- /dev/null +++ b/bricktracker/sql/schema/get_version.sql @@ -0,0 +1 @@ +PRAGMA user_version diff --git a/bricktracker/sql/schema/is_init.sql b/bricktracker/sql/schema/is_init.sql deleted file mode 100644 index 17a3a81..0000000 --- a/bricktracker/sql/schema/is_init.sql +++ /dev/null @@ -1,4 +0,0 @@ -SELECT name -FROM sqlite_master -WHERE type="table" -AND name="sets" diff --git a/bricktracker/sql/schema/set_version.sql b/bricktracker/sql/schema/set_version.sql new file mode 100644 index 0000000..53d0eb2 --- /dev/null +++ b/bricktracker/sql/schema/set_version.sql @@ -0,0 +1 @@ +PRAGMA user_version = {{ version }} diff --git a/bricktracker/sql_migration.py b/bricktracker/sql_migration.py new file mode 100644 index 0000000..0b6acc4 --- /dev/null +++ b/bricktracker/sql_migration.py @@ -0,0 +1,54 @@ +import os + +from .version import __database_version__ + + +class BrickSQLMigration(object): + description: str | None + name: str + file: str + version: int + + # Description marker + description_marker: str = '-- description:' + + def __init__(self, file: str): + self.file = file + self.name, _ = os.path.splitext(os.path.basename(self.file)) + self.version = int(self.name) + + self.description = None + + # Read the description from the migration file if it exists + def get_description(self) -> str: + if self.description is None: + self.description = '' + + # First line or ignored + with open(self.file, 'r') as file: + line = file.readline() + + # Extract a description (only the first one) + if line.startswith(self.description_marker): + self.description = line.strip()[ + len(self.description_marker): + ] + + return self.description + + # Tells whether the migration is need + def is_needed(self, current: int, /): + return self.version > current and self.version <= __database_version__ + + # Query name for the SQL loader + def get_query(self) -> str: + relative, _ = os.path.splitext( + os.path.relpath(self.file, os.path.join( + os.path.dirname(__file__), + 'sql/' + )) + ) + + print(relative) + + return relative diff --git a/bricktracker/sql_migration_list.py b/bricktracker/sql_migration_list.py new file mode 100644 index 0000000..56cbc45 --- /dev/null +++ b/bricktracker/sql_migration_list.py @@ -0,0 +1,58 @@ +from glob import glob +import logging +import os + +from .sql_migration import BrickSQLMigration + +logger = logging.getLogger(__name__) + + +class BrickSQLMigrationList(object): + migrations: list[BrickSQLMigration] + + def __init__(self): + # Load the migrations only there is none already loaded + migrations = getattr(self, 'migrations', None) + + if migrations is None: + logger.info('Loading SQL migrations list') + + BrickSQLMigrationList.migrations = [] + + path: str = os.path.join( + os.path.dirname(__file__), + 'sql/migrations/*.sql' + ) + + print(path) + + files = glob(path) + + print(files) + + for file in files: + try: + BrickSQLMigrationList.migrations.append( + BrickSQLMigration(file) + ) + # Ignore file if error + except Exception as e: + print(e) + pass + + # Get the sorted list of pending migrations + def pending( + self, + current: int, + / + ) -> list[BrickSQLMigration]: + pending: list[BrickSQLMigration] = [] + + for migration in self.migrations: + if migration.is_needed(current): + pending.append(migration) + + # Sort the list + pending.sort(key=lambda e: e.version) + + return pending diff --git a/bricktracker/version.py b/bricktracker/version.py index 1f356cc..cc0dac8 100644 --- a/bricktracker/version.py +++ b/bricktracker/version.py @@ -1 +1,4 @@ -__version__ = '1.0.0' +from typing import Final + +__version__: Final[str] = '1.0.0' +__database_version__: Final[int] = 2 diff --git a/bricktracker/views/admin.py b/bricktracker/views/admin.py index 86a126b..b97e0fa 100644 --- a/bricktracker/views/admin.py +++ b/bricktracker/views/admin.py @@ -24,6 +24,7 @@ from ..rebrickable_image import RebrickableImage from ..retired_list import BrickRetiredList from ..set import BrickSet from ..sql_counter import BrickCounter +from ..sql_migration_list import BrickSQLMigrationList from ..sql import BrickSQL from ..theme_list import BrickThemeList from .upload import upload_helper @@ -38,10 +39,10 @@ admin_page = Blueprint('admin', __name__, url_prefix='/admin') @login_required @exception_handler(__file__) def admin() -> str: - counters: dict[str, int] = {} - count_none: int = 0 - exception: Exception | None = None - is_init: bool = False + database_counters: list[BrickCounter] = [] + database_exception: Exception | None = None + database_needs_upgrade: bool = False + database_version: int = -1 nil_minifigure_name: str = '' nil_minifigure_url: str = '' nil_part_name: str = '' @@ -49,35 +50,32 @@ def admin() -> str: # This view needs to be protected against SQL errors try: - is_init = BrickSQL.is_init() - - if is_init: - counters = BrickSQL().count_records() - - record = BrickSQL().fetchone('missing/count_none') - if record is not None: - count_none = record['count'] - - nil_minifigure_name = RebrickableImage.nil_minifigure_name() - nil_minifigure_url = RebrickableImage.static_url( - nil_minifigure_name, - 'MINIFIGURES_FOLDER' - ) - - nil_part_name = RebrickableImage.nil_name() - nil_part_url = RebrickableImage.static_url( - nil_part_name, - 'PARTS_FOLDER' - ) + database = BrickSQL(failsafe=True) + database_needs_upgrade = database.needs_upgrade() + database_version = database.version + if not database_needs_upgrade: + database_counters = BrickSQL().count_records() except Exception as e: - exception = e + database_exception = e # Warning - logger.warning('An exception occured while loading the admin page: {exception}'.format( # noqa: E501 + logger.warning('A database exception occured while loading the admin page: {exception}'.format( # noqa: E501 exception=str(e), )) + nil_minifigure_name = RebrickableImage.nil_minifigure_name() + nil_minifigure_url = RebrickableImage.static_url( + nil_minifigure_name, + 'MINIFIGURES_FOLDER' + ) + + nil_part_name = RebrickableImage.nil_name() + nil_part_url = RebrickableImage.static_url( + nil_part_name, + 'PARTS_FOLDER' + ) + open_image = request.args.get('open_image', None) open_instructions = request.args.get('open_instructions', None) open_logout = request.args.get('open_logout', None) @@ -95,12 +93,12 @@ def admin() -> str: return render_template( 'admin.html', configuration=BrickConfigurationList.list(), - counters=counters, - count_none=count_none, - error=request.args.get('error'), - exception=exception, + database_counters=database_counters, + database_error=request.args.get('error'), + database_exception=database_exception, + database_needs_upgrade=database_needs_upgrade, + database_version=database_version, instructions=BrickInstructionsList(), - is_init=is_init, nil_minifigure_name=nil_minifigure_name, nil_minifigure_url=nil_minifigure_url, nil_part_name=nil_part_name, @@ -116,19 +114,6 @@ def admin() -> str: ) -# Initialize the database -@admin_page.route('/init-database', methods=['POST']) -@login_required -@exception_handler(__file__, post_redirect='admin.admin') -def init_database() -> Response: - BrickSQL.initialize() - - # Reload the instructions - BrickInstructionsList(force=True) - - return redirect(url_for('admin.admin')) - - # Delete the database @admin_page.route('/delete-database', methods=['GET']) @login_required @@ -205,6 +190,16 @@ def do_drop_database() -> Response: return redirect(url_for('admin.admin')) +# Actually upgrade the database +@admin_page.route('/upgrade-database', methods=['POST']) +@login_required +@exception_handler(__file__, post_redirect='admin.upgrade_database') +def do_upgrade_database() -> Response: + BrickSQL(failsafe=True).upgrade() + + return redirect(url_for('admin.admin')) + + # Import a database @admin_page.route('/import-database', methods=['GET']) @login_required @@ -316,3 +311,20 @@ def update_themes() -> Response: BrickThemeList(force=True) return redirect(url_for('admin.admin', open_theme=True)) + + +# Upgrade the database +@admin_page.route('/upgrade-database', methods=['GET']) +@login_required +@exception_handler(__file__, post_redirect='admin.admin') +def upgrade_database() -> str: + database = BrickSQL(failsafe=True) + + return render_template( + 'admin.html', + upgrade_database=True, + migrations=BrickSQLMigrationList().pending( + database.version + ), + error=request.args.get('error') + ) diff --git a/templates/admin.html b/templates/admin.html index e425887..d54efc5 100644 --- a/templates/admin.html +++ b/templates/admin.html @@ -18,6 +18,8 @@ {% include 'admin/database/drop.html' %} {% elif import_database %} {% include 'admin/database/import.html' %} + {% elif upgrade_database %} + {% include 'admin/database/upgrade.html' %} {% else %} {% include 'admin/logout.html' %} {% include 'admin/instructions.html' %} diff --git a/templates/admin/database.html b/templates/admin/database.html index ec64cae..f92374a 100644 --- a/templates/admin/database.html +++ b/templates/admin/database.html @@ -2,41 +2,30 @@ {{ accordion.header('Database', 'database', 'admin', expanded=open_database, icon='database-2-line') }}
Status
-{% if exception %}{% endif %} -{% if error %}{% endif %} -{% if not is_init %} +{% if database_exception %}{% endif %} +{% if database_error %}{% endif %} +{% if database_needs_upgrade %} -{% else %} - {% if count_none %} - +{% endif %} +

The database file is: {{ config['DATABASE_PATH'] }} at version {{ database_version }}

+

+ Download the database file +

+{% if database_counters %}
Records
- -