Incremental forward upgrades of the database

.dockerignore

@@ -19,6 +19,8 @@ LICENSE
 
 # Database
 *.db
+*.db-shm
+*.db-wal
 
 # Python
 **/__pycache__

.gitignore

@@ -1,6 +1,8 @@
 # Application
 .env
 *.db
+*.db-shm
+*.db-wal
 
 # Python specifics
 __pycache__/

bricktracker/sql.py

@@ -3,13 +3,15 @@ import os
 import sqlite3
 from typing import Any, Final, Tuple
 
-from .sql_stats import BrickSQLStats
-
 from flask import current_app, g
 from jinja2 import Environment, FileSystemLoader
 from werkzeug.datastructures import FileStorage
 
+from .exceptions import DatabaseException
 from .sql_counter import BrickCounter
+from .sql_migration_list import BrickSQLMigrationList
+from .sql_stats import BrickSQLStats
+from .version import __database_version__
 
 logger = logging.getLogger(__name__)
 
@@ -26,8 +28,9 @@ class BrickSQL(object):
     connection: sqlite3.Connection
     cursor: sqlite3.Cursor
     stats: BrickSQLStats
+    version: int
 
-    def __init__(self, /):
+    def __init__(self, /, failsafe: bool = False):
         # Instantiate the database connection in the Flask
         # application context so that it can be used by all
         # requests without re-opening connections
@@ -37,6 +40,9 @@ class BrickSQL(object):
         if database is not None:
             self.connection = database
             self.stats = getattr(g, 'database_stats', BrickSQLStats())
+
+            # Grab a cursor
+            self.cursor = self.connection.cursor()
         else:
             # Instantiate the stats
             self.stats = BrickSQLStats()
@@ -52,6 +58,31 @@ class BrickSQL(object):
             # Setup the row factory to get pseudo-dicts rather than tuples
             self.connection.row_factory = sqlite3.Row
 
+            # Grab a cursor
+            self.cursor = self.connection.cursor()
+
+            # Grab the version and check
+            try:
+                version = self.fetchone('schema/get_version')
+
+                if version is None:
+                    raise Exception('version is None')
+
+                self.version = version[0]
+            except Exception as e:
+                self.version = 0
+
+                raise DatabaseException('Could not get the database version: {error}'.format(  # noqa: E501
+                    error=str(e)
+                ))
+
+            if not failsafe:
+                if self.needs_upgrade():
+                    raise DatabaseException('Your database need to be upgraded from version {version} to version {required}'.format(  # noqa: E501
+                        version=self.version,
+                        required=__database_version__,
+                    ))
+
                 # Debug: Attach the debugger
                 # Uncomment manually because this is ultra verbose
                 # self.connection.set_trace_callback(print)
@@ -60,9 +91,6 @@ class BrickSQL(object):
                 g.database = self.connection
                 g.database_stats = self.stats
 
-        # Grab a cursor
-        self.cursor = self.connection.cursor()
-
     # Clear the defer stack
     def clear_defer(self, /) -> None:
         g.database_defer = []
@@ -236,11 +264,16 @@ class BrickSQL(object):
 
         return template.render(**context)
 
+    # Tells whether the database needs upgrade
+    def needs_upgrade(self) -> bool:
+        return self.version < __database_version__
+
     # Raw execute the query without any options
     def raw_execute(
         self,
         query: str,
-        parameters: dict[str, Any]
+        parameters: dict[str, Any],
+        /,
     ) -> sqlite3.Cursor:
         logger.debug('SQLite3: execute: {query}'.format(
             query=BrickSQL.clean_query(query)
@@ -248,6 +281,17 @@ class BrickSQL(object):
 
         return self.cursor.execute(query, parameters)
 
+    # Upgrade the database
+    def upgrade(self) -> None:
+        if self.needs_upgrade():
+            for pending in BrickSQLMigrationList().pending(self.version):
+                logger.info('Applying migration {version}'.format(
+                    version=pending.version)
+                )
+
+                self.executescript(pending.get_query())
+                self.execute('schema/set_version', version=pending.version)
+
     # Clean the query for debugging
     @staticmethod
     def clean_query(query: str, /) -> str:
@@ -289,11 +333,6 @@ class BrickSQL(object):
         # Info
         logger.info('The database has been initialized')
 
-    # Check if the database is initialized
-    @staticmethod
-    def is_init() -> bool:
-        return BrickSQL().fetchone('schema/is_init') is not None
-
     # Replace the database with a new file
     @staticmethod
     def upload(file: FileStorage, /) -> None:

bricktracker/sql/migrations/0001.sql

@@ -1,3 +1,4 @@
+-- description: Database initialization
 -- FROM sqlite3 app.db .schema > init.sql with extra IF NOT EXISTS and transaction
 BEGIN transaction;
 
@@ -58,9 +59,4 @@ CREATE TABLE IF NOT EXISTS missing (
         u_id TEXT
     );
 
--- Fix a bug where 'None' was inserted in missing instead of NULL
-UPDATE missing
-SET element_id = NULL
-WHERE element_id = 'None';
-
 COMMIT;

bricktracker/sql/migrations/0002.sql

@@ -0,0 +1,13 @@
+-- description: WAL journal, 'None' fix for missing table
+
+-- Set the journal mode to WAL
+PRAGMA journal_mode = WAL;
+
+BEGIN transaction;
+
+-- Fix a bug where 'None' was inserted in missing instead of NULL
+UPDATE missing
+SET element_id = NULL
+WHERE element_id = 'None';
+
+COMMIT;

bricktracker/sql/schema/get_version.sql

@@ -0,0 +1 @@
+PRAGMA user_version

bricktracker/sql/schema/is_init.sql

@@ -1,4 +0,0 @@
-SELECT name
-FROM sqlite_master
-WHERE type="table"
-AND name="sets"

bricktracker/sql/schema/set_version.sql

@@ -0,0 +1 @@
+PRAGMA user_version = {{ version }}

bricktracker/sql_migration.py

@@ -0,0 +1,54 @@
+import os
+
+from .version import __database_version__
+
+
+class BrickSQLMigration(object):
+    description: str | None
+    name: str
+    file: str
+    version: int
+
+    # Description marker
+    description_marker: str = '-- description:'
+
+    def __init__(self, file: str):
+        self.file = file
+        self.name, _ = os.path.splitext(os.path.basename(self.file))
+        self.version = int(self.name)
+
+        self.description = None
+
+    # Read the description from the migration file if it exists
+    def get_description(self) -> str:
+        if self.description is None:
+            self.description = ''
+
+            # First line or ignored
+            with open(self.file, 'r') as file:
+                line = file.readline()
+
+                # Extract a description (only the first one)
+                if line.startswith(self.description_marker):
+                    self.description = line.strip()[
+                        len(self.description_marker):
+                    ]
+
+        return self.description
+
+    # Tells whether the migration is need
+    def is_needed(self, current: int, /):
+        return self.version > current and self.version <= __database_version__
+
+    # Query name for the SQL loader
+    def get_query(self) -> str:
+        relative, _ = os.path.splitext(
+            os.path.relpath(self.file, os.path.join(
+                os.path.dirname(__file__),
+                'sql/'
+            ))
+        )
+
+        print(relative)
+
+        return relative

bricktracker/sql_migration_list.py

@@ -0,0 +1,58 @@
+from glob import glob
+import logging
+import os
+
+from .sql_migration import BrickSQLMigration
+
+logger = logging.getLogger(__name__)
+
+
+class BrickSQLMigrationList(object):
+    migrations: list[BrickSQLMigration]
+
+    def __init__(self):
+        # Load the migrations only there is none already loaded
+        migrations = getattr(self, 'migrations', None)
+
+        if migrations is None:
+            logger.info('Loading SQL migrations list')
+
+            BrickSQLMigrationList.migrations = []
+
+            path: str = os.path.join(
+                os.path.dirname(__file__),
+                'sql/migrations/*.sql'
+            )
+
+            print(path)
+
+            files = glob(path)
+
+            print(files)
+
+            for file in files:
+                try:
+                    BrickSQLMigrationList.migrations.append(
+                        BrickSQLMigration(file)
+                    )
+                # Ignore file if error
+                except Exception as e:
+                    print(e)
+                    pass
+
+    # Get the sorted list of pending migrations
+    def pending(
+        self,
+        current: int,
+        /
+    ) -> list[BrickSQLMigration]:
+        pending: list[BrickSQLMigration] = []
+
+        for migration in self.migrations:
+            if migration.is_needed(current):
+                pending.append(migration)
+
+        # Sort the list
+        pending.sort(key=lambda e: e.version)
+
+        return pending

bricktracker/version.py

@@ -1 +1,4 @@
-__version__ = '1.0.0'
+from typing import Final
+
+__version__: Final[str] = '1.0.0'
+__database_version__: Final[int] = 2

bricktracker/views/admin.py

@@ -24,6 +24,7 @@ from ..rebrickable_image import RebrickableImage
 from ..retired_list import BrickRetiredList
 from ..set import BrickSet
 from ..sql_counter import BrickCounter
+from ..sql_migration_list import BrickSQLMigrationList
 from ..sql import BrickSQL
 from ..theme_list import BrickThemeList
 from .upload import upload_helper
@@ -38,10 +39,10 @@ admin_page = Blueprint('admin', __name__, url_prefix='/admin')
 @login_required
 @exception_handler(__file__)
 def admin() -> str:
-    counters: dict[str, int] = {}
+    database_counters: list[BrickCounter] = []
-    count_none: int = 0
+    database_exception: Exception | None = None
-    exception: Exception | None = None
+    database_needs_upgrade: bool = False
-    is_init: bool = False
+    database_version: int = -1
     nil_minifigure_name: str = ''
     nil_minifigure_url: str = ''
     nil_part_name: str = ''
@@ -49,14 +50,19 @@ def admin() -> str:
 
     # This view needs to be protected against SQL errors
     try:
-        is_init = BrickSQL.is_init()
+        database = BrickSQL(failsafe=True)
+        database_needs_upgrade = database.needs_upgrade()
+        database_version = database.version
 
-        if is_init:
+        if not database_needs_upgrade:
-            counters = BrickSQL().count_records()
+            database_counters = BrickSQL().count_records()
+    except Exception as e:
+        database_exception = e
 
-        record = BrickSQL().fetchone('missing/count_none')
+        # Warning
-        if record is not None:
+        logger.warning('A database exception occured while loading the admin page: {exception}'.format(  # noqa: E501
-            count_none = record['count']
+            exception=str(e),
+        ))
 
     nil_minifigure_name = RebrickableImage.nil_minifigure_name()
     nil_minifigure_url = RebrickableImage.static_url(
@@ -70,14 +76,6 @@ def admin() -> str:
         'PARTS_FOLDER'
     )
 
-    except Exception as e:
-        exception = e
-
-        # Warning
-        logger.warning('An exception occured while loading the admin page: {exception}'.format(  # noqa: E501
-            exception=str(e),
-        ))
-
     open_image = request.args.get('open_image', None)
     open_instructions = request.args.get('open_instructions', None)
     open_logout = request.args.get('open_logout', None)
@@ -95,12 +93,12 @@ def admin() -> str:
     return render_template(
         'admin.html',
         configuration=BrickConfigurationList.list(),
-        counters=counters,
+        database_counters=database_counters,
-        count_none=count_none,
+        database_error=request.args.get('error'),
-        error=request.args.get('error'),
+        database_exception=database_exception,
-        exception=exception,
+        database_needs_upgrade=database_needs_upgrade,
+        database_version=database_version,
         instructions=BrickInstructionsList(),
-        is_init=is_init,
         nil_minifigure_name=nil_minifigure_name,
         nil_minifigure_url=nil_minifigure_url,
         nil_part_name=nil_part_name,
@@ -116,19 +114,6 @@ def admin() -> str:
     )
 
 
-# Initialize the database
-@admin_page.route('/init-database', methods=['POST'])
-@login_required
-@exception_handler(__file__, post_redirect='admin.admin')
-def init_database() -> Response:
-    BrickSQL.initialize()
-
-    # Reload the instructions
-    BrickInstructionsList(force=True)
-
-    return redirect(url_for('admin.admin'))
-
-
 # Delete the database
 @admin_page.route('/delete-database', methods=['GET'])
 @login_required
@@ -205,6 +190,16 @@ def do_drop_database() -> Response:
     return redirect(url_for('admin.admin'))
 
 
+# Actually upgrade the database
+@admin_page.route('/upgrade-database', methods=['POST'])
+@login_required
+@exception_handler(__file__, post_redirect='admin.upgrade_database')
+def do_upgrade_database() -> Response:
+    BrickSQL(failsafe=True).upgrade()
+
+    return redirect(url_for('admin.admin'))
+
+
 # Import a database
 @admin_page.route('/import-database', methods=['GET'])
 @login_required
@@ -316,3 +311,20 @@ def update_themes() -> Response:
     BrickThemeList(force=True)
 
     return redirect(url_for('admin.admin', open_theme=True))
+
+
+# Upgrade the database
+@admin_page.route('/upgrade-database', methods=['GET'])
+@login_required
+@exception_handler(__file__, post_redirect='admin.admin')
+def upgrade_database() -> str:
+    database = BrickSQL(failsafe=True)
+
+    return render_template(
+        'admin.html',
+        upgrade_database=True,
+        migrations=BrickSQLMigrationList().pending(
+            database.version
+        ),
+        error=request.args.get('error')
+    )

templates/admin.html

@@ -18,6 +18,8 @@
             {% include 'admin/database/drop.html' %}
             {% elif import_database %}
             {% include 'admin/database/import.html' %}
+            {% elif upgrade_database %}
+            {% include 'admin/database/upgrade.html' %}
             {% else %}
               {% include 'admin/logout.html' %}
               {% include 'admin/instructions.html' %}

templates/admin/database.html

@@ -2,37 +2,26 @@
 
 {{ accordion.header('Database', 'database', 'admin', expanded=open_database, icon='database-2-line') }}
 <h5 class="border-bottom">Status</h5>
-{% if exception %}<div class="alert alert-danger" role="alert">An exception occured while loading this page: {{ exception }}</div>{% endif %}
+{% if database_exception %}<div class="alert alert-danger" role="alert">An exception occured while loading this page: {{ database_exception }}</div>{% endif %}
-{% if error %}<div class="alert alert-danger" role="alert"><strong>Error:</strong> {{ error }}.</div>{% endif %}
+{% if database_error %}<div class="alert alert-danger" role="alert"><strong>Error:</strong> {{ database_error }}.</div>{% endif %}
-{% if not is_init %}
+{% if database_needs_upgrade %}
   <div class="alert alert-warning" role="alert">
-    <p>The database file is: <code>{{ config['DATABASE_PATH'] }}</code>. The database is not initialized.</p>
+    <p>Your database needs to be upgraded.</p>
     <hr>
-    <form action="{{ url_for('admin.init_database') }}" method="post" class="text-end">
+    <div class="text-end">
-        <button type="submit" class="btn btn-warning"><i class="ri-reset-right-fill"></i> Initialize the database</button>
+      <a href="{{ url_for('admin.upgrade_database') }}" class="btn btn-warning" role="button"><i class="ri-arrow-up-double-line"></i> Upgrade the database</a>
-    </form>
     </div>
-{% else %}
-  {% if count_none %}
-    <div class="alert alert-warning" role="alert">
-      <p>
-        Your <code>missing</code> table contains <code>"None"</code> entries (instead of <code>NULL</code>). <br>
-        This can lead to "phantom" missing parts appearing in your sets if you are coming from the original version of BrickTracker.
-      </p>
-      <hr>
-      <form action="{{ url_for('admin.init_database') }}" method="post" class="text-end">
-          <button type="submit" class="btn btn-warning"><i class="ri-capsule-line"></i> Apply the fix</button>
-      </form>
   </div>
-  {% endif %}
+{% endif %}
-  <p>The database file is: <code>{{ config['DATABASE_PATH'] }}</code>. <i class="ri-checkbox-circle-line"></i> The database is initialized.</p>
+<p>The database file is: <code>{{ config['DATABASE_PATH'] }}</code> at version <span class="badge rounded-pill text-bg-light border fw-normal"><i class="ri-hashtag"></i>{{ database_version }}</span></p>
-  <p>
+<p>
   <a href="{{ url_for('admin.download_database') }}" class="btn btn-primary" role="button"><i class="ri-download-line"></i> Download the database file</a>
-  </p>
+</p>
+{% if database_counters %}
   <h5 class="border-bottom">Records</h5>
   <div class="d-flex justify-content-start">
-    <ul class="list-group">
+    <ul class="list-group me-2">
-      {% for counter in counters %}
+      {% for counter in database_counters %}
       {% if not (loop.index % 5) %}
         </ul>
         <ul class="list-group">
@@ -49,9 +38,6 @@
 {{ accordion.header('Database danger zone', 'database-danger', 'admin', danger=true, class='text-end') }}
 {% if error %}<div class="alert alert-danger text-start" role="alert"><strong>Error:</strong> {{ error }}.</div>{% endif %}
 <a href="{{ url_for('admin.import_database') }}" class="btn btn-warning" role="button"><i class="ri-upload-line"></i> Import a database file</a>
-<form action="{{ url_for('admin.init_database') }}" method="post" class="d-inline">
-  <button type="submit" class="btn btn-warning"><i class="ri-reset-right-fill"></i> Initialize the database (only missing tables)</button>
-</form>
 <a href="{{ url_for('admin.drop_database') }}" class="btn btn-danger" role="button"><i class="ri-close-line"></i> Drop the database</a>
 <a href="{{ url_for('admin.delete_database') }}" class="btn btn-danger" role="button"><i class="ri-delete-bin-2-line"></i> Delete the database file</a>
 {{ accordion.footer() }}

templates/admin/database/upgrade.html

@@ -0,0 +1,29 @@
+{% import 'macro/accordion.html' as accordion %}
+
+{{ accordion.header('Database', 'database', 'admin', expanded=true, icon='database-2-line') }}
+<form action="{{ url_for('admin.do_upgrade_database') }}" method="post">
+  {% if error %}<div class="alert alert-danger text-start" role="alert"><strong>Error:</strong> {{ error }}.</div>{% endif %}
+  <div class="alert alert-warning text-center" role="alert">
+    You are about to <strong>upgrade your database file</strong>. This action is irreversible.<br>
+    The process shold be lossless, but just to be sure, grab a copy of your database before proceeding.<br>
+  </div>
+  <h5 class="border-bottom">Upgrades</h5>
+  <ul>
+    {% for migration in migrations %}
+    <li>
+      <span class="badge rounded-pill text-bg-light border fw-normal me-2"><i class="ri-hashtag"></i>{{ migration.version }}</span>
+      {% if migration.get_description() %}
+        <code>{{ migration.get_description() }}</code>
+      {% else %}
+        <span class="badge rounded-pill text-bg-secondary fst-italic">No description</span>
+      {% endif %}
+    </li>
+    {% endfor %}
+  </ul>
+  <div class="text-end">
+    <a class="btn btn-danger" href="{{ url_for('admin.admin') }}" role="button"><i class="ri-arrow-left-long-line"></i> Back to the admin</a>
+    <a href="{{ url_for('admin.download_database') }}" class="btn btn-primary" role="button"><i class="ri-download-line"></i> Download the database file</a>
+    <button type="submit" class="btn btn-warning"><i class="ri-arrow-up-double-line"></i> Upgrade the database</button>
+  </div>
+</form>
+{{ accordion.footer() }}