diff --git a/.dockerignore b/.dockerignore
index 82afab6..fa88ec8 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -19,6 +19,8 @@ LICENSE
# Database
*.db
+*.db-shm
+*.db-wal
# Python
**/__pycache__
diff --git a/.gitignore b/.gitignore
index 0d192d5..cd54740 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,8 @@
# Application
.env
*.db
+*.db-shm
+*.db-wal
# Python specifics
__pycache__/
diff --git a/bricktracker/sql.py b/bricktracker/sql.py
index e594191..eb368c9 100644
--- a/bricktracker/sql.py
+++ b/bricktracker/sql.py
@@ -3,13 +3,15 @@ import os
import sqlite3
from typing import Any, Final, Tuple
-from .sql_stats import BrickSQLStats
-
from flask import current_app, g
from jinja2 import Environment, FileSystemLoader
from werkzeug.datastructures import FileStorage
+from .exceptions import DatabaseException
from .sql_counter import BrickCounter
+from .sql_migration_list import BrickSQLMigrationList
+from .sql_stats import BrickSQLStats
+from .version import __database_version__
logger = logging.getLogger(__name__)
@@ -26,8 +28,9 @@ class BrickSQL(object):
connection: sqlite3.Connection
cursor: sqlite3.Cursor
stats: BrickSQLStats
+ version: int
- def __init__(self, /):
+ def __init__(self, /, failsafe: bool = False):
# Instantiate the database connection in the Flask
# application context so that it can be used by all
# requests without re-opening connections
@@ -37,6 +40,9 @@ class BrickSQL(object):
if database is not None:
self.connection = database
self.stats = getattr(g, 'database_stats', BrickSQLStats())
+
+ # Grab a cursor
+ self.cursor = self.connection.cursor()
else:
# Instantiate the stats
self.stats = BrickSQLStats()
@@ -52,16 +58,38 @@ class BrickSQL(object):
# Setup the row factory to get pseudo-dicts rather than tuples
self.connection.row_factory = sqlite3.Row
- # Debug: Attach the debugger
- # Uncomment manually because this is ultra verbose
- # self.connection.set_trace_callback(print)
+ # Grab a cursor
+ self.cursor = self.connection.cursor()
- # Save the connection globally for later use
- g.database = self.connection
- g.database_stats = self.stats
+ # Grab the version and check
+ try:
+ version = self.fetchone('schema/get_version')
- # Grab a cursor
- self.cursor = self.connection.cursor()
+ if version is None:
+ raise Exception('version is None')
+
+ self.version = version[0]
+ except Exception as e:
+ self.version = 0
+
+ raise DatabaseException('Could not get the database version: {error}'.format( # noqa: E501
+ error=str(e)
+ ))
+
+ if not failsafe:
+ if self.needs_upgrade():
+ raise DatabaseException('Your database need to be upgraded from version {version} to version {required}'.format( # noqa: E501
+ version=self.version,
+ required=__database_version__,
+ ))
+
+ # Debug: Attach the debugger
+ # Uncomment manually because this is ultra verbose
+ # self.connection.set_trace_callback(print)
+
+ # Save the connection globally for later use
+ g.database = self.connection
+ g.database_stats = self.stats
# Clear the defer stack
def clear_defer(self, /) -> None:
@@ -236,11 +264,16 @@ class BrickSQL(object):
return template.render(**context)
+ # Tells whether the database needs upgrade
+ def needs_upgrade(self) -> bool:
+ return self.version < __database_version__
+
# Raw execute the query without any options
def raw_execute(
self,
query: str,
- parameters: dict[str, Any]
+ parameters: dict[str, Any],
+ /,
) -> sqlite3.Cursor:
logger.debug('SQLite3: execute: {query}'.format(
query=BrickSQL.clean_query(query)
@@ -248,6 +281,17 @@ class BrickSQL(object):
return self.cursor.execute(query, parameters)
+ # Upgrade the database
+ def upgrade(self) -> None:
+ if self.needs_upgrade():
+ for pending in BrickSQLMigrationList().pending(self.version):
+ logger.info('Applying migration {version}'.format(
+ version=pending.version)
+ )
+
+ self.executescript(pending.get_query())
+ self.execute('schema/set_version', version=pending.version)
+
# Clean the query for debugging
@staticmethod
def clean_query(query: str, /) -> str:
@@ -289,11 +333,6 @@ class BrickSQL(object):
# Info
logger.info('The database has been initialized')
- # Check if the database is initialized
- @staticmethod
- def is_init() -> bool:
- return BrickSQL().fetchone('schema/is_init') is not None
-
# Replace the database with a new file
@staticmethod
def upload(file: FileStorage, /) -> None:
diff --git a/bricktracker/sql/migrations/init.sql b/bricktracker/sql/migrations/0001.sql
similarity index 91%
rename from bricktracker/sql/migrations/init.sql
rename to bricktracker/sql/migrations/0001.sql
index 85defbb..846eddc 100644
--- a/bricktracker/sql/migrations/init.sql
+++ b/bricktracker/sql/migrations/0001.sql
@@ -1,3 +1,4 @@
+-- description: Database initialization
-- FROM sqlite3 app.db .schema > init.sql with extra IF NOT EXISTS and transaction
BEGIN transaction;
@@ -58,9 +59,4 @@ CREATE TABLE IF NOT EXISTS missing (
u_id TEXT
);
--- Fix a bug where 'None' was inserted in missing instead of NULL
-UPDATE missing
-SET element_id = NULL
-WHERE element_id = 'None';
-
COMMIT;
\ No newline at end of file
diff --git a/bricktracker/sql/migrations/0002.sql b/bricktracker/sql/migrations/0002.sql
new file mode 100644
index 0000000..5dc9ed4
--- /dev/null
+++ b/bricktracker/sql/migrations/0002.sql
@@ -0,0 +1,13 @@
+-- description: WAL journal, 'None' fix for missing table
+
+-- Set the journal mode to WAL
+PRAGMA journal_mode = WAL;
+
+BEGIN transaction;
+
+-- Fix a bug where 'None' was inserted in missing instead of NULL
+UPDATE missing
+SET element_id = NULL
+WHERE element_id = 'None';
+
+COMMIT;
\ No newline at end of file
diff --git a/bricktracker/sql/schema/get_version.sql b/bricktracker/sql/schema/get_version.sql
new file mode 100644
index 0000000..693601a
--- /dev/null
+++ b/bricktracker/sql/schema/get_version.sql
@@ -0,0 +1 @@
+PRAGMA user_version
diff --git a/bricktracker/sql/schema/is_init.sql b/bricktracker/sql/schema/is_init.sql
deleted file mode 100644
index 17a3a81..0000000
--- a/bricktracker/sql/schema/is_init.sql
+++ /dev/null
@@ -1,4 +0,0 @@
-SELECT name
-FROM sqlite_master
-WHERE type="table"
-AND name="sets"
diff --git a/bricktracker/sql/schema/set_version.sql b/bricktracker/sql/schema/set_version.sql
new file mode 100644
index 0000000..53d0eb2
--- /dev/null
+++ b/bricktracker/sql/schema/set_version.sql
@@ -0,0 +1 @@
+PRAGMA user_version = {{ version }}
diff --git a/bricktracker/sql_migration.py b/bricktracker/sql_migration.py
new file mode 100644
index 0000000..0b6acc4
--- /dev/null
+++ b/bricktracker/sql_migration.py
@@ -0,0 +1,54 @@
+import os
+
+from .version import __database_version__
+
+
+class BrickSQLMigration(object):
+ description: str | None
+ name: str
+ file: str
+ version: int
+
+ # Description marker
+ description_marker: str = '-- description:'
+
+ def __init__(self, file: str):
+ self.file = file
+ self.name, _ = os.path.splitext(os.path.basename(self.file))
+ self.version = int(self.name)
+
+ self.description = None
+
+ # Read the description from the migration file if it exists
+ def get_description(self) -> str:
+ if self.description is None:
+ self.description = ''
+
+ # First line or ignored
+ with open(self.file, 'r') as file:
+ line = file.readline()
+
+ # Extract a description (only the first one)
+ if line.startswith(self.description_marker):
+ self.description = line.strip()[
+ len(self.description_marker):
+ ]
+
+ return self.description
+
+ # Tells whether the migration is need
+ def is_needed(self, current: int, /):
+ return self.version > current and self.version <= __database_version__
+
+ # Query name for the SQL loader
+ def get_query(self) -> str:
+ relative, _ = os.path.splitext(
+ os.path.relpath(self.file, os.path.join(
+ os.path.dirname(__file__),
+ 'sql/'
+ ))
+ )
+
+ print(relative)
+
+ return relative
diff --git a/bricktracker/sql_migration_list.py b/bricktracker/sql_migration_list.py
new file mode 100644
index 0000000..56cbc45
--- /dev/null
+++ b/bricktracker/sql_migration_list.py
@@ -0,0 +1,58 @@
+from glob import glob
+import logging
+import os
+
+from .sql_migration import BrickSQLMigration
+
+logger = logging.getLogger(__name__)
+
+
+class BrickSQLMigrationList(object):
+ migrations: list[BrickSQLMigration]
+
+ def __init__(self):
+ # Load the migrations only there is none already loaded
+ migrations = getattr(self, 'migrations', None)
+
+ if migrations is None:
+ logger.info('Loading SQL migrations list')
+
+ BrickSQLMigrationList.migrations = []
+
+ path: str = os.path.join(
+ os.path.dirname(__file__),
+ 'sql/migrations/*.sql'
+ )
+
+ print(path)
+
+ files = glob(path)
+
+ print(files)
+
+ for file in files:
+ try:
+ BrickSQLMigrationList.migrations.append(
+ BrickSQLMigration(file)
+ )
+ # Ignore file if error
+ except Exception as e:
+ print(e)
+ pass
+
+ # Get the sorted list of pending migrations
+ def pending(
+ self,
+ current: int,
+ /
+ ) -> list[BrickSQLMigration]:
+ pending: list[BrickSQLMigration] = []
+
+ for migration in self.migrations:
+ if migration.is_needed(current):
+ pending.append(migration)
+
+ # Sort the list
+ pending.sort(key=lambda e: e.version)
+
+ return pending
diff --git a/bricktracker/version.py b/bricktracker/version.py
index 1f356cc..cc0dac8 100644
--- a/bricktracker/version.py
+++ b/bricktracker/version.py
@@ -1 +1,4 @@
-__version__ = '1.0.0'
+from typing import Final
+
+__version__: Final[str] = '1.0.0'
+__database_version__: Final[int] = 2
diff --git a/bricktracker/views/admin.py b/bricktracker/views/admin.py
index 86a126b..b97e0fa 100644
--- a/bricktracker/views/admin.py
+++ b/bricktracker/views/admin.py
@@ -24,6 +24,7 @@ from ..rebrickable_image import RebrickableImage
from ..retired_list import BrickRetiredList
from ..set import BrickSet
from ..sql_counter import BrickCounter
+from ..sql_migration_list import BrickSQLMigrationList
from ..sql import BrickSQL
from ..theme_list import BrickThemeList
from .upload import upload_helper
@@ -38,10 +39,10 @@ admin_page = Blueprint('admin', __name__, url_prefix='/admin')
@login_required
@exception_handler(__file__)
def admin() -> str:
- counters: dict[str, int] = {}
- count_none: int = 0
- exception: Exception | None = None
- is_init: bool = False
+ database_counters: list[BrickCounter] = []
+ database_exception: Exception | None = None
+ database_needs_upgrade: bool = False
+ database_version: int = -1
nil_minifigure_name: str = ''
nil_minifigure_url: str = ''
nil_part_name: str = ''
@@ -49,35 +50,32 @@ def admin() -> str:
# This view needs to be protected against SQL errors
try:
- is_init = BrickSQL.is_init()
-
- if is_init:
- counters = BrickSQL().count_records()
-
- record = BrickSQL().fetchone('missing/count_none')
- if record is not None:
- count_none = record['count']
-
- nil_minifigure_name = RebrickableImage.nil_minifigure_name()
- nil_minifigure_url = RebrickableImage.static_url(
- nil_minifigure_name,
- 'MINIFIGURES_FOLDER'
- )
-
- nil_part_name = RebrickableImage.nil_name()
- nil_part_url = RebrickableImage.static_url(
- nil_part_name,
- 'PARTS_FOLDER'
- )
+ database = BrickSQL(failsafe=True)
+ database_needs_upgrade = database.needs_upgrade()
+ database_version = database.version
+ if not database_needs_upgrade:
+ database_counters = BrickSQL().count_records()
except Exception as e:
- exception = e
+ database_exception = e
# Warning
- logger.warning('An exception occured while loading the admin page: {exception}'.format( # noqa: E501
+ logger.warning('A database exception occured while loading the admin page: {exception}'.format( # noqa: E501
exception=str(e),
))
+ nil_minifigure_name = RebrickableImage.nil_minifigure_name()
+ nil_minifigure_url = RebrickableImage.static_url(
+ nil_minifigure_name,
+ 'MINIFIGURES_FOLDER'
+ )
+
+ nil_part_name = RebrickableImage.nil_name()
+ nil_part_url = RebrickableImage.static_url(
+ nil_part_name,
+ 'PARTS_FOLDER'
+ )
+
open_image = request.args.get('open_image', None)
open_instructions = request.args.get('open_instructions', None)
open_logout = request.args.get('open_logout', None)
@@ -95,12 +93,12 @@ def admin() -> str:
return render_template(
'admin.html',
configuration=BrickConfigurationList.list(),
- counters=counters,
- count_none=count_none,
- error=request.args.get('error'),
- exception=exception,
+ database_counters=database_counters,
+ database_error=request.args.get('error'),
+ database_exception=database_exception,
+ database_needs_upgrade=database_needs_upgrade,
+ database_version=database_version,
instructions=BrickInstructionsList(),
- is_init=is_init,
nil_minifigure_name=nil_minifigure_name,
nil_minifigure_url=nil_minifigure_url,
nil_part_name=nil_part_name,
@@ -116,19 +114,6 @@ def admin() -> str:
)
-# Initialize the database
-@admin_page.route('/init-database', methods=['POST'])
-@login_required
-@exception_handler(__file__, post_redirect='admin.admin')
-def init_database() -> Response:
- BrickSQL.initialize()
-
- # Reload the instructions
- BrickInstructionsList(force=True)
-
- return redirect(url_for('admin.admin'))
-
-
# Delete the database
@admin_page.route('/delete-database', methods=['GET'])
@login_required
@@ -205,6 +190,16 @@ def do_drop_database() -> Response:
return redirect(url_for('admin.admin'))
+# Actually upgrade the database
+@admin_page.route('/upgrade-database', methods=['POST'])
+@login_required
+@exception_handler(__file__, post_redirect='admin.upgrade_database')
+def do_upgrade_database() -> Response:
+ BrickSQL(failsafe=True).upgrade()
+
+ return redirect(url_for('admin.admin'))
+
+
# Import a database
@admin_page.route('/import-database', methods=['GET'])
@login_required
@@ -316,3 +311,20 @@ def update_themes() -> Response:
BrickThemeList(force=True)
return redirect(url_for('admin.admin', open_theme=True))
+
+
+# Upgrade the database
+@admin_page.route('/upgrade-database', methods=['GET'])
+@login_required
+@exception_handler(__file__, post_redirect='admin.admin')
+def upgrade_database() -> str:
+ database = BrickSQL(failsafe=True)
+
+ return render_template(
+ 'admin.html',
+ upgrade_database=True,
+ migrations=BrickSQLMigrationList().pending(
+ database.version
+ ),
+ error=request.args.get('error')
+ )
diff --git a/templates/admin.html b/templates/admin.html
index e425887..d54efc5 100644
--- a/templates/admin.html
+++ b/templates/admin.html
@@ -18,6 +18,8 @@
{% include 'admin/database/drop.html' %}
{% elif import_database %}
{% include 'admin/database/import.html' %}
+ {% elif upgrade_database %}
+ {% include 'admin/database/upgrade.html' %}
{% else %}
{% include 'admin/logout.html' %}
{% include 'admin/instructions.html' %}
diff --git a/templates/admin/database.html b/templates/admin/database.html
index ec64cae..f92374a 100644
--- a/templates/admin/database.html
+++ b/templates/admin/database.html
@@ -2,41 +2,30 @@
{{ accordion.header('Database', 'database', 'admin', expanded=open_database, icon='database-2-line') }}
<h5 class="border-bottom">Status</h5>
-{% if exception %}<div class="alert alert-danger" role="alert">An exception occured while loading this page: {{ exception }}</div>{% endif %}
-{% if error %}<div class="alert alert-danger" role="alert"><strong>Error:</strong> {{ error }}.</div>{% endif %}
-{% if not is_init %}
+{% if database_exception %}<div class="alert alert-danger" role="alert">An exception occured while loading this page: {{ database_exception }}</div>{% endif %}
+{% if database_error %}<div class="alert alert-danger" role="alert"><strong>Error:</strong> {{ database_error }}.</div>{% endif %}
+{% if database_needs_upgrade %}
<div class="alert alert-warning" role="alert">
- <p>The database file is: <code>{{ config['DATABASE_PATH'] }}</code>. The database is not initialized.</p>
+ <p>Your database needs to be upgraded.</p>
<hr>
- <form action="{{ url_for('admin.init_database') }}" method="post" class="text-end">
- <button type="submit" class="btn btn-warning"><i class="ri-reset-right-fill"></i> Initialize the database</button>
- </form>
- </div>
-{% else %}
- {% if count_none %}
- <div class="alert alert-warning" role="alert">
- <p>
- Your <code>missing</code> table contains <code>"None"</code> entries (instead of <code>NULL</code>). <br>
- This can lead to "phantom" missing parts appearing in your sets if you are coming from the original version of BrickTracker.
- </p>
- <hr>
- <form action="{{ url_for('admin.init_database') }}" method="post" class="text-end">
- <button type="submit" class="btn btn-warning"><i class="ri-capsule-line"></i> Apply the fix</button>
- </form>
+ <div class="text-end">
+ <a href="{{ url_for('admin.upgrade_database') }}" class="btn btn-warning" role="button"><i class="ri-arrow-up-double-line"></i> Upgrade the database</a>
</div>
- {% endif %}
- <p>The database file is: <code>{{ config['DATABASE_PATH'] }}</code>. <i class="ri-checkbox-circle-line"></i> The database is initialized.</p>
- <p>
- <a href="{{ url_for('admin.download_database') }}" class="btn btn-primary" role="button"><i class="ri-download-line"></i> Download the database file</a>
- </p>
+ </div>
+{% endif %}
+<p>The database file is: <code>{{ config['DATABASE_PATH'] }}</code> at version <span class="badge rounded-pill text-bg-light border fw-normal"><i class="ri-hashtag"></i>{{ database_version }}</span></p>
+<p>
+ <a href="{{ url_for('admin.download_database') }}" class="btn btn-primary" role="button"><i class="ri-download-line"></i> Download the database file</a>
+</p>
+{% if database_counters %}
<h5 class="border-bottom">Records</h5>
<div class="d-flex justify-content-start">
- <ul class="list-group">
- {% for counter in counters %}
- {% if not (loop.index % 5) %}
- </ul>
- <ul class="list-group">
- {% endif %}
+ <ul class="list-group me-2">
+ {% for counter in database_counters %}
+ {% if not (loop.index % 5) %}
+ </ul>
+ <ul class="list-group">
+ {% endif %}
<li class="list-group-item d-flex justify-content-between align-items-start">
<span><i class="ri-{{ counter.icon }}"></i> {{ counter.name }}</span> <span class="badge text-bg-primary rounded-pill ms-2">{{ counter.count }}</span>
</li>
@@ -49,9 +38,6 @@
{{ accordion.header('Database danger zone', 'database-danger', 'admin', danger=true, class='text-end') }}
{% if error %}<div class="alert alert-danger text-start" role="alert"><strong>Error:</strong> {{ error }}.</div>{% endif %}
<a href="{{ url_for('admin.import_database') }}" class="btn btn-warning" role="button"><i class="ri-upload-line"></i> Import a database file</a>
-<form action="{{ url_for('admin.init_database') }}" method="post" class="d-inline">
- <button type="submit" class="btn btn-warning"><i class="ri-reset-right-fill"></i> Initialize the database (only missing tables)</button>
-</form>
<a href="{{ url_for('admin.drop_database') }}" class="btn btn-danger" role="button"><i class="ri-close-line"></i> Drop the database</a>
<a href="{{ url_for('admin.delete_database') }}" class="btn btn-danger" role="button"><i class="ri-delete-bin-2-line"></i> Delete the database file</a>
{{ accordion.footer() }}
diff --git a/templates/admin/database/upgrade.html b/templates/admin/database/upgrade.html
new file mode 100644
index 0000000..763f73d
--- /dev/null
+++ b/templates/admin/database/upgrade.html
@@ -0,0 +1,29 @@
+{% import 'macro/accordion.html' as accordion %}
+
+{{ accordion.header('Database', 'database', 'admin', expanded=true, icon='database-2-line') }}
+<form action="{{ url_for('admin.do_upgrade_database') }}" method="post">
+ {% if error %}<div class="alert alert-danger text-start" role="alert"><strong>Error:</strong> {{ error }}.</div>{% endif %}
+ <div class="alert alert-warning text-center" role="alert">
+ You are about to <strong>upgrade your database file</strong>. This action is irreversible.<br>
+ The process shold be lossless, but just to be sure, grab a copy of your database before proceeding.<br>
+ </div>
+ <h5 class="border-bottom">Upgrades</h5>
+ <ul>
+ {% for migration in migrations %}
+ <li>
+ <span class="badge rounded-pill text-bg-light border fw-normal me-2"><i class="ri-hashtag"></i>{{ migration.version }}</span>
+ {% if migration.get_description() %}
+ <code>{{ migration.get_description() }}</code>
+ {% else %}
+ <span class="badge rounded-pill text-bg-secondary fst-italic">No description</span>
+ {% endif %}
+ </li>
+ {% endfor %}
+ </ul>
+ <div class="text-end">
+ <a class="btn btn-danger" href="{{ url_for('admin.admin') }}" role="button"><i class="ri-arrow-left-long-line"></i> Back to the admin</a>
+ <a href="{{ url_for('admin.download_database') }}" class="btn btn-primary" role="button"><i class="ri-download-line"></i> Download the database file</a>
+ <button type="submit" class="btn btn-warning"><i class="ri-arrow-up-double-line"></i> Upgrade the database</button>
+ </div>
+</form>
+{{ accordion.footer() }}