FrederikBaerentsen/BrickTracker
8
20
Fork 12
Code Issues 18 Pull Requests Actions Packages Projects 1 Releases 10 Wiki Activity

Incremental forward upgrades of the database

Browse Source
This commit is contained in:
Gregoo
2025-01-20 17:43:15 +01:00
parent c6e5a6a2d9
commit 5e99371b39
15 changed files with 298 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bricktracker/sql.py
+56 -17
View File
@@ -3,13 +3,15 @@ import os
import sqlite3
from typing import Any, Final, Tuple
from .sql_stats import BrickSQLStats
from flask import current_app, g
from jinja2 import Environment, FileSystemLoader
from werkzeug.datastructures import FileStorage
from .exceptions import DatabaseException
from .sql_counter import BrickCounter
from .sql_migration_list import BrickSQLMigrationList
from .sql_stats import BrickSQLStats
from .version import __database_version__
logger = logging.getLogger(__name__)
@@ -26,8 +28,9 @@ class BrickSQL(object):
connection: sqlite3.Connection
cursor: sqlite3.Cursor
stats: BrickSQLStats
version: int
def __init__(self, /):
def __init__(self, /, failsafe: bool = False):
# Instantiate the database connection in the Flask
# application context so that it can be used by all
# requests without re-opening connections
@@ -37,6 +40,9 @@ class BrickSQL(object):
if database is not None:
self.connection = database
self.stats = getattr(g, 'database_stats', BrickSQLStats())
# Grab a cursor
self.cursor = self.connection.cursor()
else:
# Instantiate the stats
self.stats = BrickSQLStats()
@@ -52,16 +58,38 @@ class BrickSQL(object):
# Setup the row factory to get pseudo-dicts rather than tuples
self.connection.row_factory = sqlite3.Row
# Debug: Attach the debugger
# Uncomment manually because this is ultra verbose
# self.connection.set_trace_callback(print)
# Grab a cursor
self.cursor = self.connection.cursor()
# Save the connection globally for later use
g.database = self.connection
g.database_stats = self.stats
# Grab the version and check
try:
version = self.fetchone('schema/get_version')
# Grab a cursor
self.cursor = self.connection.cursor()
if version is None:
raise Exception('version is None')
self.version = version[0]
except Exception as e:
self.version = 0
raise DatabaseException('Could not get the database version: {error}'.format( # noqa: E501
error=str(e)
))
if not failsafe:
if self.needs_upgrade():
raise DatabaseException('Your database need to be upgraded from version {version} to version {required}'.format( # noqa: E501
version=self.version,
required=__database_version__,
))
# Debug: Attach the debugger
# Uncomment manually because this is ultra verbose
# self.connection.set_trace_callback(print)
# Save the connection globally for later use
g.database = self.connection
g.database_stats = self.stats
# Clear the defer stack
def clear_defer(self, /) -> None:
@@ -236,11 +264,16 @@ class BrickSQL(object):
return template.render(**context)
# Tells whether the database needs upgrade
def needs_upgrade(self) -> bool:
return self.version < __database_version__
# Raw execute the query without any options
def raw_execute(
self,
query: str,
parameters: dict[str, Any]
parameters: dict[str, Any],
/,
) -> sqlite3.Cursor:
logger.debug('SQLite3: execute: {query}'.format(
query=BrickSQL.clean_query(query)
@@ -248,6 +281,17 @@ class BrickSQL(object):
return self.cursor.execute(query, parameters)
# Upgrade the database
def upgrade(self) -> None:
if self.needs_upgrade():
for pending in BrickSQLMigrationList().pending(self.version):
logger.info('Applying migration {version}'.format(
version=pending.version)
)
self.executescript(pending.get_query())
self.execute('schema/set_version', version=pending.version)
# Clean the query for debugging
@staticmethod
def clean_query(query: str, /) -> str:
@@ -289,11 +333,6 @@ class BrickSQL(object):
# Info
logger.info('The database has been initialized')
# Check if the database is initialized
@staticmethod
def is_init() -> bool:
return BrickSQL().fetchone('schema/is_init') is not None
# Replace the database with a new file
@staticmethod
def upload(file: FileStorage, /) -> None:
bricktracker/sql/migrations/init.sql → bricktracker/sql/migrations/0001.sql
+1 -5
View File
@@ -1,3 +1,4 @@
-- description: Database initialization
-- FROM sqlite3 app.db .schema > init.sql with extra IF NOT EXISTS and transaction
BEGIN transaction;
@@ -58,9 +59,4 @@ CREATE TABLE IF NOT EXISTS missing (
u_id TEXT
);
-- Fix a bug where 'None' was inserted in missing instead of NULL
UPDATE missing
SET element_id = NULL
WHERE element_id = 'None';
COMMIT;
bricktracker/sql/migrations/0002.sql
+13
View File
@@ -0,0 +1,13 @@
-- description: WAL journal, 'None' fix for missing table
-- Set the journal mode to WAL
PRAGMA journal_mode = WAL;
BEGIN transaction;
-- Fix a bug where 'None' was inserted in missing instead of NULL
UPDATE missing
SET element_id = NULL
WHERE element_id = 'None';
COMMIT;
bricktracker/sql/schema/get_version.sql
+1
View File
@@ -0,0 +1 @@
PRAGMA user_version
bricktracker/sql/schema/is_init.sql
-4
View File
@@ -1,4 +0,0 @@
SELECT name
FROM sqlite_master
WHERE type="table"
AND name="sets"
bricktracker/sql/schema/set_version.sql
+1
View File
@@ -0,0 +1 @@
PRAGMA user_version = {{ version }}
bricktracker/sql_migration.py
+54
View File
@@ -0,0 +1,54 @@
import os
from .version import __database_version__
class BrickSQLMigration(object):
description: str | None
name: str
file: str
version: int
# Description marker
description_marker: str = '-- description:'
def __init__(self, file: str):
self.file = file
self.name, _ = os.path.splitext(os.path.basename(self.file))
self.version = int(self.name)
self.description = None
# Read the description from the migration file if it exists
def get_description(self) -> str:
if self.description is None:
self.description = ''
# First line or ignored
with open(self.file, 'r') as file:
line = file.readline()
# Extract a description (only the first one)
if line.startswith(self.description_marker):
self.description = line.strip()[
len(self.description_marker):
]
return self.description
# Tells whether the migration is need
def is_needed(self, current: int, /):
return self.version > current and self.version <= __database_version__
# Query name for the SQL loader
def get_query(self) -> str:
relative, _ = os.path.splitext(
os.path.relpath(self.file, os.path.join(
os.path.dirname(__file__),
'sql/'
))
)
print(relative)
return relative
bricktracker/sql_migration_list.py
+58
View File
@@ -0,0 +1,58 @@
from glob import glob
import logging
import os
from .sql_migration import BrickSQLMigration
logger = logging.getLogger(__name__)
class BrickSQLMigrationList(object):
migrations: list[BrickSQLMigration]
def __init__(self):
# Load the migrations only there is none already loaded
migrations = getattr(self, 'migrations', None)
if migrations is None:
logger.info('Loading SQL migrations list')
BrickSQLMigrationList.migrations = []
path: str = os.path.join(
os.path.dirname(__file__),
'sql/migrations/*.sql'
)
print(path)
files = glob(path)
print(files)
for file in files:
try:
BrickSQLMigrationList.migrations.append(
BrickSQLMigration(file)
)
# Ignore file if error
except Exception as e:
print(e)
pass
# Get the sorted list of pending migrations
def pending(
self,
current: int,
/
) -> list[BrickSQLMigration]:
pending: list[BrickSQLMigration] = []
for migration in self.migrations:
if migration.is_needed(current):
pending.append(migration)
# Sort the list
pending.sort(key=lambda e: e.version)
return pending
bricktracker/version.py
+4 -1
View File
@@ -1 +1,4 @@
__version__ = '1.0.0'
from typing import Final
__version__: Final[str] = '1.0.0'
__database_version__: Final[int] = 2
bricktracker/views/admin.py
+56 -44
View File
@@ -24,6 +24,7 @@ from ..rebrickable_image import RebrickableImage
from ..retired_list import BrickRetiredList
from ..set import BrickSet
from ..sql_counter import BrickCounter
from ..sql_migration_list import BrickSQLMigrationList
from ..sql import BrickSQL
from ..theme_list import BrickThemeList
from .upload import upload_helper
@@ -38,10 +39,10 @@ admin_page = Blueprint('admin', __name__, url_prefix='/admin')
@login_required
@exception_handler(__file__)
def admin() -> str:
counters: dict[str, int] = {}
count_none: int = 0
exception: Exception | None = None
is_init: bool = False
database_counters: list[BrickCounter] = []
database_exception: Exception | None = None
database_needs_upgrade: bool = False
database_version: int = -1
nil_minifigure_name: str = ''
nil_minifigure_url: str = ''
nil_part_name: str = ''
@@ -49,35 +50,32 @@ def admin() -> str:
# This view needs to be protected against SQL errors
try:
is_init = BrickSQL.is_init()
if is_init:
counters = BrickSQL().count_records()
record = BrickSQL().fetchone('missing/count_none')
if record is not None:
count_none = record['count']
nil_minifigure_name = RebrickableImage.nil_minifigure_name()
nil_minifigure_url = RebrickableImage.static_url(
nil_minifigure_name,
'MINIFIGURES_FOLDER'
)
nil_part_name = RebrickableImage.nil_name()
nil_part_url = RebrickableImage.static_url(
nil_part_name,
'PARTS_FOLDER'
)
database = BrickSQL(failsafe=True)
database_needs_upgrade = database.needs_upgrade()
database_version = database.version
if not database_needs_upgrade:
database_counters = BrickSQL().count_records()
except Exception as e:
exception = e
database_exception = e
# Warning
logger.warning('An exception occured while loading the admin page: {exception}'.format( # noqa: E501
logger.warning('A database exception occured while loading the admin page: {exception}'.format( # noqa: E501
exception=str(e),
))
nil_minifigure_name = RebrickableImage.nil_minifigure_name()
nil_minifigure_url = RebrickableImage.static_url(
nil_minifigure_name,
'MINIFIGURES_FOLDER'
)
nil_part_name = RebrickableImage.nil_name()
nil_part_url = RebrickableImage.static_url(
nil_part_name,
'PARTS_FOLDER'
)
open_image = request.args.get('open_image', None)
open_instructions = request.args.get('open_instructions', None)
open_logout = request.args.get('open_logout', None)
@@ -95,12 +93,12 @@ def admin() -> str:
return render_template(
'admin.html',
configuration=BrickConfigurationList.list(),
counters=counters,
count_none=count_none,
error=request.args.get('error'),
exception=exception,
database_counters=database_counters,
database_error=request.args.get('error'),
database_exception=database_exception,
database_needs_upgrade=database_needs_upgrade,
database_version=database_version,
instructions=BrickInstructionsList(),
is_init=is_init,
nil_minifigure_name=nil_minifigure_name,
nil_minifigure_url=nil_minifigure_url,
nil_part_name=nil_part_name,
@@ -116,19 +114,6 @@ def admin() -> str:
)
# Initialize the database
@admin_page.route('/init-database', methods=['POST'])
@login_required
@exception_handler(__file__, post_redirect='admin.admin')
def init_database() -> Response:
BrickSQL.initialize()
# Reload the instructions
BrickInstructionsList(force=True)
return redirect(url_for('admin.admin'))
# Delete the database
@admin_page.route('/delete-database', methods=['GET'])
@login_required
@@ -205,6 +190,16 @@ def do_drop_database() -> Response:
return redirect(url_for('admin.admin'))
# Actually upgrade the database
@admin_page.route('/upgrade-database', methods=['POST'])
@login_required
@exception_handler(__file__, post_redirect='admin.upgrade_database')
def do_upgrade_database() -> Response:
BrickSQL(failsafe=True).upgrade()
return redirect(url_for('admin.admin'))
# Import a database
@admin_page.route('/import-database', methods=['GET'])
@login_required
@@ -316,3 +311,20 @@ def update_themes() -> Response:
BrickThemeList(force=True)
return redirect(url_for('admin.admin', open_theme=True))
# Upgrade the database
@admin_page.route('/upgrade-database', methods=['GET'])
@login_required
@exception_handler(__file__, post_redirect='admin.admin')
def upgrade_database() -> str:
database = BrickSQL(failsafe=True)
return render_template(
'admin.html',
upgrade_database=True,
migrations=BrickSQLMigrationList().pending(
database.version
),
error=request.args.get('error')
)