From 2c0e3f49d003794760879344b80fd84fed288444 Mon Sep 17 00:00:00 2001 From: FrederikBaerentsen Date: Wed, 19 Jun 2024 13:42:28 +0200 Subject: [PATCH] Added database scripts and updated rebrickable downloader. Fixed #1 --- app.py | 46 +++++++++++++++++----- db.py | 105 ++++++++++++++++++++++++++++++++++++++++++++++++++ downloadRB.py | 27 +++++++++++++ 3 files changed, 169 insertions(+), 9 deletions(-) create mode 100644 db.py diff --git a/app.py b/app.py index 21cf639..5c26430 100644 --- a/app.py +++ b/app.py @@ -12,7 +12,8 @@ import rebrick #rebrickable api import requests # request img from web import shutil # save img locally import eventlet -from downloadRB import download_and_unzip +from downloadRB import download_and_unzip,get_nil_images +from db import initialize_database,get_rows,delete_tables app = Flask(__name__) socketio = SocketIO(app) @@ -368,20 +369,47 @@ def new_set(set_num): count = 0 socketio.emit('task_completed', namespace='/progress') +def get_file_creation_dates(file_list): + creation_dates = {} + for file_name in file_list: + file_path = f"{file_name}" + if os.path.exists(file_path): + creation_time = os.path.getctime(file_path) + creation_dates[file_name] = time.ctime(creation_time) + else: + creation_dates[file_name] = "File not found" + return creation_dates + @app.route('/config',methods=['POST','GET']) def config(): - print(request.method) + + file_list = ['themes.csv', 'colors.csv', 'sets.csv','static/nil.png','static/nil_mf.jpg'] + creation_dates = get_file_creation_dates(file_list) + + row_counts = [0] + db_exists = Path("app.db") + if db_exists.is_file(): + db_is_there = True + row_counts = get_rows() + else: + db_is_there = False + if request.method == 'POST': - if request.form.get('CreateDB') == 'createDB': - # pass - print("Encrypted") - elif request.form.get('Get Rebrickable data') == 'rebrickUpdate': - # pass # do something else + if request.form.get('CreateDB') == 'Create Database': + initialize_database() + row_counts = get_rows() + return redirect(url_for('config')) + elif request.form.get('Update local data') == 'Update local data': urls = ["themes","sets","colors"] - for i in urls: download_and_unzip("https://cdn.rebrickable.com/media/downloads/"+i+".csv.gz") + get_nil_images() + return redirect(url_for('config')) + + elif request.form.get('deletedb') == 'Delete Database': + delete_tables() + initialize_database() else: # pass # unknown @@ -389,7 +417,7 @@ def config(): elif request.method == 'GET': # return render_template("index.html") print("No Post Back Call") - return render_template("config.html") + return render_template("config.html",db_is_there=db_is_there,creation_dates = creation_dates,row_counts=row_counts) @app.route('/missing',methods=['POST','GET']) def missing(): diff --git a/db.py b/db.py new file mode 100644 index 0000000..213e55f --- /dev/null +++ b/db.py @@ -0,0 +1,105 @@ +import os +import sqlite3 + +def initialize_database(): + db_path = 'app.db' + tables = ['sets', 'inventory', 'minifigures', 'missing'] + row_counts = {} + + # Connect to the database (this will create the file if it doesn't exist) + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + # Create the required tables if they do not exist + cursor.execute('''CREATE TABLE IF NOT EXISTS sets ( + set_num TEXT, + name TEXT, + year INTEGER, + theme_id INTEGER, + num_parts INTEGER, + set_img_url TEXT, + set_url TEXT, + last_modified_dt TEXT, + mini_col BOOLEAN, + set_check BOOLEAN, + set_col BOOLEAN, + u_id TEXT + )''') + + cursor.execute('''CREATE TABLE IF NOT EXISTS inventory ( + set_num TEXT, + id INTEGER, + part_num INTEGER, + name TEXT, + part_img_url TEXT, + part_img_url_id TEXT, + color_id INTEGER, + color_name TEXT, + quantity INTEGER, + is_spare BOOLEAN, + element_id INTEGER, + u_id TEXT + )''') + + cursor.execute('''CREATE TABLE IF NOT EXISTS minifigures ( + fig_num TEXT, + set_num TEXT, + name TEXT, + quantity INTEGER, + set_img_url TEXT, + u_id TEXT + )''') + + cursor.execute('''CREATE TABLE IF NOT EXISTS missing ( + set_num TEXT, + id INTEGER, + part_num TEXT, + part_img_url_id TEXT, + color_id INTEGER, + quantity INTEGER, + element_id INTEGER, + u_id TEXT + )''') + + # Commit the changes + conn.commit() + conn.close() + +def get_rows(): + db_path = 'app.db' + tables = ['sets', 'inventory', 'minifigures', 'missing'] + row_counts = {} + + # Connect to the database (this will create the file if it doesn't exist) + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + # Get the row count for each table + for table in tables: + cursor.execute(f"SELECT COUNT(*) FROM {table}") + row_count = cursor.fetchone()[0] + row_counts[table] = row_count + + # Close the connection + conn.close() + + return row_counts + + +def delete_tables(): + db_path = 'app.db' + tables = ['sets', 'inventory', 'minifigures', 'missing'] + row_counts = {} + + # Connect to the database (this will create the file if it doesn't exist) + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + + cursor.execute('''DROP TABLE sets''') + cursor.execute('''DROP TABLE inventory''') + cursor.execute('''DROP TABLE minifigures''') + cursor.execute('''DROP TABLE missing''') + + # Close the connection + conn.close() + diff --git a/downloadRB.py b/downloadRB.py index f18b8ab..831edcc 100644 --- a/downloadRB.py +++ b/downloadRB.py @@ -5,6 +5,33 @@ import os import sys from urllib.parse import urlparse +def get_nil_images(): + image_urls = [ + "https://rebrickable.com/static/img/nil_mf.jpg", + "https://rebrickable.com/static/img/nil.png" + ] + static_folder = "static" + + # Create the static folder if it does not exist + if not os.path.exists(static_folder): + os.makedirs(static_folder) + + for url in image_urls: + # Extract the output filename from the URL + parsed_url = urlparse(url) + output_file = os.path.join(static_folder, os.path.basename(parsed_url.path)) + + # Download the image + response = requests.get(url, stream=True) + response.raise_for_status() # Check for any request errors + + # Save the image to the static folder + with open(output_file, 'wb') as f: + f.write(response.content) + + print(f"Downloaded {output_file}") + + def download_and_unzip(url: str): # Extract the output filename from the URL parsed_url = urlparse(url)