Added download script to get new csv files

This commit is contained in:
FrederikBaerentsen 2024-06-18 19:27:51 +02:00
parent f1d093b67b
commit c11af132f0
2 changed files with 63 additions and 1 deletions

26
app.py
View File

@ -12,6 +12,7 @@ import rebrick #rebrickable api
import requests # request img from web
import shutil # save img locally
import eventlet
from downloadRB import download_and_unzip
app = Flask(__name__)
socketio = SocketIO(app)
@ -367,6 +368,29 @@ def new_set(set_num):
count = 0
socketio.emit('task_completed', namespace='/progress')
@app.route('/config',methods=['POST','GET'])
def config():
print(request.method)
if request.method == 'POST':
if request.form.get('CreateDB') == 'createDB':
# pass
print("Encrypted")
elif request.form.get('Get Rebrickable data') == 'rebrickUpdate':
# pass # do something else
urls = ["themes","sets","colors"]
for i in urls:
download_and_unzip("https://cdn.rebrickable.com/media/downloads/"+i+".csv.gz")
else:
# pass # unknown
return render_template("config.html")
elif request.method == 'GET':
# return render_template("index.html")
print("No Post Back Call")
return render_template("config.html")
@app.route('/missing',methods=['POST','GET'])
def missing():
conn = sqlite3.connect('app.db')
@ -764,4 +788,4 @@ def save_number(tmp):
return Response(status=204)
if __name__ == '__main__':
socketio.run(app.run(host='0.0.0.0', debug=False, port=3333))
socketio.run(app.run(host='0.0.0.0', debug=True, port=3333))

38
downloadRB.py Normal file
View File

@ -0,0 +1,38 @@
import requests
import gzip
import shutil
import os
import sys
from urllib.parse import urlparse
def download_and_unzip(url: str):
# Extract the output filename from the URL
parsed_url = urlparse(url)
output_file = os.path.basename(parsed_url.path).replace('.gz', '')
# Download the file
response = requests.get(url, stream=True)
response.raise_for_status() # Check for any request errors
# Write the gzipped file to the local file system
gz_file = output_file + '.gz'
with open(gz_file, 'wb') as f:
f.write(response.content)
# Unzip the file
with gzip.open(gz_file, 'rb') as f_in:
with open(output_file, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
# Optionally remove the .gz file after extraction
os.remove(gz_file)
# Usage
if __name__ == "__main__":
if len(sys.argv) != 2:
print("Usage: python3 downloadRB.py <url>")
sys.exit(1)
url = sys.argv[1]
download_and_unzip(url)