Started working on #15. Added status page for Generate

.dockerignore

@@ -0,0 +1,8 @@
+.venv/
+__pycache__/
+.env
+.git
+.gitignore
+deploy.sh
+Dockerfile
+env

.gitignore

@@ -0,0 +1,7 @@
+.venv/
+__pycache__/
+.env
+deploy.sh
+env
+thumbnails
+*.db

README.md

@@ -65,3 +65,20 @@ In the `config.py` file you need to change like 4 from `"/library"` to your comi
 
     python3 main.py
 
+## Supported Readers
+
+Any reader that supports OPDS should work, however the following have been verified to work/not work
+
+| App                                                                                                   | iOS |
+| ----------------------------------------------------------------------------  | --- |
+| KyBook 3 (iOS)                                                             | ✔️  |
+| Aldiko Next (iOS)                                                                                             | ❌  |
+| PocketBook (iOS)                                                               | ✔️  |
+| Moon+ Reader (Android) | ✔️       |
+| Panels (iOS)                                                               | ✔️  |
+| Marvin (iOS)                                                             | ✔️  |
+| Chunky (iOS)                                                             | ✔️  |
+
+# Notes
+
+5865 files in 359 seconds

config.py

@@ -1,8 +1,39 @@
 import os
 from werkzeug.security import generate_password_hash
+from sys import platform
+import sys
 
-#CONTENT_BASE_DIR = os.getenv("CONTENT_BASE_DIR", "/library")
-CONTENT_BASE_DIR = os.getenv("CONTENT_BASE_DIR", "/home/drudoo/ComicsTest/Comics")
+CONTENT_BASE_DIR = os.getenv("CONTENT_BASE_DIR", "/library") #docker
+
+#if platform == "linux" or platform == "linux2":
+#    CONTENT_BASE_DIR = os.getenv("CONTENT_BASE_DIR", "/home/drudoo/ComicsTest/Comics") #linux
+#elif platform == "win32":
+#    CONTENT_BASE_DIR = os.getenv("CONTENT_BASE_DIR", "/Comics/ComicRack") #windows
+    #CONTENT_BASE_DIR = os.getenv("CONTENT_BASE_DIR", "testlibrary") #windows test library
+
+
+# Added folder for thumbnails. These are loaded as covers for the files.
+THUMBNAIL_DIR = os.getenv("THUMBNAIL_DIR",'/thumbnails')
+
+# If using Windows, insert the drive letter of your comics here. 
+# Both the script and comics needs to be on the same drive.
+WIN_DRIVE_LETTER = 'B'
+
+# If using custom searches, then insert the default amout of results here. 
+# It is also possible to override this in the json file.
+DEFAULT_SEARCH_NUMBER = 10
+
+# Debug output
+# False: no print out in terminal
+# True: logs are printet to terminal
+DEBUG = True
+
+# Max thumbnail size
+MAXSIZE = (500,500)
+
+def _print(arg):
+    if DEBUG:
+        print(arg,file=sys.stderr)
 
 TEENYOPDS_ADMIN_PASSWORD = os.getenv("TEENYOPDS_ADMIN_PASSWORD", None)
 users = {}

db.py

@@ -0,0 +1,142 @@
+import sqlite3
+from bs4 import BeautifulSoup
+import xml.etree.ElementTree as ET
+import re
+import datetime
+
+def createdb():
+    conn = sqlite3.connect('../test_database.db') 
+    c = conn.cursor()
+
+    c.execute('''
+          CREATE TABLE IF NOT EXISTS comics
+          (
+            [book_id] TEXT PRIMARY KEY,
+            [book_path] TEXT, 
+            [series] TEXT,
+            [seriesgroup] TEXT,
+            [number] TEXT,
+            [count] INTEGER,
+            [volume] TEXT,
+            [notes] TEXT,
+            [year] INTEGER,
+            [month] INTEGER,
+            [day] INTEGER,
+            [writer] TEXT,
+            [penciller] TEXT,
+            [inker] TEXT,
+            [letterer] TEXT,
+            [colorist] TEXT,
+            [coverartist] TEXT,
+            [publisher] TEXT,
+            [genre] TEXT,
+            [pagecount] INTEGER,
+            [languageiso] TEXT,
+            [scaninformation] TEXT,
+            [pages] INTEGER,
+            [added] TEXT,
+            [filesize] INTEGER,
+            [filemodifiedtime] TEXT,
+            [filecreationtime] TEXT
+          )
+          ''')
+    conn.commit()
+
+def dropdb():
+    conn = sqlite3.connect('../test_database.db')
+    c = conn.cursor()
+    c.execute('DROP TABLE COMICS')
+    conn.commit()
+
+def checkempty(v,t):
+    r=""
+    try: 
+        r=v.find(t).text
+    except:
+        pass
+    return r
+
+def loaddata():
+    conn = sqlite3.connect('../test_database.db')
+    c = conn.cursor()
+    
+    book_id,book_path,series,seriesgroup,number="","","","",""
+    count=0
+    volume,seriesgroup,notes="","",""
+    year,month,day=0,0,0
+    writer,penciller,inker,letterer,colorist,coverartist,publiser,genre="","","","","","","",""
+    pagecount=0
+    languageiso,scaninformation="",""
+    pages=0
+    added=""
+    filesize=0
+    filemodificationtime,filecreationtime="",""
+    
+    tree = ET.parse('../ComicDb_small.xml')
+    root = tree.getroot()
+
+    for child in root:
+        #print("child: ", child.tag,child.attrib)
+        if child.tag == 'Books':
+            for grandchild in child:
+                #print("grandchild: ",grandchild.tag,grandchild.attrib)
+                #print(grandchild.attrib)
+                #print(type(grandchild.attrib))
+                book_id=grandchild.attrib['Id']
+                book_path=grandchild.attrib['File']
+                #for i,j in grandchild.attrib.items():
+                #    print(i,j)
+                #    #print(i,i["Id"])
+                #series=grandchild.attrib['Series'].text
+                #print(series)
+                #print(grandchild[0].tag)
+                #series=grandchild.find('Series').text
+                series=checkempty(grandchild,'Series')
+                number=checkempty(grandchild,'Number')
+                count=checkempty(grandchild,'Count')
+                seriesgroup=checkempty(grandchild,'SeriesGroup')
+                notes=checkempty(grandchild,'Notes')
+                year=checkempty(grandchild,'Year')
+                month=checkempty(grandchild,'Month')
+                day=checkempty(grandchild,'Day')
+                writer=checkempty(grandchild,'Writer')
+                penciller=checkempty(grandchild,'Penciller')
+                inker=checkempty(grandchild,'Inker')
+                letterer=checkempty(grandchild,'Letterer')
+                
+                c.execute("INSERT OR REPLACE INTO COMICS (book_id,book_path,series,number,count,seriesgroup,notes,year,month,day,writer,penciller, inker,letterer) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",(book_id,book_path,series,number,count,seriesgroup,notes,year,month,day,writer,penciller,inker,letterer))
+                conn.commit()
+
+                #for ggchild in grandchild:
+                #    print(ggchild.tag)
+                #    print(ggchild.text)
+                #print("----")
+
+        #for books in child.findall('Book'):
+            #print(books,type(books))
+            #print(books.tag, books.attrib)
+
+
+
+
+
+
+    #with open('ComicDb_small.xml', 'r') as f:
+    #    contents = f.read()
+    #    Bs_data = BeautifulSoup(contents, 'xml')
+    #    for i in Bs_data.find_all('Book'):
+    #        #print(i)
+    #        try:
+    #            book_id = i.find('Book',{"Id"}).text
+    #            print(book_id)
+    #        except:
+    #            pass
+    #        try:
+    #            series=i.select('Series')[0].text
+    #        except:
+    #            pass
+#dropdb()
+#createdb()
+
+loaddata()
+

docker-compose.yml

@@ -0,0 +1,14 @@
+version: '3.3'
+services:
+  comicopds:
+    image: comicopds
+    container_name: comicopds
+    restart: unless-stopped
+    ports:
+      - '5000:5000'
+    volumes:
+      #- '/opt/data/Comics/ComicRack:/library:ro'
+      #- '/home/drudoo/Pi1/Comics/ComicRack:/library:ro'
+      - '${PWD}/CT/:/library:ro'
+      - '${PWD}/thumbnails:/thumbnails'
+      - '${PWD}/:/app'

extras.py

@@ -0,0 +1,24 @@
+import os,re
+
+table = str.maketrans({
+    "<": "&lt;",
+    ">": "&gt;",
+    "&": "&amp;",
+    "'": "&apos;",
+    '"': "&quot;",
+})
+def xmlesc(txt):
+    return txt.translate(table)
+
+def get_size(file_path, unit='bytes'):
+    file_size = os.path.getsize(file_path)
+    exponents_map = {'bytes': 0, 'kb': 1, 'mb': 2, 'gb': 3}
+    if unit not in exponents_map:
+        raise ValueError("Must select from \
+        ['bytes', 'kb', 'mb', 'gb']")
+    else:
+        size = file_size / 1024 ** exponents_map[unit]
+        return round(size, 1)
+
+def get_cvdb(string):
+    return re.findall('(?<=\[CVDB)(.*)(?=].)', string[0].text)[0]

import.py

@@ -0,0 +1,51 @@
+import zipfile
+from bs4 import BeautifulSoup
+import time
+import config
+import os,sys
+import time
+import sqlite3
+import timeit
+import re
+import datetime
+
+conn = sqlite3.connect('app.db')
+list = []
+
+start_time = timeit.default_timer()
+for root, dirs, files in os.walk(os.path.abspath(config.CONTENT_BASE_DIR)):
+    for file in files:
+        f = os.path.join(root, file)
+        #try:
+        if f.endswith(".cbz"):
+            print("CBZ: " + f)
+            s = zipfile.ZipFile(f)
+            #s = gzip.GzipFile(f)
+            Bs_data = BeautifulSoup(s.open('ComicInfo.xml').read(), "xml")
+            #print(Bs_data.select('Series')[0].text, file=sys.stderr)
+            #print(Bs_data.select('Title')[0].text, file=sys.stderr)
+            CVDB=re.findall('(?<=\[CVDB)(.*)(?=].)', Bs_data.select('Notes')[0].text)
+            #list.append('CVDB'+CVDB[0] + ': '  + Bs_data.select('Series')[0].text + "(" + Bs_data.select('Volume')[0].text + ") : " + Bs_data.select('Number')[0].text  )
+            #print(list, file=sys.stdout)
+        
+            ISSUE=Bs_data.select('Number')[0].text
+            SERIES=Bs_data.select('Series')[0].text
+            VOLUME=Bs_data.select('Volume')[0].text
+            PUBLISHER=Bs_data.select('Publisher')[0].text
+            try:
+                TITLE=Bs_data.select('Title')[0].text
+            except:
+                TITLE=""
+            PATH=f 
+            UPDATED=str(datetime.datetime.now())
+            #print(UPDATED,file=sys.stdout)
+            #sql="INSERT OR REPLACE INTO COMICS (CVDB,ISSUE,SERIES,VOLUME, PUBLISHER, TITLE, FILE,PATH,UPDATED) VALUES ("+CVDB[0]+",'"+ISSUE+"','"+SERIES+"','"+VOLUME+"','"+PUBLISHER+"','"+TITLE+"','"+file+"','" + f + "','" + UPDATED + "')"
+            #print(sql,file=sys.stdout)
+            conn.execute("INSERT OR REPLACE INTO COMICS (CVDB,ISSUE,SERIES,VOLUME, PUBLISHER, TITLE, FILE,PATH,UPDATED) VALUES (?,?,?,?,?,?,?,?,?)", (CVDB[0], ISSUE, SERIES, VOLUME, PUBLISHER, TITLE, file, f, UPDATED))
+            conn.commit()
+        else:
+            print("NOT CBZ: " + f)
+
+conn.close()
+elapsed = timeit.default_timer() - start_time
+print(elapsed)

main.py

@@ -1,23 +1,36 @@
-from flask import Flask, send_from_directory, request
+from flask import Flask, redirect,url_for,  render_template, send_from_directory, request
 from flask_httpauth import HTTPBasicAuth
 from werkzeug.security import check_password_hash
 from gevent.pywsgi import WSGIServer
 import timeit
 import sqlite3
 import os
+from PIL import Image
 import zipfile
+import gzip
 from bs4 import BeautifulSoup
 import re
 import datetime
 import sys
+import time
+import json
+import numpy as np
+from pathlib import Path
+from io import BytesIO
+from threading import Thread
+
+# for debugging
+from pprint import pprint
+####
+
+generated = None
 
 from opds import fromdir
-import config
+import config,extras
 
 app = Flask(__name__, static_url_path="", static_folder="static")
 auth = HTTPBasicAuth()
 
-
 @auth.verify_password
 def verify_password(username, password):
     if not config.TEENYOPDS_ADMIN_PASSWORD:
@@ -27,58 +40,304 @@ def verify_password(username, password):
     ):
         return username
 
+@app.route("/", methods=['POST','GET'])
+def startpage():
+    #result = "Hello, World!"
+    config._print(request.method)
+    if request.method == 'POST':
+        if request.form.get('Create') == 'Create':
+            # pass
+            config._print("open")
+            conn = sqlite3.connect('app.db')
+            cursor = conn.cursor()
+        
+            cursor.execute("create table COMICS (CVDB,ISSUE,SERIES,VOLUME, YEAR, PUBLISHER, TITLE, FILE,PATH,UPDATED,PRIMARY KEY(CVDB))")
+            result = cursor.fetchall()
+            conn.close()
+            config._print("Encrypted")
+        elif  request.form.get('Import') == 'Import':
+            # pass # do something else
+            config._print("Decrypted")
+            return redirect(url_for('import2sql'))
+        elif request.form.get('Generate') == 'Generate':
+            config._print("Generate Covers from Start page")
+             
+            return redirect(url_for('generate2'))
+        else:
+            # pass # unknown
+            return render_template("first.html")
+    elif request.method == 'GET':
+        # return render_template("index.html")
+        config._print("No Post Back Call")
+    conn = sqlite3.connect('app.db')
+    cursor = conn.cursor()
+    
+    try:
+        cursor.execute("select * from comics where CVDB in (SELECT CVDB from comics order by RANDOM() LIMIT " + str(config.DEFAULT_SEARCH_NUMBER) + ");")
+        result = cursor.fetchall()
+    
+        pub_list = ["Marvel", "DC Comics","Dark Horse Comics", "Dynamite Entertainment", "Oni Press"]
+        count = []
+        for i in pub_list:
+            cursor.execute("select count(*) from comics where Publisher = '" + i + "';")
+            count.append(cursor.fetchone()[0])
+
+        #cursor.execute("SELECT volume, COUNT(volume) FROM comics GROUP BY volume ORDER BY volume;")
+        cursor.execute("SELECT year, COUNT(year) FROM comics GROUP BY year ORDER BY year;")
+        volume = cursor.fetchall()
+        
+        
+
+        x = []
+        y = []
+        for i in volume:
+            x.append(i[0])
+            y.append(i[1])
+        conn.close()
+
+        try:
+            total = np.sum(np.array(volume).astype('int')[:,1],axis=0)
+            dir_path = r'thumbnails'
+            covers = 0
+            for path in os.listdir(dir_path):
+                if os.path.isfile(os.path.join(dir_path,path)):
+                    covers += 1
+            config._print("covers: " + str(covers))
+        except Exception as e:
+            config._print(e)
+        return render_template("start.html", first=False,result=result,pub_list=pub_list,count=count,x=x,y=y,total=total,covers=covers)
+    except:
+        conn.close()
+
+        config._print('first')
+        return render_template("start.html",first=True)
+
+#@app.route("/first", methods=['GET', 'POST'])
+#def first():
+#   return render_template('first.html',result=result) 
+
+
+
 
-@app.route("/")
 @app.route("/healthz")
 def healthz():
     return "ok"
 
+@app.route('/search')
+def search():
+    args = request.args.get('q')
+    print(args)
+    conn = sqlite3.connect('app.db')
+    cursor = conn.cursor()
+    result = 'no good' 
+    try:
+        cursor.execute("select TITLE, PATH from comics where TITLE like '%" + str(args) + "%';")
+        result = cursor.fetchall()
+
+        cursor.close()
+        for i in result:
+            print(i) 
+    except Exception as e:
+        config._print(e)
+    return str(result)
+
+total = None
+#@app.route("/generate")
+def generate():
+    config._print('GENERATES NOW!!!')
+    force = 'True' #request.args.get('force')
+    global generated
+    global total
+    total = 0
+    generated = 0
+    comiccount = 0
+    files_without_comicinfo = 0
+    errorcount = 0
+    skippedcount = 0
+    errormsg = ""
+    for root, dirs, files in os.walk(os.path.abspath(config.CONTENT_BASE_DIR)):
+        for file in files:
+            f = os.path.join(root,file)
+            if f.endswith('.cbz'):
+                total = total + 1
+    for root, dirs, files in os.walk(os.path.abspath(config.CONTENT_BASE_DIR)):
+        for file in files:
+            f = os.path.join(root, file)
+            if f.endswith('.cbz'):
+                config._print(generated)
+                try:
+                    comiccount = comiccount + 1
+                    s = zipfile.ZipFile(f)
+                    filelist = zipfile.ZipFile.namelist(s)
+                    if 'ComicInfo.xml' in filelist:
+                        Bs_data = BeautifulSoup(s.open('ComicInfo.xml').read(), "xml")
+                        CVDB=extras.get_cvdb(Bs_data.select('Notes'))
+                        if force == 'True':
+                            ext = [i for i, x in enumerate(filelist) if re.search("(?i)\.jpg|png|jpeg$", x)]
+                            cover = s.open(filelist[ext[0]]).read()
+                            
+                            image = Image.open(BytesIO(cover))
+                            rgb_im = image.convert("RGB")
+                            image.thumbnail(config.MAXSIZE,Image.LANCZOS)
+                            image.save(config.THUMBNAIL_DIR + "/" + str(CVDB) + ".jpg")
+
+                            # Old way of saving without resize 
+                            #c = open(config.THUMBNAIL_DIR + "/" + str(CVDB) + ".jpg", 'wb+')
+                            #c.write(cover)
+                            #c.close()
+                            generated = generated + 1
+                        if Path(config.THUMBNAIL_DIR + "/" + str(CVDB) + ".jpg").exists() == False:
+                            config._print("generating for " + str(CVDB))
+                            try: 
+                                ext = [i for i, x in enumerate(filelist) if re.search("(?i)\.jpg|png|jpeg$", x)]
+                                #config._print(filelist)
+                                #config._print(ext)
+                                #config._print(filelist[ext[0]])
+                                cover = s.open(filelist[ext[0]]).read()
+                                #xyz = [i for i, x in enumerate(filelist) if re.match('*\.py$',x)]
+                                #config._print(xyz)
+                                image = Image.open(BytesIO(cover))
+                                image.thumbnail(config.MAXSIZE,Image.LANCZOS)
+                                image.save(config.THUMBNAIL_DIR + "/" + str(CVDB) + ".jpg")
+                                generated = generated + 1
+                            except Exception as e:
+                                errormsg = str(e)
+                                config._print(e)
+                        else:
+                            if not force:
+                                skippedcount = skippedcount + 1
+                    else:
+                        print("Error at: " + str(CVDB) + " " + str(f))
+                        files_withtout_comicinfo = files_without_comicinfo + 1
+                except Exception as e:
+                    errorcount = errorcount + 1
+                    config._print("Error (/generate): " + str(e))
+                    config._print(f)
+                    errormsg = str(e)
+    return "Forced generation: " + str(force) + "<br>Comics: " + str(comiccount) + "<br>Generated: " + str(generated) + "<br>CBZ files without ComicInfo.xml: " + str(files_without_comicinfo) + "<br>Errors: " + str(errorcount) + "<br>Skipped: " + str(skippedcount) + "<br>" + errormsg
+    config._print( "Forced generation: " + str(force) + "<br>Comics: " + str(comiccount) + "<br>Generated: " + str(generated) + "<br>CBZ files without ComicInfo.xml: " + str(files_without_comicinfo) + "<br>Errors: " + str(errorcount) + "<br>Skipped: " + str(skippedcount) + "<br>" + errormsg)
+
+@app.route("/generate2")
+def generate2():
+    t1 = Thread(target=generate)
+    t1.start()
+    return render_template('status.html')
+
+
+@app.route("/t2")
+def index():
+    t1 = Thread(target=generate)
+    t1.start()
+    return render_template('status.html')
+    
+@app.route('/status',methods=['GET'])
+def getStatus():
+    statusList = {'status':generated,'total':total}
+    return json.dumps(statusList)
+
 @app.route('/import')
 def import2sql():
     conn = sqlite3.connect('app.db')
     list = []
-
+    comiccount = 0
+    importcount = 0
+    coverscount = 0
+    skippedcount = 0
+    errorcount = 0
+    comics_with_errors = []
+    start_time = timeit.default_timer()
     for root, dirs, files in os.walk(os.path.abspath(config.CONTENT_BASE_DIR)):
         for file in files:
             f = os.path.join(root, file)
-            s = zipfile.ZipFile(f)
-            Bs_data = BeautifulSoup(s.open('ComicInfo.xml').read(), "xml")
-            #print(Bs_data.select('Series')[0].text, file=sys.stderr)
-            #print(Bs_data.select('Title')[0].text, file=sys.stderr)
-            CVDB=re.findall('(?<=\[CVDB)(.*)(?=].)', Bs_data.select('Notes')[0].text)
-            #list.append('CVDB'+CVDB[0] + ': '  + Bs_data.select('Series')[0].text + "(" + Bs_data.select('Volume')[0].text + ") : " + Bs_data.select('Number')[0].text  )
-            #print(list, file=sys.stdout)
-            
-            ISSUE=Bs_data.select('Number')[0].text
-            SERIES=Bs_data.select('Series')[0].text
-            VOLUME=Bs_data.select('Volume')[0].text
-            PUBLISHER=Bs_data.select('Publisher')[0].text
-            TITLE=Bs_data.select('Title')[0].text
-            PATH=f 
-            UPDATED=str(datetime.datetime.now())
-            print(UPDATED,file=sys.stdout)
-            sql="INSERT OR REPLACE INTO COMICS (CVDB,ISSUE,SERIES,VOLUME, PUBLISHER, TITLE, FILE,PATH,UPDATED) VALUES ("+CVDB[0]+",'"+ISSUE+"','"+SERIES+"','"+VOLUME+"','"+PUBLISHER+"','"+TITLE+"','"+file+"','" + f + "','" + UPDATED + "')"
-            print(sql,file=sys.stdout)
-            conn.execute(sql);
-            conn.commit()
-    
+            if f.endswith('.cbz'):
+                try:
+                    comiccount = comiccount + 1
+                    s = zipfile.ZipFile(f)
+                    filelist = zipfile.ZipFile.namelist(s)
+                    if filelist[0] == 'ComicInfo.xml':
+                        filemodtime = os.path.getmtime(f)
+                        Bs_data = BeautifulSoup(s.open('ComicInfo.xml').read(), "xml")
+                        CVDB=extras.get_cvdb(Bs_data.select('Notes'))
+                        ISSUE=Bs_data.select('Number')[0].text
+                        SERIES=Bs_data.select('Series')[0].text
+                        VOLUME=Bs_data.select('Volume')[0].text
+                        YEAR=Bs_data.select('Year')[0].text
+                        PUBLISHER=Bs_data.select('Publisher')[0].text
+                        try:
+                            TITLE=Bs_data.select('Title')[0].text
+                        except:
+                            TITLE="" #sometimes title is blank.
+                        PATH=f 
+                        UPDATED=filemodtime
+                        #print(UPDATED,file=sys.stdout)
+                        #sql="INSERT OR REPLACE INTO COMICS (CVDB,ISSUE,SERIES,VOLUME, PUBLISHER, TITLE, FILE,PATH,UPDATED) VALUES ("+CVDB+",'"+ISSUE+"','"+SERIES+"','"+VOLUME+"','"+PUBLISHER+"','"+TITLE+"','"+file+"','" + f + "','" + UPDATED + "')"
+                        #print(sql,file=sys.stdout)
+                        #conn.execute(sql);
+                        
+                        # CREATE TABLE IF MISSING
+                        # create table COMICS (CVDB, ISSUE, SERIES,VOLUME,PUBLISHER,TITLE,FILE,PATH,UPDATED,PRIMARY KEY(CVDB))
+                        try:
+                            query = "SELECT UPDATED FROM COMICS WHERE CVDB = '" + str(CVDB) + "';"
+                            savedmodtime = conn.execute(query).fetchone()[0]
+                        except:
+                            savedmodtime = 0
+                        if savedmodtime < filemodtime:
+                            conn.execute("INSERT OR REPLACE INTO COMICS (CVDB,ISSUE,SERIES,VOLUME, YEAR, PUBLISHER, TITLE, FILE,PATH,UPDATED) VALUES (?,?,?,?,?,?,?,?,?,?)", (CVDB, ISSUE, SERIES, VOLUME, YEAR, PUBLISHER, TITLE, file, f, UPDATED))
+                            conn.commit()
+                            config._print("Adding: " + str(CVDB))
+                            importcount = importcount + 1
+                        elif Path(config.THUMBNAIL_DIR + "/" + str(CVDB) + ".jpg").exists() == False:
+                            cover = s.open(filelist[1]).read()
+                            c = open(config.THUMBNAIL_DIR + "/" + str(CVDB) + ".jpg", 'wb+')
+                            c.write(cover)
+                            c.close()
+                            coverscount = coverscount + 1
+                        else:
+                            config._print("Skipping: " + f)
+                            skippedcount = skippedcount + 1
+                except Exception as e:
+                    errorcount = errorcount + 1
+                    comics_with_errors.append(f)
+                    config._print(e)
+    config._print(comics_with_errors)
     conn.close()
-    return "yay"
+    elapsed = timeit.default_timer() - start_time
+    elapsed_time = "IMPORTED IN: " + str(round(elapsed,2)) + "s"
+    import_stats = elapsed_time + "<br>Comics: " + str(comiccount) + "<br>Imported: " + str(importcount) + "<br>Covers: " + str(coverscount) + "<br>Skipped: " + str(skippedcount) + "<br>Errors: " + str(errorcount)
+    return import_stats #+ "<br>" + ['<li>' + x + '</li>' for x in comics_with_errors]
 
 @app.route("/content/<path:path>")
 @auth.login_required
 def send_content(path):
+    #print('content')
     return send_from_directory(config.CONTENT_BASE_DIR, path)
 
+@app.route("/image/<path:path>")
+def image(path):
+    return send_from_directory(config.THUMBNAIL_DIR,path)
+
 @app.route("/catalog")
+@app.route("/catalog/")
 @app.route("/catalog/<path:path>")
 @auth.login_required
 def catalog(path=""):
-    start_time = timeit.default_timer()
-    print(request.root_url) 
+    config._print("path: " + path)
+    config._print("root_url: " + request.root_url)
+    config._print("url: " + request.url)
+    config._print("CONTENT_BASE_DIR: " + config.CONTENT_BASE_DIR)
+    #print("PRESSED ON")
+    #start_time = timeit.default_timer()
+    #print(request.root_url) 
     c = fromdir(request.root_url, request.url, config.CONTENT_BASE_DIR, path)
-    elapsed = timeit.default_timer() - start_time
-    print(elapsed)
+    #print("c: ")
+    #pprint(vars(c))
+    #for x in c.entries:
+    #    for y in x.links:
+    #        pprint(y.href)
+    #print("------")
+    #elapsed = timeit.default_timer() - start_time
+    #print("-----------------------------------------------------------------------------------------------------------------------")
+    #print("RENDERED IN: " + str(round(elapsed,2))+"s")
     
     return c.render()
 

opds/catalog.py

@@ -5,7 +5,8 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape
 from .entry import Entry
 from .link import Link
 import sqlite3,json
-
+import config
+import extras
 
 class Catalog(object):
     def __init__(
@@ -47,15 +48,22 @@ def fromsearch(root_url, url, content_base_path, content_relative_path):
     return c
 
 def fromdir(root_url, url, content_base_path, content_relative_path):
-
-
+    
     path = os.path.join(content_base_path, content_relative_path)
 
-    #print(path)
-    c = Catalog(
-        title=os.path.basename(os.path.dirname(path)), root_url=root_url, url=url
-    )
-    #print(c.url)
+    if os.path.basename(content_relative_path) == "":
+        c = Catalog(
+            title="Comics",
+            root_url=root_url,
+            url=url
+            )
+    else:
+        c = Catalog(
+            title=extras.xmlesc(os.path.basename(content_relative_path)),
+            root_url=root_url, 
+            url=url
+        )
+    #title=os.path.basename(os.path.dirname(path)), root_url=root_url, url=url
 
     ##########WORKING AREA###########
     searchArr=[]
@@ -71,20 +79,24 @@ def fromdir(root_url, url, content_base_path, content_relative_path):
     print(searchArr) 
     ######################
 
+    
+    
 
     if not "search" in c.url:
         onlydirs = [
             f for f in os.listdir(path) if not os.path.isfile(os.path.join(path, f))
         ]
-    #print(onlydirs)
+        onlydirs.sort()
+        print(onlydirs)
         for dirname in onlydirs:
+            print(dirname)
             link = Link(
-                href=quote(f"/catalog/{content_relative_path}/{dirname}"),
+                href=quote(f"/catalog/{content_relative_path}/{dirname}").replace('//','/'), #windows fix
                 rel="subsection",
                 rpath=path,
                 type="application/atom+xml;profile=opds-catalog;kind=acquisition",
             )
-            c.add_entry(Entry(title=dirname, id=uuid4(), links=[link]))
+            c.add_entry(Entry(title=extras.xmlesc(dirname), id=uuid4(), links=[link]))
 
     
     if c.url.endswith("/catalog"):
@@ -97,86 +109,120 @@ def fromdir(root_url, url, content_base_path, content_relative_path):
                 rpath=path,
                 type="application/atom+xml;profile=opds-catalog;kind=acquisition",
             )
-            c.add_entry(Entry(title="Search["+i+"]",id=uuid4(),links=[link2]))
+            c.add_entry(Entry(title="["+i+"]",id=uuid4(),links=[link2]))
 
     if not "search" in c.url:
         onlyfiles = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
-        #print(onlyfiles)
+        onlyfiles.sort()
         for filename in onlyfiles:
+            if not filename.endswith('cbz'):
+                continue
             link = Link(
                 href=quote(f"/content/{content_relative_path}/{filename}"),
                 rel="http://opds-spec.org/acquisition",
                 rpath=path,
                 type=mimetype(filename),
             )
-            c.add_entry(Entry(title=filename.rsplit(".",1)[0], id=uuid4(), links=[link]))
+
+            #c.add_entry(Entry(title=filename.rsplit(".",1)[0], id=uuid4(), links=[link]))
+            c.add_entry(Entry(title=extras.xmlesc(filename).rsplit(".",1)[0], id=uuid4(), links=[link]))
+        
             #fixed issue with multiple . in filename
             #print(c.render()) 
     else:
         with open('test.json') as fi:
             data=json.load(fi)
-            print("--> LOADED 2 FILE") # try and get this as low as possible.
+            config._print("--> LOADED 2 FILE") # try and get this as low as possible.
         for e in data:
             for key, value in e.items():
-                print(key)
+                config._print(key)
                 searchArr.append(key)
         for i in searchArr:
-            print(i)
+            config._print("i (in searchArr): " + i)
+            config._print("quote i: " + quote(f""+i))
             if quote(f""+i) in c.url:
                 conn = sqlite3.connect('app.db')
-                print(data)
                 for e in data:
+                    config._print("e (in data): " + str(e))
                     for key, value in e.items():
-                        print(key)
+                        config._print("key: " + key)
                         if key == i:
+                            config._print("key <" + str(key) + "> matches <" + str(i) + ">")
                             query="SELECT * FROM COMICS where "
-                            for i in value:
+                            for h in value:
                                 first=True
-                                for j,k in i.items():
+                                for j,k in h.items():
+                                    
                                     if j == 'SQL':
                                         query = query + k
                                     if k != '' and j != "SQL":
-                     #                   print(j,k)
-                                        if not first:
+                                        config._print(j)
+                                        config._print(k)
+                                        config._print(query)
+                                        if not first and j != 'limit':
                                             query = query + "and "
+                                            config._print(query)
                                         if type(k) == list:
-                     #                       print(k)
+                                            config._print(k)
                                             if j == "series" or j == "title":
                                                 firstS = True
                                                 query = query + "("
+                                                config._print(query)
                                                 for l in k:
                                                     if not firstS:
                                                         query = query + "or "
+                                                        config._print(query)
                                                     query = query + j + " like '%" + l + "%' "
+                                                    config._print(query)
                                                     if firstS: 
                                                         firstS = False
                                                 query = query + ") "
+                                                config._print(query)
                                             else:
                                                 query = query + j + " in (" 
+                                                config._print(query)
                                                 firstL = True
                                                 for l in k:
                                                     if not firstL: 
                                                         query = query + ","
-                                                    query = query + "'" + l + "'"
+                                                        config._print(query)
+                                                    query = query + "'" + str(l) + "'"
+                                                    config._print(query)
                                                     if firstL:
                                                         firstL = False
                                                 query = query + ") "
+                                                config._print(query)
 
+                                        elif j != 'limit':
+                                            query = query + j + " like '%" + str(k) + "%' "
+                                            config._print(query)
+                                        elif j == 'limit':
+                                            config.DEFAULT_SEARCH_NUMBER = k
                                         else:
-                                            query = query + j + " like '%" + k + "%' "
+                                            print(">>>>>>>>>>>ERROR THIS SHOULD NOT HAPPEN<<<<<<<<<<<")
                                         if first:
                                             first = False
-                            query = query + ";"
-                            print("----> " + query)
+                                                                                
+                                query = query + " order by series asc, cast(issue as unsigned) asc "
+                                if config.DEFAULT_SEARCH_NUMBER != 0:
+                                    query = query + "LIMIT " + str(config.DEFAULT_SEARCH_NUMBER) + ";"
+                                else:
+                                    query = query + ";"
+                                break
+                        else: 
+                            config._print("key <" + str(key) + "> DOES NOT match <" + str(i) + ">")
+                    
+                config._print("----> " + query)
                                 
                 sql = query
                 #sql="SELECT * from COMICS where SERIES like '%" + i+ "%' or Title like '%" + i+ "%';"
-                print(sql)
+                #config._print(sql)
                 s = conn.execute(sql)
                 #list=[] 
                 for r in s:
-                    #print(r)
-                    tUrl=f""+r[7].replace("/home/drudoo/ComicsTest/Comics/","/content/")
+                    #config._print(r)
+                    tUrl=f""+r[7].replace('\\','/').replace(config.WIN_DRIVE_LETTER + ':','').replace(config.CONTENT_BASE_DIR,"/content")
+                    #config._print(tUrl)
                     tTitle=r[6]
                     link3 = Link(
                         #href=quote(f"/content/DC Comics/Earth Cities/Gotham City/Batgirl/Annual/(2012) Batgirl Annual/Batgirl Annual #001 - The Blood That Moves Us [December, 2012].cbz"),
@@ -185,6 +231,7 @@ def fromdir(root_url, url, content_base_path, content_relative_path):
                         rpath=path,
                         type="application/x-cbz",
                     )
+                    #config._print(link3.href)
                     c.add_entry(
                         Entry(
                             title=tTitle,
@@ -192,12 +239,12 @@ def fromdir(root_url, url, content_base_path, content_relative_path):
                             links=[link3]
                             )
                         )
-
-
+    #print(c.title)
     return c
 
 
 
+
 def mimetype(path):
     extension = path.split(".")[-1].lower()
     if extension == "pdf":

opds/entry.py

@@ -1,6 +1,10 @@
 import zipfile
 from bs4 import BeautifulSoup
 import os
+import re
+
+import extras
+import config
 
 class Entry(object):
     valid_keys = (
@@ -23,7 +27,10 @@ class Entry(object):
         "oai_updatedates",
         "authors",
         "formats",
+        "size",
         "links",
+        "cover",
+        "covertype"
     )
 
     required_keys = ("id", "title", "links")
@@ -46,29 +53,57 @@ class Entry(object):
         
         #print(">>entry.py")
         #print(kwargs)
+        print(kwargs["title"])
         #print(kwargs["links"][0].get("rpath"))
         #print("--end entry.py")
+        try:
+            if kwargs["links"][0].get("type") == 'application/x-cbz':
+                f=self.links[0].get("rpath")+"/"+self.title+".cbz"
+                if os.path.exists(f): 
+                    s = zipfile.ZipFile(f)
+                    self.size = extras.get_size(f, 'mb')
+                    data=BeautifulSoup(s.open('ComicInfo.xml').read(), features="xml")
+                    #self.cover=s.open('P00001.jpg').read()
+
+                    if data.select('Writer') != []:
+                        self.authors = data.select('Writer')[0].text.split(",")
+                    else:
+                        config._print("No Writer found: " + str(data.select('Writer')))
+                        
+                    self.cover = "/image/" + extras.get_cvdb(data.select('Notes')) + ".jpg"
+                    #if data.select('Title') != []:
+                    #    self.title = data.select('Title')[0]
+                        
+                    #    print(data.select('Title')[0])
+                    title = data.select('Title')[0].text.replace("&","&")
+                    kwargs["title"] = title
+                    print(title)
+                    if data.select('Summary') != []:
+                        #print(data.select('Summary')[0].text)
+                        self.summary = data.select('Summary')[0]
+                    else:
+                        config._print("No Summary found: " + str(data.select('Summary')))
+                    
+
+                #print(data)
+                #print(kwargs["links"][0])
+                #print(data.select('Series')[0].text)
+                #print(kwargs["links"][0].get("rpath"))
+                    if data.select('Series')[0].text in kwargs["links"][0].get("rpath"):
+                        releasedate=data.select('Year')[0].text+"-"+data.select('Month')[0].text.zfill(2)+"-"+data.select('Day')[0].text.zfill(2)
+                        try:
+                            self.title = "#"+data.select('Number')[0].text.zfill(2) + ": " + title + " (" + releasedate + ") [" + str(self.size) + "MB]"
+                        except:
+                            self.title = "#"+data.select('Number')[0].text.zfill(2) + " (" + releasedate + ") [" + str(self.size) + "MB]"
+                #print(self.title)
+                    else:
+                        self.title = title
 
-        if kwargs["links"][0].get("type") == 'application/x-cbz':
-            f=self.links[0].get("rpath")+"/"+self.title+".cbz"
-            if os.path.exists(f): 
-                s = zipfile.ZipFile(f)
-                data=BeautifulSoup(s.open('ComicInfo.xml').read(), "xml")
-            #print(data)
-            #print(kwargs["links"][0])
-            #print(data.select('Series')[0].text)
-            #print(kwargs["links"][0].get("rpath"))
-                if data.select('Series')[0].text in kwargs["links"][0].get("rpath"):
-                    releasedate=data.select('Year')[0].text+"-"+data.select('Month')[0].text.zfill(2)+"-"+data.select('Day')[0].text.zfill(2)
-                    self.title = "#"+data.select('Number')[0].text.zfill(2) + ": " + data.select('Title')[0].text + " (" + releasedate + ")"
-            #print(self.title)
                 else:
                     self.title = kwargs["title"]
-            else:
-                self.title = kwargs["title"]
-            #self.title = data.select('Title')[0].text
-            
-
+                #self.title = data.select('Title')[0].text
+        except Exception as e:
+            config._print(e)
     def get(self, key):
         return self._data.get(key, None)
 

opds/search.py

@@ -0,0 +1,237 @@
+import os
+from uuid import uuid4
+from urllib.parse import quote
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+from .entry import Entry
+from .link import Link
+import sqlite3,json
+import config
+import extras
+
+class Search(object):
+    def __init__(
+        self,
+        title,
+    ):
+        self.title = title
+
+    def render(self):
+        env = Environment(
+            loader=FileSystemLoader(
+                searchpath=os.path.join(os.path.dirname(__file__), "templates")
+            ),
+            autoescape=select_autoescape(["html", "xml"]),
+        )
+        template = env.get_template("catalog.opds.jinja2")
+        return template.render(catalog=self)
+
+def fromdir(root_url, url, content_base_path, content_relative_path):
+    
+    path = os.path.join(content_base_path, content_relative_path)
+
+    if os.path.basename(content_relative_path) == "":
+        c = Catalog(
+            title="Comics",
+            root_url=root_url,
+            url=url
+            )
+    else:
+        c = Catalog(
+            title=extras.xmlesc(os.path.basename(content_relative_path)),
+            root_url=root_url, 
+            url=url
+        )
+    #title=os.path.basename(os.path.dirname(path)), root_url=root_url, url=url
+
+    ##########WORKING AREA###########
+    searchArr=[]
+    if c.url.endswith("/catalog"):
+        with open('test.json') as fi:
+            data=json.load(fi)
+            print("--> LOADED FILE") # try and get this as low as possible.
+    #searchArr=["Girl","Bat","Part One"]
+    
+            for e in data:
+                for key, value in e.items():
+                    searchArr.append(key)
+    print(searchArr) 
+    ######################
+
+    
+    
+
+    if not "search" in c.url:
+        onlydirs = [
+            f for f in os.listdir(path) if not os.path.isfile(os.path.join(path, f))
+        ]
+        onlydirs.sort()
+        print(onlydirs)
+        for dirname in onlydirs:
+            print(dirname)
+            link = Link(
+                href=quote(f"/catalog/{content_relative_path}/{dirname}").replace('//','/'), #windows fix
+                rel="subsection",
+                rpath=path,
+                type="application/atom+xml;profile=opds-catalog;kind=acquisition",
+            )
+            c.add_entry(Entry(title=extras.xmlesc(dirname), id=uuid4(), links=[link]))
+
+    
+    if c.url.endswith("/catalog"):
+        
+        for i in searchArr:
+
+            link2 = Link(
+                href=quote(f"/catalog/search["+i+"]"),
+                rel="subsection",
+                rpath=path,
+                type="application/atom+xml;profile=opds-catalog;kind=acquisition",
+            )
+            c.add_entry(Entry(title="["+i+"]",id=uuid4(),links=[link2]))
+
+    if not "search" in c.url:
+        onlyfiles = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))]
+        onlyfiles.sort()
+        for filename in onlyfiles:
+            if not filename.endswith('cbz'):
+                continue
+            link = Link(
+                href=quote(f"/content/{content_relative_path}/{filename}"),
+                rel="http://opds-spec.org/acquisition",
+                rpath=path,
+                type=mimetype(filename),
+            )
+
+            #c.add_entry(Entry(title=filename.rsplit(".",1)[0], id=uuid4(), links=[link]))
+            c.add_entry(Entry(title=extras.xmlesc(filename).rsplit(".",1)[0], id=uuid4(), links=[link]))
+        
+            #fixed issue with multiple . in filename
+            #print(c.render()) 
+    else:
+        with open('test.json') as fi:
+            data=json.load(fi)
+            config._print("--> LOADED 2 FILE") # try and get this as low as possible.
+        for e in data:
+            for key, value in e.items():
+                config._print(key)
+                searchArr.append(key)
+        for i in searchArr:
+            config._print("i (in searchArr): " + i)
+            config._print("quote i: " + quote(f""+i))
+            if quote(f""+i) in c.url:
+                conn = sqlite3.connect('app.db')
+                for e in data:
+                    config._print("e (in data): " + str(e))
+                    for key, value in e.items():
+                        config._print("key: " + key)
+                        if key == i:
+                            config._print("key <" + str(key) + "> matches <" + str(i) + ">")
+                            query="SELECT * FROM COMICS where "
+                            for h in value:
+                                first=True
+                                for j,k in h.items():
+                                    
+                                    if j == 'SQL':
+                                        query = query + k
+                                    if k != '' and j != "SQL":
+                                        config._print(j)
+                                        config._print(k)
+                                        config._print(query)
+                                        if not first and j != 'limit':
+                                            query = query + "and "
+                                            config._print(query)
+                                        if type(k) == list:
+                                            config._print(k)
+                                            if j == "series" or j == "title":
+                                                firstS = True
+                                                query = query + "("
+                                                config._print(query)
+                                                for l in k:
+                                                    if not firstS:
+                                                        query = query + "or "
+                                                        config._print(query)
+                                                    query = query + j + " like '%" + l + "%' "
+                                                    config._print(query)
+                                                    if firstS: 
+                                                        firstS = False
+                                                query = query + ") "
+                                                config._print(query)
+                                            else:
+                                                query = query + j + " in (" 
+                                                config._print(query)
+                                                firstL = True
+                                                for l in k:
+                                                    if not firstL: 
+                                                        query = query + ","
+                                                        config._print(query)
+                                                    query = query + "'" + str(l) + "'"
+                                                    config._print(query)
+                                                    if firstL:
+                                                        firstL = False
+                                                query = query + ") "
+                                                config._print(query)
+
+                                        elif j != 'limit':
+                                            query = query + j + " like '%" + str(k) + "%' "
+                                            config._print(query)
+                                        elif j == 'limit':
+                                            config.DEFAULT_SEARCH_NUMBER = k
+                                        else:
+                                            print(">>>>>>>>>>>ERROR THIS SHOULD NOT HAPPEN<<<<<<<<<<<")
+                                        if first:
+                                            first = False
+                                                                                
+                                query = query + " order by series asc, cast(issue as unsigned) asc "
+                                if config.DEFAULT_SEARCH_NUMBER != 0:
+                                    query = query + "LIMIT " + str(config.DEFAULT_SEARCH_NUMBER) + ";"
+                                else:
+                                    query = query + ";"
+                                break
+                        else: 
+                            config._print("key <" + str(key) + "> DOES NOT match <" + str(i) + ">")
+                    
+                config._print("----> " + query)
+                                
+                sql = query
+                #sql="SELECT * from COMICS where SERIES like '%" + i+ "%' or Title like '%" + i+ "%';"
+                #config._print(sql)
+                s = conn.execute(sql)
+                #list=[] 
+                for r in s:
+                    #config._print(r)
+                    tUrl=f""+r[7].replace('\\','/').replace(config.WIN_DRIVE_LETTER + ':','').replace(config.CONTENT_BASE_DIR,"/content")
+                    #config._print(tUrl)
+                    tTitle=r[6]
+                    link3 = Link(
+                        #href=quote(f"/content/DC Comics/Earth Cities/Gotham City/Batgirl/Annual/(2012) Batgirl Annual/Batgirl Annual #001 - The Blood That Moves Us [December, 2012].cbz"),
+                        href=quote(tUrl),
+                        rel="http://opds-spec.org/acquisition",
+                        rpath=path,
+                        type="application/x-cbz",
+                    )
+                    #config._print(link3.href)
+                    c.add_entry(
+                        Entry(
+                            title=tTitle,
+                            id=uuid4(),
+                            links=[link3]
+                            )
+                        )
+    #print(c.title)
+    return c
+
+
+
+
+def mimetype(path):
+    extension = path.split(".")[-1].lower()
+    if extension == "pdf":
+        return "application/pdf"
+    elif extension == "epub":
+        return "application/epub"
+    elif extension == "mobi":
+        return "application/mobi"
+    elif extension == "cbz":
+        return "application/x-cbz"
+    else:
+        return "application/unknown"

opds/templates/catalog.opds.jinja2

@@ -27,7 +27,19 @@
   <entry>
     <title>{{ entry.title }}</title>
     <id>{{ entry.id }}</id>
+    <summary type="text">{{ entry.summary }}</summary>
+    {% for author in entry.authors %}
+    <author>
+      <name>{{ author }}</name>
+    </author>
+    {% endfor %}
     {% if entry.updated %} <updated>{{ entry.updated }}</updated> {% endif %}
+    <link rel="http://opds-spec.org/image"    
+          href="{{ entry.cover }}"
+          type="image/jpg"/>
+    <link rel="http://opds-spec.org/image/thumbnail"    
+          href="{{ entry.cover }}"
+          type="image/jpg"/>
     {% for link in entry.links %}
     <link rel="{{ link.rel }}"
           href="{{ link.href }}"

opds/test.json

@@ -1,77 +0,0 @@
-[
-    {
-        "SQL TEST": [
-            {
-                "SQL": "(series like '%Aqua%' or series like '%girl%') and issue in ('1','2','5','10') and title not like '%Annual%'"
-            }
-        ]
-    },{
-        "Man 2020,2019": [
-            {
-                "title": "Man",
-                "volume": [
-                    "2020",
-                    "2019"
-                ],
-                "publisher": "",
-                "series": "",
-                "issue": ""
-            }
-        ]
-    },
-    {
-        "DC (BAT)": [
-            {
-                "title": "",
-                "volume": "",
-                "publisher": "DC Comics",
-                "series": "Bat",
-                "issue": ""
-            }
-        ]
-    },{
-        "Marvel": [
-            {
-                "title": "",
-                "volume": "",
-                "publisher": "marvel",
-                "series": "",
-                "issue": ""
-            }
-        ]
-    },
-    {
-        "Girl": [
-            {
-                "title": ["girl","man","World"],
-                "volume": "",
-                "publisher": "",
-                "series": "girl",
-                "issue": ""
-            }
-        ]
-    },
-    {
-        "Aquaman": [
-            {
-                "title": "",
-                "volume": "",
-                "publisher": "",
-                "series": "aquaman",
-                "issue": ["2","3","5","10","22"]
-            }
-        ]
-    }
-,
-    {
-        "Girl series": [
-            {
-                "title": "",
-                "volume": "",
-                "publisher": "",
-                "series": "girl",
-                "issue": "2"
-            }
-        ]
-    }
-]

requirements.txt

@@ -1,7 +1,10 @@
 Flask==2.0.2
+Werkzeug==2.2.2
+numpy
 Jinja2==3.0.2
 requests==2.26.0
 Flask-HTTPAuth==4.5.0
 gevent==21.8.0
 bs4
 lxml
+Pillow

templates/first.html

@@ -0,0 +1,14 @@
+<html>
+    <body>
+
+        <form method="post" action="/">
+
+                <input type="submit" value="Encrypt" name="Encrypt"/>
+                <input type="submit" value="Decrypt" name="Decrypt" />
+
+        </form>
+    </body>
+</html>
+    
+
+<p>{{ result }}</p>

templates/start.html

@@ -0,0 +1,91 @@
+<html>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.5.0/Chart.min.js"></script>
+<body>
+
+  {% if first and request.args.get('first') == None %}
+    <form method="post">
+        <p>DB is missing table. <input type="submit" value="Create" name="Create"/>
+    </form>
+    {% endif %}
+
+  {% if result == [] %}
+
+    <form method="post">
+        <p>No comics imported. <input type="submit" value="Import" name="Import"/>
+    </form>
+    {% endif %}
+
+    {% if total != covers %}
+    <form method="post">
+      <p>Some covers missing <input type="submit" value="Generate" name="Generate"/>
+    </form>
+  {% endif %}
+
+    <h1>Total Comics: {{ total }}</h1>
+
+<canvas id="myChart" style="width:100%;max-width:600px"></canvas>
+
+<script>
+var xValues = {{ pub_list | safe }};
+var yValues = {{ count }};
+var barColors = ["red", "green","blue","orange", "purple"];
+
+new Chart("myChart", {
+  type: "bar",
+  data: {
+    labels: xValues,
+    datasets: [{
+      backgroundColor: barColors,
+      data: yValues
+    }]
+  },
+  options: {
+    legend: {display: false},
+    title: {
+      display: true,
+      text: "Publishers"
+    }
+  }
+});
+</script>
+
+
+<canvas id="myChart3" style="width:100%;max-width:600px"></canvas>
+
+
+<script>
+var xValues = {{ x | safe }};
+var yValues = {{ y | safe }};
+
+new Chart("myChart3", {
+  type: "line",
+  data: {
+    labels: xValues,
+    datasets: [{
+      fill: false,
+      backgroundColor: "rgba(0,0,255,1.0)",
+      borderColor: "rgba(0,0,255,0.1)",
+      data: yValues
+    }]
+  },
+  options: {
+    legend: {display: false},
+  }
+});
+</script>
+
+
+
+<table id="comics">
+  {% for i in result %}
+  <tr>
+    {% for j in range(0,9) %}
+    <td>{{ i[j] }}</td>
+    {% endfor %}
+  </tr>
+  {% endfor %}
+</table>
+
+
+</body>
+</html>

templates/status.html

@@ -0,0 +1,75 @@
+<!doctype html>
+<html>
+
+<head>
+  <meta charset="UTF-8">
+
+  <style>
+  
+  body {
+    background-color: #D64F2A;
+  }
+  
+  .progress {
+    display: flex;
+    position: absolute;
+    height: 100%;
+    width: 100%;
+  }
+  
+  .status {
+    color: white;
+    margin: auto;
+  }
+
+  .status h2 {
+    padding: 50px;
+    font-size: 80px;
+    font-weight: bold;
+  }
+  
+  </style>
+
+  <title>Status Update</title>
+
+</head>
+
+<body>
+  <div class="progress">
+    <div class="status">
+      <h2 id="innerStatus">Loading...</h2>
+    </div>
+  </div>
+</body>
+
+<script>
+var timeout;
+
+async function getStatus() {
+
+  let get;
+  
+  try {
+    const res = await fetch("/status");
+    get = await res.json();
+  } catch (e) {
+    console.error("Error: ", e);
+  }
+  
+  document.getElementById("innerStatus").innerHTML = Math.round(get.status / get.total * 100,0) + "&percnt;";
+  
+  if (get.status == get.total){
+    document.getElementById("innerStatus").innerHTML += " Done.";
+    clearTimeout(timeout);
+    // Simulate a mouse click:
+    window.location.href = "/";
+    return false;
+  }
+   
+  timeout = setTimeout(getStatus, 1000);
+}
+
+getStatus();
+</script>
+
+</html>

test.json

@@ -1,12 +1,24 @@
 [
     {
-        "SQL TEST": [
+        "Amazons": [
             {
-                "SQL": "(series like '%Aqua%' or series like '%girl%') and issue in ('1','2','5','10') and title not like '%Annual%'"
+                "SQL": "(series = 'Nubia & the Amazons' and issue in ('1','2','3','4','5','6')) or (series like 'Trial of the Amazons%' and issue in ('1','2')) or (series = 'Wonder Woman' and issue in ('785','786','787'))"
             }
         ]
-    },{
-        "Man 2020,2019": [
+    },
+    {
+        "Letter 44": [
+            {
+                "title": "",
+                "volume": "",
+                "publisher": "",
+                "series": "Letter 44",
+                "issue": ""
+            }
+        ]
+    },
+    {
+        "Man 2020 or 2019": [
             {
                 "title": "Man",
                 "volume": [
@@ -20,7 +32,7 @@
         ]
     },
     {
-        "DC (BAT)": [
+        "DC BAT": [
             {
                 "title": "",
                 "volume": "",
@@ -29,7 +41,8 @@
                 "issue": ""
             }
         ]
-    },{
+    },
+    {
         "Marvel": [
             {
                 "title": "",
@@ -43,7 +56,11 @@
     {
         "Girl": [
             {
-                "title": ["girl","man","World"],
+                "title": [
+                    "girl",
+                    "man",
+                    "World"
+                ],
                 "volume": "",
                 "publisher": "",
                 "series": "girl",
@@ -51,27 +68,63 @@
             }
         ]
     },
+    {
+        "number 1": [
+            {
+                "title": "",
+                "volume": "",
+                "publisher": "",
+                "series": "",
+                "issue": [
+                    "1"
+                ]
+            }
+        ]
+    },
     {
         "Aquaman": [
             {
-                "title": "",
+                "title": [
+                    "Tyrant King",
+                    "The Deluge Act Three",
+                    "Warhead Part One",
+                    "Black Mantra"
+                ],
                 "volume": "",
                 "publisher": "",
-                "series": "aquaman",
-                "issue": ["2","3","5","10","22"]
+                "series": "",
+                "issue": ""
             }
         ]
-    }
-,
+    },
     {
-        "Girl series": [
+        "2020-2022 DC Comics": [
             {
                 "title": "",
-                "volume": "",
+                "volume": [
+                    "2020",
+                    "2022"
+                ],
+                "publisher": "DC Comics",
+                "series": [
+                    "Batman",
+                    "Detective Comics"
+                ],
+                "issue": "",
+                "limit": 50
+            }
+        ]
+    },
+    {
+        "New Series 2023": [
+            {
+                "title": "",
+                "volume": "2023",
                 "publisher": "",
-                "series": "girl",
-                "issue": "2"
+                "series": "",
+                "issue": "1",
+                "limit": 30
             }
         ]
     }
-]
+]