FrederikBaerentsen/ComicRack_Scripts
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases 2 Wiki Activity

Compare commits

base: FrederikBaerentsen:69a6343c2e9b759f5840ac01f60ac3c93c23288f
Branches Tags
FrederikBaerentsen:master
FrederikBaerentsen:CVIssueCount.v2.1
FrederikBaerentsen:CVIssueCount.v2.0
...
compare: FrederikBaerentsen:6129aa4a7a4b4f0d89a3bb7e57927aad0638b563
Branches Tags
FrederikBaerentsen:master
FrederikBaerentsen:CVIssueCount.v2.1
FrederikBaerentsen:CVIssueCount.v2.0

2 Commits
69a6343c2e ... 6129aa4a7a

Author SHA1 Message Date
FrederikBaerentsen
6129aa4a7a Updated 20221005 2022-10-05 16:32:43 +02:00
FrederikBaerentsen
6aeb1be35d updated to python3 2022-10-05 16:31:47 +02:00
5 changed files with 1096050 additions and 839062 deletions
Show Stats Expand all files Collapse all files

129184
Update Missing/20221005_latest.mcl Normal file
View File

File diff suppressed because one or more lines are too long

1676680
Update Missing/Deleted_Comics.txt
View File

File diff suppressed because it is too large Load Diff

129069
Update Missing/archive/20220926.mcl Normal file
View File

File diff suppressed because one or more lines are too long

125
Update Missing/log/20221005.log Normal file
View File

@ -0,0 +1,125 @@
py: Reading in current database
py: Reading in current database
py: Reading in current database
py: Reading in current database
py: Reading in current database
py: Querying ComicVine for new issues
py:0/2308 Since 2022-09-26
py: Reading in current database
py: Querying ComicVine for new issues
py:0/2308 Since 2022-09-26
py:100/2308 Since 2022-09-26
py:200/2308 Since 2022-09-26
py:300/2308 Since 2022-09-26
py:400/2308 Since 2022-09-26
py:500/2308 Since 2022-09-26
py:600/2308 Since 2022-09-26
py:700/2308 Since 2022-09-26
py:800/2308 Since 2022-09-26
py:900/2308 Since 2022-09-26
py:1000/2308 Since 2022-09-26
py:1100/2308 Since 2022-09-26
py:1200/2308 Since 2022-09-26
py:1300/2308 Since 2022-09-26
py:1400/2308 Since 2022-09-26
py:1500/2308 Since 2022-09-26
py:1600/2308 Since 2022-09-26
py:1700/2308 Since 2022-09-26
py:1800/2308 Since 2022-09-26
py: Error. Trying Again...
py: Error. Trying Again...
py: Error. Trying Again...
py: Error. Trying Again...
py:1900/2308 Since 2022-09-26
py:2000/2308 Since 2022-09-26
py:2100/2308 Since 2022-09-26
py:2200/2308 Since 2022-09-26
py:2300/2308 Since 2022-09-26
py: Reading in current database
py: Querying ComicVine for new issues
py:0/2308 Since 2022-09-26
py:100/2308 Since 2022-09-26
py:200/2308 Since 2022-09-26
py:300/2308 Since 2022-09-26
py:400/2308 Since 2022-09-26
py:500/2308 Since 2022-09-26
py:600/2308 Since 2022-09-26
py:700/2308 Since 2022-09-26
py:800/2308 Since 2022-09-26
py:900/2308 Since 2022-09-26
py:1000/2308 Since 2022-09-26
py:1100/2308 Since 2022-09-26
py:1200/2308 Since 2022-09-26
py:1300/2308 Since 2022-09-26
py:1400/2308 Since 2022-09-26
py:1500/2308 Since 2022-09-26
py:1600/2308 Since 2022-09-26
py:1700/2308 Since 2022-09-26
py:1800/2308 Since 2022-09-26
py:1900/2308 Since 2022-09-26
py:2000/2308 Since 2022-09-26
py:2100/2308 Since 2022-09-26
py:2200/2308 Since 2022-09-26
py:2300/2308 Since 2022-09-26
py: Writing missings to file
py: Reading in current database
py: Querying ComicVine for new issues
py:0/2308 Since 2022-09-26
py:100/2308 Since 2022-09-26
py:200/2308 Since 2022-09-26
py:300/2308 Since 2022-09-26
py:400/2308 Since 2022-09-26
py:500/2308 Since 2022-09-26
py:600/2308 Since 2022-09-26
py:700/2308 Since 2022-09-26
py:800/2308 Since 2022-09-26
py:900/2308 Since 2022-09-26
py:1000/2308 Since 2022-09-26
py:1100/2308 Since 2022-09-26
py:1200/2308 Since 2022-09-26
py:1300/2308 Since 2022-09-26
py:1400/2308 Since 2022-09-26
py:1500/2308 Since 2022-09-26
py:1600/2308 Since 2022-09-26
py:1700/2308 Since 2022-09-26
py:1800/2308 Since 2022-09-26
py:1900/2308 Since 2022-09-26
py:2000/2308 Since 2022-09-26
py:2100/2308 Since 2022-09-26
py:2200/2308 Since 2022-09-26
py:2300/2308 Since 2022-09-26
py: Writing missings to file
py: Reading in current database
py: Querying ComicVine for new issues
py:0/2308 Since 2022-09-26
py:100/2308 Since 2022-09-26
py:200/2308 Since 2022-09-26
py: Error. Trying Again...
py:300/2308 Since 2022-09-26
py: Error. Trying Again...
py: Error. Trying Again...
py:400/2308 Since 2022-09-26
py:500/2308 Since 2022-09-26
py:600/2308 Since 2022-09-26
py:700/2308 Since 2022-09-26
py:800/2308 Since 2022-09-26
py:900/2308 Since 2022-09-26
py:1000/2308 Since 2022-09-26
py: Error. Trying Again...
py: Error. Trying Again...
py: Error. Trying Again...
py:1100/2308 Since 2022-09-26
py:1200/2308 Since 2022-09-26
py:1300/2308 Since 2022-09-26
py:1400/2308 Since 2022-09-26
py:1500/2308 Since 2022-09-26
py:1600/2308 Since 2022-09-26
py:1700/2308 Since 2022-09-26
py:1800/2308 Since 2022-09-26
py:1900/2308 Since 2022-09-26
py:2000/2308 Since 2022-09-26
py:2100/2308 Since 2022-09-26
py:2200/2308 Since 2022-09-26
py:2300/2308 Since 2022-09-26
py: Writing missings to file
py: Writing database to file

54
Update Missing/update_missing.py
View File

@ -68,7 +68,7 @@ today_date = str(date.today().strftime("%Y%m%d"))
f1=open(ROOT_DIR+'log/'+today_date+'.log', 'a') f1=open(ROOT_DIR+'log/'+today_date+'.log', 'a')
if len(sys.argv) > 5: if len(sys.argv) > 5:
print "Using argvs" print("Using argvs")
in_file = str(sys.argv[1]) # missing.mcl in_file = str(sys.argv[1]) # missing.mcl
out_file = str(sys.argv[2]) # updated_missing.mcl out_file = str(sys.argv[2]) # updated_missing.mcl
api_key = str(sys.argv[3]) # ComicVine API key api_key = str(sys.argv[3]) # ComicVine API key
@ -81,14 +81,14 @@ issues_volume = {}
skip_header = True skip_header = True
cont = 0 cont = 0
exit exit
print >>f1, "py: Reading in current database" print("py: Reading in current database")
for line in comiclist: for line in comiclist:
if skip_header: if skip_header:
skip_header = False skip_header = False
continue continue
line_split = unicode(line, encoding='utf-8').replace("\n","").split(";") line_split = line.replace("\n","").split(";")
volume_id = int(line_split[0]) volume_id = int(line_split[0])
if (line_split[1][0] == '"') and (line_split[1][len(line_split[1])] == '"'): if (line_split[1][0] == '"') and (line_split[1][len(line_split[1])] == '"'):
@ -98,14 +98,16 @@ for line in comiclist:
num_split = line_split[2].split(",") num_split = line_split[2].split(",")
for i in range(0,len(issue_split)): for i in range(0,len(issue_split)):
if issues_number.has_key(int(issue_split[i])): #if issues_number.has_key(int(issue_split[i])):
#if int(issue_split[i]) in issue_number:
if issues_number.__contains__(int(issue_split[i])):
cont += 1 cont += 1
issues_number[int(issue_split[i])] = num_split[i] issues_number[int(issue_split[i])] = num_split[i]
issues_volume[int(issue_split[i])] = volume_id issues_volume[int(issue_split[i])] = volume_id
comiclist.close() comiclist.close()
print >>f1, "py: Querying ComicVine for new issues" print("py: Querying ComicVine for new issues")
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'} headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
new_comics_cont = 0 new_comics_cont = 0
old_comics_cont = 0 old_comics_cont = 0
@ -131,14 +133,14 @@ while offset < max:
max = json_obj['number_of_total_results'] max = json_obj['number_of_total_results']
print >>f1, "py:" + str(min(offset,max)) + "/" + str(max) + " Since " + start_date print("py:" + str(min(offset,max)) + "/" + str(max) + " Since " + start_date)
for i in json_obj['results']: for i in json_obj['results']:
volume_id = i['volume']['id'] volume_id = i['volume']['id']
issue_id = i['id'] issue_id = i['id']
num = unicode(i['issue_number']).replace(",",".&@1").replace(";",".&@2").replace("\n","").replace("\r","") num = i['issue_number'].replace(",",".&@1").replace(";",".&@2").replace("\n","").replace("\r","")
if not issues_number.has_key(issue_id): if not issues_number.__contains__(issue_id):
new_comics_cont += 1 new_comics_cont += 1
issues_number[issue_id] = num issues_number[issue_id] = num
issues_volume[issue_id] = volume_id issues_volume[issue_id] = volume_id
@ -155,10 +157,10 @@ while offset < max:
FindingError = False FindingError = False
if skip == 1: if skip == 1:
print >>f1, "py: Comic with error found, id= " + str(issue_id+1) print("py: Comic with error found, id= " + str(issue_id+1))
ErrorIds += ";"+ str(issue_id+1) ErrorIds += ";"+ str(issue_id+1)
comic_skip_cont += 1 comic_skip_cont += 1
print >>f1, "py: Continue loading comics now..." print("py: Continue loading comics now...")
FindingError = True FindingError = True
skip = 0 skip = 0
@ -167,18 +169,18 @@ while offset < max:
except: except:
if retry < 4 and not FindingError: if retry < 4 and not FindingError:
print >>f1, "py: Error. Trying Again..." print("py: Error. Trying Again...")
retry += 1 retry += 1
else: else:
if not FindingError: if not FindingError:
print >>f1, "py: Finding Error in comic list: " + str(100-limit) + "%" print("py: Finding Error in comic list: " + str(100-limit) + "%")
skip = 1 skip = 1
limit -= 1 limit -= 1
if limit == 0 or FindingError: if limit == 0 or FindingError:
print >>f1, "py: Comic with error found, id= " + str(issue_id+offset) print("py: Comic with error found, id= " + str(issue_id+offset))
FindingError = True FindingError = True
limit = 1 limit = 1
offset += 1 offset += 1
@ -187,39 +189,39 @@ while offset < max:
comics = {} comics = {}
for issue_id in issues_number.keys(): for issue_id in issues_number.keys():
if not comics.has_key(issues_volume[issue_id]): if not comics.__contains__(issues_volume[issue_id]):
comics[issues_volume[issue_id]] = {} comics[issues_volume[issue_id]] = {}
comics[issues_volume[issue_id]][issue_id]=issues_number[issue_id] comics[issues_volume[issue_id]][issue_id]=issues_number[issue_id]
print >>f1, "py: Writing missings to file" print("py: Writing missings to file")
deleted_file = open(ROOT_DIR+"Deleted_Comics.txt", "wb") deleted_file = open(ROOT_DIR+"Deleted_Comics.txt", "wb")
for issue_id in non_retrieved_comics.keys(): for issue_id in non_retrieved_comics.keys():
deleted_file.write(str(issue_id)+"\n") deleted_file.write((str(issue_id)+"\n").encode())
deleted_comics_cont += 1 deleted_comics_cont += 1
deleted_file.close() deleted_file.close()
print >>f1, "py: Writing database to file" print("py: Writing database to file")
outfile = open(out_file,"wb") outfile = open(out_file,"wb")
outfile.write("Missing;" + end_date + "\n") outfile.write(("Missing;" + end_date + "\n").encode())
for volume_id in sorted(comics.iterkeys()): for volume_id in sorted(comics):
issues = "" issues = ""
nums = "" nums = ""
for issue_id in sorted(comics[volume_id].iterkeys()): for issue_id in sorted(comics[volume_id]):
issues += str(issue_id) + "," issues += str(issue_id) + ","
nums += comics[volume_id][issue_id] + "," nums += comics[volume_id][issue_id] + ","
issues = issues[:-1] issues = issues[:-1]
outfile.write(str(volume_id) + ";" + issues + ";" + nums.encode('utf-8','ignore') + "\n") outfile.write((str(volume_id) + ";" + issues + ";" + nums + "\n").encode())
outfile.close() outfile.close()
print >>f1, "py: Done! " + str(new_comics_cont) + " comics added to database! (" + str(comic_skip_cont)+ " skipped and " + str(old_comics_cont) + " comics already in database)" print("py: Done! " + str(new_comics_cont) + " comics added to database! (" + str(comic_skip_cont)+ " skipped and " + str(old_comics_cont) + " comics already in database)")
print >>f1, "py: " + str(deleted_comics_cont) + " comics in databased not retrieved in this round." print("py: " + str(deleted_comics_cont) + " comics in databased not retrieved in this round.")
print >>f1, "py: " + str(updated_comics_cont) + " comics updated in database." print("py: " + str(updated_comics_cont) + " comics updated in database.")
print >>f1, "py: Ids with error in server: " + ErrorIds[1:] print("py: Ids with error in server: " + ErrorIds[1:])
print >>f1, "py: " + str(cont) print("py: " + str(cont))
#raw_input("Press Enter to continue...") #raw_input("Press Enter to continue...")