FrederikBaerentsen/DataHoarder_scripts
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added commnets

Browse Source
This commit is contained in:
Frederik Baerentsen 2020-06-24 08:51:26 +02:00
parent d296cf0f97
commit 67312953a4
1 changed files with 30 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
lego_dl.sh
View File

@ -1,17 +1,37 @@
#!/bin/bash #!/bin/bash
##############################
#
# This scripts uses the brickset instructions file to get links and set numbers.
# In order to get themes, we get themes from rebrickable.
#
# Files are saved as:
# $tID""$setDESC""${setNAME// /_}"_("$setTHEME"_"$setYEAR").pdf"
# 1190-1_Retro_Buggy_(Town_1999).pdf
#
# If a set contains multiple filesm the file name will be:
# 1190-1_(1_of_3)_Retro_Buggy_(Town_1999).pdf
#
# Req:
# bash, awk, grep, sed, curl, wget
#
# Get themes.csv and sets.cvs from https://rebrickable.com/downloads # Get themes.csv and sets.cvs from https://rebrickable.com/downloads
#
echo "----> Starting Download of all LEGO instructions from https://brickset.com/exportscripts/instructions"
firstline=0
logfile="lego_errors.log"
downloadFolder="../Instructions"
# error_level: # error_level:
# 0 no errors reported # 0 no errors reported
# 1 Download errors reported # 1 Download errors reported
# 2 Existing files and download errors reported # 2 Existing files and download errors reported
#
##############################
echo "----> Starting Download of all LEGO instructions from https://brickset.com/exportscripts/instructions"
firstline=0
##### CHANGE HERE #####
logfile="lego_errors.log"
downloadFolder="../Instructions"
error_level=0 error_level=0
##### STOP CHANGE #####
echo "" > $logfile echo "" > $logfile
THEMES="themes.csv" THEMES="themes.csv"
@ -37,13 +57,9 @@ sed 1,1d instructions | sed -r 's/("[^",]+),([^",]+")/\1###\2/g' | awk -F, '{pri
while IFS='|' read -r ID LINK NAME DESC ADDED RETRIVED while IFS='|' read -r ID LINK NAME DESC ADDED RETRIVED
do do
#echo $ID $LINK $NAME $DESC $ADDED $RETRIVED
if [ "$firstline" = 0 ]; then if [ "$firstline" = 0 ]; then
firstline=1 firstline=1
else else
#echo "I got:$ID - $LINK - $NAME - $DESC - $ADDED - $RETRIVED"
tID=$(sed -e 's/^"//' -e 's/"$//' <<<"$ID") tID=$(sed -e 's/^"//' -e 's/"$//' <<<"$ID")
tLINK=$(sed -e 's/^"//' -e 's/"$//' <<<"$LINK") tLINK=$(sed -e 's/^"//' -e 's/"$//' <<<"$LINK")
#tNAME=$(sed -e 's/^"//' -e 's/"$//' <<<"$NAME") #tNAME=$(sed -e 's/^"//' -e 's/"$//' <<<"$NAME")
@ -57,11 +73,8 @@ do
if [[ $tempID != "" ]]; then if [[ $tempID != "" ]]; then
while [[ $tempID != "" ]] while [[ $tempID != "" ]]
do do
#echo "tempID: $tempID"
tthemeName=$(awk -F',' -v id="$tempID" '$1 == id' themes.csv) tthemeName=$(awk -F',' -v id="$tempID" '$1 == id' themes.csv)
#echo $tthemeName
IFS=',' read -r -a tArray <<< "$tthemeName" IFS=',' read -r -a tArray <<< "$tthemeName"
#echo "<${tArray[2]}>"
tempID=${tArray[2]} tempID=${tArray[2]}
themeName=${tArray[1]} themeName=${tArray[1]}
done done
@ -71,8 +84,6 @@ do
setTHEME=$(sed -e 's/[^A-Za-z0-9._-]/_/g' <<< $themeName) setTHEME=$(sed -e 's/[^A-Za-z0-9._-]/_/g' <<< $themeName)
#echo $setYEAR
tADDED=$(sed -e 's/^"//' -e 's/"$//' <<<"$ADDED") tADDED=$(sed -e 's/^"//' -e 's/"$//' <<<"$ADDED")
tDESC=$(sed -e 's/^"//' -e 's/"$//' <<<"$DESC") tDESC=$(sed -e 's/^"//' -e 's/"$//' <<<"$DESC")
ttDESC=$(echo $tDESC | grep -Eo '\s[0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/ //g' | sed 's/\//_of_/g') ttDESC=$(echo $tDESC | grep -Eo '\s[0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/ //g' | sed 's/\//_of_/g')