FrederikBaerentsen/DataHoarder_scripts
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fixed errors

Browse Source
This commit is contained in:
Frederik Baerentsen 2020-06-24 10:18:54 +02:00
parent a848096baa
commit ec88a6914f
1 changed files with 121 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
lego_dl.sh
View File

@ -1,48 +1,48 @@
#!/bin/bash #!/bin/bash
############################## ##############################
# #
# NOTE: There is still some small errors with sets with multiple booklets # NOTE: There is still some small errors with sets with multiple booklets
# #
# This scripts uses the brickset instructions file to get links and set numbers. # This scripts uses the brickset instructions file to get links and set numbers.
# In order to get themes, we get themes from rebrickable. # In order to get themes, we get themes from rebrickable.
# #
# Files are saved as: # Files are saved as:
# $tID""$setDESC""${setNAME// /_}"_("$setTHEME"_"$setYEAR").pdf" # $tID""$setDESC""${setNAME// /_}"_("$setTHEME"_"$setYEAR").pdf"
# 1190-1_Retro_Buggy_(Town_1999).pdf # 1190-1_Retro_Buggy_(Town_1999).pdf
# #
# If a set contains multiple filesm the file name will be: # If a set contains multiple filesm the file name will be:
# 2520-1_(1_of_2)_Battle_Arena_(Ninjago_2011).pdf # 2520-1_(1_of_2)_Battle_Arena_(Ninjago_2011).pdf
# 2520-1_(2_of_2)_Battle_Arena_(Ninjago_2011).pdf # 2520-1_(2_of_2)_Battle_Arena_(Ninjago_2011).pdf
# #
# Req: # Req:
# bash, awk, grep, sed, curl, wget # bash, awk, grep, sed, curl, wget
# #
# Get themes.csv and sets.cvs from https://rebrickable.com/downloads # Get themes.csv and sets.cvs from https://rebrickable.com/downloads
# #
# error_level: # error_level:
# 0 no errors reported # 0 no errors reported
# 1 Download errors reported # 1 Download errors reported
# 2 Existing files and download errors reported # 2 Existing files and download errors reported
# #
############################## ##############################
echo "----> Starting Download of all LEGO instructions from https://brickset.com/exportscripts/instructions" echo "----> Starting Download of all LEGO instructions from https://brickset.com/exportscripts/instructions"
firstline=0 firstline=0
##### CHANGE HERE ##### ##### CHANGE HERE #####
logfile="lego_errors.log" logfile="lego_errors.log"
downloadFolder="../Instructions" downloadFolder="../Instructions"
error_level=1 error_level=1
##### STOP CHANGE ##### ##### STOP CHANGE #####
echo "" > $logfile echo "" > $logfile
THEMES="themes.csv" THEMES="themes.csv"
SETS="sets.csv" SETS="sets.csv"
if [ -f "instructions" ]; then if [ -f "instructions" ]; then
echo "---> Using existing instructions file. Manually delete it to redownload..." echo "---> Using existing instructions file. Manually delete it to redownload..."
else else
echo -ne "---> Downloading csv from Brickset..." echo -ne "---> Downloading csv from Brickset..."
wget https://brickset.com/exportscripts/instructions &> /dev/null wget https://brickset.com/exportscripts/instructions &> /dev/null
if [[ "$?" != 0 ]]; then if [[ "$?" != 0 ]]; then
@ -51,15 +51,21 @@ else
else else
echo "Done!" echo "Done!"
fi fi
fi fi
echo "----> Converting csv with ',' to '|'..."
head -1 instructions | sed 's/,/|/g' > instructions.csv
sed 1,1d instructions | sed -r 's/("[^",]+),([^",]+")/\1###\2/g' | awk -F, '{print $1,$2,$3,$4,$5,$6}' | sed 's/###/,/g' | sed 's/" "/"|"/g' >> instructions.csv
while IFS='|' read -r ID LINK NAME DESC ADDED RETRIVED echo "----> Converting csv with ',' to '|'..."
do head -1 instructions | sed 's/,/|/g' > instructions.csv
sed 1,1d instructions | sed -r 's/("[^",]+),([^",]+")/\1###\2/g' | awk -F, '{print $1,$2,$3,$4,$5,$6}' | sed 's/###/,/g' | sed 's/" "/"|"/g' >> instructions.csv
LINES=$(wc -l instructions | awk '{ print $1 }')
CURRENT=0
while IFS='|' read -r ID LINK NAME DESC ADDED RETRIVED
do
#$PC=$((200*$CURRENT/$LINES % 2 + 100*$CURRENT/$LINES))
#CURRENT=$((CURRENT + 1))
if [ "$firstline" = 0 ]; then if [ "$firstline" = 0 ]; then
firstline=1 firstline=1
else else
@ -89,14 +95,16 @@ do
tADDED=$(sed -e 's/^"//' -e 's/"$//' <<<"$ADDED") tADDED=$(sed -e 's/^"//' -e 's/"$//' <<<"$ADDED")
tDESC=$(sed -e 's/^"//' -e 's/"$//' <<<"$DESC") tDESC=$(sed -e 's/^"//' -e 's/"$//' <<<"$DESC")
ttDESC=$(echo $tDESC | grep -Eo '\s[0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/ //g' | sed 's/\//_of_/g')
ttDESC=$(echo $tDESC | grep -Eo '[^0-9][0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/[^A-Za-z0-9/]//g' | sed 's/\//_of_/g')
#ttDESC=$(echo $tDESC | grep -Eo '\s[0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/ //g' | sed 's/\//_of_/g')
if [ -z "$ttDESC" ]; then if [ -z "$ttDESC" ]; then
setDESC="_" setDESC="_"
else else
setDESC="_("$ttDESC")_" setDESC="_("$ttDESC")_"
fi fi
#PC=$(echo $CURRENT $LINES | awk '{print 100*$1/$2}')
tFilename=""$tID""$setDESC""${setNAME// /_}"_("$setTHEME"_"$setYEAR").pdf" tFilename=""$tID""$setDESC""${setNAME// /_}"_("$setTHEME"_"$setYEAR").pdf"
filename=$tFilename filename=$tFilename
@ -119,13 +127,12 @@ do
else else
echo "ERROR!" echo "ERROR!"
if [[ $error_level = 1 || $error_level = 2 ]]; then if [[ $error_level = 1 || $error_level = 2 ]]; then
echo
echo "--> Not downloaded. Try again manually..." echo "--> Not downloaded. Try again manually..."
echo "$filename was not downloaded. Check CURL" >> $logfile echo "$filename was not downloaded. Check CURL" >> $logfile
fi fi
fi
#random sleep in order to not look like a script #random sleep in order to not look like a script
sleep $(( ( RANDOM % 1 ) + 1 )) sleep $(( ( RANDOM % 5 ) + 1 ))
fi
fi fi
fi fi
fi fi