Fixed errors
This commit is contained in:
Frederik Baerentsen
parent
a848096baa
commit
ec88a6914f
107
lego_dl.sh
107
lego_dl.sh
@ -1,48 +1,48 @@
|
||||
#!/bin/bash
|
||||
##############################
|
||||
#
|
||||
# NOTE: There is still some small errors with sets with multiple booklets
|
||||
#
|
||||
# This scripts uses the brickset instructions file to get links and set numbers.
|
||||
# In order to get themes, we get themes from rebrickable.
|
||||
#
|
||||
# Files are saved as:
|
||||
# $tID""$setDESC""${setNAME// /_}"_("$setTHEME"_"$setYEAR").pdf"
|
||||
# 1190-1_Retro_Buggy_(Town_1999).pdf
|
||||
#
|
||||
# If a set contains multiple filesm the file name will be:
|
||||
# 2520-1_(1_of_2)_Battle_Arena_(Ninjago_2011).pdf
|
||||
# 2520-1_(2_of_2)_Battle_Arena_(Ninjago_2011).pdf
|
||||
#
|
||||
# Req:
|
||||
# bash, awk, grep, sed, curl, wget
|
||||
#
|
||||
# Get themes.csv and sets.cvs from https://rebrickable.com/downloads
|
||||
#
|
||||
# error_level:
|
||||
# 0 no errors reported
|
||||
# 1 Download errors reported
|
||||
# 2 Existing files and download errors reported
|
||||
#
|
||||
##############################
|
||||
#!/bin/bash
|
||||
##############################
|
||||
#
|
||||
# NOTE: There is still some small errors with sets with multiple booklets
|
||||
#
|
||||
# This scripts uses the brickset instructions file to get links and set numbers.
|
||||
# In order to get themes, we get themes from rebrickable.
|
||||
#
|
||||
# Files are saved as:
|
||||
# $tID""$setDESC""${setNAME// /_}"_("$setTHEME"_"$setYEAR").pdf"
|
||||
# 1190-1_Retro_Buggy_(Town_1999).pdf
|
||||
#
|
||||
# If a set contains multiple filesm the file name will be:
|
||||
# 2520-1_(1_of_2)_Battle_Arena_(Ninjago_2011).pdf
|
||||
# 2520-1_(2_of_2)_Battle_Arena_(Ninjago_2011).pdf
|
||||
#
|
||||
# Req:
|
||||
# bash, awk, grep, sed, curl, wget
|
||||
#
|
||||
# Get themes.csv and sets.cvs from https://rebrickable.com/downloads
|
||||
#
|
||||
# error_level:
|
||||
# 0 no errors reported
|
||||
# 1 Download errors reported
|
||||
# 2 Existing files and download errors reported
|
||||
#
|
||||
##############################
|
||||
|
||||
echo "----> Starting Download of all LEGO instructions from https://brickset.com/exportscripts/instructions"
|
||||
firstline=0
|
||||
echo "----> Starting Download of all LEGO instructions from https://brickset.com/exportscripts/instructions"
|
||||
firstline=0
|
||||
|
||||
##### CHANGE HERE #####
|
||||
logfile="lego_errors.log"
|
||||
downloadFolder="../Instructions"
|
||||
error_level=1
|
||||
##### STOP CHANGE #####
|
||||
##### CHANGE HERE #####
|
||||
logfile="lego_errors.log"
|
||||
downloadFolder="../Instructions"
|
||||
error_level=1
|
||||
##### STOP CHANGE #####
|
||||
|
||||
echo "" > $logfile
|
||||
echo "" > $logfile
|
||||
|
||||
THEMES="themes.csv"
|
||||
SETS="sets.csv"
|
||||
THEMES="themes.csv"
|
||||
SETS="sets.csv"
|
||||
|
||||
if [ -f "instructions" ]; then
|
||||
if [ -f "instructions" ]; then
|
||||
echo "---> Using existing instructions file. Manually delete it to redownload..."
|
||||
else
|
||||
else
|
||||
echo -ne "---> Downloading csv from Brickset..."
|
||||
wget https://brickset.com/exportscripts/instructions &> /dev/null
|
||||
if [[ "$?" != 0 ]]; then
|
||||
@ -51,15 +51,21 @@ else
|
||||
else
|
||||
echo "Done!"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "----> Converting csv with ',' to '|'..."
|
||||
head -1 instructions | sed 's/,/|/g' > instructions.csv
|
||||
sed 1,1d instructions | sed -r 's/("[^",]+),([^",]+")/\1###\2/g' | awk -F, '{print $1,$2,$3,$4,$5,$6}' | sed 's/###/,/g' | sed 's/" "/"|"/g' >> instructions.csv
|
||||
|
||||
while IFS='|' read -r ID LINK NAME DESC ADDED RETRIVED
|
||||
do
|
||||
echo "----> Converting csv with ',' to '|'..."
|
||||
head -1 instructions | sed 's/,/|/g' > instructions.csv
|
||||
sed 1,1d instructions | sed -r 's/("[^",]+),([^",]+")/\1###\2/g' | awk -F, '{print $1,$2,$3,$4,$5,$6}' | sed 's/###/,/g' | sed 's/" "/"|"/g' >> instructions.csv
|
||||
|
||||
LINES=$(wc -l instructions | awk '{ print $1 }')
|
||||
CURRENT=0
|
||||
|
||||
while IFS='|' read -r ID LINK NAME DESC ADDED RETRIVED
|
||||
do
|
||||
|
||||
#$PC=$((200*$CURRENT/$LINES % 2 + 100*$CURRENT/$LINES))
|
||||
#CURRENT=$((CURRENT + 1))
|
||||
if [ "$firstline" = 0 ]; then
|
||||
firstline=1
|
||||
else
|
||||
@ -89,14 +95,16 @@ do
|
||||
|
||||
tADDED=$(sed -e 's/^"//' -e 's/"$//' <<<"$ADDED")
|
||||
tDESC=$(sed -e 's/^"//' -e 's/"$//' <<<"$DESC")
|
||||
ttDESC=$(echo $tDESC | grep -Eo '\s[0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/ //g' | sed 's/\//_of_/g')
|
||||
|
||||
ttDESC=$(echo $tDESC | grep -Eo '[^0-9][0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/[^A-Za-z0-9/]//g' | sed 's/\//_of_/g')
|
||||
#ttDESC=$(echo $tDESC | grep -Eo '\s[0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/ //g' | sed 's/\//_of_/g')
|
||||
|
||||
if [ -z "$ttDESC" ]; then
|
||||
setDESC="_"
|
||||
else
|
||||
setDESC="_("$ttDESC")_"
|
||||
fi
|
||||
|
||||
#PC=$(echo $CURRENT $LINES | awk '{print 100*$1/$2}')
|
||||
tFilename=""$tID""$setDESC""${setNAME// /_}"_("$setTHEME"_"$setYEAR").pdf"
|
||||
filename=$tFilename
|
||||
|
||||
@ -119,13 +127,12 @@ do
|
||||
else
|
||||
echo "ERROR!"
|
||||
if [[ $error_level = 1 || $error_level = 2 ]]; then
|
||||
echo
|
||||
echo "--> Not downloaded. Try again manually..."
|
||||
echo "$filename was not downloaded. Check CURL" >> $logfile
|
||||
fi
|
||||
fi
|
||||
#random sleep in order to not look like a script
|
||||
sleep $(( ( RANDOM % 1 ) + 1 ))
|
||||
sleep $(( ( RANDOM % 5 ) + 1 ))
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
Loading…
Reference in New Issue
Block a user