Added test to download from brickinstructions if lego.com is unavailable

This commit is contained in:
Frederik Baerentsen 2020-06-25 18:14:25 +02:00
parent affb4bb441
commit 68ddb6bf86

View File

@ -66,7 +66,7 @@
return 0 return 0
else else
return 1 return 1
fi fi
} }
while IFS='|' read -r ID LINK NAME DESC ADDED RETRIVED while IFS='|' read -r ID LINK NAME DESC ADDED RETRIVED
@ -74,106 +74,108 @@
#$PC=$((200*$CURRENT/$LINES % 2 + 100*$CURRENT/$LINES)) #$PC=$((200*$CURRENT/$LINES % 2 + 100*$CURRENT/$LINES))
#CURRENT=$((CURRENT + 1)) #CURRENT=$((CURRENT + 1))
if [ "$firstline" = 0 ]; then if [ "$firstline" = 0 ]; then
firstline=1 firstline=1
else else
tID=$(sed -e 's/^"//' -e 's/"$//' <<<"$ID") tID=$(sed -e 's/^"//' -e 's/"$//' <<<"$ID")
tLINK=$(sed -e 's/^"//' -e 's/"$//' <<<"$LINK") tLINK=$(sed -e 's/^"//' -e 's/"$//' <<<"$LINK")
#tNAME=$(sed -e 's/^"//' -e 's/"$//' <<<"$NAME") #tNAME=$(sed -e 's/^"//' -e 's/"$//' <<<"$NAME")
ttNAME=$(cut -d, -f1-2 sets.csv | grep -w $tID | cut -d, -f2) ttNAME=$(cut -d, -f1-2 sets.csv | grep -w $tID | cut -d, -f2)
setYEAR=$(grep -w $tID sets.csv | cut -d, -f3) setYEAR=$(grep -w $tID sets.csv | cut -d, -f3)
setNAME=$(sed -e 's/[^A-Za-z0-9._-]/_/g' <<< $ttNAME) setNAME=$(sed -e 's/[^A-Za-z0-9._-]/_/g' <<< $ttNAME)
themeID=$(grep -w $tID sets.csv | cut -d, -f4) themeID=$(grep -w $tID sets.csv | cut -d, -f4)
themeName=$(awk -F',' -v id="$themeID" '$1 == id' themes.csv) themeName=$(awk -F',' -v id="$themeID" '$1 == id' themes.csv)
IFS=',' read -r -a array <<< "$themeName" IFS=',' read -r -a array <<< "$themeName"
tempID=${array[2]} tempID=${array[2]}
if [[ $tempID != "" ]]; then if [[ $tempID != "" ]]; then
while [[ $tempID != "" ]] while [[ $tempID != "" ]]
do do
tthemeName=$(awk -F',' -v id="$tempID" '$1 == id' themes.csv) tthemeName=$(awk -F',' -v id="$tempID" '$1 == id' themes.csv)
IFS=',' read -r -a tArray <<< "$tthemeName" IFS=',' read -r -a tArray <<< "$tthemeName"
tempID=${tArray[2]} tempID=${tArray[2]}
themeName=${tArray[1]} themeName=${tArray[1]}
done done
else else
themeName=${array[1]} themeName=${array[1]}
fi fi
setTHEME=$(sed -e 's/[^A-Za-z0-9._-]/_/g' <<< $themeName) setTHEME=$(sed -e 's/[^A-Za-z0-9._-]/_/g' <<< $themeName)
tADDED=$(sed -e 's/^"//' -e 's/"$//' <<<"$ADDED")
tDESC=$(sed -e 's/^"//' -e 's/"$//' <<<"$DESC")
ttDESC=$(echo $tDESC | grep -Eo '[^0-9][0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/[^A-Za-z0-9/]//g' | sed 's/\//_of_/g')
#ttDESC=$(echo $tDESC | grep -Eo '\s[0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/ //g' | sed 's/\//_of_/g')
if [ -z "$ttDESC" ]; then tADDED=$(sed -e 's/^"//' -e 's/"$//' <<<"$ADDED")
setDESC="_" tDESC=$(sed -e 's/^"//' -e 's/"$//' <<<"$DESC")
else
setDESC="_("$ttDESC")_"
fi
#PC=$(echo $CURRENT $LINES | awk '{print 100*$1/$2}')
tFilename=""$tID""$setDESC""${setNAME// /_}"_("$setTHEME"_"$setYEAR").pdf"
filename=$tFilename
if [ -f "$downloadFolder/$filename" ]; then ttDESC=$(echo $tDESC | grep -Eo '[^0-9][0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/[^A-Za-z0-9/]//g' | sed 's/\//_of_/g')
if [[ $error_level = 2 ]]; then #ttDESC=$(echo $tDESC | grep -Eo '\s[0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/ //g' | sed 's/\//_of_/g')
echo "-> $tID exists. Skipping..."
echo "$filename exists." >> $logfile
fi
else
if [[ "$tDESC" = "{No longer listed at LEGO.com}" ]] ; then
echo -ne "-> $tID testing link."
if validate_url $tLINK; then if [ -z "$ttDESC" ]; then
echo -ne "Found... Downloading..." etDESC="_"
else
curl -H "Mozilla/5.0 (platform; rv:75.0) Gecko/20100101 Firefox/75.0" -L $tLINK --silent --output "$downloadFolder/$filename" tDESC="_("$ttDESC")_"
if [ -f "$downloadFolder/$filename" ]; then fi
echo "Done! > $filename" #PC=$(echo $CURRENT $LINES | awk '{print 100*$1/$2}')
else tFilename=""$tID"_"$setDESC""${setNAME// /_}"_("$setTHEME"_"$setYEAR").pdf"
echo "ERROR!" filename=$tFilename
if [[ $error_level = 1 || $error_level = 2 ]]; then
echo "--> Not downloaded. Try again manually..." if [ -f "$downloadFolder/$filename" ]; then
if [[ $error_level = 2 ]]; then
echo "-> $tID exists. Skipping..."
echo "$filename exists." >> $logfile
fi
else
if [[ "$tDESC" = "{No longer listed at LEGO.com}" ]] ; then
echo -ne "-> $tID testing link."
echo -ne "LEGO.com <$tLINK>..."
if validate_url $tLINK; then
echo -ne "Found on lego.com... Downloading..."
curl -H "Mozilla/5.0 (platform; rv:75.0) Gecko/20100101 Firefox/75.0" -L $tLINK --silent --output "$downloadFolder/$filename"
if [ -f "$downloadFolder/$filename" ]; then
echo "Done! > $filename"
else
echo "ERROR!"
if [[ $error_level = 1 || $error_level = 2 ]]; then
echo "--> Not downloaded. Try again manually..."
echo "$filename was not downloaded. Check CURL" >> $logfile
fi
fi
#random sleep in order to not look like a script
sleep $(( ( RANDOM % 5 ) + 1 ))
else
#test
biID=$(sed -e 's/[^0-9_]/_/g' <<< $tID)
biLink="https://lego.brickinstructions.com/pdfdrop/"
echo -ne "BrickInstructions <$biLink$biID.pdf>..."
if validate_url "$biLink$biID.pdf"; then
echo -ne "Found on BrickInstructions... Downloading..."
curl -H "Mozilla/5.0 (platform; rv:75.0) Gecko/20100101 Firefox/75.0" -L "$biLink$tID.pdf" --silent --output "$downloadFolder/$filename"
if [ -f "$downloadFolder/$filename" ]; then
echo "Done! > $filename"
else
echo "ERROR!"
if [[ $error_level = 1 || $error_level = 2 ]]; then
echo "--> Not downloaded. Try again manually..."
echo "$filename was not downloaded. Check CURL" >> $logfile echo "$filename was not downloaded. Check CURL" >> $logfile
fi fi
fi fi
#random sleep in order to not look like a script #random sleep in order to not look like a script
sleep $(( ( RANDOM % 5 ) + 1 )) sleep $(( ( RANDOM % 5 ) + 1 ))
else else
#test if [[ $error_level = 1 || $error_level = 2 ]]; then
biLink="https://lego.brickinstructions.com/pdfdrop/" echo "-> $tID is not available. Skipping..."
if validate_url "$biLink$tID.pdf"; then echo "$filename is not available." >> $logfile
echo -ne "Found on BrickInstructions... Downloading..."
curl -H "Mozilla/5.0 (platform; rv:75.0) Gecko/20100101 Firefox/75.0" -L "$biLink$tID.pdf" --silent --output "$downloadFolder/$filename"
if [ -f "$downloadFolder/$filename" ]; then
echo "Done! > $filename"
else
echo "ERROR!"
if [[ $error_level = 1 || $error_level = 2 ]]; then
echo "--> Not downloaded. Try again manually..."
echo "$filename was not downloaded. Check CURL" >> $logfile
fi fi
fi fi
#random sleep in order to not look like a script fi
sleep $(( ( RANDOM % 5 ) + 1 ))
fi else
fi echo -ne "--> $tID downloading now..."
curl -H "Mozilla/5.0 (platform; rv:75.0) Gecko/20100101 Firefox/75.0" -L $tLINK --silent --output "$downloadFolder/$filename"
if [ -f "$downloadFolder/$filename" ]; then
echo "Done! > $filename"
else else
if [[ $error_level = 1 || $error_level = 2 ]]; then echo "ERROR!"
echo "-> $tID is not available. Skipping..." if [[ $error_level = 1 || $error_level = 2 ]]; then
echo "$filename is not available." >> $logfile echo "--> Not downloaded. Try again manually..."
fi
fi
else
echo -ne "--> $tID downloading now..."
curl -H "Mozilla/5.0 (platform; rv:75.0) Gecko/20100101 Firefox/75.0" -L $tLINK --silent --output "$downloadFolder/$filename"
if [ -f "$downloadFolder/$filename" ]; then
echo "Done! > $filename"
else
echo "ERROR!"
if [[ $error_level = 1 || $error_level = 2 ]]; then
echo "--> Not downloaded. Try again manually..."
echo "$filename was not downloaded. Check CURL" >> $logfile echo "$filename was not downloaded. Check CURL" >> $logfile
fi fi
#random sleep in order to not look like a script #random sleep in order to not look like a script
@ -182,6 +184,5 @@
fi fi
fi fi
fi fi
done < instructions.csv done < instructions.csv