Added test to download from brickinstructions if lego.com is unavailable
This commit is contained in:
Frederik Baerentsen
parent
affb4bb441
commit
68ddb6bf86
177
lego_dl.sh
177
lego_dl.sh
@ -66,7 +66,7 @@
|
|||||||
return 0
|
return 0
|
||||||
else
|
else
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
while IFS='|' read -r ID LINK NAME DESC ADDED RETRIVED
|
while IFS='|' read -r ID LINK NAME DESC ADDED RETRIVED
|
||||||
@ -74,106 +74,108 @@
|
|||||||
|
|
||||||
#$PC=$((200*$CURRENT/$LINES % 2 + 100*$CURRENT/$LINES))
|
#$PC=$((200*$CURRENT/$LINES % 2 + 100*$CURRENT/$LINES))
|
||||||
#CURRENT=$((CURRENT + 1))
|
#CURRENT=$((CURRENT + 1))
|
||||||
if [ "$firstline" = 0 ]; then
|
if [ "$firstline" = 0 ]; then
|
||||||
firstline=1
|
firstline=1
|
||||||
else
|
else
|
||||||
tID=$(sed -e 's/^"//' -e 's/"$//' <<<"$ID")
|
tID=$(sed -e 's/^"//' -e 's/"$//' <<<"$ID")
|
||||||
tLINK=$(sed -e 's/^"//' -e 's/"$//' <<<"$LINK")
|
tLINK=$(sed -e 's/^"//' -e 's/"$//' <<<"$LINK")
|
||||||
#tNAME=$(sed -e 's/^"//' -e 's/"$//' <<<"$NAME")
|
#tNAME=$(sed -e 's/^"//' -e 's/"$//' <<<"$NAME")
|
||||||
ttNAME=$(cut -d, -f1-2 sets.csv | grep -w $tID | cut -d, -f2)
|
ttNAME=$(cut -d, -f1-2 sets.csv | grep -w $tID | cut -d, -f2)
|
||||||
setYEAR=$(grep -w $tID sets.csv | cut -d, -f3)
|
setYEAR=$(grep -w $tID sets.csv | cut -d, -f3)
|
||||||
setNAME=$(sed -e 's/[^A-Za-z0-9._-]/_/g' <<< $ttNAME)
|
setNAME=$(sed -e 's/[^A-Za-z0-9._-]/_/g' <<< $ttNAME)
|
||||||
themeID=$(grep -w $tID sets.csv | cut -d, -f4)
|
themeID=$(grep -w $tID sets.csv | cut -d, -f4)
|
||||||
themeName=$(awk -F',' -v id="$themeID" '$1 == id' themes.csv)
|
themeName=$(awk -F',' -v id="$themeID" '$1 == id' themes.csv)
|
||||||
IFS=',' read -r -a array <<< "$themeName"
|
IFS=',' read -r -a array <<< "$themeName"
|
||||||
tempID=${array[2]}
|
tempID=${array[2]}
|
||||||
if [[ $tempID != "" ]]; then
|
if [[ $tempID != "" ]]; then
|
||||||
while [[ $tempID != "" ]]
|
while [[ $tempID != "" ]]
|
||||||
do
|
do
|
||||||
tthemeName=$(awk -F',' -v id="$tempID" '$1 == id' themes.csv)
|
tthemeName=$(awk -F',' -v id="$tempID" '$1 == id' themes.csv)
|
||||||
IFS=',' read -r -a tArray <<< "$tthemeName"
|
IFS=',' read -r -a tArray <<< "$tthemeName"
|
||||||
tempID=${tArray[2]}
|
tempID=${tArray[2]}
|
||||||
themeName=${tArray[1]}
|
themeName=${tArray[1]}
|
||||||
done
|
done
|
||||||
else
|
else
|
||||||
themeName=${array[1]}
|
themeName=${array[1]}
|
||||||
fi
|
fi
|
||||||
|
|
||||||
setTHEME=$(sed -e 's/[^A-Za-z0-9._-]/_/g' <<< $themeName)
|
setTHEME=$(sed -e 's/[^A-Za-z0-9._-]/_/g' <<< $themeName)
|
||||||
|
|
||||||
tADDED=$(sed -e 's/^"//' -e 's/"$//' <<<"$ADDED")
|
tADDED=$(sed -e 's/^"//' -e 's/"$//' <<<"$ADDED")
|
||||||
tDESC=$(sed -e 's/^"//' -e 's/"$//' <<<"$DESC")
|
tDESC=$(sed -e 's/^"//' -e 's/"$//' <<<"$DESC")
|
||||||
|
|
||||||
ttDESC=$(echo $tDESC | grep -Eo '[^0-9][0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/[^A-Za-z0-9/]//g' | sed 's/\//_of_/g')
|
ttDESC=$(echo $tDESC | grep -Eo '[^0-9][0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/[^A-Za-z0-9/]//g' | sed 's/\//_of_/g')
|
||||||
#ttDESC=$(echo $tDESC | grep -Eo '\s[0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/ //g' | sed 's/\//_of_/g')
|
#ttDESC=$(echo $tDESC | grep -Eo '\s[0-9]{1}\s?\/\s?[0-9]{1,2}' | sed 's/ //g' | sed 's/\//_of_/g')
|
||||||
|
|
||||||
if [ -z "$ttDESC" ]; then
|
if [ -z "$ttDESC" ]; then
|
||||||
setDESC="_"
|
etDESC="_"
|
||||||
else
|
else
|
||||||
setDESC="_("$ttDESC")_"
|
tDESC="_("$ttDESC")_"
|
||||||
fi
|
fi
|
||||||
#PC=$(echo $CURRENT $LINES | awk '{print 100*$1/$2}')
|
#PC=$(echo $CURRENT $LINES | awk '{print 100*$1/$2}')
|
||||||
tFilename=""$tID""$setDESC""${setNAME// /_}"_("$setTHEME"_"$setYEAR").pdf"
|
tFilename=""$tID"_"$setDESC""${setNAME// /_}"_("$setTHEME"_"$setYEAR").pdf"
|
||||||
filename=$tFilename
|
filename=$tFilename
|
||||||
|
|
||||||
if [ -f "$downloadFolder/$filename" ]; then
|
if [ -f "$downloadFolder/$filename" ]; then
|
||||||
if [[ $error_level = 2 ]]; then
|
if [[ $error_level = 2 ]]; then
|
||||||
echo "-> $tID exists. Skipping..."
|
echo "-> $tID exists. Skipping..."
|
||||||
echo "$filename exists." >> $logfile
|
echo "$filename exists." >> $logfile
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
if [[ "$tDESC" = "{No longer listed at LEGO.com}" ]] ; then
|
if [[ "$tDESC" = "{No longer listed at LEGO.com}" ]] ; then
|
||||||
echo -ne "-> $tID testing link."
|
echo -ne "-> $tID testing link."
|
||||||
|
echo -ne "LEGO.com <$tLINK>..."
|
||||||
|
if validate_url $tLINK; then
|
||||||
|
echo -ne "Found on lego.com... Downloading..."
|
||||||
|
curl -H "Mozilla/5.0 (platform; rv:75.0) Gecko/20100101 Firefox/75.0" -L $tLINK --silent --output "$downloadFolder/$filename"
|
||||||
|
if [ -f "$downloadFolder/$filename" ]; then
|
||||||
|
echo "Done! > $filename"
|
||||||
|
else
|
||||||
|
echo "ERROR!"
|
||||||
|
if [[ $error_level = 1 || $error_level = 2 ]]; then
|
||||||
|
echo "--> Not downloaded. Try again manually..."
|
||||||
|
echo "$filename was not downloaded. Check CURL" >> $logfile
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
#random sleep in order to not look like a script
|
||||||
|
sleep $(( ( RANDOM % 5 ) + 1 ))
|
||||||
|
else
|
||||||
|
#test
|
||||||
|
|
||||||
if validate_url $tLINK; then
|
biID=$(sed -e 's/[^0-9_]/_/g' <<< $tID)
|
||||||
echo -ne "Found... Downloading..."
|
biLink="https://lego.brickinstructions.com/pdfdrop/"
|
||||||
|
echo -ne "BrickInstructions <$biLink$biID.pdf>..."
|
||||||
curl -H "Mozilla/5.0 (platform; rv:75.0) Gecko/20100101 Firefox/75.0" -L $tLINK --silent --output "$downloadFolder/$filename"
|
if validate_url "$biLink$biID.pdf"; then
|
||||||
if [ -f "$downloadFolder/$filename" ]; then
|
echo -ne "Found on BrickInstructions... Downloading..."
|
||||||
echo "Done! > $filename"
|
curl -H "Mozilla/5.0 (platform; rv:75.0) Gecko/20100101 Firefox/75.0" -L "$biLink$tID.pdf" --silent --output "$downloadFolder/$filename"
|
||||||
else
|
if [ -f "$downloadFolder/$filename" ]; then
|
||||||
echo "ERROR!"
|
echo "Done! > $filename"
|
||||||
if [[ $error_level = 1 || $error_level = 2 ]]; then
|
else
|
||||||
echo "--> Not downloaded. Try again manually..."
|
echo "ERROR!"
|
||||||
|
if [[ $error_level = 1 || $error_level = 2 ]]; then
|
||||||
|
echo "--> Not downloaded. Try again manually..."
|
||||||
echo "$filename was not downloaded. Check CURL" >> $logfile
|
echo "$filename was not downloaded. Check CURL" >> $logfile
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
#random sleep in order to not look like a script
|
#random sleep in order to not look like a script
|
||||||
sleep $(( ( RANDOM % 5 ) + 1 ))
|
sleep $(( ( RANDOM % 5 ) + 1 ))
|
||||||
else
|
else
|
||||||
#test
|
if [[ $error_level = 1 || $error_level = 2 ]]; then
|
||||||
biLink="https://lego.brickinstructions.com/pdfdrop/"
|
echo "-> $tID is not available. Skipping..."
|
||||||
if validate_url "$biLink$tID.pdf"; then
|
echo "$filename is not available." >> $logfile
|
||||||
echo -ne "Found on BrickInstructions... Downloading..."
|
|
||||||
curl -H "Mozilla/5.0 (platform; rv:75.0) Gecko/20100101 Firefox/75.0" -L "$biLink$tID.pdf" --silent --output "$downloadFolder/$filename"
|
|
||||||
if [ -f "$downloadFolder/$filename" ]; then
|
|
||||||
echo "Done! > $filename"
|
|
||||||
else
|
|
||||||
echo "ERROR!"
|
|
||||||
if [[ $error_level = 1 || $error_level = 2 ]]; then
|
|
||||||
echo "--> Not downloaded. Try again manually..."
|
|
||||||
echo "$filename was not downloaded. Check CURL" >> $logfile
|
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
#random sleep in order to not look like a script
|
fi
|
||||||
sleep $(( ( RANDOM % 5 ) + 1 ))
|
|
||||||
fi
|
else
|
||||||
fi
|
echo -ne "--> $tID downloading now..."
|
||||||
|
curl -H "Mozilla/5.0 (platform; rv:75.0) Gecko/20100101 Firefox/75.0" -L $tLINK --silent --output "$downloadFolder/$filename"
|
||||||
|
if [ -f "$downloadFolder/$filename" ]; then
|
||||||
|
echo "Done! > $filename"
|
||||||
else
|
else
|
||||||
if [[ $error_level = 1 || $error_level = 2 ]]; then
|
echo "ERROR!"
|
||||||
echo "-> $tID is not available. Skipping..."
|
if [[ $error_level = 1 || $error_level = 2 ]]; then
|
||||||
echo "$filename is not available." >> $logfile
|
echo "--> Not downloaded. Try again manually..."
|
||||||
fi
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo -ne "--> $tID downloading now..."
|
|
||||||
curl -H "Mozilla/5.0 (platform; rv:75.0) Gecko/20100101 Firefox/75.0" -L $tLINK --silent --output "$downloadFolder/$filename"
|
|
||||||
if [ -f "$downloadFolder/$filename" ]; then
|
|
||||||
echo "Done! > $filename"
|
|
||||||
else
|
|
||||||
echo "ERROR!"
|
|
||||||
if [[ $error_level = 1 || $error_level = 2 ]]; then
|
|
||||||
echo "--> Not downloaded. Try again manually..."
|
|
||||||
echo "$filename was not downloaded. Check CURL" >> $logfile
|
echo "$filename was not downloaded. Check CURL" >> $logfile
|
||||||
fi
|
fi
|
||||||
#random sleep in order to not look like a script
|
#random sleep in order to not look like a script
|
||||||
@ -182,6 +184,5 @@
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
done < instructions.csv
|
done < instructions.csv
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user