FrederikBaerentsen/DataHoarder_scripts
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added check if link exists if brickset says it isnt available

Browse Source
This commit is contained in:
Frederik Baerentsen 2020-06-24 13:05:22 +02:00
parent ec88a6914f
commit def111ed6a
1 changed files with 31 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
lego_dl.sh
View File

@ -61,6 +61,14 @@
LINES=$(wc -l instructions | awk '{ print $1 }') LINES=$(wc -l instructions | awk '{ print $1 }')
CURRENT=0 CURRENT=0
function validate_url(){
if [[ `wget -S --spider $1 2>&1 | grep 'HTTP/1.1 200 OK'` ]]; then
return 0
else
return 1
fi
}
while IFS='|' read -r ID LINK NAME DESC ADDED RETRIVED while IFS='|' read -r ID LINK NAME DESC ADDED RETRIVED
do do
@ -115,9 +123,29 @@
fi fi
else else
if [[ "$tDESC" = "{No longer listed at LEGO.com}" ]] ; then if [[ "$tDESC" = "{No longer listed at LEGO.com}" ]] ; then
if [[ $error_level = 1 || $error_level = 2 ]]; then echo -ne "-> $tID testing link."
echo "-> $tID is not available. Skipping..."
echo "$filename is not available." >> $logfile if validate_url $tLINK; then
echo -ne "Found... Downloading..."
curl -H "Mozilla/5.0 (platform; rv:75.0) Gecko/20100101 Firefox/75.0" -L $tLINK --silent --output "$downloadFolder/$filename"
if [ -f "$downloadFolder/$filename" ]; then
echo "Done! > $filename"
else
echo "ERROR!"
if [[ $error_level = 1 || $error_level = 2 ]]; then
echo "--> Not downloaded. Try again manually..."
echo "$filename was not downloaded. Check CURL" >> $logfile
fi
#random sleep in order to not look like a script
sleep $(( ( RANDOM % 5 ) + 1 ))
fi
else
if [[ $error_level = 1 || $error_level = 2 ]]; then
echo "-> $tID is not available. Skipping..."
echo "$filename is not available." >> $logfile
fi
fi fi
else else
echo -ne "--> $tID downloading now..." echo -ne "--> $tID downloading now..."