From def111ed6a816fd6d2f78d74829bf8481df2b4be Mon Sep 17 00:00:00 2001
From: Frederik Baerentsen <drudoo@drudoo.com>
Date: Wed, 24 Jun 2020 13:05:22 +0200
Subject: [PATCH] Added check if link exists if brickset says it isnt available

---
 lego_dl.sh | 34 +++++++++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/lego_dl.sh b/lego_dl.sh
index df2a7b3..249f612 100644
--- a/lego_dl.sh
+++ b/lego_dl.sh
@@ -61,6 +61,14 @@
  LINES=$(wc -l instructions | awk '{ print $1 }')
  CURRENT=0
 
+ function validate_url(){
+	if [[ `wget -S --spider $1  2>&1 | grep 'HTTP/1.1 200 OK'` ]]; then
+		return 0
+	else
+		return 1
+ 	fi 
+ }  
+
  while IFS='|' read -r ID LINK NAME DESC ADDED RETRIVED
  do
 
@@ -115,9 +123,29 @@
 			 fi 	
 		 else
 			 if [[ "$tDESC" = "{No longer listed at LEGO.com}" ]] ; then
-				 if [[ $error_level = 1 || $error_level = 2 ]]; then
-					 echo "-> $tID is not available. Skipping..."
-					 echo "$filename is not available." >> $logfile
+				 echo -ne "-> $tID testing link."
+
+				 if validate_url $tLINK; then     
+					 echo -ne "Found... Downloading..."   
+				 
+				 		curl -H "Mozilla/5.0 (platform; rv:75.0) Gecko/20100101 Firefox/75.0" -L $tLINK --silent --output "$downloadFolder/$filename"
+				 		if [ -f "$downloadFolder/$filename" ]; then
+					 		echo "Done! > $filename"
+				 		else
+					 		echo "ERROR!"
+					 		if [[ $error_level = 1 || $error_level = 2 ]]; then  
+						 	echo "--> Not downloaded. Try again manually..."
+							echo "$filename was not downloaded. Check CURL" >> $logfile
+						fi
+						#random sleep in order to not look like a script
+						sleep $(( ( RANDOM % 5 )  + 1 ))
+				fi
+					
+				else
+				   if [[ $error_level = 1 || $error_level = 2 ]]; then
+					   echo "-> $tID is not available. Skipping..."
+					   echo "$filename is not available." >> $logfile
+				   fi
 				 fi
 			 else  
 				 echo -ne "--> $tID downloading now..."