DataHoarder_scripts/catalog.py

30 lines
885 B
Python
Raw Normal View History

import requests
import urllib.request
import time
from bs4 import BeautifulSoup
import re
import os
url = 'https://brickset.com/library/catalogues'
response = requests.get(url)
soup = BeautifulSoup(response.text, "html.parser")
line_count = 1 #variable to track what line you are on
for idx, val in enumerate(soup.findAll('h3')):
if idx != 0:
year=val.string
title=soup.find_all('p')[idx].contents[0]
link=soup.find_all('p')[idx].contents[3].get('href')
print(link," ",title," ",year)
dpath="../Catalogues/"+year+"-"+title.strip().replace(' ', '_').replace('/', '-')+".pdf"
print(dpath)
if not os.path.exists(dpath):
r = requests.get(link, stream=True)
with open(dpath, 'wb') as f:
f.write(r.content)
time.sleep(1) #pause the code for a sec
line_count +=1