30 lines
885 B
Python
30 lines
885 B
Python
|
import requests
|
||
|
|
import urllib.request
|
||
|
|
import time
|
||
|
|
from bs4 import BeautifulSoup
|
||
|
|
import re
|
||
|
|
import os
|
||
|
|
|
||
|
|
url = 'https://brickset.com/library/catalogues'
|
||
|
|
|
||
|
|
|
||
|
|
response = requests.get(url)
|
||
|
|
soup = BeautifulSoup(response.text, "html.parser")
|
||
|
|
line_count = 1 #variable to track what line you are on
|
||
|
|
|
||
|
|
for idx, val in enumerate(soup.findAll('h3')):
|
||
|
|
if idx != 0:
|
||
|
|
year=val.string
|
||
|
|
title=soup.find_all('p')[idx].contents[0]
|
||
|
|
link=soup.find_all('p')[idx].contents[3].get('href')
|
||
|
|
print(link," ",title," ",year)
|
||
|
|
|
||
|
|
dpath="../Catalogues/"+year+"-"+title.strip().replace(' ', '_').replace('/', '-')+".pdf"
|
||
|
|
print(dpath)
|
||
|
|
if not os.path.exists(dpath):
|
||
|
|
r = requests.get(link, stream=True)
|
||
|
|
with open(dpath, 'wb') as f:
|
||
|
|
f.write(r.content)
|
||
|
|
time.sleep(1) #pause the code for a sec
|
||
|
|
line_count +=1
|