Merge pull request 'Fixed the rebrickable scraping to deal with changes' (#81) from hiddenside/BrickTracker:fix-instructions-download into master

Reviewed-on: FrederikBaerentsen/BrickTracker#81

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## Unreleased
+
+- Fix legibility of "Damaged" and "Missing" fields for tiny screen by reducing horizontal padding
+
 ## 1.2.2:
 
 This release fixes a bug where orphaned parts in the `inventory` table are blocking the database upgrade.

bricktracker/instructions.py

@@ -1,6 +1,7 @@
 from datetime import datetime, timezone
 import logging
 import os
+from urllib.parse import urljoin
 from shutil import copyfileobj
 import traceback
 from typing import Tuple, TYPE_CHECKING
@@ -11,6 +12,8 @@ import humanize
 import requests
 from werkzeug.datastructures import FileStorage
 from werkzeug.utils import secure_filename
+import re
+import cloudscraper
 
 from .exceptions import ErrorException, DownloadException
 if TYPE_CHECKING:
@@ -89,91 +92,72 @@ class BrickInstructions(object):
 
     # Download an instruction file
     def download(self, path: str, /) -> None:
+        """
+        Streams the PDF in chunks and uses self.socket.update_total
+        + self.socket.progress_count to drive a determinate bar.
+        """
         try:
-            # Just to make sure that the progress is initiated
-            self.socket.progress(
-                message='Downloading {file}'.format(
-                    file=self.filename,
-                )
-            )
-
             target = self.path(filename=secure_filename(self.filename))
 
-            # Skipping rather than failing here
+            # Skip if we already have it
             if os.path.isfile(target):
-                self.socket.complete(
+                return self.socket.complete(
-                    message='File {file} already exists, skipped'.format(
+                    message=f"File {self.filename} already exists, skipped"
-                        file=self.filename,
-                    )
                 )
 
-            else:
+            # Fetch PDF via cloudscraper (to bypass Cloudflare)
-                url = current_app.config['REBRICKABLE_LINK_INSTRUCTIONS_PATTERN'].format(  # noqa: E501
+            scraper = cloudscraper.create_scraper()
-                    path=path
+            scraper.headers.update({
-                )
+                "User-Agent": current_app.config['REBRICKABLE_USER_AGENT']
-                trimmed_url = current_app.config['REBRICKABLE_LINK_INSTRUCTIONS_PATTERN'].format(  # noqa: E501
+            })
-                    path=path.partition('/')[0]
+            resp = scraper.get(path, stream=True)
-                )
+            if not resp.ok:
+                raise DownloadException(f"Failed to download: HTTP {resp.status_code}")
 
-                # Request the file
+            # Tell the socket how many bytes in total
-                self.socket.progress(
+            total = int(resp.headers.get("Content-Length", 0))
-                    message='Requesting {url}'.format(
+            self.socket.update_total(total)
-                        url=trimmed_url,
-                    )
-                )
 
-                response = requests.get(url, stream=True)
+            # Reset the counter and kick off at 0%
-                if response.ok:
+            self.socket.progress_count = 0
+            self.socket.progress(message=f"Starting download {self.filename}")
 
-                    # Store the content header as size
+            # Write out in 8 KiB chunks and update the counter
-                    try:
+            with open(target, "wb") as f:
-                        self.size = int(
+                for chunk in resp.iter_content(chunk_size=8192):
-                            response.headers.get('Content-length', 0)
+                    if not chunk:
-                        )
+                        continue
-                    except Exception:
+                    f.write(chunk)
-                        self.size = 0
 
-                    # Downloading the file
+                    # Bump the internal counter and emit
+                    self.socket.progress_count += len(chunk)
                     self.socket.progress(
-                        message='Downloading {url} ({size})'.format(
+                        message=(
-                            url=trimmed_url,
+                            f"Downloading {self.filename} "
-                            size=self.human_size(),
+                            f"({humanize.naturalsize(self.socket.progress_count)}/"
+                            f"{humanize.naturalsize(self.socket.progress_total)})"
                         )
                     )
 
-                    with open(target, 'wb') as f:
+            # Done!
-                        copyfileobj(response.raw, f)
+            logger.info(f"Downloaded {self.filename}")
-                else:
+            self.socket.complete(
-                    raise DownloadException('failed to download: {code}'.format(  # noqa: E501
+                message=f"File {self.filename} downloaded ({self.human_size()})"
-                        code=response.status_code
-                    ))
-
-                # Info
-                logger.info('The instruction file {file} has been downloaded'.format(  # noqa: E501
-                    file=self.filename
-                ))
-
-                # Complete
-                self.socket.complete(
-                    message='File {file} downloaded ({size})'.format(  # noqa: E501
-                        file=self.filename,
-                        size=self.human_size()
-                    )
-                )
-
-        except Exception as e:
-            self.socket.fail(
-                message='Error while downloading instruction {file}: {error}'.format(  # noqa: E501
-                    file=self.filename,
-                    error=e,
-                )
             )
 
+        except Exception as e:
             logger.debug(traceback.format_exc())
+            self.socket.fail(
+                message=f"Error downloading {self.filename}: {e}"
+            )
 
     # Display the size in a human format
     def human_size(self) -> str:
-        return humanize.naturalsize(self.size)
+        try:
+            size = self.size
+        except AttributeError:
+            size = os.path.getsize(self.path())
+        return humanize.naturalsize(size)
 
     # Display the time in a human format
     def human_time(self) -> str:
@@ -250,40 +234,52 @@ class BrickInstructions(object):
     # Find the instructions for a set
     @staticmethod
     def find_instructions(set: str, /) -> list[Tuple[str, str]]:
-        response = requests.get(
+        """
-            current_app.config['REBRICKABLE_LINK_INSTRUCTIONS_PATTERN'].format(
+        Scrape Rebrickable’s HTML and return a list of
-                path=set,
+        (filename_slug, download_url). Duplicate slugs get _1, _2, …
-            ),
+        """
-            headers={
+        page_url = f"https://rebrickable.com/instructions/{set}/"
-                'User-Agent': current_app.config['REBRICKABLE_USER_AGENT']
+        logger.debug(f"[find_instructions] fetching HTML from {page_url!r}")
-            }
-        )
 
-        if not response.ok:
+        # Solve Cloudflare’s challenge
-            raise ErrorException('Failed to load the Rebrickable instructions page. Status code: {code}'.format(  # noqa: E501
+        scraper = cloudscraper.create_scraper()
-                code=response.status_code
+        scraper.headers.update({'User-Agent': current_app.config['REBRICKABLE_USER_AGENT']})
-            ))
+        resp = scraper.get(page_url)
+        if not resp.ok:
+            raise ErrorException(f'Failed to load instructions page for {set}. HTTP {resp.status_code}')
 
-        # Parse the HTML content
+        soup = BeautifulSoup(resp.content, 'html.parser')
-        soup = BeautifulSoup(response.content, 'html.parser')
+        link_re = re.compile(r'^/instructions/\d+/.+/download/')
 
-        # Collect all <img> tags with "LEGO Building Instructions" in the
+        raw: list[tuple[str, str]] = []
-        # alt attribute
+        for a in soup.find_all('a', href=link_re):
-        found_tags: list[Tuple[str, str]] = []
+            img = a.find('img', alt=True)
-        for a_tag in soup.find_all('a', href=True):
+            if not img or set not in img['alt']:
-            img_tag = a_tag.find('img', alt=True)
+                continue
-            if img_tag and "LEGO Building Instructions" in img_tag['alt']:
-                found_tags.append(
-                    (
-                        img_tag['alt'].removeprefix('LEGO Building Instructions for '),  # noqa: E501
-                        a_tag['href']
-                    )
-                )  # Save alt and href
 
-        # Raise an error if nothing found
+            # Turn the alt text into a slug
-        if not len(found_tags):
+            alt_text = img['alt'].removeprefix('LEGO Building Instructions for ')
-            raise ErrorException('No instruction found for set {set}'.format(
+            slug = re.sub(r'[^A-Za-z0-9]+', '-', alt_text).strip('-')
-                set=set
-            ))
 
-        return found_tags
+            # Build the absolute download URL
+            download_url = urljoin('https://rebrickable.com', a['href'])
+            raw.append((slug, download_url))
+
+        if not raw:
+            raise ErrorException(f'No download links found on instructions page for {set}')
+
+        # Disambiguate duplicate slugs by appending _1, _2, …
+        from collections import Counter, defaultdict
+        counts = Counter(name for name, _ in raw)
+        seen: dict[str, int] = defaultdict(int)
+        unique: list[tuple[str, str]] = []
+        for name, url in raw:
+            idx = seen[name]
+            if counts[name] > 1 and idx > 0:
+                final_name = f"{name}_{idx}"
+            else:
+                final_name = name
+            seen[name] += 1
+            unique.append((final_name, url))
+
+        return unique

docs/quickstart.md

@@ -61,9 +61,9 @@ docker compose up -d
 
 2. Access BrickTracker at `http://localhost:3333`
 
-Please refer to [Environment Variables Reference](docs/env.md) for a list of available variables.
+Please refer to [Environment Variables Reference](env.md) for a list of available variables.
 
-3. Read more in [First steps](docs/first-steps.md)
+3. Read more in [First steps](first-steps.md)
 
 ## Troubleshooting
 
@@ -85,6 +85,6 @@ Please refer to [Environment Variables Reference](docs/env.md) for a list of ava
    - Check for any syntax errors in `.env` file
    - Verify no conflicting environment variables are set in the shell
 
-For more troubleshooting, take a look at [Common Errors](docs/common-errors.md)
+For more troubleshooting, take a look at [Common Errors](common-errors.md)
 
-Please refer to [Setup](docs/setup.md) for more information.
+Please refer to [Setup](setup.md) for more information.

docs/setup.md

@@ -53,7 +53,7 @@ services:
 The [.env.sample](../.env.sample) file provides ample documentation on all the configurable options. Have a look at it.
 You can make a copy of `.env.sample` as `.env` with your options or create an `.env` file from scratch.
 
-[Environment Variables Reference](docs/env.md) contains a table of the available variables.
+[Environment Variables Reference](env.md) contains a table of the available variables.
 
 ## Database file
 

requirements.txt

@@ -9,3 +9,4 @@ rebrick
 requests
 tzdata
 bs4
+cloudscraper

templates/macro/form.html

@@ -1,6 +1,6 @@
 {% macro checkbox(name, id, prefix, url, checked, parent=none, delete=false) %}
   {% if g.login.is_authenticated() %}
-    <input class="form-check-input" type="checkbox" id="{{ prefix }}-{{ id }}" {% if checked %}checked{% endif %}
+    <input class="form-check-input px-1" type="checkbox" id="{{ prefix }}-{{ id }}" {% if checked %}checked{% endif %}
     {% if not delete %}
       data-changer-id="{{ id }}" data-changer-prefix="{{ prefix }}" data-changer-url="{{ url }}" {% if parent %}data-changer-parent="{{ parent }}"{% endif %}
     {% else %}
@@ -22,8 +22,8 @@
   {% else %}
     <label class="visually-hidden" for="{{ prefix }}-{{ id }}">{{ name }}</label>
     <div class="input-group">
-      {% if icon %}<span class="input-group-text"><i class="ri-{{ icon }} me-1"></i><span class="ms-1 d-none d-md-inline"> {{ name }}</span></span>{% endif %}
+      {% if icon %}<span class="input-group-text px-1"><i class="ri-{{ icon }} me-1"></i><span class="ms-1 d-none d-md-inline"> {{ name }}</span></span>{% endif %}
-      <input class="form-control form-control-sm flex-shrink-1" type="text" id="{{ prefix }}-{{ id }}" value="{% if value %}{{ value }}{% endif %}"
+      <input class="form-control form-control-sm flex-shrink-1 px-1" type="text" id="{{ prefix }}-{{ id }}" value="{% if value %}{{ value }}{% endif %}"
       {% if g.login.is_authenticated() %}
         data-changer-id="{{ id }}" data-changer-prefix="{{ prefix }}" data-changer-url="{{ url }}"
         {% if date %}data-changer-date="true"{% endif %}
@@ -31,12 +31,12 @@
         disabled
       {% endif %}
         autocomplete="off">
-      {% if suffix %}<span class="input-group-text d-none d-md-inline">{{ suffix }}</span>{% endif %}
+      {% if suffix %}<span class="input-group-text d-none d-md-inline px-1">{{ suffix }}</span>{% endif %}
       {% if g.login.is_authenticated() %}
-        <span id="status-{{ prefix }}-{{ id }}" class="input-group-text ri-save-line"></span>
+        <span id="status-{{ prefix }}-{{ id }}" class="input-group-text ri-save-line px-1"></span>
-        <button id="clear-{{ prefix }}-{{ id }}" type="button" class="btn btn-sm btn-light btn-outline-danger border"><i class="ri-eraser-line"></i></button>
+        <button id="clear-{{ prefix }}-{{ id }}" type="button" class="btn btn-sm btn-light btn-outline-danger border px-1"><i class="ri-eraser-line"></i></button>
       {% else %}
-        <span class="input-group-text ri-prohibited-line"></span>
+        <span class="input-group-text ri-prohibited-line px-1"></span>
       {% endif %}
     </div>
   {% endif %}
@@ -46,8 +46,8 @@
   {% if g.login.is_authenticated() %}
     <label class="visually-hidden" for="{{ prefix }}-{{ id }}">{{ name }}</label>
     <div class="input-group">
-      {% if icon %}<span class="input-group-text"><i class="ri-{{ icon }} me-1"></i><span class="ms-1 d-none d-md-inline"> {{ name }}</span></span>{% endif %}
+      {% if icon %}<span class="input-group-text px-1"><i class="ri-{{ icon }} me-1"></i><span class="ms-1 d-none d-md-inline"> {{ name }}</span></span>{% endif %}
-      <select id="{{ prefix }}-{{ id }}" class="form-select"
+      <select id="{{ prefix }}-{{ id }}" class="form-select px-1"
         {% if not delete %}
           data-changer-id="{{ id }}" data-changer-prefix="{{ prefix }}" data-changer-url="{{ url }}"
         {% else %}
@@ -59,8 +59,8 @@
           <option value="{{ metadata.fields.id }}" {% if metadata.fields.id == value %}selected{% endif %}>{{ metadata.fields.name }}</option>
         {% endfor %}
       </select>
-      <span id="status-{{ prefix }}-{{ id }}" class="input-group-text ri-save-line"></span>
+      <span id="status-{{ prefix }}-{{ id }}" class="input-group-text ri-save-line px-1"></span>
-      <button id="clear-{{ prefix }}-{{ id }}" type="button" class="btn btn-sm btn-light btn-outline-danger border"><i class="ri-eraser-line"></i></button>
+      <button id="clear-{{ prefix }}-{{ id }}" type="button" class="btn btn-sm btn-light btn-outline-danger border px-1"><i class="ri-eraser-line"></i></button>
     </div>
   {% endif %}
 {% endmacro %}

templates/sets.html

@@ -10,7 +10,7 @@
       <label class="visually-hidden" for="grid-search">Search</label>
       <div class="input-group">
         <span class="input-group-text"><i class="ri-search-line"></i><span class="ms-1 d-none d-md-inline"> Search</span></span>
-        <input id="grid-search" data-search-exact="name,number,parts,searchPurchaseLocation,searchStorage,theme,year" data-search-list="searchOwner,searchTag" class="form-control form-control-sm" type="text" placeholder="Set, name, number of parts, theme, year, owner, purchase location, storage, tag" value="">
+        <input id="grid-search" data-search-exact="name,number,parts,searchPurchaseLocation,searchStorage,theme,year" data-search-list="searchOwner,searchTag" class="form-control form-control-sm px-1" type="text" placeholder="Set, name, number of parts, theme, year, owner, purchase location, storage, tag" value="">
         <button id="grid-search-clear" type="button" class="btn btn-light btn-outline-danger border"><i class="ri-eraser-line"></i></button>
       </div>
     </div>