Scraping Website With Beautiful Soup Library in Django Project

Create a simple scraper to get information from a web page. With BeautifulSoup & Requests libraries to analyze Web Page using text formatting HTML html.parser. Extract data of links referenced in this web page. This code pre-processes data and sends to the template an object that contains a list of anchors. Render scraped data of all links in this web page.
Scraped Data

Show Code

views.py Code

def scrape_data(request):
    # URL of the web page you want to scrape
    url = "https://portfolio-mparraf.herokuapp.com"
    search_line = "def scrape_data(request):"
    pattern = "def xml_books(request):"

    try:
        with open(__file__, 'r') as file:   # este módulo se documenta a sí mismo
            content = file.readlines()
            index = None

            for i, line in enumerate(content):
                if search_line in line and index is None:
                    index = i

                if line.startswith(pattern) and index is not None:
                    pos = i

        # static_dir apunta al directorio real de estáticos (mparraf/static).
        content_path = os.path.join(static_dir, 'json/data-science/content.json')
        with open(content_path, 'r') as file:
            content_json = file.read()

        content_raw = json.loads(content_json)
        content_filter = {key: value for key, value in content_raw.items() if key.startswith("scrape")}
    except FileNotFoundError:
        # Sin la metadata la página sigue siendo útil: se muestran los enlaces
        # y el código, solo falta el texto descriptivo.
        content_filter = {}

    views = content[index: pos]
    # The page being scraped is this very site's home page. It is rendered in-process, not
    # fetched over HTTP: since sep 2026 the dyno runs a single sync worker (Procfile), so a
    # request to our own host would wait for itself until the 20 s timeout and return 500.
    # The HTML is cached for 15 minutes: one render per window instead of one per visitor.
    def fetch_home():
        from portfolio.views import homeportfolio  # local import: portfolio no importa dataScience
        response = homeportfolio(request)
        return response.content if response.status_code == 200 else None

    html = cache.get_or_set("scrape_demo:home_html", fetch_home, 60 * 15)

    if html is not None:
        # Module to analyze files with text formatting HTML.
        soup = BeautifulSoup(html, 'html.parser')

        # Extract data of links referenced in this web page. This code pre-processes data and sends to the template an object that contains a list of anchors.
        links = [{'data': url + link.get('data'), 'title': link.get('title')} for link in soup.find_all('object')]

        # Render scraped data of all links in this web page.
        context = {'links': links, 'views': views, 'content': content_filter}
        return render(request, 'data-science/scrape/scraped_links.html', context)
    else:
        # Handle the case where the request was not successful
        return render(request, 'data-science/scrape/scraper_error.html')




# start xml
import os
import io
import ssl
import unicodedata
import requests
import xml.etree.ElementTree as ET
from urllib.parse import urlencode
from django.core.cache import cache
from django.core.paginator import Paginator, PageNotAnInteger, EmptyPage
from django.template.exceptions import TemplateDoesNotExist
from django.shortcuts import render

def _norm(s: str) -> str:
    """Normaliza texto: minúsculas, sin acentos, espacios compactados."""
    if not s:
        return ""
    s = unicodedata.normalize("NFKD", s)
    s = "".join(c for c in s if not unicodedata.combining(c))
    return " ".join(s.lower().strip().split())

def _fetch_books_catalog(file_id):
    """Descarga books.xml desde Drive y devuelve el catálogo completo normalizado.

    Es la parte cara (~12 MB de descarga + parseo de ~10.000 items), así que su
    resultado se cachea; los campos *_norm precomputados dejan el filtro por
    request en una pasada barata sobre la lista.
    """
    xml_bytes = download_gdrive_public_file(file_id)
    root = parse_xml_bytes(xml_bytes)
    catalog = []
    # Recorremos items; las etiquetas son <item>, <auth>, <book>, etc.
    for node in root.findall(".//item"):
        def tx(tag):
            el = node.find(tag)
            return (el.text if el is not None and el.text is not None else "").strip()

        author_raw = tx("auth")
        title_raw  = tx("book")

        img_raw = tx("img_url")
        if img_raw and not img_raw.startswith(("http://", "https://")):
            img_raw = "https://" + img_raw.lstrip("/")

        catalog.append({
            "id":           tx("isbn"),
            "title":        title_raw,
            "title_norm":   _norm(title_raw),
            "author_norm":  _norm(author_raw),
            "language":     tx("lang").capitalize(),
            "price":        tx("euro"),
            "publish_date": tx("year"),
            "description":  tx("about"),
            "publisher":    tx("publ").capitalize(),
            "tags":         tx("tags"),
            "img":          img_raw,
            "page":         tx("page"),
        })
    return catalog
Template
