Scrapping Website With Beautiful Soup Library in Django Project

Create a simple scraper to get information from a web page. With BeautifulSoup & Requests libraries to analyze Web Page using text formatting HTML html.parser. Extract data of links referenced in this web page. This code pre-processes data and sends to the template an object that contains a list of anchors. Render scraped data of all links in this web page.

Scraped Data

Show Code

views.py Code

def scrape_data(request):
    # URL of the web page you want to scrape
    url = "https://portfolio-mparraf.herokuapp.com"
    module_dir = os.path.dirname(__file__)
    parent_directory = os.path.dirname(module_dir)
    search_line = "def scrape_data(request):"
    pattern = "def xml_books(request):"

    try:
        with open(os.path.join(module_dir, 'views.py'), 'r') as file:
            content = file.readlines()
            index = None

            for i, line in enumerate(content):
                if search_line in line and index is None:
                    index = i

                if line.startswith(pattern) and index is not None:
                    pos = i

        with open(os.path.join(parent_directory, 'mparraf/static/json/data-science/content.json'), 'r') as file:
            content_json = file.read()

        content_raw = json.loads(content_json)
        content_filter = {key: value for key, value in content_raw.items() if key.startswith("scrape")}
    except FileNotFoundError:
        result = "The file could not be found."

    views = content[index: pos]
    # Send an HTTP GET request to the URL
    response = requests.get(url)

    if response.status_code == 200:
        # Module to analyze files with text formatting HTML.
        soup = BeautifulSoup(response.content, 'html.parser')

        # Extract data of links referenced in this web page. This code pre-processes data and sends to the template an object that contains a list of anchors.
        links = [{'data': url + link.get('data'), 'title': link.get('title')} for link in soup.find_all('object')]

        # Render scraped data of all links in this web page.
        context = {'links': links, 'views': views, 'content': content_filter}
        return render(request, 'data-science/scrape/scraped_links.html', context)
    else:
        # Handle the case where the request was not successful
        return render(request, 'data-science/scrape/scraper_error.html')

# start xml

Template