Как избежать извлечения повторяющихся ссылок в Python Scrapy?

Чтобы предотвратить извлечение повторяющихся ссылок при сканировании веб-сайта с помощью Scrapy, вы можете сохранить набор посещенных URL-адресов и проверить, была ли посещена каждая ссылка ранее.

import json
import mimetypes
from typing import Any, Optional
import scrapy.linkextractors
import scrapy.spiders
from scrapy.http import Response
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

# Define the allowed MIME types for downloading
ALLOWED_DOWNLOAD_TYPES = {
    "application/pdf",
    "application/msword",
    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
}

# Function to check if a MIME type is allowed for download
def is_download_type(typ: str) -> bool:
    return typ in ALLOWED_DOWNLOAD_TYPES

# Define the spider class
class QuotesSpider(scrapy.Spider):
    name = "quotes"

    allowed_domains = [
        "quotes.toscrape.com"
    ]

    start_urls = [
        "https://quotes.toscrape.com/"
    ]

    def __init__(self, name: Optional[str] = None, **kwargs: Any):
        super().__init__(name, **kwargs)

        if not mimetypes.inited:
            mimetypes.init()

        self.link_extractor = scrapy.linkextractors.LinkExtractor()

        # Track visited URLs to avoid duplicates
        self.visited_urls = set()
        
    def parse(self, response: Response, **kwargs: Any) -> Any:
        with open("output.txt", "w", encoding = "utf-8") as output_file:
            for link in self.link_extractor.extract_links(response):
                # Normalize URL for comparison
                normalized_url = response.urljoin(link.url)
                if normalized_url not in self.visited_urls:
                    self.visited_urls.add(normalized_url)
                    typ, _ = mimetypes.guess_type(link.url)
                    if typ is not None:
                        if typ == "text/html":
                            yield response.follow(link, callback=self.parse)
                        if is_download_type(typ):
                            result = {"nofollow": link.nofollow, "url": link.url, "text": link.text}
                            output_file.write(json.dumps(result) + '\n')

# Execute the spider
if __name__ == "__main__":
    process = CrawlerProcess(get_project_settings())
    process.crawl(QuotesSpider)
    process.start()