Dynamic Rotating Mobile Proxies for High-Volume Web Scraping

import scrapy
import requests
import time

class ECommerceScraper(scrapy.Spider):
    name = "ecommerce"
    start_urls = ["https://example.com/products"]
    
    # XProxy settings
    proxy_url = "http://user:[email protected]:10001"
    rotation_url = "http://192.168.1.100:8000/api/rotate?port=10001"

    def rotate_ip(self):
        self.logger.info("Rate limit hit! Triggering XProxy cellular IP rotation...")
        res = requests.get(self.rotation_url)
        if res.status_code == 200:
            time.sleep(4)  # Wait for hardware modem handshake
            self.logger.info("IP rotated successfully.")
        else:
            self.logger.error("Failed to rotate IP.")

    def parse(self, response):
        # If blocked by anti-bot firewall, rotate cellular IP and retry
        if "captcha" in response.url or response.status == 429:
            self.rotate_ip()
            yield scrapy.Request(response.url, callback=self.parse, dont_filter=True)
            return

        # Normal parsing logic
        for product in response.css(".product-card"):
            yield {
                "name": product.css(".title::text").get(),
                "price": product.css(".price::text").get(),
            }