@dataclass
class ScraperConfig:
    """Configuration for the web scraper"""
    max_pages: int = 100
    wait_time: float = 2.0
    page_scroll_wait: float = 2.0
    batch_size: int = 5
    requests_per_second: float = 1.0
    image_quality: int = 85
    convert_to_webp: bool = True
    save_original: bool = True
    output_formats: List[str] = field(default_factory=lambda: ['json', 'html', 'markdown'])
    user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
    manual_mode: bool = False
    manual_wait_time: int = 30  # Seconds to wait during manual browsing
    accept_cookies: bool = True  # Automatically try to accept cookies
    wait_for_js: bool = True    # Wait for JavaScript to complete
    ignore_https_errors: bool = False  # Ignore HTTPS errors
    browser: str = "chromium"   # Browser to use: "chromium", "firefox", or "webkit"
    use_keypress_fallback: bool = False  # Use keypress fallback for manual interaction
 
class RateLimiter:
    """Rate limiting for respectful crawling"""
    def __init__(self, requests_per_second: float):
        self.delay = 1.0 / requests_per_second
        self.last_request_time = 0
 
    async def wait(self):
        now = time.time()
        delay_needed = self.last_request_time + self.delay - now
        if delay_needed > 0:
            await asyncio.sleep(delay_needed)
        self.last_request_time = time.time()
 
class EnhancedWebScraper:
    """Enhanced web scraper with improved image handling and organization"""
    
    def __init__(self, config: Optional[ScraperConfig] = None):
        self.config = config or ScraperConfig()
        self.setup_logging()
        self.setup_directories()
        self.rate_limiter = RateLimiter(self.config.requests_per_second)
        self.visited_urls: Set[str] = set()
        self.processed_resources: Set[str] = set()  # Track by hash

async def process_batch(self, urls: List[str]):
    """Process multiple URLs in parallel"""
    tasks = []
    for i in range(0, len(urls), self.config.batch_size):
        batch = urls[i:i + self.config.batch_size]
        batch_tasks = [self.process_url(url) for url in batch]
        tasks.extend(batch_tasks)
    
    results = await asyncio.gather(*tasks, return_exceptions=True)
    return [r for r in results if not isinstance(r, Exception)]

async def handle_lazy_images(self, page):
    """Handle lazy-loaded images by scrolling"""
    self.logger.info("Handling lazy-loaded images...")
    try:
        # Scroll to bottom in steps
        viewport_height = await page.evaluate('window.innerHeight')
        page_height = await page.evaluate('document.documentElement.scrollHeight')
        
        for position in range(0, page_height, viewport_height):
            await page.evaluate(f'window.scrollTo(0, {position})')
            await asyncio.sleep(self.config.page_scroll_wait)
        
        # Scroll back to top
        await page.evaluate('window.scrollTo(0, 0)')
        await asyncio.sleep(self.config.page_scroll_wait)
        
    except Exception as e:
        self.logger.error(f"Error handling lazy images: {e}")

async def extract_images(self, page, soup: BeautifulSoup, base_url: str) -> List[ExtractedResource]:
    """Extract images from the page"""
    resources = []
    
    # Handle lazy-loaded images
    await self.handle_lazy_images(page)
    
    # Extract images from various sources
    sources = {
        'img_tags': soup.find_all('img', src=True),
        'picture_tags': soup.find_all('picture'),
        'background_images': soup.find_all(style=re.compile(r'background-image')),
        'svg_tags': soup.find_all('svg'),
        'css_images': await self.extract_css_images(page)
    }
    
    for source_type, elements in sources.items():
        for element in elements:
            try:
                resource = await self.process_element(element, source_type, base_url)
                if resource and resource.hash not in self.processed_resources:
                    resources.append(resource)
                    self.processed_resources.add(resource.hash)
            except Exception as e:
                self.logger.error(f"Error processing {source_type} element: {e}")
    
    return resources

async def process_image(self, resource: ExtractedResource) -> List[Path]:
    """Process and save image in multiple formats if needed"""
    saved_paths = []
    try:
        # Create image object
        img = Image.open(BytesIO(resource.content))
        
        # Save original if configured
        if self.config.save_original:
            original_path = self.dirs['images'] / resource.filename
            img.save(original_path, optimize=True, quality=self.config.image_quality)
            saved_paths.append(original_path)
        
        # Convert to WebP if configured
        if self.config.convert_to_webp:
            webp_path = self.dirs['images_webp'] / f"{resource.filename.rsplit('.', 1)[0]}.webp"
            if img.mode in ('RGBA', 'LA'):
                background = Image.new('RGB', img.size, (255, 255, 255))
                background.paste(img, mask=img.split()[-1])
                img = background
            img.save(webp_path, 'WEBP', quality=self.config.image_quality)
            saved_paths.append(webp_path)
        
    except Exception as e:
        self.logger.error(f"Error processing image {resource.url}: {e}")
    
    return saved_paths

async def extract_css_images(self, page) -> List[Dict]:
    """Extract images from CSS using JavaScript"""
    try:
        css_images = await page.evaluate("""() => {
            const images = [];
            const styleSheets = Array.from(document.styleSheets);
            
            styleSheets.forEach(sheet => {
                try {
                    const rules = Array.from(sheet.cssRules || []);
                    rules.forEach(rule => {
                        if (rule.style && rule.style.backgroundImage) {
                            const urls = rule.style.backgroundImage.match(/url\\(['"]?(.*?)['"]?\\)/g);
                            if (urls) {
                                urls.forEach(url => {
                                    const cleanUrl = url.replace(/url\\(['"]?|['"]?\\)/g, '');
                                    if (cleanUrl && !cleanUrl.startsWith('data:')) {
                                        images.push({
                                            url: cleanUrl,
                                            selector: rule.selectorText || 'unknown'
                                        });
                                    }
                                });
                            }
                        }
                    });
                } catch (e) {
                    // Handle CORS errors
                    console.warn('Could not access stylesheet rules');
                }
            });
            
            return images;
        }""")
        
        return css_images
    except Exception as e:
        self.logger.error(f"Error extracting CSS images: {e}")
        return []

async def accept_cookies(self, page) -> bool:
    """Try to automatically accept cookies on the page"""
    try:
        self.logger.info("Attempting to auto-accept cookies...")
        
        # Common patterns for cookie buttons and forms
        cookie_button_selectors = [
            "button[id*='cookie' i]",
            "button[class*='cookie' i]",
            "button[id*='accept' i]",
            "button[class*='accept' i]",
            "a[id*='cookie' i]",
            "a[class*='cookie' i]",
            "a[id*='accept' i]",
            "a[class*='accept' i]",
            "[id*='cookie-consent' i] button",
            "#cookieChoiceDismiss",
            ".cookie-banner button",
            "#consent-btn",
            "#onetrust-accept-btn-handler",
            ".cookie-notice-action button",
            "[class*='CookieConsent'] button",
            "[data-cookieconsent='accept']",
            "[aria-label*='Accept cookies' i]",
            "[title*='Accept cookies' i]"
        ]
        
        # Try each selector
        for selector in cookie_button_selectors:
            if await page.query_selector(selector):
                self.logger.info(f"Found potential cookie button with selector: {selector}")
                try:
                    await page.click(selector)
                    self.logger.info("Clicked cookie button")
                    # Wait a moment for the banner to disappear
                    await asyncio.sleep(1)
                    return True
                except Exception as e:
                    self.logger.warning(f"Failed to click cookie button: {e}")
        
        self.logger.info("No cookie buttons found or all attempts failed")
        return False
        
    except Exception as e:
        self.logger.error(f"Error during cookie acceptance: {e}")
        return False

try:
    # Processing logic...
except Exception as e:
    self.logger.error(f"Error processing URL {url}: {e}")
    
    # Add macOS-specific troubleshooting advice
    if "Target page, context or browser has been closed" in str(e) or "Browser has been closed" in str(e):
        print("\nTROUBLESHOOTING: Browser launch failed. This is common on macOS due to security settings.")
        print("Try the following:")
        print("1. Go to System Preferences > Security & Privacy > Privacy > Automation")
        print("2. Make sure Terminal or your Python IDE has permission to control 'Chrome' or 'Chromium'")
        print("3. Try using Firefox instead by editing the script to use playwright.firefox")
        print("4. Try running with headless=True (automatic mode) if you just need the scraping functionality")
        print("5. Restart and use the keypress fallback option when prompted\n")
        
        # Ask if user wants to try keypress fallback
        if self.config.manual_mode and not self.config.use_keypress_fallback:
            retry = input("Would you like to try the keypress fallback approach? (y/n): ").strip().lower() == 'y'
            if retry:
                print("Retrying with keypress fallback...\n")
                self.config.use_keypress_fallback = True
                return await self._process_url_with_keypress_fallback(url)

# Detect and handle platform-specific issues
if os.name == 'nt':
    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 
# MacOS-specific browser handling
if platform.system() == 'Darwin':
    # Use simpler launch options on MacOS to avoid security issues
    browser = await browser_engine.launch(
        headless=False,
        args=['--start-maximized'] if self.config.browser == 'chromium' else None
    )

async def _process_url_with_keypress_fallback(self, url: str) -> Dict:
    """Alternative approach that uses external browser and waits for user keypress"""
    start_time = time.time()
    resources = []
    screenshots = []
    
    try:
        # Print instructions
        print(f"\n{'='*70}")
        print(f"KEYPRESS FALLBACK MODE ACTIVE")
        print(f"Since the automated browser couldn't be launched, we'll use this alternative method.")
        print(f"INSTRUCTIONS:")
        print(f"1. Open {url} in your browser manually")
        print(f"2. Navigate to the content you want to extract")
        print(f"3. When ready, come back to this window and press ENTER")
        print(f"4. The script will ask for the HTML of the page")
        print(f"{'='*70}\n")
        
        # Wait for user to press ENTER
        input("Press ENTER when you've browsed to the content and are ready to extract it...")
        
        # Process HTML from manual browser...

# Track already processed resources by hash
self.processed_resources: Set[str] = set()
 
# Add only unique resources
if resource.hash not in self.processed_resources:
    resources.append(resource)
    self.processed_resources.add(resource.hash)

async def create_report(self, url: str, resources: List[ExtractedResource], 
                      execution_time: float, screenshots: List[Path] = None) -> Dict:
    """Create a detailed report of the scraping session"""
    report = {
        'url': url,
        'timestamp': datetime.now().isoformat(),
        'execution_time_seconds': execution_time,
        'resources': {
            'total': len(resources),
            'by_type': {},
            'details': []
        },
        'screenshots': [str(path) for path in (screenshots or [])],
        'config': {
            # Configuration details...
        }
    }
    
    # Compile resource statistics
    for resource in resources:
        # Update type counts
        if resource.type not in report['resources']['by_type']:
            report['resources']['by_type'][resource.type] = 0
        report['resources']['by_type'][resource.type] += 1
        
        # Add resource details
        report['resources']['details'].append({
            'type': resource.type,
            'url': resource.url,
            'filename': resource.filename,
            'metadata': resource.metadata
        })
    
    # Save report
    report_path = self.dirs['reports'] / f"report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
    async with aiofiles.open(report_path, 'w', encoding='utf-8') as f:
        await f.write(json.dumps(report, indent=2))
    
    return report

@dataclass
class ExtractedResource:
    """Represents an extracted resource (image, document, etc.)"""
    url: str
    type: str
    content: bytes
    filename: str
    metadata: Dict = field(default_factory=dict)
    hash: Optional[str] = None
 
    def __post_init__(self):
        if not self.hash and self.content:
            self.hash = hashlib.md5(self.content).hexdigest()

def setup_logging(self):
    """Configure logging system"""
    log_dir = Path('logs')
    log_dir.mkdir(exist_ok=True)
    
    log_file = log_dir / f'scraper_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log'
    
    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s - %(levelname)s - [%(name)s] - %(message)s',
        handlers=[
            logging.StreamHandler(),
            logging.FileHandler(log_file, encoding='utf-8')
        ]
    )
    self.logger = logging.getLogger(__name__)