web2context.py

"""This file runs the entire crawler process and manages PDF outputs."""
import argparse
import asyncio
import os
import sys
import warnings
from pathlib import Path

# Suppress asyncio warnings about unclosed loops
warnings.filterwarnings('ignore', category=RuntimeWarning, message='.*coroutine.*was never awaited')
warnings.filterwarnings('ignore', category=RuntimeWarning, message='.*unclosed.*')

# Store original exception hook
_original_excepthook = sys.excepthook

def _quiet_excepthook(exc_type, exc_value, exc_traceback):
    """Suppress certain exceptions during shutdown."""
    # Suppress KeyboardInterrupt exceptions during shutdown
    if exc_type is KeyboardInterrupt:
        return
    
    # Suppress RuntimeError about ignored coroutines
    if exc_type is RuntimeError:
        error_msg = str(exc_value).lower()
        if 'coroutine' in error_msg or 'generatorexit' in error_msg:
            return
    
    # Suppress GeneratorExit exceptions
    if exc_type is GeneratorExit:
        return
    
    # Suppress threading module exceptions during shutdown
    if exc_traceback:
        frame = exc_traceback
        while frame:
            filename = frame.tb_frame.f_code.co_filename
            # Suppress exceptions from threading.py and concurrent/futures during shutdown
            if 'threading.py' in filename or 'concurrent/futures' in filename:
                if exc_type is KeyboardInterrupt or 'shutdown' in str(exc_value).lower():
                    return
            frame = frame.tb_next
    
    # Call original handler for other exceptions
    _original_excepthook(exc_type, exc_value, exc_traceback)

# Set custom exception hook
sys.excepthook = _quiet_excepthook

from crawler_components.orchestrator import Web2Context


async def _run_crawler(crawler):
    """Run crawler with proper exception handling."""
    try:
        await crawler.crawl()
    except (KeyboardInterrupt, asyncio.CancelledError):
        # Re-raise to be handled by crawl() method
        raise
    except Exception:
        # Re-raise other exceptions
        raise


def _exception_handler(loop, context):
    """Handle unhandled exceptions in event loop (suppress cancellation errors)."""
    exception = context.get('exception')
    message = context.get('message', '')
    
    if exception:
        # Suppress cancellation and interruption errors
        if isinstance(exception, (asyncio.CancelledError, KeyboardInterrupt)):
            return
        
        # Suppress Playwright errors that occur during cancellation
        exception_str = str(exception)
        exception_type_str = type(exception).__name__
        
        if ('ERR_ABORTED' in exception_str or 
            'TargetClosedError' in exception_type_str or
            'Target page, context or browser has been closed' in exception_str or
            'coroutine ignored' in exception_str or
            'GeneratorExit' in exception_str):
            return
    
    # Suppress "Future exception was never retrieved" warnings for cancelled tasks
    if ('Future exception was never retrieved' in message or
        'Exception ignored' in message):
        return
    
    # For other exceptions, suppress them silently (they're likely cancellation-related)
    # This prevents noisy error messages during graceful shutdown
    pass


def main():
    """Main entry point."""
    try:
        parser = argparse.ArgumentParser(
            description='Web2Context: Crawl a website and convert pages to PDF.',
            formatter_class=argparse.RawDescriptionHelpFormatter,
            epilog="""
Examples:
  python web2context.py https://example.com
  python web2context.py https://example.com --llm
  python web2context.py https://example.com --llm --fast
            """
        )
        
        parser.add_argument(
            'url',
            help='Starting URL to crawl'
        )
        
        parser.add_argument(
            '--llm',
            action='store_true',
            help='Optimize for LLMs (Clean output, ignore query params, update mode)'
        )
        
        parser.add_argument(
            '--debug',
            action='store_true',
            help='Enable debug logging'
        )
        
        parser.add_argument(
            '--fast',
            action='store_true',
            help='Turbo mode (10 workers, min delay)'
        )
        
        args = parser.parse_args()
        
        # Default settings
        workers = 3
        delay = 1.0
        clean = False
        ignore_query = False
        if_exists = 'ask'
        auto_scope = False
        
        # Apply --llm settings
        if args.llm:
            clean = True
            ignore_query = True
            if_exists = 'update'
            auto_scope = True  # Enable smart scope detection
            
        # Apply --fast settings
        if args.fast:
            workers = 10
            delay = 0.1
        
        # Create crawler and run with proper exception handling
        crawler = Web2Context(
            args.url,
            output_dir=None, # Auto-generated based on domain
            workers=workers,
            delay=delay,
            debug=args.debug,
            if_exists=if_exists,
            ignore_query=ignore_query,
            clean_output=clean,
            auto_scope=auto_scope
        )
        
        # Create event loop with custom exception handler
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        loop.set_exception_handler(_exception_handler)
        
        try:
            loop.run_until_complete(_run_crawler(crawler))
        finally:
            # Clean shutdown: cancel all tasks and close loop properly
            try:
                # Get all pending tasks
                pending = [t for t in asyncio.all_tasks(loop) if not t.done()]
                
                # Cancel all pending tasks
                for task in pending:
                    task.cancel()
                
                # Wait for cancellation to complete
                if pending:
                    # Use gather with return_exceptions to avoid raising
                    loop.run_until_complete(
                        asyncio.gather(*pending, return_exceptions=True)
                    )
                
                # Give loop a chance to process cancellations
                loop.run_until_complete(asyncio.sleep(0))
            except Exception:
                # Ignore errors during cleanup
                pass
            finally:
                try:
                    # Close the loop
                    loop.close()
                except Exception:
                    pass
    
    except KeyboardInterrupt:
        # This should be caught by crawl() method, but just in case
        print("\n\nProcess stopped.", file=sys.stderr)
        # Suppress threading cleanup exceptions by exiting immediately
        os._exit(0)
    
    except Exception as e:
        print(f"\nUnexpected error: {str(e)}", file=sys.stderr)
        sys.exit(1)


if __name__ == '__main__':
    main()