7_code_executor_links.py

import os
from dotenv import load_dotenv
# For 'import autogen' to work, had to 'pip install packaging'
from autogen import AssistantAgent, UserProxyAgent
from autogen.coding import DockerCommandLineCodeExecutor

load_dotenv()

# For some reason I need proxyman running and OPENAI_BASE_URL set to http://localhost:10800/v1 for this to work 🤷‍♂️
llm_config = {
    'model': 'gpt-4o',
    'api_key': os.getenv('OPENAI_API_KEY'),
    'base_url': os.getenv('OPENAI_BASE_URL') if os.getenv('OPENAI_BASE_URL') else None,
}


directory = '/Users/johnny/Downloads/instructor-help'

# Create a Docker command line code executor.
executor = DockerCommandLineCodeExecutor(
    # Execute code using the given docker image name.
    image="python:3.12-slim",
    timeout=100,  # Timeout for each code execution in seconds.
    # Use the temporary directory to store the code files.
    work_dir=directory,
)

code_executor_agent = UserProxyAgent(
    "code_executor_agent",
    # Use the docker command line code executor.
    code_execution_config={"executor": executor},
)

code_writer_agent = AssistantAgent(
    "code_writer_agent",
    llm_config=llm_config,
)

website = 'https://instructor-help.examind.io/'

chat_result = code_executor_agent.initiate_chat(
    code_writer_agent,
    message=f"Write Python code to scrape the webpage at {website}, "
    f"find all absolute links that start with {website} or relative links, "
    "and convert relative links to absolute fully qualified links. "
    f"Follow these links and find more links that are either absolute {website} links or relative links. "
    "Continue this process recursively until no more relevant links are found. "
    "Do not follow links that have already been crawled. "
    "Save the output to a file named links.json that stores an array of unique link strings in JSON format."
)

# When the code executor is no longer used, stop it to release the resources.
# executor.stop()