Move docker code exec to autogen-ext (#3733)

* move docker code exec to autogen-ext

* fix test

* rename docker subpackage

* add missing renamed package

---------

Co-authored-by: Leonardo Pinheiro <[email protected]>

python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/guides/code-execution.ipynb

@@ -315,8 +315,8 @@
    "source": [
     "from autogen_agentchat.agents import CodeExecutorAgent, CodingAssistantAgent\n",
     "from autogen_agentchat.teams import RoundRobinGroupChat, StopMessageTermination\n",
-    "from autogen_core.components.code_executor import DockerCommandLineCodeExecutor\n",
     "from autogen_core.components.models import OpenAIChatCompletionClient\n",
+    "from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor\n",
     "\n",
     "async with DockerCommandLineCodeExecutor(work_dir=\"coding\") as code_executor:  # type: ignore[syntax]\n",
     "    code_executor_agent = CodeExecutorAgent(\"code_executor\", code_executor=code_executor)\n",

python/packages/autogen-core/docs/src/user-guide/agentchat-user-guide/stocksnippet.md

@@ -8,7 +8,7 @@ from autogen_agentchat import EVENT_LOGGER_NAME
 from autogen_agentchat.agents import CodeExecutorAgent, CodingAssistantAgent
 from autogen_agentchat.logging import ConsoleLogHandler
 from autogen_agentchat.teams import RoundRobinGroupChat, StopMessageTermination
-from autogen_core.components.code_executor import DockerCommandLineCodeExecutor
+from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor
 from autogen_core.components.models import OpenAIChatCompletionClient
 
 logger = logging.getLogger(EVENT_LOGGER_NAME)

python/packages/autogen-core/docs/src/user-guide/core-user-guide/cookbook/tool-use-with-intervention.ipynb

@@ -23,7 +23,6 @@
     "from autogen_core.base import AgentId, AgentType, MessageContext\n",
     "from autogen_core.base.intervention import DefaultInterventionHandler, DropMessage\n",
     "from autogen_core.components import FunctionCall, RoutedAgent, message_handler\n",
-    "from autogen_core.components.code_executor import DockerCommandLineCodeExecutor\n",
     "from autogen_core.components.models import (\n",
     "    ChatCompletionClient,\n",
     "    LLMMessage,\n",
@@ -32,7 +31,8 @@
     "    UserMessage,\n",
     ")\n",
     "from autogen_core.components.tool_agent import ToolAgent, ToolException, tool_agent_caller_loop\n",
-    "from autogen_core.components.tools import PythonCodeExecutionTool, ToolSchema"
+    "from autogen_core.components.tools import PythonCodeExecutionTool, ToolSchema\n",
+    "from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor"
    ]
   },
   {
@@ -157,7 +157,7 @@
    "source": [
     "In this example, we will use a tool for Python code execution.\n",
     "First, we create a Docker-based command-line code executor\n",
-    "using {py:class}`~autogen_core.components.code_executor.DockerCommandLineCodeExecutor`,\n",
+    "using {py:class}`~autogen_core.components.code_executor.docker_executorCommandLineCodeExecutor`,\n",
     "and then use it to instantiate a built-in Python code execution tool\n",
     "{py:class}`~autogen_core.components.tools.PythonCodeExecutionTool`\n",
     "that runs code in a Docker container."

python/packages/autogen-core/docs/src/user-guide/core-user-guide/framework/command-line-code-executors.ipynb

@@ -10,12 +10,12 @@
     "Generally speaking, it will save each code block to a file and the execute that file.\n",
     "This means that each code block is executed in a new process. There are two forms of this executor:\n",
     "\n",
-    "- Docker ({py:class}`~autogen_core.components.code_executor.DockerCommandLineCodeExecutor`) - this is where all commands are executed in a Docker container\n",
+    "- Docker ({py:class}`~autogen_ext.code_executor.docker_executor.DockerCommandLineCodeExecutor`) - this is where all commands are executed in a Docker container\n",
     "- Local ({py:class}`~autogen_core.components.code_executor.LocalCommandLineCodeExecutor`) - this is where all commands are executed on the host machine\n",
     "\n",
     "## Docker\n",
     "\n",
-    "The {py:class}`~autogen_core.components.code_executor.DockerCommandLineCodeExecutor` will create a Docker container and run all commands within that container. \n",
+    "The {py:class}`~autogen_ext.code_executor.docker_executor.DockerCommandLineCodeExecutor` will create a Docker container and run all commands within that container. \n",
     "The default image that is used is `python:3-slim`, this can be customized by passing the `image` parameter to the constructor. \n",
     "If the image is not found locally then the class will try to pull it. \n",
     "Therefore, having built the image locally is enough. The only thing required for this image to be compatible with the executor is to have `sh` and `python` installed. \n",
@@ -50,7 +50,8 @@
     "from pathlib import Path\n",
     "\n",
     "from autogen_core.base import CancellationToken\n",
-    "from autogen_core.components.code_executor import CodeBlock, DockerCommandLineCodeExecutor\n",
+    "from autogen_core.components.code_executor import CodeBlock\n",
+    "from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor\n",
     "\n",
     "work_dir = Path(\"coding\")\n",
     "work_dir.mkdir(exist_ok=True)\n",

python/packages/autogen-core/docs/src/user-guide/core-user-guide/framework/tools.ipynb

@@ -44,8 +44,8 @@
    ],
    "source": [
     "from autogen_core.base import CancellationToken\n",
-    "from autogen_core.components.code_executor import DockerCommandLineCodeExecutor\n",
     "from autogen_core.components.tools import PythonCodeExecutionTool\n",
+    "from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor\n",
     "\n",
     "# Create the tool.\n",
     "code_executor = DockerCommandLineCodeExecutor()\n",
@@ -63,7 +63,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "The {py:class}`~autogen_core.components.code_executor.DockerCommandLineCodeExecutor`\n",
+    "The {py:class}`~autogen_core.components.code_executor.docker_executorCommandLineCodeExecutor`\n",
     "class is a built-in code executor that runs Python code snippets in a subprocess\n",
     "in the local command line environment.\n",
     "The {py:class}`~autogen_core.components.tools.PythonCodeExecutionTool` class wraps the code executor\n",

python/packages/autogen-core/docs/src/user-guide/core-user-guide/quickstart.ipynb

@@ -312,8 +312,8 @@
     "import tempfile\n",
     "\n",
     "from autogen_core.application import SingleThreadedAgentRuntime\n",
-    "from autogen_core.components.code_executor import DockerCommandLineCodeExecutor\n",
     "from autogen_core.components.models import OpenAIChatCompletionClient\n",
+    "from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor\n",
     "\n",
     "work_dir = tempfile.mkdtemp()\n",
     "\n",

python/packages/autogen-core/pyproject.toml

@@ -23,7 +23,6 @@ dependencies = [
     "grpcio~=1.62.0",
     "protobuf~=4.25.1",
     "tiktoken",
-    "docker~=7.0",
     "opentelemetry-api~=1.27.0",
     "asyncio_atexit"
 ]

python/packages/autogen-core/samples/coding_pub_sub.py

@@ -20,7 +20,6 @@
 from autogen_core.application import SingleThreadedAgentRuntime
 from autogen_core.base import MessageContext
 from autogen_core.components import DefaultSubscription, DefaultTopicId, FunctionCall, RoutedAgent, message_handler
-from autogen_core.components.code_executor import DockerCommandLineCodeExecutor
 from autogen_core.components.models import (
     AssistantMessage,
     ChatCompletionClient,
@@ -31,6 +30,7 @@
     UserMessage,
 )
 from autogen_core.components.tools import PythonCodeExecutionTool, Tool
+from autogen_ext.code_executor.docker_executor import DockerCommandLineCodeExecutor
 from common.utils import get_chat_completion_client_from_envs
 
 

python/packages/autogen-core/src/autogen_core/components/code_executor/__init__.py

@@ -10,9 +10,8 @@
     with_requirements,
 )
 from ._impl.command_line_code_result import CommandLineCodeResult
-from ._impl.docker_command_line_code_executor import DockerCommandLineCodeExecutor
 from ._impl.local_commandline_code_executor import LocalCommandLineCodeExecutor
-from ._impl.utils import get_required_packages, lang_to_cmd
+from ._impl.utils import get_file_name_from_content, get_required_packages, lang_to_cmd, silence_pip
 from ._utils import extract_markdown_code_blocks
 
 __all__ = [
@@ -31,7 +30,8 @@
     "extract_markdown_code_blocks",
     "get_required_packages",
     "build_python_functions_file",
-    "DockerCommandLineCodeExecutor",
     "get_required_packages",
     "lang_to_cmd",
+    "get_file_name_from_content",
+    "silence_pip",
 ]

python/packages/autogen-core/tests/execution/test_commandline_code_executor.py

@@ -2,7 +2,6 @@
 # Credit to original authors
 
 import asyncio
-import os
 import sys
 import tempfile
 from pathlib import Path
@@ -12,48 +11,22 @@
 import pytest_asyncio
 from aiofiles import open
 from autogen_core.base import CancellationToken
-from autogen_core.components.code_executor import CodeBlock, DockerCommandLineCodeExecutor, LocalCommandLineCodeExecutor
-
-
-def docker_tests_enabled() -> bool:
-    if os.environ.get("SKIP_DOCKER", "unset").lower() == "true":
-        return False
-
-    try:
-        import docker
-        from docker.errors import DockerException
-    except ImportError:
-        return False
-
-    try:
-        client = docker.from_env()
-        client.ping()  # type: ignore
-        return True
-    except DockerException:
-        return False
+from autogen_core.components.code_executor import CodeBlock, LocalCommandLineCodeExecutor
 
 
 @pytest_asyncio.fixture(scope="function")  # type: ignore
 async def executor_and_temp_dir(
     request: pytest.FixtureRequest,
-) -> AsyncGenerator[tuple[LocalCommandLineCodeExecutor | DockerCommandLineCodeExecutor, str], None]:
-    if request.param == "local":
-        with tempfile.TemporaryDirectory() as temp_dir:
-            yield LocalCommandLineCodeExecutor(work_dir=temp_dir), temp_dir
-    elif request.param == "docker":
-        if not docker_tests_enabled():
-            pytest.skip("Docker tests are disabled")
-
-        with tempfile.TemporaryDirectory() as temp_dir:
-            async with DockerCommandLineCodeExecutor(work_dir=temp_dir) as executor:
-                yield executor, temp_dir
+) -> AsyncGenerator[tuple[LocalCommandLineCodeExecutor, str], None]:
+    with tempfile.TemporaryDirectory() as temp_dir:
+        yield LocalCommandLineCodeExecutor(work_dir=temp_dir), temp_dir
 
 
-ExecutorFixture: TypeAlias = tuple[LocalCommandLineCodeExecutor | DockerCommandLineCodeExecutor, str]
+ExecutorFixture: TypeAlias = tuple[LocalCommandLineCodeExecutor, str]
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize("executor_and_temp_dir", ["local", "docker"], indirect=True)
+@pytest.mark.parametrize("executor_and_temp_dir", ["local"], indirect=True)
 async def test_execute_code(executor_and_temp_dir: ExecutorFixture) -> None:
     executor, _temp_dir = executor_and_temp_dir
     cancellation_token = CancellationToken()
@@ -101,7 +74,7 @@ async def test_execute_code(executor_and_temp_dir: ExecutorFixture) -> None:
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize("executor_and_temp_dir", ["local", "docker"], indirect=True)
+@pytest.mark.parametrize("executor_and_temp_dir", ["local"], indirect=True)
 async def test_commandline_code_executor_timeout(executor_and_temp_dir: ExecutorFixture) -> None:
     executor, temp_dir = executor_and_temp_dir
     cancellation_token = CancellationToken()
@@ -111,7 +84,6 @@ async def test_commandline_code_executor_timeout(executor_and_temp_dir: Executor
     assert code_result.exit_code and "Timeout" in code_result.output
 
 
-# TODO: add docker when cancellation is supported
 @pytest.mark.asyncio
 async def test_commandline_code_executor_cancellation() -> None:
     with tempfile.TemporaryDirectory() as temp_dir:
@@ -136,7 +108,7 @@ async def test_local_commandline_code_executor_restart() -> None:
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize("executor_and_temp_dir", ["local", "docker"], indirect=True)
+@pytest.mark.parametrize("executor_and_temp_dir", ["local"], indirect=True)
 async def test_invalid_relative_path(executor_and_temp_dir: ExecutorFixture) -> None:
     executor, _temp_dir = executor_and_temp_dir
     cancellation_token = CancellationToken()
@@ -151,7 +123,7 @@ async def test_invalid_relative_path(executor_and_temp_dir: ExecutorFixture) ->
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize("executor_and_temp_dir", ["local", "docker"], indirect=True)
+@pytest.mark.parametrize("executor_and_temp_dir", ["local"], indirect=True)
 async def test_valid_relative_path(executor_and_temp_dir: ExecutorFixture) -> None:
     executor, temp_dir_str = executor_and_temp_dir
 
@@ -171,24 +143,3 @@ async def test_valid_relative_path(executor_and_temp_dir: ExecutorFixture) -> No
     assert "test.py" in result.code_file
     assert (temp_dir / Path("test.py")).resolve() == Path(result.code_file).resolve()
     assert (temp_dir / Path("test.py")).exists()
-
-
-@pytest.mark.asyncio
-async def test_docker_commandline_code_executor_start_stop() -> None:
-    if not docker_tests_enabled():
-        pytest.skip("Docker tests are disabled")
-
-    with tempfile.TemporaryDirectory() as temp_dir:
-        executor = DockerCommandLineCodeExecutor(work_dir=temp_dir)
-        await executor.start()
-        await executor.stop()
-
-
-@pytest.mark.asyncio
-async def test_docker_commandline_code_executor_start_stop_context_manager() -> None:
-    if not docker_tests_enabled():
-        pytest.skip("Docker tests are disabled")
-
-    with tempfile.TemporaryDirectory() as temp_dir:
-        async with DockerCommandLineCodeExecutor(work_dir=temp_dir) as _exec:
-            pass

python/packages/autogen-ext/pyproject.toml

@@ -22,6 +22,7 @@ dependencies = [
 [project.optional-dependencies]
 langchain-tools = ["langchain >= 0.3.1"]
 azure-code-executor = ["azure-core"]
+docker-code-executor = ["docker~=7.0"]
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/autogen_ext"]
@@ -47,3 +48,8 @@ include = "../../shared_tasks.toml"
 
 [tool.poe.tasks]
 test = "pytest -n auto"
+
+[tool.mypy]
+[[tool.mypy.overrides]]
+module = "docker.*"
+ignore_missing_imports = true

python/packages/autogen-ext/src/autogen_ext/code_executor/docker_executor/__init__.py

@@ -0,0 +1,3 @@
+from ._impl import DockerCommandLineCodeExecutor
+
+__all__ = ["DockerCommandLineCodeExecutor"]

python/packages/autogen-core/src/autogen_core/components/code_executor/_impl/docker_command_line_code_executor.py → python/packages/autogen-ext/src/autogen_ext/code_executor/docker_executor/_impl.py

@@ -18,16 +18,19 @@
 import docker
 import docker.models
 import docker.models.containers
-from docker.errors import ImageNotFound, NotFound
-
-from ....base._cancellation_token import CancellationToken
-from ....components.code_executor._base import CodeBlock, CodeExecutor
-from ....components.code_executor._func_with_reqs import FunctionWithRequirements, FunctionWithRequirementsStr
-from ....components.code_executor._impl.command_line_code_result import CommandLineCodeResult
-from .._func_with_reqs import (
+from autogen_core.base import CancellationToken
+from autogen_core.components.code_executor import (
+    CodeBlock,
+    CodeExecutor,
+    CommandLineCodeResult,
+    FunctionWithRequirements,
+    FunctionWithRequirementsStr,
     build_python_functions_file,
+    get_file_name_from_content,
+    lang_to_cmd,
+    silence_pip,
 )
-from .utils import get_file_name_from_content, lang_to_cmd, silence_pip
+from docker.errors import ImageNotFound, NotFound
 
 if sys.version_info >= (3, 11):
     from typing import Self