From d78ef5c8c3d7441f25153ee78af1b3086e10ab32 Mon Sep 17 00:00:00 2001 From: Graham Neubig Date: Fri, 23 Aug 2024 17:58:00 -0400 Subject: [PATCH] Fix break in app config (#10) * Fix break in app config * Fix warning --- github_resolver/resolve_issues.py | 5 +- poetry.lock | 22 ++-- tests/test_resolve_issues.py | 161 ++++++++++++++++++++++++------ 3 files changed, 142 insertions(+), 46 deletions(-) diff --git a/github_resolver/resolve_issues.py b/github_resolver/resolve_issues.py index 0983768..0535540 100644 --- a/github_resolver/resolve_issues.py +++ b/github_resolver/resolve_issues.py @@ -97,7 +97,7 @@ async def initialize_runtime( logger.info('-' * 30) obs: CmdOutputObservation - action = CmdRunAction(command=f'cd /workspace') + action = CmdRunAction(command='cd /workspace') logger.info(action, extra={'msg_type': 'ACTION'}) obs = cast(CmdOutputObservation, await runtime.run_action(action)) logger.info(obs, extra={'msg_type': 'OBSERVATION'}) @@ -129,7 +129,7 @@ async def complete_runtime( logger.info('-' * 30) obs: CmdOutputObservation - action = CmdRunAction(command=f'cd /workspace') + action = CmdRunAction(command='cd /workspace') logger.info(action, extra={'msg_type': 'ACTION'}) obs = cast(CmdOutputObservation, await runtime.run_action(action)) logger.info(obs, extra={'msg_type': 'OBSERVATION'}) @@ -254,7 +254,6 @@ async def process_issue( config = AppConfig( default_agent="CodeActAgent", - run_as_devin=False, runtime='eventstream', max_budget_per_task=4, max_iterations=max_iterations, diff --git a/poetry.lock b/poetry.lock index bd090bc..c19a4bc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -370,18 +370,18 @@ files = [ [[package]] name = "boto3" -version = "1.35.4" +version = "1.35.5" description = "The AWS SDK for Python" category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.35.4-py3-none-any.whl", hash = "sha256:96c39593afb7b55ebb74d08c8e3201041d105b557c8c8536c9054c9f13da5f2a"}, - {file = "boto3-1.35.4.tar.gz", hash = "sha256:d997b82c468bd5c2d5cd29810d47079b66b178d2b5ae021aebe262c4d78d4c94"}, + {file = "boto3-1.35.5-py3-none-any.whl", hash = "sha256:2cef3aa476181395c260f4b6e6c5565e5a3022a874fb6b579d8e6b169f94e0b3"}, + {file = "boto3-1.35.5.tar.gz", hash = "sha256:5724ddeda8e18c7614c20a09c20159ed87ff7439755cf5e250a1a3feaf9afb7e"}, ] [package.dependencies] -botocore = ">=1.35.4,<1.36.0" +botocore = ">=1.35.5,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -390,14 +390,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.35.4" +version = "1.35.5" description = "Low-level, data-driven core of boto 3." category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.35.4-py3-none-any.whl", hash = "sha256:10195e5ca764745f02b9a51df048b996ddbdc1899a44a2caf35dfb225dfea489"}, - {file = "botocore-1.35.4.tar.gz", hash = "sha256:4cc51a6a486915aedc140f9d027b7e156646b7a0f7b33b1000762c81aff9a12f"}, + {file = "botocore-1.35.5-py3-none-any.whl", hash = "sha256:8116b72c7ae845c195146e437e2afd9d17538a37b3f3548dcf67c12c86ba0742"}, + {file = "botocore-1.35.5.tar.gz", hash = "sha256:3a0086c7124cb3b0d9f98563d00ffd14a942c3f9e731d8d1ccf0d3a1ac7ed884"}, ] [package.dependencies] @@ -1034,14 +1034,14 @@ torch = ["torch"] [[package]] name = "faker" -version = "27.4.0" +version = "28.0.0" description = "Faker is a Python package that generates fake data for you." category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "Faker-27.4.0-py3-none-any.whl", hash = "sha256:1c44d4bdcad7237516c9a829b6a0bcb031c6a4cb0506207c480c79f74d8922bf"}, - {file = "faker-27.4.0.tar.gz", hash = "sha256:4ce108fc96053bbba3abf848e3a2885f05faa938deb987f97e4420deaec541c4"}, + {file = "Faker-28.0.0-py3-none-any.whl", hash = "sha256:6a3a08be54c37e05f7943d7ba5211d252c1de737687a46ad6f29209d8d5db11f"}, + {file = "faker-28.0.0.tar.gz", hash = "sha256:0d3c0399204aaf8205cc1750db443474ca0436f177126b2c27b798e8336cc74f"}, ] [package.dependencies] @@ -3113,7 +3113,7 @@ zope-interface = "7.0.1" type = "git" url = "https://github.com/All-Hands-AI/openhands.git" reference = "fix-pypi-package" -resolved_reference = "a0c0234e1916567e528586d11ccdd5064cfa29d5" +resolved_reference = "2b55e9a4bc6b92002a9e358521a790b5808ee58e" [[package]] name = "packaging" diff --git a/tests/test_resolve_issues.py b/tests/test_resolve_issues.py index 89dea18..302a301 100644 --- a/tests/test_resolve_issues.py +++ b/tests/test_resolve_issues.py @@ -1,50 +1,71 @@ +import os +import tempfile import pytest -from unittest.mock import patch, MagicMock + +from unittest.mock import AsyncMock, patch, MagicMock from github_resolver.resolve_issues import ( create_git_patch, initialize_runtime, complete_runtime, get_instruction, + process_issue, ) from github_resolver.github_issue import GithubIssue from openhands.events.action import CmdRunAction -from openhands.events.observation import CmdOutputObservation +from openhands.events.observation import CmdOutputObservation, NullObservation +from github_resolver.resolver_output import ResolverOutput +from openhands.core.config import LLMConfig + + +@pytest.fixture +def mock_output_dir(): + with tempfile.TemporaryDirectory() as temp_dir: + repo_path = os.path.join(temp_dir, "repo") + # Initialize a GitHub repo in "repo" and add a commit with "README.md" + os.makedirs(repo_path) + os.system(f"git init {repo_path}") + readme_path = os.path.join(repo_path, "README.md") + with open(readme_path, "w") as f: + f.write("hello world") + os.system(f"git -C {repo_path} add README.md") + os.system(f"git -C {repo_path} commit -m 'Initial commit'") + yield temp_dir @pytest.fixture def mock_subprocess(): - with patch('subprocess.check_output') as mock_check_output: + with patch("subprocess.check_output") as mock_check_output: yield mock_check_output @pytest.fixture def mock_os(): - with patch('os.system') as mock_system, patch('os.path.join') as mock_join: + with patch("os.system") as mock_system, patch("os.path.join") as mock_join: yield mock_system, mock_join def test_create_git_patch(mock_subprocess, mock_os): - mock_subprocess.return_value = b'abcdef1234567890' + mock_subprocess.return_value = b"abcdef1234567890" mock_os[0].return_value = 0 - mock_os[1].return_value = '/path/to/workspace/123.patch' + mock_os[1].return_value = "/path/to/workspace/123.patch" - with patch('builtins.open', MagicMock()) as mock_open: + with patch("builtins.open", MagicMock()) as mock_open: mock_open.return_value.__enter__.return_value.read.return_value = ( - 'patch content' + "patch content" ) git_id, patch_content = create_git_patch( - '/path/to/workspace', 'main', 'fix', 123 + "/path/to/workspace", "main", "fix", 123 ) - assert git_id == 'abcdef1234567890' - assert patch_content == 'patch content' - mock_subprocess.assert_called_once_with(['git', 'rev-parse', 'main']) + assert git_id == "abcdef1234567890" + assert patch_content == "patch content" + mock_subprocess.assert_called_once_with(["git", "rev-parse", "main"]) mock_os[0].assert_called_once_with( - 'cd /path/to/workspace && git diff main fix > 123.patch' + "cd /path/to/workspace && git diff main fix > 123.patch" ) - mock_open.assert_called_once_with('/path/to/workspace/123.patch', 'r') + mock_open.assert_called_once_with("/path/to/workspace/123.patch", "r") async def create_cmd_output( @@ -60,11 +81,11 @@ async def test_initialize_runtime(): mock_runtime = MagicMock() mock_runtime.run_action.side_effect = [ create_cmd_output( - exit_code=0, content='', command_id=1, command='cd /workspace' + exit_code=0, content="", command_id=1, command="cd /workspace" ), create_cmd_output( exit_code=0, - content='', + content="", command_id=2, command='git config --global core.pager ""', ), @@ -73,7 +94,7 @@ async def test_initialize_runtime(): await initialize_runtime(mock_runtime) assert mock_runtime.run_action.call_count == 2 - mock_runtime.run_action.assert_any_call(CmdRunAction(command='cd /workspace')) + mock_runtime.run_action.assert_any_call(CmdRunAction(command="cd /workspace")) mock_runtime.run_action.assert_any_call( CmdRunAction(command='git config --global core.pager ""') ) @@ -84,50 +105,126 @@ async def test_complete_runtime(): mock_runtime = MagicMock() mock_runtime.run_action.side_effect = [ create_cmd_output( - exit_code=0, content='', command_id=1, command='cd /workspace' + exit_code=0, content="", command_id=1, command="cd /workspace" ), create_cmd_output( exit_code=0, - content='', + content="", command_id=2, command='git config --global core.pager ""', ), create_cmd_output( exit_code=0, - content='', + content="", command_id=3, - command='git diff base_commit_hash fix', + command="git diff base_commit_hash fix", ), create_cmd_output( - exit_code=0, content='git diff content', command_id=4, command='git apply' + exit_code=0, content="git diff content", command_id=4, command="git apply" ), ] - result = await complete_runtime(mock_runtime, 'base_commit_hash') + result = await complete_runtime(mock_runtime, "base_commit_hash") - assert result == {'git_patch': 'git diff content'} + assert result == {"git_patch": "git diff content"} assert mock_runtime.run_action.call_count == 4 +@pytest.mark.asyncio +async def test_process_issue(mock_output_dir): + # Mock dependencies + mock_create_runtime = AsyncMock() + mock_initialize_runtime = AsyncMock() + mock_run_controller = AsyncMock() + mock_complete_runtime = AsyncMock() + mock_guess_success = MagicMock() + + # Set up test data + issue = GithubIssue( + owner="test_owner", + repo="test_repo", + number=1, + title="Test Issue", + body="This is a test issue", + ) + base_commit = "abcdef1234567890" + max_iterations = 5 + llm_config = LLMConfig(model="test_model", api_key="test_api_key") + container_image = "test_image:latest" + + # Mock return values + mock_create_runtime.return_value = MagicMock() + mock_run_controller.return_value = MagicMock( + history=MagicMock( + get_events=MagicMock(return_value=[NullObservation(content="")]) + ), + metrics=MagicMock(get=MagicMock(return_value={"test_result": "passed"})), + last_error=None, + ) + mock_complete_runtime.return_value = {"git_patch": "test patch"} + mock_guess_success.return_value = (True, "Issue resolved successfully") + + # Patch the necessary functions + with patch( + "github_resolver.resolve_issues.create_runtime", mock_create_runtime + ), patch( + "github_resolver.resolve_issues.initialize_runtime", mock_initialize_runtime + ), patch( + "github_resolver.resolve_issues.run_controller", mock_run_controller + ), patch( + "github_resolver.resolve_issues.complete_runtime", mock_complete_runtime + ), patch( + "github_resolver.resolve_issues.guess_success", mock_guess_success + ), patch( + "github_resolver.resolve_issues.logger" + ): + + # Call the function + result = await process_issue( + issue, + base_commit, + max_iterations, + llm_config, + mock_output_dir, + container_image, + ) + + # Assert the result + assert isinstance(result, ResolverOutput) + assert result.issue == issue + assert result.base_commit == base_commit + assert result.git_patch == "test patch" + assert result.success + assert result.success_explanation == "Issue resolved successfully" + assert result.error is None + + # Assert that the mocked functions were called + mock_create_runtime.assert_called_once() + mock_initialize_runtime.assert_called_once() + mock_run_controller.assert_called_once() + mock_complete_runtime.assert_called_once() + mock_guess_success.assert_called_once() + + def test_get_instruction(): issue = GithubIssue( - owner='test_owner', - repo='test_repo', + owner="test_owner", + repo="test_repo", number=123, - title='Test Issue', - body='This is a test issue', + title="Test Issue", + body="This is a test issue", ) instruction = get_instruction(issue) assert ( - 'Please fix the following issue for the repository in /workspace.' + "Please fix the following issue for the repository in /workspace." in instruction ) - assert 'This is a test issue' in instruction + assert "This is a test issue" in instruction assert ( - 'You should ONLY interact with the environment provided to you' in instruction + "You should ONLY interact with the environment provided to you" in instruction ) -if __name__ == '__main__': +if __name__ == "__main__": pytest.main()