From ec1851ec84ed2147c3b28c99a928600f18a27bab Mon Sep 17 00:00:00 2001 From: Talion Date: Tue, 18 Jun 2024 23:05:11 -0400 Subject: [PATCH 01/10] create an initial config.toml --- config.toml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 config.toml diff --git a/config.toml b/config.toml new file mode 100644 index 0000000000..c001d2340d --- /dev/null +++ b/config.toml @@ -0,0 +1,34 @@ +# config.toml +# Unified configuration file for the gpt-engineer project + +# API Configuration +[api] +# API key for OpenAPI +# OPENAI_API_KEY=Your personal OpenAI API key from https://platform.openai.com/account/api-keys +OPENAI_API_KEY = "your_api_key_here" +ANTHROPIC_API_KEY = "your_anthropic_api_key_here" +# Model configurations +[model] +model_name = "gpt-4o" +# Controls randomness: lower values for more focused, deterministic outputs +temperature = 0.1 +# Endpoint for your Azure OpenAI Service (https://xx.openai.azure.com). +# In that case, the given model is the deployment name chosen in the Azure AI Studio. +azure_endpoint = "" + +# improve mode Configuration +[imporve] +# Enable or disable linting (true/false) +is_linting = true +# Enable or disable file selection. "true" will open your default editor to select the file. (true/false) +is_file_selection = true + +# Git Filter Configuration +[git_filter] +# File extension settings for the git filter +file_extensions = ["py", "toml", "md"] + +# Self-Healing Mechanism Configuration +[self_healing] +# Number of retry attempts for self-healing mechanisms (0-2) +retry_attempts = 1 From 030f9016ecf44ca48737f218026bb807344be663 Mon Sep 17 00:00:00 2001 From: Talion Date: Wed, 19 Jun 2024 22:03:29 -0400 Subject: [PATCH 02/10] updates on config parsing file --- gpt_engineer/applications/cli/main.py | 6 +++ gpt_engineer/core/project_config.py | 57 +++++++++++---------------- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/gpt_engineer/applications/cli/main.py b/gpt_engineer/applications/cli/main.py index 7f93a07f23..e5b9f07ec6 100644 --- a/gpt_engineer/applications/cli/main.py +++ b/gpt_engineer/applications/cli/main.py @@ -57,6 +57,7 @@ from gpt_engineer.core.files_dict import FilesDict from gpt_engineer.core.git import stage_uncommitted_to_git from gpt_engineer.core.preprompts_holder import PrepromptsHolder +from gpt_engineer.core.project_config import Config from gpt_engineer.core.prompt import Prompt from gpt_engineer.tools.custom_steps import clarified_gen, lite_gen, self_heal @@ -410,6 +411,11 @@ def main( path = Path(project_path) print("Running gpt-engineer in", path.absolute(), "\n") + # read the configuration file from the root directory + config = Config("config.toml") + + # todo: apply configuration here + prompt = load_prompt( DiskMemory(path), improve_mode, diff --git a/gpt_engineer/core/project_config.py b/gpt_engineer/core/project_config.py index 137a5558c8..8bf5fd94e9 100644 --- a/gpt_engineer/core/project_config.py +++ b/gpt_engineer/core/project_config.py @@ -8,7 +8,7 @@ import tomlkit -default_config_filename = "gpt-engineer.toml" +default_config_filename = "config.toml" example_config = """ [run] @@ -38,22 +38,22 @@ class _PathsConfig: @dataclass -class _RunConfig: - build: str | None = None - test: str | None = None - lint: str | None = None - format: str | None = None +class _ApiConfig: + OPENAI_API_KEY: str | None = None + ANTHROPIC_API_KEY: str | None = None @dataclass -class _OpenApiConfig: - url: str +class _ModelConfig: + model_name: str | None = None + temperature: float | None = None + azure_endpoint: str | None = None @dataclass -class _GptEngineerAppConfig: - project_id: str - openapi: list[_OpenApiConfig] | None = None +class _ImproveConfig: + is_linting: bool | None = None + is_file_selection: bool | None = None def filter_none(d: dict) -> dict: @@ -74,8 +74,9 @@ class Config: """Configuration for the GPT Engineer CLI and gptengineer.app via `gpt-engineer.toml`.""" paths: _PathsConfig = field(default_factory=_PathsConfig) - run: _RunConfig = field(default_factory=_RunConfig) - gptengineer_app: _GptEngineerAppConfig | None = None + api_config: _ApiConfig = field(default_factory=_ApiConfig) + model_config: _ModelConfig = field(default_factory=_ModelConfig) + improve_config: _ImproveConfig = field(default_factory=_ImproveConfig) @classmethod def from_toml(cls, config_file: Path | str): @@ -86,27 +87,17 @@ def from_toml(cls, config_file: Path | str): @classmethod def from_dict(cls, config_dict: dict): - run = _RunConfig(**config_dict.get("run", {})) paths = _PathsConfig(**config_dict.get("paths", {})) - - # load optional gptengineer-app section - gptengineer_app_dict = config_dict.get("gptengineer-app", {}) - gptengineer_app = None - if gptengineer_app_dict: - assert ( - "project_id" in gptengineer_app_dict - ), "project_id is required in gptengineer-app section" - gptengineer_app = _GptEngineerAppConfig( - # required if gptengineer-app section is present - project_id=gptengineer_app_dict["project_id"], - openapi=[ - _OpenApiConfig(**openapi) - for openapi in gptengineer_app_dict.get("openapi", []) - ] - or None, - ) - - return cls(paths=paths, run=run, gptengineer_app=gptengineer_app) + api_config = _ApiConfig(**config_dict.get("api_config", {})) + model_config = _ModelConfig(**config_dict.get("model_config", {})) + improve_config = _ImproveConfig(**config_dict.get("improve_config", {})) + + return cls( + paths=paths, + api_config=api_config, + model_config=model_config, + improve_config=improve_config, + ) def to_dict(self) -> dict: d = asdict(self) From 34f4fab7bb6274c1fced263420aa6729f45bfe94 Mon Sep 17 00:00:00 2001 From: Talion Date: Sun, 23 Jun 2024 12:20:08 -0400 Subject: [PATCH 03/10] access config.toml in main.py;fix typo; --- config.toml | 2 +- gpt_engineer/applications/cli/main.py | 6 +++--- gpt_engineer/core/project_config.py | 25 ++++++++++++++++++++----- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/config.toml b/config.toml index c001d2340d..747c8a26e8 100644 --- a/config.toml +++ b/config.toml @@ -17,7 +17,7 @@ temperature = 0.1 azure_endpoint = "" # improve mode Configuration -[imporve] +[improve] # Enable or disable linting (true/false) is_linting = true # Enable or disable file selection. "true" will open your default editor to select the file. (true/false) diff --git a/gpt_engineer/applications/cli/main.py b/gpt_engineer/applications/cli/main.py index e5b9f07ec6..0e95c73249 100644 --- a/gpt_engineer/applications/cli/main.py +++ b/gpt_engineer/applications/cli/main.py @@ -251,7 +251,7 @@ def prompt_yesno() -> bool: def main( project_path: str = typer.Argument(".", help="path"), model: str = typer.Option( - os.environ.get("MODEL_NAME", "gpt-4o"), "--model", "-m", help="model id string" + os.environ.get("MODEL_NAME", "gpt-4"), "--model", "-m", help="model id string" ), temperature: float = typer.Option( 0.1, @@ -412,8 +412,8 @@ def main( print("Running gpt-engineer in", path.absolute(), "\n") # read the configuration file from the root directory - config = Config("config.toml") - + config = Config() + config.from_toml(Path(os.getcwd()) / "config.toml").to_dict() # todo: apply configuration here prompt = load_prompt( diff --git a/gpt_engineer/core/project_config.py b/gpt_engineer/core/project_config.py index 8bf5fd94e9..961a761bde 100644 --- a/gpt_engineer/core/project_config.py +++ b/gpt_engineer/core/project_config.py @@ -87,10 +87,23 @@ def from_toml(cls, config_file: Path | str): @classmethod def from_dict(cls, config_dict: dict): - paths = _PathsConfig(**config_dict.get("paths", {})) - api_config = _ApiConfig(**config_dict.get("api_config", {})) - model_config = _ModelConfig(**config_dict.get("model_config", {})) - improve_config = _ImproveConfig(**config_dict.get("improve_config", {})) + paths = _PathsConfig(**config_dict.get("paths", {"base": None, "src": None})) + api_config = _ApiConfig( + **config_dict.get( + "api", {"OPENAI_API_KEY": None, "ANTHROPIC_API_KEY": None} + ) + ) + model_config = _ModelConfig( + **config_dict.get( + "model", + {"model_name": None, "temperature": None, "azure_endpoint": None}, + ) + ) + improve_config = _ImproveConfig( + **config_dict.get( + "improve", {"is_linting": None, "is_file_selection": None} + ) + ) return cls( paths=paths, @@ -101,7 +114,9 @@ def from_dict(cls, config_dict: dict): def to_dict(self) -> dict: d = asdict(self) - d["gptengineer-app"] = d.pop("gptengineer_app", None) + d["api"] = d.pop("api_config", None) + d["model"] = d.pop("model_config", None) + d["improve"] = d.pop("improve_config", None) # Drop None values and empty dictionaries # Needed because tomlkit.dumps() doesn't handle None values, From 3b358d8a5e9568a653b9f27ea034e11f47a421c0 Mon Sep 17 00:00:00 2001 From: Talion Date: Sun, 30 Jun 2024 00:45:15 -0400 Subject: [PATCH 04/10] extract linting option to config --- config.toml | 6 ++-- .../applications/cli/file_selector.py | 31 ++----------------- gpt_engineer/applications/cli/main.py | 15 +++++++-- 3 files changed, 18 insertions(+), 34 deletions(-) diff --git a/config.toml b/config.toml index 747c8a26e8..0bca49ba49 100644 --- a/config.toml +++ b/config.toml @@ -2,11 +2,12 @@ # Unified configuration file for the gpt-engineer project # API Configuration -[api] +[API] # API key for OpenAPI # OPENAI_API_KEY=Your personal OpenAI API key from https://platform.openai.com/account/api-keys OPENAI_API_KEY = "your_api_key_here" ANTHROPIC_API_KEY = "your_anthropic_api_key_here" + # Model configurations [model] model_name = "gpt-4o" @@ -18,8 +19,9 @@ azure_endpoint = "" # improve mode Configuration [improve] +# Linting with BLACK (Python) enhances code suggestions from LLMs. # Enable or disable linting (true/false) -is_linting = true +is_linting = false # Enable or disable file selection. "true" will open your default editor to select the file. (true/false) is_file_selection = true diff --git a/gpt_engineer/applications/cli/file_selector.py b/gpt_engineer/applications/cli/file_selector.py index a80608620c..29ed1d2874 100644 --- a/gpt_engineer/applications/cli/file_selector.py +++ b/gpt_engineer/applications/cli/file_selector.py @@ -53,16 +53,11 @@ class FileSelector: IGNORE_FOLDERS = {"site-packages", "node_modules", "venv", "__pycache__"} FILE_LIST_NAME = "file_selection.toml" COMMENT = ( - "# Remove '#' to select a file or turn off linting.\n\n" - "# Linting with BLACK (Python) enhances code suggestions from LLMs. " - "To disable linting, uncomment the relevant option in the linting settings.\n\n" + "# Remove '#' to select a file\n\n" "# gpt-engineer can only read selected files. " "Including irrelevant files will degrade performance, " "cost additional tokens and potentially overflow token limit.\n\n" ) - LINTING_STRING = '[linting]\n# "linting" = "off"\n\n' - is_linting = True - def __init__(self, project_path: Union[str, Path]): """ Initializes the FileSelector with a given project path. @@ -117,7 +112,7 @@ def ask_for_files(self) -> tuple[FilesDict, bool]: except UnicodeDecodeError: print(f"Warning: File not UTF-8 encoded {file_path}, skipping") - return FilesDict(content_dict), self.is_linting + return FilesDict(content_dict) def editor_file_selector( self, input_path: Union[str, Path], init: bool = True @@ -159,7 +154,6 @@ def editor_file_selector( # Write to the toml file with open(toml_file, "w") as f: f.write(self.COMMENT) - f.write(self.LINTING_STRING) f.write(s) else: @@ -167,16 +161,6 @@ def editor_file_selector( all_files = self.get_current_files(root_path) s = toml.dumps({"files": {x: "selected" for x in all_files}}) - # get linting status from the toml file - with open(toml_file, "r") as file: - linting_status = toml.load(file) - if ( - "linting" in linting_status - and linting_status["linting"].get("linting", "").lower() == "off" - ): - self.is_linting = False - self.LINTING_STRING = '[linting]\n"linting" = "off"\n\n' - print("\nLinting is disabled") with open(toml_file, "r") as file: selected_files = toml.load(file) @@ -195,7 +179,6 @@ def editor_file_selector( # Write the merged list back to the .toml for user review and modification with open(toml_file, "w") as file: file.write(self.COMMENT) # Ensure to write the comment - file.write(self.LINTING_STRING) file.write(s) print( @@ -293,16 +276,6 @@ def get_files_from_toml( selected_files = [] edited_tree = toml.load(toml_file) # Load the edited .toml file - # check if users have disabled linting or not - if ( - "linting" in edited_tree - and edited_tree["linting"].get("linting", "").lower() == "off" - ): - self.is_linting = False - print("\nLinting is disabled") - else: - self.is_linting = True - # Iterate through the files in the .toml and append selected files to the list for file, _ in edited_tree["files"].items(): selected_files.append(file) diff --git a/gpt_engineer/applications/cli/main.py b/gpt_engineer/applications/cli/main.py index 0e95c73249..5644fe9bae 100644 --- a/gpt_engineer/applications/cli/main.py +++ b/gpt_engineer/applications/cli/main.py @@ -411,9 +411,18 @@ def main( path = Path(project_path) print("Running gpt-engineer in", path.absolute(), "\n") + # ask if the user wants to change the configuration + print("The configuration file(config.toml) is located in the root directory. You can edit it with your preferred " + "text editor.") + # todo: interface to edit the configuration + # read the configuration file from the root directory config = Config() - config.from_toml(Path(os.getcwd()) / "config.toml").to_dict() + config_dict = config.from_toml(Path(os.getcwd()) / "config.toml").to_dict() + + + + # todo: apply configuration here prompt = load_prompt( @@ -463,10 +472,10 @@ def main( files = FileStore(project_path) if not no_execution: if improve_mode: - files_dict_before, is_linting = FileSelector(project_path).ask_for_files() + files_dict_before = FileSelector(project_path).ask_for_files() # lint the code - if is_linting: + if config_dict["improve"]["is_linting"]: files_dict_before = files.linting(files_dict_before) files_dict = handle_improve_mode(prompt, agent, memory, files_dict_before) From 3a83be590fd8d3e9de1185e9ce4e7c9aa14eaa88 Mon Sep 17 00:00:00 2001 From: Talion Date: Sun, 30 Jun 2024 00:45:37 -0400 Subject: [PATCH 05/10] fix the tests for config --- gpt_engineer/core/project_config.py | 48 ++++++++----------- tests/test_project_config.py | 71 ++++++++--------------------- 2 files changed, 37 insertions(+), 82 deletions(-) diff --git a/gpt_engineer/core/project_config.py b/gpt_engineer/core/project_config.py index 961a761bde..84a9295820 100644 --- a/gpt_engineer/core/project_config.py +++ b/gpt_engineer/core/project_config.py @@ -11,32 +11,23 @@ default_config_filename = "config.toml" example_config = """ -[run] -build = "npm run build" -test = "npm run test" -lint = "quick-lint-js" - -[paths] -base = "./frontend" # base directory to operate in (for monorepos) -src = "./src" # source directory (under the base directory) from which context will be retrieved - -[gptengineer-app] # this namespace is used for gptengineer.app, may be used for internal experiments -project_id = "..." - -# we support multiple OpenAPI schemas, used as context for the LLM -openapi = [ - { url = "https://api.gptengineer.app/openapi.json" }, - { url = "https://some-color-translating-api/openapi.json" }, -] +# API Configuration +[API] +OPENAI_API_KEY = "..." +ANTHROPIC_API_KEY = "..." + +# Model configurations +[model] +model_name = "gpt-4o" +temperature = 0.1 +azure_endpoint = "" + +# improve mode Configuration +[improve] +is_linting = false +is_file_selection = true """ - -@dataclass -class _PathsConfig: - base: str | None = None - src: str | None = None - - @dataclass class _ApiConfig: OPENAI_API_KEY: str | None = None @@ -71,9 +62,8 @@ def filter_none(d: dict) -> dict: @dataclass class Config: - """Configuration for the GPT Engineer CLI and gptengineer.app via `gpt-engineer.toml`.""" + """Configuration for the GPT Engineer project""" - paths: _PathsConfig = field(default_factory=_PathsConfig) api_config: _ApiConfig = field(default_factory=_ApiConfig) model_config: _ModelConfig = field(default_factory=_ModelConfig) improve_config: _ImproveConfig = field(default_factory=_ImproveConfig) @@ -87,10 +77,9 @@ def from_toml(cls, config_file: Path | str): @classmethod def from_dict(cls, config_dict: dict): - paths = _PathsConfig(**config_dict.get("paths", {"base": None, "src": None})) api_config = _ApiConfig( **config_dict.get( - "api", {"OPENAI_API_KEY": None, "ANTHROPIC_API_KEY": None} + "API", {"OPENAI_API_KEY": None, "ANTHROPIC_API_KEY": None} ) ) model_config = _ModelConfig( @@ -106,7 +95,6 @@ def from_dict(cls, config_dict: dict): ) return cls( - paths=paths, api_config=api_config, model_config=model_config, improve_config=improve_config, @@ -114,7 +102,7 @@ def from_dict(cls, config_dict: dict): def to_dict(self) -> dict: d = asdict(self) - d["api"] = d.pop("api_config", None) + d["API"] = d.pop("api_config", None) d["model"] = d.pop("model_config", None) d["improve"] = d.pop("improve_config", None) diff --git a/tests/test_project_config.py b/tests/test_project_config.py index 8aab8a2e7e..d7d5304798 100644 --- a/tests/test_project_config.py +++ b/tests/test_project_config.py @@ -4,10 +4,8 @@ from gpt_engineer.core.project_config import ( Config, - _GptEngineerAppConfig, - _OpenApiConfig, example_config, - filter_none, + filter_none, _ImproveConfig, ) @@ -19,22 +17,12 @@ def test_config_load(): # load the config from the file config = Config.from_toml(f.name) - assert config.paths.base == "./frontend" - assert config.paths.src == "./src" - assert config.run.build == "npm run build" - assert config.run.test == "npm run test" - assert config.run.lint == "quick-lint-js" - assert config.gptengineer_app - assert config.gptengineer_app.project_id == "..." - assert config.gptengineer_app.openapi - assert ( - config.gptengineer_app.openapi[0].url - == "https://api.gptengineer.app/openapi.json" - ) - assert ( - config.gptengineer_app.openapi[1].url - == "https://some-color-translating-api/openapi.json" - ) + assert config.api_config.OPENAI_API_KEY == "..." + assert config.api_config.ANTHROPIC_API_KEY == "..." + assert config.model_config.model_name == "gpt-4o" + assert config.model_config.temperature == 0.1 + assert config.improve_config.is_linting is False + assert config.improve_config.is_file_selection is True assert config.to_dict() assert config.to_toml(f.name, save=False) @@ -44,7 +32,6 @@ def test_config_load(): def test_config_defaults(): config = Config() - assert config.paths.base is None with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: config.to_toml(f.name) @@ -57,48 +44,28 @@ def test_config_defaults(): def test_config_from_dict(): - d = {"gptengineer-app": {"project_id": "..."}} # minimal example + d = {"improve": {"is_linting": "..."}} # minimal example config = Config.from_dict(d) - assert config.gptengineer_app - assert config.gptengineer_app.project_id == "..." + assert config.improve_config + assert config.improve_config.is_linting == "..." config_dict = config.to_dict() # check that the config dict matches the input dict exactly (no keys/defaults added) assert config_dict == d -def test_config_from_dict_with_openapi(): - # A good test because it has 3 levels of nesting - d = { - "gptengineer-app": { - "project_id": "...", - "openapi": [ - {"url": "https://api.gptengineer.app/openapi.json"}, - ], - } - } - config = Config.from_dict(d) - assert config.gptengineer_app - assert config.gptengineer_app.project_id == "..." - assert config.gptengineer_app.openapi - assert ( - config.gptengineer_app.openapi[0].url - == "https://api.gptengineer.app/openapi.json" - ) - - def test_config_load_partial(): # Loads a partial config, and checks that the rest is not set (i.e. None) example_config = """ -[gptengineer-app] -project_id = "..." +[improve] +is_linting = "..." """.strip() with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: f.write(example_config) config = Config.from_toml(f.name) - assert config.gptengineer_app - assert config.gptengineer_app.project_id == "..." + assert config.improve_config + assert config.improve_config.is_linting == "..." assert config.to_dict() toml_str = config.to_toml(f.name, save=False) assert toml_str == example_config @@ -109,15 +76,15 @@ def test_config_load_partial(): def test_config_update(): example_config = """ -[gptengineer-app] -project_id = "..." +[improve] +is_linting = "..." """.strip() with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: f.write(example_config) config = Config.from_toml(f.name) - config.gptengineer_app = _GptEngineerAppConfig( - project_id="...", - openapi=[_OpenApiConfig(url="https://api.gptengineer.app/openapi.json")], + config.improve_config = _ImproveConfig( + is_linting=False, + is_file_selection=True ) config.to_toml(f.name) assert Config.from_toml(f.name) == config From e7b1f4da7600aa62341860e9998589f7264abc39 Mon Sep 17 00:00:00 2001 From: Talion Date: Sun, 30 Jun 2024 00:53:02 -0400 Subject: [PATCH 06/10] fix pre-commit --- gpt_engineer/applications/cli/file_selector.py | 2 +- gpt_engineer/applications/cli/main.py | 9 ++++----- gpt_engineer/core/project_config.py | 1 + tests/test_project_config.py | 8 +++----- 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/gpt_engineer/applications/cli/file_selector.py b/gpt_engineer/applications/cli/file_selector.py index 29ed1d2874..8e77d0f916 100644 --- a/gpt_engineer/applications/cli/file_selector.py +++ b/gpt_engineer/applications/cli/file_selector.py @@ -58,6 +58,7 @@ class FileSelector: "Including irrelevant files will degrade performance, " "cost additional tokens and potentially overflow token limit.\n\n" ) + def __init__(self, project_path: Union[str, Path]): """ Initializes the FileSelector with a given project path. @@ -161,7 +162,6 @@ def editor_file_selector( all_files = self.get_current_files(root_path) s = toml.dumps({"files": {x: "selected" for x in all_files}}) - with open(toml_file, "r") as file: selected_files = toml.load(file) diff --git a/gpt_engineer/applications/cli/main.py b/gpt_engineer/applications/cli/main.py index 5644fe9bae..4447b75285 100644 --- a/gpt_engineer/applications/cli/main.py +++ b/gpt_engineer/applications/cli/main.py @@ -412,17 +412,16 @@ def main( print("Running gpt-engineer in", path.absolute(), "\n") # ask if the user wants to change the configuration - print("The configuration file(config.toml) is located in the root directory. You can edit it with your preferred " - "text editor.") + print( + "The configuration file(config.toml) is located in the root directory. You can edit it with your preferred " + "text editor." + ) # todo: interface to edit the configuration # read the configuration file from the root directory config = Config() config_dict = config.from_toml(Path(os.getcwd()) / "config.toml").to_dict() - - - # todo: apply configuration here prompt = load_prompt( diff --git a/gpt_engineer/core/project_config.py b/gpt_engineer/core/project_config.py index 84a9295820..61e27fc5cf 100644 --- a/gpt_engineer/core/project_config.py +++ b/gpt_engineer/core/project_config.py @@ -28,6 +28,7 @@ is_file_selection = true """ + @dataclass class _ApiConfig: OPENAI_API_KEY: str | None = None diff --git a/tests/test_project_config.py b/tests/test_project_config.py index d7d5304798..5021100569 100644 --- a/tests/test_project_config.py +++ b/tests/test_project_config.py @@ -4,8 +4,9 @@ from gpt_engineer.core.project_config import ( Config, + _ImproveConfig, example_config, - filter_none, _ImproveConfig, + filter_none, ) @@ -82,10 +83,7 @@ def test_config_update(): with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: f.write(example_config) config = Config.from_toml(f.name) - config.improve_config = _ImproveConfig( - is_linting=False, - is_file_selection=True - ) + config.improve_config = _ImproveConfig(is_linting=False, is_file_selection=True) config.to_toml(f.name) assert Config.from_toml(f.name) == config From ce067d5401ca77cf06ba45f2147a72cbd652d7e4 Mon Sep 17 00:00:00 2001 From: Talion Date: Sat, 3 Aug 2024 19:11:39 -0400 Subject: [PATCH 07/10] temp save --- gpt_engineer/applications/cli/main.py | 139 ++++++++++++++++---------- 1 file changed, 85 insertions(+), 54 deletions(-) diff --git a/gpt_engineer/applications/cli/main.py b/gpt_engineer/applications/cli/main.py index 4447b75285..d6d1606f2e 100644 --- a/gpt_engineer/applications/cli/main.py +++ b/gpt_engineer/applications/cli/main.py @@ -31,6 +31,7 @@ import sys from pathlib import Path +from typing import Optional import openai import typer @@ -250,85 +251,77 @@ def prompt_yesno() -> bool: ) def main( project_path: str = typer.Argument(".", help="path"), - model: str = typer.Option( - os.environ.get("MODEL_NAME", "gpt-4"), "--model", "-m", help="model id string" - ), - temperature: float = typer.Option( - 0.1, + model: Optional[str] = typer.Option(None, "--model", "-m", help="model id string"), + temperature: Optional[float] = typer.Option( + None, "--temperature", "-t", help="Controls randomness: lower values for more focused, deterministic outputs", ), - improve_mode: bool = typer.Option( - False, + improve_mode: Optional[bool] = typer.Option( + None, "--improve", "-i", help="Improve an existing project by modifying the files.", ), - lite_mode: bool = typer.Option( - False, + lite_mode: Optional[bool] = typer.Option( + None, "--lite", "-l", help="Lite mode: run a generation using only the main prompt.", ), - clarify_mode: bool = typer.Option( - False, + clarify_mode: Optional[bool] = typer.Option( + None, "--clarify", "-c", help="Clarify mode - discuss specification with AI before implementation.", ), - self_heal_mode: bool = typer.Option( - False, + self_heal_mode: Optional[bool] = typer.Option( + None, "--self-heal", "-sh", help="Self-heal mode - fix the code by itself when it fails.", ), - azure_endpoint: str = typer.Option( - "", + azure_endpoint: Optional[str] = typer.Option( + None, "--azure", "-a", - help="""Endpoint for your Azure OpenAI Service (https://xx.openai.azure.com). - In that case, the given model is the deployment name chosen in the Azure AI Studio.""", + help="Endpoint for your Azure OpenAI Service (https://xx.openai.azure.com). In that case, the given model is the deployment name chosen in the Azure AI Studio.", ), - use_custom_preprompts: bool = typer.Option( - False, + use_custom_preprompts: Optional[bool] = typer.Option( + None, "--use-custom-preprompts", - help="""Use your project's custom preprompts instead of the default ones. - Copies all original preprompts to the project's workspace if they don't exist there.""", + help="Use your project's custom preprompts instead of the default ones. Copies all original preprompts to the project's workspace if they don't exist there.", ), - llm_via_clipboard: bool = typer.Option( - False, + llm_via_clipboard: Optional[bool] = typer.Option( + None, "--llm-via-clipboard", help="Use the clipboard to communicate with the AI.", ), - verbose: bool = typer.Option( - False, "--verbose", "-v", help="Enable verbose logging for debugging." + verbose: Optional[bool] = typer.Option( + None, "--verbose", "-v", help="Enable verbose logging for debugging." ), - debug: bool = typer.Option( - False, "--debug", "-d", help="Enable debug mode for debugging." + debug: Optional[bool] = typer.Option( + None, "--debug", "-d", help="Enable debug mode for debugging." ), - prompt_file: str = typer.Option( - "prompt", - "--prompt_file", - help="Relative path to a text file containing a prompt.", + prompt_file: Optional[str] = typer.Option( + None, "--prompt_file", help="Relative path to a text file containing a prompt." ), - entrypoint_prompt_file: str = typer.Option( - "", + entrypoint_prompt_file: Optional[str] = typer.Option( + None, "--entrypoint_prompt", help="Relative path to a text file containing a file that specifies requirements for you entrypoint.", ), - image_directory: str = typer.Option( - "", - "--image_directory", - help="Relative path to a folder containing images.", + image_directory: Optional[str] = typer.Option( + None, "--image_directory", help="Relative path to a folder containing images." ), - use_cache: bool = typer.Option( - False, + use_cache: Optional[bool] = typer.Option( + None, "--use_cache", help="Speeds up computations and saves tokens when running the same prompt multiple times by caching the LLM response.", ), - no_execution: bool = typer.Option( - False, + no_execution: Optional[bool] = typer.Option( + None, "--no_execution", help="Run setup but to not call LLM or write any code. For testing purposes.", ), @@ -378,6 +371,57 @@ def main( None """ + # ask if the user wants to change the configuration + print( + "The configuration file(config.toml) is located in the root directory. You can edit it with your preferred " + "text editor." + ) + # todo: interface to edit the configuration + + # read the configuration file from the root directory + config = Config() + config_dict = config.from_toml(Path(os.getcwd()) / "config.toml").to_dict() + + # todo: apply configuration here + + # Override with CLI options if provided + model = model or config_dict["model"]["model_name"] + temperature = ( + temperature if temperature is not None else config_dict["model"]["temperature"] + ) + improve_mode = ( + improve_mode + if improve_mode is not None + else config.get("improve", {}).get("is_file_selection", False) + ) + lite_mode = ( + lite_mode + if lite_mode is not None + else config.get("improve", {}).get("is_linting", False) + ) + clarify_mode = clarify_mode or False # not provided in the config, default to False + self_heal_mode = ( + self_heal_mode or False + ) # not provided in the config, default to False + azure_endpoint = azure_endpoint or config.get("model", {}).get("azure_endpoint", "") + use_custom_preprompts = ( + use_custom_preprompts or False + ) # not provided in the config, default to False + llm_via_clipboard = ( + llm_via_clipboard or False + ) # not provided in the config, default to False + verbose = verbose or False # not provided in the config, default to False + debug = debug or False # not provided in the config, default to False + prompt_file = ( + prompt_file or "prompt" + ) # not provided in the config, default to 'prompt' + entrypoint_prompt_file = ( + entrypoint_prompt_file or "" + ) # not provided in the config, default to '' + image_directory = image_directory or "" # not provided in the config, default to '' + use_cache = use_cache or False # not provided in the config, default to False + no_execution = no_execution or False # not provided in the config, default to False + if debug: import pdb @@ -411,19 +455,6 @@ def main( path = Path(project_path) print("Running gpt-engineer in", path.absolute(), "\n") - # ask if the user wants to change the configuration - print( - "The configuration file(config.toml) is located in the root directory. You can edit it with your preferred " - "text editor." - ) - # todo: interface to edit the configuration - - # read the configuration file from the root directory - config = Config() - config_dict = config.from_toml(Path(os.getcwd()) / "config.toml").to_dict() - - # todo: apply configuration here - prompt = load_prompt( DiskMemory(path), improve_mode, From fb83fe9baae889803f305e0f3d783e8034551e15 Mon Sep 17 00:00:00 2001 From: Talion Date: Thu, 8 Aug 2024 20:30:25 -0400 Subject: [PATCH 08/10] add more parts to config parser; load config to main scope --- gpt_engineer/applications/cli/main.py | 58 +++++++------ gpt_engineer/core/project_config.py | 118 ++++++++++++-------------- 2 files changed, 85 insertions(+), 91 deletions(-) diff --git a/gpt_engineer/applications/cli/main.py b/gpt_engineer/applications/cli/main.py index d6d1606f2e..fbe65653d6 100644 --- a/gpt_engineer/applications/cli/main.py +++ b/gpt_engineer/applications/cli/main.py @@ -384,43 +384,49 @@ def main( # todo: apply configuration here - # Override with CLI options if provided + # Loading the configuration from the config_dict + model = model or config_dict["model"]["model_name"] temperature = ( temperature if temperature is not None else config_dict["model"]["temperature"] ) + azure_endpoint = azure_endpoint or config_dict["model"]["azure_endpoint"] + + # Improve mode configuration improve_mode = ( improve_mode if improve_mode is not None - else config.get("improve", {}).get("is_file_selection", False) + else config_dict["improve"]["is_file_selection"] ) lite_mode = ( - lite_mode - if lite_mode is not None - else config.get("improve", {}).get("is_linting", False) + lite_mode if lite_mode is not None else config_dict["improve"]["is_linting"] ) - clarify_mode = clarify_mode or False # not provided in the config, default to False + + # Self-healing mechanism configuration self_heal_mode = ( - self_heal_mode or False - ) # not provided in the config, default to False - azure_endpoint = azure_endpoint or config.get("model", {}).get("azure_endpoint", "") - use_custom_preprompts = ( - use_custom_preprompts or False - ) # not provided in the config, default to False - llm_via_clipboard = ( - llm_via_clipboard or False - ) # not provided in the config, default to False - verbose = verbose or False # not provided in the config, default to False - debug = debug or False # not provided in the config, default to False - prompt_file = ( - prompt_file or "prompt" - ) # not provided in the config, default to 'prompt' - entrypoint_prompt_file = ( - entrypoint_prompt_file or "" - ) # not provided in the config, default to '' - image_directory = image_directory or "" # not provided in the config, default to '' - use_cache = use_cache or False # not provided in the config, default to False - no_execution = no_execution or False # not provided in the config, default to False + self_heal_mode + if self_heal_mode is not None + else config_dict["self_healing"]["retry_attempts"] + ) + + # Git filter configuration + config_dict["git_filter"]["file_extensions"] # Assuming this is needed somewhere + + # API keys + config_dict["API"]["OPENAI_API_KEY"] + config_dict["API"]["ANTHROPIC_API_KEY"] + + # Default values for optional parameters + clarify_mode = clarify_mode or False + use_custom_preprompts = use_custom_preprompts or False + llm_via_clipboard = llm_via_clipboard or False + verbose = verbose or False + debug = debug or False + prompt_file = prompt_file or "prompt" + entrypoint_prompt_file = entrypoint_prompt_file or "" + image_directory = image_directory or "" + use_cache = use_cache or False + no_execution = no_execution or False if debug: import pdb diff --git a/gpt_engineer/core/project_config.py b/gpt_engineer/core/project_config.py index 61e27fc5cf..4488d6503c 100644 --- a/gpt_engineer/core/project_config.py +++ b/gpt_engineer/core/project_config.py @@ -1,10 +1,6 @@ -""" -Functions for reading and writing the `gpt-engineer.toml` configuration file. - -The `gpt-engineer.toml` file is a TOML file that contains project-specific configuration used by the GPT Engineer CLI and gptengineer.app. -""" -from dataclasses import asdict, dataclass, field +from dataclasses import dataclass, field from pathlib import Path +from typing import Any, Dict import tomlkit @@ -26,48 +22,29 @@ [improve] is_linting = false is_file_selection = true -""" - - -@dataclass -class _ApiConfig: - OPENAI_API_KEY: str | None = None - ANTHROPIC_API_KEY: str | None = None - - -@dataclass -class _ModelConfig: - model_name: str | None = None - temperature: float | None = None - azure_endpoint: str | None = None +# Git Filter Configuration +[git_filter] +file_extensions = ["py", "toml", "md"] -@dataclass -class _ImproveConfig: - is_linting: bool | None = None - is_file_selection: bool | None = None - - -def filter_none(d: dict) -> dict: - # Drop None values and empty dictionaries from a dictionary - return { - k: v - for k, v in ( - (k, filter_none(v) if isinstance(v, dict) else v) - for k, v in d.items() - if v is not None - ) - if not (isinstance(v, dict) and not v) # Check for non-empty after filtering - } +# Self-Healing Mechanism Configuration +[self_healing] +retry_attempts = 1 +""" @dataclass class Config: """Configuration for the GPT Engineer project""" - api_config: _ApiConfig = field(default_factory=_ApiConfig) - model_config: _ModelConfig = field(default_factory=_ModelConfig) - improve_config: _ImproveConfig = field(default_factory=_ImproveConfig) + api_config: Dict[str, Any] = field(default_factory=dict) + model_config: Dict[str, Any] = field(default_factory=dict) + improve_config: Dict[str, Any] = field(default_factory=dict) + git_filter_config: Dict[str, Any] = field(default_factory=dict) + self_healing_config: Dict[str, Any] = field(default_factory=dict) + other_sections: Dict[str, Any] = field( + default_factory=dict + ) # To handle any other sections dynamically @classmethod def from_toml(cls, config_file: Path | str): @@ -78,38 +55,36 @@ def from_toml(cls, config_file: Path | str): @classmethod def from_dict(cls, config_dict: dict): - api_config = _ApiConfig( - **config_dict.get( - "API", {"OPENAI_API_KEY": None, "ANTHROPIC_API_KEY": None} - ) - ) - model_config = _ModelConfig( - **config_dict.get( - "model", - {"model_name": None, "temperature": None, "azure_endpoint": None}, - ) - ) - improve_config = _ImproveConfig( - **config_dict.get( - "improve", {"is_linting": None, "is_file_selection": None} - ) - ) + api_config = config_dict.get("API", {}) + model_config = config_dict.get("model", {}) + improve_config = config_dict.get("improve", {}) + git_filter_config = config_dict.get("git_filter", {}) + self_healing_config = config_dict.get("self_healing", {}) + + # Extract other sections not explicitly handled + handled_keys = {"API", "model", "improve", "git_filter", "self_healing"} + other_sections = {k: v for k, v in config_dict.items() if k not in handled_keys} return cls( api_config=api_config, model_config=model_config, improve_config=improve_config, + git_filter_config=git_filter_config, + self_healing_config=self_healing_config, + other_sections=other_sections, ) def to_dict(self) -> dict: - d = asdict(self) - d["API"] = d.pop("api_config", None) - d["model"] = d.pop("model_config", None) - d["improve"] = d.pop("improve_config", None) + d = { + "API": self.api_config, + "model": self.model_config, + "improve": self.improve_config, + "git_filter": self.git_filter_config, + "self_healing": self.self_healing_config, + } + d.update(self.other_sections) # Add other dynamic sections # Drop None values and empty dictionaries - # Needed because tomlkit.dumps() doesn't handle None values, - # and we don't want to write empty sections. d = filter_none(d) return d @@ -124,15 +99,15 @@ def to_toml(self, config_file: Path | str, save=True) -> str: default_config = Config().to_dict() for k, v in self.to_dict().items(): # only write values that are already explicitly set, or that differ from defaults - if k in config or v != default_config[k]: + if k in config or v != default_config.get(k): if isinstance(v, dict): config[k] = { k2: v2 for k2, v2 in v.items() if ( - k2 in config[k] + k2 in config.get(k, {}) or default_config.get(k) is None - or v2 != default_config[k].get(k2) + or v2 != default_config.get(k, {}).get(k2) ) } else: @@ -151,3 +126,16 @@ def read_config(config_file: Path) -> tomlkit.TOMLDocument: assert config_file.exists(), f"Config file {config_file} does not exist" with open(config_file, "r") as f: return tomlkit.load(f) + + +def filter_none(d: dict) -> dict: + """Drop None values and empty dictionaries from a dictionary""" + return { + k: v + for k, v in ( + (k, filter_none(v) if isinstance(v, dict) else v) + for k, v in d.items() + if v is not None + ) + if not (isinstance(v, dict) and not v) # Check for non-empty after filtering + } From 5eab6a46adaad46ad3a8ed3b63abeb244f835a10 Mon Sep 17 00:00:00 2001 From: Talion Date: Sat, 10 Aug 2024 10:12:02 -0400 Subject: [PATCH 09/10] update test --- tests/test_project_config.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/test_project_config.py b/tests/test_project_config.py index 5021100569..2258c29657 100644 --- a/tests/test_project_config.py +++ b/tests/test_project_config.py @@ -2,12 +2,7 @@ import pytest -from gpt_engineer.core.project_config import ( - Config, - _ImproveConfig, - example_config, - filter_none, -) +from gpt_engineer.core.project_config import Config, example_config, filter_none def test_config_load(): @@ -76,15 +71,18 @@ def test_config_load_partial(): def test_config_update(): - example_config = """ + initial_config = """ [improve] is_linting = "..." """.strip() with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: - f.write(example_config) + f.write(initial_config) + config = Config.from_toml(f.name) - config.improve_config = _ImproveConfig(is_linting=False, is_file_selection=True) + config.improve_config = {"is_linting": False, "is_file_selection": True} config.to_toml(f.name) + + # Check that updated values are written and read correctly assert Config.from_toml(f.name) == config From 94505c5e8d5a67a91b818f402d70adbb0d8dee7a Mon Sep 17 00:00:00 2001 From: Talion Date: Sat, 10 Aug 2024 16:46:51 -0400 Subject: [PATCH 10/10] modify tests to access Configuration data dict --- tests/test_project_config.py | 38 +++++++++++++++++------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/tests/test_project_config.py b/tests/test_project_config.py index 2258c29657..f5d0182d7b 100644 --- a/tests/test_project_config.py +++ b/tests/test_project_config.py @@ -6,23 +6,23 @@ def test_config_load(): - # write example config to a file + # Write example config to a file with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: f.write(example_config) - # load the config from the file + # Load the config from the file config = Config.from_toml(f.name) - assert config.api_config.OPENAI_API_KEY == "..." - assert config.api_config.ANTHROPIC_API_KEY == "..." - assert config.model_config.model_name == "gpt-4o" - assert config.model_config.temperature == 0.1 - assert config.improve_config.is_linting is False - assert config.improve_config.is_file_selection is True + assert config.api_config["OPENAI_API_KEY"] == "..." + assert config.api_config["ANTHROPIC_API_KEY"] == "..." + assert config.model_config["model_name"] == "gpt-4o" + assert config.model_config["temperature"] == 0.1 + assert config.improve_config["is_linting"] is False + assert config.improve_config["is_file_selection"] is True assert config.to_dict() assert config.to_toml(f.name, save=False) - # check that write+read is idempotent + # Check that write+read is idempotent assert Config.from_toml(f.name) == config @@ -40,33 +40,31 @@ def test_config_defaults(): def test_config_from_dict(): - d = {"improve": {"is_linting": "..."}} # minimal example + d = {"improve": {"is_linting": "..."}} # Minimal example config = Config.from_dict(d) - assert config.improve_config - assert config.improve_config.is_linting == "..." + assert config.improve_config["is_linting"] == "..." config_dict = config.to_dict() - # check that the config dict matches the input dict exactly (no keys/defaults added) + # Check that the config dict matches the input dict exactly (no keys/defaults added) assert config_dict == d def test_config_load_partial(): - # Loads a partial config, and checks that the rest is not set (i.e. None) - example_config = """ + # Loads a partial config, and checks that the rest is not set (i.e., None) + partial_config = """ [improve] is_linting = "..." """.strip() with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: - f.write(example_config) + f.write(partial_config) config = Config.from_toml(f.name) - assert config.improve_config - assert config.improve_config.is_linting == "..." + assert config.improve_config["is_linting"] == "..." assert config.to_dict() toml_str = config.to_toml(f.name, save=False) - assert toml_str == example_config + assert toml_str.strip() == partial_config - # check that write+read is idempotent + # Check that write+read is idempotent assert Config.from_toml(f.name) == config