From c3c1b95bc15db576ca81b86c9ca12f3f2b8036ce Mon Sep 17 00:00:00 2001 From: Nina Bernick Date: Fri, 19 Jul 2024 16:34:51 -0700 Subject: [PATCH 1/4] add dev readme --- README.md | 4 +- docs/HOWTO-working-with-platformics.md | 51 ++++++++++++++++++++++++++ platformics/api/core/deps.py | 2 +- platformics/database/connect.py | 2 +- platformics/settings.py | 1 + test_app/docker-compose.yml | 1 + 6 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 docs/HOWTO-working-with-platformics.md diff --git a/README.md b/README.md index 980a7d3..ae338a5 100644 --- a/README.md +++ b/README.md @@ -52,13 +52,13 @@ The libraries and tools that make Platformics work: 1. Install the `Dev Containers` extension for vscode 2. Open a new VSCode window in your api directory. It will read the `.devcontainer/devcontainer.json` configuration and prompt you to reopen the directory in a container (lower right side of the screen). Click "Reopen in container" 3. Click the "Run and Debug" icon in the icon bar on the right side of the VSCode window (or ctrl+shift+d). Then click the "start debugging" icon at the top of the run and debug panel (or press F5). This will launch a secondary instance of the API service that listens on port 9008. -4. Set all the breakpoints you want. Browse to the api at http://localhost:9008 to trigger them. Remember that the application restarts when files change, so you'll have to start and stop the debugger to pick up any changes you make! +4. Set all the breakpoints you want. Browse to the api at http://localhost:9008/graphql to trigger them. Remember that the application restarts when files change, so you'll have to start and stop the debugger to pick up any changes you make! ## Debugging and developing platformics itself 1. Run `make dev` in the root of this directory. This launches a compose service called `dev-app` that has the `platformics` directory in this repo mounted inside the `test_app` application as a sub-module, so it can be edited directly and be debugged via the VSCode debugger. 2. Open a new VSCode window in the root of this reopo. It will read the `.devcontainer/devcontainer.json` configuration and prompt you to reopen the directory in a container (lower right side of the screen). Click "Reopen in container" 3. Click the "Run and Debug" icon in the icon bar on the right side of the VSCode window (or ctrl+shift+d). Then click the "start debugging" icon at the top of the run and debug panel (or press F5). This will launch a secondary instance of the API service that listens on port 9008. -4. Set all the breakpoints you want. Browse to the api at http://localhost:9008 to trigger them. Remember that the application restarts when files change, so you'll have to start and stop the debugger to pick up any changes you make! +4. Set all the breakpoints you want. Browse to the api at http://localhost:9008/graphql to trigger them. Remember that the application restarts when files change, so you'll have to start and stop the debugger to pick up any changes you make! ## HOWTO - [Extend the generated API](docs/HOWTO-extend-generated-api.md) diff --git a/docs/HOWTO-working-with-platformics.md b/docs/HOWTO-working-with-platformics.md new file mode 100644 index 0000000..dd3bee9 --- /dev/null +++ b/docs/HOWTO-working-with-platformics.md @@ -0,0 +1,51 @@ +# How To: Working with Platformics + +## Structure + +### Platformics +Notable files and subdirectories: +* `api/` - base code and utilities for setting up API + * `core/` + * `deps.py` - dependencies injected to FastAPI endpoints + * `query_builder.py` - functions for querying DB given GraphQL queries + * `gql_loaders.py` - dataloaders for relationships to avoid GQL N+1 queries + * `strawberry_extensions.py` - extensions to apply dependencies to resolvers + * `types/` + * `entities.py` - base entity code + * `files.py` - GQL types, mutations, queries for files +* `codegen/` + * `lib/linkml_wrappers.py` - convenience functions for converting LinkML to generated code + * `templates/` - all Jinja templates for codegen. Entity-related templates can be overridden with [custom templates](https://github.com/chanzuckerberg/platformics/tree/main/platformics/docs/HOWTO-customize-templates.md). + * `generator.py` - script handling all logic of applying Jinja templates to LinkML schema to generate code +* `database/` + * `models/` + * `base.py` - SQLAlchemy model for base entity + * `file.py` - SQLAlchemy model and methods for file + * `connect.py` - functions for connecting to database +* `support/` - miscellaneous support enums, functions for files +* `test_infra/` - contains base entity and file factories +* `settings.py` - config variables + + + + +### Test app +Notable files and subdirectories: +* `api/` - entrypoint for GQL API service + * `helpers/` - generated GQL types and helper functions for GROUPBY queries + * `types/` - generated GQL types + * `validators/` - generated Pydantic validators for create and update inputs + * `mutations.py` - generated mutations (create, update, delete) for each entity type + * `queries.py` - generated queries (list and aggregate) for each entity type + * `schema.graphql` - GQL format schema + * `schema.json` - JSON format schema +* `cerbos` - generated access policies for user actions for each entity type +* `database/` - code related to establishing DB connections / sessions + * `migrations/` - alembic migrations + * `models/` - generated SQLAlchemy models +* `schema/` + * `schema.yaml` - LinkML schema used to codegen entity-related files +* `test_infra/` + * `factories/` - FactoryBoy factories generated for each entity type +* `tests/` - your custom tests (not codegenned) +* `etc/` - some basic setup configuration diff --git a/platformics/api/core/deps.py b/platformics/api/core/deps.py index b284ccc..0642cb6 100644 --- a/platformics/api/core/deps.py +++ b/platformics/api/core/deps.py @@ -30,7 +30,7 @@ async def get_engine( settings: APISettings = Depends(get_settings), ) -> typing.AsyncGenerator[AsyncDB, None]: """Wrap resolvers in a DB engine""" - engine = init_async_db(settings.DB_URI) + engine = init_async_db(settings.DB_URI, echo=settings.DB_ECHO) try: yield engine finally: diff --git a/platformics/database/connect.py b/platformics/database/connect.py index d147172..7401765 100644 --- a/platformics/database/connect.py +++ b/platformics/database/connect.py @@ -40,7 +40,7 @@ def session(self) -> sessionmaker[Session]: def init_async_db(db_uri: str, **kwargs: dict[str, Any]) -> AsyncDB: - engine = create_async_engine(db_uri, echo=False, pool_size=5, max_overflow=5, future=True, **kwargs) + engine = create_async_engine(db_uri, pool_size=5, max_overflow=5, future=True, **kwargs) return AsyncDB(engine) diff --git a/platformics/settings.py b/platformics/settings.py index ea8b13c..5d50031 100644 --- a/platformics/settings.py +++ b/platformics/settings.py @@ -39,6 +39,7 @@ class Settings(BaseSettings): DEFAULT_UPLOAD_PROTOCOL: str BOTO_ENDPOINT_URL: typing.Optional[str] = None AWS_REGION: str + DB_ECHO: bool = False ############################################################################ # Computed properties diff --git a/test_app/docker-compose.yml b/test_app/docker-compose.yml index 0615a28..fcc155c 100644 --- a/test_app/docker-compose.yml +++ b/test_app/docker-compose.yml @@ -67,6 +67,7 @@ services: - AWS_REGION=us-west-2 - AWS_ACCESS_KEY_ID=test - AWS_SECRET_ACCESS_KEY=test + - DB_ECHO=true # TODO - these are keypairs for testing only! Do not use in prod!! - JWK_PUBLIC_KEY_FILE=/app/etc/public_key.pem - JWK_PRIVATE_KEY_FILE=/app/etc/private_key.pem From bf17d720bcdfcc9695bf5b2d466fede61d69b3fa Mon Sep 17 00:00:00 2001 From: Nina Bernick Date: Mon, 22 Jul 2024 15:50:16 -0700 Subject: [PATCH 2/4] container docs --- README.md | 13 +-------- docs/HOWTO-working-with-platformics.md | 37 +++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index ae338a5..bb36e2d 100644 --- a/README.md +++ b/README.md @@ -48,19 +48,8 @@ The libraries and tools that make Platformics work: 2. Run `make codegen` to re-run code gen and restart the API service 3. If your changes require DB schema changes, run `make alembic-autogenerate` and `make alembic-upgrade-head` to generate DB migrations and run them. -## Debugging -1. Install the `Dev Containers` extension for vscode -2. Open a new VSCode window in your api directory. It will read the `.devcontainer/devcontainer.json` configuration and prompt you to reopen the directory in a container (lower right side of the screen). Click "Reopen in container" -3. Click the "Run and Debug" icon in the icon bar on the right side of the VSCode window (or ctrl+shift+d). Then click the "start debugging" icon at the top of the run and debug panel (or press F5). This will launch a secondary instance of the API service that listens on port 9008. -4. Set all the breakpoints you want. Browse to the api at http://localhost:9008/graphql to trigger them. Remember that the application restarts when files change, so you'll have to start and stop the debugger to pick up any changes you make! - -## Debugging and developing platformics itself -1. Run `make dev` in the root of this directory. This launches a compose service called `dev-app` that has the `platformics` directory in this repo mounted inside the `test_app` application as a sub-module, so it can be edited directly and be debugged via the VSCode debugger. -2. Open a new VSCode window in the root of this reopo. It will read the `.devcontainer/devcontainer.json` configuration and prompt you to reopen the directory in a container (lower right side of the screen). Click "Reopen in container" -3. Click the "Run and Debug" icon in the icon bar on the right side of the VSCode window (or ctrl+shift+d). Then click the "start debugging" icon at the top of the run and debug panel (or press F5). This will launch a secondary instance of the API service that listens on port 9008. -4. Set all the breakpoints you want. Browse to the api at http://localhost:9008/graphql to trigger them. Remember that the application restarts when files change, so you'll have to start and stop the debugger to pick up any changes you make! - ## HOWTO +- [Work with platformics](docs/HOWTO-working-with-platformics.md) - [Extend the generated API](docs/HOWTO-extend-generated-api.md) - [Customize Codegen templates](docs/HOWTO-customize-templates.md) diff --git a/docs/HOWTO-working-with-platformics.md b/docs/HOWTO-working-with-platformics.md index dd3bee9..c96f18a 100644 --- a/docs/HOWTO-working-with-platformics.md +++ b/docs/HOWTO-working-with-platformics.md @@ -39,7 +39,7 @@ Notable files and subdirectories: * `queries.py` - generated queries (list and aggregate) for each entity type * `schema.graphql` - GQL format schema * `schema.json` - JSON format schema -* `cerbos` - generated access policies for user actions for each entity type +* `cerbos/` - generated access policies for user actions for each entity type * `database/` - code related to establishing DB connections / sessions * `migrations/` - alembic migrations * `models/` - generated SQLAlchemy models @@ -49,3 +49,38 @@ Notable files and subdirectories: * `factories/` - FactoryBoy factories generated for each entity type * `tests/` - your custom tests (not codegenned) * `etc/` - some basic setup configuration + +## Containers +There are two main ways of developing: making changes in the test app only, and making changes to the core platformics library. + +### Developing in the test app +To develop in the test app, `make init` will build the `platformics` image from the latest base image and start up the test app on port 9009. Changes within the `platformics` repo will not be picked up unless the image is rebuilt. + +Containers (`test_app/docker-compose.yml`) +* `motoserver`: mock of S3 services to run test app entirely locally for development +* `cerbos`: resource authorization +* `platformics-db`: Postgres database +* `graphql-api`: API + +When developing on `platformics` itself, running `make dev` will start all of the above containers, then stop the `graphql-api` container and start a new `dev-app` compose service. +`dev-app` is very similar to the `graphql-api` container, but mounts `platformics` directory into the test app as a submodule so that changes to `platformics` are picked up without rebuilding (see below in "Debugging and developing platformics itself"). `graphql-api` and `dev-app` share a port, so the `graphql-api` container is stopped before starting the `dev-app` container. + + +For either of these two flows, the main app will be listening on port 9009 and debugging sessions will listen on port 9008. + + +## Debugging +### Test app +1. Install the `Dev Containers` extension for vscode +2. Open a new VSCode window in your api directory. It will read the `.devcontainer/devcontainer.json` configuration and prompt you to reopen the directory in a container (lower right side of the screen). Click "Reopen in container" +3. Click the "Run and Debug" icon in the icon bar on the right side of the VSCode window (or ctrl+shift+d). Then click the "start debugging" icon at the top of the run and debug panel (or press F5). This will launch a secondary instance of the API service that listens on port 9008. +4. Set all the breakpoints you want. Browse to the api at http://localhost:9008/graphql to trigger them. Remember that the application restarts when files change, so you'll have to start and stop the debugger to pick up any changes you make! + +### Debugging and developing platformics itself +1. Run `make dev` in the root of this directory. This launches a compose service called `dev-app` that has the `platformics` directory in this repo mounted inside the `test_app` application as a sub-module, so it can be edited directly and be debugged via the VSCode debugger. +2. Open a new VSCode window in the root of this reopo. It will read the `.devcontainer/devcontainer.json` configuration and prompt you to reopen the directory in a container (lower right side of the screen). Click "Reopen in container" +3. Click the "Run and Debug" icon in the icon bar on the right side of the VSCode window (or ctrl+shift+d). Then click the "start debugging" icon at the top of the run and debug panel (or press F5). This will launch a secondary instance of the API service that listens on port 9008. +4. Set all the breakpoints you want. Browse to the api at http://localhost:9008/graphql to trigger them. Remember that the application restarts when files change, so you'll have to start and stop the debugger to pick up any changes you make! + +### Queries +To view SQL logs for queries, set `DB_ECHO=true` in `docker-compose.yml` \ No newline at end of file From c9ea95c2977c44151a3bcfdc5feb6e981dd4e019 Mon Sep 17 00:00:00 2001 From: Nina Bernick Date: Mon, 22 Jul 2024 15:55:48 -0700 Subject: [PATCH 3/4] consolidate --- docs/HOWTO-working-with-platformics.md | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/docs/HOWTO-working-with-platformics.md b/docs/HOWTO-working-with-platformics.md index c96f18a..6bcfe67 100644 --- a/docs/HOWTO-working-with-platformics.md +++ b/docs/HOWTO-working-with-platformics.md @@ -51,10 +51,9 @@ Notable files and subdirectories: * `etc/` - some basic setup configuration ## Containers -There are two main ways of developing: making changes in the test app only, and making changes to the core platformics library. +There are two main ways of running the test app depending on what kind of development you're doing: making changes in the test app only, and making changes to the core platformics library. -### Developing in the test app -To develop in the test app, `make init` will build the `platformics` image from the latest base image and start up the test app on port 9009. Changes within the `platformics` repo will not be picked up unless the image is rebuilt. +To develop in the test app, `make init` will build the `platformics` image from the latest base image and start up the test app listening on port 9009. Changes within the `platformics` repo will not be picked up unless the image is rebuilt. Containers (`test_app/docker-compose.yml`) * `motoserver`: mock of S3 services to run test app entirely locally for development @@ -63,24 +62,21 @@ Containers (`test_app/docker-compose.yml`) * `graphql-api`: API When developing on `platformics` itself, running `make dev` will start all of the above containers, then stop the `graphql-api` container and start a new `dev-app` compose service. -`dev-app` is very similar to the `graphql-api` container, but mounts `platformics` directory into the test app as a submodule so that changes to `platformics` are picked up without rebuilding (see below in "Debugging and developing platformics itself"). `graphql-api` and `dev-app` share a port, so the `graphql-api` container is stopped before starting the `dev-app` container. +The compose service called `dev-app` has the `platformics` directory in this repo mounted inside the `test_app` application as a sub-module, so it can be edited directly and be debugged via the VSCode debugger. +`graphql-api` and `dev-app` share a port, so the `graphql-api` container is stopped before starting the `dev-app` container. For either of these two flows, the main app will be listening on port 9009 and debugging sessions will listen on port 9008. ## Debugging -### Test app + +### Using VSCode debugger 1. Install the `Dev Containers` extension for vscode 2. Open a new VSCode window in your api directory. It will read the `.devcontainer/devcontainer.json` configuration and prompt you to reopen the directory in a container (lower right side of the screen). Click "Reopen in container" 3. Click the "Run and Debug" icon in the icon bar on the right side of the VSCode window (or ctrl+shift+d). Then click the "start debugging" icon at the top of the run and debug panel (or press F5). This will launch a secondary instance of the API service that listens on port 9008. 4. Set all the breakpoints you want. Browse to the api at http://localhost:9008/graphql to trigger them. Remember that the application restarts when files change, so you'll have to start and stop the debugger to pick up any changes you make! -### Debugging and developing platformics itself -1. Run `make dev` in the root of this directory. This launches a compose service called `dev-app` that has the `platformics` directory in this repo mounted inside the `test_app` application as a sub-module, so it can be edited directly and be debugged via the VSCode debugger. -2. Open a new VSCode window in the root of this reopo. It will read the `.devcontainer/devcontainer.json` configuration and prompt you to reopen the directory in a container (lower right side of the screen). Click "Reopen in container" -3. Click the "Run and Debug" icon in the icon bar on the right side of the VSCode window (or ctrl+shift+d). Then click the "start debugging" icon at the top of the run and debug panel (or press F5). This will launch a secondary instance of the API service that listens on port 9008. -4. Set all the breakpoints you want. Browse to the api at http://localhost:9008/graphql to trigger them. Remember that the application restarts when files change, so you'll have to start and stop the debugger to pick up any changes you make! ### Queries To view SQL logs for queries, set `DB_ECHO=true` in `docker-compose.yml` \ No newline at end of file From a2586e70fdd53bd639dc1e39cb29cf4f404131ea Mon Sep 17 00:00:00 2001 From: Nina Bernick Date: Tue, 23 Jul 2024 09:06:27 -0700 Subject: [PATCH 4/4] tweaks --- docs/HOWTO-working-with-platformics.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/HOWTO-working-with-platformics.md b/docs/HOWTO-working-with-platformics.md index 6bcfe67..c6b45d7 100644 --- a/docs/HOWTO-working-with-platformics.md +++ b/docs/HOWTO-working-with-platformics.md @@ -24,9 +24,7 @@ Notable files and subdirectories: * `connect.py` - functions for connecting to database * `support/` - miscellaneous support enums, functions for files * `test_infra/` - contains base entity and file factories -* `settings.py` - config variables - - +* `settings.py` - config variables using [Pydantic Settings](https://docs.pydantic.dev/latest/concepts/pydantic_settings/) ### Test app @@ -79,4 +77,4 @@ For either of these two flows, the main app will be listening on port 9009 and d ### Queries -To view SQL logs for queries, set `DB_ECHO=true` in `docker-compose.yml` \ No newline at end of file +To view SQL logs for queries, set `DB_ECHO=true` in `docker-compose.yml`. Run `make start` or `docker compose up -d` to apply the change. \ No newline at end of file