From ee0f0335681208c298d01af4366d73d808451cb0 Mon Sep 17 00:00:00 2001 From: Yian Shang Date: Thu, 10 Oct 2024 19:16:43 -0700 Subject: [PATCH 1/3] Fix an issue where retrieving cube SQL fails because the node object loaded from the database did not have cube metadata on it --- datajunction-server/datajunction_server/api/sql.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/datajunction-server/datajunction_server/api/sql.py b/datajunction-server/datajunction_server/api/sql.py index 7d2229bca..a766266dc 100644 --- a/datajunction-server/datajunction_server/api/sql.py +++ b/datajunction-server/datajunction_server/api/sql.py @@ -124,6 +124,10 @@ async def build_and_save_node_sql( # pylint: disable=too-many-locals # If it's a cube, we'll build SQL for the metrics in the cube, along with any additional # dimensions or filters provided in the arguments if node.type == NodeType.CUBE: + node = cast( + Node, + await Node.get_cube_by_name(session, node_name), + ) dimensions = list( OrderedDict.fromkeys(node.current.cube_node_dimensions + dimensions), ) From e56e87cb8eebc69bc52c483e855b5fbf24d3d581 Mon Sep 17 00:00:00 2001 From: Yian Shang Date: Thu, 10 Oct 2024 20:16:20 -0700 Subject: [PATCH 2/3] Add refresh of cache for every metrics SQL request --- .../datajunction_server/api/sql.py | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/datajunction-server/datajunction_server/api/sql.py b/datajunction-server/datajunction_server/api/sql.py index a766266dc..fe1778b10 100644 --- a/datajunction-server/datajunction_server/api/sql.py +++ b/datajunction-server/datajunction_server/api/sql.py @@ -366,6 +366,7 @@ async def get_sql_for_metrics( # pylint: disable=too-many-locals ), ignore_errors: Optional[bool] = True, use_materialized: Optional[bool] = True, + background_tasks: BackgroundTasks, ) -> TranslatedSQL: """ Return SQL for a set of metrics with dimensions and filters @@ -388,6 +389,21 @@ async def get_sql_for_metrics( # pylint: disable=too-many-locals engine_version=engine_version, query_type=QueryBuildType.METRICS, ): + # Update the node SQL in a background task to keep it up-to-date + background_tasks.add_task( + build_and_save_sql_for_metrics, + session=session, + metrics=metrics, + dimensions=dimensions, + filters=filters, + orderby=orderby, + limit=limit, + engine_name=engine_name, + engine_version=engine_version, + access_control=access_control, + ignore_errors=ignore_errors, + use_materialized=use_materialized, + ) engine = ( await get_engine(session, engine_name, engine_version) # type: ignore if engine_name @@ -399,6 +415,37 @@ async def get_sql_for_metrics( # pylint: disable=too-many-locals dialect=engine.dialect if engine else None, ) + return await build_and_save_sql_for_metrics( + session, + metrics, + dimensions, + filters, + orderby, + limit, + engine_name, + engine_version, + access_control, + ignore_errors=ignore_errors, # type: ignore + use_materialized=use_materialized, # type: ignore + ) + + +async def build_and_save_sql_for_metrics( # pylint: disable=too-many-arguments,too-many-locals + session: AsyncSession, + metrics: List[str], + dimensions: List[str], + filters: List[str] = None, + orderby: List[str] = None, + limit: Optional[int] = None, + engine_name: Optional[str] = None, + engine_version: Optional[str] = None, + access_control: Optional[access.AccessControlStore] = None, + ignore_errors: bool = True, + use_materialized: bool = True, +): + """ + Builds and saves SQL for metrics. + """ translated_sql, _, _ = await build_sql_for_multiple_metrics( session, metrics, @@ -417,8 +464,8 @@ async def get_sql_for_metrics( # pylint: disable=too-many-locals session=session, nodes=metrics, dimensions=dimensions, - filters=filters, - orderby=orderby, + filters=filters, # type: ignore + orderby=orderby, # type: ignore limit=limit, engine_name=engine_name, engine_version=engine_version, From c9be191c6e4450556a368a515e3345b9fff6a8d8 Mon Sep 17 00:00:00 2001 From: Yian Shang Date: Thu, 10 Oct 2024 22:36:35 -0700 Subject: [PATCH 3/3] Fix pdm lock check --- .github/workflows/test.yml | 10 +++++----- .pre-commit-config.yaml | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4f2800b61..daca31cd3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,6 +64,11 @@ jobs: prerelease: true enable-pep582: true + - uses: pre-commit/action@v3.0.0 + name: Force check of all pdm.lock files + with: + extra_args: pdm-lock-check --all-files + - name: Run Tests if: | (matrix.library == 'client' && steps.filter.outputs.client == 'true') || @@ -84,11 +89,6 @@ jobs: export MODULE=${{ matrix.library == 'server' && 'datajunction_server' || matrix.library == 'client' && 'datajunction' || matrix.library == 'djqs' && 'djqs' || matrix.library == 'djrs' && 'datajunction_reflection'}} pdm run pytest ${{ (matrix.library == 'server' || matrix.library == 'client') && '-n auto' || '' }} --cov-fail-under=100 --cov=$MODULE --cov-report term-missing -vv tests/ --doctest-modules $MODULE --without-integration --without-slow-integration - - uses: pre-commit/action@v3.0.0 - name: Force check of all pdm.lock files - with: - extra_args: pdm-lock-check --all-files - build-javascript: runs-on: ubuntu-latest strategy: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a17fea649..322442b37 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -110,7 +110,7 @@ repos: entry: pdm lock --check --project . files: ^pyproject.toml$ - repo: https://github.com/pdm-project/pdm - rev: 2.11.1 + rev: 2.18.2 hooks: - id: pdm-lock-check name: pdm-lock-check-server @@ -124,7 +124,7 @@ repos: entry: pdm lock --check --project datajunction-query files: ^datajunction-query/pyproject.toml$ - repo: https://github.com/pdm-project/pdm - rev: 2.8.1 + rev: 2.18.2 hooks: - id: pdm-lock-check name: pdm-lock-check-reflection