DM-45263: Add new tap_schema module

Makefile

@@ -20,6 +20,9 @@ print_target:
 build:
 	@uv pip install --force-reinstall --no-deps -e .
 
+deps:
+	@uv pip install --upgrade -r requirements.txt
+
 docs:
 	@rm -rf docs/dev/internals docs/_build
 	@tox -e docs

docs/changes/DM-45263.feature.rst

@@ -0,0 +1,3 @@
+Added a new ``tap_schema_`` module designed to deprecate and eventually replace the ``tap`` module.
+This module provides utilities for translating a Felis schema into a TAP_SCHEMA representation.
+The command ``felis load-tap-schema`` can be used to activate this functionality.

docs/dev/internals.rst

@@ -7,33 +7,41 @@ Python API
 .. automodapi:: felis.datamodel
    :include-all-objects:
 
-.. automodapi:: felis.metadata
+.. automodapi:: felis.db.dialects
    :include-all-objects:
    :no-inheritance-diagram:
 
-.. automodapi:: felis.tap
+.. automodapi:: felis.db.sqltypes
    :include-all-objects:
    :no-inheritance-diagram:
 
-.. automodapi:: felis.types
-   :include-all-objects:
-
-.. automodapi:: felis.db.dialects
+.. automodapi:: felis.db.utils
    :include-all-objects:
    :no-inheritance-diagram:
 
-.. automodapi:: felis.db.sqltypes
+.. automodapi:: felis.db.variants
    :include-all-objects:
    :no-inheritance-diagram:
 
-.. automodapi:: felis.db.utils
+.. automodapi:: felis.metadata
    :include-all-objects:
    :no-inheritance-diagram:
 
-.. automodapi:: felis.db.variants
+.. automodapi:: felis.tap
    :include-all-objects:
    :no-inheritance-diagram:
 
+.. automodapi:: felis.tap_schema
+    :include-all-objects:
+    :no-inheritance-diagram:
+
 .. automodapi:: felis.tests.postgresql
     :include-all-objects:
     :no-inheritance-diagram:
+
+.. automodapi:: felis.tests.utils
+    :include-all-objects:
+    :no-inheritance-diagram:
+
+.. automodapi:: felis.types
+   :include-all-objects:

docs/documenteer.toml

@@ -21,6 +21,8 @@ nitpick_ignore = [
     ["py:class", "sqlalchemy.orm.decl_api.Base"],
     ["py:class", "sqlalchemy.engine.mock.MockConnection"],
     ["py:class", "pydantic.main.BaseModel"],
+    ["py:exc", "pydantic.ValidationError"],
+    ["py:exc", "yaml.YAMLError"]
 ]
 nitpick_ignore_regex = [
     # Bug in autodoc_pydantic.
@@ -29,5 +31,6 @@ nitpick_ignore_regex = [
 python_api_dir = "dev/internals"
 
 [sphinx.intersphinx.projects]
-python = "https://docs.python.org/3/"
-sqlalchemy = "https://docs.sqlalchemy.org/en/latest/"
+python = "https://docs.python.org/3"
+sqlalchemy = "https://docs.sqlalchemy.org/en/latest"
+lsst = "https://pipelines.lsst.io/v/weekly"

docs/user-guide/datatypes.rst

@@ -74,7 +74,7 @@ The following table shows these mapping:
 +-----------+---------------+----------+------------------+--------------+
 | unicode   | NVARCHAR      | NVARCHAR | VARCHAR          | unicodeChar  |
 +-----------+---------------+----------+------------------+--------------+
-| text      | TEXT          | LONGTEXT | TEXT             | uncodeChar   |
+| text      | TEXT          | LONGTEXT | TEXT             | char         |
 +-----------+---------------+----------+------------------+--------------+
 | binary    | BLOB          | LONGBLOB | BYTEA            | unsignedByte |
 +-----------+---------------+----------+------------------+--------------+

pyproject.toml

@@ -26,7 +26,8 @@ dependencies = [
     "click >= 7",
     "pyyaml >= 6",
     "pydantic >= 2, < 3",
-    "lsst-utils"
+    "lsst-utils",
+    "lsst-resources"
 ]
 requires-python = ">=3.11.0"
 dynamic = ["version"]
@@ -55,7 +56,7 @@ zip-safe = true
 license-files = ["COPYRIGHT", "LICENSE"]
 
 [tool.setuptools.package-data]
-"felis" = ["py.typed"]
+"felis" = ["py.typed", "schemas/*.yaml"]
 
 [tool.setuptools.dynamic]
 version = { attr = "lsst_versions.get_lsst_version" }

python/felis/cli.py

@@ -23,22 +23,21 @@
 
 from __future__ import annotations
 
-import io
 import logging
 from collections.abc import Iterable
 from typing import IO
 
 import click
-import yaml
 from pydantic import ValidationError
 from sqlalchemy.engine import Engine, create_engine, make_url
-from sqlalchemy.engine.mock import MockConnection
+from sqlalchemy.engine.mock import MockConnection, create_mock_engine
 
 from . import __version__
 from .datamodel import Schema
-from .db.utils import DatabaseContext
+from .db.utils import DatabaseContext, is_mock_url
 from .metadata import MetaDataBuilder
 from .tap import Tap11Base, TapLoadingVisitor, init_tables
+from .tap_schema import DataLoader, TableManager
 
 __all__ = ["cli"]
 
@@ -107,7 +106,7 @@
     dry_run: bool,
     output_file: IO[str] | None,
     ignore_constraints: bool,
-    file: IO,
+    file: IO[str],
 ) -> None:
     """Create database objects from the Felis file.
 
@@ -133,8 +132,7 @@
         Felis file to read.
     """
     try:
-        yaml_data = yaml.safe_load(file)
-        schema = Schema.model_validate(yaml_data, context={"id_generation": ctx.obj["id_generation"]})
+        schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
         url = make_url(engine_url)
         if schema_name:
             logger.info(f"Overriding schema name with: {schema_name}")
@@ -261,7 +259,7 @@
     tap_keys_table: str,
     tap_key_columns_table: str,
     tap_schema_index: int,
-    file: io.TextIOBase,
+    file: IO[str],
 ) -> None:
     """Load TAP metadata from a Felis file.
 
@@ -304,8 +302,7 @@
     The data will be loaded into the TAP_SCHEMA from the engine URL. The
     tables must have already been initialized or an error will occur.
     """
-    yaml_data = yaml.load(file, Loader=yaml.SafeLoader)
-    schema = Schema.model_validate(yaml_data)
+    schema = Schema.from_stream(file)
 
     tap_tables = init_tables(
         tap_schema_name,
@@ -345,6 +342,79 @@
         tap_visitor.visit_schema(schema)
 
 
+@cli.command("load-tap-schema", help="Load metadata from a Felis file into a TAP_SCHEMA database")
+@click.option("--engine-url", envvar="FELIS_ENGINE_URL", help="SQLAlchemy Engine URL")
+@click.option("--tap-schema-name", help="Name of the TAP_SCHEMA schema in the database")
+@click.option(
+    "--tap-tables-postfix", help="Postfix which is applied to standard TAP_SCHEMA table names", default=""
+)
+@click.option("--tap-schema-index", type=int, help="TAP_SCHEMA index of the schema in this environment")
+@click.option("--dry-run", is_flag=True, help="Execute dry run only. Does not insert any data.")
+@click.option("--echo", is_flag=True, help="Print out the generated insert statements to stdout")
+@click.option("--output-file", type=click.Path(), help="Write SQL commands to a file")
+@click.argument("file", type=click.File())
+@click.pass_context
+def load_tap_schema(
+    ctx: click.Context,
+    engine_url: str,
+    tap_schema_name: str,
+    tap_tables_postfix: str,
+    tap_schema_index: int,
+    dry_run: bool,
+    echo: bool,
+    output_file: str | None,
+    file: IO[str],
+) -> None:
+    """Load TAP metadata from a Felis file.
+
+    Parameters
+    ----------
+    engine_url
+        SQLAlchemy Engine URL.
+    tap_tables_postfix
+        Postfix which is applied to standard TAP_SCHEMA table names.
+    tap_schema_index
+        TAP_SCHEMA index of the schema in this environment.
+    dry_run
+        Execute dry run only. Does not insert any data.
+    echo
+        Print out the generated insert statements to stdout.
+    output_file
+        Output file for writing generated SQL.
+    file
+        Felis file to read.
+
+    Notes
+    -----
+    The TAP_SCHEMA database must already exist or the command will fail. This
+    command will not initialize the TAP_SCHEMA tables.
+    """
+    url = make_url(engine_url)
+    engine: Engine | MockConnection
+    if dry_run or is_mock_url(url):
+        engine = create_mock_engine(url, executor=None)
+    else:
+        engine = create_engine(engine_url)
+    mgr = TableManager(
+        engine=engine,
+        apply_schema_to_metadata=False if engine.dialect.name == "sqlite" else True,
+        schema_name=tap_schema_name,
+        table_name_postfix=tap_tables_postfix,
+    )
+
+    schema = Schema.from_stream(file, context={"id_generation": ctx.obj["id_generation"]})
+
+    DataLoader(
+        schema,
+        mgr,
+        engine,
+        tap_schema_index=tap_schema_index,
+        dry_run=dry_run,
+        print_sql=echo,
+        output_path=output_file,
+    ).load()
+
+
 @cli.command("validate", help="Validate one or more Felis YAML files")
 @click.option(
     "--check-description", is_flag=True, help="Check that all objects have a description", default=False
@@ -372,7 +442,7 @@
     check_redundant_datatypes: bool,
     check_tap_table_indexes: bool,
     check_tap_principal: bool,
-    files: Iterable[io.TextIOBase],
+    files: Iterable[IO[str]],
 ) -> None:
     """Validate one or more felis YAML files.
 
@@ -406,9 +476,8 @@
         file_name = getattr(file, "name", None)
         logger.info(f"Validating {file_name}")
         try:
-            data = yaml.load(file, Loader=yaml.SafeLoader)
-            Schema.model_validate(
-                data,
+            Schema.from_stream(
+                file,
                 context={
                     "check_description": check_description,
                     "check_redundant_datatypes": check_redundant_datatypes,