From b637e1070fecf18adca2adc8346786a24203b484 Mon Sep 17 00:00:00 2001 From: Sung Yun <107272191+sungwy@users.noreply.github.com> Date: Sat, 26 Oct 2024 23:57:56 +0000 Subject: [PATCH] set up polaris docker image --- dev/docker-compose-integration.yml | 9 +- test.ipynb | 239 +++++++++++++++++++++++++++++ tests/conftest.py | 14 -- tests/integration/conftest.py | 162 +++++++++++++++++++ 4 files changed, 402 insertions(+), 22 deletions(-) create mode 100644 test.ipynb create mode 100644 tests/integration/conftest.py diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index fccdcdc757..43c65d9356 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -41,19 +41,12 @@ services: - hive:hive - minio:minio rest: - image: tabulario/iceberg-rest + build: https://github.com/apache/polaris.git container_name: pyiceberg-rest networks: iceberg_net: ports: - 8181:8181 - environment: - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 - - CATALOG_WAREHOUSE=s3://warehouse/ - - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO - - CATALOG_S3_ENDPOINT=http://minio:9000 minio: image: minio/minio container_name: pyiceberg-minio diff --git a/test.ipynb b/test.ipynb new file mode 100644 index 0000000000..73290f46cb --- /dev/null +++ b/test.ipynb @@ -0,0 +1,239 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from requests import HTTPError, Session\n", + "\n", + "PRINCIPAL_TOKEN=\"principal:root;realm:default-realm\"\n", + "POLARIS_URL=\"http://localhost:8181\"\n", + "PRINCIPAL_NAME=\"iceberg\"\n", + "CATALOG_NAME=\"polaris\"\n", + "CATALOG_ROLE=\"admin_role\"\n", + "PRINCIPAL_ROLE = \"admin_principal_role\"" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [], + "source": [ + "def create_principal(session: Session) -> str:\n", + " response = session.get(url=f\"{POLARIS_URL}/api/management/v1/principals/{PRINCIPAL_NAME}\")\n", + " try:\n", + " # rotate creds\n", + " response.raise_for_status()\n", + " response = session.delete(\n", + " url=f\"{POLARIS_URL}/api/management/v1/principals/{PRINCIPAL_NAME}\",\n", + " )\n", + " finally:\n", + " # create principal\n", + " data = {\"principal\": {\"name\": PRINCIPAL_NAME}, \"credentialRotationRequired\": 'false'}\n", + " response = session.post(\n", + " url=f\"{POLARIS_URL}/api/management/v1/principals\", data=json.dumps(data),\n", + " )\n", + " credentials = response.json()[\"credentials\"]\n", + "\n", + " principal_credential = f\"{credentials['clientId']}:{credentials['clientSecret']}\"\n", + " return principal_credential\n", + "\n", + "def create_catalog(session: Session) -> str:\n", + " response = session.get(\n", + " url=f\"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}\",\n", + " )\n", + " try:\n", + " response.raise_for_status()\n", + " except HTTPError:\n", + " # Create Catalog\n", + " data = {\n", + " \"catalog\": {\n", + " \"name\": CATALOG_NAME,\n", + " \"type\": \"INTERNAL\",\n", + " \"readOnly\": False,\n", + " \"properties\": {\n", + " \"default-base-location\": \"file:///warehouse\"\n", + " },\n", + " \"storageConfigInfo\": {\n", + " \"storageType\": \"FILE\",\n", + " \"allowedLocations\": [\n", + " \"file:///warehouse\"\n", + " ]\n", + " }\n", + " }\n", + " }\n", + " response = session.post(\n", + " url=f\"{POLARIS_URL}/api/management/v1/catalogs\", data=json.dumps(data),\n", + " )\n", + " response.raise_for_status()\n", + "\n", + "def create_catalog_role(session: Session) -> None:\n", + " try:\n", + " response = session.get(\n", + " url=f\"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}/catalog-roles/{CATALOG_ROLE}\"\n", + " )\n", + " response.raise_for_status()\n", + " except HTTPError:\n", + " # Create Catalog Role\n", + " data = {\n", + " \"catalogRole\": {\n", + " \"name\": CATALOG_ROLE,\n", + " }\n", + " }\n", + " response = session.post(\n", + " url=f\"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}/catalog-roles\", data=json.dumps(data),\n", + " )\n", + " response.raise_for_status()\n", + "\n", + "def grant_catalog_privileges(session: Session) -> None:\n", + " # Grant Catalog privileges to the catalog role\n", + " data = {\n", + " \"grant\": {\n", + " \"type\": \"catalog\",\n", + " \"privilege\": \"CATALOG_MANAGE_CONTENT\"\n", + " }\n", + " }\n", + " response = session.put(\n", + " url=f\"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}/catalog-roles/{CATALOG_ROLE}/grants\", data=json.dumps(data),\n", + " )\n", + " response.raise_for_status()\n", + "\n", + "def create_principal_role(session: Session) -> None:\n", + " try:\n", + " response = session.get(\n", + " url=f\"{POLARIS_URL}/api/management/v1/principal-roles/{PRINCIPAL_ROLE}\",\n", + " )\n", + " response.raise_for_status()\n", + " except HTTPError:\n", + " # Create a principal role\n", + " data = {\n", + " \"principalRole\": {\n", + " \"name\": PRINCIPAL_ROLE,\n", + " }\n", + " }\n", + " response = session.post(\n", + " url=f\"{POLARIS_URL}/api/management/v1/principal-roles\", data=json.dumps(data),\n", + " )\n", + " response.raise_for_status()\n", + " \n", + " # Assign the catalog role to the principal role\n", + " data = {\n", + " \"catalogRole\": {\n", + " \"name\": CATALOG_ROLE,\n", + " }\n", + " }\n", + " response = session.put(\n", + " url=f\"{POLARIS_URL}/api/management/v1/principal-roles/{PRINCIPAL_ROLE}/catalog-roles/{CATALOG_NAME}\", data=json.dumps(data),\n", + " )\n", + " response.raise_for_status()\n", + "\n", + " # Assign the principal role to the root principal\n", + " data = {\n", + " \"principalRole\": {\n", + " \"name\": PRINCIPAL_ROLE,\n", + " }\n", + " }\n", + " response = session.put(\n", + " url=f\"{POLARIS_URL}/api/management/v1/principals/{PRINCIPAL_NAME}/principal-roles\", data=json.dumps(data),\n", + " )\n", + " response.raise_for_status()" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [], + "source": [ + "session = Session()\n", + "session.headers[\"Content-type\"] = \"application/json\"\n", + "session.headers[\"Accept\"] = \"application/json\"\n", + "session.headers[\"Authorization\"] = f\"Bearer {PRINCIPAL_TOKEN}\"\n", + "\n", + "principal_credential = create_principal(session)\n", + "create_catalog(session)\n", + "create_catalog_role(session)\n", + "grant_catalog_privileges(session)\n", + "create_principal_role(session)" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/workspaces/iceberg-python/pyiceberg/utils/deprecated.py:51: DeprecationWarning: Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI configuration and defaults to http://localhost:8181/api/catalogoauth/tokens. This automatic fallback will be removed in a future Iceberg release.It is recommended to configure the OAuth2 endpoint using the 'oauth2-server-uri'property to be prepared. This warning will disappear if the OAuth2endpoint is explicitly configured. See https://github.com/apache/iceberg/issues/10537\n", + " _deprecation_warning(message)\n" + ] + } + ], + "source": [ + "from pyiceberg.catalog import load_catalog\n", + "\n", + "\n", + "catalog = load_catalog(\n", + " \"local\",\n", + " **{\n", + " \"type\": \"rest\",\n", + " \"credential\": principal_credential,\n", + " \"uri\": \"http://localhost:8181/api/catalog\",\n", + " \"s3.endpoint\": \"http://localhost:9000\",\n", + " \"s3.access-key-id\": \"admin\",\n", + " \"s3.secret-access-key\": \"password\",\n", + " \"warehouse\": \"polaris\",\n", + " \"scope\": \"PRINCIPAL_ROLE:ALL\"\n", + " },\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[('test',)]" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "catalog.list_namespaces()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pyiceberg-FsHa-ZgB-py3.10", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/conftest.py b/tests/conftest.py index b05947ebe6..56c779452e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2185,20 +2185,6 @@ def bound_reference_uuid() -> BoundReference[str]: return BoundReference(field=NestedField(1, "field", UUIDType(), required=False), accessor=Accessor(position=0, inner=None)) -@pytest.fixture(scope="session") -def session_catalog() -> Catalog: - return load_catalog( - "local", - **{ - "type": "rest", - "uri": "http://localhost:8181", - "s3.endpoint": "http://localhost:9000", - "s3.access-key-id": "admin", - "s3.secret-access-key": "password", - }, - ) - - @pytest.fixture(scope="session") def session_catalog_hive() -> Catalog: return load_catalog( diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000000..e99673fc65 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,162 @@ +import pytest +import json +from requests import HTTPError, Session + +from pyiceberg.catalog import Catalog, load_catalog + +PRINCIPAL_TOKEN="principal:root;realm:default-realm" +POLARIS_URL="http://localhost:8181" +PRINCIPAL_NAME="iceberg" +CATALOG_NAME="polaris" +CATALOG_ROLE="admin_role" +PRINCIPAL_ROLE = "admin_principal_role" + +def create_principal(session: Session) -> str: + response = session.get(url=f"{POLARIS_URL}/api/management/v1/principals/{PRINCIPAL_NAME}") + try: + # rotate creds + response.raise_for_status() + response = session.delete( + url=f"{POLARIS_URL}/api/management/v1/principals/{PRINCIPAL_NAME}", + ) + finally: + # create principal + data = {"principal": {"name": PRINCIPAL_NAME}, "credentialRotationRequired": 'false'} + response = session.post( + url=f"{POLARIS_URL}/api/management/v1/principals", data=json.dumps(data), + ) + credentials = response.json()["credentials"] + + principal_credential = f"{credentials['clientId']}:{credentials['clientSecret']}" + return principal_credential + +def create_catalog(session: Session) -> str: + response = session.get( + url=f"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}", + ) + try: + response.raise_for_status() + except HTTPError: + # Create Catalog + data = { + "catalog": { + "name": CATALOG_NAME, + "type": "INTERNAL", + "readOnly": False, + "properties": { + "default-base-location": "file:///warehouse" + }, + "storageConfigInfo": { + "storageType": "FILE", + "allowedLocations": [ + "file:///warehouse" + ] + } + } + } + response = session.post( + url=f"{POLARIS_URL}/api/management/v1/catalogs", data=json.dumps(data), + ) + response.raise_for_status() + +def create_catalog_role(session: Session) -> None: + try: + response = session.get( + url=f"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}/catalog-roles/{CATALOG_ROLE}" + ) + response.raise_for_status() + except HTTPError: + # Create Catalog Role + data = { + "catalogRole": { + "name": CATALOG_ROLE, + } + } + response = session.post( + url=f"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}/catalog-roles", data=json.dumps(data), + ) + response.raise_for_status() + +def grant_catalog_privileges(session: Session) -> None: + # Grant Catalog privileges to the catalog role + data = { + "grant": { + "type": "catalog", + "privilege": "CATALOG_MANAGE_CONTENT" + } + } + response = session.put( + url=f"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}/catalog-roles/{CATALOG_ROLE}/grants", data=json.dumps(data), + ) + response.raise_for_status() + +def create_principal_role(session: Session) -> None: + try: + response = session.get( + url=f"{POLARIS_URL}/api/management/v1/principal-roles/{PRINCIPAL_ROLE}", + ) + response.raise_for_status() + except HTTPError: + # Create a principal role + data = { + "principalRole": { + "name": PRINCIPAL_ROLE, + } + } + response = session.post( + url=f"{POLARIS_URL}/api/management/v1/principal-roles", data=json.dumps(data), + ) + response.raise_for_status() + + # Assign the catalog role to the principal role + data = { + "catalogRole": { + "name": CATALOG_ROLE, + } + } + response = session.put( + url=f"{POLARIS_URL}/api/management/v1/principal-roles/{PRINCIPAL_ROLE}/catalog-roles/{CATALOG_NAME}", data=json.dumps(data), + ) + response.raise_for_status() + + # Assign the principal role to the root principal + data = { + "principalRole": { + "name": PRINCIPAL_ROLE, + } + } + response = session.put( + url=f"{POLARIS_URL}/api/management/v1/principals/{PRINCIPAL_NAME}/principal-roles", data=json.dumps(data), + ) + response.raise_for_status() + +@pytest.fixture(scope="session") +def principal_credential() -> str: + session = Session() + session.headers["Content-type"] = "application/json" + session.headers["Accept"] = "application/json" + session.headers["Authorization"] = f"Bearer {PRINCIPAL_TOKEN}" + + principal_credential = create_principal(session) + create_catalog(session) + create_catalog_role(session) + grant_catalog_privileges(session) + create_principal_role(session) + return principal_credential + + +@pytest.fixture(scope="session") +def session_catalog() -> Catalog: + return load_catalog( + "local", + **{ + "type": "rest", + "credential": principal_credential, + "uri": "http://localhost:8181/api/catalog", + "s3.endpoint": "http://localhost:9000", + "s3.access-key-id": "admin", + "s3.secret-access-key": "password", + "warehouse": "polaris", + "scope": "PRINCIPAL_ROLE:ALL" + }, + ) \ No newline at end of file