Skip to content

Commit

Permalink
set up polaris docker image
Browse files Browse the repository at this point in the history
  • Loading branch information
@sungwy
sungwy committed Oct 26, 2024
1 parent de976fe commit b637e10
Show file tree
Hide file tree
Showing 4 changed files with 402 additions and 22 deletions.
9 changes: 1 addition & 8 deletions dev/docker-compose-integration.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,19 +41,12 @@ services:
- hive:hive
- minio:minio
rest:
image: tabulario/iceberg-rest
build: https://github.com/apache/polaris.git
container_name: pyiceberg-rest
networks:
iceberg_net:
ports:
- 8181:8181
environment:
- AWS_ACCESS_KEY_ID=admin
- AWS_SECRET_ACCESS_KEY=password
- AWS_REGION=us-east-1
- CATALOG_WAREHOUSE=s3://warehouse/
- CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
- CATALOG_S3_ENDPOINT=http://minio:9000
minio:
image: minio/minio
container_name: pyiceberg-minio
Expand Down
239 changes: 239 additions & 0 deletions test.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,239 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 43,
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"from requests import HTTPError, Session\n",
"\n",
"PRINCIPAL_TOKEN=\"principal:root;realm:default-realm\"\n",
"POLARIS_URL=\"http://localhost:8181\"\n",
"PRINCIPAL_NAME=\"iceberg\"\n",
"CATALOG_NAME=\"polaris\"\n",
"CATALOG_ROLE=\"admin_role\"\n",
"PRINCIPAL_ROLE = \"admin_principal_role\""
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [],
"source": [
"def create_principal(session: Session) -> str:\n",
" response = session.get(url=f\"{POLARIS_URL}/api/management/v1/principals/{PRINCIPAL_NAME}\")\n",
" try:\n",
" # rotate creds\n",
" response.raise_for_status()\n",
" response = session.delete(\n",
" url=f\"{POLARIS_URL}/api/management/v1/principals/{PRINCIPAL_NAME}\",\n",
" )\n",
" finally:\n",
" # create principal\n",
" data = {\"principal\": {\"name\": PRINCIPAL_NAME}, \"credentialRotationRequired\": 'false'}\n",
" response = session.post(\n",
" url=f\"{POLARIS_URL}/api/management/v1/principals\", data=json.dumps(data),\n",
" )\n",
" credentials = response.json()[\"credentials\"]\n",
"\n",
" principal_credential = f\"{credentials['clientId']}:{credentials['clientSecret']}\"\n",
" return principal_credential\n",
"\n",
"def create_catalog(session: Session) -> str:\n",
" response = session.get(\n",
" url=f\"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}\",\n",
" )\n",
" try:\n",
" response.raise_for_status()\n",
" except HTTPError:\n",
" # Create Catalog\n",
" data = {\n",
" \"catalog\": {\n",
" \"name\": CATALOG_NAME,\n",
" \"type\": \"INTERNAL\",\n",
" \"readOnly\": False,\n",
" \"properties\": {\n",
" \"default-base-location\": \"file:///warehouse\"\n",
" },\n",
" \"storageConfigInfo\": {\n",
" \"storageType\": \"FILE\",\n",
" \"allowedLocations\": [\n",
" \"file:///warehouse\"\n",
" ]\n",
" }\n",
" }\n",
" }\n",
" response = session.post(\n",
" url=f\"{POLARIS_URL}/api/management/v1/catalogs\", data=json.dumps(data),\n",
" )\n",
" response.raise_for_status()\n",
"\n",
"def create_catalog_role(session: Session) -> None:\n",
" try:\n",
" response = session.get(\n",
" url=f\"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}/catalog-roles/{CATALOG_ROLE}\"\n",
" )\n",
" response.raise_for_status()\n",
" except HTTPError:\n",
" # Create Catalog Role\n",
" data = {\n",
" \"catalogRole\": {\n",
" \"name\": CATALOG_ROLE,\n",
" }\n",
" }\n",
" response = session.post(\n",
" url=f\"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}/catalog-roles\", data=json.dumps(data),\n",
" )\n",
" response.raise_for_status()\n",
"\n",
"def grant_catalog_privileges(session: Session) -> None:\n",
" # Grant Catalog privileges to the catalog role\n",
" data = {\n",
" \"grant\": {\n",
" \"type\": \"catalog\",\n",
" \"privilege\": \"CATALOG_MANAGE_CONTENT\"\n",
" }\n",
" }\n",
" response = session.put(\n",
" url=f\"{POLARIS_URL}/api/management/v1/catalogs/{CATALOG_NAME}/catalog-roles/{CATALOG_ROLE}/grants\", data=json.dumps(data),\n",
" )\n",
" response.raise_for_status()\n",
"\n",
"def create_principal_role(session: Session) -> None:\n",
" try:\n",
" response = session.get(\n",
" url=f\"{POLARIS_URL}/api/management/v1/principal-roles/{PRINCIPAL_ROLE}\",\n",
" )\n",
" response.raise_for_status()\n",
" except HTTPError:\n",
" # Create a principal role\n",
" data = {\n",
" \"principalRole\": {\n",
" \"name\": PRINCIPAL_ROLE,\n",
" }\n",
" }\n",
" response = session.post(\n",
" url=f\"{POLARIS_URL}/api/management/v1/principal-roles\", data=json.dumps(data),\n",
" )\n",
" response.raise_for_status()\n",
" \n",
" # Assign the catalog role to the principal role\n",
" data = {\n",
" \"catalogRole\": {\n",
" \"name\": CATALOG_ROLE,\n",
" }\n",
" }\n",
" response = session.put(\n",
" url=f\"{POLARIS_URL}/api/management/v1/principal-roles/{PRINCIPAL_ROLE}/catalog-roles/{CATALOG_NAME}\", data=json.dumps(data),\n",
" )\n",
" response.raise_for_status()\n",
"\n",
" # Assign the principal role to the root principal\n",
" data = {\n",
" \"principalRole\": {\n",
" \"name\": PRINCIPAL_ROLE,\n",
" }\n",
" }\n",
" response = session.put(\n",
" url=f\"{POLARIS_URL}/api/management/v1/principals/{PRINCIPAL_NAME}/principal-roles\", data=json.dumps(data),\n",
" )\n",
" response.raise_for_status()"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [],
"source": [
"session = Session()\n",
"session.headers[\"Content-type\"] = \"application/json\"\n",
"session.headers[\"Accept\"] = \"application/json\"\n",
"session.headers[\"Authorization\"] = f\"Bearer {PRINCIPAL_TOKEN}\"\n",
"\n",
"principal_credential = create_principal(session)\n",
"create_catalog(session)\n",
"create_catalog_role(session)\n",
"grant_catalog_privileges(session)\n",
"create_principal_role(session)"
]
},
{
"cell_type": "code",
"execution_count": 41,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/workspaces/iceberg-python/pyiceberg/utils/deprecated.py:51: DeprecationWarning: Deprecated in 0.8.0, will be removed in 1.0.0. Iceberg REST client is missing the OAuth2 server URI configuration and defaults to http://localhost:8181/api/catalogoauth/tokens. This automatic fallback will be removed in a future Iceberg release.It is recommended to configure the OAuth2 endpoint using the 'oauth2-server-uri'property to be prepared. This warning will disappear if the OAuth2endpoint is explicitly configured. See https://github.com/apache/iceberg/issues/10537\n",
" _deprecation_warning(message)\n"
]
}
],
"source": [
"from pyiceberg.catalog import load_catalog\n",
"\n",
"\n",
"catalog = load_catalog(\n",
" \"local\",\n",
" **{\n",
" \"type\": \"rest\",\n",
" \"credential\": principal_credential,\n",
" \"uri\": \"http://localhost:8181/api/catalog\",\n",
" \"s3.endpoint\": \"http://localhost:9000\",\n",
" \"s3.access-key-id\": \"admin\",\n",
" \"s3.secret-access-key\": \"password\",\n",
" \"warehouse\": \"polaris\",\n",
" \"scope\": \"PRINCIPAL_ROLE:ALL\"\n",
" },\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[('test',)]"
]
},
"execution_count": 42,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"catalog.list_namespaces()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "pyiceberg-FsHa-ZgB-py3.10",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.13"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
14 changes: 0 additions & 14 deletions tests/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2185,20 +2185,6 @@ def bound_reference_uuid() -> BoundReference[str]:
return BoundReference(field=NestedField(1, "field", UUIDType(), required=False), accessor=Accessor(position=0, inner=None))


@pytest.fixture(scope="session")
def session_catalog() -> Catalog:
return load_catalog(
"local",
**{
"type": "rest",
"uri": "http://localhost:8181",
"s3.endpoint": "http://localhost:9000",
"s3.access-key-id": "admin",
"s3.secret-access-key": "password",
},
)


@pytest.fixture(scope="session")
def session_catalog_hive() -> Catalog:
return load_catalog(
Expand Down
Loading

0 comments on commit b637e10

Please sign in to comment.