Skip to content

Commit

Permalink
Added integration test for sharepoint
Browse files Browse the repository at this point in the history
  • Loading branch information
mateuszkuprowski committed Jan 9, 2025
1 parent 9a3ab36 commit 1ce2811
Show file tree
Hide file tree
Showing 2 changed files with 87 additions and 1 deletion.
86 changes: 86 additions & 0 deletions test/integration/connectors/test_sharepoint.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
# test_sharepoint_integration.py
import os

import pytest

from test.integration.connectors.utils.validation import (
SourceValidationConfigs,
source_connector_validation,
)
from test.integration.utils import requires_env
from unstructured_ingest.v2.processes.connectors.sharepoint import (
CONNECTOR_TYPE,
SharepointAccessConfig,
SharepointConnectionConfig,
SharepointDownloader,
SharepointDownloaderConfig,
SharepointIndexer,
SharepointIndexerConfig,
SharepointPermissionsConfig,
)

SOURCE_TAG = "source"


@pytest.mark.asyncio
@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
@requires_env("SP_CLIENT_ID", "SP_CLIENT_CRED", "SP_SITE_URL", "SP_TENANT")
async def test_sharepoint_source(temp_dir):
"""
Integration test that:
1) Creates a SharepointIndexer to list/enumerate items in a given site
2) Creates a SharepointDownloader to fetch each enumerated item
3) Runs a validation helper to confirm the end-to-end pipeline
"""
client_id = os.getenv("SHAREPOINT_CLIENT_ID")
client_cred = os.getenv("SHAREPOINT_CRED")
tenant = os.getenv("SHAREPOINT_PERMISSIONS_TENANT")
site_url = os.getenv("SHAREPOINT_SITE")

access_config = SharepointAccessConfig(client_cred=client_cred)

permissions_config = SharepointPermissionsConfig(
permissions_application_id=None,
permissions_tenant=tenant,
permissions_client_cred=None, # or SecretStr(...)
)

connection_config = SharepointConnectionConfig(
client_id=client_id,
site=site_url,
access_config=access_config,
permissions_config=permissions_config,
)

index_config = SharepointIndexerConfig(
path=None,
recursive=True,
omit_files=False,
omit_pages=False,
omit_lists=True,
)

download_config = SharepointDownloaderConfig(
download_dir=temp_dir # Directory where the files get saved
)

indexer = SharepointIndexer(
connection_config=connection_config,
index_config=index_config,
)
downloader = SharepointDownloader(
connection_config=connection_config,
download_config=download_config,
)

expected_files = 7

await source_connector_validation(
indexer=indexer,
downloader=downloader,
configs=SourceValidationConfigs(
test_id="sharepoint",
expected_num_files=expected_files,
validate_downloaded_files=True,
),
)
2 changes: 1 addition & 1 deletion unstructured_ingest/v2/processes/connectors/sharepoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ def get_client(self) -> "ClientContext":
authority=f"{self.permissions_config.authority_url.get_secret_value()}/"
f"{self.permissions_config.permissions_tenant}",
client_id=self.permissions_config.permissions_application_id,
client_credential=self.permissions_config.permissions_client_cred.get_secret_value(), # noqa: E501
client_credential=self.permissions_config.permissions_client_cred.get_secret_value(), # noqa: E501
)
token_result = app.acquire_token_for_client(
scopes=[
Expand Down

0 comments on commit 1ce2811

Please sign in to comment.