Integration tests executed on a real deployment as part of the CICD -…

… Redshift Datasets (#1636)

⚠️ MERGE AFTER #1628
### Feature or Bugfix
- Feature: Testing

### Detail
Add integration tests for Redshift datasets
It also includes the global conftest fixtures that will be used for
redshift dataset sharing

At the moment it assumes Redshift infrastructure is already provided; we
might want to implement the deployment of this pre-required infra, but
that can be done on a separate PR.

## Testing
Tested locally:

![image](https://github.com/user-attachments/assets/564ff84f-61ac-4055-9551-e7186cacdd69)


### Relates
- #1619 
- #1220 

### Security
Please answer the questions below briefly where applicable, or write
`N/A`. Based on
[OWASP 10](https://owasp.org/Top10/en/).

- Does this PR introduce or modify any input fields or queries - this
includes
fetching data from storage outside the application (e.g. a database, an
S3 bucket)?
  - Is the input sanitized?
- What precautions are you taking before deserializing the data you
consume?
  - Is injection prevented by parametrizing queries?
  - Have you ensured no `eval` or similar functions are used?
- Does this PR introduce any functionality or component that requires
authorization?
- How have you ensured it respects the existing AuthN/AuthZ mechanisms?
  - Are you logging failed auth attempts?
- Are you using or adding any cryptographic features?
  - Do you use a standard proven implementations?
  - Are the used keys controlled by the customer? Where are they stored?
- Are you introducing any new policies/roles/users?
  - Have you used the least-privilege principle? How?


By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

tests_new/integration_tests/modules/redshift_datasets/dataset_queries.py

@@ -210,15 +210,14 @@ def import_redshift_dataset(
     description,
     tags,
     owner,
-    topics,
     group_uri,
     confidentiality,
     auto_approval_enabled,
     connection_uri,
     schema,
     tables,
 ):
-    mutation = {
+    query = {
         'operationName': 'importRedshiftDataset',
         'variables': {
             'input': {
@@ -228,7 +227,6 @@ def import_redshift_dataset(
                 'description': description,
                 'tags': tags,
                 'owner': owner,
-                'topics': topics,
                 'SamlAdminGroupName': group_uri,
                 'confidentiality': confidentiality,
                 'autoApprovalEnabled': auto_approval_enabled,
@@ -243,16 +241,19 @@ def import_redshift_dataset(
                 datasetUri
                 label
                 userRoleForDataset
+                connection {
+                 connectionUri
+                 } 
               }
             }
         """,
     }
-    response = client.query(mutation=mutation)
+    response = client.query(query=query)
     return response.data.importRedshiftDataset
 
 
 def update_redshift_dataset(client, dataset_uri, description):
-    mutation = {
+    query = {
         'operationName': 'updateRedshiftDataset',
         'variables': {
             'datasetUri': dataset_uri,
@@ -267,16 +268,17 @@ def update_redshift_dataset(client, dataset_uri, description):
                 datasetUri
                 label
                 userRoleForDataset
+                description
               }
             }
         """,
     }
-    response = client.query(mutation=mutation)
+    response = client.query(query=query)
     return response.data.updateRedshiftDataset
 
 
 def delete_redshift_dataset(client, dataset_uri):
-    mutation = {
+    query = {
         'operationName': 'deleteRedshiftDataset',
         'variables': {'datasetUri': dataset_uri},
         'query': """
@@ -285,12 +287,12 @@ def delete_redshift_dataset(client, dataset_uri):
             }
         """,
     }
-    response = client.query(mutation=mutation)
+    response = client.query(query=query)
     return response.data.deleteRedshiftDataset
 
 
 def add_redshift_dataset_tables(client, dataset_uri, tables):
-    mutation = {
+    query = {
         'operationName': 'addRedshiftDatasetTables',
         'variables': {
             'datasetUri': dataset_uri,
@@ -305,12 +307,12 @@ def add_redshift_dataset_tables(client, dataset_uri, tables):
             }
         """,
     }
-    response = client.query(mutation=mutation)
+    response = client.query(query=query)
     return response.data.addRedshiftDatasetTables
 
 
 def delete_redshift_dataset_table(client, rs_table_uri):
-    mutation = {
+    query = {
         'operationName': 'deleteRedshiftDatasetTable',
         'variables': {'rsTableUri': rs_table_uri},
         'query': """
@@ -319,12 +321,12 @@ def delete_redshift_dataset_table(client, rs_table_uri):
             }
         """,
     }
-    response = client.query(mutation=mutation)
+    response = client.query(query=query)
     return response.data.deleteRedshiftDatasetTable
 
 
 def update_redshift_dataset_table(client, rs_table_uri, description):
-    mutation = {
+    query = {
         'operationName': 'updateRedshiftDatasetTable',
         'variables': {
             'rsTableUri': rs_table_uri,
@@ -338,9 +340,10 @@ def update_redshift_dataset_table(client, rs_table_uri, description):
               updateRedshiftDatasetTable(rsTableUri: $rsTableUri, input: $input) {
                 rsTableUri
                 label
+                description
               }
             }
         """,
     }
-    response = client.query(mutation=mutation)
+    response = client.query(query=query)
     return response.data.updateRedshiftDatasetTable

tests_new/integration_tests/modules/redshift_datasets/global_conftest.py

@@ -1,7 +1,7 @@
 import logging
 
 import pytest
-from integration_tests.core.stack.utils import check_stack_ready
+from integration_tests.core.stack.utils import check_stack_ready, check_stack_in_progress
 from integration_tests.conftest import RedshiftConnection
 from integration_tests.modules.redshift_datasets.connection_queries import (
     create_redshift_connection,
@@ -10,10 +10,18 @@
     delete_redshift_connection_group_permissions,
 )
 
+from integration_tests.modules.redshift_datasets.dataset_queries import (
+    import_redshift_dataset,
+    delete_redshift_dataset,
+    list_redshift_dataset_tables,
+)
+
 log = logging.getLogger(__name__)
 
 REDSHIFT_DATABASE = 'dev'
 REDSHIFT_SCHEMA = 'public'
+REDSHIFT_TABLE1 = 'region'
+REDSHIFT_TABLE2 = 'nation'
 
 
 def create_connection(client, env, group, name, conn_type, red_type, connection_data=RedshiftConnection):
@@ -31,6 +39,14 @@ def create_connection(client, env, group, name, conn_type, red_type, connection_
         redshift_user=None,
         secret_arn=connection_data.secret_arn,
     )
+    # The connection creation updates the permissions of the pivot role in the environment stack
+    check_stack_in_progress(
+        client=client,
+        env_uri=env.environmentUri,
+        stack_uri=env.stack.stackUri,
+        target_uri=env.environmentUri,
+        target_type='environment',
+    )
     check_stack_ready(
         client=client,
         env_uri=env.environmentUri,
@@ -140,3 +156,65 @@ def session_connection_cluster_data_user(client5, group5, session_cross_acc_env_
     finally:
         if connection:
             delete_redshift_connection(client=client5, connection_uri=connection.connectionUri)
+
+
+@pytest.fixture(scope='session')
+def session_redshift_dataset_serverless(
+    client1, group1, user1, session_env1, org1, session_connection_serverless_data_user, session_id
+):
+    dataset = None
+    try:
+        dataset = import_redshift_dataset(
+            client=client1,
+            label='session_redshift_serverless_dataset1',
+            org_uri=org1.organizationUri,
+            env_uri=session_env1.environmentUri,
+            description='Used for integration test',
+            tags=[session_id],
+            owner=user1.username,
+            group_uri=group1,
+            confidentiality='Unclassified',
+            auto_approval_enabled=False,
+            connection_uri=session_connection_serverless_data_user.connectionUri,
+            schema=REDSHIFT_SCHEMA,
+            tables=[REDSHIFT_TABLE1],
+        )
+        yield dataset
+    finally:
+        if dataset:
+            delete_redshift_dataset(client=client1, dataset_uri=dataset.datasetUri)
+
+
+@pytest.fixture(scope='session')
+def session_redshift_dataset_serverless_table(client1, session_redshift_dataset_serverless):
+    tables = list_redshift_dataset_tables(
+        client=client1, dataset_uri=session_redshift_dataset_serverless.datasetUri, term=REDSHIFT_TABLE1
+    )
+    yield tables.nodes[0]
+
+
+@pytest.fixture(scope='session')
+def session_redshift_dataset_cluster(
+    client5, group5, user5, session_cross_acc_env_1, org1, session_connection_cluster_data_user, session_id
+):
+    dataset = None
+    try:
+        dataset = import_redshift_dataset(
+            client=client5,
+            label='session_redshift_cluster_dataset1',
+            org_uri=org1.organizationUri,
+            env_uri=session_cross_acc_env_1.environmentUri,
+            description='Used for integration test',
+            tags=[session_id],
+            owner=user5.username,
+            group_uri=group5,
+            confidentiality='Secret',
+            auto_approval_enabled=False,
+            connection_uri=session_connection_cluster_data_user.connectionUri,
+            schema=REDSHIFT_SCHEMA,
+            tables=[],
+        )
+        yield dataset
+    finally:
+        if dataset:
+            delete_redshift_dataset(client=client5, dataset_uri=dataset.datasetUri)