diff --git a/clients/export-client/export-api-spec.yaml b/clients/export-client/export-api-spec.yaml index ea417e3885..e90520fd87 100644 --- a/clients/export-client/export-api-spec.yaml +++ b/clients/export-client/export-api-spec.yaml @@ -39,11 +39,7 @@ paths: requestBody: required: true content: - application/json: - schema: - type: string - format: binary - application/csv: + application/octet-stream: schema: type: string format: binary diff --git a/src/main/java/org/candlepin/subscriptions/capacity/CapacityIngressConfiguration.java b/src/main/java/org/candlepin/subscriptions/capacity/CapacityIngressConfiguration.java index 03c9807a36..a057a6c3c5 100644 --- a/src/main/java/org/candlepin/subscriptions/capacity/CapacityIngressConfiguration.java +++ b/src/main/java/org/candlepin/subscriptions/capacity/CapacityIngressConfiguration.java @@ -21,7 +21,6 @@ package org.candlepin.subscriptions.capacity; import org.candlepin.subscriptions.db.RhsmSubscriptionsDataSourceConfiguration; -import org.candlepin.subscriptions.export.ExportConfiguration; import org.candlepin.subscriptions.resteasy.ResteasyConfiguration; import org.springframework.boot.autoconfigure.AutoConfigurationExcludeFilter; import org.springframework.boot.context.TypeExcludeFilter; @@ -43,14 +42,11 @@ @Import({ ResteasyConfiguration.class, RhsmSubscriptionsDataSourceConfiguration.class, - ExportConfiguration.class }) @ComponentScan( basePackages = { "org.candlepin.subscriptions.capacity", "org.candlepin.subscriptions.product", - "org.candlepin.subscriptions.export", - "org.candlepin.subscriptions.subscription.export", }, // Prevent TestConfiguration annotated classes from being picked up by ComponentScan excludeFilters = { diff --git a/src/main/resources/application-capacity-ingress.yaml b/src/main/resources/application-capacity-ingress.yaml index 3525e8893e..39139b3497 100644 --- a/src/main/resources/application-capacity-ingress.yaml +++ b/src/main/resources/application-capacity-ingress.yaml @@ -1,15 +1,2 @@ -SUBSCRIPTION_EXPORT_TOPIC: ${clowder.kafka.topics."platform.export.requests".name:platform.export.requests} - - rhsm-subscriptions: - export-service: - url: ${clowder.privateEndpoints.export-service-service.url:http://localhost:10000} - truststore: ${clowder.privateEndpoints.export-service-service.trust-store-path} - truststore-password: ${clowder.privateEndpoints.export-service-service.trust-store-password} - truststore-type: ${clowder.privateEndpoints.export-service-service.trust-store-type} - psk: ${SWATCH_EXPORT_PSK:placeholder} subscription-sync-enabled: ${SUBSCRIPTION_SYNC_ENABLED:true} - export: - tasks: - topic: ${SUBSCRIPTION_EXPORT_TOPIC} - kafka-group-id: swatch-subscription-export diff --git a/src/test/java/org/candlepin/subscriptions/export/BaseDataExporterServiceTest.java b/src/test/java/org/candlepin/subscriptions/export/BaseDataExporterServiceTest.java index 7f43a49479..aab4c606a0 100644 --- a/src/test/java/org/candlepin/subscriptions/export/BaseDataExporterServiceTest.java +++ b/src/test/java/org/candlepin/subscriptions/export/BaseDataExporterServiceTest.java @@ -51,7 +51,7 @@ import org.candlepin.subscriptions.db.model.Offering; import org.candlepin.subscriptions.db.model.ServiceLevel; import org.candlepin.subscriptions.db.model.Usage; -import org.candlepin.subscriptions.json.SubscriptionsExportJson; +import org.candlepin.subscriptions.json.InstancesExportJson; import org.candlepin.subscriptions.rbac.RbacApiException; import org.candlepin.subscriptions.rbac.RbacService; import org.candlepin.subscriptions.task.TaskQueueProperties; @@ -193,7 +193,7 @@ protected void verifyNoRequestsWereSentToExportServiceWithUploadData() { protected void verifyRequestWasSentToExportServiceWithNoDataFound() { verifyRequestWasSentToExportServiceWithUploadData( - request, toJson(new SubscriptionsExportJson().withData(new ArrayList<>()))); + request, toJson(new InstancesExportJson().withData(new ArrayList<>()))); } protected void updateOffering() { diff --git a/src/test/java/org/candlepin/subscriptions/subscription/export/PerformanceSubscriptionDataExporterServiceIT.java b/src/test/java/org/candlepin/subscriptions/subscription/export/PerformanceSubscriptionDataExporterServiceIT.java deleted file mode 100644 index 0570a239ab..0000000000 --- a/src/test/java/org/candlepin/subscriptions/subscription/export/PerformanceSubscriptionDataExporterServiceIT.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright Red Hat, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * Red Hat trademarks are not licensed under GPLv3. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package org.candlepin.subscriptions.subscription.export; - -import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; -import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.redhat.cloud.event.apps.exportservice.v1.Format; -import com.redhat.swatch.configuration.util.MetricIdUtils; -import java.time.Duration; -import java.util.Map; -import java.util.UUID; -import org.candlepin.clock.ApplicationClock; -import org.candlepin.subscriptions.db.model.BillingProvider; -import org.candlepin.subscriptions.db.model.Subscription; -import org.candlepin.subscriptions.db.model.SubscriptionMeasurementKey; -import org.candlepin.subscriptions.export.BaseDataExporterServiceTest; -import org.candlepin.subscriptions.test.ExtendWithSwatchDatabase; -import org.hibernate.SessionFactory; -import org.hibernate.StatelessSession; -import org.hibernate.Transaction; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ActiveProfiles; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -@Disabled -@SpringBootTest(useMainMethod = SpringBootTest.UseMainMethod.ALWAYS) -@ActiveProfiles(value = {"kafka-queue", "test-inventory", "capacity-ingress"}) -class PerformanceSubscriptionDataExporterServiceIT extends BaseDataExporterServiceTest - implements ExtendWithSwatchDatabase { - - @Autowired SessionFactory sessionFactory; - @Autowired ApplicationClock clock; - - @Override - protected String resourceType() { - return "subscriptions"; - } - - @BeforeEach - @Override - public void setup() { - super.setup(); - - StatelessSession session = sessionFactory.openStatelessSession(); - Transaction tx = session.beginTransaction(); - // According to https://github.com/RedHatInsights/rhsm-subscriptions/pull/3145, - // the file size limit is reached when having 500k subscriptions, where - // the following exception "RESTEASY005081: File limit of 50MB has been reached." is thrown. - // To avoid this exception and un-limit the file size the client can handle, - // we need to set the system property "dev.resteasy.entity.file.threshold" to "-1". - // This setting is being done in org.candlepin.subscriptions.BootApplication. - for (int num = 0; num < 500_000; num++) { - session.insert(givenSubscriptionWithMeasurement()); - } - - tx.commit(); - session.close(); - } - - @Test - void testLargeDataDealingWithRestEasyClientFileLimit() { - givenExportRequestWithPermissions(Format.JSON); - whenReceiveExportRequest(); - verifyRequestWasSentToExportService(); - } - - private Subscription givenSubscriptionWithMeasurement() { - Subscription subscription = new Subscription(); - subscription.setSubscriptionId(UUID.randomUUID().toString()); - subscription.setStartDate(clock.now()); - subscription.setOffering(offering); - subscription.setOrgId(ORG_ID); - subscription.setBillingProvider(BillingProvider.AWS); - offering.getProductTags().clear(); - offering.getProductTags().add(RHEL_FOR_X86); - updateOffering(); - subscription.setBillingAccountId("123"); - subscription.setSubscriptionMeasurements( - Map.of( - new SubscriptionMeasurementKey(MetricIdUtils.getCores().toString(), "HYPERVISOR"), - 5.0)); - return subscription; - } - - @Override - protected void verifyRequestWasSentToExportService() { - Awaitility.await() - .atMost(Duration.ofMinutes(3)) - .untilAsserted( - () -> { - var calls = - EXPORT_SERVICE_WIRE_MOCK_SERVER.findAll( - postRequestedFor( - urlEqualTo( - String.format( - "/app/export/v1/%s/subscriptions/%s/upload", - request.getData().getResourceRequest().getExportRequestUUID(), - request.getData().getResourceRequest().getUUID())))); - assertEquals(1, calls.size()); - assertTrue(calls.get(0).getBody().length > 50_000_000); - }); - } -} diff --git a/src/test/java/org/candlepin/subscriptions/subscription/export/SubscriptionDataExporterServiceTest.java b/src/test/java/org/candlepin/subscriptions/subscription/export/SubscriptionDataExporterServiceTest.java deleted file mode 100644 index 1beca9981a..0000000000 --- a/src/test/java/org/candlepin/subscriptions/subscription/export/SubscriptionDataExporterServiceTest.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Copyright Red Hat, Inc. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - * Red Hat trademarks are not licensed under GPLv3. No permission is - * granted to use or replicate Red Hat trademarks that are incorporated - * in this software or its documentation. - */ -package org.candlepin.subscriptions.subscription.export; - -import static com.redhat.swatch.export.ExportRequestHandler.MISSING_PERMISSIONS; -import static org.candlepin.subscriptions.db.SubscriptionCapacityViewRepository.orgIdEquals; -import static org.candlepin.subscriptions.subscription.export.SubscriptionDataExporterService.PRODUCT_ID; - -import com.redhat.cloud.event.apps.exportservice.v1.Format; -import com.redhat.swatch.configuration.util.MetricIdUtils; -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.candlepin.clock.ApplicationClock; -import org.candlepin.subscriptions.db.SubscriptionCapacityViewRepository; -import org.candlepin.subscriptions.db.SubscriptionRepository; -import org.candlepin.subscriptions.db.model.BillingProvider; -import org.candlepin.subscriptions.db.model.Subscription; -import org.candlepin.subscriptions.db.model.SubscriptionCapacityView; -import org.candlepin.subscriptions.db.model.SubscriptionMeasurementKey; -import org.candlepin.subscriptions.export.BaseDataExporterServiceTest; -import org.candlepin.subscriptions.json.SubscriptionsExportCsvItem; -import org.candlepin.subscriptions.json.SubscriptionsExportJson; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.junit.jupiter.params.provider.EnumSource; -import org.junit.jupiter.params.provider.ValueSource; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.test.context.ActiveProfiles; - -@ActiveProfiles(value = {"kafka-queue", "test-inventory", "capacity-ingress"}) -class SubscriptionDataExporterServiceTest extends BaseDataExporterServiceTest { - - @Autowired SubscriptionRepository subscriptionRepository; - @Autowired SubscriptionCapacityViewRepository subscriptionCapacityViewRepository; - @Autowired ApplicationClock clock; - @Autowired SubscriptionCsvDataMapperService csvDataMapperService; - @Autowired SubscriptionJsonDataMapperService jsonDataMapperService; - - @AfterEach - public void tearDown() { - subscriptionRepository.deleteAll(); - } - - @Override - protected String resourceType() { - return "subscriptions"; - } - - @Test - void testRequestWithoutPermissions() { - givenExportRequestWithoutPermissions(); - whenReceiveExportRequest(); - verifyRequestWasSentToExportServiceWithError(request, MISSING_PERMISSIONS); - } - - @Test - void testRequestWithPermissions() { - givenExportRequestWithPermissions(Format.JSON); - whenReceiveExportRequest(); - verifyRequestWasSentToExportService(); - } - - @Test - void testRequestShouldBeUploadedWithSubscriptionsAsJson() { - givenSubscriptionWithMeasurement(RHEL_FOR_X86); - givenExportRequestWithPermissions(Format.JSON); - whenReceiveExportRequest(); - verifyRequestWasSentToExportService(); - } - - @Test - void testRequestShouldBeUploadedWithSubscriptionsUsingPrometheusEnabledProductAsJson() { - givenSubscriptionWithMeasurement(ROSA); - givenExportRequestWithPermissions(Format.JSON); - givenFilterInExportRequest(PRODUCT_ID, ROSA); - whenReceiveExportRequest(); - verifyRequestWasSentToExportService(); - } - - @Test - void testRequestShouldBeUploadedWithSubscriptionsAsCsv() { - givenSubscriptionWithMeasurement(RHEL_FOR_X86); - givenExportRequestWithPermissions(Format.CSV); - whenReceiveExportRequest(); - verifyRequestWasSentToExportService(); - } - - @ParameterizedTest - @EnumSource(value = Format.class) - void testGivenDuplicateSubscriptionsThenItReturnsOnlyOneRecordAndCapacityIsSum(Format format) { - givenSubscriptionWithMeasurement(RHEL_FOR_X86); - givenSameSubscriptionWithOtherMeasurement(); - givenExportRequestWithPermissions(format); - whenReceiveExportRequest(); - verifyRequestWasSentToExportService(); - } - - @ParameterizedTest - @CsvSource( - value = { - PRODUCT_ID + "," + RHEL_FOR_X86, - "usage,production", - "category,hypervisor", - "sla,premium", - "metric_id,Cores", - "billing_provider,aws", - "billing_account_id,123" - }) - void testFiltersFoundData(String filterName, String exists) { - givenSubscriptionWithMeasurement(RHEL_FOR_X86); - givenExportRequestWithPermissions(Format.JSON); - givenFilterInExportRequest(filterName, exists); - whenReceiveExportRequest(); - verifyRequestWasSentToExportService(); - verifyNoRequestsWereSentToExportServiceWithError(); - } - - @ParameterizedTest - @CsvSource( - value = { - PRODUCT_ID + "," + ROSA, - "usage,disaster recovery", - "category,cloud", - "sla,standard", - "metric_id,Sockets", - "billing_provider,azure", - "billing_account_id,345" - }) - void testFiltersDoesNotFoundDataAndReportIsEmpty(String filterName, String doesNotExist) { - givenSubscriptionWithMeasurement(RHEL_FOR_X86); - givenExportRequestWithPermissions(Format.JSON); - givenFilterInExportRequest(filterName, doesNotExist); - whenReceiveExportRequest(); - verifyRequestWasSentToExportServiceWithNoDataFound(); - verifyNoRequestsWereSentToExportServiceWithError(); - } - - @ParameterizedTest - @ValueSource(strings = {PRODUCT_ID, "usage", "category", "sla", "metric_id", "billing_provider"}) - void testFiltersAreInvalid(String filterName) { - givenSubscriptionWithMeasurement(RHEL_FOR_X86); - givenExportRequestWithPermissions(Format.JSON); - givenFilterInExportRequest(filterName, "wrong!"); - whenReceiveExportRequest(); - verifyNoRequestsWereSentToExportServiceWithUploadData(); - verifyRequestWasSentToExportServiceWithError(request); - } - - @Test - void testRequestShouldFilterByOrgId() { - givenSubscriptionWithMeasurementForAnotherOrgId(RHEL_FOR_X86); - givenSubscriptionWithMeasurement(RHEL_FOR_X86); - givenExportRequestWithPermissions(Format.JSON); - whenReceiveExportRequest(); - verifyRequestWasSentToExportService(); - } - - @Override - protected void verifyRequestWasSentToExportService() { - boolean isCsvFormat = request.getData().getResourceRequest().getFormat() == Format.CSV; - List data = new ArrayList<>(); - for (SubscriptionCapacityView subscription : - subscriptionCapacityViewRepository.findAll(orgIdEquals(ORG_ID))) { - if (isCsvFormat) { - data.addAll(csvDataMapperService.mapDataItem(subscription, null)); - } else { - data.addAll(jsonDataMapperService.mapDataItem(subscription, null)); - } - } - - if (isCsvFormat) { - verifyRequestWasSentToExportServiceWithUploadCsvData(data); - } else { - verifyRequestWasSentToExportServiceWithUploadJsonData( - new SubscriptionsExportJson().withData(data)); - } - } - - private void verifyRequestWasSentToExportServiceWithUploadCsvData(List data) { - verifyRequestWasSentToExportServiceWithUploadData( - request, toCsv(data, SubscriptionsExportCsvItem.class)); - } - - private void verifyRequestWasSentToExportServiceWithUploadJsonData(SubscriptionsExportJson data) { - verifyRequestWasSentToExportServiceWithUploadData(request, toJson(data)); - } - - private void givenSubscriptionWithMeasurementForAnotherOrgId(String productId) { - givenSubscriptionWithMeasurement(UUID.randomUUID().toString(), productId); - } - - private void givenSubscriptionWithMeasurement(String productId) { - givenSubscriptionWithMeasurement(ORG_ID, productId); - } - - private void givenSubscriptionWithMeasurement(String orgId, String productId) { - Subscription subscription = new Subscription(); - subscription.setSubscriptionId(UUID.randomUUID().toString()); - subscription.setSubscriptionNumber(UUID.randomUUID().toString()); - subscription.setStartDate(OffsetDateTime.parse("2024-04-23T11:48:15.888129Z")); - subscription.setEndDate(OffsetDateTime.parse("2028-05-23T11:48:15.888129Z")); - subscription.setOffering(offering); - subscription.setOrgId(orgId); - subscription.setBillingProvider(BillingProvider.AWS); - offering.getProductTags().clear(); - offering.getProductTags().add(productId); - updateOffering(); - subscription.setBillingAccountId("123"); - subscription.setSubscriptionMeasurements( - Map.of( - new SubscriptionMeasurementKey(MetricIdUtils.getCores().toString(), "HYPERVISOR"), - 5.0)); - subscriptionRepository.save(subscription); - } - - private void givenSameSubscriptionWithOtherMeasurement() { - var subscriptions = subscriptionRepository.findAll(); - if (subscriptions.isEmpty()) { - throw new RuntimeException( - "No subscriptions found. Use 'givenSubscriptionWithMeasurement' to add one."); - } - - var existing = subscriptions.get(0); - Subscription subscription = new Subscription(); - subscription.setSubscriptionId(existing.getSubscriptionId()); - subscription.setSubscriptionNumber(existing.getSubscriptionNumber()); - subscription.setStartDate(OffsetDateTime.parse("2024-05-23T11:48:15.888129Z")); - subscription.setEndDate(OffsetDateTime.parse("2024-06-23T11:48:15.888129Z")); - subscription.setOffering(offering); - subscription.setOrgId(ORG_ID); - subscription.setBillingProvider(BillingProvider.AWS); - subscription.setBillingAccountId(existing.getBillingAccountId()); - subscription.setSubscriptionMeasurements( - Map.of( - new SubscriptionMeasurementKey(MetricIdUtils.getCores().toString(), "HYPERVISOR"), - 10.0)); - subscriptionRepository.save(subscription); - } -} diff --git a/swatch-contracts/build.gradle b/swatch-contracts/build.gradle index 7c8fc2066a..de8cd033e9 100644 --- a/swatch-contracts/build.gradle +++ b/swatch-contracts/build.gradle @@ -42,8 +42,10 @@ dependencies { implementation project(':swatch-common-panache') implementation project(':swatch-common-smallrye-fault-tolerance') implementation project(':swatch-common-splunk') + implementation project(':swatch-common-export') implementation project(':clients:quarkus:rbac-client') implementation project(':clients:quarkus:product-client') + implementation project(':clients:quarkus:export-client') implementation project(':swatch-product-configuration') implementation project(':swatch-common-resteasy-client') implementation project(":clients:rh-partner-gateway-client") @@ -67,6 +69,8 @@ dependencies { testImplementation libraries["awaitility"] testImplementation("io.quarkus:quarkus-jdbc-h2") testImplementation("io.quarkus:quarkus-test-kafka-companion") + testImplementation 'org.testcontainers:postgresql' + testImplementation project(':swatch-common-testcontainers') testAnnotationProcessor libraries["mapstruct-processor"] // if you are using mapstruct in test code annotationProcessor libraries["lombok-mapstruct-binding"] @@ -106,7 +110,9 @@ openApiGenerate { } jsonSchema2Pojo { - source = files("${projectDir}/../swatch-core/schemas/enabled_orgs_request.yaml", "${projectDir}/../swatch-core/schemas/enabled_orgs_response.yaml") + source = files("${projectDir}/../swatch-core/schemas/enabled_orgs_request.yaml", + "${projectDir}/../swatch-core/schemas/enabled_orgs_response.yaml", + "${projectDir}/schemas") targetPackage = "com.redhat.swatch.contract.model" targetDirectory = file("${buildDir}/generated/src/gen/java") includeAdditionalProperties = false diff --git a/swatch-contracts/deploy/clowdapp.yaml b/swatch-contracts/deploy/clowdapp.yaml index ae252b88fc..1d997588d7 100644 --- a/swatch-contracts/deploy/clowdapp.yaml +++ b/swatch-contracts/deploy/clowdapp.yaml @@ -84,6 +84,10 @@ parameters: value: '3' - name: KAFKA_OFFERING_SYNC_PARTITIONS value: '3' + - name: KAFKA_SUBSCRIPTIONS_EXPORT_REPLICAS + value: '3' + - name: KAFKA_SUBSCRIPTIONS_EXPORT_PARTITIONS + value: '3' - name: CURL_CRON_IMAGE value: quay.io/app-sre/ubi8-ubi-minimal - name: CURL_CRON_IMAGE_TAG @@ -117,6 +121,7 @@ objects: dependencies: - swatch-tally - rbac + - export-service kafkaTopics: - replicas: ${{KAFKA_ENABLED_ORGS_REPLICAS}} @@ -131,6 +136,9 @@ objects: - replicas: ${{KAFKA_OFFERING_SYNC_REPLICAS}} partitions: ${{KAFKA_OFFERING_SYNC_PARTITIONS}} topicName: platform.rhsm-subscriptions.offering-sync + - replicas: ${{KAFKA_SUBSCRIPTIONS_EXPORT_REPLICAS}} + partitions: ${{KAFKA_SUBSCRIPTIONS_EXPORT_PARTITIONS}} + topicName: platform.export.requests # Creates a database if local mode, or uses RDS in production # database: @@ -244,6 +252,11 @@ objects: secretKeyRef: name: swatch-psks key: self + - name: SWATCH_EXPORT_PSK + valueFrom: + secretKeyRef: + name: export-psk + key: export_psk - name: QUARKUS_PROFILE value: ${QUARKUS_PROFILE} - name: KEYSTORE_PASSWORD @@ -384,6 +397,13 @@ objects: data: self: cGxhY2Vob2xkZXI= + - apiVersion: v1 + kind: Secret + metadata: + name: export-psk + data: + export_psk: dGVzdGluZy1hLXBzaw== + - apiVersion: v1 kind: ConfigMap metadata: diff --git a/swatch-core/schemas/subscriptions_export_csv_item.yaml b/swatch-contracts/schemas/subscriptions_export_csv_item.yaml similarity index 100% rename from swatch-core/schemas/subscriptions_export_csv_item.yaml rename to swatch-contracts/schemas/subscriptions_export_csv_item.yaml diff --git a/swatch-core/schemas/subscriptions_export_json.yaml b/swatch-contracts/schemas/subscriptions_export_json.yaml similarity index 100% rename from swatch-core/schemas/subscriptions_export_json.yaml rename to swatch-contracts/schemas/subscriptions_export_json.yaml diff --git a/swatch-core/schemas/subscriptions_export_json_item.yaml b/swatch-contracts/schemas/subscriptions_export_json_item.yaml similarity index 100% rename from swatch-core/schemas/subscriptions_export_json_item.yaml rename to swatch-contracts/schemas/subscriptions_export_json_item.yaml diff --git a/swatch-core/schemas/subscriptions_export_json_measurement.yaml b/swatch-contracts/schemas/subscriptions_export_json_measurement.yaml similarity index 100% rename from swatch-core/schemas/subscriptions_export_json_measurement.yaml rename to swatch-contracts/schemas/subscriptions_export_json_measurement.yaml diff --git a/swatch-contracts/src/main/java/com/redhat/swatch/contract/config/Channels.java b/swatch-contracts/src/main/java/com/redhat/swatch/contract/config/Channels.java index 5fcae4d416..3aed5371cf 100644 --- a/swatch-contracts/src/main/java/com/redhat/swatch/contract/config/Channels.java +++ b/swatch-contracts/src/main/java/com/redhat/swatch/contract/config/Channels.java @@ -30,6 +30,7 @@ public final class Channels { public static final String OFFERING_SYNC = "offering-sync"; public static final String OFFERING_SYNC_TASK_TOPIC = "offering-sync-task"; public static final String OFFERING_SYNC_TASK_UMB = "offering-sync-umb"; + public static final String EXPORT_REQUESTS_TOPIC = "export-requests"; private Channels() {} } diff --git a/swatch-contracts/src/main/java/com/redhat/swatch/contract/config/ExportConfiguration.java b/swatch-contracts/src/main/java/com/redhat/swatch/contract/config/ExportConfiguration.java new file mode 100644 index 0000000000..d6842aa120 --- /dev/null +++ b/swatch-contracts/src/main/java/com/redhat/swatch/contract/config/ExportConfiguration.java @@ -0,0 +1,83 @@ +/* + * Copyright Red Hat, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Red Hat trademarks are not licensed under GPLv3. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.swatch.contract.config; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.introspect.JacksonAnnotationIntrospector; +import com.fasterxml.jackson.databind.util.StdDateFormat; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fasterxml.jackson.module.jakarta.xmlbind.JakartaXmlBindAnnotationModule; +import com.redhat.cloud.event.parser.ConsoleCloudEventParser; +import com.redhat.swatch.contract.service.export.SubscriptionDataExporterService; +import com.redhat.swatch.export.CsvExportFileWriter; +import com.redhat.swatch.export.ExportRequestHandler; +import com.redhat.swatch.export.JsonExportFileWriter; +import com.redhat.swatch.export.api.ExportDelegate; +import com.redhat.swatch.export.api.RbacDelegate; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Produces; +import jakarta.enterprise.inject.Typed; +import java.util.List; + +@ApplicationScoped +public class ExportConfiguration { + + @ApplicationScoped + @Produces + ExportRequestHandler exportService( + ExportDelegate exportDelegate, + RbacDelegate rbacDelegate, + SubscriptionDataExporterService subscriptionDataExporterService, + ObjectMapper objectMapper, + CsvMapper csvMapper) { + return new ExportRequestHandler( + exportDelegate, + rbacDelegate, + List.of(subscriptionDataExporterService), + new ConsoleCloudEventParser(objectMapper), + new JsonExportFileWriter(objectMapper), + new CsvExportFileWriter(csvMapper)); + } + + @ApplicationScoped + @Produces + @Typed(CsvMapper.class) + CsvMapper csvMapper() { + CsvMapper csvMapper = new CsvMapper(); + csvMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + csvMapper.setDateFormat(new StdDateFormat().withColonInTimeZone(true)); + csvMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + csvMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + csvMapper.setAnnotationIntrospector(new JacksonAnnotationIntrospector()); + // Explicitly load the modules we need rather than use ObjectMapper.findAndRegisterModules in + // order to avoid com.fasterxml.jackson.module.scala.DefaultScalaModule, which was causing + // deserialization to ignore @JsonProperty on OpenApi classes. + csvMapper.registerModule(new JakartaXmlBindAnnotationModule()); + csvMapper.registerModule(new JavaTimeModule()); + csvMapper.registerModule(new Jdk8Module()); + return csvMapper; + } +} diff --git a/swatch-contracts/src/main/java/com/redhat/swatch/contract/config/RbacConfiguration.java b/swatch-contracts/src/main/java/com/redhat/swatch/contract/config/RbacConfiguration.java new file mode 100644 index 0000000000..7085848645 --- /dev/null +++ b/swatch-contracts/src/main/java/com/redhat/swatch/contract/config/RbacConfiguration.java @@ -0,0 +1,36 @@ +/* + * Copyright Red Hat, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Red Hat trademarks are not licensed under GPLv3. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.swatch.contract.config; + +import com.redhat.swatch.clients.rbac.RbacApi; +import com.redhat.swatch.clients.rbac.RbacClient; +import com.redhat.swatch.clients.rbac.RbacService; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Produces; + +@ApplicationScoped +public class RbacConfiguration { + @ApplicationScoped + @Produces + RbacService rbacService(@RbacClient RbacApi rbacApi) { + return new RbacService(rbacApi); + } +} diff --git a/swatch-contracts/src/main/java/com/redhat/swatch/contract/repository/SubscriptionCapacityViewRepository.java b/swatch-contracts/src/main/java/com/redhat/swatch/contract/repository/SubscriptionCapacityViewRepository.java index fb9a17419c..0b6c6fa290 100644 --- a/swatch-contracts/src/main/java/com/redhat/swatch/contract/repository/SubscriptionCapacityViewRepository.java +++ b/swatch-contracts/src/main/java/com/redhat/swatch/contract/repository/SubscriptionCapacityViewRepository.java @@ -38,15 +38,15 @@ public class SubscriptionCapacityViewRepository implements PanacheSpecificationSupport { - static final String PHYSICAL = "PHYSICAL"; - static final String HYPERVISOR = "HYPERVISOR"; + public static final String PHYSICAL = "PHYSICAL"; + public static final String HYPERVISOR = "HYPERVISOR"; public Stream streamBy( Specification criteria) { return find(SubscriptionCapacityView.class, criteria).stream(); } - static Specification buildSearchSpecification(String orgId) { + public static Specification buildSearchSpecification(String orgId) { return buildSearchSpecification(orgId, null, null, null, null, null, null, null); } @@ -102,40 +102,40 @@ public static Specification buildSearchSpecification( return searchCriteria; } - static Specification productIdEquals(ProductId productId) { + public static Specification productIdEquals(ProductId productId) { return (root, query, builder) -> builder.equal(root.get(SubscriptionCapacityView_.productTag), productId.getValue()); } - static Specification hasBillingProviderId() { + public static Specification hasBillingProviderId() { return (root, query, builder) -> builder.and( builder.isNotNull(root.get(SubscriptionCapacityView_.billingProviderId)), builder.notEqual(root.get(SubscriptionCapacityView_.billingProviderId), "")); } - static Specification billingAccountIdStartsWith(String value) { + public static Specification billingAccountIdStartsWith(String value) { return (root, query, builder) -> builder.like(root.get(SubscriptionCapacityView_.billingAccountId), value + "%"); } - static Specification slaEquals(ServiceLevel sla) { + public static Specification slaEquals(ServiceLevel sla) { return (root, query, builder) -> builder.equal(root.get(SubscriptionCapacityView_.serviceLevel), sla); } - static Specification usageEquals(Usage usage) { + public static Specification usageEquals(Usage usage) { return (root, query, builder) -> builder.equal(root.get(SubscriptionCapacityView_.usage), usage); } - static Specification billingProviderEquals( + public static Specification billingProviderEquals( BillingProvider billingProvider) { return (root, query, builder) -> builder.equal(root.get(SubscriptionCapacityView_.billingProvider), billingProvider); } - static Specification hasCategory(ReportCategory value) { + public static Specification hasCategory(ReportCategory value) { String measurementType = getMeasurementTypeFromCategory(value); if (measurementType != null) { return handleMetricsFilter("measurement_type", measurementType); @@ -144,11 +144,11 @@ static Specification hasCategory(ReportCategory value) return null; } - static Specification hasMetricId(String value) { + public static Specification hasMetricId(String value) { return handleMetricsFilter("metric_id", MetricId.fromString(value).toString()); } - static String getMeasurementTypeFromCategory(ReportCategory value) { + public static String getMeasurementTypeFromCategory(ReportCategory value) { var category = HypervisorReportCategory.mapCategory(value); if (category != null) { return switch (category) { @@ -159,7 +159,7 @@ static String getMeasurementTypeFromCategory(ReportCategory value) { return null; } - static Specification orgIdEquals(String orgId) { + public static Specification orgIdEquals(String orgId) { return (root, query, builder) -> builder.equal(root.get(SubscriptionCapacityView_.orgId), orgId); } diff --git a/swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/ExportDelegateImpl.java b/swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/ExportDelegateImpl.java new file mode 100644 index 0000000000..4a591d7b20 --- /dev/null +++ b/swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/ExportDelegateImpl.java @@ -0,0 +1,72 @@ +/* + * Copyright Red Hat, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Red Hat trademarks are not licensed under GPLv3. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.swatch.contract.service.export; + +import static com.redhat.swatch.export.ExportRequestHandler.INTERNAL_ERROR; + +import com.redhat.swatch.clients.export.api.model.DownloadExportErrorRequest; +import com.redhat.swatch.clients.export.api.resources.ApiException; +import com.redhat.swatch.clients.export.api.resources.ExportApi; +import com.redhat.swatch.export.ExportServiceException; +import com.redhat.swatch.export.ExportServiceRequest; +import com.redhat.swatch.export.api.ExportDelegate; +import jakarta.enterprise.context.ApplicationScoped; +import java.io.File; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.microprofile.rest.client.inject.RestClient; + +@Slf4j +@ApplicationScoped +public class ExportDelegateImpl implements ExportDelegate { + @RestClient ExportApi exportApi; + + @Override + public void sendExportError(ExportServiceRequest request, Integer error, String message) + throws ExportServiceException { + try { + exportApi.downloadExportError( + request.getExportRequestUUID(), + request.getApplication(), + request.getRequest().getUUID(), + new DownloadExportErrorRequest().error(error).message(message)); + } catch (ApiException e) { + log.error( + "Error marking the export as error for request {}", request.getExportRequestUUID(), e); + throw new ExportServiceException( + INTERNAL_ERROR, "Error marking the export as error with message: " + e.getMessage(), e); + } + } + + @Override + public void upload(File file, ExportServiceRequest request) throws ExportServiceException { + try { + exportApi.downloadExportUpload( + request.getExportRequestUUID(), + request.getApplication(), + request.getRequest().getUUID(), + file); + } catch (ApiException e) { + log.error("Error sending the upload for request {}", request.getExportRequestUUID(), e); + throw new ExportServiceException( + INTERNAL_ERROR, "Error sending upload request with message: " + e.getMessage(), e); + } + } +} diff --git a/swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/ExportRequestConsumer.java b/swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/ExportRequestConsumer.java new file mode 100644 index 0000000000..8416c5b3e5 --- /dev/null +++ b/swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/ExportRequestConsumer.java @@ -0,0 +1,47 @@ +/* + * Copyright Red Hat, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Red Hat trademarks are not licensed under GPLv3. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.swatch.contract.service.export; + +import static com.redhat.swatch.contract.config.Channels.EXPORT_REQUESTS_TOPIC; + +import com.redhat.swatch.export.ExportRequestHandler; +import com.redhat.swatch.export.ExportServiceException; +import io.micrometer.core.annotation.Timed; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.transaction.Transactional; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.microprofile.reactive.messaging.Incoming; + +@ApplicationScoped +@Slf4j +@AllArgsConstructor +public class ExportRequestConsumer { + + private final ExportRequestHandler exportService; + + @Timed("rhsm-subscriptions.exports.upload") + @Transactional + @Incoming(EXPORT_REQUESTS_TOPIC) + public void receive(String exportEvent) throws ExportServiceException { + exportService.handle(exportEvent); + } +} diff --git a/swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/RbacDelegateImpl.java b/swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/RbacDelegateImpl.java new file mode 100644 index 0000000000..d93be41969 --- /dev/null +++ b/swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/RbacDelegateImpl.java @@ -0,0 +1,47 @@ +/* + * Copyright Red Hat, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Red Hat trademarks are not licensed under GPLv3. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.swatch.contract.service.export; + +import com.redhat.swatch.clients.rbac.RbacApiException; +import com.redhat.swatch.clients.rbac.RbacService; +import com.redhat.swatch.export.ExportServiceException; +import com.redhat.swatch.export.api.RbacDelegate; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.ws.rs.core.Response; +import java.util.List; +import lombok.AllArgsConstructor; + +@AllArgsConstructor +@ApplicationScoped +public class RbacDelegateImpl implements RbacDelegate { + private final RbacService rbacService; + + @Override + public List getPermissions(String application, String xRhIdentity) + throws ExportServiceException { + try { + return rbacService.getPermissions(application, xRhIdentity); + } catch (RbacApiException e) { + throw new ExportServiceException( + Response.Status.NOT_FOUND.getStatusCode(), e.getMessage(), e); + } + } +} diff --git a/src/main/java/org/candlepin/subscriptions/subscription/export/SubscriptionCsvDataMapperService.java b/swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/SubscriptionCsvDataMapperService.java similarity index 79% rename from src/main/java/org/candlepin/subscriptions/subscription/export/SubscriptionCsvDataMapperService.java rename to swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/SubscriptionCsvDataMapperService.java index 8a40662033..522dfbd99c 100644 --- a/src/main/java/org/candlepin/subscriptions/subscription/export/SubscriptionCsvDataMapperService.java +++ b/swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/SubscriptionCsvDataMapperService.java @@ -18,24 +18,22 @@ * granted to use or replicate Red Hat trademarks that are incorporated * in this software or its documentation. */ -package org.candlepin.subscriptions.subscription.export; - -import static org.candlepin.subscriptions.db.SubscriptionCapacityViewRepository.PHYSICAL; -import static org.candlepin.subscriptions.subscription.export.SubscriptionDataExporterService.groupMetrics; +package com.redhat.swatch.contract.service.export; +import com.redhat.swatch.contract.model.SubscriptionsExportCsvItem; +import com.redhat.swatch.contract.repository.ServiceLevel; +import com.redhat.swatch.contract.repository.SubscriptionCapacityView; +import com.redhat.swatch.contract.repository.SubscriptionCapacityViewRepository; +import com.redhat.swatch.contract.repository.Usage; +import com.redhat.swatch.contract.resource.api.v1.ApiModelMapperV1; import com.redhat.swatch.export.DataMapperService; import com.redhat.swatch.export.ExportServiceRequest; +import jakarta.enterprise.context.ApplicationScoped; import java.util.List; import java.util.Optional; import lombok.AllArgsConstructor; -import org.candlepin.subscriptions.db.model.ServiceLevel; -import org.candlepin.subscriptions.db.model.SubscriptionCapacityView; -import org.candlepin.subscriptions.db.model.Usage; -import org.candlepin.subscriptions.json.SubscriptionsExportCsvItem; -import org.candlepin.subscriptions.util.ApiModelMapperV1; -import org.springframework.stereotype.Service; -@Service +@ApplicationScoped @AllArgsConstructor public class SubscriptionCsvDataMapperService implements DataMapperService { @@ -48,7 +46,7 @@ public List mapDataItem(SubscriptionCapacityView dataItem, ExportService return List.of(); } - return groupMetrics(mapper, dataItem, request).stream() + return SubscriptionDataExporterService.groupMetrics(mapper, dataItem, request).stream() .map( m -> { var item = new SubscriptionsExportCsvItem(); @@ -58,7 +56,7 @@ public List mapDataItem(SubscriptionCapacityView dataItem, ExportService item.setEnd(dataItem.getEndDate()); item.setQuantity((double) dataItem.getQuantity()); item.setMetricId(m.getMetricId()); - if (PHYSICAL.equals(m.getMeasurementType())) { + if (SubscriptionCapacityViewRepository.PHYSICAL.equals(m.getMeasurementType())) { item.setHypervisorCapacity(m.getCapacity()); } else { item.setCapacity(m.getCapacity()); diff --git a/src/main/java/org/candlepin/subscriptions/subscription/export/SubscriptionDataExporterService.java b/swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/SubscriptionDataExporterService.java similarity index 78% rename from src/main/java/org/candlepin/subscriptions/subscription/export/SubscriptionDataExporterService.java rename to swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/SubscriptionDataExporterService.java index 227d91765d..935b11f184 100644 --- a/src/main/java/org/candlepin/subscriptions/subscription/export/SubscriptionDataExporterService.java +++ b/swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/SubscriptionDataExporterService.java @@ -18,25 +18,27 @@ * granted to use or replicate Red Hat trademarks that are incorporated * in this software or its documentation. */ -package org.candlepin.subscriptions.subscription.export; - -import static org.candlepin.subscriptions.db.SubscriptionCapacityViewRepository.billingAccountIdStartsWith; -import static org.candlepin.subscriptions.db.SubscriptionCapacityViewRepository.billingProviderEquals; -import static org.candlepin.subscriptions.db.SubscriptionCapacityViewRepository.buildSearchSpecification; -import static org.candlepin.subscriptions.db.SubscriptionCapacityViewRepository.getMeasurementTypeFromCategory; -import static org.candlepin.subscriptions.db.SubscriptionCapacityViewRepository.hasCategory; -import static org.candlepin.subscriptions.db.SubscriptionCapacityViewRepository.hasMetricId; -import static org.candlepin.subscriptions.db.SubscriptionCapacityViewRepository.productIdEquals; -import static org.candlepin.subscriptions.db.SubscriptionCapacityViewRepository.slaEquals; -import static org.candlepin.subscriptions.db.SubscriptionCapacityViewRepository.usageEquals; -import static org.candlepin.subscriptions.resource.ResourceUtils.ANY; +package com.redhat.swatch.contract.service.export; import com.redhat.swatch.configuration.registry.MetricId; import com.redhat.swatch.configuration.registry.ProductId; +import com.redhat.swatch.contract.model.SubscriptionsExportJsonMeasurement; +import com.redhat.swatch.contract.openapi.model.ReportCategory; +import com.redhat.swatch.contract.repository.BillingProvider; +import com.redhat.swatch.contract.repository.ServiceLevel; +import com.redhat.swatch.contract.repository.SubscriptionCapacityView; +import com.redhat.swatch.contract.repository.SubscriptionCapacityViewMetric; +import com.redhat.swatch.contract.repository.SubscriptionCapacityViewRepository; +import com.redhat.swatch.contract.repository.Usage; +import com.redhat.swatch.contract.resource.ResourceUtils; +// NOTE(khowell): this couples our export implementation to the v1 REST API +import com.redhat.swatch.contract.resource.api.v1.ApiModelMapperV1; import com.redhat.swatch.export.DataExporterService; import com.redhat.swatch.export.DataMapperService; import com.redhat.swatch.export.ExportServiceException; import com.redhat.swatch.export.ExportServiceRequest; +import com.redhat.swatch.panache.Specification; +import jakarta.enterprise.context.ApplicationScoped; import jakarta.ws.rs.core.Response; import java.util.ArrayList; import java.util.HashMap; @@ -50,23 +52,9 @@ import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; -import org.candlepin.subscriptions.db.SubscriptionCapacityViewRepository; -import org.candlepin.subscriptions.db.model.BillingProvider; -import org.candlepin.subscriptions.db.model.ServiceLevel; -import org.candlepin.subscriptions.db.model.SubscriptionCapacityView; -import org.candlepin.subscriptions.db.model.SubscriptionCapacityViewMetric; -import org.candlepin.subscriptions.db.model.Usage; -import org.candlepin.subscriptions.json.SubscriptionsExportJsonMeasurement; -// NOTE(khowell): this couples our export implementation to the v1 REST API -import org.candlepin.subscriptions.util.ApiModelMapperV1; -import org.candlepin.subscriptions.utilization.api.v1.model.ReportCategory; -import org.springframework.context.annotation.Profile; -import org.springframework.data.jpa.domain.Specification; -import org.springframework.stereotype.Component; @Slf4j -@Component -@Profile("capacity-ingress") +@ApplicationScoped @AllArgsConstructor public class SubscriptionDataExporterService implements DataExporterService { @@ -117,7 +105,8 @@ public DataMapperService getMapper(ExportServiceReques private Specification extractExportFilter( ExportServiceRequest request) { - Specification criteria = buildSearchSpecification(request.getOrgId()); + Specification criteria = + SubscriptionCapacityViewRepository.buildSearchSpecification(request.getOrgId()); if (request.getFilters() != null) { var requestedFilters = request.getFilters().entrySet(); try { @@ -144,14 +133,14 @@ private Specification extractExportFilter( } private static Specification handleProductIdFilter(String value) { - return productIdEquals(ProductId.fromString(value)); + return SubscriptionCapacityViewRepository.productIdEquals(ProductId.fromString(value)); } private static Specification handleUsageFilter(String value) { Usage usage = Usage.fromString(value); if (value.equalsIgnoreCase(usage.getValue())) { if (!Usage._ANY.equals(usage)) { - return usageEquals(usage); + return SubscriptionCapacityViewRepository.usageEquals(usage); } } else { throw new IllegalArgumentException(String.format("usage: %s not supported", value)); @@ -164,7 +153,7 @@ private static Specification handleSlaFilter(String va ServiceLevel serviceLevel = ServiceLevel.fromString(value); if (value.equalsIgnoreCase(serviceLevel.getValue())) { if (!ServiceLevel._ANY.equals(serviceLevel)) { - return slaEquals(serviceLevel); + return SubscriptionCapacityViewRepository.slaEquals(serviceLevel); } } else { throw new IllegalArgumentException(String.format("sla: %s not supported", value)); @@ -174,18 +163,19 @@ private static Specification handleSlaFilter(String va } private Specification handleCategoryFilter(String value) { - return hasCategory(mapper.map(ReportCategory.fromValue(value))); + return SubscriptionCapacityViewRepository.hasCategory( + mapper.map(ReportCategory.fromValue(value))); } private static Specification handleMetricIdFilter(String value) { - return hasMetricId(MetricId.fromString(value).toString()); + return SubscriptionCapacityViewRepository.hasMetricId(MetricId.fromString(value).toString()); } private static Specification handleBillingProviderFilter(String value) { BillingProvider billingProvider = BillingProvider.fromString(value); - if (value.equalsIgnoreCase(billingProvider.getValue())) { + if (billingProvider != null && value.equalsIgnoreCase(billingProvider.getValue())) { if (!BillingProvider._ANY.equals(billingProvider)) { - return billingProviderEquals(billingProvider); + return SubscriptionCapacityViewRepository.billingProviderEquals(billingProvider); } } else { throw new IllegalArgumentException( @@ -197,8 +187,8 @@ private static Specification handleBillingProviderFilt private static Specification handleBillingAccountIdFilter( String value) { - if (!ANY.equalsIgnoreCase(value)) { - return billingAccountIdStartsWith(value); + if (!ResourceUtils.ANY.equalsIgnoreCase(value)) { + return SubscriptionCapacityViewRepository.billingAccountIdStartsWith(value); } return null; @@ -248,7 +238,8 @@ private static String getMeasurementTypeFilter( if (request != null && request.getFilters() != null && request.getFilters().get(CATEGORY) instanceof String value) { - return getMeasurementTypeFromCategory(mapper.map(ReportCategory.fromValue(value))); + return SubscriptionCapacityViewRepository.getMeasurementTypeFromCategory( + mapper.map(ReportCategory.fromValue(value))); } return null; diff --git a/src/main/java/org/candlepin/subscriptions/subscription/export/SubscriptionJsonDataMapperService.java b/swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/SubscriptionJsonDataMapperService.java similarity index 81% rename from src/main/java/org/candlepin/subscriptions/subscription/export/SubscriptionJsonDataMapperService.java rename to swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/SubscriptionJsonDataMapperService.java index 4da31d1902..869f1f5daf 100644 --- a/src/main/java/org/candlepin/subscriptions/subscription/export/SubscriptionJsonDataMapperService.java +++ b/swatch-contracts/src/main/java/com/redhat/swatch/contract/service/export/SubscriptionJsonDataMapperService.java @@ -18,24 +18,24 @@ * granted to use or replicate Red Hat trademarks that are incorporated * in this software or its documentation. */ -package org.candlepin.subscriptions.subscription.export; +package com.redhat.swatch.contract.service.export; -import static org.candlepin.subscriptions.subscription.export.SubscriptionDataExporterService.groupMetrics; +import static com.redhat.swatch.contract.service.export.SubscriptionDataExporterService.groupMetrics; +import com.redhat.swatch.contract.model.SubscriptionsExportJsonItem; +import com.redhat.swatch.contract.repository.ServiceLevel; +import com.redhat.swatch.contract.repository.SubscriptionCapacityView; +import com.redhat.swatch.contract.repository.Usage; +import com.redhat.swatch.contract.resource.api.v1.ApiModelMapperV1; import com.redhat.swatch.export.DataMapperService; import com.redhat.swatch.export.ExportServiceRequest; +import jakarta.enterprise.context.ApplicationScoped; import java.util.ArrayList; import java.util.List; import java.util.Optional; import lombok.AllArgsConstructor; -import org.candlepin.subscriptions.db.model.ServiceLevel; -import org.candlepin.subscriptions.db.model.SubscriptionCapacityView; -import org.candlepin.subscriptions.db.model.Usage; -import org.candlepin.subscriptions.json.SubscriptionsExportJsonItem; -import org.candlepin.subscriptions.util.ApiModelMapperV1; -import org.springframework.stereotype.Service; -@Service +@ApplicationScoped @AllArgsConstructor public class SubscriptionJsonDataMapperService implements DataMapperService { diff --git a/swatch-contracts/src/main/resources/application.properties b/swatch-contracts/src/main/resources/application.properties index cf1e931b02..b326e33769 100644 --- a/swatch-contracts/src/main/resources/application.properties +++ b/swatch-contracts/src/main/resources/application.properties @@ -217,6 +217,12 @@ quarkus.rest-client."com.redhat.swatch.clients.rbac.api.resources.AccessApi".tru quarkus.rest-client."com.redhat.swatch.clients.rbac.api.resources.AccessApi".trust-store-password=${clowder.endpoints.rbac-service.trust-store-password} quarkus.rest-client."com.redhat.swatch.clients.rbac.api.resources.AccessApi".trust-store-type=${clowder.endpoints.rbac-service.trust-store-type} +quarkus.rest-client."com.redhat.swatch.clients.export.api.resources.ExportApi".url=${clowder.privateEndpoints.export-service-service.url:http://localhost:10000} +quarkus.rest-client."com.redhat.swatch.clients.export.api.resources.ExportApi".trust-store=${clowder.privateEndpoints.export-service-service.trust-store-path} +quarkus.rest-client."com.redhat.swatch.clients.export.api.resources.ExportApi".trust-store-password=${clowder.privateEndpoints.export-service-service.trust-store-password} +quarkus.rest-client."com.redhat.swatch.clients.export.api.resources.ExportApi".trust-store-type=${clowder.privateEndpoints.export-service-service.trust-store-type} +quarkus.rest-client."com.redhat.swatch.clients.export.api.resources.ExportApi".headers."x-rh-exports-psk"=${SWATCH_EXPORT_PSK:placeholder} + # avoid duplicate/wrong enum names and other problems by preventing generated interfaces from being # added back into the API spec mp.openapi.scan.exclude.packages=com.redhat.swatch.contract.openapi @@ -338,6 +344,11 @@ mp.messaging.outgoing.offering-sync.topic=platform.rhsm-subscriptions.offering-s mp.messaging.outgoing.offering-sync.group.id=offering-worker mp.messaging.outgoing.offering-sync.value.serializer=io.quarkus.kafka.client.serialization.ObjectMapperSerializer +mp.messaging.incoming.export-requests.connector=smallrye-kafka +mp.messaging.incoming.export-requests.topic=platform.export.requests +mp.messaging.incoming.export-requests.group.id=swatch-subscription-export +mp.messaging.incoming.export-requests.failure-strategy=ignore + mp.messaging.incoming.contracts.connector=smallrye-in-memory mp.messaging.outgoing.contractstest.connector=smallrye-in-memory %ephemeral.mp.messaging.incoming.contracts.connector=smallrye-amqp diff --git a/swatch-contracts/src/main/resources/db/202410171500-move-subscription-capacity-view_h2_only.xml b/swatch-contracts/src/main/resources/db/202410171500-move-subscription-capacity-view_h2_only.xml new file mode 100644 index 0000000000..d347442570 --- /dev/null +++ b/swatch-contracts/src/main/resources/db/202410171500-move-subscription-capacity-view_h2_only.xml @@ -0,0 +1,71 @@ + + + + + Move subscription_capacity_view ownership to swatch-contracts + + + + = now()) + group by s.subscription_id, s.subscription_number, s.sku, o.has_unlimited_usage, o.description, o.sla, o.usage , s.org_id, s.billing_provider, s.billing_provider_id, s.billing_account_id, spt.product_tag,s.quantity, sm.metric_id, sm.value, sm.measurement_type, s.start_date, s.end_date + order by s.subscription_id asc + ]]> + + + + = now()) + group by s.subscription_id, s.subscription_number, s.sku, o.has_unlimited_usage, o.description, o.sla, o.usage , s.org_id, s.billing_provider, s.billing_provider_id, s.billing_account_id, spt.product_tag,s.quantity, sm.metric_id, sm.value, sm.measurement_type, s.start_date, s.end_date + order by s.subscription_id asc + ]]> + + + + diff --git a/swatch-contracts/src/main/resources/db/changeLog.xml b/swatch-contracts/src/main/resources/db/changeLog.xml index 0a49659d2e..40ca6f885e 100644 --- a/swatch-contracts/src/main/resources/db/changeLog.xml +++ b/swatch-contracts/src/main/resources/db/changeLog.xml @@ -19,4 +19,5 @@ + diff --git a/swatch-contracts/src/test/java/com/redhat/swatch/contract/service/export/SubscriptionDataExporterServiceTest.java b/swatch-contracts/src/test/java/com/redhat/swatch/contract/service/export/SubscriptionDataExporterServiceTest.java new file mode 100644 index 0000000000..51b94165fe --- /dev/null +++ b/swatch-contracts/src/test/java/com/redhat/swatch/contract/service/export/SubscriptionDataExporterServiceTest.java @@ -0,0 +1,440 @@ +/* + * Copyright Red Hat, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Red Hat trademarks are not licensed under GPLv3. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.swatch.contract.service.export; + +import static com.redhat.swatch.contract.service.export.SubscriptionDataExporterService.PRODUCT_ID; +import static com.redhat.swatch.export.ExportRequestHandler.ADMIN_ROLE; +import static com.redhat.swatch.export.ExportRequestHandler.MISSING_PERMISSIONS; +import static com.redhat.swatch.export.ExportRequestHandler.SWATCH_APP; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.redhat.cloud.event.apps.exportservice.v1.Format; +import com.redhat.cloud.event.apps.exportservice.v1.ResourceRequest; +import com.redhat.cloud.event.apps.exportservice.v1.ResourceRequestClass; +import com.redhat.cloud.event.parser.ConsoleCloudEventParser; +import com.redhat.cloud.event.parser.GenericConsoleCloudEvent; +import com.redhat.swatch.clients.rbac.RbacApiException; +import com.redhat.swatch.clients.rbac.RbacService; +import com.redhat.swatch.configuration.util.MetricIdUtils; +import com.redhat.swatch.contract.config.Channels; +import com.redhat.swatch.contract.model.SubscriptionsExportCsvItem; +import com.redhat.swatch.contract.model.SubscriptionsExportJson; +import com.redhat.swatch.contract.repository.BillingProvider; +import com.redhat.swatch.contract.repository.OfferingEntity; +import com.redhat.swatch.contract.repository.OfferingRepository; +import com.redhat.swatch.contract.repository.ServiceLevel; +import com.redhat.swatch.contract.repository.SubscriptionCapacityViewRepository; +import com.redhat.swatch.contract.repository.SubscriptionEntity; +import com.redhat.swatch.contract.repository.SubscriptionMeasurementKey; +import com.redhat.swatch.contract.repository.SubscriptionRepository; +import com.redhat.swatch.contract.repository.Usage; +import com.redhat.swatch.contract.test.resources.ExportServiceWireMockResource; +import com.redhat.swatch.contract.test.resources.InMemoryMessageBrokerKafkaResource; +import com.redhat.swatch.contract.test.resources.InjectWireMock; +import io.quarkus.test.InjectMock; +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.junit.QuarkusTest; +import io.smallrye.reactive.messaging.memory.InMemoryConnector; +import io.smallrye.reactive.messaging.memory.InMemorySource; +import jakarta.enterprise.inject.Any; +import jakarta.inject.Inject; +import jakarta.transaction.Transactional; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.ValueSource; + +@QuarkusTest +@QuarkusTestResource(ExportServiceWireMockResource.class) +@QuarkusTestResource(InMemoryMessageBrokerKafkaResource.class) +class SubscriptionDataExporterServiceTest { + + private static final String RHEL_FOR_X86 = "RHEL for x86"; + private static final String ROSA = "rosa"; + private static final String ORG_ID = "13259775"; + + @Inject SubscriptionRepository subscriptionRepository; + @Inject SubscriptionCapacityViewRepository subscriptionCapacityViewRepository; + @Inject SubscriptionCsvDataMapperService csvDataMapperService; + @Inject SubscriptionJsonDataMapperService jsonDataMapperService; + @Inject ObjectMapper objectMapper; + @Inject CsvMapper csvMapper; + @Inject OfferingRepository offeringRepository; + @InjectMock RbacService rbacService; + @InjectWireMock ExportServiceWireMockResource wireMockResource; + @Inject @Any InMemoryConnector connector; + + private ConsoleCloudEventParser parser; + private OfferingEntity offering; + private GenericConsoleCloudEvent request; + private InMemorySource exportChannel; + + @Transactional + @BeforeEach + public void setup() { + exportChannel = connector.source(Channels.EXPORT_REQUESTS_TOPIC); + parser = new ConsoleCloudEventParser(objectMapper); + + offering = new OfferingEntity(); + offering.setSku("MKU001"); + offering.setUsage(Usage.PRODUCTION); + offering.setServiceLevel(ServiceLevel.PREMIUM); + offeringRepository.persist(offering); + } + + @Transactional + @AfterEach + public void tearDown() { + subscriptionRepository.deleteAll(); + } + + @Test + void testRequestWithoutPermissions() { + givenExportRequestWithoutPermissions(); + whenReceiveExportRequest(); + verifyRequestWasSentToExportServiceWithError(request, MISSING_PERMISSIONS); + } + + @Test + void testRequestWithPermissions() { + givenExportRequestWithPermissions(Format.JSON); + whenReceiveExportRequest(); + verifyRequestWasSentToExportService(); + } + + @Test + void testRequestShouldBeUploadedWithSubscriptionsAsJson() { + givenSubscriptionWithMeasurement(RHEL_FOR_X86); + givenExportRequestWithPermissions(Format.JSON); + whenReceiveExportRequest(); + verifyRequestWasSentToExportService(); + } + + @Disabled( + value = + """ + This test needs a real PostgreSQL instance with all the migrations executed: the ones from the + monolith and from the swatch-contracts service. + This should be addressed after SWATCH-2820. + """) + @Test + void testRequestShouldBeUploadedWithSubscriptionsUsingPrometheusEnabledProductAsJson() { + givenSubscriptionWithMeasurement(ROSA); + givenExportRequestWithPermissions(Format.JSON); + givenFilterInExportRequest(PRODUCT_ID, ROSA); + whenReceiveExportRequest(); + verifyRequestWasSentToExportService(); + } + + @Test + void testRequestShouldBeUploadedWithSubscriptionsAsCsv() { + givenSubscriptionWithMeasurement(RHEL_FOR_X86); + givenExportRequestWithPermissions(Format.CSV); + whenReceiveExportRequest(); + verifyRequestWasSentToExportService(); + } + + @Disabled( + value = + """ + This test needs a real PostgreSQL instance with all the migrations executed: the ones from the + monolith and from the swatch-contracts service. + This should be addressed after SWATCH-2820. + """) + @ParameterizedTest + @EnumSource(value = Format.class) + void testGivenDuplicateSubscriptionsThenItReturnsOnlyOneRecordAndCapacityIsSum(Format format) { + givenSubscriptionWithMeasurement(RHEL_FOR_X86); + givenSameSubscriptionWithOtherMeasurement(); + givenExportRequestWithPermissions(format); + whenReceiveExportRequest(); + verifyRequestWasSentToExportService(); + } + + @ParameterizedTest + @CsvSource( + value = { + PRODUCT_ID + "," + RHEL_FOR_X86, + "usage,production", + // Uncommented after SWATCH-2820 is done + // "category,hypervisor", + "sla,premium", + // Uncommented after SWATCH-2820 is done + // "metric_id,Cores", + "billing_provider,aws", + "billing_account_id,123" + }) + void testFiltersFoundData(String filterName, String exists) { + givenSubscriptionWithMeasurement(RHEL_FOR_X86); + givenExportRequestWithPermissions(Format.JSON); + givenFilterInExportRequest(filterName, exists); + whenReceiveExportRequest(); + verifyRequestWasSentToExportService(); + verifyNoRequestsWereSentToExportServiceWithError(); + } + + @ParameterizedTest + @CsvSource( + value = { + PRODUCT_ID + "," + ROSA, + "usage,disaster recovery", + // Uncommented after SWATCH-2820 is done + // "category,cloud", + "sla,standard", + // Uncommented after SWATCH-2820 is done + // "metric_id,Sockets", + "billing_provider,azure", + "billing_account_id,345" + }) + void testFiltersDoesNotFoundDataAndReportIsEmpty(String filterName, String doesNotExist) { + givenSubscriptionWithMeasurement(RHEL_FOR_X86); + givenExportRequestWithPermissions(Format.JSON); + givenFilterInExportRequest(filterName, doesNotExist); + whenReceiveExportRequest(); + verifyRequestWasSentToExportServiceWithNoDataFound(); + verifyNoRequestsWereSentToExportServiceWithError(); + } + + @ParameterizedTest + @ValueSource(strings = {PRODUCT_ID, "usage", "category", "sla", "metric_id", "billing_provider"}) + void testFiltersAreInvalid(String filterName) { + givenSubscriptionWithMeasurement(RHEL_FOR_X86); + givenExportRequestWithPermissions(Format.JSON); + givenFilterInExportRequest(filterName, "wrong!"); + whenReceiveExportRequest(); + verifyNoRequestsWereSentToExportServiceWithUploadData(); + verifyRequestWasSentToExportServiceWithError(request); + } + + @Test + void testRequestShouldFilterByOrgId() { + givenSubscriptionWithMeasurementForAnotherOrgId(RHEL_FOR_X86); + givenSubscriptionWithMeasurement(RHEL_FOR_X86); + givenExportRequestWithPermissions(Format.JSON); + whenReceiveExportRequest(); + verifyRequestWasSentToExportService(); + } + + private void verifyRequestWasSentToExportService() { + boolean isCsvFormat = request.getData().getResourceRequest().getFormat() == Format.CSV; + List data = new ArrayList<>(); + subscriptionCapacityViewRepository + .streamBy(SubscriptionCapacityViewRepository.orgIdEquals(ORG_ID)) + .forEach( + subscription -> { + if (isCsvFormat) { + data.addAll(csvDataMapperService.mapDataItem(subscription, null)); + } else { + data.addAll(jsonDataMapperService.mapDataItem(subscription, null)); + } + }); + + if (isCsvFormat) { + verifyRequestWasSentToExportServiceWithUploadCsvData(data); + } else { + verifyRequestWasSentToExportServiceWithUploadJsonData( + new SubscriptionsExportJson().withData(data)); + } + } + + private void verifyRequestWasSentToExportServiceWithUploadCsvData(List data) { + verifyRequestWasSentToExportServiceWithUploadData( + request, toCsv(data, SubscriptionsExportCsvItem.class)); + } + + private void verifyRequestWasSentToExportServiceWithUploadJsonData(SubscriptionsExportJson data) { + verifyRequestWasSentToExportServiceWithUploadData(request, toJson(data)); + } + + private void givenSubscriptionWithMeasurementForAnotherOrgId(String productId) { + givenSubscriptionWithMeasurement(UUID.randomUUID().toString(), productId); + } + + private void givenSubscriptionWithMeasurement(String productId) { + givenSubscriptionWithMeasurement(ORG_ID, productId); + } + + @Transactional + void givenSubscriptionWithMeasurement(String orgId, String productId) { + SubscriptionEntity subscription = new SubscriptionEntity(); + subscription.setSubscriptionId(UUID.randomUUID().toString()); + subscription.setSubscriptionNumber(UUID.randomUUID().toString()); + subscription.setStartDate(OffsetDateTime.parse("2024-04-23T11:48:15.888129Z")); + subscription.setEndDate(OffsetDateTime.parse("2028-05-23T11:48:15.888129Z")); + subscription.setOffering(offering); + subscription.setOrgId(orgId); + subscription.setBillingProvider(BillingProvider.AWS); + offering.getProductTags().clear(); + offering.getProductTags().add(productId); + updateOffering(); + subscription.setBillingAccountId("123"); + subscription.setSubscriptionMeasurements( + Map.of( + new SubscriptionMeasurementKey(MetricIdUtils.getCores().toString(), "HYPERVISOR"), + 5.0)); + subscriptionRepository.persist(subscription); + } + + @Transactional + void givenSameSubscriptionWithOtherMeasurement() { + var subscriptions = subscriptionRepository.listAll(); + if (subscriptions.isEmpty()) { + throw new RuntimeException( + "No subscriptions found. Use 'givenSubscriptionWithMeasurement' to add one."); + } + + var existing = subscriptions.get(0); + SubscriptionEntity subscription = new SubscriptionEntity(); + subscription.setSubscriptionId(existing.getSubscriptionId()); + subscription.setSubscriptionNumber(existing.getSubscriptionNumber()); + subscription.setStartDate(OffsetDateTime.parse("2024-05-23T11:48:15.888129Z")); + subscription.setEndDate(OffsetDateTime.parse("2024-06-23T11:48:15.888129Z")); + subscription.setOffering(offering); + subscription.setOrgId(ORG_ID); + subscription.setBillingProvider(BillingProvider.AWS); + subscription.setBillingAccountId(existing.getBillingAccountId()); + subscription.setSubscriptionMeasurements( + Map.of( + new SubscriptionMeasurementKey(MetricIdUtils.getCores().toString(), "HYPERVISOR"), + 10.0)); + subscriptionRepository.persist(subscription); + } + + private void givenExportRequestWithoutPermissions() { + givenExportRequest(Format.JSON); + givenRbacPermissions(List.of()); + } + + private void givenExportRequestWithPermissions(Format format) { + givenExportRequest(format); + givenRbacPermissions(List.of(SWATCH_APP + ADMIN_ROLE)); + } + + private void givenExportRequest(Format format) { + request = new GenericConsoleCloudEvent<>(); + request.setId(UUID.randomUUID()); + request.setSource("urn:redhat:source:console:app:export-service"); + request.setSpecVersion("1.0"); + request.setType("com.redhat.console.export-service.request"); + request.setDataSchema( + "https://console.redhat.com/api/schemas/apps/export-service/v1/resource-request.json"); + request.setTime(LocalDateTime.now()); + request.setOrgId(ORG_ID); + + var resourceRequest = new ResourceRequest(); + resourceRequest.setResourceRequest(new ResourceRequestClass()); + resourceRequest.getResourceRequest().setExportRequestUUID(UUID.randomUUID()); + resourceRequest.getResourceRequest().setUUID(UUID.randomUUID()); + resourceRequest.getResourceRequest().setApplication("subscriptions"); + resourceRequest.getResourceRequest().setFormat(format); + resourceRequest.getResourceRequest().setResource("subscriptions"); + resourceRequest.getResourceRequest().setXRhIdentity("MTMyNTk3NzU="); + resourceRequest.getResourceRequest().setFilters(new HashMap<>()); + + request.setData(resourceRequest); + } + + private void givenFilterInExportRequest(String filter, String value) { + request.getData().getResourceRequest().getFilters().put(filter, value); + } + + private void givenRbacPermissions(List SWATCH_APP) { + try { + when(rbacService.getPermissions( + request.getData().getResourceRequest().getApplication(), + request.getData().getResourceRequest().getXRhIdentity())) + .thenReturn(SWATCH_APP); + } catch (RbacApiException e) { + Assertions.fail("Failed to call the get permissions method", e); + } + } + + private void whenReceiveExportRequest() { + exportChannel.send(parser.toJson(request)); + } + + private void verifyNoRequestsWereSentToExportServiceWithError() { + wireMockResource.verifyNoRequestsWereSentToExportServiceWithError(request); + } + + private void verifyNoRequestsWereSentToExportServiceWithUploadData() { + wireMockResource.verifyNoRequestsWereSentToExportServiceWithUploadData(request); + } + + private void verifyRequestWasSentToExportServiceWithNoDataFound() { + wireMockResource.verifyRequestWasSentToExportServiceWithUploadData( + request, toJson(new SubscriptionsExportJson().withData(new ArrayList<>()))); + } + + private void verifyRequestWasSentToExportServiceWithUploadData( + GenericConsoleCloudEvent request, String expected) { + wireMockResource.verifyRequestWasSentToExportServiceWithUploadData(request, expected); + } + + private void verifyRequestWasSentToExportServiceWithError( + GenericConsoleCloudEvent request) { + verifyRequestWasSentToExportServiceWithError(request, ""); + } + + private void verifyRequestWasSentToExportServiceWithError( + GenericConsoleCloudEvent request, String message) { + wireMockResource.verifyRequestWasSentToExportServiceWithError(request, message); + } + + private void updateOffering() { + offeringRepository.persist(offering); + } + + private String toJson(Object data) { + try { + return objectMapper.writeValueAsString(data); + } catch (JsonProcessingException e) { + Assertions.fail("Failed to serialize the export data", e); + return null; + } + } + + private String toCsv(List data, Class dataItemClass) { + try { + var csvSchema = csvMapper.schemaFor(dataItemClass).withUseHeader(true); + var writer = csvMapper.writer(csvSchema); + return writer.writeValueAsString(data); + } catch (JsonProcessingException e) { + Assertions.fail("Failed to serialize the export data", e); + return null; + } + } +} diff --git a/swatch-contracts/src/test/java/com/redhat/swatch/contract/test/resources/ExportServiceWireMockResource.java b/swatch-contracts/src/test/java/com/redhat/swatch/contract/test/resources/ExportServiceWireMockResource.java new file mode 100644 index 0000000000..b808063d8d --- /dev/null +++ b/swatch-contracts/src/test/java/com/redhat/swatch/contract/test/resources/ExportServiceWireMockResource.java @@ -0,0 +1,137 @@ +/* + * Copyright Red Hat, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * Red Hat trademarks are not licensed under GPLv3. No permission is + * granted to use or replicate Red Hat trademarks that are incorporated + * in this software or its documentation. + */ +package com.redhat.swatch.contract.test.resources; + +import static com.github.tomakehurst.wiremock.client.WireMock.containing; +import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathMatching; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; + +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.common.ConsoleNotifier; +import com.redhat.cloud.event.apps.exportservice.v1.ResourceRequest; +import com.redhat.cloud.event.parser.GenericConsoleCloudEvent; +import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; +import java.util.Map; +import org.testcontainers.shaded.org.awaitility.Awaitility; + +public class ExportServiceWireMockResource implements QuarkusTestResourceLifecycleManager { + + private WireMockServer wireMockServer; + + @Override + public Map start() { + wireMockServer = startWireMockServer(); + wireMockServer.resetAll(); + wireMockServer.stubFor(post(urlPathMatching("/app/export/.*"))); + return Map.of("clowder.privateEndpoints.export-service-service.url", wireMockServer.baseUrl()); + } + + @Override + public void stop() { + if (wireMockServer != null) { + wireMockServer.stop(); + wireMockServer = null; + } + } + + @Override + public void inject(TestInjector testInjector) { + testInjector.injectIntoFields( + this, + new TestInjector.AnnotatedAndMatchesType( + InjectWireMock.class, ExportServiceWireMockResource.class)); + } + + public void verifyNoRequestsWereSentToExportServiceWithError( + GenericConsoleCloudEvent request) { + wireMockServer.verify( + 0, + postRequestedFor( + urlEqualTo( + String.format( + "/app/export/v1/%s/subscriptions/%s/error", + request.getData().getResourceRequest().getUUID(), + request.getData().getResourceRequest().getExportRequestUUID())))); + } + + public void verifyRequestWasSentToExportServiceWithError( + GenericConsoleCloudEvent request, String message) { + Awaitility.await() + .untilAsserted( + () -> + wireMockServer.verify( + postRequestedFor( + urlEqualTo( + String.format( + "/app/export/v1/%s/subscriptions/%s/error", + request.getData().getResourceRequest().getExportRequestUUID(), + request.getData().getResourceRequest().getUUID()))) + .withRequestBody(containing(message)))); + } + + public void verifyNoRequestsWereSentToExportServiceWithUploadData( + GenericConsoleCloudEvent request) { + wireMockServer.verify( + 0, + postRequestedFor( + urlEqualTo( + String.format( + "/app/export/v1/%s/subscriptions/%s/upload", + request.getData().getResourceRequest().getExportRequestUUID(), + request.getData().getResourceRequest().getUUID())))); + } + + public void verifyRequestWasSentToExportServiceWithUploadData( + GenericConsoleCloudEvent request, String expected) { + Awaitility.await() + .untilAsserted( + () -> + wireMockServer.verify( + postRequestedFor( + urlEqualTo( + String.format( + "/app/export/v1/%s/subscriptions/%s/upload", + request.getData().getResourceRequest().getExportRequestUUID(), + request.getData().getResourceRequest().getUUID()))) + .withRequestBody(equalTo(expected)))); + } + + private static WireMockServer startWireMockServer() { + var wireMockServer = + new WireMockServer( + wireMockConfig() + .dynamicPort() + .notifier(new ConsoleNotifier(true)) + // This is mandatory to handle large files, otherwise Wiremock returns 500 Server + // Error + .jettyHeaderRequestSize(16384) + .jettyHeaderResponseSize(80000) + .stubRequestLoggingDisabled(true) + .maxLoggedResponseSize(1000)); + wireMockServer.start(); + System.out.printf("Running export service on port %d%n", wireMockServer.port()); + return wireMockServer; + } +} diff --git a/swatch-contracts/src/test/java/com/redhat/swatch/contract/test/resources/InMemoryMessageBrokerKafkaResource.java b/swatch-contracts/src/test/java/com/redhat/swatch/contract/test/resources/InMemoryMessageBrokerKafkaResource.java index 46d01b02fb..424cfcec49 100644 --- a/swatch-contracts/src/test/java/com/redhat/swatch/contract/test/resources/InMemoryMessageBrokerKafkaResource.java +++ b/swatch-contracts/src/test/java/com/redhat/swatch/contract/test/resources/InMemoryMessageBrokerKafkaResource.java @@ -23,6 +23,7 @@ import static com.redhat.swatch.contract.config.Channels.CAPACITY_RECONCILE; import static com.redhat.swatch.contract.config.Channels.CAPACITY_RECONCILE_TASK; import static com.redhat.swatch.contract.config.Channels.ENABLED_ORGS; +import static com.redhat.swatch.contract.config.Channels.EXPORT_REQUESTS_TOPIC; import static com.redhat.swatch.contract.config.Channels.OFFERING_SYNC; import static com.redhat.swatch.contract.config.Channels.SUBSCRIPTION_SYNC_TASK_TOPIC; @@ -41,6 +42,7 @@ public Map start() { env.putAll(InMemoryConnector.switchIncomingChannelsToInMemory(CAPACITY_RECONCILE_TASK)); env.putAll(InMemoryConnector.switchOutgoingChannelsToInMemory(CAPACITY_RECONCILE)); env.putAll(InMemoryConnector.switchOutgoingChannelsToInMemory(OFFERING_SYNC)); + env.putAll(InMemoryConnector.switchIncomingChannelsToInMemory(EXPORT_REQUESTS_TOPIC)); return env; } diff --git a/swatch-subscription-sync/deploy/clowdapp.yaml b/swatch-subscription-sync/deploy/clowdapp.yaml index dc710f0f8a..e4de62c596 100644 --- a/swatch-subscription-sync/deploy/clowdapp.yaml +++ b/swatch-subscription-sync/deploy/clowdapp.yaml @@ -115,10 +115,6 @@ parameters: value: '3' - name: KAFKA_SUBSCRIPTIONS_PRUNE_PARTITIONS value: '3' - - name: KAFKA_SUBSCRIPTIONS_EXPORT_REPLICAS - value: '3' - - name: KAFKA_SUBSCRIPTIONS_EXPORT_PARTITIONS - value: '3' - name: CURL_CRON_IMAGE value: quay.io/app-sre/ubi8-ubi-minimal - name: CURL_CRON_IMAGE_TAG @@ -158,17 +154,11 @@ objects: # dependencies: # - rhsm - kafkaTopics: - - replicas: ${{KAFKA_SUBSCRIPTIONS_EXPORT_REPLICAS}} - partitions: ${{KAFKA_SUBSCRIPTIONS_EXPORT_PARTITIONS}} - topicName: platform.export.requests - database: sharedDbAppName: swatch-tally dependencies: - swatch-tally - rbac - - export-service # Creates a database if local mode, or uses RDS in production # database: @@ -369,11 +359,6 @@ objects: secretKeyRef: name: swatch-psks key: self - - name: SWATCH_EXPORT_PSK - valueFrom: - secretKeyRef: - name: export-psk - key: export_psk - name: USER_HOST value: ${USER_HOST} - name: USER_MAX_CONNECTIONS @@ -471,10 +456,3 @@ objects: name: swatch-psks data: self: cGxhY2Vob2xkZXI= - - - apiVersion: v1 - kind: Secret - metadata: - name: export-psk - data: - export_psk: dGVzdGluZy1hLXBzaw==