[connectors] Implement cleanup for Zendesk brands and categories (#8314)

* feat: implement brand-wise and category-wise cleanup

* refactor: move the cleanup to the Resources

* fix: fix the delete methods

* revert: move back the cleanup to the activities

This reverts commit 842ad0f.

* refactor: move the methods that invoke a certain model in the dedicated Resource class

* perf: add batch delete operations

* lint

* refactor: rename a method

* perf: batch the deletion of the categories

* perf: batch the deletion of the articles for an entire brand

* perf: get rid of a loop over the categories when deleting articles for a brand

* refactor: add a function deleteCategoryChildren

* misc: add comments relative to the fact that brand have 2 permission fields

* refactor: remove the `deleteUpserted...` functions

connectors/src/connectors/zendesk/lib/brand_permissions.ts

@@ -96,6 +96,6 @@ export async function revokeSyncZendeskBrand({
     return null;
   }
 
-  await brand.revokePermissions();
+  await brand.revokeAllPermissions();
   return brand;
 }

connectors/src/connectors/zendesk/lib/help_center_permissions.ts

@@ -7,6 +7,7 @@ import {
 } from "@connectors/connectors/zendesk/lib/zendesk_api";
 import logger from "@connectors/logger/logger";
 import {
+  ZendeskArticleResource,
   ZendeskBrandResource,
   ZendeskCategoryResource,
 } from "@connectors/resources/zendesk_resources";
@@ -104,7 +105,17 @@ export async function revokeSyncZendeskHelpCenter({
     return null;
   }
 
+  // updating the field helpCenterPermission to "none" for the brand
   await brand.revokeHelpCenterPermissions();
+  // revoking the permissions for all the children categories and articles
+  await ZendeskCategoryResource.revokePermissionsForBrand({
+    connectorId,
+    brandId,
+  });
+  await ZendeskArticleResource.revokePermissionsForBrand({
+    connectorId,
+    brandId,
+  });
   return brand;
 }
 

connectors/src/connectors/zendesk/lib/permissions.ts

@@ -21,8 +21,10 @@ import {
 import logger from "@connectors/logger/logger";
 import { ConnectorResource } from "@connectors/resources/connector_resource";
 import {
+  ZendeskArticleResource,
   ZendeskBrandResource,
   ZendeskCategoryResource,
+  ZendeskTicketResource,
 } from "@connectors/resources/zendesk_resources";
 
 /**
@@ -111,7 +113,7 @@ export async function retrieveChildrenNodes({
   if (!parentInternalId) {
     if (isReadPermissionsOnly) {
       const brandsInDatabase =
-        await ZendeskBrandResource.fetchBrandsWithHelpCenter({ connectorId });
+        await ZendeskBrandResource.fetchAllWithHelpCenter({ connectorId });
       nodes = brandsInDatabase.map((brand) =>
         brand.toContentNode({ connectorId })
       );
@@ -186,10 +188,11 @@ export async function retrieveChildrenNodes({
       // If the parent is a brand's tickets, we retrieve the list of tickets for the brand.
       case "tickets": {
         if (isReadPermissionsOnly) {
-          const ticketsInDb = await ZendeskBrandResource.fetchReadOnlyTickets({
-            connectorId,
-            brandId: objectId,
-          });
+          const ticketsInDb =
+            await ZendeskTicketResource.fetchByBrandIdReadOnly({
+              connectorId,
+              brandId: objectId,
+            });
           nodes = ticketsInDb.map((ticket) =>
             ticket.toContentNode({ connectorId })
           );
@@ -199,7 +202,7 @@ export async function retrieveChildrenNodes({
       // If the parent is a brand's help center, we retrieve the list of Categories for this brand.
       case "help-center": {
         const categoriesInDatabase =
-          await ZendeskBrandResource.fetchReadOnlyCategories({
+          await ZendeskCategoryResource.fetchByBrandIdReadOnly({
             connectorId,
             brandId: objectId,
           });
@@ -239,7 +242,7 @@ export async function retrieveChildrenNodes({
       case "category": {
         if (isReadPermissionsOnly) {
           const articlesInDb =
-            await ZendeskCategoryResource.fetchReadOnlyArticles({
+            await ZendeskArticleResource.fetchByCategoryIdReadOnly({
               connectorId,
               categoryId: objectId,
             });

connectors/src/connectors/zendesk/lib/ticket_permissions.ts

@@ -3,7 +3,10 @@ import type { ModelId } from "@dust-tt/types";
 import { getZendeskAccessToken } from "@connectors/connectors/zendesk/lib/zendesk_access_token";
 import { createZendeskClient } from "@connectors/connectors/zendesk/lib/zendesk_api";
 import logger from "@connectors/logger/logger";
-import { ZendeskBrandResource } from "@connectors/resources/zendesk_resources";
+import {
+  ZendeskBrandResource,
+  ZendeskTicketResource,
+} from "@connectors/resources/zendesk_resources";
 
 export async function allowSyncZendeskTickets({
   subdomain,
@@ -81,6 +84,12 @@ export async function revokeSyncZendeskTickets({
     return null;
   }
 
+  // updating the field ticketsPermission to "none" for the brand
   await brand.revokeTicketsPermissions();
+  // revoking the permissions for all the children tickets
+  await ZendeskTicketResource.revokePermissionsForBrand({
+    connectorId,
+    brandId,
+  });
   return brand;
 }

connectors/src/connectors/zendesk/temporal/activities.ts

@@ -1,18 +1,26 @@
 import type { ModelId } from "@dust-tt/types";
 
+import {
+  getArticleInternalId,
+  getTicketInternalId,
+} from "@connectors/connectors/zendesk/lib/id_conversions";
 import { getZendeskAccessToken } from "@connectors/connectors/zendesk/lib/zendesk_access_token";
 import {
   changeZendeskClientSubdomain,
   createZendeskClient,
 } from "@connectors/connectors/zendesk/lib/zendesk_api";
 import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config";
+import { deleteFromDataSource } from "@connectors/lib/data_sources";
 import { syncStarted, syncSucceeded } from "@connectors/lib/sync_status";
 import { ConnectorResource } from "@connectors/resources/connector_resource";
 import {
+  ZendeskArticleResource,
   ZendeskBrandResource,
   ZendeskCategoryResource,
   ZendeskConfigurationResource,
+  ZendeskTicketResource,
 } from "@connectors/resources/zendesk_resources";
+import type { DataSourceConfig } from "@connectors/types/data_source_config";
 
 async function _getZendeskConnectorOrRaise(connectorId: ModelId) {
   const connector = await ConnectorResource.fetchById(connectorId);
@@ -82,12 +90,6 @@ export async function syncZendeskBrandActivity({
 }): Promise<{ helpCenterAllowed: boolean; ticketsAllowed: boolean }> {
   const connector = await _getZendeskConnectorOrRaise(connectorId);
   const dataSourceConfig = dataSourceConfigFromConnector(connector);
-  const loggerArgs = {
-    workspaceId: dataSourceConfig.workspaceId,
-    connectorId,
-    provider: "zendesk",
-    dataSourceId: dataSourceConfig.dataSourceId,
-  };
   const configuration = await _getZendeskConfigurationOrRaise(connectorId);
 
   const brandInDb = await ZendeskBrandResource.fetchByBrandId({
@@ -105,7 +107,7 @@ export async function syncZendeskBrandActivity({
     brandInDb.helpCenterPermission === "none" &&
     brandInDb.ticketsPermission === "none"
   ) {
-    await brandInDb.remove({ dataSourceConfig, loggerArgs });
+    await brandInDb.delete();
     return { helpCenterAllowed: false, ticketsAllowed: false };
   }
 
@@ -120,12 +122,13 @@ export async function syncZendeskBrandActivity({
     result: { brand: fetchedBrand },
   } = await zendeskApiClient.brand.show(brandId);
   if (!fetchedBrand) {
-    await brandInDb.remove({ dataSourceConfig, loggerArgs });
+    await deleteBrandChildren({ connectorId, brandId, dataSourceConfig });
+    await brandInDb.delete();
     return { helpCenterAllowed: false, ticketsAllowed: false };
   }
 
   const categoriesWithReadPermissions =
-    await ZendeskBrandResource.fetchReadOnlyCategories({
+    await ZendeskCategoryResource.fetchByBrandIdReadOnly({
       connectorId,
       brandId,
     });
@@ -134,7 +137,7 @@ export async function syncZendeskBrandActivity({
   if (noMoreAllowedCategories) {
     // if the tickets and all children categories are not allowed anymore, we delete the brand data
     if (brandInDb.ticketsPermission !== "read") {
-      await brandInDb.remove({ dataSourceConfig, loggerArgs });
+      await deleteBrandChildren({ connectorId, brandId, dataSourceConfig });
       return { helpCenterAllowed: false, ticketsAllowed: false };
     }
     await brandInDb.update({ helpCenterPermission: "none" });
@@ -245,12 +248,6 @@ export async function syncZendeskCategoryActivity({
 }): Promise<boolean> {
   const connector = await _getZendeskConnectorOrRaise(connectorId);
   const dataSourceConfig = dataSourceConfigFromConnector(connector);
-  const loggerArgs = {
-    workspaceId: dataSourceConfig.workspaceId,
-    connectorId,
-    provider: "zendesk",
-    dataSourceId: dataSourceConfig.dataSourceId,
-  };
   const configuration = await _getZendeskConfigurationOrRaise(connectorId);
   const categoryInDb = await ZendeskCategoryResource.fetchByCategoryId({
     connectorId,
@@ -264,7 +261,8 @@ export async function syncZendeskCategoryActivity({
 
   // if all rights were revoked, we delete the category data.
   if (categoryInDb.permission === "none") {
-    await categoryInDb.remove({ dataSourceConfig, loggerArgs });
+    await deleteCategoryChildren({ connectorId, dataSourceConfig, categoryId });
+    await categoryInDb.delete();
     return false;
   }
 
@@ -278,7 +276,8 @@ export async function syncZendeskCategoryActivity({
   const { result: fetchedCategory } =
     await zendeskApiClient.helpcenter.categories.show(categoryId);
   if (!fetchedCategory) {
-    await categoryInDb.remove({ dataSourceConfig, loggerArgs });
+    await deleteCategoryChildren({ connectorId, categoryId, dataSourceConfig });
+    await categoryInDb.delete();
     return false;
   }
 
@@ -317,3 +316,84 @@ export async function syncZendeskTicketsActivity({}: {
 }): Promise<{ hasMore: boolean; afterCursor: string }> {
   return { hasMore: false, afterCursor: "" };
 }
+
+/**
+ * Deletes all the data stored in the db and in the data source relative to a brand (category, articles and tickets).
+ */
+async function deleteBrandChildren({
+  connectorId,
+  brandId,
+  dataSourceConfig,
+}: {
+  connectorId: number;
+  brandId: number;
+  dataSourceConfig: DataSourceConfig;
+}) {
+  /// deleting the articles in the data source
+  const articles = await ZendeskArticleResource.fetchByBrandId({
+    connectorId,
+    brandId,
+  });
+  await Promise.all(
+    articles.map((article) =>
+      deleteFromDataSource(
+        dataSourceConfig,
+        getArticleInternalId(connectorId, article.articleId)
+      )
+    )
+  );
+  /// deleting the tickets in the data source
+  const tickets = await ZendeskTicketResource.fetchByBrandId({
+    connectorId: connectorId,
+    brandId: brandId,
+  });
+  await Promise.all([
+    tickets.map((ticket) =>
+      deleteFromDataSource(
+        dataSourceConfig,
+        getTicketInternalId(ticket.connectorId, ticket.ticketId)
+      )
+    ),
+  ]);
+  /// deleting the articles stored in the db
+  await ZendeskArticleResource.deleteByBrandId({
+    connectorId,
+    brandId,
+  });
+  /// deleting the categories stored in the db
+  await ZendeskCategoryResource.deleteByBrandId({ connectorId, brandId });
+  /// deleting the tickets stored in the db
+  await ZendeskTicketResource.deleteByBrandId({ connectorId, brandId });
+}
+
+/**
+ * Deletes all the data stored in the db and in the data source relative to a category (articles).
+ */
+async function deleteCategoryChildren({
+  connectorId,
+  categoryId,
+  dataSourceConfig,
+}: {
+  connectorId: number;
+  categoryId: number;
+  dataSourceConfig: DataSourceConfig;
+}) {
+  /// deleting the articles in the data source
+  const articles = await ZendeskArticleResource.fetchByCategoryId({
+    connectorId,
+    categoryId,
+  });
+  await Promise.all(
+    articles.map((article) =>
+      deleteFromDataSource(
+        dataSourceConfig,
+        getArticleInternalId(connectorId, article.articleId)
+      )
+    )
+  );
+  /// deleting the articles stored in the db
+  await ZendeskArticleResource.deleteByCategoryId({
+    connectorId,
+    categoryId,
+  });
+}