feat: queue products for metadata fetching on create

When a product is created, the backend attempted to get metadata/images for the product. However this process can be very slow (30+ seconds), and the requester to the API might have a smaller timeout, breaking the client. This makes it so the API returns immediately and attempts to fetch the metadata later in a cronjob.
livraria-papelaria-espaco · Oct 27, 2024 · c861e01 · c861e01
1 parent c26ee6a
commit c861e01
Show file tree

Hide file tree

Showing 5 changed files with 92 additions and 59 deletions.
diff --git a/backend/api/product/controllers/Product.js b/backend/api/product/controllers/Product.js
@@ -171,24 +171,12 @@ module.exports = {
       }
 
       if (metadataServices.isISBN(data.reference)) {
-        const metadata = await metadataServices.fetchMetadataFromWook(data.reference);
-        if (metadata) {
-          const category = await strapi.services.category.findOne({ name: 'Livraria' });
-
-          const slug = await strapi.plugins['content-manager'].services.uid.generateUIDField({
-            contentTypeUID: 'application::product.product',
-            field: 'slug',
-            data: metadata,
-          });
-          data = {
-            ...data,
-            show: true,
-            ...metadata,
-            slug,
-            images: await metadataServices.fetchAndUploadImages(data.reference, slug),
-            category: category ? category.id : undefined,
-          };
-        }
+        data = {
+          ...data,
+          show: false,
+          // Instead of blocking on this request, set a flag and fetch metadata in a cron job
+          waitingForMetadata: true,
+        };
       }
 
       if (!data.slug)

diff --git a/backend/api/product/models/Product.settings.json b/backend/api/product/models/Product.settings.json
@@ -40,6 +40,13 @@
       "private": true,
       "index": true
     },
+    "waitingForMetadata": {
+      "required": true,
+      "default": false,
+      "type": "boolean",
+      "private": true,
+      "index": true
+    },
     "reference": {
       "type": "string",
       "required": false,

diff --git a/backend/api/product/services/Product.js b/backend/api/product/services/Product.js
@@ -1,5 +1,8 @@
 'use strict';
 
+// https://github.com/strapi/strapi/blob/v3.6.8/packages/strapi-plugin-content-manager/services/uid.js#L4
+const slugify = require('@sindresorhus/slugify');
+
 const convertSortQueryParams = (sortQuery) => {
   if (typeof sortQuery !== 'string') {
     throw new Error(`convertSortQueryParams expected a string, got ${typeof sortQuery}`);
@@ -41,6 +44,9 @@ const convertLimitQueryParams = (limitQuery) => {
   return limitAsANumber;
 };
 
+// Only one instance of the fetchMetadataForQueuedProducts function may be running at a given time
+let LOCK_METADATA_QUEUE = false;
+
 module.exports = {
   searchEnhanced: async (params) => {
     let searchQuery;
@@ -142,4 +148,70 @@ module.exports = {
       return { ref, qnt };
     }
   },
+
+  fetchMetadataForQueuedProducts: async () => {
+    if (LOCK_METADATA_QUEUE) {
+      // Only allow one instance of this function to be running
+      return;
+    }
+    LOCK_METADATA_QUEUE = true;
+
+    try {
+      let metadataServices = strapi.plugins['metadata-fetcher'].services['metadata-fetcher'];
+
+      let queuedProducts = await strapi.services.product.find({
+        waitingForMetadata: true,
+        _sort: 'updatedAt:asc',
+      });
+
+      for (let product of queuedProducts) {
+        const metadata = await metadataServices.fetchMetadataFromWook(product.reference);
+
+        let data = {
+          waitingForMetadata: false,
+        };
+
+        if (metadata) {
+          const category = await strapi.services.category.findOne({ name: 'Livraria' });
+
+          const calculatedSlug = slugify(metadata.name);
+          let slug = product.slug;
+          if (calculatedSlug !== slug) {
+            slug = await strapi.plugins['content-manager'].services.uid.generateUIDField({
+              contentTypeUID: 'application::product.product',
+              field: 'slug',
+              data: metadata,
+            });
+          }
+
+          data = {
+            ...data,
+            show: true,
+            ...metadata,
+            slug,
+            images: await metadataServices.fetchAndUploadImages(product.reference, slug),
+            category: category ? category.id : undefined,
+          };
+        }
+
+        const res = await strapi.query('product').model.updateOne({ _id: product._id }, data);
+
+        if (res.nModified !== 1) {
+          throw new Error(`Failed to save updated metadata for product ${product.reference}`);
+        }
+
+        await strapi.services.productsearch
+          .updateProduct({
+            ...product,
+            ...data,
+          })
+          .catch(strapi.log.error);
+      }
+    } catch (e) {
+      console.error(e);
+      strapi.log.error("Error while fetching metadata for queued products:", e);
+    } finally {
+      LOCK_METADATA_QUEUE = false;
+    }
+  }
 };
diff --git a/backend/config/functions/cron.js b/backend/config/functions/cron.js
@@ -19,4 +19,10 @@ module.exports = {
 
     strapi.services.order.cancelExpiredOrders();
   },
+  // Every 5 minutes
+  '*/5 * * * *': () => {
+    if (process.env.NODE_ENV === 'production' && process.env.NODE_APP_INSTANCE !== '0') return;
+
+    strapi.services.product.fetchMetadataForQueuedProducts();
+  },
 };
diff --git a/backend/plugins/metadata-fetcher/services/metadata-fetcher.js b/backend/plugins/metadata-fetcher/services/metadata-fetcher.js
@@ -1,33 +1,13 @@
 'use strict';
 const axios = require('axios');
-//const sharp = require('sharp');
 
 /**
  * metadata-fetcher.js service
  *
  * @description: A set of functions similar to controller's actions to avoid code duplication.
  */
 
-const WOOK_REGEX = /<script type="application\/ld\+json">([^]+?)<\/script>/;
 const NEWLINE_REGEX = /\n/g;
-/*const FNAC_SEARCH_REGEX = /<a href="(.+?)" class=".*?Article-title js-minifa-title js-Search-hashLink.*?">.+?<\/a>/;
-const FNAC_REGEX = /<script type="application\/json" class="js-configuration">[^]*?({.+})[^]*?<\/script>/;
-const FNAC_HEADERS = {
-  'User-Agent':
-    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36',
-  'Accept-Language': 'en-US,en;q=0.9',
-};*/
-
-/*const fetchFnacImage = async (url, i) => {
-  try {
-    const response = await axios.get(url, { responseType: 'arraybuffer' });
-    const buffer = Buffer.from(response.data, 'binary');
-    const trimmedBuffer = await sharp(buffer).trim().jpeg().toBuffer();
-    return trimmedBuffer;
-  } catch (e) {
-    return undefined;
-  }
-};*/
 
 const fetchImagesFromFnac = async (isbn) => {
   try {
@@ -36,26 +16,6 @@ const fetchImagesFromFnac = async (isbn) => {
     });
     const buffer = Buffer.from(response.data, 'binary');
     return [buffer];
-    /*const searchResponse = await axios.get(
-      `https://www.fnac.pt/SearchResult/ResultList.aspx?Search=${isbn}`,
-      {
-        headers: FNAC_HEADERS,
-      }
-    );
-    const productUrl = (FNAC_SEARCH_REGEX.exec(searchResponse.data) || [])[1];
-
-    const response = await axios.get(productUrl, {
-      headers: FNAC_HEADERS,
-    });
-    const dataString = (FNAC_REGEX.exec(response.data) || [])[1];
-    const data = JSON.parse(dataString);
-    console.log(data);
-    const images = await Promise.all(
-      data.productData.images.map((imgSet, i) =>
-        fetchFnacImage(imgSet.zoom || imgSet.image || imgSet.thumb)
-      )
-    );
-    return images.filter((i) => !!i);*/
   } catch (e) {
     return [];
   }
@@ -123,4 +83,4 @@ module.exports = {
       })
     );
   },
-};
+};