fix #83 transient fetch issue on search candidate

- improve bluesky search post error handler - fetch failed case - transient fetch issue on search are now retried 2 times - on 3 error, search transient issue are marked failed internally only
boly38 · Sep 5, 2024 · 9709bf0 · 9709bf0
1 parent 6a5a139
commit 9709bf0
Show file tree

Hide file tree

Showing 8 changed files with 57 additions and 18 deletions.
diff --git a/src/exceptions/ServerInternalErrorException.js b/src/exceptions/ServerInternalErrorException.js
@@ -3,9 +3,10 @@ import i18n from "i18n";
 import {isSet} from "../lib/Common.js";
 
 export default class InternalServerErrorException {
-    constructor(message = null) {
+    constructor(message = null, mustBeReported = true) {
         this.code = ReasonPhrases.INTERNAL_SERVER_ERROR;
         this.status = StatusCodes.INTERNAL_SERVER_ERROR;
         this.message = isSet(message) ? message : i18n.__('server.error.internal');
+        this.mustBeReported = mustBeReported;
     }
 }
diff --git a/src/exceptions/ServiceUnavailableException.js b/src/exceptions/ServiceUnavailableException.js
@@ -3,9 +3,10 @@ import i18n from "i18n";
 import {isSet} from "../lib/Common.js";
 
 export default class ServiceUnavailableException {
-    constructor(message = null) {
+    constructor(message = null, mustBeReported = false) {
         this.code = ReasonPhrases.SERVICE_UNAVAILABLE;
         this.status = StatusCodes.SERVICE_UNAVAILABLE;
         this.message = isSet(message) ? message : i18n.__('server.error.unavailable');
+        this.mustBeReported = mustBeReported;
     }
 }
diff --git a/src/services/BotService.js b/src/services/BotService.js
@@ -112,8 +112,8 @@ export default class BotService {
 export const pluginResolve = (text, html, status = 200, post = 0) => {
     return {text, html, status, post};
 }
-export const pluginReject = (text, html, status, shortResponseMessage) => {
-    return {text, html, status, message: shortResponseMessage};
+export const pluginReject = (text, html, status, shortResponseMessage, mustBeReported=false) => {
+    return {text, html, status, message: shortResponseMessage, mustBeReported};
 }
 
 export const dataSimulationDirectory = "src/data/simulation"
diff --git a/src/services/ExpressServer.js b/src/services/ExpressServer.js
@@ -10,7 +10,7 @@ import {
     cacheGetVersion
 } from "../lib/MemoryCache.js";
 import {unauthorized} from "../lib/CommonApi.js";
-import {generateErrorId} from "../lib/Common.js";
+import {generateErrorId, isSet} from "../lib/Common.js";
 import {StatusCodes} from "http-status-codes";
 import ApplicationConfig from "../config/ApplicationConfig.js";
 
@@ -55,7 +55,7 @@ export default class ExpressServer {
         // as singleton // https://github.com/mashpie/i18n-node?tab=readme-ov-file#as-singleton
         i18n.configure({
             locales: ['fr', 'en'],
-            directory: path.join(__dirname, 'src','locales'),
+            directory: path.join(__dirname, 'src', 'locales'),
             defaultLocale: 'en',
             queryParameter: 'lang',// query parameter to switch locale (ie. /home?lang=ch) - defaults to NULL
             cookie: 'lang'
@@ -76,7 +76,11 @@ export default class ExpressServer {
         expressServer.app.use(expressServer.errorHandlerMiddleware.bind(this));// error handler
 
         // build initial cache
-        await this.summaryService.cacheGetWeekSummary({})
+        try {
+            await this.summaryService.cacheGetWeekSummary({})
+        } catch (summaryError) {
+            this.logger.error(`Initial getWeekSummary Error msg:${summaryError.message} stack:${summaryError.stack}`);
+        }
 
         // register inactivity listener
         this.inactivityDetector.registerOnInactivityListener(
@@ -140,7 +144,8 @@ export default class ExpressServer {
                 unauthorized(res, UNAUTHORIZED_FRIENDLY);
             }
         } catch (error) {
-            if (error.status !== StatusCodes.INTERNAL_SERVER_ERROR && error.message) {
+            const {status, message, mustBeReported} = error;
+            if (status !== StatusCodes.INTERNAL_SERVER_ERROR && isSet(message)) {
                 const {status, message} = error;
                 res.status(status).json({success: false, message});
                 return;
@@ -151,11 +156,15 @@ export default class ExpressServer {
             this.logger.error(errorInternalDetails);
             this.auditLogsService.createAuditLog(errorInternalDetails);
             // user
-            const unexpectedError = res.__('server.error.unexpected');
-            const pleaseReportIssue = res.__('server.pleaseReportIssue');
-            const issueLink = res.__('server.issueLinkLabel');
-            let userErrorTxt =  `${unexpectedError}, ${pleaseReportIssue} ${BES_ISSUES} - ${errId}`;
-            let userErrorHtml = `${unexpectedError},${pleaseReportIssue} <a href="${BES_ISSUES}">${issueLink}</a> - ${errId}`;
+            let unexpectedError = res.__('server.error.unexpected');
+            let userErrorTxt = unexpectedError;
+            let userErrorHtml = unexpectedError;
+            if (mustBeReported !== false) {
+                const pleaseReportIssue = res.__('server.pleaseReportIssue');
+                const issueLink = res.__('server.issueLinkLabel');
+                userErrorTxt = `${unexpectedError}, ${pleaseReportIssue} ${BES_ISSUES} - ${errId}`;
+                userErrorHtml = `${unexpectedError}, ${pleaseReportIssue} <a href="${BES_ISSUES}">${issueLink}</a> - ${errId}`;
+            }
             this.newsService.add(userErrorHtml);
             res.status(500).json({success: false, message: userErrorTxt});
         }

diff --git a/src/services/PluginsCommonService.js b/src/services/PluginsCommonService.js
@@ -78,15 +78,18 @@ export default class PluginsCommonService {
     }
 
     rejectWithIdentifyError(pluginName, step, candidate, err, context) {
+        const {status, message, mustBeReported} = err;
         let plantnetTxtError = `[${step}] Impossible d'identifier l'image avec ${pluginName}`;
         let plantnetHtmlError = plantnetTxtError;
         if (isSet(candidate)) {
             plantnetTxtError = `[${step}] Impossible d'identifier l'image de ${postLinkOf(candidate)} avec ${pluginName}`;
             plantnetHtmlError = `<b>Post</b>: <div class="bg-warning">${postHtmlOf(candidate)}</div>` +
                 `<b>Erreur [${step}]</b>: impossible d'identifier l'image avec ${pluginName}`;
         }
-        this.logger.error(`${plantnetTxtError} : ${err.message}`, context);
-        return Promise.reject(pluginReject(plantnetTxtError, plantnetHtmlError, 500, `${pluginName} unexpected error`));
+        this.logger.error(`${plantnetTxtError} : ${message}`, context);
+        return Promise.reject(pluginReject(plantnetTxtError, plantnetHtmlError,
+            isSet(status) ? status: 500,
+            `${pluginName} unexpected error`, mustBeReported));
     }
 
     async handleWithoutScoredResult(pluginName, minimalPercent, options) {

diff --git a/src/services/SummaryService.js b/src/services/SummaryService.js
@@ -24,7 +24,7 @@ export default class SummaryService {
             "maxHoursOld": 7 * 24,// now-7d ... now
             "limit": 100
         })
-        logger.info(`Summary - ${botPosts.length} post(s)`, context);
+        logger.info(`Summary - ${botPosts?.length} post(s)`, context);
         const analytics = {
             posts: 0, likes: 0, replies: 0, reposts: 0,
             bestScore: 0, bestScorePosts: [],

diff --git a/src/servicesExternal/BlueSkyService.js b/src/servicesExternal/BlueSkyService.js
@@ -2,6 +2,7 @@ import {BskyAgent, RichText} from '@atproto/api'
 import {descriptionOfPostAuthor, didOfPostAuthor, postLinkOf, postsFilterSearchResults} from "../domain/post.js";
 import {getEncodingBufferAndBase64FromUri, isSet, nowISO8601, nowMinusHoursUTCISO} from "../lib/Common.js";
 import InternalServerErrorException from "../exceptions/ServerInternalErrorException.js";
+import ServiceUnavailableException from "../exceptions/ServiceUnavailableException.js";
 
 const BLUESKY_POST_LENGTH_MAX = 300;// https://github.com/bluesky-social/bsky-docs/issues/162
 
@@ -72,12 +73,30 @@ export default class BlueSkyService {
             params["until"] = nowMinusHoursUTCISO(0);
         }
         this.logger.info(`searchPosts ${JSON.stringify(params)}`);
-        const response = await this.api.app.bsky.feed.searchPosts(params, {});
-        const posts = postsFilterSearchResults(response.data.posts, hasImages, hasNoReply, isNotMuted);
+        const response = await this.resilientSearchPostsWithRetry(params, {}, 2);
+        const posts = postsFilterSearchResults(response?.data?.posts, hasImages, hasNoReply, isNotMuted);
         this.logger.debug(`posts`, JSON.stringify(posts, null, 2));
         return posts;
     }
 
+    // https://github.com/bluesky-social/atproto/issues/2786 - bluesky api client should provide more than `TypeError: fetch failed`
+    async resilientSearchPostsWithRetry(params, options, retryAttempts) {
+        if (retryAttempts === 0) {
+            throw new ServiceUnavailableException("Bluesky search service is not available for now.");
+        }
+        try {
+            return await this.api.app.bsky.feed.searchPosts(params, options);
+        } catch (error) {
+            this.logger.warn(`resilientSearchPostsWithRetry ${retryAttempts} error:${error}`);
+            if (error.message === "TypeError: fetch failed") {
+                await this.safeSleepMs(5000);// await 5 sec before retrying
+                return await this.resilientSearchPostsWithRetry(params, options, retryAttempts - 1);
+            } else {
+                throw error;
+            }
+        }
+    }
+
     /**
      * reply to a given POST with a given TEXT
      * bluesky replyTo doc : https://docs.bsky.app/docs/tutorials/creating-a-post#replies
@@ -227,4 +246,8 @@ export default class BlueSkyService {
         this.logger.info(`postsArray`, JSON.stringify(postsArray, null, 2));
     }
     */
+
+    async safeSleepMs(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
 }
diff --git a/tests/libTest.js b/tests/libTest.js
@@ -41,6 +41,8 @@ export const initEnv = () => {
 }
 export const _expectNoError = (err) => {
     console.error("_expectNoError", err);
+    const {status,message,success,error} = err;
+    console.error("_expectNoError details attempt", {status,message,success,error} );
     console.trace();// print stack
     expect.fail(err);
 }