Truncate Bodies Script

libs/model/src/models/comment.ts

@@ -9,6 +9,7 @@ import type {
   ReactionAttributes,
   ThreadInstance,
 } from '.';
+import { beforeValidateCommentsHook } from './utils';
 
 export type CommentAttributes = z.infer<typeof Comment> & {
   // associations
@@ -68,6 +69,9 @@ export default (
     },
     {
       hooks: {
+        beforeValidate(instance: CommentInstance) {
+          beforeValidateCommentsHook(instance);
+        },
         afterCreate: async (comment, options) => {
           await (
             sequelize.models.Thread as Sequelize.ModelStatic<ThreadInstance>
@@ -85,7 +89,6 @@ export default (
             thread_id: String(comment.thread_id),
           });
         },
-
         afterDestroy: async ({ thread_id }, options) => {
           await (
             sequelize.models.Thread as Sequelize.ModelStatic<ThreadInstance>

libs/model/src/models/comment_version_history.ts

@@ -3,6 +3,7 @@ import Sequelize from 'sequelize';
 import { z } from 'zod';
 import { CommentAttributes } from './comment';
 import type { ModelInstance } from './types';
+import { beforeValidateCommentsHook } from './utils';
 
 export type CommentVersionHistoryAttributes = z.infer<
   typeof CommentVersionHistory
@@ -30,5 +31,10 @@ export default (
       tableName: 'CommentVersionHistories',
       timestamps: false,
       indexes: [{ fields: ['comment_id'] }],
+      hooks: {
+        beforeValidate(instance: CommentVersionHistoryInstance) {
+          beforeValidateCommentsHook(instance);
+        },
+      },
     },
   );

libs/model/src/models/thread.ts

@@ -8,6 +8,7 @@ import { AddressAttributes } from './address';
 import type { CommunityAttributes } from './community';
 import type { ThreadSubscriptionAttributes } from './thread_subscriptions';
 import type { ModelInstance } from './types';
+import { beforeValidateThreadsHook } from './utils';
 
 export type ThreadAttributes = z.infer<typeof Thread> & {
   // associations
@@ -26,7 +27,7 @@ export default (
       address_id: { type: Sequelize.INTEGER, allowNull: true },
       created_by: { type: Sequelize.STRING, allowNull: true },
       title: { type: Sequelize.TEXT, allowNull: false },
-      body: { type: Sequelize.TEXT, allowNull: true },
+      body: { type: Sequelize.TEXT, allowNull: false },
       kind: { type: Sequelize.STRING, allowNull: false },
       stage: {
         type: Sequelize.TEXT,
@@ -116,6 +117,9 @@ export default (
         { fields: ['canvas_msg_id'] },
       ],
       hooks: {
+        beforeValidate(instance: ThreadInstance) {
+          beforeValidateThreadsHook(instance);
+        },
         afterCreate: async (
           thread: ThreadInstance,
           options: Sequelize.CreateOptions<ThreadAttributes>,

libs/model/src/models/thread_version_history.ts

@@ -3,6 +3,7 @@ import Sequelize from 'sequelize';
 import { z } from 'zod';
 import { ThreadAttributes } from './thread';
 import type { ModelInstance } from './types';
+import { beforeValidateThreadsHook } from './utils';
 
 export type ThreadVersionHistoryAttributes = z.infer<
   typeof ThreadVersionHistory
@@ -31,5 +32,10 @@ export default (
       tableName: 'ThreadVersionHistories',
       timestamps: false,
       indexes: [{ fields: ['thread_id'] }],
+      hooks: {
+        beforeValidate(instance: ThreadVersionHistoryInstance) {
+          beforeValidateThreadsHook(instance);
+        },
+      },
     },
   );

libs/model/src/models/utils.ts

@@ -1,4 +1,8 @@
-import { decamelize } from '@hicommonwealth/shared';
+import {
+  decamelize,
+  MAX_TRUNCATED_CONTENT_LENGTH,
+  safeTruncateBody,
+} from '@hicommonwealth/shared';
 import {
   Model,
   Sequelize,
@@ -278,3 +282,45 @@ export const syncHooks = {
     options.logging = false;
   },
 };
+
+export const beforeValidateThreadsHook = (instance: {
+  body: string;
+  content_url?: string | null | undefined;
+}) => {
+  if (!instance.body || instance.body.length <= MAX_TRUNCATED_CONTENT_LENGTH)
+    return;
+
+  if (!instance.content_url) {
+    throw new Error(
+      'content_url must be defined if body ' +
+        `length is greater than ${MAX_TRUNCATED_CONTENT_LENGTH}`,
+    );
+  } else
+    instance.body = safeTruncateBody(
+      instance.body,
+      MAX_TRUNCATED_CONTENT_LENGTH,
+    );
+  return instance;
+};
+
+// TODO: merge with beforeValidateThreadsHook after
+//  https://github.com/hicommonwealth/commonwealth/issues/9673
+export const beforeValidateCommentsHook = (instance: {
+  text: string;
+  content_url?: string | null | undefined;
+}) => {
+  if (!instance.text || instance.text.length <= MAX_TRUNCATED_CONTENT_LENGTH)
+    return;
+
+  if (!instance.content_url) {
+    throw new Error(
+      'content_url must be defined if body ' +
+        `length is greater than ${MAX_TRUNCATED_CONTENT_LENGTH}`,
+    );
+  } else
+    instance.text = safeTruncateBody(
+      instance.text,
+      MAX_TRUNCATED_CONTENT_LENGTH,
+    );
+  return instance;
+};

libs/model/test/contest-worker/contest-worker-policy.spec.ts

@@ -97,6 +97,7 @@ describe('Contest Worker Policy', () => {
             canvas_msg_id: '',
             kind: '',
             stage: '',
+            body: '',
             view_count: 0,
             reaction_count: 0,
             reaction_weights_sum: '0',

libs/model/test/thread/thread-lifecycle.spec.ts

@@ -21,6 +21,7 @@ import * as schemas from '@hicommonwealth/schemas';
 import { TopicWeightedVoting } from '@hicommonwealth/schemas';
 import {
   CANVAS_TOPIC,
+  MAX_TRUNCATED_CONTENT_LENGTH,
   getTestSigner,
   sign,
   toCanvasSignedDataApiArgs,
@@ -283,7 +284,6 @@ describe('Thread lifecycle', () => {
             payload: await signPayload(actors[role].address!, instancePayload),
           });
           expect(_thread?.title).to.equal(instancePayload.title);
-          expect(_thread?.body).to.equal(instancePayload.body);
           expect(_thread?.stage).to.equal(instancePayload.stage);
           // capture as admin author for other tests
           if (!thread) thread = _thread!;
@@ -296,6 +296,12 @@ describe('Thread lifecycle', () => {
                 key: _thread!.content_url!.split('/').pop()!,
               }),
             ).toBeTruthy();
+
+            expect(_thread?.body).to.equal(
+              instancePayload.body.slice(0, MAX_TRUNCATED_CONTENT_LENGTH),
+            );
+          } else {
+            expect(_thread?.body).to.equal(instancePayload.body);
           }
         });
       } else {
@@ -313,7 +319,7 @@ describe('Thread lifecycle', () => {
 
   describe('updates', () => {
     test('should patch content', async () => {
-      const body = {
+      const payloadContent = {
         title: 'hello',
         body: chance.paragraph({ sentences: 50 }),
         canvas_msg_id: '',
@@ -323,10 +329,13 @@ describe('Thread lifecycle', () => {
         actor: actors.admin,
         payload: {
           thread_id: thread.id!,
-          ...body,
+          ...payloadContent,
         },
       });
-      expect(updated).to.contain(body);
+      expect(updated).to.contain({
+        ...payloadContent,
+        body: payloadContent.body.slice(0, MAX_TRUNCATED_CONTENT_LENGTH),
+      });
       expect(updated?.content_url).toBeTruthy();
       expect(
         await blobStorage({ key: R2_ADAPTER_KEY }).exists({
@@ -336,15 +345,15 @@ describe('Thread lifecycle', () => {
       ).toBeTruthy();
       expect(updated?.ThreadVersionHistories?.length).to.equal(2);
 
-      body.body = 'wasup';
+      payloadContent.body = 'wasup';
       updated = await command(UpdateThread(), {
         actor: actors.admin,
         payload: {
           thread_id: thread.id!,
-          ...body,
+          ...payloadContent,
         },
       });
-      expect(updated).to.contain(body);
+      expect(updated).to.contain(payloadContent);
       expect(updated?.content_url).toBeFalsy();
       expect(updated!.ThreadVersionHistories?.length).to.equal(3);
       const sortedHistory = updated!.ThreadVersionHistories!.sort(
@@ -572,7 +581,7 @@ describe('Thread lifecycle', () => {
       });
       expect(firstComment).to.include({
         thread_id: thread!.id,
-        text,
+        text: text.slice(0, MAX_TRUNCATED_CONTENT_LENGTH),
         community_id: thread!.community_id,
       });
       expect(firstComment?.content_url).toBeTruthy();
@@ -713,7 +722,7 @@ describe('Thread lifecycle', () => {
       });
       expect(updated).to.include({
         thread_id: thread!.id,
-        text,
+        text: text.slice(0, MAX_TRUNCATED_CONTENT_LENGTH),
         community_id: thread!.community_id,
       });
       expect(updated?.content_url).toBeTruthy();

libs/model/test/util-tests/getCommentDepth.spec.ts

@@ -24,6 +24,7 @@ describe('getCommentDepth', () => {
     });
     const thread = await models.Thread.create({
       community_id,
+      body: 'test',
       address_id: address!.id!,
       title: 'Testing',
       kind: 'discussion',

libs/schemas/src/entities/thread.schemas.ts

@@ -21,7 +21,7 @@ export const Thread = z.object({
   title: z.string(),
   kind: z.string(),
   stage: z.string().optional(),
-  body: z.string().nullish(),
+  body: z.string(),
   url: z.string().nullish(),
   topic_id: PG_INT.nullish(),
   pinned: z.boolean().nullish(),

libs/shared/src/constants.ts

@@ -16,3 +16,8 @@ export const DISCORD_BOT_ADDRESS = '0xdiscordbot';
 export const DEFAULT_NAME = 'Anonymous';
 
 export const MAX_RECIPIENTS_PER_WORKFLOW_TRIGGER = 1_000;
+
+// The maximum number of characters allowed in 'body' and 'text'
+// columns of Threads, Comments, and version history models.
+// Full content found by fetching from 'content_url'.
+export const MAX_TRUNCATED_CONTENT_LENGTH = 2_000;