Merge pull request #55 from ShivTestOrg/questionmissing

fix: skip plugin exec if question is missing

evals/handlers/setup-context.ts

@@ -69,7 +69,7 @@ export async function fetchContext(context: Context, question: string): Promise<
   logger.debug(`Fetched similar issues: ${JSON.stringify(similarIssues)}`);
 
   // Rerank similar content
-  const { similarIssues: rerankedIssues, similarComments: rerankedComments } = await reranker.reRankSimilarContent(question, similarIssues, similarComments);
+  const { similarIssues: rerankedIssues, similarComments: rerankedComments } = await reranker.reRankSimilarContent(similarIssues, similarComments, question);
 
   // Calculate token usage from reranked content
   const similarText = [

src/adapters/voyage/helpers/rerankers.ts

@@ -120,9 +120,9 @@ export class Rerankers extends SuperVoyage {
   }
 
   async reRankSimilarContent(
-    query: string,
     similarIssues: SimilarIssue[],
     similarComments: SimilarComment[],
+    query: string,
     topK: number = 5
   ): Promise<{ similarIssues: SimilarIssue[]; similarComments: SimilarComment[] }> {
     try {

src/handlers/ask-llm.ts

@@ -46,7 +46,7 @@ export async function askQuestion(context: Context, question: string): Promise<C
     logger.debug(`Fetched similar issues: ${JSON.stringify(similarIssues)}`);
 
     // Rerank similar content
-    const { similarIssues: rerankedIssues, similarComments: rerankedComments } = await reranker.reRankSimilarContent(question, similarIssues, similarComments);
+    const { similarIssues: rerankedIssues, similarComments: rerankedComments } = await reranker.reRankSimilarContent(similarIssues, similarComments, question);
 
     // Calculate token usage from reranked content
     const similarText = [

src/handlers/comment-created-callback.ts

@@ -17,9 +17,8 @@ export async function processCommentCallback(context: Context<"issue_comment.cre
     question = command.parameters.question;
   } else if (payload.comment.body.trim().startsWith("/ask")) {
     question = payload.comment.body.trim().replace("/ask", "").trim();
-  }
-  if (!question) {
-    throw logger.error("No question provided");
+  } else if (!question) {
+    return { status: 200, reason: logger.info("No question found in comment. Skipping.").logMessage.raw };
   }
 
   try {

src/helpers/format-chat-history.ts

@@ -359,7 +359,7 @@ async function processNodeContent(
       for (const comment of sortedComments) {
         if (!comment.body?.trim()) continue;
 
-        const commentLine = `${childPrefix}├── ${comment.commentType || "issuecomment"}-${comment.id}: ${comment.user}: ${comment.body.trim()}`;
+        const commentLine = `${childPrefix}├── ${comment.commentType || "issue_comment"}-${comment.id}: ${comment.user}: ${comment.body.trim()}`;
 
         if (!updateTokenCount(commentLine, testTokenLimits)) {
           break;
@@ -389,7 +389,7 @@ async function processNodeContent(
     ] as const) {
       if (!items?.length) continue;
 
-      const typeHeader = `${childPrefix}${type}:`;
+      const typeHeader = `${childPrefix}${type}`;
       if (!updateTokenCount(typeHeader, testTokenLimits)) break;
       output.push(typeHeader);
 
@@ -553,10 +553,10 @@ export async function formatChatHistory(
   const headerLine = "Issue Tree Structure:";
   treeOutput.push(headerLine, "");
 
-  const tokenLimitsnew = createDefaultTokenLimits(context);
+  const tokenLimitsNew = createDefaultTokenLimits(context);
 
-  const isSuccess = await processTreeNode(reRankedChat, "", treeOutput, tokenLimitsnew);
-  logger.debug(`Tree processing ${isSuccess ? "succeeded" : "failed"} with tokens: ${tokenLimitsnew.runningTokenCount}/${tokenLimitsnew.tokensRemaining}`);
+  const isSuccess = await processTreeNode(reRankedChat, "", treeOutput, tokenLimitsNew);
+  logger.debug(`Tree processing ${isSuccess ? "succeeded" : "failed"} with tokens: ${tokenLimitsNew.runningTokenCount}/${tokenLimitsNew.tokensRemaining}`);
   logger.debug(`Tree fetching tokens: ${tokenLimits.runningTokenCount}/${tokenLimits.tokensRemaining}`);
   return treeOutput;
 }

tests/__mocks__/handlers.ts

@@ -1,11 +1,140 @@
-import { http, HttpResponse } from "msw";
+import { http, HttpResponse, graphql } from "msw";
 import { db } from "./db";
 import issueTemplate from "./issue-template";
 
 /**
  * Intercepts the routes and returns a custom payload
  */
 export const handlers = [
+  // GraphQL handler for fetching comment by ID
+  graphql.query("GetCommentById", ({ variables }) => {
+    // Try to find an issue comment first
+    const comment = db.comments.findFirst({
+      where: { id: { equals: Number(variables.id) } },
+    });
+
+    if (comment) {
+      // If it's an issue comment
+      if (comment.issue_url) {
+        const issue = db.issue.findFirst({
+          where: { number: { equals: comment.issue_number } },
+        });
+
+        return HttpResponse.json({
+          data: {
+            node: {
+              __typename: "IssueComment",
+              id: String(comment.id),
+              body: comment.body,
+              author: {
+                login: comment.user.login,
+              },
+              issue: {
+                id: String(issue?.id),
+                number: issue?.number,
+                title: issue?.title,
+                url: issue?.html_url,
+                repository: {
+                  name: comment.repo,
+                  owner: {
+                    login: comment.owner,
+                  },
+                },
+              },
+            },
+          },
+        });
+      }
+      // If it's a pull request review comment
+      else if (comment.pull_request_url) {
+        const pull = db.pull.findFirst({
+          where: { number: { equals: comment.issue_number } },
+        });
+
+        return HttpResponse.json({
+          data: {
+            node: {
+              __typename: "PullRequestReviewComment",
+              id: String(comment.id),
+              body: comment.body,
+              author: {
+                login: comment.user.login,
+              },
+              pullRequest: {
+                id: String(pull?.id),
+                number: pull?.number,
+                title: pull?.title,
+                url: pull?.html_url,
+                repository: {
+                  name: comment.repo,
+                  owner: {
+                    login: comment.owner,
+                  },
+                },
+              },
+            },
+          },
+        });
+      }
+    }
+
+    // If no comment found
+    return HttpResponse.json({
+      data: {
+        node: null,
+      },
+    });
+  }),
+
+  // GraphQL handler for fetching issue by ID
+  graphql.query("GetIssueById", ({ variables }) => {
+    const issue = db.issue.findFirst({
+      where: { id: { equals: Number(variables.id) } },
+    });
+
+    if (!issue) {
+      return HttpResponse.json({
+        data: {
+          node: null,
+        },
+      });
+    }
+
+    const comments = db.comments.findMany({
+      where: { issue_number: { equals: issue.number } },
+    });
+
+    return HttpResponse.json({
+      data: {
+        node: {
+          id: String(issue.id),
+          number: issue.number,
+          title: issue.title,
+          body: issue.body,
+          url: issue.html_url,
+          repository: {
+            name: issue.repo,
+            owner: {
+              login: issue.owner,
+            },
+          },
+          author: {
+            login: "ubiquity",
+          },
+          comments: {
+            nodes: comments.map((comment) => ({
+              id: String(comment.id),
+              body: comment.body,
+              author: {
+                login: comment.user.login,
+              },
+            })),
+          },
+        },
+      },
+    });
+  }),
+
   http.post("https://api.openai.com/v1/chat/completions", () => {
     const answer = `${JSON.stringify(["This is a mock response from OpenAI"])}`;
 

tests/main.test.ts

@@ -12,13 +12,17 @@ import { CompletionsType } from "../src/adapters/openai/helpers/completions";
 import { logger } from "../src/helpers/errors";
 import { Octokit } from "@octokit/rest";
 import { createKey } from "../src/helpers/issue-fetching";
+import { SimilarComment, SimilarIssue, TreeNode } from "../src/types/github-types";
 
 const TEST_QUESTION = "what is pi?";
 const LOG_CALLER = "_Logs.<anonymous>";
 const ISSUE_ID_2_CONTENT = "More context here #2";
 const ISSUE_ID_3_CONTENT = "More context here #3";
 const MOCK_ANSWER = "This is a mock answer for the chat";
 const SPEC = "This is a demo spec for a demo task just perfect for testing.";
+const BASE_LINK = "https://github.com/ubiquity/test-repo/issues/";
+const ISSUE_BODY_BASE = "Related to issue";
+const ISSUE_BODY_BASE_2 = "Just another issue";
 
 type Comment = {
   id: number;
@@ -94,6 +98,7 @@ describe("Ask plugin tests", () => {
 
   it("should construct the chat history correctly", async () => {
     const ctx = createContext(TEST_QUESTION);
+    const debugSpy = jest.spyOn(ctx.logger, "debug");
     const infoSpy = jest.spyOn(ctx.logger, "info");
     createComments([
       transformCommentTemplate(1, 1, ISSUE_ID_2_CONTENT, "ubiquity", "test-repo", true, "2"),
@@ -106,36 +111,41 @@ describe("Ask plugin tests", () => {
     await issueCommentCreatedCallback(ctx);
 
     const expectedOutput = [
-      "Formatted chat history Issue Tree Structure:",
+      "Formatted chat history: Issue Tree Structure:",
       "",
-      "Issue #1 (https://github.com/ubiquity/test-repo/issues/1)",
+      "Issue #1 (" + BASE_LINK + "1)",
       "Body:",
       `      ${SPEC}`,
       "",
       "Comments: 2",
-      `├── issuecomment-1: ubiquity: More context here #2 [#2](https://github.com/ubiquity/test-repo/issues/2)`,
-      `└── issuecomment-2: ubiquity: ${TEST_QUESTION} [#1](https://github.com/ubiquity/test-repo/issues/1)`,
+      `├── issue_comment-2: ubiquity: ${TEST_QUESTION} [#1](${BASE_LINK}1)`,
+      `├── issue_comment-1: ubiquity: ${ISSUE_ID_2_CONTENT} [#2](${BASE_LINK}2)`,
       "",
-      "      └── Issue #2 (https://github.com/ubiquity/test-repo/issues/2)",
-      "          Body:",
-      `              Related to issue #3`,
+      "Similar Issues:",
+      "- Issue #2 (" + BASE_LINK + "2) - Similarity: 50.00%",
+      `  ${ISSUE_BODY_BASE} #3`,
+      "- Issue #3 (" + BASE_LINK + "3) - Similarity: 30.00%",
+      `  ${ISSUE_BODY_BASE_2}`,
       "",
-      "          Comments: 1",
-      `          └── issuecomment-3: ubiquity: ${ISSUE_ID_3_CONTENT} [#3](https://github.com/ubiquity/test-repo/issues/3)`,
+      "└── Issue #3 (" + BASE_LINK + "3)",
+      "    Body:",
+      `        ${ISSUE_BODY_BASE_2}`,
+      "    Comments: 1",
+      `    ├── issue_comment-4: ubiquity: Just a comment [#1](${BASE_LINK}1)`,
       "",
-      "              └── Issue #3 (https://github.com/ubiquity/test-repo/issues/3)",
-      "                  Body:",
-      `                      Just another issue`,
-      "                  Comments: 1",
-      `                  └── issuecomment-4: ubiquity: Just a comment [#1](https://github.com/ubiquity/test-repo/issues/1)`,
+      "    └── Issue #2 (" + BASE_LINK + "2)",
+      "        Body:",
+      `            ${ISSUE_BODY_BASE} #3`,
+      "        Comments: 1",
+      `        ├── issue_comment-3: ubiquity: ${ISSUE_ID_3_CONTENT} [#3](${BASE_LINK}3)`,
       "",
     ].join("\n");
 
     // Find the index of the formatted chat history log
-    const chatHistoryLogIndex = infoSpy.mock.calls.findIndex((call) => (call[0] as string).startsWith("Formatted chat history"));
+    const chatHistoryLogIndex = debugSpy.mock.calls.findIndex((call) => (call[0] as string).startsWith("Formatted chat history: Issue Tree Structure:"));
 
     const normalizedExpected = normalizeString(expectedOutput);
-    const normalizedReceived = normalizeString(infoSpy.mock.calls[chatHistoryLogIndex][0] as string);
+    const normalizedReceived = normalizeString(debugSpy.mock.calls[chatHistoryLogIndex][0] as string);
     expect(normalizedReceived).toEqual(normalizedExpected);
 
     // Find the index of the answer log
@@ -173,7 +183,7 @@ function transformCommentTemplate(commentId: number, issueNumber: number, body:
     },
     body: body,
     url: "https://api.github.com/repos/ubiquity/test-repo/issues/comments/1",
-    html_url: "https://github.com/ubiquity/test-repo/issues/1",
+    html_url: BASE_LINK + "1",
     owner: "ubiquity",
     repo: "test-repo",
     issue_number: 1,
@@ -219,17 +229,17 @@ async function setupTests() {
     ...issueTemplate,
     id: 2,
     number: 2,
-    body: "Related to issue #3",
-    html_url: "https://github.com/ubiquity/test-repo/issues/2",
+    body: `${ISSUE_BODY_BASE} #3`,
+    html_url: BASE_LINK + "2",
     url: "https://api.github.com/repos/ubiquity/test-repo/issues/2",
   });
 
   db.issue.create({
     ...issueTemplate,
     id: 3,
     number: 3,
-    body: "Just another issue",
-    html_url: "https://github.com/ubiquity/test-repo/issues/3",
+    body: ISSUE_BODY_BASE_2,
+    html_url: BASE_LINK + "3",
     url: "https://api.github.com/repos/ubiquity/test-repo/issues/3",
   });
 }
@@ -293,7 +303,7 @@ function createContext(body = TEST_QUESTION) {
             return [
               {
                 issue_id: "2",
-                issue_plaintext: "Related to issue #3",
+                issue_plaintext: `${ISSUE_BODY_BASE} #3`,
                 similarity: 0.5,
               },
               {
@@ -388,6 +398,15 @@ function createContext(body = TEST_QUESTION) {
           reRankResults: async (similarText: string[]) => {
             return similarText;
           },
+          reRankSimilarContent: async (similarIssues: SimilarIssue[], similarComments: SimilarComment[]) => {
+            return {
+              similarIssues,
+              similarComments,
+            };
+          },
+          reRankTreeNodes: async (rootNode: TreeNode) => {
+            return rootNode;
+          },
         },
       },
       openai: {
@@ -409,6 +428,9 @@ function createContext(body = TEST_QUESTION) {
               },
             };
           },
+          getPromptTokens: async (query: string): Promise<number> => {
+            return query ? query.length : 100;
+          },
           findTokenLength: async () => {
             return 1000;
           },