Skip to content

Commit

Permalink
[Console Insights] Render direct citations from Search RAG
Browse files Browse the repository at this point in the history 
Screenshot: https://i.imgur.com/68Crmvd.png

Bug: 381228083
Change-Id: I613c86da3cfe41a40a124134436a85d3f0e9365f
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/6138609
Commit-Queue: Alex Rudenko <[email protected]>
Auto-Submit: Wolfgang Beyer <[email protected]>
Reviewed-by: Alex Rudenko <[email protected]>
  • Loading branch information
@wolfib
wolfib authored and Devtools-frontend LUCI CQ committed Jan 8, 2025
1 parent e32aa43 commit 45e6cc5
Show file tree
Hide file tree
Showing 7 changed files with 209 additions and 24 deletions.
6 changes: 3 additions & 3 deletions front_end/core/host/AidaClient.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ describeWithEnvironment('AidaClient', () => {
metadata: {
rpcGlobalId: 123,
attributionMetadata:
{attributionAction: 'CITE', citations: [{startIndex: 0, endIndex: 1, url: 'https://example.com'}]},
{attributionAction: 'CITE', citations: [{startIndex: 0, endIndex: 1, uri: 'https://example.com'}]},
},
},
]);
Expand All @@ -321,7 +321,7 @@ describeWithEnvironment('AidaClient', () => {
rpcGlobalId: 123,
attributionMetadata: {
attributionAction: Host.AidaClient.RecitationAction.CITE,
citations: [{startIndex: 0, endIndex: 1, url: 'https://example.com'}],
citations: [{startIndex: 0, endIndex: 1, uri: 'https://example.com'}],
},
},
completed: false,
Expand All @@ -333,7 +333,7 @@ describeWithEnvironment('AidaClient', () => {
rpcGlobalId: 123,
attributionMetadata: {
attributionAction: Host.AidaClient.RecitationAction.CITE,
citations: [{startIndex: 0, endIndex: 1, url: 'https://example.com'}],
citations: [{startIndex: 0, endIndex: 1, uri: 'https://example.com'}],
},
},
functionCalls: undefined,
Expand Down
16 changes: 13 additions & 3 deletions front_end/core/host/AidaClient.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,10 +194,20 @@ export enum RecitationAction {
EXEMPT_FOUND_IN_PROMPT = 'EXEMPT_FOUND_IN_PROMPT',
}

export enum CitationSourceType {
CITATION_SOURCE_TYPE_UNSPECIFIED = 'CITATION_SOURCE_TYPE_UNSPECIFIED',
TRAINING_DATA = 'TRAINING_DATA',
WORLD_FACTS = 'WORLD_FACTS',
LOCAL_FACTS = 'LOCAL_FACTS',
INDIRECT = 'INDERECT',
}

export interface Citation {
startIndex: number;
endIndex: number;
url: string;
startIndex?: number;
endIndex?: number;
uri?: string;
sourceType?: CitationSourceType;
repository?: string;
}

export interface AttributionMetadata {
Expand Down
62 changes: 62 additions & 0 deletions front_end/panels/explain/components/ConsoleInsight.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,4 +271,66 @@ describeWithEnvironment('ConsoleInsight', () => {
assert.strictEqual(xLinks[1].textContent?.trim(), 'https://www.anotherSource.test/page');
assert.strictEqual(xLinks[1].getAttribute('href'), 'https://www.anotherSource.test/page');
});

it('displays direct citations', async () => {
function getAidaClientWithMetadata() {
return {
async *
fetch() {
yield {
explanation: 'This is not a real answer, it is just a test.',
metadata: {
rpcGlobalId: 0,
attributionMetadata: {
attributionAction: Host.AidaClient.RecitationAction.CITE,
citations: [
{
startIndex: 0,
endIndex: 10,
uri: 'https://www.wiki.test/directSource',
sourceType: Host.AidaClient.CitationSourceType.WORLD_FACTS,
},
{
startIndex: 20,
endIndex: 25,
uri: 'https://www.world-fact.test/',
sourceType: Host.AidaClient.CitationSourceType.WORLD_FACTS,
},
],
},
factualityMetadata: {
facts: [
{sourceUri: 'https://www.firstSource.test/someInfo'},
],
},
},
completed: true,
};
},
registerClientEvent: sinon.spy(),
};
}

component = new Explain.ConsoleInsight(
getTestPromptBuilder(), getAidaClientWithMetadata(), Host.AidaClient.AidaAccessPreconditions.AVAILABLE);
renderElementIntoDOM(component);
await drainMicroTasks();

const markdownView = component.shadowRoot!.querySelector('devtools-markdown-view');
assert.strictEqual(
getCleanTextContentFromElements(markdownView!.shadowRoot!, '.message')[0],
'This is not [1] a real answer [2] , it is just a test.');
const details = component.shadowRoot!.querySelector('details');
assert.strictEqual(details!.querySelector('summary')!.textContent?.trim(), 'Sources and related content');
const directCitations = details!.querySelectorAll('ol x-link');
assert.lengthOf(directCitations, 2);
assert.strictEqual(directCitations[0].textContent?.trim(), 'https://www.wiki.test/directSource');
assert.strictEqual(directCitations[0].getAttribute('href'), 'https://www.wiki.test/directSource');
assert.strictEqual(directCitations[1].textContent?.trim(), 'https://www.world-fact.test/');
assert.strictEqual(directCitations[1].getAttribute('href'), 'https://www.world-fact.test/');
const relatedContent = details!.querySelectorAll('ul x-link');
assert.lengthOf(relatedContent, 1);
assert.strictEqual(relatedContent[0].textContent?.trim(), 'https://www.firstSource.test/someInfo');
assert.strictEqual(relatedContent[0].getAttribute('href'), 'https://www.firstSource.test/someInfo');
});
});
120 changes: 104 additions & 16 deletions front_end/panels/explain/components/ConsoleInsight.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ type StateData = {
sources: Source[],
isPageReloadRecommended: boolean,
completed: boolean,
directCitationUrls: string[],
}&Host.AidaClient.AidaResponse|{
type: State.ERROR,
error: string,
Expand All @@ -211,6 +212,27 @@ type StateData = {
type: State.OFFLINE,
};

const markedExtension = {
name: 'citation',
level: 'inline',
start(src: string) {
return src.match(/<cite/)?.index;
},
tokenizer(src: string) {
const match = src.match(/^<cite href="(\S+)">(\S+)<\/cite>/);
if (match) {
return {
type: 'citation',
raw: match[0],
linkTarget: match[1],
linkText: match[2],
};
}
return false;
},
renderer: () => '',
};

export class ConsoleInsight extends HTMLElement {
static async create(promptBuilder: PublicPromptBuilder, aidaClient: PublicAidaClient): Promise<ConsoleInsight> {
const aidaAvailability = await Host.AidaClient.AidaClient.checkAccessPreconditions();
Expand All @@ -232,6 +254,7 @@ export class ConsoleInsight extends HTMLElement {
#consoleInsightsEnabledSetting: Common.Settings.Setting<boolean>|undefined;
#aidaAvailability: Host.AidaClient.AidaAccessPreconditions;
#boundOnAidaAvailabilityChange: () => Promise<void>;
#marked: Marked.Marked.Marked;

constructor(
promptBuilder: PublicPromptBuilder, aidaClient: PublicAidaClient,
Expand All @@ -241,6 +264,7 @@ export class ConsoleInsight extends HTMLElement {
this.#aidaClient = aidaClient;
this.#aidaAvailability = aidaAvailability;
this.#consoleInsightsEnabledSetting = this.#getConsoleInsightsEnabledSetting();
this.#marked = new Marked.Marked.Marked({extensions: [markedExtension]});

this.#state = this.#getStateFromAidaAvailability();
this.#boundOnAidaAvailabilityChange = this.#onAidaAvailabilityChange.bind(this);
Expand Down Expand Up @@ -461,10 +485,42 @@ export class ConsoleInsight extends HTMLElement {
await this.#generateInsight();
}

#insertCitations(explanation: string, metadata: Host.AidaClient.AidaResponseMetadata):
{explanationWithCitations: string, directCitationUrls: string[]} {
const directCitationUrls: string[] = [];
if (!this.#isSearchRagResponse(metadata) || !metadata.attributionMetadata) {
return {explanationWithCitations: explanation, directCitationUrls};
}

const {attributionMetadata} = metadata;
const sortedCitations =
attributionMetadata.citations
.filter(citation => citation.sourceType === Host.AidaClient.CitationSourceType.WORLD_FACTS)
.sort((a, b) => (b.endIndex || 0) - (a.endIndex || 0));
let explanationWithCitations = explanation;
for (const [index, citation] of sortedCitations.entries()) {
// Matches optional punctuation mark followed by whitespace.
// Ensures citation is placed at the end of a word.
const myRegex = /[.,:;!?]*\s/g;
myRegex.lastIndex = citation.endIndex || 0;
const result = myRegex.exec(explanationWithCitations);
if (result && citation.uri) {
explanationWithCitations = explanationWithCitations.slice(0, result.index) +
`<cite href="${citation.uri}">${sortedCitations.length - index}</cite>` +
explanationWithCitations.slice(result.index);
directCitationUrls.push(citation.uri);
}
}

directCitationUrls.reverse();
return {explanationWithCitations, directCitationUrls};
}

async #generateInsight(): Promise<void> {
try {
for await (const {sources, isPageReloadRecommended, explanation, metadata, completed} of this.#getInsight()) {
const tokens = this.#validateMarkdown(explanation);
const {explanationWithCitations, directCitationUrls} = this.#insertCitations(explanation, metadata);
const tokens = this.#validateMarkdown(explanationWithCitations);
const valid = tokens !== false;
this.#transitionTo({
type: State.INSIGHT,
Expand All @@ -475,6 +531,7 @@ export class ConsoleInsight extends HTMLElement {
metadata,
isPageReloadRecommended,
completed,
directCitationUrls,
});
}
Host.userMetrics.actionTaken(Host.UserMetrics.Action.InsightGenerated);
Expand All @@ -492,7 +549,7 @@ export class ConsoleInsight extends HTMLElement {
*/
#validateMarkdown(text: string): Marked.Marked.TokensList|false {
try {
const tokens = Marked.Marked.lexer(text);
const tokens = this.#marked.lexer(text);
for (const token of tokens) {
this.#renderer.renderToken(token);
}
Expand Down Expand Up @@ -567,34 +624,59 @@ export class ConsoleInsight extends HTMLElement {
// clang-format on
}

#maybeRenderSources(): LitHtml.LitTemplate {
if (this.#state.type !== State.INSIGHT || !this.#state.directCitationUrls.length) {
return LitHtml.nothing;
}

// clang-format off
return html`
<ol class="sources-list">
${this.#state.directCitationUrls.map(url => html`
<li>
<x-link
href=${url}
class="link"
jslog=${VisualLogging.link('references.console-insights').track({click: true})}
>
${url}
</x-link>
</li>
`)}
</ol>
`;
// clang-format on
}

#maybeRenderRelatedContent(): LitHtml.LitTemplate {
if (this.#state.type !== State.INSIGHT || !this.#state.metadata.factualityMetadata?.facts.length) {
return LitHtml.nothing;
}
// clang-format off
return html`
<details jslog=${VisualLogging.expand('references').track({click: true})}>
<summary>${i18nString(UIStrings.references)}</summary>
<ul>
<ul>
${this.#state.metadata?.factualityMetadata?.facts.map(fact => {
return fact.sourceUri ? html`
<li>
<x-link
href=${fact.sourceUri}
class="link"
jslog=${VisualLogging.link('references.console-insights').track({click: true})}
>
<li>
<x-link
href=${fact.sourceUri}
class="link"
jslog=${VisualLogging.link('references.console-insights').track({click: true})}
>
${fact.sourceUri}
</x-link>
</li>
</x-link>
</li>
` : LitHtml.nothing;
})}
</ul>
</details>
</ul>
`;
// clang-format on
}

#isSearchRagResponse(metadata: Host.AidaClient.AidaResponseMetadata): boolean {
return Boolean(metadata.factualityMetadata?.facts.length);
}

#renderMain(): LitHtml.TemplateResult {
const jslog = `${VisualLogging.section(this.#state.type).track({resize: true})}`;
// clang-format off
Expand All @@ -619,7 +701,13 @@ export class ConsoleInsight extends HTMLElement {
.data=${{tokens: this.#state.tokens, renderer: this.#renderer, animationEnabled: true} as MarkdownView.MarkdownView.MarkdownViewData}>
</devtools-markdown-view>`: this.#state.explanation
}
${this.#maybeRenderRelatedContent()}
${this.#isSearchRagResponse(this.#state.metadata) ? html`
<details jslog=${VisualLogging.expand('references').track({click: true})}>
<summary>${i18nString(UIStrings.references)}</summary>
${this.#maybeRenderSources()}
${this.#maybeRenderRelatedContent()}
</details>
` : LitHtml.nothing}
<details jslog=${VisualLogging.expand('sources').track({click: true})}>
<summary>${i18nString(UIStrings.inputData)}</summary>
<devtools-console-insight-sources-list .sources=${this.#state.sources} .isPageReloadRecommended=${this.#state.isPageReloadRecommended}>
Expand Down
15 changes: 15 additions & 0 deletions front_end/panels/explain/components/consoleInsight.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,3 +332,18 @@ h2:focus-visible {
.close-button {
align-self: flex-start;
}

.sources-list {
padding-left: 15px;
margin-bottom: 10px;
list-style: none;
counter-reset: sources;
}

.sources-list li {
counter-increment: sources;
}

.sources-list li::marker {
content: "[" counter(sources) "]";
}
13 changes: 11 additions & 2 deletions front_end/ui/components/markdown_view/MarkdownView.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import './MarkdownImage.js';
import './MarkdownLink.js';

import type * as Marked from '../../../third_party/marked/marked.js';
import * as VisualLogging from '../../../ui/visual_logging/visual_logging.js';
import * as UI from '../../legacy/legacy.js';
import * as LitHtml from '../../lit-html/lit-html.js';

Expand Down Expand Up @@ -294,7 +295,7 @@ export class MarkdownInsightRenderer extends MarkdownLitRenderer {
return '';
}

override templateForToken(token: Marked.Marked.MarkedToken): LitHtml.TemplateResult|null {
override templateForToken(token: Marked.Marked.Token): LitHtml.TemplateResult|null {
switch (token.type) {
case 'heading':
return html`<strong>${this.renderText(token)}</strong>`;
Expand All @@ -309,9 +310,17 @@ export class MarkdownInsightRenderer extends MarkdownLitRenderer {
case 'code':
return html`<devtools-code-block
.code=${this.unescape(token.text)}
.codeLang=${this.detectCodeLanguage(token)}
.codeLang=${this.detectCodeLanguage(token as Marked.Marked.Tokens.Code)}
.displayNotice=${true}>
</devtools-code-block>`;
case 'citation':
return html`<sup>
<x-link href=${token.linkTarget} class="devtools-link" jslog=${VisualLogging.link('inline-citation').track({
click: true,
})}>
[${token.linkText}]
</x-link>
</sup>`;
}
return super.templateForToken(token as Marked.Marked.MarkedToken);
}
Expand Down
1 change: 1 addition & 0 deletions front_end/ui/visual_logging/KnownContextValues.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1682,6 +1682,7 @@ export const knownContextValues = new Set([
'initiator-address-space',
'initiator-tree',
'inline',
'inline-citation',
'inline-size',
'inline-variable-values',
'inline-variable-values-false',
Expand Down

0 comments on commit 45e6cc5

Please sign in to comment.