Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

telemetry(amazonq): Adding cwsprChatProgrammingLanguage parameter to insertAtCursor and CopyAtClipboard events #4959

Merged
merged 19 commits into from
Oct 16, 2024
Merged

telemetry(amazonq): Adding cwsprChatProgrammingLanguage parameter to insertAtCursor and CopyAtClipboard events #4959

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
fa8c411
Adding Acceptance metrics for UTG
laileni-aws Oct 9, 2024
89ae57f
Fixing lint errors
laileni-aws Oct 9, 2024
d0b4b50
Merge branch 'main' into main
laileni-aws Oct 9, 2024
4fb43b7
Merge branch 'main' into main
laileni-aws Oct 9, 2024
fefcb99
Adding Language to user events
laileni-aws Oct 10, 2024
a883685
Merge branch 'testMetrics' into main
laileni-aws Oct 10, 2024
3aacba4
minor fixes
laileni-aws Oct 10, 2024
f4ae707
Merge branch 'main' into main
laileni-aws Oct 10, 2024
7a6e925
Merge branch 'main' into main
laileni-aws Oct 10, 2024
7b6fbd1
minor edits
laileni-aws Oct 11, 2024
d9a0fbf
Merge branch 'main' into main
laileni-aws Oct 11, 2024
fc6a8a1
minor edits in Test files
laileni-aws Oct 11, 2024
da650e8
Addressing comments and adding cwsprChatProgrammingLanguage to common…
laileni-aws Oct 15, 2024
29d8df3
minor edits
laileni-aws Oct 15, 2024
20237b1
Bumping the telemetry version
laileni-aws Oct 15, 2024
1e18908
Moved extractCodeBlockLanguage to TextUtils and added test cases
laileni-aws Oct 16, 2024
ee23f93
Minor edits
laileni-aws Oct 16, 2024
b3a7e35
Minor edits
laileni-aws Oct 16, 2024
d1e87d9
Merge branch 'main' into main
laileni-aws Oct 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion gradle/libs.versions.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ mockitoKotlin = "5.4.0"
mockk = "1.13.10"
nimbus-jose-jwt = "9.40"
node-gradle = "7.0.2"
telemetryGenerator = "1.0.262"
telemetryGenerator = "1.0.272"
testLogger = "4.0.0"
testRetry = "1.5.10"
# test-only; platform provides slf4j transitively at runtime
Expand Down
8 changes: 8 additions & 0 deletions ...ftware/aws/toolkits/jetbrains/services/cwc/controller/chat/messenger/ChatPromptHandler.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import kotlinx.coroutines.flow.onStart
import software.amazon.awssdk.awscore.exception.AwsServiceException
import software.amazon.awssdk.services.codewhispererstreaming.model.CodeWhispererStreamingException
import software.aws.toolkits.core.utils.convertMarkdownToHTML
import software.aws.toolkits.core.utils.extractCodeBlockLanguage
import software.aws.toolkits.jetbrains.services.cwc.clients.chat.exceptions.ChatApiException
import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatRequestData
import software.aws.toolkits.jetbrains.services.cwc.clients.chat.model.ChatResponseEvent
Expand All @@ -33,6 +34,8 @@ class ChatPromptHandler(private val telemetryHelper: TelemetryHelper) {
private val codeReferences = mutableListOf<CodeReference>()
private var requestId: String = ""
private var statusCode: Int = 0
private val defaultTestGenResponseLanguage: String = "plaintext"
private var codeBlockLanguage: String = defaultTestGenResponseLanguage

companion object {
private val CODE_BLOCK_PATTERN = Regex("<pre>\\s*<code")
Expand Down Expand Up @@ -209,6 +212,10 @@ class ChatPromptHandler(private val telemetryHelper: TelemetryHelper) {
} else {
responseText.toString()
}
if (codeBlockLanguage == defaultTestGenResponseLanguage) {
// To get the language of generated code in Q chat.
codeBlockLanguage = extractCodeBlockLanguage(message)
}
ChatMessage(
tabId = tabId,
triggerId = triggerId,
Expand All @@ -217,6 +224,7 @@ class ChatPromptHandler(private val telemetryHelper: TelemetryHelper) {
message = message,
codeReference = codeReferences,
userIntent = data.userIntent,
codeBlockLanguage = codeBlockLanguage,
)
} else {
null
Expand Down
6 changes: 4 additions & 2 deletions ...software/aws/toolkits/jetbrains/services/cwc/controller/chat/telemetry/TelemetryHelper.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,8 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
credentialStartUrl = getStartUrl(context.project),
cwsprChatCodeBlockIndex = message.codeBlockIndex?.toLong(),
cwsprChatTotalCodeBlocks = message.totalCodeBlocks?.toLong(),
cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId)
cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId),
cwsprChatProgrammingLanguage = message.codeBlockLanguage,
)
ChatInteractWithMessageEvent.builder().apply {
conversationId(getConversationId(message.tabId).orEmpty())
Expand All @@ -244,7 +245,8 @@ class TelemetryHelper(private val context: AmazonQAppInitContext, private val se
credentialStartUrl = getStartUrl(context.project),
cwsprChatCodeBlockIndex = message.codeBlockIndex?.toLong(),
cwsprChatTotalCodeBlocks = message.totalCodeBlocks?.toLong(),
cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId)
cwsprChatHasProjectContext = getMessageHasProjectContext(message.messageId),
cwsprChatProgrammingLanguage = message.codeBlockLanguage,
)
ChatInteractWithMessageEvent.builder().apply {
conversationId(getConversationId(message.tabId).orEmpty())
Expand Down
3 changes: 3 additions & 0 deletions ...tbrains-community/src/software/aws/toolkits/jetbrains/services/cwc/messages/CwcMessage.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ sealed interface IncomingCwcMessage : CwcMessage {
val eventId: String?,
val codeBlockIndex: Int?,
val totalCodeBlocks: Int?,
val codeBlockLanguage: String?,
) : IncomingCwcMessage

data class InsertCodeAtCursorPosition(
Expand All @@ -84,6 +85,7 @@ sealed interface IncomingCwcMessage : CwcMessage {
val eventId: String?,
val codeBlockIndex: Int?,
val totalCodeBlocks: Int?,
val codeBlockLanguage: String?,
) : IncomingCwcMessage

data class TriggerTabIdReceived(
Expand Down Expand Up @@ -214,6 +216,7 @@ data class ChatMessage(
val relatedSuggestions: List<Suggestion>? = null,
val codeReference: List<CodeReference>? = null,
val userIntent: UserIntent? = null,
val codeBlockLanguage: String? = "plaintext",
) : UiMessage(
tabId = tabId,
type = "chatMessage",
Expand Down
6 changes: 4 additions & 2 deletions ...ins-community/tst/software/aws/toolkits/jetbrains/services/amazonq/TelemetryHelperTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -420,7 +420,8 @@ class TelemetryHelperTest {
"insertionTargetType",
"eventId",
codeBlockIndex,
totalCodeBlocks
totalCodeBlocks,
lang
)
)

Expand Down Expand Up @@ -484,7 +485,8 @@ class TelemetryHelperTest {
emptyList(),
eventId,
codeBlockIndex,
totalCodeBlocks
totalCodeBlocks,
lang
)
)

Expand Down
10 changes: 8 additions & 2 deletions plugins/amazonq/mynah-ui/src/mynah-ui/ui/apps/cwChatConnector.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ export class Connector {
codeBlockIndex?: number,
totalCodeBlocks?: number,
userIntent?: string,
codeBlockLanguage?: string,
): void => {
this.sendMessageToExtension({
tabID: tabID,
Expand All @@ -113,7 +114,8 @@ export class Connector {
eventId,
codeBlockIndex,
totalCodeBlocks,
userIntent
userIntent,
codeBlockLanguage
})
}

Expand All @@ -127,6 +129,7 @@ export class Connector {
codeBlockIndex?: number,
totalCodeBlocks?: number,
userIntent?: string,
codeBlockLanguage?: string,
): void => {
this.sendMessageToExtension({
tabID: tabID,
Expand All @@ -139,7 +142,8 @@ export class Connector {
eventId,
codeBlockIndex,
totalCodeBlocks,
userIntent
userIntent,
codeBlockLanguage
})
}

Expand Down Expand Up @@ -271,6 +275,7 @@ export class Connector {
canBeVoted: true,
codeReference: messageData.codeReference,
userIntent: messageData.userIntent,
codeBlockLanguage: messageData.codeBlockLanguage,
}

// If it is not there we will not set it
Expand Down Expand Up @@ -304,6 +309,7 @@ export class Connector {
messageId: messageData.messageId,
codeReference: messageData.codeReference,
userIntent: messageData.userIntent,
codeBlockLanguage: messageData.codeBlockLanguage,
followUp:
messageData.followUps !== undefined && messageData.followUps.length > 0
? {
Expand Down
15 changes: 10 additions & 5 deletions plugins/amazonq/mynah-ui/src/mynah-ui/ui/connector.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ export interface ChatPayload {
}

export interface CWCChatItem extends ChatItem {
userIntent?: string
userIntent?: string,
codeBlockLanguage?: string,
}

export interface ConnectorProps {
Expand Down Expand Up @@ -235,7 +236,8 @@ export class Connector {
eventId?: string,
codeBlockIndex?: number,
totalCodeBlocks?: number,
userIntent?: string
userIntent?: string,
codeBlockLanguage?: string
): void => {
switch (this.tabsStorage.getTab(tabID)?.type) {
case 'cwc':
Expand All @@ -248,7 +250,8 @@ export class Connector {
eventId,
codeBlockIndex,
totalCodeBlocks,
userIntent
userIntent,
codeBlockLanguage
)
break
case 'featuredev':
Expand All @@ -266,7 +269,8 @@ export class Connector {
eventId?: string,
codeBlockIndex?: number,
totalCodeBlocks?: number,
userIntent?: string
userIntent?: string,
codeBlockLanguage?: string
): void => {
switch (this.tabsStorage.getTab(tabID)?.type) {
case 'cwc':
Expand All @@ -279,7 +283,8 @@ export class Connector {
eventId,
codeBlockIndex,
totalCodeBlocks,
userIntent
userIntent,
codeBlockLanguage
)
break
case 'featuredev':
Expand Down
12 changes: 7 additions & 5 deletions plugins/amazonq/mynah-ui/src/mynah-ui/ui/main.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export const createMynahUI = (ideApi: any, featureDevInitEnabled: boolean, codeT
let mynahUI: MynahUI
// eslint-disable-next-line prefer-const
let connector: Connector
const messageUserIntentMap = new Map<string, string>()
const responseMetadata = new Map<string, string[]>()

const tabsStorage = new TabsStorage({
onTabTimeout: tabID => {
Expand Down Expand Up @@ -254,8 +254,8 @@ export const createMynahUI = (ideApi: any, featureDevInitEnabled: boolean, codeT
? { type: ChatItemType.CODE_RESULT, fileList: item.fileList }
: {}),
})
if (item.messageId !== undefined && item.userIntent !== undefined) {
messageUserIntentMap.set(item.messageId, item.userIntent)
if (item.messageId !== undefined && item.userIntent !== undefined && item.codeBlockLanguage !== undefined) {
responseMetadata.set(item.messageId, [item.userIntent, item.codeBlockLanguage])
}
return
}
Expand Down Expand Up @@ -466,7 +466,8 @@ export const createMynahUI = (ideApi: any, featureDevInitEnabled: boolean, codeT
eventId,
codeBlockIndex,
totalCodeBlocks,
messageUserIntentMap.get(messageId) ?? undefined
responseMetadata.get(messageId)?.[0] ?? undefined,
responseMetadata.get(messageId)?.[1] ?? undefined
)
break
case 'copy':
Expand All @@ -479,7 +480,8 @@ export const createMynahUI = (ideApi: any, featureDevInitEnabled: boolean, codeT
eventId,
codeBlockIndex,
totalCodeBlocks,
messageUserIntentMap.get(messageId) ?? undefined
responseMetadata.get(messageId)?.[0] ?? undefined,
responseMetadata.get(messageId)?.[1] ?? undefined
)
mynahUI.notify({
type: NotificationType.SUCCESS,
Expand Down
19 changes: 19 additions & 0 deletions plugins/core/core/src/software/aws/toolkits/core/utils/TextUtils.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,25 @@ fun convertMarkdownToHTML(markdown: String): String {
return htmlRenderer.render(document)
}

fun extractCodeBlockLanguage(message: String): String {
// This fulfills both the cases of unit test generation(java, python) and general use case(Non java and Non python) languages.
val defaultTestGenResponseLanguage = "plaintext"
val indexStart = 3
val codeBlockStart = message.indexOf("```")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

are there any other edge cases with this we need to worry about?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  • If we use this extractCodeBlokLanguage, we assume code generated should start or at-least will have 3 backticks with language name.
  • So If generated code doesn't provide language after ``` we assume the language to be plaintext
  • Even for counting number of code blocks, we are implementing this approach for recognizing code blocks with 3 backticks, until backend provide IDE this data.
  • Right now this is used only for telemetry purposes

if (codeBlockStart == -1) {
return defaultTestGenResponseLanguage
}

val languageStart = codeBlockStart + indexStart
val languageEnd = message.indexOf('\n', languageStart)

if (languageEnd == -1) {
return defaultTestGenResponseLanguage
}

return message.substring(languageStart, languageEnd).trim().ifEmpty { defaultTestGenResponseLanguage }
}

class CodeBlockRenderer(private val html: HtmlWriter) : NodeRenderer {
override fun getNodeTypes(): Set<Class<out Node>> = setOf(FencedCodeBlock::class.java)
override fun render(node: Node?) {
Expand Down
56 changes: 0 additions & 56 deletions plugins/core/jetbrains-community/resources/telemetryOverride.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,11 +99,6 @@
"type": "int",
"description": "Cpu usage of LSP server"
},
{
"name": "cwsprChatProgrammingLanguage",
"type": "string",
"description": "Programming language associated with the message"
},
{
"name": "cwsprChatConversationType",
"type": "string",
Expand Down Expand Up @@ -644,57 +639,6 @@
}
]
},
{
"name": "amazonq_interactWithMessage",
"description": "When a user interacts with a message in the conversation",
"metadata": [
{
"type": "cwsprChatConversationId"
},
{
"type": "cwsprChatMessageId"
},
{
"type": "cwsprChatUserIntent",
"required": false
},
{
"type": "cwsprChatInteractionType"
},
{
"type": "cwsprChatInteractionTarget",
"required": false
},
{
"type": "cwsprChatCodeBlockIndex",
"required": false
},
{
"type": "cwsprChatTotalCodeBlocks",
"required": false
},
{
"type": "cwsprChatAcceptedCharactersLength",
"required": false
},
{
"type": "cwsprChatAcceptedNumberOfLines",
"required": false
},
{
"type": "cwsprChatHasReference",
"required": false
},
{
"type": "credentialStartUrl",
"required": false
},
{
"type": "cwsprChatHasProjectContext",
"required": false
}
]
},
{
"name": "amazonq_modifyCode",
"description": "Percentage of code modified by the user after copying/inserting code from a message",
Expand Down
Loading
Loading