Skip to content

Commit

Permalink
chore: update service and settings
Browse files Browse the repository at this point in the history
  • Loading branch information
dtopalov committed Sep 26, 2024
1 parent e5bd015 commit fb9e61d
Show file tree
Hide file tree
Showing 3 changed files with 107 additions and 90 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<kendo-combobox
[data]="disabilitiesData"
[valuePrimitive]="true"
(valueChange)="onValueChange($event)"
(valueChange)="applyAIRecommendedSettings($event)"
[value]="comboboxValue"
textField="text"
valueField="text" class="k-mb-4 !k-w-full"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ import { defaultFont, SettingsService } from '../../settings.service';
import { groupBy } from '@progress/kendo-data-query';
import { SVGIcon, arrowRotateCcwIcon, fontFamilyIcon, imageResizeIcon, pauseSmIcon, underlineIcon } from '@progress/kendo-svg-icons';
import { contrastIcon, darkModeIcon, dyslexiaFontIcon, microphoneIcon } from './svg-icons';
import { map, Subscription } from 'rxjs';
import { HttpService } from '../../http.service';
import { Subscription } from 'rxjs';
import { ComboBoxComponent } from '@progress/kendo-angular-dropdowns';
import { IWindow } from '../../models/window.model';
import { OpenAIService } from '../../openai.service';

@Component({
selector: 'app-settings-list-component',
Expand Down Expand Up @@ -46,6 +46,11 @@ export class SettingsListComponent {
text: 'ADHD'
}], [{field: 'type'}]);

private isDisabilitySupported(disability: string): boolean {
const supportedDisabilities = this.disabilitiesData.flatMap(group => group.items.map(item => item.text));
return supportedDisabilities.includes(disability);
};

public recognition: any;

public resetIcon: SVGIcon = arrowRotateCcwIcon;
Expand All @@ -64,36 +69,36 @@ export class SettingsListComponent {

constructor(
private settingsService: SettingsService,
private httpService: HttpService) { }
private openAIService: OpenAIService) { }

public ngAfterViewInit(): void {
// Using a custom window interface as some of the speech recognition classes not showing up
const myWindow: IWindow = window as any;
const SpeechRecognition = myWindow.SpeechRecognition || myWindow.webkitSpeechRecognition;
this.recognition = new SpeechRecognition();
this.recognition.continuous = false;
this.recognition.lang = 'en-US';
this.recognition.interimResults = false;
this.recognition.onresult = (event: any) => {
console.log("Speech Recognition end");
const transcript: string = event.results[0][0].transcript;
console.log(`Result received: ${transcript}`)
const filtered: any = Array.from(event.results).filter(
(r: any) => r.isFinal && r[0].confidence > 0.9
);
if (filtered.length != 0) {
this.combo.searchbar.handleInput({
target: { value: transcript},
});
const myWindow: IWindow = window as any;

const SpeechRecognition = myWindow.SpeechRecognition || myWindow.webkitSpeechRecognition;

this.recognition = new SpeechRecognition();
this.recognition.continuous = false;
this.recognition.lang = 'en-US';
this.recognition.interimResults = false;

this.recognition.onresult = (event: any) => {
console.log("Speech Recognition end");

const transcript: string = event.results[0][0].transcript;
console.log(`Result received: ${transcript}`)

const filtered: any = Array.from(event.results).filter(
(r: any) => r.isFinal && r[0].confidence > 0.9
);
if (filtered.length != 0) {
this.combo.searchbar.handleInput({
target: { value: transcript},
});
this.combo.selectClick();
this.combo.togglePopup(false);
}
};
}
};
}

public getSetting(prop: string): any {
Expand All @@ -106,15 +111,30 @@ export class SettingsListComponent {

public resetSettings(): void {
this.settingsService.resetSettings();
this.combo.reset();
}

public onValueChange(value: string) {
console.log(`value change`);
this.comboboxValue = value;
async applyAIRecommendedSettings(value: string) {
if (!this.isDisabilitySupported(value)) {
console.log(`Disability "${value}" is not supported. Skipping AI call.`);
return;
}
// reset old selected disability settings before applying new
this.settingsService.resetSettings();
let settings = await this.openAIService.getAIRecommendedSettings(value);
// let settings = `{"colorTheme": "contrast", "underlineLinks": true}`;
settings = JSON.parse(settings); // parse JSON string into object

for (const [setting, settingValue] of Object.entries(settings)) {
const settingObj = { [setting]: settingValue };
console.log(`setting: ${setting}`);
console.log(`settingValue: ${settingValue}`);
this.settingChange(settingObj);
}
this.settingsExpanded = true;
}

public activateSpeech() {
public activateSpeech() {
this.recognition.start();
console.log("Speech Recognition start");
}
Expand Down
113 changes: 55 additions & 58 deletions examples-standalone/coffee-warehouse/src/app/openai.service.ts
Original file line number Diff line number Diff line change
@@ -1,78 +1,75 @@
import { Injectable } from '@angular/core';

import { AzureOpenAI } from "openai";
import type {
ChatCompletionCreateParamsNonStreaming,
ChatCompletionCreateParamsNonStreaming,
} from "openai/resources/index";

// You will need to set these environment variables or edit the following values
const apiKey = 'asd';
const endpoint = 'https://dx-hackathon-2024q3.openai.azure.com/';


// Required Azure OpenAI deployment name and API version
const apiVersion = "2024-08-01-preview";
const GPT_4o_MINI = 'gpt-4o-mini-dx-hackathon';

@Injectable({
providedIn: 'root',
providedIn: 'root',
})
export class OpenAIService {
private client: AzureOpenAI;

constructor() {
this.client = new AzureOpenAI({
endpoint,
apiKey,
apiVersion,
deployment: GPT_4o_MINI,
dangerouslyAllowBrowser: true
});
}

public async getAIResponse(disabilities: string[]): Promise<string> {
const messages = this.createMessages(disabilities);

const completion = await this.client.chat.completions.create(messages);
const result = completion.choices[0].message.content;
console.log(result);
return result;
}

private createMessages(selectedDisabilities: string[]): ChatCompletionCreateParamsNonStreaming {
const prompt = `
Based on the following disabilities: ${selectedDisabilities.join(', ')},
suggest the appropriate accessibility settings as a single object.
Only include the settings that are applicable to the disabilities.
The possible settings are:
- textSize: Numeric value for text size (e.g. 16)
- colorTheme: Either 'contrast' or 'dark'
- font: Either 'legible' or 'dyslexia'
- underlineLinks: Boolean (true or false)
- pauseAnimations: Boolean (true or false)
- lgSizeWidgets: Boolean (true or false)
- lineHeight: Numeric value for line height (e.g. 1.2)
- letterSpacing: Numeric value for letter spacing (e.g. 1)
private client: AzureOpenAI;

Respond with only an object that contains the applicable settings.
Do not wrap the object in any syntax, such as \`\`\`json\`.
For example: { textSize: 18, font: 'dyslexia' }
`;
constructor() {
this.client = new AzureOpenAI({
endpoint,
apiKey,
apiVersion,
deployment: GPT_4o_MINI,
dangerouslyAllowBrowser: true
});
}

return {
messages: [
{
role: "system",
content: "You are a helpful assistant who recommends accessibility settings based on disabilities provided."
},
{
role: "user",
content: prompt,
},
],
model: "gpt-4",
};
}
public async getAIRecommendedSettings(disability: string): Promise<string> {
const messages = this.createMessages(disability);
console.log(`Disability: ${disability}`);

const completion = await this.client.chat.completions.create(messages);
const result = completion.choices[0].message.content;
console.log(`Result: ${result}`);
return result;
}

private createMessages(selectedDisabilities: string): ChatCompletionCreateParamsNonStreaming {
const prompt = `
Based on the following disabilities: ${selectedDisabilities},
suggest the appropriate accessibility settings as a single object.
Only include the settings that are applicable to the disabilities.
The possible settings are:
- fontSize: Numeric value for text size
- colorTheme: Either "contrast" or "dark"
- fontFamily: Either "legible" or "dyslexia"
- underlineLinks: Boolean (true or false)
- pauseAnimations: Boolean (true or false)
- lineHeight: Numerical value for line height - e.g. 1.5
- letterSpacing: Integer (whole) value for letter spacing - e.g. 1 instead of 0.8
Respond with a valid JSON string that contains the applicable settings which can then be parsed into an object using JSON.parse.
Do not include any extra syntax like \`\`\`json or any text before or after the object.
Example: {"fontSize": 18, "fontFamily": "dyslexia"}
`;

return {
messages: [
{
role: "system",
content: "You are a helpful assistant who recommends accessibility settings based on disabilities provided."
},
{
role: "user",
content: prompt,
},
],
model: "gpt-4",
};
}
}

0 comments on commit fb9e61d

Please sign in to comment.