From f82b4ee236957f6da80d72d44da651556519d559 Mon Sep 17 00:00:00 2001 From: Kerwin Date: Sun, 16 Apr 2023 14:16:32 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E6=95=8F=E6=84=9F?= =?UTF-8?q?=E8=AF=8D=E5=AE=A1=E6=A0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.en.md | 7 ++ README.md | 7 ++ docker-compose/docker-compose.yml | 7 ++ package.json | 2 +- service/src/chatgpt/index.ts | 36 +++++- service/src/index.ts | 34 +++++- service/src/storage/config.ts | 36 +++++- service/src/storage/model.ts | 18 +++ service/src/utils/is.ts | 16 +++ service/src/utils/textAudit.ts | 91 +++++++++++++++ src/api/index.ts | 16 ++- src/components/common/Setting/Audit.vue | 143 ++++++++++++++++++++++++ src/components/common/Setting/Mail.vue | 16 +-- src/components/common/Setting/index.vue | 8 ++ src/components/common/Setting/model.ts | 21 ++++ src/locales/en-US.ts | 9 ++ src/locales/ko-KR.ts | 9 ++ src/locales/zh-CN.ts | 9 ++ src/locales/zh-TW.ts | 9 ++ 19 files changed, 474 insertions(+), 20 deletions(-) create mode 100644 service/src/utils/textAudit.ts create mode 100644 src/components/common/Setting/Audit.vue diff --git a/README.en.md b/README.en.md index f9545de3..fd557cb3 100644 --- a/README.en.md +++ b/README.en.md @@ -266,6 +266,13 @@ services: SMTP_TSL: true SMTP_USERNAME: noreply@examile.com SMTP_PASSWORD: xxx + # Enable sensitive word review, because the response result is streaming, so there is currently no review. + AUDIT_ENABLED: false + # https://ai.baidu.com/ai-doc/ANTIPORN/Vk3h6xaga + AUDIT_PROVIDER: baidu + AUDIT_API_KEY: xxx + AUDIT_API_SECRET: xxx + AUDIT_TEXT_LABEL: xxx links: - database diff --git a/README.md b/README.md index 9f8b0f91..55496da3 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,13 @@ services: SMTP_TSL: true SMTP_USERNAME: noreply@examile.com SMTP_PASSWORD: xxx + # 是否开启敏感词审核, 因为响应结果是流式 所以暂时没审核 + AUDIT_ENABLED: false + # https://ai.baidu.com/ai-doc/ANTIPORN/Vk3h6xaga + AUDIT_PROVIDER: baidu + AUDIT_API_KEY: xxx + AUDIT_API_SECRET: xxx + AUDIT_TEXT_LABEL: xxx links: - database diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index eb71a164..77b35747 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -57,6 +57,13 @@ services: SMTP_TSL: true SMTP_USERNAME: ${SMTP_USERNAME} SMTP_PASSWORD: ${SMTP_PASSWORD} + # 是否开启敏感词审核, 因为响应结果是流式 所以暂时没审核 + AUDIT_ENABLED: false + # https://ai.baidu.com/ai-doc/ANTIPORN/Vk3h6xaga + AUDIT_PROVIDER: baidu + AUDIT_API_KEY: + AUDIT_API_SECRET: + AUDIT_TEXT_LABEL: links: - database diff --git a/package.json b/package.json index c74b2e01..b8a226f1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "chatgpt-web", - "version": "2.11.7", + "version": "2.12.0", "private": false, "description": "ChatGPT Web", "author": "ChenZhaoYu ", diff --git a/service/src/chatgpt/index.ts b/service/src/chatgpt/index.ts index c66f6e92..c6fea55e 100644 --- a/service/src/chatgpt/index.ts +++ b/service/src/chatgpt/index.ts @@ -5,6 +5,9 @@ import { ChatGPTAPI, ChatGPTUnofficialProxyAPI } from 'chatgpt' import { SocksProxyAgent } from 'socks-proxy-agent' import httpsProxyAgent from 'https-proxy-agent' import fetch from 'node-fetch' +import type { AuditConfig } from 'src/storage/model' +import type { TextAuditService } from '../utils/textAudit' +import { textAuditServices } from '../utils/textAudit' import { getCacheConfig, getOriginConfig } from '../storage/config' import { sendResponse } from '../utils' import { isNotEmptyString } from '../utils/is' @@ -26,6 +29,7 @@ const ErrorCodeMessage: Record = { let apiModel: ApiModel let api: ChatGPTAPI | ChatGPTUnofficialProxyAPI +let auditService: TextAuditService export async function initApi() { // More Info: https://github.com/transitive-bullshit/chatgpt-api @@ -85,6 +89,10 @@ async function chatReplyProcess(options: RequestOptions) { const config = await getCacheConfig() const model = isNotEmptyString(config.apiModel) ? config.apiModel : 'gpt-3.5-turbo' const { message, lastContext, process, systemMessage, temperature, top_p } = options + + if ((config.auditConfig?.enabled ?? false) && !await auditText(config.auditConfig, message)) + return sendResponse({ type: 'Fail', message: '含有敏感词 | Contains sensitive words' }) + try { const timeoutMs = (await getCacheConfig()).timeoutMs let options: SendMessageOptions = { timeoutMs } @@ -120,9 +128,28 @@ async function chatReplyProcess(options: RequestOptions) { } } +export function initAuditService(audit: AuditConfig) { + if (!audit || !audit.options || !audit.options.apiKey || !audit.options.apiSecret) + throw new Error('未配置 | Not configured.') + const Service = textAuditServices[audit.provider] + auditService = new Service(audit.options) +} + +async function auditText(audit: AuditConfig, text: string): Promise { + if (!auditService) + initAuditService(audit) + + return await auditService.audit(text) +} +let cachedBanlance: number | undefined +let cacheExpiration = 0 + async function fetchBalance() { - // 计算起始日期和结束日期 const now = new Date().getTime() + if (cachedBanlance && cacheExpiration > now) + return Promise.resolve(cachedBanlance.toFixed(3)) + + // 计算起始日期和结束日期 const startDate = new Date(now - 90 * 24 * 60 * 60 * 1000) const endDate = new Date(now + 24 * 60 * 60 * 1000) @@ -165,9 +192,10 @@ async function fetchBalance() { const totalUsage = usageData.total_usage / 100 // 计算剩余额度 - const balance = totalAmount - totalUsage + cachedBanlance = totalAmount - totalUsage + cacheExpiration = now + 10 * 60 * 1000 - return Promise.resolve(balance.toFixed(3)) + return Promise.resolve(cachedBanlance.toFixed(3)) } catch { return Promise.resolve('-') @@ -226,4 +254,4 @@ initApi() export type { ChatContext, ChatMessage } -export { chatReplyProcess, chatConfig, currentModel } +export { chatReplyProcess, chatConfig, currentModel, auditText } diff --git a/service/src/index.ts b/service/src/index.ts index f1627b08..d6e821fa 100644 --- a/service/src/index.ts +++ b/service/src/index.ts @@ -3,10 +3,10 @@ import jwt from 'jsonwebtoken' import * as dotenv from 'dotenv' import type { RequestProps } from './types' import type { ChatContext, ChatMessage } from './chatgpt' -import { chatConfig, chatReplyProcess, currentModel, initApi } from './chatgpt' +import { auditText, chatConfig, chatReplyProcess, currentModel, initApi, initAuditService } from './chatgpt' import { auth } from './middleware/auth' import { clearConfigCache, getCacheConfig, getOriginConfig } from './storage/config' -import type { ChatInfo, ChatOptions, Config, MailConfig, SiteConfig, UsageResponse, UserInfo } from './storage/model' +import type { AuditConfig, ChatInfo, ChatOptions, Config, MailConfig, SiteConfig, UsageResponse, UserInfo } from './storage/model' import { Status } from './storage/model' import { clearChat, @@ -594,6 +594,36 @@ router.post('/mail-test', rootAuth, async (req, res) => { } }) +router.post('/setting-audit', rootAuth, async (req, res) => { + try { + const config = req.body as AuditConfig + + const thisConfig = await getOriginConfig() + thisConfig.auditConfig = config + const result = await updateConfig(thisConfig) + clearConfigCache() + initAuditService(config) + res.send({ status: 'Success', message: '操作成功 | Successfully', data: result.auditConfig }) + } + catch (error) { + res.send({ status: 'Fail', message: error.message, data: null }) + } +}) + +router.post('/audit-test', rootAuth, async (req, res) => { + try { + const { audit, text } = req.body as { audit: AuditConfig; text: string } + const config = await getCacheConfig() + initAuditService(audit) + const result = await auditText(audit, text) + initAuditService(config.auditConfig) + res.send({ status: 'Success', message: !result ? '含敏感词 | Contains sensitive words' : '不含敏感词 | Does not contain sensitive words.', data: null }) + } + catch (error) { + res.send({ status: 'Fail', message: error.message, data: null }) + } +}) + app.use('', router) app.use('/api', router) app.set('trust proxy', 1) diff --git a/service/src/storage/config.ts b/service/src/storage/config.ts index 03fc2838..04186918 100644 --- a/service/src/storage/config.ts +++ b/service/src/storage/config.ts @@ -1,7 +1,8 @@ import { ObjectId } from 'mongodb' import * as dotenv from 'dotenv' -import { isNotEmptyString } from '../utils/is' -import { Config, MailConfig, SiteConfig } from './model' +import type { TextAuditServiceProvider } from 'src/utils/textAudit' +import { isNotEmptyString, isTextAuditServiceProvider } from '../utils/is' +import { AuditConfig, Config, MailConfig, SiteConfig, TextAudioType } from './model' import { getConfig } from './mongo' dotenv.config() @@ -69,9 +70,40 @@ export async function getOriginConfig() { if (config.siteConfig.registerReview === undefined) config.siteConfig.registerReview = process.env.REGISTER_REVIEW === 'true' } + + if (config.auditConfig === undefined) { + config.auditConfig = new AuditConfig( + process.env.AUDIT_ENABLED === 'true', + isTextAuditServiceProvider(process.env.AUDIT_PROVIDER) + ? process.env.AUDIT_PROVIDER as TextAuditServiceProvider + : 'baidu', + { + apiKey: process.env.AUDIT_API_KEY, + apiSecret: process.env.AUDIT_API_SECRET, + label: process.env.AUDIT_TEXT_LABEL, + }, + getTextAuditServiceOptionFromString(process.env.AUDIT_TEXT_TYPE), + ) + } return config } +function getTextAuditServiceOptionFromString(value: string): TextAudioType { + if (value === undefined) + return TextAudioType.None + + switch (value.toLowerCase()) { + case 'request': + return TextAudioType.Request + case 'response': + return TextAudioType.Response + case 'all': + return TextAudioType.All + default: + return TextAudioType.None + } +} + export function clearConfigCache() { cacheExpiration = 0 cachedConfig = null diff --git a/service/src/storage/model.ts b/service/src/storage/model.ts index 1902d61b..4333417d 100644 --- a/service/src/storage/model.ts +++ b/service/src/storage/model.ts @@ -1,4 +1,5 @@ import type { ObjectId } from 'mongodb' +import type { TextAuditServiceOptions, TextAuditServiceProvider } from 'src/utils/textAudit' export enum Status { Normal = 0, @@ -129,6 +130,7 @@ export class Config { public httpsProxy?: string, public siteConfig?: SiteConfig, public mailConfig?: MailConfig, + public auditConfig?: AuditConfig, ) { } } @@ -153,3 +155,19 @@ export class MailConfig { public smtpPassword: string, ) { } } + +export class AuditConfig { + constructor( + public enabled: boolean, + public provider: TextAuditServiceProvider, + public options: TextAuditServiceOptions, + public textType: TextAudioType, + ) { } +} + +export enum TextAudioType { + None = 0, + Request = 1 << 0, // 二进制 01 + Response = 1 << 1, // 二进制 10 + All = Request | Response, // 二进制 11 +} diff --git a/service/src/utils/is.ts b/service/src/utils/is.ts index aa2d040c..05e8af0c 100644 --- a/service/src/utils/is.ts +++ b/service/src/utils/is.ts @@ -1,3 +1,6 @@ +import { TextAudioType } from '../storage/model' +import type { TextAuditServiceProvider } from './textAudit' + export function isNumber(value: T | unknown): value is number { return Object.prototype.toString.call(value) === '[object Number]' } @@ -21,3 +24,16 @@ export function isFunction any | void | never>(val export function isEmail(value: any): boolean { return isNotEmptyString(value) && /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/.test(value) } + +export function isTextAuditServiceProvider(value: any): value is TextAuditServiceProvider { + return value === 'baidu' // || value === 'ali' +} + +export function isTextAudioType(value: any): value is TextAudioType { + return ( + value === TextAudioType.None + || value === TextAudioType.Request + || value === TextAudioType.Response + || value === TextAudioType.All + ) +} diff --git a/service/src/utils/textAudit.ts b/service/src/utils/textAudit.ts new file mode 100644 index 00000000..fa4bc022 --- /dev/null +++ b/service/src/utils/textAudit.ts @@ -0,0 +1,91 @@ +import fetch from 'node-fetch' + +export interface TextAuditServiceOptions { + apiKey: string + apiSecret: string + label: string +} + +export interface TextAuditService { + audit(text: string): Promise +} + +/** + * https://ai.baidu.com/ai-doc/ANTIPORN/Vk3h6xaga + */ +export class BaiduTextAuditService implements TextAuditService { + private accessToken: string + private expiredTime: number + + constructor(private options: TextAuditServiceOptions) { } + + async audit(text: string): Promise { + if (!await this.refreshAccessToken()) + return + const url = `https://aip.baidubce.com/rest/2.0/solution/v1/text_censor/v2/user_defined?access_token=${this.accessToken}` + let headers: { + 'Content-Type': 'application/x-www-form-urlencoded' + 'Accept': 'application/json' + } + const response = await fetch(url, { headers, method: 'POST', body: `text=${encodeURIComponent(text)}` }) + const data = await response.json() as { conclusionType: number; data: any } + + if (data.error_msg) + throw new Error(data.error_msg) + + // 审核结果类型，可取值1、2、3、4，分别代表1：合规，2：不合规，3：疑似，4：审核失败 + if (data.conclusionType === 1) + return true + + // https://ai.baidu.com/ai-doc/ANTIPORN/Nk3h6xbb2#%E7%BB%86%E5%88%86%E6%A0%87%E7%AD%BE%E5%AF%B9%E7%85%A7%E8%A1%A8 + + // 3 仅政治 + const safe = data.data.filter(d => d.subType === 3).length <= 0 + if (!safe || !this.options.label) + return safe + const str = JSON.stringify(data) + for (const l of this.options.label.split(',')) { + if (str.indexOf(l)) + return false + } + return true + } + + async refreshAccessToken() { + if (!this.options.apiKey || !this.options.apiSecret) + throw new Error('未配置 | Not configured.') + + try { + if (this.accessToken && Math.floor(new Date().getTime() / 1000) <= this.expiredTime) + return true + + const url = `https://aip.baidubce.com/oauth/2.0/token?client_id=${this.options.apiKey}&client_secret=${this.options.apiSecret}&grant_type=client_credentials` + let headers: { + 'Content-Type': 'application/json' + 'Accept': 'application/json' + } + const response = await fetch(url, { headers }) + const data = (await response.json()) as { access_token: string; expires_in: number } + + this.accessToken = data.access_token + this.expiredTime = Math.floor(new Date().getTime() / 1000) + (+data.expires_in) + return true + } + catch (error) { + global.console.error(`百度审核${error}`) + } + return false + } +} + +export type TextAuditServiceProvider = 'baidu' // | 'ali' + +export type TextAuditServices = { + [key in TextAuditServiceProvider]: new ( + options: TextAuditServiceOptions, + ) => TextAuditService; +} + +export const textAuditServices: TextAuditServices = { + baidu: BaiduTextAuditService, +} diff --git a/src/api/index.ts b/src/api/index.ts index c97c6b68..d1a520a1 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -1,6 +1,6 @@ import type { AxiosProgressEvent, GenericAbortSignal } from 'axios' import { get, post } from '@/utils/request' -import type { ConfigState, MailConfig, SiteConfig } from '@/components/common/Setting/model' +import type { AuditConfig, ConfigState, MailConfig, SiteConfig } from '@/components/common/Setting/model' import { useAuthStore, useSettingStore } from '@/store' export function fetchChatAPI( @@ -168,6 +168,20 @@ export function fetchTestMail(mail: MailConfig) { }) } +export function fetchUpdateAudit(audit: AuditConfig) { + return post({ + url: '/setting-audit', + data: audit, + }) +} + +export function fetchTestAudit(text: string, audit: AuditConfig) { + return post({ + url: '/audit-test', + data: { audit, text }, + }) +} + export function fetchUpdateSite(config: SiteConfig) { return post({ url: '/setting-site', diff --git a/src/components/common/Setting/Audit.vue b/src/components/common/Setting/Audit.vue new file mode 100644 index 00000000..00106af7 --- /dev/null +++ b/src/components/common/Setting/Audit.vue @@ -0,0 +1,143 @@ + + + diff --git a/src/components/common/Setting/Mail.vue b/src/components/common/Setting/Mail.vue index befec869..0ccec352 100644 --- a/src/components/common/Setting/Mail.vue +++ b/src/components/common/Setting/Mail.vue @@ -24,12 +24,10 @@ async function fetchConfig() { } } -async function updateMailInfo(mail?: MailConfig) { - if (!mail) - return +async function updateMailInfo() { saving.value = true try { - const { data } = await fetchUpdateMail(mail) + const { data } = await fetchUpdateMail(config.value as MailConfig) config.value = data ms.success(t('common.success')) } @@ -39,12 +37,10 @@ async function updateMailInfo(mail?: MailConfig) { saving.value = false } -async function testMail(mail?: MailConfig) { - if (!mail) - return +async function testMail() { testing.value = true try { - const { message } = await fetchTestMail(mail) as { status: string; message: string } + const { message } = await fetchTestMail(config.value as MailConfig) as { status: string; message: string } ms.success(message) } catch (error: any) { @@ -112,10 +108,10 @@ onMounted(() => {
- + {{ $t('common.save') }} - + {{ $t('common.test') }}
diff --git a/src/components/common/Setting/index.vue b/src/components/common/Setting/index.vue index d7c7bd19..992065e9 100644 --- a/src/components/common/Setting/index.vue +++ b/src/components/common/Setting/index.vue @@ -6,6 +6,7 @@ import Advanced from './Advanced.vue' import About from './About.vue' import Site from './Site.vue' import Mail from './Mail.vue' +import Audit from './Audit.vue' import { SvgIcon } from '@/components/common' import { useAuthStore, useUserStore } from '@/store' @@ -81,6 +82,13 @@ const show = computed({ + + + +
diff --git a/src/components/common/Setting/model.ts b/src/components/common/Setting/model.ts index 1cf7ac54..b1f93b4d 100644 --- a/src/components/common/Setting/model.ts +++ b/src/components/common/Setting/model.ts @@ -11,6 +11,7 @@ export class ConfigState { balance?: number siteConfig?: SiteConfig mailConfig?: MailConfig + auditConfig?: AuditConfig } export class SiteConfig { @@ -30,3 +31,23 @@ export class MailConfig { smtpUserName?: string smtpPassword?: string } +export type TextAuditServiceProvider = 'baidu' // | 'ali' + +export interface TextAuditServiceOptions { + apiKey: string + apiSecret: string + label?: string +} +export enum TextAudioType { + None = 0, + Request = 1 << 0, // 二进制 01 + Response = 1 << 1, // 二进制 10 + All = Request | Response, // 二进制 11 +} + +export class AuditConfig { + enabled?: boolean + provider?: TextAuditServiceProvider + options?: TextAuditServiceOptions + textType?: TextAudioType +} diff --git a/src/locales/en-US.ts b/src/locales/en-US.ts index 21b44aa1..34720457 100644 --- a/src/locales/en-US.ts +++ b/src/locales/en-US.ts @@ -68,6 +68,7 @@ export default { config: 'Base Config', siteConfig: 'Site Config', mailConfig: 'Mail Config', + auditConfig: 'Audit Config', avatarLink: 'Avatar Link', name: 'Name', description: 'Description', @@ -102,6 +103,14 @@ export default { loginSalt: 'Login Salt', loginSaltTip: 'Changes will invalidate all logged in', monthlyUsage: 'Monthly Usage', + auditEnabled: 'Audit Enabled', + auditProvider: 'Provider', + auditApiKey: 'Api Key', + auditApiSecret: 'Api Secret', + auditTest: 'Test Text', + auditBaiduLabel: 'Label', + auditBaiduLabelTip: 'English comma separated, If empty, only politics.', + auditBaiduLabelLink: 'Goto Label Detail', }, store: { siderButton: 'Prompt Store', diff --git a/src/locales/ko-KR.ts b/src/locales/ko-KR.ts index c7ae3ba9..922fb484 100644 --- a/src/locales/ko-KR.ts +++ b/src/locales/ko-KR.ts @@ -68,6 +68,7 @@ export default { config: '기본 구성', siteConfig: '사이트 구성', mailConfig: '메일 구성', + auditConfig: '감사 구성', avatarLink: '아바타 링크', name: '이름', description: '설명', @@ -100,6 +101,14 @@ export default { loginSalt: '로그인 정보 암호화 Salt', loginSaltTip: '변경하면 모든사용자의 로그인이 풀립니다.', monthlyUsage: '월간 사용량', + auditEnabled: '승인 상태', + auditProvider: '공급자', + auditApiKey: 'Api Key', + auditApiSecret: 'Api Secret', + auditTest: '테스트 텍스트', + auditBaiduLabel: 'Label', + auditBaiduLabelTip: '영어 쉼표로 구분, If empty, only politics.', + auditBaiduLabelLink: '레이블 세부 정보로 이동', }, store: { siderButton: '프롬프트 스토어', diff --git a/src/locales/zh-CN.ts b/src/locales/zh-CN.ts index e9815a75..56d12c8a 100644 --- a/src/locales/zh-CN.ts +++ b/src/locales/zh-CN.ts @@ -68,6 +68,7 @@ export default { config: '基本配置', siteConfig: '网站配置', mailConfig: '邮箱配置', + auditConfig: '敏感词审核', avatarLink: '头像链接', name: '名称', description: '描述', @@ -102,6 +103,14 @@ export default { loginSalt: '登录混淆盐', loginSaltTip: '变更会导致所有已登录失效', monthlyUsage: '本月使用量', + auditEnabled: '审核状态', + auditProvider: '提供商', + auditApiKey: 'Api Key', + auditApiSecret: 'Api Secret', + auditTest: '测试文本', + auditBaiduLabel: '二级分类', + auditBaiduLabelTip: '英文逗号分隔, 如果空, 仅政治', + auditBaiduLabelLink: '查看细分类型', }, store: { siderButton: '提示词商店', diff --git a/src/locales/zh-TW.ts b/src/locales/zh-TW.ts index ea0de532..02391e0b 100644 --- a/src/locales/zh-TW.ts +++ b/src/locales/zh-TW.ts @@ -68,6 +68,7 @@ export default { config: '基本設定', siteConfig: '网站配置', mailConfig: '邮箱配置', + auditConfig: '敏感词审核', avatarLink: '頭貼連結', name: '名稱', description: '描述', @@ -102,6 +103,14 @@ export default { loginSalt: '登录混淆盐', loginSaltTip: '变更会导致所有已登录失效', monthlyUsage: '本月使用量', + auditEnabled: '审核状态', + auditProvider: '提供商', + auditApiKey: 'Api Key', + auditApiSecret: 'Api Secret', + auditTest: '测试文本', + auditBaiduLabel: '二级分类', + auditBaiduLabelTip: '英文逗号分隔, 如果空, 仅政治', + auditBaiduLabelLink: '查看细分类型', }, store: { siderButton: '提示詞商店',