diff --git a/.gitignore b/.gitignore index 31053b0..d8aed2b 100644 --- a/.gitignore +++ b/.gitignore @@ -128,4 +128,5 @@ dist/ # Python *.pyc __pycache__/ -*.egg-info/ \ No newline at end of file +*.egg-info/ +node_modules diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..60a4dee --- /dev/null +++ b/package-lock.json @@ -0,0 +1,8587 @@ +{ + "name": "tamingllms", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "promptfoo": "^0.100.5" + } + }, + "node_modules/@adaline/anthropic": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/@adaline/anthropic/-/anthropic-0.19.0.tgz", + "integrity": "sha512-jZugMU25pICy7MkZPasE5INNrd1ZS+NVTdY175k48WA+5j4aT3MSCoMAupgpb4B+hrB5UGrnOqL+dtz2Q8YyTw==", + "dependencies": { + "@adaline/provider": "0.17.0", + "@adaline/types": "0.15.0", + "zod": "^3.23.8" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@adaline/azure": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/@adaline/azure/-/azure-0.9.1.tgz", + "integrity": "sha512-T1lABZu5/YqGhepZaFRFJQiKzCN7EvFyc2g/AarPhKr9WXe4e7qxkv4J8NZqNk+6VIrxI12q8Ihi1uCxsn3Rrg==", + "dependencies": { + "@adaline/openai": "0.21.0", + "@adaline/provider": "0.17.0", + "@adaline/types": "0.15.0", + "zod": "^3.23.8" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@adaline/gateway": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@adaline/gateway/-/gateway-0.23.0.tgz", + "integrity": "sha512-ZrR5b9uyWt7zOvOT0qaFtSawsqNA9uir4Veuglr28TwZ/HBDnEhFKQV2uzxZBIKlrYFHXb9mM+Qn9TkmmoQGqw==", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/semantic-conventions": "^1.27.0", + "@smithy/protocol-http": "^4.0.4", + "@smithy/signature-v4": "^4.0.0", + "axios": "^1.7.2", + "crypto-js": "^4.2.0", + "dotenv": "^16.4.5", + "env-paths": "^3.0.0", + "lodash": "^4.17.21", + "lru-cache": "^11.0.1", + "proxy-agent": "^6.4.0", + "uuid": "^10.0.0", + "zod": "^3.23.8" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@adaline/gateway/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/@adaline/google": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@adaline/google/-/google-0.8.0.tgz", + "integrity": "sha512-cXqfIyNKbLqSy/6PFekEZcmur104LvzyAScw6Nv7BzEr+HXdaXzYF/tJfx7+ZnY3LmauMR4J2Godmefepf1iAw==", + "dependencies": { + "@adaline/provider": "0.17.0", + "@adaline/types": "0.15.0", + "zod": "^3.23.8" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@adaline/groq": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@adaline/groq/-/groq-0.8.1.tgz", + "integrity": "sha512-sDFlaCDYH8QUTmW+QhK1gMTA1sU9s5ELPXy/Cz+xShRk7IODRkCmEk4gQnDOH0gH/BSvt9C+dFR9kCUSlDfNKQ==", + "dependencies": { + "@adaline/openai": "0.21.0", + "@adaline/provider": "0.17.0", + "@adaline/types": "0.15.0", + "zod": "^3.23.8" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@adaline/open-router": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@adaline/open-router/-/open-router-0.8.0.tgz", + "integrity": "sha512-BGyhe8jFNTSrDwpnmMQF96CTdrAzKwCM81/seuwQF9/MKuA46r/t1IW4sU0ViXu4FV5ndr3kb5rO/5/LIW/r8Q==", + "dependencies": { + "@adaline/provider": "0.17.0", + "@adaline/types": "0.15.0", + "zod": "^3.23.8" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@adaline/openai": { + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/@adaline/openai/-/openai-0.21.0.tgz", + "integrity": "sha512-Meq6m3Fm/+1CR5OXr0OW1MFnc6AbQWd5ScWqaIpQNCmeKttGNp3+ZeUn6MBtos5pfX4HprDcUr+HVTebxVALXw==", + "dependencies": { + "@adaline/provider": "0.17.0", + "@adaline/types": "0.15.0", + "zod": "^3.23.8" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@adaline/provider": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/@adaline/provider/-/provider-0.17.0.tgz", + "integrity": "sha512-ab+jpn0bLhnqzpKHXy99KVBD+sKiIRbboHJ4S/mv4aeBYPSOqqBhmNqFBWXr1upIAxxmgXhFeQMzBL75/es/mw==", + "dependencies": { + "@adaline/types": "0.15.0", + "zod": "^3.23.8" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@adaline/together-ai": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@adaline/together-ai/-/together-ai-0.8.0.tgz", + "integrity": "sha512-nd9zl8HMXPgJLa13Lo8/2WlqwH9RFxRJX/TvJP/kTbJcyT7TiI49jATrzPkGqv/8s/OTUoy41p5YxpAhK+fMEg==", + "dependencies": { + "@adaline/provider": "0.17.0", + "@adaline/types": "0.15.0", + "zod": "^3.23.8" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@adaline/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@adaline/types/-/types-0.15.0.tgz", + "integrity": "sha512-2dZvXzzn+pLwg07C4VPj36Rywk51qHz4vvIihrccK7GaljQCnZ+qb2+y+QgJ9HBeAqyzjsY3/SQ8W79pfyJzXg==", + "dependencies": { + "json-schema": "^0.4.0", + "jsonschema": "^1.4.1", + "zod": "^3.23.8" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@adaline/vertex": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@adaline/vertex/-/vertex-0.8.1.tgz", + "integrity": "sha512-4i+M9Lc8t7J3uWiwp4lEZRmAxuAm87xrwR6TLdY0w8dfuBiimafcUdSycjCKnkdolfX1IzoyidfbpB3L9VYUsQ==", + "dependencies": { + "@adaline/google": "0.8.0", + "@adaline/provider": "0.17.0", + "@adaline/types": "0.15.0", + "zod": "^3.23.8" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@ai-zen/node-fetch-event-source": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@ai-zen/node-fetch-event-source/-/node-fetch-event-source-2.1.4.tgz", + "integrity": "sha512-OHFwPJecr+qwlyX5CGmTvKAKPZAdZaxvx/XDqS1lx4I2ZAk9riU0XnEaRGOOAEFrdcLZ98O5yWqubwjaQc0umg==", + "peer": true, + "dependencies": { + "cross-fetch": "^4.0.0" + } + }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.32.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.32.1.tgz", + "integrity": "sha512-U9JwTrDvdQ9iWuABVsMLj8nJVwAyQz6QXvgLsVhryhCEPkLsbcP/MXxm+jYcAwLoV8ESbaTTjnD4kuAFa+Hyjg==", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + } + }, + "node_modules/@apidevtools/json-schema-ref-parser": { + "version": "11.7.3", + "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-11.7.3.tgz", + "integrity": "sha512-WApSdLdXEBb/1FUPca2lteASewEfpjEYJ8oXZP+0gExK5qSfsEKBKcA+WjY6Q4wvXwyv0+W6Kvc372pSceib9w==", + "dependencies": { + "@jsdevtools/ono": "^7.1.3", + "@types/json-schema": "^7.0.15", + "js-yaml": "^4.1.0" + }, + "engines": { + "node": ">= 16" + }, + "funding": { + "url": "https://github.com/sponsors/philsturgeon" + } + }, + "node_modules/@aws-crypto/crc32": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", + "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", + "peer": true, + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "peer": true, + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "peer": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "peer": true, + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "peer": true, + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "peer": true, + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/util/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/util/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/util/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-bedrock-runtime": { + "version": "3.706.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.706.0.tgz", + "integrity": "sha512-Wxzv0VgC1JFDFEnnmCCpKCXYpe+CfANY+ljY7HBSF2zJDasBHKpM4z28sN5nuffJgsBbkIDWltWbNnXKP/cONA==", + "peer": true, + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.699.0", + "@aws-sdk/client-sts": "3.699.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-node": "3.699.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/eventstream-serde-browser": "^3.0.13", + "@smithy/eventstream-serde-config-resolver": "^3.0.10", + "@smithy/eventstream-serde-node": "^3.0.12", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-node": "^3.0.10", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-stream": "^3.3.1", + "@smithy/util-utf8": "^3.0.0", + "@types/uuid": "^9.0.1", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sso": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.696.0.tgz", + "integrity": "sha512-q5TTkd08JS0DOkHfUL853tuArf7NrPeqoS5UOvqJho8ibV9Ak/a/HO4kNvy9Nj3cib/toHYHsQIEtecUPSUUrQ==", + "peer": true, + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-node": "^3.0.10", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sso-oidc": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.699.0.tgz", + "integrity": "sha512-u8a1GorY5D1l+4FQAf4XBUC1T10/t7neuwT21r0ymrtMFSK2a9QqVHKMoLkvavAwyhJnARSBM9/UQC797PFOFw==", + "peer": true, + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-node": "3.699.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-node": "^3.0.10", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.699.0" + } + }, + "node_modules/@aws-sdk/client-sts": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.699.0.tgz", + "integrity": "sha512-++lsn4x2YXsZPIzFVwv3fSUVM55ZT0WRFmPeNilYIhZClxHLmVAWKH4I55cY9ry60/aTKYjzOXkWwyBKGsGvQg==", + "peer": true, + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.699.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-node": "3.699.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-node": "^3.0.10", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.696.0.tgz", + "integrity": "sha512-3c9III1k03DgvRZWg8vhVmfIXPG6hAciN9MzQTzqGngzWAELZF/WONRTRQuDFixVtarQatmLHYVw/atGeA2Byw==", + "peer": true, + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/core": "^2.5.3", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/property-provider": "^3.1.9", + "@smithy/protocol-http": "^4.1.7", + "@smithy/signature-v4": "^4.2.2", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/util-middleware": "^3.0.10", + "fast-xml-parser": "4.4.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.696.0.tgz", + "integrity": "sha512-T9iMFnJL7YTlESLpVFT3fg1Lkb1lD+oiaIC8KMpepb01gDUBIpj9+Y+pA/cgRWW0yRxmkDXNazAE2qQTVFGJzA==", + "peer": true, + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.696.0.tgz", + "integrity": "sha512-GV6EbvPi2eq1+WgY/o2RFA3P7HGmnkIzCNmhwtALFlqMroLYWKE7PSeHw66Uh1dFQeVESn0/+hiUNhu1mB0emA==", + "peer": true, + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/property-provider": "^3.1.9", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/util-stream": "^3.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.699.0.tgz", + "integrity": "sha512-dXmCqjJnKmG37Q+nLjPVu22mNkrGHY8hYoOt3Jo9R2zr5MYV7s/NHsCHr+7E+BZ+tfZYLRPeB1wkpTeHiEcdRw==", + "peer": true, + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-env": "3.696.0", + "@aws-sdk/credential-provider-http": "3.696.0", + "@aws-sdk/credential-provider-process": "3.696.0", + "@aws-sdk/credential-provider-sso": "3.699.0", + "@aws-sdk/credential-provider-web-identity": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/credential-provider-imds": "^3.2.6", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.699.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.699.0.tgz", + "integrity": "sha512-MmEmNDo1bBtTgRmdNfdQksXu4uXe66s0p1hi1YPrn1h59Q605eq/xiWbGL6/3KdkViH6eGUuABeV2ODld86ylg==", + "peer": true, + "dependencies": { + "@aws-sdk/credential-provider-env": "3.696.0", + "@aws-sdk/credential-provider-http": "3.696.0", + "@aws-sdk/credential-provider-ini": "3.699.0", + "@aws-sdk/credential-provider-process": "3.696.0", + "@aws-sdk/credential-provider-sso": "3.699.0", + "@aws-sdk/credential-provider-web-identity": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/credential-provider-imds": "^3.2.6", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.696.0.tgz", + "integrity": "sha512-mL1RcFDe9sfmyU5K1nuFkO8UiJXXxLX4JO1gVaDIOvPqwStpUAwi3A1BoeZhWZZNQsiKI810RnYGo0E0WB/hUA==", + "peer": true, + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.699.0.tgz", + "integrity": "sha512-Ekp2cZG4pl9D8+uKWm4qO1xcm8/MeiI8f+dnlZm8aQzizeC+aXYy9GyoclSf6daK8KfRPiRfM7ZHBBL5dAfdMA==", + "peer": true, + "dependencies": { + "@aws-sdk/client-sso": "3.696.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/token-providers": "3.699.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.696.0.tgz", + "integrity": "sha512-XJ/CVlWChM0VCoc259vWguFUjJDn/QwDqHwbx+K9cg3v6yrqXfK5ai+p/6lx0nQpnk4JzPVeYYxWRpaTsGC9rg==", + "peer": true, + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.696.0" + } + }, + "node_modules/@aws-sdk/middleware-host-header": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.696.0.tgz", + "integrity": "sha512-zELJp9Ta2zkX7ELggMN9qMCgekqZhFC5V2rOr4hJDEb/Tte7gpfKSObAnw/3AYiVqt36sjHKfdkoTsuwGdEoDg==", + "peer": true, + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-logger": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.696.0.tgz", + "integrity": "sha512-KhkHt+8AjCxcR/5Zp3++YPJPpFQzxpr+jmONiT/Jw2yqnSngZ0Yspm5wGoRx2hS1HJbyZNuaOWEGuJoxLeBKfA==", + "peer": true, + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.696.0.tgz", + "integrity": "sha512-si/maV3Z0hH7qa99f9ru2xpS5HlfSVcasRlNUXKSDm611i7jFMWwGNLUOXFAOLhXotPX5G3Z6BLwL34oDeBMug==", + "peer": true, + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.696.0.tgz", + "integrity": "sha512-Lvyj8CTyxrHI6GHd2YVZKIRI5Fmnugt3cpJo0VrKKEgK5zMySwEZ1n4dqPK6czYRWKd5+WnYHYAuU+Wdk6Jsjw==", + "peer": true, + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@smithy/core": "^2.5.3", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/region-config-resolver": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.696.0.tgz", + "integrity": "sha512-7EuH142lBXjI8yH6dVS/CZeiK/WZsmb/8zP6bQbVYpMrppSTgB3MzZZdxVZGzL5r8zPQOU10wLC4kIMy0qdBVQ==", + "peer": true, + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/types": "^3.7.1", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.10", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.699.0.tgz", + "integrity": "sha512-kuiEW9DWs7fNos/SM+y58HCPhcIzm1nEZLhe2/7/6+TvAYLuEWURYsbK48gzsxXlaJ2k/jGY3nIsA7RptbMOwA==", + "peer": true, + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sso-oidc": "^3.699.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.696.0.tgz", + "integrity": "sha512-9rTvUJIAj5d3//U5FDPWGJ1nFJLuWb30vugGOrWk7aNZ6y9tuA3PI7Cc9dP8WEXKVyK1vuuk8rSFP2iqXnlgrw==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/util-endpoints": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.696.0.tgz", + "integrity": "sha512-T5s0IlBVX+gkb9g/I6CLt4yAZVzMSiGnbUqWihWsHvQR1WOoIcndQy/Oz/IJXT9T2ipoy7a80gzV6a5mglrioA==", + "peer": true, + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/types": "^3.7.1", + "@smithy/util-endpoints": "^2.1.6", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/util-locate-window": { + "version": "3.693.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.693.0.tgz", + "integrity": "sha512-ttrag6haJLWABhLqtg1Uf+4LgHWIMOVSYL+VYZmAp2v4PUGOwWmWQH0Zk8RM7YuQcLfH/EoR72/Yxz6A4FKcuw==", + "peer": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.696.0.tgz", + "integrity": "sha512-Z5rVNDdmPOe6ELoM5AhF/ja5tSjbe6ctSctDPb0JdDf4dT0v2MfwhJKzXju2RzX8Es/77Glh7MlaXLE0kCB9+Q==", + "peer": true, + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/types": "^3.7.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.696.0.tgz", + "integrity": "sha512-KhKqcfyXIB0SCCt+qsu4eJjsfiOrNzK5dCV7RAW2YIpp+msxGUUX0NdRE9rkzjiv+3EMktgJm3eEIS+yxtlVdQ==", + "peer": true, + "dependencies": { + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, + "node_modules/@azure-rest/core-client": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure-rest/core-client/-/core-client-1.4.0.tgz", + "integrity": "sha512-ozTDPBVUDR5eOnMIwhggbnVmOrka4fXCs8n8mvUo4WLLc38kki6bAOByDoVZZPz/pZy2jMt2kwfpvy/UjALj6w==", + "peer": true, + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.3.0", + "@azure/core-rest-pipeline": "^1.5.0", + "@azure/core-tracing": "^1.0.1", + "@azure/core-util": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/abort-controller": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", + "peer": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.9.0.tgz", + "integrity": "sha512-FPwHpZywuyasDSLMqJ6fhbOK3TqUdviZNF8OqRGA4W5Ewib2lEEZ+pBsYcBa88B2NGO/SEnYPGhyBqNlE8ilSw==", + "peer": true, + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-client": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.9.2.tgz", + "integrity": "sha512-kRdry/rav3fUKHl/aDLd/pDLcB+4pOFwPPTVEExuMyaI5r+JBbMWqRbCY1pn5BniDaU3lRxO9eaQ1AmSMehl/w==", + "peer": true, + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.4.0", + "@azure/core-rest-pipeline": "^1.9.1", + "@azure/core-tracing": "^1.0.0", + "@azure/core-util": "^1.6.1", + "@azure/logger": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.18.1.tgz", + "integrity": "sha512-/wS73UEDrxroUEVywEm7J0p2c+IIiVxyfigCGfsKvCxxCET4V/Hef2aURqltrXMRjNmdmt5IuOgIpl8f6xdO5A==", + "peer": true, + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.8.0", + "@azure/core-tracing": "^1.0.1", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.0.0", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.2.0.tgz", + "integrity": "sha512-UKTiEJPkWcESPYJz3X5uKRYyOcJD+4nYph+KpfdPRnQJVrZfk0KJgdnaAWKfhsBBtAf/D58Az4AvCJEmWgIBAg==", + "peer": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.11.0.tgz", + "integrity": "sha512-DxOSLua+NdpWoSqULhjDyAZTXFdP/LKkqtYuxxz1SCN289zk3OG8UOpnCQAz/tygyACBtWp/BoO72ptK7msY8g==", + "peer": true, + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/identity": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/@azure/identity/-/identity-4.5.0.tgz", + "integrity": "sha512-EknvVmtBuSIic47xkOqyNabAme0RYTw52BTMz8eBgU1ysTyMrD1uOoM+JdS0J/4Yfp98IBT3osqq3BfwSaNaGQ==", + "peer": true, + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.2", + "@azure/core-rest-pipeline": "^1.17.0", + "@azure/core-tracing": "^1.0.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.0.0", + "@azure/msal-browser": "^3.26.1", + "@azure/msal-node": "^2.15.0", + "events": "^3.0.0", + "jws": "^4.0.0", + "open": "^8.0.0", + "stoppable": "^1.1.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/logger": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.1.4.tgz", + "integrity": "sha512-4IXXzcCdLdlXuCG+8UKEwLA1T1NHqUfanhXYHiQTn+6sfWCZXduqbtXDGceg3Ce5QxTGo7EqmbV6Bi+aqKuClQ==", + "peer": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/msal-browser": { + "version": "3.27.0", + "resolved": "https://registry.npmjs.org/@azure/msal-browser/-/msal-browser-3.27.0.tgz", + "integrity": "sha512-+b4ZKSD8+vslCtVRVetkegEhOFMLP3rxDWJY212ct+2r6jVg6OSQKc1Qz3kCoXo0FgwaXkb+76TMZfpHp8QtgA==", + "peer": true, + "dependencies": { + "@azure/msal-common": "14.16.0" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@azure/msal-common": { + "version": "14.16.0", + "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-14.16.0.tgz", + "integrity": "sha512-1KOZj9IpcDSwpNiQNjt0jDYZpQvNZay7QAEi/5DLubay40iGYtLzya/jbjRPLyOTZhEKyL1MzPuw2HqBCjceYA==", + "peer": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@azure/msal-node": { + "version": "2.16.2", + "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-2.16.2.tgz", + "integrity": "sha512-An7l1hEr0w1HMMh1LU+rtDtqL7/jw74ORlc9Wnh06v7TU/xpG39/Zdr1ZJu3QpjUfKJ+E0/OXMW8DRSWTlh7qQ==", + "peer": true, + "dependencies": { + "@azure/msal-common": "14.16.0", + "jsonwebtoken": "^9.0.0", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@azure/msal-node/node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "peer": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/@azure/openai-assistants": { + "version": "1.0.0-beta.6", + "resolved": "https://registry.npmjs.org/@azure/openai-assistants/-/openai-assistants-1.0.0-beta.6.tgz", + "integrity": "sha512-gINKKcqTpR0neF+36Owe0Q1u1JO3IK6clBzWTfZ+9V/TkQq+LoUgp5F8dKvSv/YChfwEpZA2r1DWCwNE07eYIQ==", + "peer": true, + "dependencies": { + "@azure-rest/core-client": "^1.1.4", + "@azure/core-auth": "^1.5.0", + "@azure/core-client": "^1.7.3", + "@azure/core-rest-pipeline": "^1.13.0", + "@azure/core-util": "^1.6.1", + "@azure/logger": "^1.0.4", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", + "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==", + "dependencies": { + "@babel/helper-validator-identifier": "^7.25.9", + "js-tokens": "^4.0.0", + "picocolors": "^1.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/generator": { + "version": "7.26.3", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.3.tgz", + "integrity": "sha512-6FF/urZvD0sTeO7k6/B15pMLC4CHUv1426lzr3N01aHJTl046uCAh9LXW/fzeXXjPNCJ6iABW5XaWOsIZB93aQ==", + "dependencies": { + "@babel/parser": "^7.26.3", + "@babel/types": "^7.26.3", + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.25.9.tgz", + "integrity": "sha512-tnUA4RsrmflIM6W6RFTLFSXITtl0wKjgpnLgXyowocVPrbYrLUXSBXDgTs8BlbmIzIdlBySRQjINYs2BAkiLtw==", + "dependencies": { + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", + "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", + "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.26.3", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.3.tgz", + "integrity": "sha512-WJ/CvmY8Mea8iDXo6a7RK2wbmJITT5fN3BEkRuFlxVyNx8jOKIIhmC4fSkTcPcf8JyavbBwIe6OpiCOBXt/IcA==", + "dependencies": { + "@babel/types": "^7.26.3" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.0.tgz", + "integrity": "sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw==", + "dependencies": { + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/template": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.9.tgz", + "integrity": "sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==", + "dependencies": { + "@babel/code-frame": "^7.25.9", + "@babel/parser": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.26.4", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.26.4.tgz", + "integrity": "sha512-fH+b7Y4p3yqvApJALCPJcwb0/XaOSgtK4pzV6WVjPR5GLFQBRI7pfoX2V2iM48NXvX07NUxxm1Vw98YjqTcU5w==", + "dependencies": { + "@babel/code-frame": "^7.26.2", + "@babel/generator": "^7.26.3", + "@babel/parser": "^7.26.3", + "@babel/template": "^7.25.9", + "@babel/types": "^7.26.3", + "debug": "^4.3.1", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.26.3", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.3.tgz", + "integrity": "sha512-vN5p+1kl59GVKMvTHt55NzzmYVxprfJD+ql7U9NFIfKCBkYE55LYtS+WtPlaYOyzydrKI8Nezd+aZextrd+FMA==", + "dependencies": { + "@babel/helper-string-parser": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@colors/colors": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", + "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==", + "optional": true, + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/@dabh/diagnostics": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", + "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", + "dependencies": { + "colorspace": "1.1.x", + "enabled": "2.0.x", + "kuler": "^2.0.0" + } + }, + "node_modules/@emotion/babel-plugin": { + "version": "11.13.5", + "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.13.5.tgz", + "integrity": "sha512-pxHCpT2ex+0q+HH91/zsdHkw/lXd468DIN2zvfvLtPKLLMo6gQj7oLObq8PhkrxOZb/gGCq03S3Z7PDhS8pduQ==", + "dependencies": { + "@babel/helper-module-imports": "^7.16.7", + "@babel/runtime": "^7.18.3", + "@emotion/hash": "^0.9.2", + "@emotion/memoize": "^0.9.0", + "@emotion/serialize": "^1.3.3", + "babel-plugin-macros": "^3.1.0", + "convert-source-map": "^1.5.0", + "escape-string-regexp": "^4.0.0", + "find-root": "^1.1.0", + "source-map": "^0.5.7", + "stylis": "4.2.0" + } + }, + "node_modules/@emotion/cache": { + "version": "11.14.0", + "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.14.0.tgz", + "integrity": "sha512-L/B1lc/TViYk4DcpGxtAVbx0ZyiKM5ktoIyafGkH6zg/tj+mA+NE//aPYKG0k8kCHSHVJrpLpcAlOBEXQ3SavA==", + "dependencies": { + "@emotion/memoize": "^0.9.0", + "@emotion/sheet": "^1.4.0", + "@emotion/utils": "^1.4.2", + "@emotion/weak-memoize": "^0.4.0", + "stylis": "4.2.0" + } + }, + "node_modules/@emotion/hash": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.2.tgz", + "integrity": "sha512-MyqliTZGuOm3+5ZRSaaBGP3USLw6+EGykkwZns2EPC5g8jJ4z9OrdZY9apkl3+UP9+sdz76YYkwCKP5gh8iY3g==" + }, + "node_modules/@emotion/is-prop-valid": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-1.3.1.tgz", + "integrity": "sha512-/ACwoqx7XQi9knQs/G0qKvv5teDMhD7bXYns9N/wM8ah8iNb8jZ2uNO0YOgiq2o2poIvVtJS2YALasQuMSQ7Kw==", + "dependencies": { + "@emotion/memoize": "^0.9.0" + } + }, + "node_modules/@emotion/memoize": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.9.0.tgz", + "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==" + }, + "node_modules/@emotion/react": { + "version": "11.14.0", + "resolved": "https://registry.npmjs.org/@emotion/react/-/react-11.14.0.tgz", + "integrity": "sha512-O000MLDBDdk/EohJPFUqvnp4qnHeYkVP5B0xEG0D/L7cOKP9kefu2DXn8dj74cQfsEzUqh+sr1RzFqiL1o+PpA==", + "dependencies": { + "@babel/runtime": "^7.18.3", + "@emotion/babel-plugin": "^11.13.5", + "@emotion/cache": "^11.14.0", + "@emotion/serialize": "^1.3.3", + "@emotion/use-insertion-effect-with-fallbacks": "^1.2.0", + "@emotion/utils": "^1.4.2", + "@emotion/weak-memoize": "^0.4.0", + "hoist-non-react-statics": "^3.3.1" + }, + "peerDependencies": { + "react": ">=16.8.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@emotion/serialize": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.3.3.tgz", + "integrity": "sha512-EISGqt7sSNWHGI76hC7x1CksiXPahbxEOrC5RjmFRJTqLyEK9/9hZvBbiYn70dw4wuwMKiEMCUlR6ZXTSWQqxA==", + "dependencies": { + "@emotion/hash": "^0.9.2", + "@emotion/memoize": "^0.9.0", + "@emotion/unitless": "^0.10.0", + "@emotion/utils": "^1.4.2", + "csstype": "^3.0.2" + } + }, + "node_modules/@emotion/sheet": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.4.0.tgz", + "integrity": "sha512-fTBW9/8r2w3dXWYM4HCB1Rdp8NLibOw2+XELH5m5+AkWiL/KqYX6dc0kKYlaYyKjrQ6ds33MCdMPEwgs2z1rqg==" + }, + "node_modules/@emotion/styled": { + "version": "11.14.0", + "resolved": "https://registry.npmjs.org/@emotion/styled/-/styled-11.14.0.tgz", + "integrity": "sha512-XxfOnXFffatap2IyCeJyNov3kiDQWoR08gPUQxvbL7fxKryGBKUZUkG6Hz48DZwVrJSVh9sJboyV1Ds4OW6SgA==", + "dependencies": { + "@babel/runtime": "^7.18.3", + "@emotion/babel-plugin": "^11.13.5", + "@emotion/is-prop-valid": "^1.3.0", + "@emotion/serialize": "^1.3.3", + "@emotion/use-insertion-effect-with-fallbacks": "^1.2.0", + "@emotion/utils": "^1.4.2" + }, + "peerDependencies": { + "@emotion/react": "^11.0.0-rc.0", + "react": ">=16.8.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@emotion/unitless": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.10.0.tgz", + "integrity": "sha512-dFoMUuQA20zvtVTuxZww6OHoJYgrzfKM1t52mVySDJnMSEa08ruEvdYQbhvyu6soU+NeLVd3yKfTfT0NeV6qGg==" + }, + "node_modules/@emotion/use-insertion-effect-with-fallbacks": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@emotion/use-insertion-effect-with-fallbacks/-/use-insertion-effect-with-fallbacks-1.2.0.tgz", + "integrity": "sha512-yJMtVdH59sxi/aVJBpk9FQq+OR8ll5GT8oWd57UpeaKEVGab41JWaCFA7FRLoMLloOZF/c/wsPoe+bfGmRKgDg==", + "peerDependencies": { + "react": ">=16.8.0" + } + }, + "node_modules/@emotion/utils": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.4.2.tgz", + "integrity": "sha512-3vLclRofFziIa3J2wDh9jjbkUz9qk5Vi3IZ/FSTKViB0k+ef0fPV7dYrUIugbgupYDx7v9ud/SjrtEP8Y4xLoA==" + }, + "node_modules/@emotion/weak-memoize": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.4.0.tgz", + "integrity": "sha512-snKqtPW01tN0ui7yu9rGv69aJXr/a/Ywvl11sUjNtEcRc+ng/mQriFL0wLXMef74iHa/EkftbDzU9F8iFbH+zg==" + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.23.1.tgz", + "integrity": "sha512-6VhYk1diRqrhBAqpJEdjASR/+WVRtfjpqKuNw11cLiaWpAT/Uu+nokB+UJnevzy/P9C/ty6AOe0dwueMrGh/iQ==", + "cpu": [ + "ppc64" + ], + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.23.1.tgz", + "integrity": "sha512-uz6/tEy2IFm9RYOyvKl88zdzZfwEfKZmnX9Cj1BHjeSGNuGLuMD1kR8y5bteYmwqKm1tj8m4cb/aKEorr6fHWQ==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.23.1.tgz", + "integrity": "sha512-xw50ipykXcLstLeWH7WRdQuysJqejuAGPd30vd1i5zSyKK3WE+ijzHmLKxdiCMtH1pHz78rOg0BKSYOSB/2Khw==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.23.1.tgz", + "integrity": "sha512-nlN9B69St9BwUoB+jkyU090bru8L0NA3yFvAd7k8dNsVH8bi9a8cUAUSEcEEgTp2z3dbEDGJGfP6VUnkQnlReg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.23.1.tgz", + "integrity": "sha512-YsS2e3Wtgnw7Wq53XXBLcV6JhRsEq8hkfg91ESVadIrzr9wO6jJDMZnCQbHm1Guc5t/CdDiFSSfWP58FNuvT3Q==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.23.1.tgz", + "integrity": "sha512-aClqdgTDVPSEGgoCS8QDG37Gu8yc9lTHNAQlsztQ6ENetKEO//b8y31MMu2ZaPbn4kVsIABzVLXYLhCGekGDqw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.23.1.tgz", + "integrity": "sha512-h1k6yS8/pN/NHlMl5+v4XPfikhJulk4G+tKGFIOwURBSFzE8bixw1ebjluLOjfwtLqY0kewfjLSrO6tN2MgIhA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.23.1.tgz", + "integrity": "sha512-lK1eJeyk1ZX8UklqFd/3A60UuZ/6UVfGT2LuGo3Wp4/z7eRTRYY+0xOu2kpClP+vMTi9wKOfXi2vjUpO1Ro76g==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.23.1.tgz", + "integrity": "sha512-CXXkzgn+dXAPs3WBwE+Kvnrf4WECwBdfjfeYHpMeVxWE0EceB6vhWGShs6wi0IYEqMSIzdOF1XjQ/Mkm5d7ZdQ==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.23.1.tgz", + "integrity": "sha512-/93bf2yxencYDnItMYV/v116zff6UyTjo4EtEQjUBeGiVpMmffDNUyD9UN2zV+V3LRV3/on4xdZ26NKzn6754g==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.23.1.tgz", + "integrity": "sha512-VTN4EuOHwXEkXzX5nTvVY4s7E/Krz7COC8xkftbbKRYAl96vPiUssGkeMELQMOnLOJ8k3BY1+ZY52tttZnHcXQ==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.23.1.tgz", + "integrity": "sha512-Vx09LzEoBa5zDnieH8LSMRToj7ir/Jeq0Gu6qJ/1GcBq9GkfoEAoXvLiW1U9J1qE/Y/Oyaq33w5p2ZWrNNHNEw==", + "cpu": [ + "loong64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.23.1.tgz", + "integrity": "sha512-nrFzzMQ7W4WRLNUOU5dlWAqa6yVeI0P78WKGUo7lg2HShq/yx+UYkeNSE0SSfSure0SqgnsxPvmAUu/vu0E+3Q==", + "cpu": [ + "mips64el" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.23.1.tgz", + "integrity": "sha512-dKN8fgVqd0vUIjxuJI6P/9SSSe/mB9rvA98CSH2sJnlZ/OCZWO1DJvxj8jvKTfYUdGfcq2dDxoKaC6bHuTlgcw==", + "cpu": [ + "ppc64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.23.1.tgz", + "integrity": "sha512-5AV4Pzp80fhHL83JM6LoA6pTQVWgB1HovMBsLQ9OZWLDqVY8MVobBXNSmAJi//Csh6tcY7e7Lny2Hg1tElMjIA==", + "cpu": [ + "riscv64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.23.1.tgz", + "integrity": "sha512-9ygs73tuFCe6f6m/Tb+9LtYxWR4c9yg7zjt2cYkjDbDpV/xVn+68cQxMXCjUpYwEkze2RcU/rMnfIXNRFmSoDw==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.23.1.tgz", + "integrity": "sha512-EV6+ovTsEXCPAp58g2dD68LxoP/wK5pRvgy0J/HxPGB009omFPv3Yet0HiaqvrIrgPTBuC6wCH1LTOY91EO5hQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.23.1.tgz", + "integrity": "sha512-aevEkCNu7KlPRpYLjwmdcuNz6bDFiE7Z8XC4CPqExjTvrHugh28QzUXVOZtiYghciKUacNktqxdpymplil1beA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.23.1.tgz", + "integrity": "sha512-3x37szhLexNA4bXhLrCC/LImN/YtWis6WXr1VESlfVtVeoFJBRINPJ3f0a/6LV8zpikqoUg4hyXw0sFBt5Cr+Q==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.23.1.tgz", + "integrity": "sha512-aY2gMmKmPhxfU+0EdnN+XNtGbjfQgwZj43k8G3fyrDM/UdZww6xrWxmDkuz2eCZchqVeABjV5BpildOrUbBTqA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.23.1.tgz", + "integrity": "sha512-RBRT2gqEl0IKQABT4XTj78tpk9v7ehp+mazn2HbUeZl1YMdaGAQqhapjGTCe7uw7y0frDi4gS0uHzhvpFuI1sA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.23.1.tgz", + "integrity": "sha512-4O+gPR5rEBe2FpKOVyiJ7wNDPA8nGzDuJ6gN4okSA1gEOYZ67N8JPk58tkWtdtPeLz7lBnY6I5L3jdsr3S+A6A==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.23.1.tgz", + "integrity": "sha512-BcaL0Vn6QwCwre3Y717nVHZbAa4UBEigzFm6VdsVdT/MbZ38xoj1X9HPkZhbmaBGUD1W8vxAfffbDe8bA6AKnQ==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.23.1.tgz", + "integrity": "sha512-BHpFFeslkWrXWyUPnbKm+xYYVYruCinGcftSBaa8zoF9hZO4BcSCFUvHVTtzpIY6YzUnYtuEhZ+C9iEXjxnasg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@fal-ai/serverless-client": { + "version": "0.14.3", + "resolved": "https://registry.npmjs.org/@fal-ai/serverless-client/-/serverless-client-0.14.3.tgz", + "integrity": "sha512-xroGr51Xi5XlOs+cWKrEY8NvD7VngCGJDIJmBLQ7WLa/HRiPWmarHz00noSQ3Go/508RsdPcstDZnEFLBL3mgw==", + "peer": true, + "dependencies": { + "@msgpack/msgpack": "^3.0.0-beta2", + "eventsource-parser": "^1.1.2", + "robot3": "^0.4.1" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@googleapis/sheets": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/@googleapis/sheets/-/sheets-9.3.1.tgz", + "integrity": "sha512-nPgzOiDs/FSFhE+dX2KfkmsmkXM3WfXYP06FoW8cXvHshwxHSI3FbXwe5XJYstDAWXP9YA7AMSvmwnuD4OAl2w==", + "dependencies": { + "googleapis-common": "^7.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@ibm-cloud/watsonx-ai": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@ibm-cloud/watsonx-ai/-/watsonx-ai-1.3.0.tgz", + "integrity": "sha512-V4PorMPhxwYiayWxycryun4Bjxn3PJrQqJGca+maQd61Q7s+/PUJAHWjwzVSVHxiher17zFHf4NwqB8J6bWj4w==", + "peer": true, + "dependencies": { + "@types/node": "^18.0.0", + "extend": "3.0.2", + "ibm-cloud-sdk-core": "^5.0.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@ibm-generative-ai/node-sdk": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@ibm-generative-ai/node-sdk/-/node-sdk-2.0.6.tgz", + "integrity": "sha512-RX+2FMVZEy6giDlrMmbahZA/nrIZmyEO9Ss1I+a5yaask4Zs1YkrEI/+CjZI8x7MU9Mk6RRyOnLC33EdiJLKTw==", + "peer": true, + "dependencies": { + "@ai-zen/node-fetch-event-source": "^2.1.2", + "fetch-retry": "^5.0.6", + "http-status-codes": "^2.3.0", + "openapi-fetch": "^0.8.2", + "p-queue-compat": "^1.0.225", + "yaml": "^2.3.3" + }, + "peerDependencies": { + "@langchain/core": ">=0.1.0" + } + }, + "node_modules/@inquirer/checkbox": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-3.0.1.tgz", + "integrity": "sha512-0hm2nrToWUdD6/UHnel/UKGdk1//ke5zGUpHIvk5ZWmaKezlGxZkOJXNSWsdxO/rEqTkbB3lNC2J6nBElV2aAQ==", + "dependencies": { + "@inquirer/core": "^9.2.1", + "@inquirer/figures": "^1.0.6", + "@inquirer/type": "^2.0.0", + "ansi-escapes": "^4.3.2", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/confirm": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-4.0.1.tgz", + "integrity": "sha512-46yL28o2NJ9doViqOy0VDcoTzng7rAb6yPQKU7VDLqkmbCaH4JqK4yk4XqlzNWy9PVC5pG1ZUXPBQv+VqnYs2w==", + "dependencies": { + "@inquirer/core": "^9.2.1", + "@inquirer/type": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/core": { + "version": "9.2.1", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-9.2.1.tgz", + "integrity": "sha512-F2VBt7W/mwqEU4bL0RnHNZmC/OxzNx9cOYxHqnXX3MP6ruYvZUZAW9imgN9+h/uBT/oP8Gh888J2OZSbjSeWcg==", + "dependencies": { + "@inquirer/figures": "^1.0.6", + "@inquirer/type": "^2.0.0", + "@types/mute-stream": "^0.0.4", + "@types/node": "^22.5.5", + "@types/wrap-ansi": "^3.0.0", + "ansi-escapes": "^4.3.2", + "cli-width": "^4.1.0", + "mute-stream": "^1.0.0", + "signal-exit": "^4.1.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^6.2.0", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/core/node_modules/@types/node": { + "version": "22.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.1.tgz", + "integrity": "sha512-qKgsUwfHZV2WCWLAnVP1JqnpE6Im6h3Y0+fYgMTasNQ7V++CBX5OT1as0g0f+OyubbFqhf6XVNIsmN4IIhEgGQ==", + "dependencies": { + "undici-types": "~6.20.0" + } + }, + "node_modules/@inquirer/core/node_modules/undici-types": { + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==" + }, + "node_modules/@inquirer/editor": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-3.0.1.tgz", + "integrity": "sha512-VA96GPFaSOVudjKFraokEEmUQg/Lub6OXvbIEZU1SDCmBzRkHGhxoFAVaF30nyiB4m5cEbDgiI2QRacXZ2hw9Q==", + "dependencies": { + "@inquirer/core": "^9.2.1", + "@inquirer/type": "^2.0.0", + "external-editor": "^3.1.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/expand": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-3.0.1.tgz", + "integrity": "sha512-ToG8d6RIbnVpbdPdiN7BCxZGiHOTomOX94C2FaT5KOHupV40tKEDozp12res6cMIfRKrXLJyexAZhWVHgbALSQ==", + "dependencies": { + "@inquirer/core": "^9.2.1", + "@inquirer/type": "^2.0.0", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/figures": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.8.tgz", + "integrity": "sha512-tKd+jsmhq21AP1LhexC0pPwsCxEhGgAkg28byjJAd+xhmIs8LUX8JbUc3vBf3PhLxWiB5EvyBE5X7JSPAqMAqg==", + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/input": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-3.0.1.tgz", + "integrity": "sha512-BDuPBmpvi8eMCxqC5iacloWqv+5tQSJlUafYWUe31ow1BVXjW2a5qe3dh4X/Z25Wp22RwvcaLCc2siHobEOfzg==", + "dependencies": { + "@inquirer/core": "^9.2.1", + "@inquirer/type": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/number": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-2.0.1.tgz", + "integrity": "sha512-QpR8jPhRjSmlr/mD2cw3IR8HRO7lSVOnqUvQa8scv1Lsr3xoAMMworcYW3J13z3ppjBFBD2ef1Ci6AE5Qn8goQ==", + "dependencies": { + "@inquirer/core": "^9.2.1", + "@inquirer/type": "^2.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/password": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-3.0.1.tgz", + "integrity": "sha512-haoeEPUisD1NeE2IanLOiFr4wcTXGWrBOyAyPZi1FfLJuXOzNmxCJPgUrGYKVh+Y8hfGJenIfz5Wb/DkE9KkMQ==", + "dependencies": { + "@inquirer/core": "^9.2.1", + "@inquirer/type": "^2.0.0", + "ansi-escapes": "^4.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/prompts": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-6.0.1.tgz", + "integrity": "sha512-yl43JD/86CIj3Mz5mvvLJqAOfIup7ncxfJ0Btnl0/v5TouVUyeEdcpknfgc+yMevS/48oH9WAkkw93m7otLb/A==", + "dependencies": { + "@inquirer/checkbox": "^3.0.1", + "@inquirer/confirm": "^4.0.1", + "@inquirer/editor": "^3.0.1", + "@inquirer/expand": "^3.0.1", + "@inquirer/input": "^3.0.1", + "@inquirer/number": "^2.0.1", + "@inquirer/password": "^3.0.1", + "@inquirer/rawlist": "^3.0.1", + "@inquirer/search": "^2.0.1", + "@inquirer/select": "^3.0.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/rawlist": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-3.0.1.tgz", + "integrity": "sha512-VgRtFIwZInUzTiPLSfDXK5jLrnpkuSOh1ctfaoygKAdPqjcjKYmGh6sCY1pb0aGnCGsmhUxoqLDUAU0ud+lGXQ==", + "dependencies": { + "@inquirer/core": "^9.2.1", + "@inquirer/type": "^2.0.0", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/search": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-2.0.1.tgz", + "integrity": "sha512-r5hBKZk3g5MkIzLVoSgE4evypGqtOannnB3PKTG9NRZxyFRKcfzrdxXXPcoJQsxJPzvdSU2Rn7pB7lw0GCmGAg==", + "dependencies": { + "@inquirer/core": "^9.2.1", + "@inquirer/figures": "^1.0.6", + "@inquirer/type": "^2.0.0", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/select": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-3.0.1.tgz", + "integrity": "sha512-lUDGUxPhdWMkN/fHy1Lk7pF3nK1fh/gqeyWXmctefhxLYxlDsc7vsPBEpxrfVGDsVdyYJsiJoD4bJ1b623cV1Q==", + "dependencies": { + "@inquirer/core": "^9.2.1", + "@inquirer/figures": "^1.0.6", + "@inquirer/type": "^2.0.0", + "ansi-escapes": "^4.3.2", + "yoctocolors-cjs": "^2.1.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-2.0.0.tgz", + "integrity": "sha512-XvJRx+2KR3YXyYtPUUy+qd9i7p+GO9Ko6VIIpWlBrpWwXDv8WLFeHTxz35CfQFUiBMLXlGHhGzys7lqit9gWag==", + "dependencies": { + "mute-stream": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.1.0.tgz", + "integrity": "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", + "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==", + "dependencies": { + "@jridgewell/set-array": "^1.2.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", + "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@jsdevtools/ono": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", + "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==" + }, + "node_modules/@langchain/core": { + "version": "0.3.22", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.3.22.tgz", + "integrity": "sha512-9rwEbxJi3Fgs8XuealNYxB6s0FCOnvXLnpiV5/oKgmEJtCRS91IqgJCWA8d59s4YkaEply/EsZVc2azNPK6Wjw==", + "peer": true, + "dependencies": { + "ansi-styles": "^5.0.0", + "camelcase": "6", + "decamelize": "1.2.0", + "js-tiktoken": "^1.0.12", + "langsmith": "^0.2.8", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "p-retry": "4", + "uuid": "^10.0.0", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@langchain/core/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "peer": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/@libsql/client-wasm": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@libsql/client-wasm/-/client-wasm-0.14.0.tgz", + "integrity": "sha512-gB/jtz0xuwrqAHApBv9e9JSew2030Fhj2edyZ83InZ4yPj/Q2LTUlEhaspEYT0T0xsAGqPy38uGrmq/OGS+DdQ==", + "bundleDependencies": [ + "@libsql/libsql-wasm-experimental" + ], + "peer": true, + "dependencies": { + "@libsql/core": "^0.14.0", + "@libsql/libsql-wasm-experimental": "^0.0.2", + "js-base64": "^3.7.5" + } + }, + "node_modules/@libsql/client-wasm/node_modules/@libsql/libsql-wasm-experimental": { + "version": "0.0.2", + "inBundle": true, + "license": "Apache-2.0", + "peer": true, + "bin": { + "sqlite-wasm": "bin/index.js" + } + }, + "node_modules/@libsql/core": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@libsql/core/-/core-0.14.0.tgz", + "integrity": "sha512-nhbuXf7GP3PSZgdCY2Ecj8vz187ptHlZQ0VRc751oB2C1W8jQUXKKklvt7t1LJiUTQBVJuadF628eUk+3cRi4Q==", + "peer": true, + "dependencies": { + "js-base64": "^3.7.5" + } + }, + "node_modules/@msgpack/msgpack": { + "version": "3.0.0-beta2", + "resolved": "https://registry.npmjs.org/@msgpack/msgpack/-/msgpack-3.0.0-beta2.tgz", + "integrity": "sha512-y+l1PNV0XDyY8sM3YtuMLK5vE3/hkfId+Do8pLo/OPxfxuFAUwcGz3oiiUuV46/aBpwTzZ+mRWVMtlSKbradhw==", + "peer": true, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@mui/core-downloads-tracker": { + "version": "6.1.10", + "resolved": "https://registry.npmjs.org/@mui/core-downloads-tracker/-/core-downloads-tracker-6.1.10.tgz", + "integrity": "sha512-LY5wdiLCBDY7u+Od8UmFINZFGN/5ZU90fhAslf/ZtfP+5RhuY45f679pqYIxe0y54l6Gkv9PFOc8Cs10LDTBYg==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + } + }, + "node_modules/@mui/material": { + "version": "6.1.10", + "resolved": "https://registry.npmjs.org/@mui/material/-/material-6.1.10.tgz", + "integrity": "sha512-txnwYObY4N9ugv5T2n5h1KcbISegZ6l65w1/7tpSU5OB6MQCU94YkP8n/3slDw2KcEfRk4+4D8EUGfhSPMODEQ==", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/core-downloads-tracker": "^6.1.10", + "@mui/system": "^6.1.10", + "@mui/types": "^7.2.19", + "@mui/utils": "^6.1.10", + "@popperjs/core": "^2.11.8", + "@types/react-transition-group": "^4.4.11", + "clsx": "^2.1.1", + "csstype": "^3.1.3", + "prop-types": "^15.8.1", + "react-is": "^18.3.1", + "react-transition-group": "^4.4.5" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@emotion/react": "^11.5.0", + "@emotion/styled": "^11.3.0", + "@mui/material-pigment-css": "^6.1.10", + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + }, + "@mui/material-pigment-css": { + "optional": true + }, + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/private-theming": { + "version": "6.1.10", + "resolved": "https://registry.npmjs.org/@mui/private-theming/-/private-theming-6.1.10.tgz", + "integrity": "sha512-DqgsH0XFEweeG3rQfVkqTkeXcj/E76PGYWag8flbPdV8IYdMo+DfVdFlZK8JEjsaIVD2Eu1kJg972XnH5pfnBQ==", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/utils": "^6.1.10", + "prop-types": "^15.8.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/styled-engine": { + "version": "6.1.10", + "resolved": "https://registry.npmjs.org/@mui/styled-engine/-/styled-engine-6.1.10.tgz", + "integrity": "sha512-+NV9adKZYhslJ270iPjf2yzdVJwav7CIaXcMlPSi1Xy1S/zRe5xFgZ6BEoMdmGRpr34lIahE8H1acXP2myrvRw==", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@emotion/cache": "^11.13.5", + "@emotion/serialize": "^1.3.3", + "@emotion/sheet": "^1.4.0", + "csstype": "^3.1.3", + "prop-types": "^15.8.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@emotion/react": "^11.4.1", + "@emotion/styled": "^11.3.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + } + } + }, + "node_modules/@mui/system": { + "version": "6.1.10", + "resolved": "https://registry.npmjs.org/@mui/system/-/system-6.1.10.tgz", + "integrity": "sha512-5YNIqxETR23SIkyP7MY2fFnXmplX/M4wNi2R+10AVRd3Ub+NLctWY/Vs5vq1oAMF0eSDLhRTGUjaUe+IGSfWqg==", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/private-theming": "^6.1.10", + "@mui/styled-engine": "^6.1.10", + "@mui/types": "^7.2.19", + "@mui/utils": "^6.1.10", + "clsx": "^2.1.1", + "csstype": "^3.1.3", + "prop-types": "^15.8.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@emotion/react": "^11.5.0", + "@emotion/styled": "^11.3.0", + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + }, + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/types": { + "version": "7.2.19", + "resolved": "https://registry.npmjs.org/@mui/types/-/types-7.2.19.tgz", + "integrity": "sha512-6XpZEM/Q3epK9RN8ENoXuygnqUQxE+siN/6rGRi2iwJPgBUR25mphYQ9ZI87plGh58YoZ5pp40bFvKYOCDJ3tA==", + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/utils": { + "version": "6.1.10", + "resolved": "https://registry.npmjs.org/@mui/utils/-/utils-6.1.10.tgz", + "integrity": "sha512-1ETuwswGjUiAf2dP9TkBy8p49qrw2wXa+RuAjNTRE5+91vtXJ1HKrs7H9s8CZd1zDlQVzUcUAPm9lpQwF5ogTw==", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/types": "^7.2.19", + "@types/prop-types": "^15.7.13", + "clsx": "^2.1.1", + "prop-types": "^15.8.1", + "react-is": "^18.3.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", + "integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==", + "engines": { + "node": ">=14" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@playwright/browser-chromium": { + "version": "1.49.1", + "resolved": "https://registry.npmjs.org/@playwright/browser-chromium/-/browser-chromium-1.49.1.tgz", + "integrity": "sha512-LLeyllKSucbojsJBOpdJshwW27ZXZs3oypqffkVWLUvxX2azHJMOevsOcWpjCfoYbpevkaEozM2xHeSUGF00lg==", + "hasInstallScript": true, + "peer": true, + "dependencies": { + "playwright-core": "1.49.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@popperjs/core": { + "version": "2.11.8", + "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz", + "integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/popperjs" + } + }, + "node_modules/@smithy/abort-controller": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.9.tgz", + "integrity": "sha512-yiW0WI30zj8ZKoSYNx90no7ugVn3khlyH/z5W8qtKBtVE6awRALbhSG+2SAHA1r6bO/6M9utxYKVZ3PCJ1rWxw==", + "peer": true, + "dependencies": { + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/config-resolver": { + "version": "3.0.13", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.13.tgz", + "integrity": "sha512-Gr/qwzyPaTL1tZcq8WQyHhTZREER5R1Wytmz4WnVGL4onA3dNk6Btll55c8Vr58pLdvWZmtG8oZxJTw3t3q7Jg==", + "peer": true, + "dependencies": { + "@smithy/node-config-provider": "^3.1.12", + "@smithy/types": "^3.7.2", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.11", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/core": { + "version": "2.5.5", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.5.5.tgz", + "integrity": "sha512-G8G/sDDhXA7o0bOvkc7bgai6POuSld/+XhNnWAbpQTpLv2OZPvyqQ58tLPPlz0bSNsXktldDDREIv1LczFeNEw==", + "peer": true, + "dependencies": { + "@smithy/middleware-serde": "^3.0.11", + "@smithy/protocol-http": "^4.1.8", + "@smithy/types": "^3.7.2", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-middleware": "^3.0.11", + "@smithy/util-stream": "^3.3.2", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "3.2.8", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.2.8.tgz", + "integrity": "sha512-ZCY2yD0BY+K9iMXkkbnjo+08T2h8/34oHd0Jmh6BZUSZwaaGlGCyBT/3wnS7u7Xl33/EEfN4B6nQr3Gx5bYxgw==", + "peer": true, + "dependencies": { + "@smithy/node-config-provider": "^3.1.12", + "@smithy/property-provider": "^3.1.11", + "@smithy/types": "^3.7.2", + "@smithy/url-parser": "^3.0.11", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/eventstream-codec": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-3.1.10.tgz", + "integrity": "sha512-323B8YckSbUH0nMIpXn7HZsAVKHYHFUODa8gG9cHo0ySvA1fr5iWaNT+iIL0UCqUzG6QPHA3BSsBtRQou4mMqQ==", + "peer": true, + "dependencies": { + "@aws-crypto/crc32": "5.2.0", + "@smithy/types": "^3.7.2", + "@smithy/util-hex-encoding": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/eventstream-serde-browser": { + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-3.0.14.tgz", + "integrity": "sha512-kbrt0vjOIihW3V7Cqj1SXQvAI5BR8SnyQYsandva0AOR307cXAc+IhPngxIPslxTLfxwDpNu0HzCAq6g42kCPg==", + "peer": true, + "dependencies": { + "@smithy/eventstream-serde-universal": "^3.0.13", + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-config-resolver": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-3.0.11.tgz", + "integrity": "sha512-P2pnEp4n75O+QHjyO7cbw/vsw5l93K/8EWyjNCAAybYwUmj3M+hjSQZ9P5TVdUgEG08ueMAP5R4FkuSkElZ5tQ==", + "peer": true, + "dependencies": { + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-node": { + "version": "3.0.13", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-3.0.13.tgz", + "integrity": "sha512-zqy/9iwbj8Wysmvi7Lq7XFLeDgjRpTbCfwBhJa8WbrylTAHiAu6oQTwdY7iu2lxigbc9YYr9vPv5SzYny5tCXQ==", + "peer": true, + "dependencies": { + "@smithy/eventstream-serde-universal": "^3.0.13", + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-universal": { + "version": "3.0.13", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-3.0.13.tgz", + "integrity": "sha512-L1Ib66+gg9uTnqp/18Gz4MDpJPKRE44geOjOQ2SVc0eiaO5l255ADziATZgjQjqumC7yPtp1XnjHlF1srcwjKw==", + "peer": true, + "dependencies": { + "@smithy/eventstream-codec": "^3.1.10", + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-4.1.2.tgz", + "integrity": "sha512-R7rU7Ae3ItU4rC0c5mB2sP5mJNbCfoDc8I5XlYjIZnquyUwec7fEo78F6DA3SmgJgkU1qTMcZJuGblxZsl10ZA==", + "peer": true, + "dependencies": { + "@smithy/protocol-http": "^4.1.8", + "@smithy/querystring-builder": "^3.0.11", + "@smithy/types": "^3.7.2", + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/hash-node": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.11.tgz", + "integrity": "sha512-emP23rwYyZhQBvklqTtwetkQlqbNYirDiEEwXl2v0GYWMnCzxst7ZaRAnWuy28njp5kAH54lvkdG37MblZzaHA==", + "peer": true, + "dependencies": { + "@smithy/types": "^3.7.2", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/invalid-dependency": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.11.tgz", + "integrity": "sha512-NuQmVPEJjUX6c+UELyVz8kUx8Q539EDeNwbRyu4IIF8MeV7hUtq1FB3SHVyki2u++5XLMFqngeMKk7ccspnNyQ==", + "peer": true, + "dependencies": { + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-content-length": { + "version": "3.0.13", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.13.tgz", + "integrity": "sha512-zfMhzojhFpIX3P5ug7jxTjfUcIPcGjcQYzB9t+rv0g1TX7B0QdwONW+ATouaLoD7h7LOw/ZlXfkq4xJ/g2TrIw==", + "peer": true, + "dependencies": { + "@smithy/protocol-http": "^4.1.8", + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-endpoint": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.2.5.tgz", + "integrity": "sha512-VhJNs/s/lyx4weiZdXSloBgoLoS8osV0dKIain8nGmx7of3QFKu5BSdEuk1z/U8x9iwes1i+XCiNusEvuK1ijg==", + "peer": true, + "dependencies": { + "@smithy/core": "^2.5.5", + "@smithy/middleware-serde": "^3.0.11", + "@smithy/node-config-provider": "^3.1.12", + "@smithy/shared-ini-file-loader": "^3.1.12", + "@smithy/types": "^3.7.2", + "@smithy/url-parser": "^3.0.11", + "@smithy/util-middleware": "^3.0.11", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-retry": { + "version": "3.0.30", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.30.tgz", + "integrity": "sha512-6323RL2BvAR3VQpTjHpa52kH/iSHyxd/G9ohb2MkBk2Ucu+oMtRXT8yi7KTSIS9nb58aupG6nO0OlXnQOAcvmQ==", + "peer": true, + "dependencies": { + "@smithy/node-config-provider": "^3.1.12", + "@smithy/protocol-http": "^4.1.8", + "@smithy/service-error-classification": "^3.0.11", + "@smithy/smithy-client": "^3.5.0", + "@smithy/types": "^3.7.2", + "@smithy/util-middleware": "^3.0.11", + "@smithy/util-retry": "^3.0.11", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-serde": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.11.tgz", + "integrity": "sha512-KzPAeySp/fOoQA82TpnwItvX8BBURecpx6ZMu75EZDkAcnPtO6vf7q4aH5QHs/F1s3/snQaSFbbUMcFFZ086Mw==", + "peer": true, + "dependencies": { + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-stack": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.11.tgz", + "integrity": "sha512-1HGo9a6/ikgOMrTrWL/WiN9N8GSVYpuRQO5kjstAq4CvV59bjqnh7TbdXGQ4vxLD3xlSjfBjq5t1SOELePsLnA==", + "peer": true, + "dependencies": { + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/node-config-provider": { + "version": "3.1.12", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.12.tgz", + "integrity": "sha512-O9LVEu5J/u/FuNlZs+L7Ikn3lz7VB9hb0GtPT9MQeiBmtK8RSY3ULmsZgXhe6VAlgTw0YO+paQx4p8xdbs43vQ==", + "peer": true, + "dependencies": { + "@smithy/property-provider": "^3.1.11", + "@smithy/shared-ini-file-loader": "^3.1.12", + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.3.2.tgz", + "integrity": "sha512-t4ng1DAd527vlxvOfKFYEe6/QFBcsj7WpNlWTyjorwXXcKw3XlltBGbyHfSJ24QT84nF+agDha9tNYpzmSRZPA==", + "peer": true, + "dependencies": { + "@smithy/abort-controller": "^3.1.9", + "@smithy/protocol-http": "^4.1.8", + "@smithy/querystring-builder": "^3.0.11", + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/property-provider": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.11.tgz", + "integrity": "sha512-I/+TMc4XTQ3QAjXfOcUWbSS073oOEAxgx4aZy8jHaf8JQnRkq2SZWw8+PfDtBvLUjcGMdxl+YwtzWe6i5uhL/A==", + "peer": true, + "dependencies": { + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/protocol-http": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.8.tgz", + "integrity": "sha512-hmgIAVyxw1LySOwkgMIUN0kjN8TG9Nc85LJeEmEE/cNEe2rkHDUWhnJf2gxcSRFLWsyqWsrZGw40ROjUogg+Iw==", + "dependencies": { + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/querystring-builder": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.11.tgz", + "integrity": "sha512-u+5HV/9uJaeLj5XTb6+IEF/dokWWkEqJ0XiaRRogyREmKGUgZnNecLucADLdauWFKUNbQfulHFEZEdjwEBjXRg==", + "peer": true, + "dependencies": { + "@smithy/types": "^3.7.2", + "@smithy/util-uri-escape": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/querystring-parser": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.11.tgz", + "integrity": "sha512-Je3kFvCsFMnso1ilPwA7GtlbPaTixa3WwC+K21kmMZHsBEOZYQaqxcMqeFFoU7/slFjKDIpiiPydvdJm8Q/MCw==", + "peer": true, + "dependencies": { + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/service-error-classification": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.11.tgz", + "integrity": "sha512-QnYDPkyewrJzCyaeI2Rmp7pDwbUETe+hU8ADkXmgNusO1bgHBH7ovXJiYmba8t0fNfJx75fE8dlM6SEmZxheog==", + "peer": true, + "dependencies": { + "@smithy/types": "^3.7.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/shared-ini-file-loader": { + "version": "3.1.12", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.12.tgz", + "integrity": "sha512-1xKSGI+U9KKdbG2qDvIR9dGrw3CNx+baqJfyr0igKEpjbHL5stsqAesYBzHChYHlelWtb87VnLWlhvfCz13H8Q==", + "peer": true, + "dependencies": { + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-4.2.4.tgz", + "integrity": "sha512-5JWeMQYg81TgU4cG+OexAWdvDTs5JDdbEZx+Qr1iPbvo91QFGzjy0IkXAKaXUHqmKUJgSHK0ZxnCkgZpzkeNTA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/protocol-http": "^4.1.8", + "@smithy/types": "^3.7.2", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-middleware": "^3.0.11", + "@smithy/util-uri-escape": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/smithy-client": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.5.0.tgz", + "integrity": "sha512-Y8FeOa7gbDfCWf7njrkoRATPa5eNLUEjlJS5z5rXatYuGkCb80LbHcu8AQR8qgAZZaNHCLyo2N+pxPsV7l+ivg==", + "peer": true, + "dependencies": { + "@smithy/core": "^2.5.5", + "@smithy/middleware-endpoint": "^3.2.5", + "@smithy/middleware-stack": "^3.0.11", + "@smithy/protocol-http": "^4.1.8", + "@smithy/types": "^3.7.2", + "@smithy/util-stream": "^3.3.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.2.tgz", + "integrity": "sha512-bNwBYYmN8Eh9RyjS1p2gW6MIhSO2rl7X9QeLM8iTdcGRP+eDiIWDt66c9IysCc22gefKszZv+ubV9qZc7hdESg==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/url-parser": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.11.tgz", + "integrity": "sha512-TmlqXkSk8ZPhfc+SQutjmFr5FjC0av3GZP4B/10caK1SbRwe/v+Wzu/R6xEKxoNqL+8nY18s1byiy6HqPG37Aw==", + "peer": true, + "dependencies": { + "@smithy/querystring-parser": "^3.0.11", + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/util-base64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", + "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", + "peer": true, + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-body-length-browser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", + "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", + "peer": true, + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/util-body-length-node": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", + "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", + "peer": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-config-provider": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", + "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", + "peer": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-defaults-mode-browser": { + "version": "3.0.30", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.30.tgz", + "integrity": "sha512-nLuGmgfcr0gzm64pqF2UT4SGWVG8UGviAdayDlVzJPNa6Z4lqvpDzdRXmLxtOdEjVlTOEdpZ9dd3ZMMu488mzg==", + "peer": true, + "dependencies": { + "@smithy/property-provider": "^3.1.11", + "@smithy/smithy-client": "^3.5.0", + "@smithy/types": "^3.7.2", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@smithy/util-defaults-mode-node": { + "version": "3.0.30", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.30.tgz", + "integrity": "sha512-OD63eWoH68vp75mYcfYyuVH+p7Li/mY4sYOROnauDrtObo1cS4uWfsy/zhOTW8F8ZPxQC1ZXZKVxoxvMGUv2Ow==", + "peer": true, + "dependencies": { + "@smithy/config-resolver": "^3.0.13", + "@smithy/credential-provider-imds": "^3.2.8", + "@smithy/node-config-provider": "^3.1.12", + "@smithy/property-provider": "^3.1.11", + "@smithy/smithy-client": "^3.5.0", + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@smithy/util-endpoints": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.1.7.tgz", + "integrity": "sha512-tSfcqKcN/Oo2STEYCABVuKgJ76nyyr6skGl9t15hs+YaiU06sgMkN7QYjo0BbVw+KT26zok3IzbdSOksQ4YzVw==", + "peer": true, + "dependencies": { + "@smithy/node-config-provider": "^3.1.12", + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-middleware": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.11.tgz", + "integrity": "sha512-dWpyc1e1R6VoXrwLoLDd57U1z6CwNSdkM69Ie4+6uYh2GC7Vg51Qtan7ITzczuVpqezdDTKJGJB95fFvvjU/ow==", + "dependencies": { + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-retry": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.11.tgz", + "integrity": "sha512-hJUC6W7A3DQgaee3Hp9ZFcOxVDZzmBIRBPlUAk8/fSOEl7pE/aX7Dci0JycNOnm9Mfr0KV2XjIlUOcGWXQUdVQ==", + "peer": true, + "dependencies": { + "@smithy/service-error-classification": "^3.0.11", + "@smithy/types": "^3.7.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-stream": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.3.2.tgz", + "integrity": "sha512-sInAqdiVeisUGYAv/FrXpmJ0b4WTFmciTRqzhb7wVuem9BHvhIG7tpiYHLDWrl2stOokNZpTTGqz3mzB2qFwXg==", + "peer": true, + "dependencies": { + "@smithy/fetch-http-handler": "^4.1.2", + "@smithy/node-http-handler": "^3.3.2", + "@smithy/types": "^3.7.2", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@socket.io/component-emitter": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@socket.io/component-emitter/-/component-emitter-3.1.2.tgz", + "integrity": "sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==" + }, + "node_modules/@tokenizer/token": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@tokenizer/token/-/token-0.3.0.tgz", + "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", + "peer": true + }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==" + }, + "node_modules/@types/cookie": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@types/cookie/-/cookie-0.4.1.tgz", + "integrity": "sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q==" + }, + "node_modules/@types/cors": { + "version": "2.8.17", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.17.tgz", + "integrity": "sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "peer": true, + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==" + }, + "node_modules/@types/ms": { + "version": "0.7.34", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", + "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==", + "peer": true + }, + "node_modules/@types/mute-stream": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.4.tgz", + "integrity": "sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/node": { + "version": "18.19.67", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.67.tgz", + "integrity": "sha512-wI8uHusga+0ZugNp0Ol/3BqQfEcCCNfojtO6Oou9iVNGPTL6QNSdnUdqq85fRgIorLhLMuPIKpsN98QE9Nh+KQ==", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/node-fetch": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, + "node_modules/@types/parse-json": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", + "integrity": "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==" + }, + "node_modules/@types/pegjs": { + "version": "0.10.6", + "resolved": "https://registry.npmjs.org/@types/pegjs/-/pegjs-0.10.6.tgz", + "integrity": "sha512-eLYXDbZWXh2uxf+w8sXS8d6KSoXTswfps6fvCUuVAGN8eRpfe7h9eSRydxiSJvo9Bf+GzifsDOr9TMQlmJdmkw==", + "peer": true + }, + "node_modules/@types/prop-types": { + "version": "15.7.14", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.14.tgz", + "integrity": "sha512-gNMvNH49DJ7OJYv+KAKn0Xp45p8PLl6zo2YnvDIbTd4J6MER2BmWN49TG7n9LvkyihINxeKW8+3bfS2yDC9dzQ==" + }, + "node_modules/@types/react": { + "version": "19.0.1", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.0.1.tgz", + "integrity": "sha512-YW6614BDhqbpR5KtUYzTA+zlA7nayzJRA9ljz9CQoxthR0sDisYZLuvSMsil36t4EH/uAt8T52Xb4sVw17G+SQ==", + "dependencies": { + "csstype": "^3.0.2" + } + }, + "node_modules/@types/react-transition-group": { + "version": "4.4.11", + "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.11.tgz", + "integrity": "sha512-RM05tAniPZ5DZPzzNFP+DmrcOdD0efDUxMy3145oljWSl3x9ZV5vhme98gTxFrj2lhXvmGNnUiuDyJgY9IKkNA==", + "dependencies": { + "@types/react": "*" + } + }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "peer": true + }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==", + "peer": true + }, + "node_modules/@types/triple-beam": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", + "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==" + }, + "node_modules/@types/uuid": { + "version": "9.0.8", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz", + "integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==", + "peer": true + }, + "node_modules/@types/wrap-ansi": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/wrap-ansi/-/wrap-ansi-3.0.0.tgz", + "integrity": "sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==" + }, + "node_modules/a-sync-waterfall": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/a-sync-waterfall/-/a-sync-waterfall-1.0.1.tgz", + "integrity": "sha512-RYTOHHdWipFUliRFMCS4X2Yn2X8M87V/OpSqWzKKOGhzqyUxzyVmhHDH9sAvG+ZuQf/TAOFsLCpMw09I1ufUnA==" + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/agent-base": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", + "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/agentkeepalive": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", + "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, + "node_modules/arr-union": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", + "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==" + }, + "node_modules/asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" + }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/axios": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.4.tgz", + "integrity": "sha512-DukmaFRnY6AzAALSH4J2M3k6PkaC+MfaAGdEERRWcC9q3/TWQwLpHR8ZRLKTdQ3aBDL64EdluRDjJqKw+BPZEw==", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/babel-plugin-macros": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/babel-plugin-macros/-/babel-plugin-macros-3.1.0.tgz", + "integrity": "sha512-Cg7TFGpIr01vOQNODXOOaGz2NpCU5gl8x1qJFbb6hbZxR7XrcE2vtbAsTAbJ7/xwJtUuJEw8K8Zr/AE0LHlesg==", + "dependencies": { + "@babel/runtime": "^7.12.5", + "cosmiconfig": "^7.0.0", + "resolve": "^1.19.0" + }, + "engines": { + "node": ">=10", + "npm": ">=6" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/base64id": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/base64id/-/base64id-2.0.0.tgz", + "integrity": "sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog==", + "engines": { + "node": "^4.5.0 || >= 5.9" + } + }, + "node_modules/basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/better-sqlite3": { + "version": "11.7.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.7.0.tgz", + "integrity": "sha512-mXpa5jnIKKHeoGzBrUJrc65cXFKcILGZpU3FXR0pradUEm9MA7UZz02qfEejaMcm9iXrSOCenwwYMJ/tZ1y5Ig==", + "hasInstallScript": true, + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + } + }, + "node_modules/big-integer": { + "version": "1.6.52", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", + "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", + "peer": true, + "engines": { + "node": ">=0.6" + } + }, + "node_modules/bignumber.js": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", + "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", + "engines": { + "node": "*" + } + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/body-parser": { + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.13.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/body-parser/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/bowser": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", + "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==", + "peer": true + }, + "node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/cache-manager": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-4.1.0.tgz", + "integrity": "sha512-ZGM6dLxrP65bfOZmcviWMadUOCICqpLs92+P/S5tj8onz+k+tB7Gr+SAgOUHCQtfm2gYEQDHiKeul4+tYPOJ8A==", + "dependencies": { + "async": "3.2.3", + "lodash.clonedeep": "^4.5.0", + "lru-cache": "^7.10.1" + } + }, + "node_modules/cache-manager-fs-hash": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cache-manager-fs-hash/-/cache-manager-fs-hash-1.1.0.tgz", + "integrity": "sha512-5D4Y2cnioxiy830a7QrWtRmsrfZCW1z3BOIZ0jessuFHIj/8e8mI4MsDYTaEz6aPn0EC4YAWWtQGJVsqccXW/w==", + "dependencies": { + "lockfile": "^1.0.4" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/cache-manager/node_modules/async": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.3.tgz", + "integrity": "sha512-spZRyzKL5l5BZQrr/6m/SqFdBN0q3OCI0f9rjfBzCMBIP4p75P620rR3gTmaksNOhmzgdxcaxdNfMy6anrbM0g==" + }, + "node_modules/cache-manager/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/call-bind": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", + "integrity": "sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.0", + "es-define-property": "^1.0.0", + "get-intrinsic": "^1.2.4", + "set-function-length": "^1.2.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.1.tgz", + "integrity": "sha512-BhYE+WDaywFg2TBWYNXAE+8B1ATnThNBqXHP5nQu0jWJdVvY2hvkpyB3qOmtmDePiS5/BDQ8wASEWGMWRG148g==", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/chalk/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/chalk/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/chardet": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==" + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "node_modules/cli-progress": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.12.0.tgz", + "integrity": "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==", + "dependencies": { + "string-width": "^4.2.3" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/cli-table3": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.5.tgz", + "integrity": "sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==", + "dependencies": { + "string-width": "^4.2.0" + }, + "engines": { + "node": "10.* || >= 12.*" + }, + "optionalDependencies": { + "@colors/colors": "1.5.0" + } + }, + "node_modules/cli-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", + "engines": { + "node": ">= 12" + } + }, + "node_modules/clone": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", + "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/clone-deep": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-0.2.4.tgz", + "integrity": "sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg==", + "peer": true, + "dependencies": { + "for-own": "^0.1.3", + "is-plain-object": "^2.0.1", + "kind-of": "^3.0.2", + "lazy-cache": "^1.0.3", + "shallow-clone": "^0.1.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "engines": { + "node": ">=6" + } + }, + "node_modules/color": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", + "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", + "dependencies": { + "color-convert": "^1.9.3", + "color-string": "^1.6.0" + } + }, + "node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/colorspace": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", + "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", + "dependencies": { + "color": "^3.1.3", + "text-hex": "1.0.x" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "peer": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/complex.js": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/complex.js/-/complex.js-2.4.2.tgz", + "integrity": "sha512-qtx7HRhPGSCBtGiST4/WGHuW+zeaND/6Ld+db6PbrulIB1i2Ev/2UPiqcmpQNPSyfBKraC0EOvOKCB5dGZKt3g==", + "engines": { + "node": "*" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/rawify" + } + }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression": { + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.5.tgz", + "integrity": "sha512-bQJ0YRck5ak3LgtnpKkiabX5pNF7tMUh1BSy2ZBOTh0Dim0BUu6aPPwByIns6/A5Prh8PufSPerMDUklpzes2Q==", + "dependencies": { + "bytes": "3.1.2", + "compressible": "~2.0.18", + "debug": "2.6.9", + "negotiator": "~0.6.4", + "on-headers": "~1.0.2", + "safe-buffer": "5.2.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/compression/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/compression/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "peer": true + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/convert-source-map": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz", + "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==" + }, + "node_modules/cookie": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==" + }, + "node_modules/cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/cosmiconfig": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.1.0.tgz", + "integrity": "sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==", + "dependencies": { + "@types/parse-json": "^4.0.0", + "import-fresh": "^3.2.1", + "parse-json": "^5.0.0", + "path-type": "^4.0.0", + "yaml": "^1.10.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/cosmiconfig/node_modules/yaml": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", + "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/cross-fetch": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz", + "integrity": "sha512-e4a5N8lVvuLgAWgnCrLr2PP0YyDOTHa9H/Rj54dirp61qXnNq46m82bRhNqIA5VccJtWBvPTFRV3TtvHUKPB1g==", + "peer": true, + "dependencies": { + "node-fetch": "^2.6.12" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/crypto-js": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz", + "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==" + }, + "node_modules/csstype": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", + "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==" + }, + "node_modules/csv-parse": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.6.0.tgz", + "integrity": "sha512-l3nz3euub2QMg5ouu5U09Ew9Wf6/wQ8I++ch1loQ0ljmzhmfZYrH9fflS22i/PQEvsPvxCwxgz5q7UB8K1JO4Q==" + }, + "node_modules/csv-stringify": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.2.tgz", + "integrity": "sha512-RFPahj0sXcmUyjrObAK+DOWtMvMIFV328n4qZJhgX3x2RqkQgOTU2mCUmiFR0CzM6AzChlRSUErjiJeEt8BaQA==" + }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/debounce": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/debounce/-/debounce-2.2.0.tgz", + "integrity": "sha512-Xks6RUDLZFdz8LIdR6q0MTH44k7FikOmnh5xkSjMig6ch45afc8sjTjRQf3P6ax8dMgcQrYO/AR2RGWURrruqw==", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/decimal.js": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz", + "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==" + }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/dedent": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.3.tgz", + "integrity": "sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==", + "peerDependencies": { + "babel-plugin-macros": "^3.1.0" + }, + "peerDependenciesMeta": { + "babel-plugin-macros": { + "optional": true + } + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-lazy-prop": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz", + "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==", + "peer": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/detect-libc": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "engines": { + "node": ">=8" + } + }, + "node_modules/dom-helpers": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.2.1.tgz", + "integrity": "sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==", + "dependencies": { + "@babel/runtime": "^7.8.7", + "csstype": "^3.0.2" + } + }, + "node_modules/dotenv": { + "version": "16.4.7", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz", + "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.0.tgz", + "integrity": "sha512-9+Sj30DIu+4KvHqMfLUGLFYL2PkURSYMVXJyXe92nFRvlYq5hBjLEhblKB+vkd/WVlUYMWigiY07T91Fkk0+4A==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/enabled": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/engine.io": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/engine.io/-/engine.io-6.6.2.tgz", + "integrity": "sha512-gmNvsYi9C8iErnZdVcJnvCpSKbWTt1E8+JZo8b+daLninywUWi5NQ5STSHZ9rFjFO7imNcvb8Pc5pe/wMR5xEw==", + "dependencies": { + "@types/cookie": "^0.4.1", + "@types/cors": "^2.8.12", + "@types/node": ">=10.0.0", + "accepts": "~1.3.4", + "base64id": "2.0.0", + "cookie": "~0.7.2", + "cors": "~2.8.5", + "debug": "~4.3.1", + "engine.io-parser": "~5.2.1", + "ws": "~8.17.1" + }, + "engines": { + "node": ">=10.2.0" + } + }, + "node_modules/engine.io-parser": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-5.2.3.tgz", + "integrity": "sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q==", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/engine.io/node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/engine.io/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/engine.io/node_modules/ws": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", + "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/env-paths": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-3.0.0.tgz", + "integrity": "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/esbuild": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.23.1.tgz", + "integrity": "sha512-VVNz/9Sa0bs5SELtn3f7qhJCDPCF5oMEl5cO9/SSinpE9hbPVvxbd572HH5AKiP7WD8INO53GgfDDhRjkylHEg==", + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.23.1", + "@esbuild/android-arm": "0.23.1", + "@esbuild/android-arm64": "0.23.1", + "@esbuild/android-x64": "0.23.1", + "@esbuild/darwin-arm64": "0.23.1", + "@esbuild/darwin-x64": "0.23.1", + "@esbuild/freebsd-arm64": "0.23.1", + "@esbuild/freebsd-x64": "0.23.1", + "@esbuild/linux-arm": "0.23.1", + "@esbuild/linux-arm64": "0.23.1", + "@esbuild/linux-ia32": "0.23.1", + "@esbuild/linux-loong64": "0.23.1", + "@esbuild/linux-mips64el": "0.23.1", + "@esbuild/linux-ppc64": "0.23.1", + "@esbuild/linux-riscv64": "0.23.1", + "@esbuild/linux-s390x": "0.23.1", + "@esbuild/linux-x64": "0.23.1", + "@esbuild/netbsd-x64": "0.23.1", + "@esbuild/openbsd-arm64": "0.23.1", + "@esbuild/openbsd-x64": "0.23.1", + "@esbuild/sunos-x64": "0.23.1", + "@esbuild/win32-arm64": "0.23.1", + "@esbuild/win32-ia32": "0.23.1", + "@esbuild/win32-x64": "0.23.1" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==" + }, + "node_modules/escape-latex": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/escape-latex/-/escape-latex-1.2.0.tgz", + "integrity": "sha512-nV5aVWW1K0wEiUIEdZ4erkGGH8mDxGyxSeqPzRNtWP7ataw+/olFObw7hujFWlVjNsaDFw5VZ5NzVSIqRgfTiw==" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/escodegen/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "peer": true + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/eventsource-parser": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-1.1.2.tgz", + "integrity": "sha512-v0eOBUbiaFojBu2s2NPBfYUoRR9GjcDNvCXVaqEf5vVfpIAh9f8RCo4vXTP8c63QRKCFwoLpMpTdPwwhEKVgzA==", + "peer": true, + "engines": { + "node": ">=14.18" + } + }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "engines": { + "node": ">=6" + } + }, + "node_modules/express": { + "version": "4.21.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.2.tgz", + "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.3", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.7.1", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.3.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.12", + "proxy-addr": "~2.0.7", + "qs": "6.13.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.19.0", + "serve-static": "1.16.2", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/express/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/express/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, + "node_modules/external-editor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", + "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", + "dependencies": { + "chardet": "^0.7.0", + "iconv-lite": "^0.4.24", + "tmp": "^0.0.33" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" + }, + "node_modules/fast-uri": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.0.3.tgz", + "integrity": "sha512-aLrHthzCjH5He4Z2H9YZ+v6Ujb9ocRuW6ZzkJQOrTxleEijANq4v1TsaPaVG1PZcuurEzrLcWRyYBYXD5cEiaw==" + }, + "node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "peer": true, + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/fastest-levenshtein": { + "version": "1.0.16", + "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", + "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", + "engines": { + "node": ">= 4.9.1" + } + }, + "node_modules/fecha": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", + "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==" + }, + "node_modules/fetch-retry": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/fetch-retry/-/fetch-retry-5.0.6.tgz", + "integrity": "sha512-3yurQZ2hD9VISAhJJP9bpYFNQrHHBXE2JxxjY5aLEcDi46RmAzJE2OC9FAde0yis5ElW0jTTzs0zfg/Cca4XqQ==", + "peer": true + }, + "node_modules/file-type": { + "version": "16.5.4", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-16.5.4.tgz", + "integrity": "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw==", + "peer": true, + "dependencies": { + "readable-web-to-node-stream": "^3.0.0", + "strtok3": "^6.2.4", + "token-types": "^4.1.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/file-type?sponsor=1" + } + }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/finalhandler": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.1.tgz", + "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/finalhandler/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/finalhandler/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/find-root": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/find-root/-/find-root-1.1.0.tgz", + "integrity": "sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==" + }, + "node_modules/fn.name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" + }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/for-in": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", + "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/for-own": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/for-own/-/for-own-0.1.5.tgz", + "integrity": "sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw==", + "peer": true, + "dependencies": { + "for-in": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/foreground-child": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz", + "integrity": "sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==", + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==" + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fraction.js": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-5.2.1.tgz", + "integrity": "sha512-Ah6t/7YCYjrPUFUFsOsRLMXAdnYM+aQwmojD2Ayb/Ezr82SwES0vuyQ8qZ3QO8n9j7W14VJuVZZet8U3bhSdQQ==", + "engines": { + "node": ">= 12" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/rawify" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, + "node_modules/fs-extra": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", + "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "peer": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "peer": true + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gaxios": { + "version": "6.7.1", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.7.1.tgz", + "integrity": "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", + "dependencies": { + "gaxios": "^6.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/get-intrinsic": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.5.tgz", + "integrity": "sha512-Y4+pKa7XeRUPWFNvOOYHkRYrfzW07oraURSvjDmRVOJ748OrVmeXtpE4+GCEHncjCjkTxPNRt8kEbxDhsn6VTg==", + "dependencies": { + "call-bind-apply-helpers": "^1.0.0", + "dunder-proto": "^1.0.0", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-tsconfig": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.8.1.tgz", + "integrity": "sha512-k9PN+cFBmaLWtVz29SkUoqU5O0slLuHJXt/2P+tMVFT+phsSGXGkp9t3rQIqdz0e+06EHNGs3oM6ZX1s2zHxRg==", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, + "node_modules/get-uri": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.4.tgz", + "integrity": "sha512-E1b1lFFLvLgak2whF2xDBcOy6NLVGZBqqjJjsIhvopKfWWEi64pLVTWWehV8KlLerZkfNTA95sTe2OdJKm1OzQ==", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" + }, + "node_modules/glob": { + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "engines": { + "node": ">=4" + } + }, + "node_modules/google-auth-library": { + "version": "9.15.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.0.tgz", + "integrity": "sha512-7ccSEJFDFO7exFbO6NRyC+xH8/mZ1GZGG2xxx9iHxZWcjUjJpjWxIMw3cofAKcueZ6DATiukmmprD7yavQHOyQ==", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/googleapis-common": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/googleapis-common/-/googleapis-common-7.2.0.tgz", + "integrity": "sha512-/fhDZEJZvOV3X5jmD+fKxMqma5q2Q9nZNSF3kn1F18tpxmA86BcTxAGBQdM0N89Z3bEaIs+HVznSmFJEAmMTjA==", + "dependencies": { + "extend": "^3.0.2", + "gaxios": "^6.0.3", + "google-auth-library": "^9.7.0", + "qs": "^6.7.0", + "url-template": "^2.0.8", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "peer": true + }, + "node_modules/groq-sdk": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/groq-sdk/-/groq-sdk-0.9.0.tgz", + "integrity": "sha512-Kdbl65yOwd5ga+UZd3KCULnoOlcPV7qJ+FfemivVpu3bOZeT7WJWh/ywAuivwJ7aJzcXmwf6x5t9x0MV0ZmR0A==", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + } + }, + "node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hoist-non-react-statics": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", + "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==", + "dependencies": { + "react-is": "^16.7.0" + } + }, + "node_modules/hoist-non-react-statics/node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==" + }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/http-status-codes": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/http-status-codes/-/http-status-codes-2.3.0.tgz", + "integrity": "sha512-RJ8XvFvpPM/Dmc5SV+dC4y5PCeOhT3x1Hq0NU3rjGeg5a/CqlhZ7uudknPwZFz4aeAXDcbAyaeP7GAo9lvngtA==", + "peer": true + }, + "node_modules/http-z": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/http-z/-/http-z-7.1.1.tgz", + "integrity": "sha512-Vt2tRLs8d4YqEj7p8oyXwiZGd+8UnyhwfgiLCGktDxcXZ9gLn3M5IKRC3WQacogrhLNANfg1vky+S9lV0O9uBA==", + "dependencies": { + "lodash": "^4.17.21" + }, + "engines": { + "node": ">=16", + "pnpm": ">=8" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dependencies": { + "ms": "^2.0.0" + } + }, + "node_modules/ibm-cloud-sdk-core": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/ibm-cloud-sdk-core/-/ibm-cloud-sdk-core-5.1.0.tgz", + "integrity": "sha512-KJCbPz3tiXB1NGAD7cL4JtwpWV8yd/C7jsaHsxvedMo2ZblNG8emMyvSpGhiKAQVZmi3c0ujz6eJdy22NHuUWQ==", + "peer": true, + "dependencies": { + "@types/debug": "^4.1.12", + "@types/node": "~10.14.19", + "@types/tough-cookie": "^4.0.0", + "axios": "1.7.4", + "camelcase": "^6.3.0", + "debug": "^4.3.4", + "dotenv": "^16.4.5", + "extend": "3.0.2", + "file-type": "16.5.4", + "form-data": "4.0.0", + "isstream": "0.1.2", + "jsonwebtoken": "^9.0.2", + "mime-types": "2.1.35", + "retry-axios": "^2.6.0", + "tough-cookie": "^4.1.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/ibm-cloud-sdk-core/node_modules/@types/node": { + "version": "10.14.22", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.22.tgz", + "integrity": "sha512-9taxKC944BqoTVjE+UT3pQH0nHZlTvITwfsOZqyc+R3sfJuxaTtxWjfn1K2UlxyPcKHf0rnaXcVFrS9F9vf0bw==", + "peer": true + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/import-fresh": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "peer": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" + }, + "node_modules/inquirer": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-11.1.0.tgz", + "integrity": "sha512-CmLAZT65GG/v30c+D2Fk8+ceP6pxD6RL+hIUOWAltCmeyEqWYwqu9v76q03OvjyZ3AB0C1Ala2stn1z/rMqGEw==", + "dependencies": { + "@inquirer/core": "^9.2.1", + "@inquirer/prompts": "^6.0.1", + "@inquirer/type": "^2.0.0", + "@types/mute-stream": "^0.0.4", + "ansi-escapes": "^4.3.2", + "mute-stream": "^1.0.0", + "run-async": "^3.0.0", + "rxjs": "^7.8.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==" + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "peer": true + }, + "node_modules/is-core-module": { + "version": "2.15.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.15.1.tgz", + "integrity": "sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ==", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "peer": true, + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "peer": true, + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "peer": true, + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==", + "peer": true + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/javascript-natural-sort": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/javascript-natural-sort/-/javascript-natural-sort-0.7.1.tgz", + "integrity": "sha512-nO6jcEfZWQXDhOiBtG2KvKyEptz7RVbpGP4vTD2hLBdmNQSsCiicO2Ioinv6UI4y9ukqnBpy+XZ9H6uLNgJTlw==" + }, + "node_modules/js-base64": { + "version": "3.7.7", + "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.7.tgz", + "integrity": "sha512-7rCnleh0z2CkXhH67J8K1Ytz0b2Y+yxTPL+/KOJoa20hfnVQ/3/T6W/KflYI4bRHRagNeXeU2bkNGI3v1oS/lw==", + "peer": true + }, + "node_modules/js-rouge": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/js-rouge/-/js-rouge-3.0.0.tgz", + "integrity": "sha512-+rXcOCf53xU0uR8y75OQVHWWQaNFGWa+3g7OjnP9/Pau4EyZ56lpAO2GpVi4pjMkMmNKHwcxkiHNqCkMZK3T7A==", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/js-tiktoken": { + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.15.tgz", + "integrity": "sha512-65ruOWWXDEZHHbAo7EjOcNxOGasQKbL4Fq3jEr2xsCqSsoOo6VVSqzWQb6PRIqypFSDcma4jO90YP0w5X8qVXQ==", + "peer": true, + "dependencies": { + "base64-js": "^1.5.1" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==" + }, + "node_modules/jsesc": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz", + "integrity": "sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==" + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, + "node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "peer": true, + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/jsonschema": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsonschema/-/jsonschema-1.4.1.tgz", + "integrity": "sha512-S6cATIPVv1z0IlxdN+zUk5EPjkGCdnhN4wVSBlvoUO1tOLJootbo9CquNJmbIh4yikWHiUedhRYrNPn1arpEmQ==", + "engines": { + "node": "*" + } + }, + "node_modules/jsonwebtoken": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz", + "integrity": "sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==", + "peer": true, + "dependencies": { + "jws": "^3.2.2", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=12", + "npm": ">=6" + } + }, + "node_modules/jsonwebtoken/node_modules/jwa": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", + "peer": true, + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jsonwebtoken/node_modules/jws": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "peer": true, + "dependencies": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "peer": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/kuler": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" + }, + "node_modules/langfuse": { + "version": "3.31.1", + "resolved": "https://registry.npmjs.org/langfuse/-/langfuse-3.31.1.tgz", + "integrity": "sha512-V74AWfB3aLz2Z0TiuIgXK7uG5eMYE20qfp5zYyKAS4DvQSAiSqC9Sgg3xm6OG8Jc1pK/X7s5WeLvswWn7PwfOA==", + "peer": true, + "dependencies": { + "langfuse-core": "^3.31.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/langfuse-core": { + "version": "3.31.1", + "resolved": "https://registry.npmjs.org/langfuse-core/-/langfuse-core-3.31.1.tgz", + "integrity": "sha512-mLFya8KJhCeqBlUy5iNzHoorOTb05Y/Oa9lfcI7PBH+RBmtLB7+QyYyTaP7EmVRkK21uRhn0U/BU0oN8UKxdYQ==", + "peer": true, + "dependencies": { + "mustache": "^4.2.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/langsmith": { + "version": "0.2.11", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.2.11.tgz", + "integrity": "sha512-rVPUN/jQEHjTuYaoVKGjfb3NsYNLGTQT9LXcgJvka5M0EDcXciC598A+DsAQrl6McdfSJCFJDelgRPqVoF2xNA==", + "peer": true, + "dependencies": { + "@types/uuid": "^10.0.0", + "commander": "^10.0.1", + "p-queue": "^6.6.2", + "p-retry": "4", + "semver": "^7.6.3", + "uuid": "^10.0.0" + }, + "peerDependencies": { + "openai": "*" + }, + "peerDependenciesMeta": { + "openai": { + "optional": true + } + } + }, + "node_modules/langsmith/node_modules/@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", + "peer": true + }, + "node_modules/langsmith/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "peer": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/lazy-cache": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", + "integrity": "sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ==", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==" + }, + "node_modules/lockfile": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/lockfile/-/lockfile-1.0.4.tgz", + "integrity": "sha512-cvbTwETRfsFh4nHsL1eGWapU1XFi5Ot9E85sWAwia7Y7EgB7vfqcZhTKZ+l7hCGxSPoushMv5GKhT5PdLv03WA==", + "dependencies": { + "signal-exit": "^3.0.2" + } + }, + "node_modules/lockfile/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "node_modules/lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==" + }, + "node_modules/lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", + "peer": true + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", + "peer": true + }, + "node_modules/lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", + "peer": true + }, + "node_modules/lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==", + "peer": true + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", + "peer": true + }, + "node_modules/lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", + "peer": true + }, + "node_modules/lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", + "peer": true + }, + "node_modules/logform": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.7.0.tgz", + "integrity": "sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ==", + "dependencies": { + "@colors/colors": "1.6.0", + "@types/triple-beam": "^1.3.2", + "fecha": "^4.2.0", + "ms": "^2.1.1", + "safe-stable-stringify": "^2.3.1", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/logform/node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lru-cache": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.0.2.tgz", + "integrity": "sha512-123qHRfJBmo2jXDbo/a5YOQrJoHF/GNQTLzQ5+IdK5pWpceK17yRc6ozlWd25FxvGKQbIUs91fDFkXmDHTKcyA==", + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/mathjs": { + "version": "14.0.0", + "resolved": "https://registry.npmjs.org/mathjs/-/mathjs-14.0.0.tgz", + "integrity": "sha512-MR3me92c6pKBqzUXosqL5KMIZDrb1x0MGOy+Ss6fQllD1zhAFloG6DJnG6X5b0VYAMA9sgGfAR2tYi5HPNNQBQ==", + "dependencies": { + "@babel/runtime": "^7.25.7", + "complex.js": "^2.2.5", + "decimal.js": "^10.4.3", + "escape-latex": "^1.2.0", + "fraction.js": "^5.2.1", + "javascript-natural-sort": "^0.7.1", + "seedrandom": "^3.0.5", + "tiny-emitter": "^2.1.0", + "typed-function": "^4.2.1" + }, + "bin": { + "mathjs": "bin/cli.js" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-deep": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.3.tgz", + "integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==", + "peer": true, + "dependencies": { + "arr-union": "^3.1.0", + "clone-deep": "^0.2.4", + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/mixin-object": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mixin-object/-/mixin-object-2.0.1.tgz", + "integrity": "sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA==", + "peer": true, + "dependencies": { + "for-in": "^0.1.3", + "is-extendable": "^0.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mixin-object/node_modules/for-in": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-0.1.8.tgz", + "integrity": "sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g==", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "peer": true, + "bin": { + "mustache": "bin/mustache" + } + }, + "node_modules/mute-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-1.0.0.tgz", + "integrity": "sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" + }, + "node_modules/negotiator": { + "version": "0.6.4", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.4.tgz", + "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/node-abi": { + "version": "3.71.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.71.0.tgz", + "integrity": "sha512-SZ40vRiy/+wRTf21hxkkEjPJZpARzUMVcJoQse2EF8qkUWbbO2z7vd5oA/H6bVH6SZQ5STGcu0KRDS7biNRfxw==", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-cache": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz", + "integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==", + "dependencies": { + "clone": "2.x" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-ensure": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/node-ensure/-/node-ensure-0.0.0.tgz", + "integrity": "sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw==", + "peer": true + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-sql-parser": { + "version": "5.3.5", + "resolved": "https://registry.npmjs.org/node-sql-parser/-/node-sql-parser-5.3.5.tgz", + "integrity": "sha512-DOR2PAe3gY5/fXmSvpTOpyv8+HkQ5jA9KShAEy7fVLaV8kVbeg8aa7qo0MdfqCDskwqjcakp0lS/Rnmg+iY/XQ==", + "peer": true, + "dependencies": { + "@types/pegjs": "^0.10.0", + "big-integer": "^1.6.48" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/nunjucks": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/nunjucks/-/nunjucks-3.2.4.tgz", + "integrity": "sha512-26XRV6BhkgK0VOxfbU5cQI+ICFUtMLixv1noZn1tGU38kQH5A5nmmbk/O45xdyBhD1esk47nKrY0mvQpZIhRjQ==", + "dependencies": { + "a-sync-waterfall": "^1.0.0", + "asap": "^2.0.3", + "commander": "^5.1.0" + }, + "bin": { + "nunjucks-precompile": "bin/precompile" + }, + "engines": { + "node": ">= 6.9.0" + }, + "peerDependencies": { + "chokidar": "^3.3.0" + }, + "peerDependenciesMeta": { + "chokidar": { + "optional": true + } + } + }, + "node_modules/nunjucks/node_modules/commander": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", + "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.3", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.3.tgz", + "integrity": "sha512-kDCGIbxkDSXE3euJZZXzc6to7fCrKHNI/hSRQnRuQ+BWjFNzZwiFF8fj/6o2t2G9/jTj8PSIYTfCLelLZEeRpA==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/on-headers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", + "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/one-time": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", + "dependencies": { + "fn.name": "1.x.x" + } + }, + "node_modules/open": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", + "integrity": "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==", + "peer": true, + "dependencies": { + "define-lazy-prop": "^2.0.0", + "is-docker": "^2.1.1", + "is-wsl": "^2.2.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/openai": { + "version": "4.76.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.76.0.tgz", + "integrity": "sha512-QBGIetjX1C9xDp5XGa/3mPnfKI9BgAe2xHQX6PmO98wuW9qQaurBaumcYptQWc9LHZZq7cH/Y1Rjnsr6uUDdVw==", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + }, + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/openapi-fetch": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.8.2.tgz", + "integrity": "sha512-4g+NLK8FmQ51RW6zLcCBOVy/lwYmFJiiT+ckYZxJWxUxH4XFhsNcX2eeqVMfVOi+mDNFja6qDXIZAz2c5J/RVw==", + "peer": true, + "dependencies": { + "openapi-typescript-helpers": "^0.0.5" + } + }, + "node_modules/openapi-typescript-helpers": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/openapi-typescript-helpers/-/openapi-typescript-helpers-0.0.5.tgz", + "integrity": "sha512-MRffg93t0hgGZbYTxg60hkRIK2sRuEOHEtCUgMuLgbCC33TMQ68AmxskzUlauzZYD47+ENeGV/ElI7qnWqrAxA==", + "peer": true + }, + "node_modules/opener": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz", + "integrity": "sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==", + "bin": { + "opener": "bin/opener-bin.js" + } + }, + "node_modules/os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "peer": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "peer": true, + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-queue-compat": { + "version": "1.0.227", + "resolved": "https://registry.npmjs.org/p-queue-compat/-/p-queue-compat-1.0.227.tgz", + "integrity": "sha512-uNKILAU2Wg6oj7hlvcDJgY4Xl1qzqqOEv/Mq8YtA0ApQQxJDa6DQoRElCPMa55mQ/A3jWeT6qSmcwzNI1cFSrA==", + "peer": true, + "dependencies": { + "eventemitter3": "5.x", + "p-timeout-compat": "^1.0.3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/p-queue-compat/node_modules/eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==", + "peer": true + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "peer": true, + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "peer": true, + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-timeout-compat": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/p-timeout-compat/-/p-timeout-compat-1.0.5.tgz", + "integrity": "sha512-JUyLagABYj4bu58nXmMEO+JpHlimYomLY3xdatcPWKRRVVQ3ky55Ki4Mdj2OUGDfkV2BPt87zqSzISk0ztwruw==", + "peer": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/pac-proxy-agent": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.1.0.tgz", + "integrity": "sha512-Z5FnLVVZSnX7WjBg0mhDtydeRZ1xMcATZThjySQUHqr+0ksP8kqaw23fNKkaaN/Z8gwLUs/W7xdl0I75eP2Xyw==", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==" + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" + }, + "node_modules/path-to-regexp": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==" + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "engines": { + "node": ">=8" + } + }, + "node_modules/pdf-parse": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pdf-parse/-/pdf-parse-1.1.1.tgz", + "integrity": "sha512-v6ZJ/efsBpGrGGknjtq9J/oC8tZWq0KWL5vQrk2GlzLEQPUDB1ex+13Rmidl1neNN358Jn9EHZw5y07FFtaC7A==", + "peer": true, + "dependencies": { + "debug": "^3.1.0", + "node-ensure": "^0.0.0" + }, + "engines": { + "node": ">=6.8.1" + } + }, + "node_modules/pdf-parse/node_modules/debug": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", + "integrity": "sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==", + "peer": true, + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/peek-readable": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz", + "integrity": "sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==", + "peer": true, + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/playwright": { + "version": "1.49.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.49.1.tgz", + "integrity": "sha512-VYL8zLoNTBxVOrJBbDuRgDWa3i+mfQgDTrL8Ah9QXZ7ax4Dsj0MSq5bYgytRnDVVe+njoKnfsYkH3HzqVj5UZA==", + "peer": true, + "dependencies": { + "playwright-core": "1.49.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.49.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.49.1.tgz", + "integrity": "sha512-BzmpVcs4kE2CH15rWfzpjzVGhWERJfmnXmniSyKeRZUs9Ws65m+RGIi7mjJK/euCegfn3i7jvqWeWyHe9y3Vgg==", + "peer": true, + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright-extra": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/playwright-extra/-/playwright-extra-4.3.6.tgz", + "integrity": "sha512-q2rVtcE8V8K3vPVF1zny4pvwZveHLH8KBuVU2MoE3Jw4OKVoBWsHI9CH9zPydovHHOCDxjGN2Vg+2m644q3ijA==", + "peer": true, + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "playwright": "*", + "playwright-core": "*" + }, + "peerDependenciesMeta": { + "playwright": { + "optional": true + }, + "playwright-core": { + "optional": true + } + } + }, + "node_modules/prebuild-install": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.2.tgz", + "integrity": "sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ==", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "optional": true, + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/promptfoo": { + "version": "0.100.5", + "resolved": "https://registry.npmjs.org/promptfoo/-/promptfoo-0.100.5.tgz", + "integrity": "sha512-2c6Q84hsa1FY6eOXIrOdv9dm9fKaBXn3tUEnBar2sLt6YghQJt9aWfloBiZjgobVyDPA1x2qPFU5i8C8YB0h7g==", + "dependencies": { + "@adaline/anthropic": "0.19.0", + "@adaline/azure": "0.9.1", + "@adaline/gateway": "0.23.0", + "@adaline/google": "0.8.0", + "@adaline/groq": "0.8.1", + "@adaline/open-router": "0.8.0", + "@adaline/openai": "0.21.0", + "@adaline/provider": "0.17.0", + "@adaline/together-ai": "0.8.0", + "@adaline/types": "0.15.0", + "@adaline/vertex": "0.8.1", + "@anthropic-ai/sdk": "^0.32.1", + "@apidevtools/json-schema-ref-parser": "^11.7.2", + "@emotion/react": "^11.13.5", + "@emotion/styled": "^11.13.5", + "@googleapis/sheets": "^9.3.1", + "@mui/material": "^6.1.10", + "ajv": "^8.17.1", + "ajv-formats": "^3.0.1", + "async": "^3.2.6", + "better-sqlite3": "^11.6.0", + "cache-manager": "^4.1.0", + "cache-manager-fs-hash": "^1.0.0", + "chalk": "^4.1.2", + "chokidar": "^3.6.0", + "cli-progress": "^3.12.0", + "cli-table3": "^0.6.5", + "commander": "^12.1.0", + "compression": "^1.7.5", + "cors": "^2.8.5", + "csv-parse": "^5.6.0", + "csv-stringify": "^6.5.2", + "debounce": "^2.2.0", + "dedent": "^1.5.3", + "dotenv": "^16.4.7", + "drizzle-orm": "^0.35.2", + "express": "^4.21.1", + "fast-deep-equal": "^3.1.3", + "fast-xml-parser": "^4.5.0", + "fastest-levenshtein": "^1.0.16", + "glob": "^10.4.3", + "groq-sdk": "^0.9.0", + "http-z": "^7.1.1", + "inquirer": "^11.1.0", + "js-rouge": "^3.0.0", + "js-yaml": "^4.1.0", + "mathjs": "^14.0.0", + "node-cache": "^5.1.2", + "nunjucks": "^3.2.4", + "openai": "^4.75.0", + "opener": "^1.5.2", + "proxy-agent": "^6.4.0", + "python-shell": "^5.0.0", + "replicate": "^0.34.1", + "rfdc": "^1.4.1", + "semver": "^7.6.3", + "socket.io": "^4.8.1", + "tiny-invariant": "^1.3.3", + "tsx": "^4.19.2", + "uuid": "^10.0.0", + "winston": "^3.17.0", + "ws": "^8.18.0", + "zod": "^3.23.8", + "zod-validation-error": "^3.4.0" + }, + "bin": { + "pf": "dist/src/main.js", + "promptfoo": "dist/src/main.js" + }, + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-bedrock-runtime": "^3.602.0", + "@aws-sdk/credential-provider-sso": "^3.686.0", + "@azure/identity": "^4.0.0", + "@azure/openai-assistants": "^1.0.0-beta.5", + "@fal-ai/serverless-client": "^0.14.3", + "@ibm-cloud/watsonx-ai": "^1.1.0", + "@ibm-generative-ai/node-sdk": "^2.0.6", + "@playwright/browser-chromium": "^1.47.2", + "@smithy/node-http-handler": "^3.1.1", + "google-auth-library": "^9.7.0", + "ibm-cloud-sdk-core": "^5.0.2", + "langfuse": "^3.7.0", + "node-sql-parser": "^5.2.0", + "pdf-parse": "^1.1.1", + "playwright": "^1.47.2", + "playwright-extra": "^4.3.6", + "puppeteer-extra-plugin-stealth": "^2.11.2" + } + }, + "node_modules/promptfoo/node_modules/commander": { + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-12.1.0.tgz", + "integrity": "sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==", + "engines": { + "node": ">=18" + } + }, + "node_modules/promptfoo/node_modules/drizzle-orm": { + "version": "0.35.3", + "resolved": "https://registry.npmjs.org/drizzle-orm/-/drizzle-orm-0.35.3.tgz", + "integrity": "sha512-Uv6N+b36x4BaZlxc96e+ag7RnMapBLGhc4SSi2F7RDwqYJipWjaU/P68RUp1FbW9r+mxoDp8nMz2Eece8PJxfA==", + "peerDependencies": { + "@aws-sdk/client-rds-data": ">=3", + "@cloudflare/workers-types": ">=3", + "@electric-sql/pglite": ">=0.1.1", + "@libsql/client": ">=0.10.0", + "@libsql/client-wasm": ">=0.10.0", + "@neondatabase/serverless": ">=0.1", + "@op-engineering/op-sqlite": ">=2", + "@opentelemetry/api": "^1.4.1", + "@planetscale/database": ">=1", + "@prisma/client": "*", + "@tidbcloud/serverless": "*", + "@types/better-sqlite3": "*", + "@types/pg": "*", + "@types/react": ">=18", + "@types/sql.js": "*", + "@vercel/postgres": ">=0.8.0", + "@xata.io/client": "*", + "better-sqlite3": ">=7", + "bun-types": "*", + "expo-sqlite": ">=13.2.0", + "knex": "*", + "kysely": "*", + "mysql2": ">=2", + "pg": ">=8", + "postgres": ">=3", + "react": ">=18", + "sql.js": ">=1", + "sqlite3": ">=5" + }, + "peerDependenciesMeta": { + "@aws-sdk/client-rds-data": { + "optional": true + }, + "@cloudflare/workers-types": { + "optional": true + }, + "@electric-sql/pglite": { + "optional": true + }, + "@libsql/client": { + "optional": true + }, + "@neondatabase/serverless": { + "optional": true + }, + "@op-engineering/op-sqlite": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@planetscale/database": { + "optional": true + }, + "@prisma/client": { + "optional": true + }, + "@tidbcloud/serverless": { + "optional": true + }, + "@types/better-sqlite3": { + "optional": true + }, + "@types/pg": { + "optional": true + }, + "@types/react": { + "optional": true + }, + "@types/sql.js": { + "optional": true + }, + "@vercel/postgres": { + "optional": true + }, + "@xata.io/client": { + "optional": true + }, + "better-sqlite3": { + "optional": true + }, + "bun-types": { + "optional": true + }, + "expo-sqlite": { + "optional": true + }, + "knex": { + "optional": true + }, + "kysely": { + "optional": true + }, + "mysql2": { + "optional": true + }, + "pg": { + "optional": true + }, + "postgres": { + "optional": true + }, + "prisma": { + "optional": true + }, + "react": { + "optional": true + }, + "sql.js": { + "optional": true + }, + "sqlite3": { + "optional": true + } + } + }, + "node_modules/promptfoo/node_modules/fast-xml-parser": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz", + "integrity": "sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/promptfoo/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "dependencies": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + } + }, + "node_modules/prop-types/node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/proxy-agent": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", + "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.6", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.1.0", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, + "node_modules/psl": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.15.0.tgz", + "integrity": "sha512-JZd3gMVBAVQkSs6HdNZo9Sdo0LNcQeMNP3CozBJb3JYC/QUYZTnKxP+f8oWRX4rHP5EurWxqAHTSwUCjlNKa1w==", + "peer": true, + "dependencies": { + "punycode": "^2.3.1" + }, + "funding": { + "url": "https://github.com/sponsors/lupomontero" + } + }, + "node_modules/pump": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "peer": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/puppeteer-extra-plugin": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.3.tgz", + "integrity": "sha512-6RNy0e6pH8vaS3akPIKGg28xcryKscczt4wIl0ePciZENGE2yoaQJNd17UiEbdmh5/6WW6dPcfRWT9lxBwCi2Q==", + "peer": true, + "dependencies": { + "@types/debug": "^4.1.0", + "debug": "^4.1.1", + "merge-deep": "^3.0.1" + }, + "engines": { + "node": ">=9.11.2" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin-stealth": { + "version": "2.11.2", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.2.tgz", + "integrity": "sha512-bUemM5XmTj9i2ZerBzsk2AN5is0wHMNE6K0hXBzBXOzP5m5G3Wl0RHhiqKeHToe/uIH8AoZiGhc1tCkLZQPKTQ==", + "peer": true, + "dependencies": { + "debug": "^4.1.1", + "puppeteer-extra-plugin": "^3.2.3", + "puppeteer-extra-plugin-user-preferences": "^2.4.1" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin-user-data-dir": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.4.1.tgz", + "integrity": "sha512-kH1GnCcqEDoBXO7epAse4TBPJh9tEpVEK/vkedKfjOVOhZAvLkHGc9swMs5ChrJbRnf8Hdpug6TJlEuimXNQ+g==", + "peer": true, + "dependencies": { + "debug": "^4.1.1", + "fs-extra": "^10.0.0", + "puppeteer-extra-plugin": "^3.2.3", + "rimraf": "^3.0.2" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin-user-preferences": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.4.1.tgz", + "integrity": "sha512-i1oAZxRbc1bk8MZufKCruCEC3CCafO9RKMkkodZltI4OqibLFXF3tj6HZ4LZ9C5vCXZjYcDWazgtY69mnmrQ9A==", + "peer": true, + "dependencies": { + "debug": "^4.1.1", + "deepmerge": "^4.2.2", + "puppeteer-extra-plugin": "^3.2.3", + "puppeteer-extra-plugin-user-data-dir": "^2.4.1" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/python-shell": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/python-shell/-/python-shell-5.0.0.tgz", + "integrity": "sha512-RUOOOjHLhgR1MIQrCtnEqz/HJ1RMZBIN+REnpSUrfft2bXqXy69fwJASVziWExfFXsR1bCY0TznnHooNsCo0/w==", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/qs": { + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "dependencies": { + "side-channel": "^1.0.6" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==", + "peer": true + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/react": { + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.0.0.tgz", + "integrity": "sha512-V8AVnmPIICiWpGfm6GLzCR/W5FXLchHop40W4nXBmdlEceh16rCN8O8LNWm5bh5XUX91fh7KpA+W0TgMKmgTpQ==", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.0.0.tgz", + "integrity": "sha512-4GV5sHFG0e/0AD4X+ySy6UJd3jVl1iNsNHdpad0qhABJ11twS3TTBnseqsKurKcsNqCEFeGL3uLpVChpIO3QfQ==", + "peer": true, + "dependencies": { + "scheduler": "^0.25.0" + }, + "peerDependencies": { + "react": "^19.0.0" + } + }, + "node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==" + }, + "node_modules/react-transition-group": { + "version": "4.4.5", + "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz", + "integrity": "sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==", + "dependencies": { + "@babel/runtime": "^7.5.5", + "dom-helpers": "^5.0.1", + "loose-envify": "^1.4.0", + "prop-types": "^15.6.2" + }, + "peerDependencies": { + "react": ">=16.6.0", + "react-dom": ">=16.6.0" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readable-web-to-node-stream": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/readable-web-to-node-stream/-/readable-web-to-node-stream-3.0.2.tgz", + "integrity": "sha512-ePeK6cc1EcKLEhJFt/AebMCLL+GgSKhuygrZ/GLaKZYEecIgIECf4UaUuaByiGtzckwR4ain9VzUh95T1exYGw==", + "peer": true, + "dependencies": { + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" + }, + "node_modules/replicate": { + "version": "0.34.1", + "resolved": "https://registry.npmjs.org/replicate/-/replicate-0.34.1.tgz", + "integrity": "sha512-kQ5ULqowkZsx34WdUhlAtp9IcpalIfkaSRrFPUGP3gEpXouCxGsjXpn57e3Ic7K3mNw74cLkIrtAgcrlP+pzvg==", + "engines": { + "git": ">=2.11.0", + "node": ">=18.0.0", + "npm": ">=7.19.0", + "yarn": ">=1.7.0" + }, + "optionalDependencies": { + "readable-stream": ">=4.0.0" + } + }, + "node_modules/replicate/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "optional": true, + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/replicate/node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "optional": true, + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "peer": true + }, + "node_modules/resolve": { + "version": "1.22.8", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", + "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", + "dependencies": { + "is-core-module": "^2.13.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "engines": { + "node": ">=4" + } + }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "peer": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/retry-axios": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/retry-axios/-/retry-axios-2.6.0.tgz", + "integrity": "sha512-pOLi+Gdll3JekwuFjXO3fTq+L9lzMQGcSq7M5gIjExcl3Gu1hd4XXuf5o3+LuSBsaULQH7DiNbsqPd1chVpQGQ==", + "peer": true, + "engines": { + "node": ">=10.7.0" + }, + "peerDependencies": { + "axios": "*" + } + }, + "node_modules/rfdc": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.4.1.tgz", + "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==" + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "peer": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rimraf/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "peer": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/rimraf/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "peer": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rimraf/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "peer": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/robot3": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/robot3/-/robot3-0.4.1.tgz", + "integrity": "sha512-hzjy826lrxzx8eRgv80idkf8ua1JAepRc9Efdtj03N3KNJuznQCPlyCJ7gnUmDFwZCLQjxy567mQVKmdv2BsXQ==", + "peer": true + }, + "node_modules/run-async": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-3.0.0.tgz", + "integrity": "sha512-540WwVDOMxA6dN6We19EcT9sc3hkXPw5mzRNGM3FkdN/vtE9NFvj5lFAPNwUDmJjXidm3v7TC1cTE7t17Ulm1Q==", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/rxjs": { + "version": "7.8.1", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.1.tgz", + "integrity": "sha512-AA3TVj+0A2iuIoQkWEK/tqFjBq2j+6PO6Y0zJcvzLAFhEFIO3HL0vls9hWLncZbAAbK0mar7oZ4V079I/qPMxg==", + "dependencies": { + "tslib": "^2.1.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "engines": { + "node": ">=10" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "node_modules/scheduler": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.25.0.tgz", + "integrity": "sha512-xFVuu11jh+xcO7JOAGJNOXld8/TcEHK/4CituBUeUb5hqxJLj9YuemAEuvm9gQ/+pgXYfbQuqAkiYu+u7YEsNA==", + "peer": true + }, + "node_modules/seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" + }, + "node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/send/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/send/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/serve-static": { + "version": "1.16.2", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.19.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/set-function-length": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==", + "dependencies": { + "define-data-property": "^1.1.4", + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.4", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "node_modules/shallow-clone": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-0.1.2.tgz", + "integrity": "sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw==", + "peer": true, + "dependencies": { + "is-extendable": "^0.1.1", + "kind-of": "^2.0.1", + "lazy-cache": "^0.2.3", + "mixin-object": "^2.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shallow-clone/node_modules/kind-of": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-2.0.1.tgz", + "integrity": "sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg==", + "peer": true, + "dependencies": { + "is-buffer": "^1.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shallow-clone/node_modules/lazy-cache": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-0.2.7.tgz", + "integrity": "sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ==", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "engines": { + "node": ">=8" + } + }, + "node_modules/side-channel": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", + "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", + "dependencies": { + "call-bind": "^1.0.7", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.4", + "object-inspect": "^1.13.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/simple-swizzle/node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socket.io": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/socket.io/-/socket.io-4.8.1.tgz", + "integrity": "sha512-oZ7iUCxph8WYRHHcjBEc9unw3adt5CmSNlppj/5Q4k2RIrhl8Z5yY2Xr4j9zj0+wzVZ0bxmYoGSzKJnRl6A4yg==", + "dependencies": { + "accepts": "~1.3.4", + "base64id": "~2.0.0", + "cors": "~2.8.5", + "debug": "~4.3.2", + "engine.io": "~6.6.0", + "socket.io-adapter": "~2.5.2", + "socket.io-parser": "~4.2.4" + }, + "engines": { + "node": ">=10.2.0" + } + }, + "node_modules/socket.io-adapter": { + "version": "2.5.5", + "resolved": "https://registry.npmjs.org/socket.io-adapter/-/socket.io-adapter-2.5.5.tgz", + "integrity": "sha512-eLDQas5dzPgOWCk9GuuJC2lBqItuhKI4uxGgo9aIV7MYbk2h9Q6uULEh8WBzThoI7l+qU9Ast9fVUmkqPP9wYg==", + "dependencies": { + "debug": "~4.3.4", + "ws": "~8.17.1" + } + }, + "node_modules/socket.io-adapter/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/socket.io-adapter/node_modules/ws": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", + "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/socket.io-parser": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-4.2.4.tgz", + "integrity": "sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==", + "dependencies": { + "@socket.io/component-emitter": "~3.1.0", + "debug": "~4.3.1" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/socket.io-parser/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/socket.io/node_modules/debug": { + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz", + "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/socks": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", + "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", + "dependencies": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==" + }, + "node_modules/stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "engines": { + "node": "*" + } + }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/stoppable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz", + "integrity": "sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==", + "peer": true, + "engines": { + "node": ">=4", + "npm": ">=6" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, + "node_modules/strtok3": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-6.3.0.tgz", + "integrity": "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw==", + "peer": true, + "dependencies": { + "@tokenizer/token": "^0.3.0", + "peek-readable": "^4.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/stylis": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.2.0.tgz", + "integrity": "sha512-Orov6g6BB1sDfYgzWfTHDOxamtX1bE/zo104Dh9e6fqJ3PooipYyfJ0pUmrZO2wAvO8YbEyeFrkV91XTsGMSrw==" + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" + }, + "node_modules/tiny-emitter": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-2.1.0.tgz", + "integrity": "sha512-NB6Dk1A9xgQPMoGqC5CVXn123gWyte215ONT5Pp5a0yt4nlEoO1ZWeCwpncaekPHXO60i47ihFnZPiRPjRMq4Q==" + }, + "node_modules/tiny-invariant": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", + "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==" + }, + "node_modules/tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "dependencies": { + "os-tmpdir": "~1.0.2" + }, + "engines": { + "node": ">=0.6.0" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/token-types": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/token-types/-/token-types-4.2.1.tgz", + "integrity": "sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==", + "peer": true, + "dependencies": { + "@tokenizer/token": "^0.3.0", + "ieee754": "^1.2.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Borewit" + } + }, + "node_modules/tough-cookie": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.4.tgz", + "integrity": "sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==", + "peer": true, + "dependencies": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.2.0", + "url-parse": "^1.5.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tough-cookie/node_modules/universalify": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", + "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", + "peer": true, + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "node_modules/triple-beam": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", + "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" + }, + "node_modules/tsx": { + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.19.2.tgz", + "integrity": "sha512-pOUl6Vo2LUq/bSa8S5q7b91cgNSjctn9ugq/+Mvow99qW6x/UZYwzxy/3NmqoT66eHYfCVvFvACC58UBPFf28g==", + "dependencies": { + "esbuild": "~0.23.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/typed-function": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/typed-function/-/typed-function-4.2.1.tgz", + "integrity": "sha512-EGjWssW7Tsk4DGfE+5yluuljS1OGYWiI1J6e8puZz9nTMM51Oug8CD5Zo4gWMsOhq5BI+1bF+rWTm4Vbj3ivRA==", + "engines": { + "node": ">= 18" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "peer": true, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "peer": true, + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, + "node_modules/url-template": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", + "integrity": "sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==" + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/winston": { + "version": "3.17.0", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.17.0.tgz", + "integrity": "sha512-DLiFIXYC5fMPxaRg832S6F5mJYvePtmO5G9v9IgUFPhXm9/GkXarH/TUrBAVzhTCzAj9anE/+GjrgXp/54nOgw==", + "dependencies": { + "@colors/colors": "^1.6.0", + "@dabh/diagnostics": "^2.0.2", + "async": "^3.2.3", + "is-stream": "^2.0.0", + "logform": "^2.7.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "safe-stable-stringify": "^2.3.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.9.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-transport": { + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.9.0.tgz", + "integrity": "sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A==", + "dependencies": { + "logform": "^2.7.0", + "readable-stream": "^3.6.2", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston/node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/wrap-ansi/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "node_modules/ws": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/yaml": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.6.1.tgz", + "integrity": "sha512-7r0XPzioN/Q9kXBro/XPnA6kznR73DHq+GXh5ON7ZozRO6aMjbmiBuKste2wslTFkC5d1dw0GooOCepZXJ2SAg==", + "peer": true, + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/yoctocolors-cjs": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yoctocolors-cjs/-/yoctocolors-cjs-2.1.2.tgz", + "integrity": "sha512-cYVsTjKl8b+FrnidjibDWskAv7UKOfcwaVZdp/it9n1s9fU3IkgDbhdIRKCW4JDsAlECJY0ytoVPT3sK6kideA==", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "3.24.0", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.0.tgz", + "integrity": "sha512-Hz+wiY8yD0VLA2k/+nsg2Abez674dDGTai33SwNvMPuf9uIrBC9eFgIMQxBBbHFxVXi8W+5nX9DcAh9YNSQm/w==", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.23.5", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.23.5.tgz", + "integrity": "sha512-5wlSS0bXfF/BrL4jPAbz9da5hDlDptdEppYfe+x4eIJ7jioqKG9uUxOwPzqof09u/XeVdrgFu29lZi+8XNDJtA==", + "peer": true, + "peerDependencies": { + "zod": "^3.23.3" + } + }, + "node_modules/zod-validation-error": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/zod-validation-error/-/zod-validation-error-3.4.0.tgz", + "integrity": "sha512-ZOPR9SVY6Pb2qqO5XHt+MkkTRxGXb4EVtnjc9JpXUOtUB1T9Ru7mZOT361AN3MsetVe7R0a1KZshJDZdgp9miQ==", + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "zod": "^3.18.0" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..ae587f3 --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "promptfoo": "^0.100.5" + } +} diff --git a/tamingllms/_build/.doctrees/environment.pickle b/tamingllms/_build/.doctrees/environment.pickle index d97236f..42cf12d 100644 Binary files a/tamingllms/_build/.doctrees/environment.pickle and b/tamingllms/_build/.doctrees/environment.pickle differ diff --git a/tamingllms/_build/.doctrees/markdown/toc.doctree b/tamingllms/_build/.doctrees/markdown/toc.doctree index 5505584..11dd0e2 100644 Binary files a/tamingllms/_build/.doctrees/markdown/toc.doctree and b/tamingllms/_build/.doctrees/markdown/toc.doctree differ diff --git a/tamingllms/_build/.doctrees/notebooks/evals.doctree b/tamingllms/_build/.doctrees/notebooks/evals.doctree index 7b8f783..42cbbd0 100644 Binary files a/tamingllms/_build/.doctrees/notebooks/evals.doctree and b/tamingllms/_build/.doctrees/notebooks/evals.doctree differ diff --git a/tamingllms/_build/.doctrees/notebooks/output_size_limit.doctree b/tamingllms/_build/.doctrees/notebooks/output_size_limit.doctree index 5c22cf7..1c067f6 100644 Binary files a/tamingllms/_build/.doctrees/notebooks/output_size_limit.doctree and b/tamingllms/_build/.doctrees/notebooks/output_size_limit.doctree differ diff --git a/tamingllms/_build/.doctrees/notebooks/structured_output.doctree b/tamingllms/_build/.doctrees/notebooks/structured_output.doctree index a345ea0..a5dc98e 100644 Binary files a/tamingllms/_build/.doctrees/notebooks/structured_output.doctree and b/tamingllms/_build/.doctrees/notebooks/structured_output.doctree differ diff --git a/tamingllms/_build/html/_images/promptfoo1.png b/tamingllms/_build/html/_images/promptfoo1.png new file mode 100644 index 0000000..cb8dc67 Binary files /dev/null and b/tamingllms/_build/html/_images/promptfoo1.png differ diff --git a/tamingllms/_build/html/_sources/markdown/toc.md b/tamingllms/_build/html/_sources/markdown/toc.md index b06c890..e66ebb8 100644 --- a/tamingllms/_build/html/_sources/markdown/toc.md +++ b/tamingllms/_build/html/_sources/markdown/toc.md @@ -139,3 +139,21 @@ Abstract: *The current discourse around Large Language Models (LLMs) tends to fo - A.2 Monitoring Solutions - A.3 Open Source Models - A.4 Community Resources + + +## Citation +[![CC BY-NC-SA 4.0][cc-by-nc-sa-image]][cc-by-nc-sa] + +[cc-by-nc-sa]: http://creativecommons.org/licenses/by-nc-sa/4.0/ +[cc-by-nc-sa-image]: https://licensebuttons.net/l/by-nc-sa/4.0/88x31.png +[cc-by-nc-sa-shield]: https://img.shields.io/badge/License-CC-BY--NC--SA-4.0-lightgrey.svg + +``` +@misc{tharsistpsouza2024tamingllms, + author = {Tharsis T. P. Souza}, + title = {Taming LLMs: A Practical Guide to LLM Pitfalls with Open Source Software}, + year = {2024}, + journal = {GitHub repository}, + url = {https://github.com/souzatharsis/tamingLLMs) +} +``` \ No newline at end of file diff --git a/tamingllms/_build/html/_sources/notebooks/evals.ipynb b/tamingllms/_build/html/_sources/notebooks/evals.ipynb index 6b5b1ca..75041e9 100644 --- a/tamingllms/_build/html/_sources/notebooks/evals.ipynb +++ b/tamingllms/_build/html/_sources/notebooks/evals.ipynb @@ -37,7 +37,11 @@ "\n", "- Temperature = 0: Most deterministic, but potentially repetitive\n", "- Temperature = 1: Balanced creativity and coherence\n", - "- Temperature > 1: Increased randomness, potentially incoherent" + "- Temperature > 1: Increased randomness, potentially incoherent\n", + "\n", + "A temperature of 1 represents the unscaled probability scores for each token in the vocabulary. Decreasing the temperature closer to 0 sharpens the distribution, so the most likely token will have an even higher probability score. Conversely, increasing the temperature makes the distribution more uniform {cite}`build-llms-from-scratch-book`.\n", + "\n", + "In this simple experiment, we use an LLM to write a single-statement executive summary of an input financial filing. We observe that even a simple parameter like temperature can dramatically alter model behavior in ways that are difficult to systematically assess. At temperature 0.0, responses are consistent but potentially too rigid. At 1.0, outputs become more varied but less predictable. At 2.0, responses can be wildly different and often incoherent. This non-deterministic behavior makes traditional software testing approaches inadequate." ] }, { @@ -171,13 +175,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "A temperature of 1 represents the unscaled probability scores for each token in the vocabulary. Decreasing the temperature closer to 0 sharpens the distribution, so the most likely token will have an even higher probability score. Conversely, increasing the temperature makes the distribution more uniform {cite}`build-llms-from-scratch-book`.\n", - "\n", - "In this simple experiment, we use an LLM to write a single-statement executive summary of an input financial filing. We observe that even a simple parameter like temperature can dramatically alter model behavior in ways that are difficult to systematically assess. At temperature 0.0, responses are consistent but potentially too rigid. At 1.0, outputs become more varied but less predictable. At 2.0, responses can be wildly different and often incoherent. This non-deterministic behavior makes traditional software testing approaches inadequate.\n", - "\n", - "\n", - "\n", - "The implications for evaluation are critical. How can one effectively test an LLM-powered system when the same prompt can yield radically different outputs based on a single parameter? Traditional testing relies on predictable inputs and outputs, but LLMs force us to grapple with probabilistic behavior. While lower temperatures may seem safer for critical applications, they don't necessarily eliminate the underlying uncertainty. This highlights the need for new evaluation paradigms that can handle both deterministic and probabilistic aspects of LLM behavior." + "How can one effectively test an LLM-powered system when the same prompt can yield radically different outputs based on a single parameter? Traditional testing relies on predictable inputs and outputs, but LLMs force us to grapple with probabilistic behavior. While lower temperatures may seem safer for critical applications, they don't necessarily eliminate the underlying uncertainty. This highlights the need for new evaluation paradigms that can handle both deterministic and probabilistic aspects of LLM behavior." ] }, { @@ -187,7 +185,9 @@ "\n", "## Emerging Properties\n", "\n", - "Beyond their non-deterministic nature, LLMs present another fascinating challenge: emergent abilities that spontaneously arise as models scale up in size. These abilities - from basic question answering to complex reasoning - aren't explicitly programmed but rather emerge \"naturally\" as the models grow larger and are trained on more data. This makes evaluation fundamentally different from traditional software testing, where capabilities are explicitly coded and can be tested against clear specifications.\n", + "Beyond their non-deterministic nature, LLMs present another fascinating characteristic: emergent abilities that spontaneously arise as models scale up in size. These abilities - from basic question answering to complex reasoning - aren't explicitly programmed but rather emerge \"naturally\" as the models grow larger and are trained on more data. This makes evaluation fundamentally different from traditional software testing, where capabilities are explicitly coded and can be tested against pre-defined specifications.\n", + "\n", + "{numref}`emerging-properties` provides a list of emergent abilities of large language models and the scale. The relationship between model scale and emergent abilities follows a fascinating non-linear pattern. Below certain size thresholds, specific abilities may be completely absent from the model - it simply cannot perform certain tasks, no matter how much you try to coax them out. However, once the model reaches critical points in its scaling journey, these abilities can suddenly manifest in what researchers call a phase transition - a dramatic shift from inability to capability. This unpredictable emergence of capabilities stands in stark contrast to traditional software development, where features are deliberately implemented and can be systematically tested.\n", "\n", "```{figure} ../_static/evals/emerging.png\n", "---\n", @@ -200,9 +200,7 @@ "Emergent abilities of large language models and the scale {cite}`wei2022emergentabilitieslargelanguage`.\n", "```\n", "\n", - " {numref}`emerging-properties` provides a list of emergent abilities of large language models and the scale. The relationship between model scale and emergent abilities follows a fascinating non-linear pattern. Below certain size thresholds, specific abilities may be completely absent from the model - it simply cannot perform certain tasks, no matter how much you try to coax them out. However, once the model reaches critical points in its scaling journey, these abilities can suddenly manifest in what researchers call a phase transition - a dramatic shift from inability to capability. This unpredictable emergence of capabilities stands in stark contrast to traditional software development, where features are deliberately implemented and can be systematically tested.\n", - "\n", - "The implications for evaluation are pressing. While conventional software testing relies on stable test suites and well-defined acceptance criteria, LLM evaluation must contend with a constantly shifting landscape of capabilities. What worked to evaluate a 7B parameter model may be completely inadequate for a 70B parameter model that has developed new emergent abilities. This dynamic nature of LLM capabilities forces us to fundamentally rethink our approach to testing and evaluation.\n", + "The implications for evaluation are critical. While conventional software testing relies on stable test suites and well-defined acceptance criteria, LLM evaluation must contend with a constantly shifting landscape of capabilities. What worked to evaluate a 7B parameter model may be completely inadequate for a 70B parameter model that has developed new emergent abilities. This dynamic nature of LLM capabilities forces us to fundamentally rethink our approach to testing and evaluation.\n", "\n", "## Problem Statement\n", "\n", @@ -216,11 +214,11 @@ "The challenge becomes even more complex when we consider that traditional software evaluation methods simply weren't designed for these kinds of systems - There is an **Evals Gap** between traditional software testing and LLM evaluation. We need new frameworks that can account for both the deterministic aspects we're used to testing and the emergent properties that make LLMs unique. \n", "\n", "{numref}`evals-table` explores how LLM evaluation differs from traditional software testing across several key dimensions:\n", - "- **Capability Assessment vs Functional Testing**: Traditional software testing validates specific functionality against predefined requirements. LLM evaluation must assess not necessarily pre-defined \"emergent properties\" like reasoning, creativity, and language understanding that extend beyond explicit programming.\n", + "- **Capability Assessment vs Functional Testing**: Traditional software testing validates specific functionality against predefined requirements. LLM evaluation must assess not necessarily pre-defined behavior but also \"emergent properties\" like reasoning, creativity, and language understanding that extend beyond explicit programming.\n", "\n", - "- **Metrics and Measurement Challenges**: While traditional software metrics can usually be precisely defined and measured, LLM evaluation often involves subjective qualities like \"helpfulness\" or \"naturalness\" that resist straightforward quantification. Even when we try to break these down into numeric scores, the underlying judgment remains inherently human and context-dependent.\n", + "- **Metrics and Measurement Challenges**: While traditional software metrics can usually be precisely defined and measured, LLM evaluation often involves subjective qualities like \"helpfulness\" or \"naturalness\" that resist straightforward quantification. Even when we try to break these down into numeric scores, the underlying judgment often remains inherently human and context-dependent.\n", "\n", - "- **Dataset Contamination**: Traditional software testing uses carefully crafted test cases with known inputs and expected outputs (e.g., unit tests, integration tests). In contrast, LLMs trained on massive internet-scale datasets risk having already seen and memorized evaluation examples during training, which can lead to artificially inflated performance scores. This requires careful dataset curation to ensure test sets are truly unseen by the model and rigorous cross-validation approaches.\n", + "- **Dataset Contamination**: Traditional software testing uses carefully crafted test cases with known inputs and expected outputs (e.g., unit tests). In contrast, LLMs trained on massive internet-scale datasets risk having already seen and memorized evaluation examples during training, which can lead to artificially inflated performance scores. This requires careful dataset curation to ensure test sets are truly unseen by the model and rigorous cross-validation approaches.\n", "\n", "- **Benchmark Evolution**: Traditional software maintains stable test suites over time. LLM benchmarks continuously evolve as capabilities advance, making longitudinal performance comparisons difficult and potentially obsoleting older evaluation methods.\n", "\n", @@ -252,7 +250,7 @@ "- Evaluation must align with business objectives\n", "- A great LLM doesn't guarantee a great application!\n", "\n", - "Examples of key requirements for validation are listed in {numref}`validation-requirements`.\n", + "Examples of key requirements for validation are listed in {numref}`validation-requirements` ranging from Safety, Cognitive, Technical, Meta-Cognitive, to Ethical aspects.\n", "\n", "```{table} LLM Application Testing Requirements Matrix\n", ":name: validation-requirements\n", @@ -1524,18 +1522,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/tobias/src/tamingLLMs/tamingllms/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], + "outputs": [], "source": [ "import evaluate as hf_evaluate # HuggingFace's evaluate\n", "from langsmith import evaluate as langsmith_evaluate # LangSmith's evaluate\n", @@ -2098,8 +2087,428 @@ "source": [ "### PromptFoo\n", "\n", - "PromptFoo {cite}`promptfoo` is a framework for evaluating the quality of prompts for LLMs.\n", - "\n" + "Promptfoo {cite}`promptfoo2024` is an open-source framework designed for evaluating applications that utilize large language models (LLMs). Key features include:\n", + "\n", + "1. **Automated Testing**: Promptfoo provides automated testing capabilities, allowing developers to run custom evaluations tailored to their applications.\n", + "\n", + "2. **Custom Probes**: Developers can create custom probes to focus on specific use cases for instance decoupling prompts from tests cases.\n", + "\n", + "3. **User-Friendly CLI**: The framework features a command-line interface that supports live reloads and caching, facilitating rapid testing and iteration.\n", + "\n", + "We will use promptfoo's command line interface in the following examples. Please follow installation instructions [here](https://www.promptfoo.dev/docs/installation/#for-command-line-usage).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Evals are defined in a configuration file `promptfooconfig.yaml`, which defines elements such as providers, prompts, test cases, and assertions." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the following example, we will perform a two-step evaluation:\n", + "\n", + "1. Evaluate the performance of different LLM models given a set of constraints.\n", + "2. Evaluate the quality of different prompts for the best performing model from 1." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "description: Best model eval\n", + "prompts:\n", + "- file://prompt1.txt\n", + "providers:\n", + "- openai:gpt-4o-mini\n", + "- openai:gpt-4\n", + "- openai:gpt-3.5-turbo\n", + "defaultTest:\n", + " assert:\n", + " - type: cost\n", + " threshold: 0.001\n", + " - type: latency\n", + " threshold: 1000\n", + " - type: python\n", + " value: len(output) < 200\n", + " - type: llm-rubric\n", + " value: Does the summary look like it was written by an expert analyst [Yes/No]?\n", + "tests: file://tests.csv\n", + "\n" + ] + } + ], + "source": [ + "import yaml\n", + "\n", + "# Read the YAML file\n", + "with open('promptfoo/model_comparison/promptfooconfig.yaml', 'r') as f:\n", + " config = yaml.safe_load(f)\n", + "\n", + "# Pretty print the YAML content\n", + "print(yaml.dump(config, default_flow_style=False, sort_keys=False))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The configuration file shows how PromptFoo can be used to evaluate different LLM models. The YAML configuration defines three providers (GPT-4o-mini, GPT-4, and GPT-3.5-turbo) and sets up test assertions to validate their outputs. These assertions check important constraints:\n", + "\n", + "1. Cost efficiency: Each inference must cost less than $0.001\n", + "2. Latency requirements: Response time must be under 1000ms \n", + "3. Output length: Generated text must be less than 200 characters\n", + "4. Output quality: An LLM-based rubric evaluates if the output appears to be written by an expert (uses openai's gpt-4o model)\n", + "\n", + "The prompts are loaded from an external file (prompt1.txt) and test cases are defined in tests.csv. This structured approach enables systematic evaluation of model performance across multiple decoupled dimensions, namely prompts and test cases.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```bash\n", + "promptfoo eval --no-cache --output eval.json\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This command will run the evaluation and store the results in eval.json while making sure that the evaluation is not cached so we are measuring actual latency of the LLMs. The code below processes the PromptFoo evaluation results stored in eval.json. It reads the evaluation data from the JSON file and extracts key metrics including:\n", + "\n", + "- Provider name (e.g. gpt-4, gpt-3.5-turbo)\n", + "- Latency in milliseconds \n", + "- Token usage statistics\n", + "- Cost per request\n", + "- Number of passed/failed assertions\n", + "- Prompt token count\n", + "- Total number of API requests" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import pandas as pd\n", + "\n", + "# Read the JSON file\n", + "with open('promptfoo/model_comparison/eval.json', 'r') as f:\n", + " eval_data = json.load(f)\n", + "\n", + "# Extract results into a list of dictionaries\n", + "results = []\n", + "for prompt in eval_data['results']['prompts']:\n", + " result = {\n", + " 'provider': prompt['provider'],\n", + " 'latency_ms': prompt['metrics']['totalLatencyMs'],\n", + " 'token_usage': prompt['metrics']['tokenUsage']['total'],\n", + " 'cost': prompt['metrics']['cost'],\n", + " 'assert_pass': prompt['metrics']['assertPassCount'], \n", + " 'assert_fail': prompt['metrics']['assertFailCount'],\n", + " 'prompt_tokens': prompt['metrics']['tokenUsage']['prompt'],\n", + " 'num_requests': prompt['metrics']['tokenUsage']['numRequests']\n", + " }\n", + " results.append(result)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import display, Markdown" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "### PromptFoo Evaluation Results" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " provider latency_ms token_usage cost assert_pass \\\n", + "0 openai:gpt-4o-mini 2463 97 0.000035 6 \n", + "1 openai:gpt-4 3773 103 0.004620 4 \n", + "2 openai:gpt-3.5-turbo 1669 95 0.000091 7 \n", + "\n", + " assert_fail prompt_tokens num_requests \n", + "0 2 52 2 \n", + "1 4 52 2 \n", + "2 1 52 2 \n" + ] + } + ], + "source": [ + "# Convert to DataFrame\n", + "df = pd.DataFrame(results)\n", + "display(Markdown(\"### PromptFoo Evaluation Results\"))\n", + "print(df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The evaluation results reveal interesting performance characteristics across different OpenAI models. GPT-3.5-turbo demonstrates the best overall performance given our criteria with the lowest latency (1669ms), lowest token usage (95), and highest number of passed assertions (7). While GPT-4 shows higher token usage (103) and latency (3773ms), it also has the highest cost per request ($0.00462). The GPT-4-mini variant offers a middle ground, with moderate latency and token usage, while maintaining relatively good assertion performance (6 passes). These results suggest that for this particular evaluation task, GPT-3.5-turbo provides the best balance of performance, reliability, and cost-effectiveness.\n", + "\n", + "Promptfool also offers a web interface for visualizing the evaluation results as shown in {numref}`promptfoo1`. \n", + "\n", + "```bash\n", + "promptfoo view\n", + "```\n", + "\n", + "We can observe results per test case (i.e. section of the SEC filing) and per provider. Humans can also manually review the results and provide feedback as well as generate new test cases." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```{figure} ../_static/evals/promptfoo1.png\n", + "---\n", + "name: promptfoo1\n", + "alt: PromptFoo Evaluation Results\n", + "scale: 30%\n", + "align: center\n", + "---\n", + "PromptFoo evaluation results showing performance metrics across different models.\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we have established `GPT-3.5-turbo` as our model of choice given the minimum required criteria based on cost, latency and basic qualitative evaluation, we can compare the performance of different prompts as a next evaluation step. Can we improve the quality of the summaries by using different prompts?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, we redefine our evaluation criteria. We now would like to select the prompt that delivers the most \"detailed\" summaries. Our updated promptfoo configuration file is shown below." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "description: Best model eval\n", + "prompts:\n", + "- file://prompt1.txt\n", + "- file://prompt2.txt\n", + "- file://prompt3.txt\n", + "providers:\n", + "- openai:gpt-3.5-turbo\n", + "defaultTest:\n", + " assert:\n", + " - type: llm-rubric\n", + " value: 'Evaluate the output based on how detailed it is. Grade it on a scale\n", + " of 0.0 to 1.0, where:\n", + "\n", + " Score of 0.1: Not much detail.\n", + "\n", + " Score of 0.5: Some detail.\n", + "\n", + " Score of 1.0: Very detailed.\n", + "\n", + " '\n", + "tests: file://tests.csv\n", + "\n" + ] + } + ], + "source": [ + "# Read the YAML file\n", + "with open('promptfoo/prompt_comparison/promptfooconfig.yaml', 'r') as f:\n", + " config = yaml.safe_load(f)\n", + "\n", + "# Pretty print the YAML content\n", + "print(yaml.dump(config, default_flow_style=False, sort_keys=False))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that we are now passing 3 different prompts. And we have updated our assertions to check if the output is detailed. Now, let's define 3 prompt variations we would like to test aiming at improving the quality/detail of the summaries." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "### prompt1.txt\n", + "---\n", + "'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'\n", + "\n", + "\n", + "### prompt2.txt\n", + "---\n", + "'ROLE: You are a financial analyst. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'\n", + "\n", + "\n", + "### prompt3.txt\n", + "---\n", + "'ROLE: You are a financial analyst. REQUIREMENTS: BE DETAILED. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Display the prompt variations\n", + "from IPython.display import display, Markdown\n", + "\n", + "prompt_files = ['prompt1.txt', 'prompt2.txt', 'prompt3.txt']\n", + "prompt_content = []\n", + "\n", + "for file in prompt_files:\n", + " with open(f'promptfoo/prompt_comparison/{file}', 'r') as f:\n", + " content = f.read().strip()\n", + " prompt_content.append(f\"### {file}\\n---\\n{content}\\n\")\n", + "\n", + "display(Markdown(\"\\n\\n\".join(prompt_content)))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The first prompt matches our previous prompt. The second prompt adds a \"financial analyst\" role to the prompt. The third prompt expands on second prompt and add a requirement \"BE DETAILED\"." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now run the evaluation again." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```bash\n", + "promptfoo eval --output eval.json\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "### Prompt Comparison Results by Section" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Prompt Section prompt1.txt prompt2.txt prompt3.txt\n", + "0 Legal Proceedings 0.1 0.5 1.0\n", + "1 Risk Factors 0.1 0.5 0.5\n" + ] + } + ], + "source": [ + "# Read the evaluation results from JSON file\n", + "import json\n", + "with open('promptfoo/prompt_comparison/eval.json', 'r') as f:\n", + " eval_data = json.load(f)\n", + "\n", + "# Create a list to store the data\n", + "data = []\n", + "\n", + "# Extract results for each test case\n", + "for result in eval_data['results']['results']:\n", + " section = result['vars']['section']\n", + " prompt_id = result['promptId']\n", + " score = result['gradingResult']['score'] if 'gradingResult' in result else 0.0\n", + " \n", + " # Find the corresponding prompt file\n", + " for prompt in eval_data['results']['prompts']:\n", + " if prompt['id'] == prompt_id:\n", + " prompt_file = prompt['label'].split(':')[0]\n", + " break\n", + " \n", + " # Add to data list\n", + " data.append([section, prompt_file, score])\n", + "\n", + "# Convert to DataFrame\n", + "df_raw = pd.DataFrame(data, columns=['Section', 'Prompt', 'Score'])\n", + "\n", + "# Pivot to get desired format\n", + "df = df_raw.pivot(index='Section', columns='Prompt', values='Score').reset_index()\n", + "df = df[['Section', 'prompt1.txt', 'prompt2.txt', 'prompt3.txt']]\n", + "\n", + "display(Markdown(\"### Prompt Comparison Results by Section\"))\n", + "print(df)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The results show that prompt3.txt performs best for Legal Proceedings sections, achieving a perfect score of 1.0 compared to 0.5 for prompt2.txt and 0.1 for prompt1.txt. For Risk Factors sections, both prompt2.txt and prompt3.txt achieve moderate scores of 0.5, while prompt1.txt scores poorly at 0.1. This suggests that prompt3.txt is generally more effective at extracting detailed information, particularly for legal content. In summary, defining a Role and a requirement for the output to be detailed is a good way to improve the quality of the summaries at least for this specific task, model and criteria.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In conclusion, Promptfoo can serve as an effective LLM application evaluation tool particularly for its ability to decouple several components of the evaluation process. Hence enabling the user to focus on the most important aspects of the evaluation given the particular application and criteria making it a valuable and flexible tool for LLM application development." ] }, { diff --git a/tamingllms/_build/html/markdown/toc.html b/tamingllms/_build/html/markdown/toc.html index bbe602a..44a8f0f 100644 --- a/tamingllms/_build/html/markdown/toc.html +++ b/tamingllms/_build/html/markdown/toc.html @@ -379,6 +379,19 @@

Appendix A: Tools and Resources +

Citation

+

CC BY-NC-SA 4.0

+
@misc{tharsistpsouza2024tamingllms,
+  author = {Tharsis T. P. Souza},
+  title = {Taming LLMs: A Practical Guide to LLM Pitfalls with Open Source Software},
+  year = {2024},
+  journal = {GitHub repository},
+  url = {https://github.com/souzatharsis/tamingLLMs)
+}
+
+
+
diff --git a/tamingllms/_build/html/notebooks/evals.html b/tamingllms/_build/html/notebooks/evals.html index 977fe26..e9bf532 100644 --- a/tamingllms/_build/html/notebooks/evals.html +++ b/tamingllms/_build/html/notebooks/evals.html @@ -193,7 +193,7 @@
-

4. The Evals Gap

+

4. The Evals Gap

It doesn’t matter how beautiful your theory is,
it doesn’t matter how smart you are.
@@ -203,45 +203,45 @@

Contents
-

4.1. Non-Deterministic Generative Machines

+

4.1. Non-Deterministic Generative Machines

One of the most fundamental challenges when building products with Large Language Models (LLMs) is their generative and non-deterministic nature. Unlike traditional software systems where the same input reliably produces the same output, LLMs can generate novel text that may not exist in their training data, and produce different responses each time they’re queried - even with identical prompts and input data. This behavior is both a strength and a significant engineering challenge and product challenge.

When you ask an LLM the same question multiple times, you’ll likely get different responses. This isn’t a bug - it’s a fundamental feature of how these models work. The “temperature” parameter, which controls the randomness of outputs, allows models to be creative and generate diverse responses. However, this same feature makes it difficult to build reliable, testable systems.

Consider a financial services company using LLMs to generate investment advice. The non-deterministic nature of these models means that:

@@ -252,7 +252,7 @@

-

4.1.1. Temperature and Sampling

+

4.1.1. Temperature and Sampling

The primary source of non-determinism in LLMs comes from their sampling strategies. During text generation, the model:

  1. Calculates probability distributions for each next token

    2. @@ -261,12 +261,14 @@

    -

    4.1.2. The Temperature Spectrum

    +

    4.1.2. The Temperature Spectrum

    • Temperature = 0: Most deterministic, but potentially repetitive

    • Temperature = 1: Balanced creativity and coherence

    • Temperature > 1: Increased randomness, potentially incoherent

    +

    A temperature of 1 represents the unscaled probability scores for each token in the vocabulary. Decreasing the temperature closer to 0 sharpens the distribution, so the most likely token will have an even higher probability score. Conversely, increasing the temperature makes the distribution more uniform [Raschka, 2024].

    +

    In this simple experiment, we use an LLM to write a single-statement executive summary of an input financial filing. We observe that even a simple parameter like temperature can dramatically alter model behavior in ways that are difficult to systematically assess. At temperature 0.0, responses are consistent but potentially too rigid. At 1.0, outputs become more varied but less predictable. At 2.0, responses can be wildly different and often incoherent. This non-deterministic behavior makes traditional software testing approaches inadequate.

    from dotenv import load_dotenv
@@ -376,25 +378,23 @@ 
    [Raschka, 2024].
    
-
    In this simple experiment, we use an LLM to write a single-statement executive summary of an input financial filing. We observe that even a simple parameter like temperature can dramatically alter model behavior in ways that are difficult to systematically assess. At temperature 0.0, responses are consistent but potentially too rigid. At 1.0, outputs become more varied but less predictable. At 2.0, responses can be wildly different and often incoherent. This non-deterministic behavior makes traditional software testing approaches inadequate.
    
-
    The implications for evaluation are critical. How can one effectively test an LLM-powered system when the same prompt can yield radically different outputs based on a single parameter? Traditional testing relies on predictable inputs and outputs, but LLMs force us to grapple with probabilistic behavior. While lower temperatures may seem safer for critical applications, they don’t necessarily eliminate the underlying uncertainty. This highlights the need for new evaluation paradigms that can handle both deterministic and probabilistic aspects of LLM behavior.
    
+
    How can one effectively test an LLM-powered system when the same prompt can yield radically different outputs based on a single parameter? Traditional testing relies on predictable inputs and outputs, but LLMs force us to grapple with probabilistic behavior. While lower temperatures may seem safer for critical applications, they don’t necessarily eliminate the underlying uncertainty. This highlights the need for new evaluation paradigms that can handle both deterministic and probabilistic aspects of LLM behavior.

-

4.2. Emerging Properties

-

Beyond their non-deterministic nature, LLMs present another fascinating challenge: emergent abilities that spontaneously arise as models scale up in size. These abilities - from basic question answering to complex reasoning - aren’t explicitly programmed but rather emerge “naturally” as the models grow larger and are trained on more data. This makes evaluation fundamentally different from traditional software testing, where capabilities are explicitly coded and can be tested against clear specifications.

+

4.2. Emerging Properties

+

Beyond their non-deterministic nature, LLMs present another fascinating characteristic: emergent abilities that spontaneously arise as models scale up in size. These abilities - from basic question answering to complex reasoning - aren’t explicitly programmed but rather emerge “naturally” as the models grow larger and are trained on more data. This makes evaluation fundamentally different from traditional software testing, where capabilities are explicitly coded and can be tested against pre-defined specifications.

+

Fig. 4.1 provides a list of emergent abilities of large language models and the scale. The relationship between model scale and emergent abilities follows a fascinating non-linear pattern. Below certain size thresholds, specific abilities may be completely absent from the model - it simply cannot perform certain tasks, no matter how much you try to coax them out. However, once the model reaches critical points in its scaling journey, these abilities can suddenly manifest in what researchers call a phase transition - a dramatic shift from inability to capability. This unpredictable emergence of capabilities stands in stark contrast to traditional software development, where features are deliberately implemented and can be systematically tested.

Emerging Properties

Fig. 4.1 Emergent abilities of large language models and the scale [Wei et al., 2022].

-

Fig. 4.1 provides a list of emergent abilities of large language models and the scale. The relationship between model scale and emergent abilities follows a fascinating non-linear pattern. Below certain size thresholds, specific abilities may be completely absent from the model - it simply cannot perform certain tasks, no matter how much you try to coax them out. However, once the model reaches critical points in its scaling journey, these abilities can suddenly manifest in what researchers call a phase transition - a dramatic shift from inability to capability. This unpredictable emergence of capabilities stands in stark contrast to traditional software development, where features are deliberately implemented and can be systematically tested.

-

The implications for evaluation are pressing. While conventional software testing relies on stable test suites and well-defined acceptance criteria, LLM evaluation must contend with a constantly shifting landscape of capabilities. What worked to evaluate a 7B parameter model may be completely inadequate for a 70B parameter model that has developed new emergent abilities. This dynamic nature of LLM capabilities forces us to fundamentally rethink our approach to testing and evaluation.

+

The implications for evaluation are critical. While conventional software testing relies on stable test suites and well-defined acceptance criteria, LLM evaluation must contend with a constantly shifting landscape of capabilities. What worked to evaluate a 7B parameter model may be completely inadequate for a 70B parameter model that has developed new emergent abilities. This dynamic nature of LLM capabilities forces us to fundamentally rethink our approach to testing and evaluation.

-

4.3. Problem Statement

+

4.3. Problem Statement

Consider a practical example that illustrates these challenges: building a Math AI tutoring system for children powered by an LLM. In traditional software development, you would define specific features (like presenting math problems or checking answers) and write tests to verify each function. But with LLMs, you’re not just testing predefined features - you’re trying to evaluate emergent capabilities like adapting explanations to a child’s level, maintaining engagement through conversational learning, and providing age-appropriate safety-bound content.

This fundamental difference raises critical questions about evaluation:

    @@ -405,9 +405,9 @@

    explores how LLM evaluation differs from traditional software testing across several key dimensions:

      -

    • Capability Assessment vs Functional Testing: Traditional software testing validates specific functionality against predefined requirements. LLM evaluation must assess not necessarily pre-defined “emergent properties” like reasoning, creativity, and language understanding that extend beyond explicit programming.

      • -

    • Metrics and Measurement Challenges: While traditional software metrics can usually be precisely defined and measured, LLM evaluation often involves subjective qualities like “helpfulness” or “naturalness” that resist straightforward quantification. Even when we try to break these down into numeric scores, the underlying judgment remains inherently human and context-dependent.

      • -

    • Dataset Contamination: Traditional software testing uses carefully crafted test cases with known inputs and expected outputs (e.g., unit tests, integration tests). In contrast, LLMs trained on massive internet-scale datasets risk having already seen and memorized evaluation examples during training, which can lead to artificially inflated performance scores. This requires careful dataset curation to ensure test sets are truly unseen by the model and rigorous cross-validation approaches.

      • +

    • Capability Assessment vs Functional Testing: Traditional software testing validates specific functionality against predefined requirements. LLM evaluation must assess not necessarily pre-defined behavior but also “emergent properties” like reasoning, creativity, and language understanding that extend beyond explicit programming.

      • +

    • Metrics and Measurement Challenges: While traditional software metrics can usually be precisely defined and measured, LLM evaluation often involves subjective qualities like “helpfulness” or “naturalness” that resist straightforward quantification. Even when we try to break these down into numeric scores, the underlying judgment often remains inherently human and context-dependent.

      • +

    • Dataset Contamination: Traditional software testing uses carefully crafted test cases with known inputs and expected outputs (e.g., unit tests). In contrast, LLMs trained on massive internet-scale datasets risk having already seen and memorized evaluation examples during training, which can lead to artificially inflated performance scores. This requires careful dataset curation to ensure test sets are truly unseen by the model and rigorous cross-validation approaches.

    • Benchmark Evolution: Traditional software maintains stable test suites over time. LLM benchmarks continuously evolve as capabilities advance, making longitudinal performance comparisons difficult and potentially obsoleting older evaluation methods.

    • Human Evaluation Requirements: Traditional software testing automates most validation. LLM evaluation may demand significant human oversight to assess output quality, appropriateness, and potential biases through structured annotation and systematic review processes.

    @@ -444,7 +444,7 @@

    -

    4.4. Evals Design

    +

    4.4. Evals Design

    First, it’s important to make a distinction between evaluating an LLM versus evaluating an LLM-based application. While the latter offers foundation capabilities and are typically general-purpose, the former is more specific and tailored to a particular use case. Here, we define an LLM-based application as a system that uses one or more LLMs to perform a specific task. More specifically, an LLM-based application is the combination of one or more LLM models, their associated prompts and parameters to solve a particular business problem.

    That differentiation is important because it changes the scope of evaluation. LLMs are usually evaluated based on their capabilities, which include things like language understanding, reasoning and knowledge. LLM-based applications, instead, should be evaluated based on their end-to-end functionality, performance, and how well they meet business requirements. That distinction has key implications for the design of evaluation systems:

      @@ -452,7 +452,7 @@

      Table 4.2.

      +

      Examples of key requirements for validation are listed in Table 4.2 ranging from Safety, Cognitive, Technical, Meta-Cognitive, to Ethical aspects.

      @@ -531,7 +531,7 @@

      -

      4.4.1. Conceptual Overview

      +

      4.4.1. Conceptual Overview

      Fig. 4.2 demonstrates a conceptual design of key components of LLM Application evaluation.

      Conceptual Overview @@ -612,7 +612,7 @@

      -

      4.4.2. Design Considerations

      +

      4.4.2. Design Considerations

      The design of an LLM application evaluation system depends heavily on the specific use case and business requirements. Here we list important questions for planning an LLM application evaluation system pertaining to each of the key components previously introduced:

      1. Examples (Input Dataset):

        @@ -697,7 +697,7 @@

        -

        4.5. Metrics

        +

        4.5. Metrics

        The choice of metric depends on the specific task and desired evaluation criteria. However, one can categorize metrics into two broad categories: intrinsic and extrinsic.

        • Intrinsic metrics focus on the model’s performance on its primary training objective, which is typically to predict the next token in a sequence. Perplexity is a common intrinsic metric that measures how well the model predicts a given sample of text.

          • @@ -1007,9 +1007,9 @@

          4.6. Evaluators

          +

          4.6. Evaluators

          -

          4.6.1. Model-Based Evaluation

          +

          4.6.1. Model-Based Evaluation

          Traditional metrics like BLEU or ROUGE often fall short in capturing the nuanced, contextual, and creative outputs of LLMs. As an alternative we can consider a “Model-based evaluation” approach. A common approach is to use an LLM as a judge. This is an approach that leverages language models themselves to assess the quality of outputs from other language models. This method involves using a model (often a more capable one) to act as an automated judge, evaluating aspects like accuracy, coherence, and relevance of generated content. Unlike traditional metrics that rely on exact matching or statistical measures, model-based evaluation can capture nuanced aspects of language and provide more contextual assessment.

          As discussed in the paper [Li et al., 2024], LLM-based evaluation approaches generally fall into two main categories:

            @@ -1249,11 +1249,11 @@

            -

            4.6.2. Human-Based Evaluation

            +

            4.6.2. Human-Based Evaluation

            Human assessors can judge aspects like fluency, coherence, and factual accuracy, providing a more comprehensive evaluation. However, human evaluation can be subjective and resource-intensive.

          -

          4.6.3. Evaluating Evaluators

          +

          4.6.3. Evaluating Evaluators

          We have discussed how LLMs can be used to evaluate LLM-based aplications. However, how can we evaluate the performance of LLMs that evaluate other LLMs? This is the question that meta evaluation aims to answer. Clearly, the discussion can become quite meta as we need to evaluate the performance of the evaluator to evaluate the performance of the evaluated model. However, one can make a case for two general options:

          1. Use a gold-standard dataset that is used to evaluate the performance of LLM evaluators using a “metrics-based” approach.

            2. @@ -1297,7 +1297,7 @@

            -

            4.7. Benchmarks and Leaderboards

            +

            4.7. Benchmarks and Leaderboards

            Benchmarks act as standardized tests for LLMs, evaluating their performance across a spectrum of tasks. These tasks simulate real-world applications such as answering questions, generating coherent text, solving mathematical problems, or even writing computer code. They also assess more abstract qualities like fairness, robustness, and cultural understanding.

            Benchmarks can be thought as comprehensive “exams” that probe different “subjects” in order to certify an LLM. They help researchers and developers compare models systematically, in a way LLM performance is comparable while enabling the identification of emergent behaviors or capabilities as models evolve in scale and sophistication.

            The history of LLM benchmarks reflects the evolving priorities of artificial intelligence research, starting with foundational tasks and moving toward complex, real-world challenges. It began in 2018 with the introduction of GLUE(General Language Understanding Evaluation) [Wang et al., 2019], which set a new standard for evaluating natural language understanding. GLUE measured performance on tasks like sentiment analysis and textual entailment, providing a baseline for assessing the fundamental capabilities of language models. A year later, SuperGLUE [Wang et al., 2019] expanded on this foundation by introducing more nuanced tasks that tested reasoning and language comprehension at a deeper level, challenging the limits of models like BERT and its successors.

            @@ -1342,9 +1342,9 @@

            -

            4.8. Tools

            +

            4.8. Tools

            -

            4.8.1. LightEval

            +

            4.8.1. LightEval

            LightEval [Fourrier et al., 2023] is a lightweight framework for evaluation of LLMs across a variety of standard and bespoke metrics and tasks across multiple inference backends via Python SDK and CLI.

            As a motivating example, consider a scenario where financial data has been extracted from SEC financial filings and require econometric analysis. Tasks like estimating autoregressive models for time series forecasting or conducting hypothesis tests on market efficiency are common in financial analysis. Let’s evaluate how well different models perform on this type of task.

            First, we need to select a benchmark to assess LLMs capabilities in this domain. MMLU has a sub-benchmark called Econometrics we can use for this task. Table 4.4 shows a sample of the benchmark dataset from MMLU Econometrics. It consists of multiple-choice questions from econometrics and expected answers.

            @@ -1533,7 +1533,7 @@

            [Hugging Face, 2024]. Its integration with the Hugging Face ecosystem and modular architecture make it particularly powerful for evaluating open source models. For further details, visit the official repository [Fourrier et al., 2023].

            -

            4.8.2. LangSmith

            +

            4.8.2. LangSmith

            Let’s revisit our evaluation example when we were interested in evaluating the quality of summaries generated by different (smaller and cheaper) LLM models compared to a benchmark model (larger and more expensive). Recal the setup:

            • Benchmark model: gpt-4o

              • @@ -1562,12 +1562,6 @@

              -
              /home/tobias/src/tamingLLMs/tamingllms/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html
-  from .autonotebook import tqdm as notebook_tqdm
-
              -
              -

              The code below creates a dataset in LangSmith that will serve as our golden dataset for evaluation. The dataset consists of test cases where we create a single example with the following content:

                @@ -1947,12 +1941,273 @@

                -

                4.8.3. PromptFoo

                -

                PromptFoo [PromptFoo, 2024] is a framework for evaluating the quality of prompts for LLMs.

                +

                4.8.3. PromptFoo

                +

                Promptfoo [promptfoo, 2024] is an open-source framework designed for evaluating applications that utilize large language models (LLMs). Key features include:

                +
                  +

                1. Automated Testing: Promptfoo provides automated testing capabilities, allowing developers to run custom evaluations tailored to their applications.

                  2. +

                2. Custom Probes: Developers can create custom probes to focus on specific use cases for instance decoupling prompts from tests cases.

                  3. +

                3. User-Friendly CLI: The framework features a command-line interface that supports live reloads and caching, facilitating rapid testing and iteration.

                  4. +
                +

                We will use promptfoo’s command line interface in the following examples. Please follow installation instructions here.

                +

                Evals are defined in a configuration file promptfooconfig.yaml, which defines elements such as providers, prompts, test cases, and assertions.

                +

                In the following example, we will perform a two-step evaluation:

                +
                  +

                1. Evaluate the performance of different LLM models given a set of constraints.

                  2. +

                2. Evaluate the quality of different prompts for the best performing model from 1.

                  3. +
                +
                +
                +
                import yaml
+
+# Read the YAML file
+with open('promptfoo/model_comparison/promptfooconfig.yaml', 'r') as f:
+    config = yaml.safe_load(f)
+
+# Pretty print the YAML content
+print(yaml.dump(config, default_flow_style=False, sort_keys=False))
+
                +
                +
                +
                +
                description: Best model eval
+prompts:
+- file://prompt1.txt
+providers:
+- openai:gpt-4o-mini
+- openai:gpt-4
+- openai:gpt-3.5-turbo
+defaultTest:
+  assert:
+  - type: cost
+    threshold: 0.001
+  - type: latency
+    threshold: 1000
+  - type: python
+    value: len(output) < 200
+  - type: llm-rubric
+    value: Does the summary look like it was written by an expert analyst [Yes/No]?
+tests: file://tests.csv
+
                +
                +
                +
                +

                The configuration file shows how PromptFoo can be used to evaluate different LLM models. The YAML configuration defines three providers (GPT-4o-mini, GPT-4, and GPT-3.5-turbo) and sets up test assertions to validate their outputs. These assertions check important constraints:

                +
                  +

                1. Cost efficiency: Each inference must cost less than $0.001

                  2. +

                2. Latency requirements: Response time must be under 1000ms

                  3. +

                3. Output length: Generated text must be less than 200 characters

                  4. +

                4. Output quality: An LLM-based rubric evaluates if the output appears to be written by an expert (uses openai’s gpt-4o model)

                  5. +
                +

                The prompts are loaded from an external file (prompt1.txt) and test cases are defined in tests.csv. This structured approach enables systematic evaluation of model performance across multiple decoupled dimensions, namely prompts and test cases.

                +
                promptfoo eval --no-cache --output eval.json
+
                +
                +

                This command will run the evaluation and store the results in eval.json while making sure that the evaluation is not cached so we are measuring actual latency of the LLMs. The code below processes the PromptFoo evaluation results stored in eval.json. It reads the evaluation data from the JSON file and extracts key metrics including:

                +
                  +

                • Provider name (e.g. gpt-4, gpt-3.5-turbo)

                  • +

                • Latency in milliseconds

                  • +

                • Token usage statistics

                  • +

                • Cost per request

                  • +

                • Number of passed/failed assertions

                  • +

                • Prompt token count

                  • +

                • Total number of API requests

                  • +
                +
                +
                +
                import json
+import pandas as pd
+
+# Read the JSON file
+with open('promptfoo/model_comparison/eval.json', 'r') as f:
+    eval_data = json.load(f)
+
+# Extract results into a list of dictionaries
+results = []
+for prompt in eval_data['results']['prompts']:
+    result = {
+        'provider': prompt['provider'],
+        'latency_ms': prompt['metrics']['totalLatencyMs'],
+        'token_usage': prompt['metrics']['tokenUsage']['total'],
+        'cost': prompt['metrics']['cost'],
+        'assert_pass': prompt['metrics']['assertPassCount'], 
+        'assert_fail': prompt['metrics']['assertFailCount'],
+        'prompt_tokens': prompt['metrics']['tokenUsage']['prompt'],
+        'num_requests': prompt['metrics']['tokenUsage']['numRequests']
+    }
+    results.append(result)
+
                +
                +
                +
                +
                +
                +
                from IPython.display import display, Markdown
+
                +
                +
                +
                +
                +
                +
                # Convert to DataFrame
+df = pd.DataFrame(results)
+display(Markdown("### PromptFoo Evaluation Results"))
+print(df)
+
                +
                +
                +
                +

                PromptFoo Evaluation Results

                +
                               provider  latency_ms  token_usage      cost  assert_pass  \
+0    openai:gpt-4o-mini        2463           97  0.000035            6   
+1          openai:gpt-4        3773          103  0.004620            4   
+2  openai:gpt-3.5-turbo        1669           95  0.000091            7   
+
+   assert_fail  prompt_tokens  num_requests  
+0            2             52             2  
+1            4             52             2  
+2            1             52             2  
+
                +
                +
                +
                +

                The evaluation results reveal interesting performance characteristics across different OpenAI models. GPT-3.5-turbo demonstrates the best overall performance given our criteria with the lowest latency (1669ms), lowest token usage (95), and highest number of passed assertions (7). While GPT-4 shows higher token usage (103) and latency (3773ms), it also has the highest cost per request ($0.00462). The GPT-4-mini variant offers a middle ground, with moderate latency and token usage, while maintaining relatively good assertion performance (6 passes). These results suggest that for this particular evaluation task, GPT-3.5-turbo provides the best balance of performance, reliability, and cost-effectiveness.

                +

                Promptfool also offers a web interface for visualizing the evaluation results as shown in Fig. 4.12.

                +
                promptfoo view
+
                +
                +

                We can observe results per test case (i.e. section of the SEC filing) and per provider. Humans can also manually review the results and provide feedback as well as generate new test cases.

                +
                +PromptFoo Evaluation Results +
                +

                Fig. 4.12 PromptFoo evaluation results showing performance metrics across different models.

                +
                +
                +

                Now that we have established GPT-3.5-turbo as our model of choice given the minimum required criteria based on cost, latency and basic qualitative evaluation, we can compare the performance of different prompts as a next evaluation step. Can we improve the quality of the summaries by using different prompts?

                +

                First, we redefine our evaluation criteria. We now would like to select the prompt that delivers the most “detailed” summaries. Our updated promptfoo configuration file is shown below.

                +
                +
                +
                # Read the YAML file
+with open('promptfoo/prompt_comparison/promptfooconfig.yaml', 'r') as f:
+    config = yaml.safe_load(f)
+
+# Pretty print the YAML content
+print(yaml.dump(config, default_flow_style=False, sort_keys=False))
+
                +
                +
                +
                +
                description: Best model eval
+prompts:
+- file://prompt1.txt
+- file://prompt2.txt
+- file://prompt3.txt
+providers:
+- openai:gpt-3.5-turbo
+defaultTest:
+  assert:
+  - type: llm-rubric
+    value: 'Evaluate the output based on how detailed it is.  Grade it on a scale
+      of 0.0 to 1.0, where:
+
+      Score of 0.1: Not much detail.
+
+      Score of 0.5: Some detail.
+
+      Score of 1.0: Very detailed.
+
+      '
+tests: file://tests.csv
+
                +
                +
                +
                +

                Note that we are now passing 3 different prompts. And we have updated our assertions to check if the output is detailed. Now, let’s define 3 prompt variations we would like to test aiming at improving the quality/detail of the summaries.

                +
                +
                +
                # Display the prompt variations
+from IPython.display import display, Markdown
+
+prompt_files = ['prompt1.txt', 'prompt2.txt', 'prompt3.txt']
+prompt_content = []
+
+for file in prompt_files:
+    with open(f'promptfoo/prompt_comparison/{file}', 'r') as f:
+        content = f.read().strip()
+        prompt_content.append(f"### {file}\n---\n{content}\n")
+
+display(Markdown("\n\n".join(prompt_content)))
+
                +
                +
                +
                +

                prompt1.txt

                +
                +

                ‘Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}’

                +

                prompt2.txt

                +
                +

                ‘ROLE: You are a financial analyst. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}’

                +

                prompt3.txt

                +
                +

                ‘ROLE: You are a financial analyst. REQUIREMENTS: BE DETAILED. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}’

                +
                +
                +

                The first prompt matches our previous prompt. The second prompt adds a “financial analyst” role to the prompt. The third prompt expands on second prompt and add a requirement “BE DETAILED”.

                +

                We can now run the evaluation again.

                +
                promptfoo eval --output eval.json
+
                +
                +
                +
                +
                # Read the evaluation results from JSON file
+import json
+with open('promptfoo/prompt_comparison/eval.json', 'r') as f:
+    eval_data = json.load(f)
+
+# Create a list to store the data
+data = []
+
+# Extract results for each test case
+for result in eval_data['results']['results']:
+    section = result['vars']['section']
+    prompt_id = result['promptId']
+    score = result['gradingResult']['score'] if 'gradingResult' in result else 0.0
+    
+    # Find the corresponding prompt file
+    for prompt in eval_data['results']['prompts']:
+        if prompt['id'] == prompt_id:
+            prompt_file = prompt['label'].split(':')[0]
+            break
+            
+    # Add to data list
+    data.append([section, prompt_file, score])
+
+# Convert to DataFrame
+df_raw = pd.DataFrame(data, columns=['Section', 'Prompt', 'Score'])
+
+# Pivot to get desired format
+df = df_raw.pivot(index='Section', columns='Prompt', values='Score').reset_index()
+df = df[['Section', 'prompt1.txt', 'prompt2.txt', 'prompt3.txt']]
+
+display(Markdown("### Prompt Comparison Results by Section"))
+print(df)
+
                +
                +
                +
                +

                Prompt Comparison Results by Section

                +
                Prompt            Section  prompt1.txt  prompt2.txt  prompt3.txt
+0       Legal Proceedings          0.1          0.5          1.0
+1            Risk Factors          0.1          0.5          0.5
+
                +
                +
                +
                +

                The results show that prompt3.txt performs best for Legal Proceedings sections, achieving a perfect score of 1.0 compared to 0.5 for prompt2.txt and 0.1 for prompt1.txt. For Risk Factors sections, both prompt2.txt and prompt3.txt achieve moderate scores of 0.5, while prompt1.txt scores poorly at 0.1. This suggests that prompt3.txt is generally more effective at extracting detailed information, particularly for legal content. In summary, defining a Role and a requirement for the output to be detailed is a good way to improve the quality of the summaries at least for this specific task, model and criteria.

                +

                In conclusion, Promptfoo can serve as an effective LLM application evaluation tool particularly for its ability to decouple several components of the evaluation process. Hence enabling the user to focus on the most important aspects of the evaluation given the particular application and criteria making it a valuable and flexible tool for LLM application development.

          -

          4.9. References

          +

          4.9. References

          [ALB+24] @@ -2044,6 +2299,10 @@

          [LHE22]

          Stephanie Lin, Jacob Hilton, and Owain Evans. Truthfulqa: measuring how models mimic human falsehoods. 2022. URL: https://arxiv.org/abs/2109.07958, arXiv:2109.07958.

          +
          +[pro24] +

          promptfoo. Promptfoo: llm testing and evaluation framework. 2024. Open source framework for testing and evaluating LLM prompts. URL: https://www.promptfoo.dev/.

          +
          [Ras24]

          Sebastian Raschka. Build A Large Language Model (From Scratch). Manning, 2024. ISBN 978-1633437166. URL: https://www.manning.com/books/build-a-large-language-model-from-scratch.

          @@ -2084,10 +2343,6 @@

          [MetaAI24]

          Meta AI. Meta llama models on hugging face. https://huggingface.co/meta-llama, 2024. Accessed: 2024.

          -
          -[PromptFoo24] -

          PromptFoo. Promptfoo - open-source prompt engineering toolkit. https://www.promptfoo.dev/, 2024. Accessed: 12/06/2024.

          -
          diff --git a/tamingllms/_build/html/notebooks/output_size_limit.html b/tamingllms/_build/html/notebooks/output_size_limit.html index c588fc0..b889dd7 100644 --- a/tamingllms/_build/html/notebooks/output_size_limit.html +++ b/tamingllms/_build/html/notebooks/output_size_limit.html @@ -194,7 +194,7 @@
          -

          2. Output Size Limitations

          +

          2. Output Size Limitations

          Only those who will risk going too far can possibly find out how far one can go.

          —T.S. Eliot

          @@ -202,34 +202,34 @@

          Contents
          -

          2.1. What are Token Limits?

          +

          2.1. What are Token Limits?

          Tokens are the basic units that LLMs process text with. A token can be as short as a single character or as long as a complete word. In English, a general rule of thumb is that 1 token ≈ 4 characters or ¾ of a word.

          The max_output_tokens is parameter often available in modern LLMs that determines the maximum length of text that an LLM can generate in a single response. Table 2.1 shows the max_output_tokens for several key models, which typically range between 4096 and 16384 tokens. Contrary to what one might expect, the model does not “summarizes the answer” such that it does not surpass max_output_tokens limit. Instead, it will stop once it reaches this limit, even mid-sentence, i.e. the response may be truncated.

      Table 4.2 LLM Application Testing Requirements Matrix
      @@ -289,7 +289,7 @@

      -

      2.2. Problem Statement

      +

      2.2. Problem Statement

      The max_output_tokens limit in LLMs poses a significant challenge for users who need to generate long outputs, as it may result in truncated content and/or incomplete information.

      1. Truncated Content: Users aiming to generate extensive content, such as detailed reports or comprehensive articles, may find their outputs abruptly cut off due to the max_output_tokens limit. This truncation can result in incomplete information and disrupt the flow of the content.

        2. @@ -298,7 +298,7 @@

        -

        2.3. Content Chunking with Contextual Linking

        +

        2.3. Content Chunking with Contextual Linking

        Content chunking with contextual linking is a technique used to manage the max_output_tokens limitation by breaking down long-form content into smaller, manageable chunks. This approach allows the LLM to focus on smaller sections of the input, enabling it to generate more complete and detailed responses for each chunk while maintaining coherence and context across the entire output.

        1. Chunking the Content: The input content is split into smaller chunks. This allows the LLM to process each chunk individually, focusing on generating a complete and detailed response for that specific section of the input.

          2. @@ -309,7 +309,7 @@

          max_output_tokens limitation and generate coherent long-form content without truncation.

          Let’s examine an example implementation of this technique.

          -

          2.3.1. Generating long-form content

          +

          2.3.1. Generating long-form content

          • Goal: Generate a long-form report analyzing a company’s financial statement.

          • Input: A company’s 10K SEC filing.

            • @@ -322,7 +322,7 @@

            Fig. 2.1 illustrates the process we will follow for handling long-form content generation with Large Language Models through “Content Chunking with Contextual Linking.” It shows how input content is first split into manageable chunks using a chunking function (e.g. CharacterTextSplitter with tiktoken tokenizer), then each chunk is processed sequentially while maintaining context from previous chunks. For each chunk, the system updates the context, generates a dynamic prompt with specific parameters, makes a call to the LLM chain, and stores the response. After all chunks are processed, the individual responses are combined with newlines to create the final report, effectively working around the token limit constraints of LLMs while maintaining coherence across the generated content.

            -

            2.3.1.1. Step 1: Chunking the Content

            +

            2.3.1.1. Step 1: Chunking the Content

            There are different methods for chunking, and each of them might be appropriate for different situations. However, we can broadly group chunking strategies in two types:

            • Fixed-size Chunking: This is the most common and straightforward approach to chunking. We simply decide the number of tokens in our chunk and, optionally, whether there should be any overlap between them. In general, we will want to keep some overlap between chunks to make sure that the semantic context doesn’t get lost between chunks. Fixed-sized chunking may be a reasonable path in many common cases. Compared to other forms of chunking, fixed-sized chunking is computationally cheap and simple to use since it doesn’t require the use of any specialied techniques or libraries.

              • @@ -359,7 +359,7 @@

              -

              2.3.1.2. Step 2: Writing the Base Prompt Template

              +

              2.3.1.2. Step 2: Writing the Base Prompt Template

              We will write a base prompt template which will serve as a foundational structure for all chunks, ensuring consistency in the instructions and context provided to the language model. The template includes the following parameters:

              • role: Defines the role or persona the model should assume.

                • @@ -426,7 +426,7 @@

                -

                2.3.1.3. Step 3: Constructing Dynamic Prompt Parameters

                +

                2.3.1.3. Step 3: Constructing Dynamic Prompt Parameters

                Now, we will write a function (get_dynamic_prompt_template) that constructs prompt parameters dynamically for each chunk.

                @@ -479,7 +479,7 @@

                -

                2.3.1.4. Step 4: Generating the Report

                +

                2.3.1.4. Step 4: Generating the Report

                Finally, we will write a function that generates the actual report by calling the LLMChain with the dynamically updated prompt parameters for each chunk and concatenating the results at the end.

                @@ -538,7 +538,7 @@

                -

                2.3.1.5. Example Usage

                +

                2.3.1.5. Example Usage

                # Load the text from sample 10K SEC filing
@@ -606,7 +606,7 @@ 
                
-
                2.3.2. Discussion
                
+
                2.3.2. Discussion
                
 
                Results from the generated report present a few interesting aspects:
                
 
                  
 
                • Coherence: The generated report demonstrates a high level of coherence. The sections are logically structured, and the flow of information is smooth. Each part of the report builds upon the previous sections, providing a comprehensive analysis of Apple Inc.’s financial performance and key risk factors. The use of headings and subheadings helps in maintaining clarity and organization throughout the document.
                  • 
@@ -620,7 +620,7 @@ 
                  
-
                  2.4. Implications
                  
+
                  2.4. Implications
                  
 
                  Implementing context chunking with contextual linking is a practical solution to manage the output size limitations of LLMs. However, this approach comes with its own set of implications that developers must consider.
                  
 
                    
 
                  1. Increased Development Complexity: Implementing strategies to overcome the maximum output token length introduces additional layers of complexity to the application design. It necessitates meticulous management of context across multiple outputs to maintain coherence. Ensuring that each chunk retains the necessary context for the conversation or document can be challenging and often requires advanced logic to handle transitions seamlessly.
                    2. 
@@ -630,7 +630,7 @@ 
                    
-
                    2.5. Future Considerations
                    
+
                    2.5. Future Considerations
                    
 
                    As models evolve, we can expect several advancements that will significantly impact how we handle output size limitations:
                    
 
                      
 
                    1. Contextual Awareness: Future LLMs will likely have improved contextual awareness - or as Mustafa Suleyman would call “infinite memory”, enabling them to better understand and manage the context of a conversation or document over long interactions. This will reduce the need for repetitive context setting and improve the overall user experience.
                      2. 
@@ -642,11 +642,11 @@ 
                      
-
                      2.6. Conclusion
                      
+
                      2.6. Conclusion
                      
 
                      In conclusion, while managing output size limitations in LLMs presents significant challenges, it also drives innovation in application design and optimization strategies. By implementing techniques such as context chunking, efficient prompt templates, and graceful fallbacks, developers can mitigate these limitations and enhance the performance and cost-effectiveness of their applications. As the technology evolves, advancements in contextual awareness, token efficiency, and memory management will further empower developers to build more robust and scalable LLM-powered systems. It is crucial to stay informed about these developments and continuously adapt to leverage the full potential of LLMs while addressing their inherent constraints.
            -

            2.7. References

            +

            2.7. References

            [LangChain24] diff --git a/tamingllms/_build/html/notebooks/structured_output.html b/tamingllms/_build/html/notebooks/structured_output.html index 8c03a27..c2b94de 100644 --- a/tamingllms/_build/html/notebooks/structured_output.html +++ b/tamingllms/_build/html/notebooks/structured_output.html @@ -198,7 +198,7 @@
            -

            3. Wrestling with Structured Output

            +

            3. Wrestling with Structured Output

            In limits, there is freedom. Creativity thrives within structure.

            —Julia B. Cameron

            @@ -206,42 +206,42 @@

            Contents
            -

            3.1. Introduction

            +

            3.1. Introduction

            Large language models (LLMs) excel at generating human-like text, but they often struggle to produce output in a structured format consistently. This poses a significant challenge when we need LLMs to generate data that can be easily processed by other systems, such as databases, APIs, or other software applications. Sometimes, even with a well-crafted prompt, an LLM might produce an unstructured response when a structured one is expected. This can be particularly challenging when integrating LLMs into systems that require specific data formats.

            As a motivating example, consider the following simple task: Given a segment of a SEC financial filing, generate a two-person discussion about the key financial data from the text in JSON format, simulating what would be a real-world discussion about the underlying companies’ disclosed financial information. We would like to generate a structured output that can be easily parsed and integrated with other systems.

            Throughout this notebook, we will consider as input a segment of a sample SEC filing of Apple Inc.

            @@ -347,7 +347,7 @@

            -

            3.2. Problem Statement

            +

            3.2. Problem Statement

            Obtaining structured output from LLMs presents several significant challenges:

            • Inconsistency: LLMs often produce unpredictable results, sometimes generating well-structured output and other times deviating from the expected format.

              • @@ -356,7 +356,7 @@

              -

              3.3. User Needs

              +

              3.3. User Needs

              What user needs drive the demand for LLM output constraints when building LLM-based applications? In a recent work by Google Research [Liu et al., 2024], the authors explore the user need for constraints on the output of large language models, drawing on a survey of 51 industry professionals who use LLMs in their work. These needs can be broadly categorized as follows:

              1. Improving Developer Efficiency and Workflow

                @@ -379,10 +379,10 @@

                -

                3.4. Solutions

                +

                3.4. Solutions

                Several strategies and tools can be employed to address the challenges of structured output from LLMs.

                -

                3.4.1. Strategies

                +

                3.4.1. Strategies

                • Schema Guidance: Providing the LLM with a clear schema or blueprint of the desired output structure helps to constrain its generation and improve consistency. This can be achieved by using tools like Pydantic to define the expected data structure and then using that definition to guide the LLM’s output.

                • Output Parsing: When LLMs don’t natively support structured output, parsing their text output using techniques like regular expressions or dedicated parsing libraries can extract the desired information. For example, you can use regular expressions to extract specific patterns from the LLM’s output, or you can use libraries like Pydantic to parse the output into structured data objects.

                  • @@ -390,9 +390,9 @@

                  -

                  3.4.2. Techniques and Tools

                  +

                  3.4.2. Techniques and Tools

                  -

                  3.4.2.1. One-Shot Prompts

                  +

                  3.4.2.1. One-Shot Prompts

                  In one-shot prompting, you provide a single example of the desired output format within the prompt.

                  @@ -459,7 +459,7 @@

                  -

                  3.4.2.2. Structured Output with Provider-Specific APIs

                  +

                  3.4.2.2. Structured Output with Provider-Specific APIs

                  One-shot prompting is a simple technique that can lead to material improvements in structured output, though may not be sufficient for complex (e.g. nested) structures and / or when the model’s output needs to be restricted to a specific set of options or types.

                  Provider-specific APIs can offer ways to handle those challenges. We will explore two approaches here using OpenAI’s API:

                    @@ -468,7 +468,7 @@

                    -

                    3.4.2.3. JSON Mode

                    +

                    3.4.2.3. JSON Mode

                    JSON mode is a feature provided by most LLM API providers, such as OpenAI, that allows the model to generate output in JSON format. This is particularly useful when you need structured data as a result, such as when parsing the output programmatically or integrating it with other systems that require JSON input. As depicted in Fig. 3.1, JSON mode is implemented by instructing theLLM model to use JSON as response format and optionally defining a target schema.

                    JSON Mode @@ -606,7 +606,7 @@

                    -

                    3.4.3. LangChain

                    +

                    3.4.3. LangChain

                    LangChain is a framework designed to simplify the development of LLM applications. It provider an abstraction layer over many LLM providers, including OpenAI, that offers several tools for parsing structured output.

                    In particular, LangChain offers the with_structured_output method, which can be used with LLMs that support structured output APIs, allowing you to enforce a schema directly within the prompt.

                    @@ -666,7 +666,7 @@

                    .with_structured_output() can be found here.

                  -

                  3.4.4. Outlines

                  +

                  3.4.4. Outlines

                  Outlines [Outlines, 2024] is a library specifically focused on structured text generation from LLMs. Under the hood, Outlines works by adjusting the probability distribution of the model’s output logits - the raw scores from the final layer of the neural network that are normally converted into text tokens. By introducing carefully crafted logit biases, Outlines can guide the model to prefer certain tokens over others, effectively constraining its outputs to a predefined set of valid options.

                  The authors solve the general guided generation problem [Willard and Louf, 2023], which as a consequence solves the problem of structured output generation, in LLMs by introducing an efficient indexing approach that reformulates neural text generation using finite-state machines (FSMs).

                  They define the next token generation as a random variable:

                  @@ -699,7 +699,7 @@

                  \(\tilde{s}_{t+1}\) is the next token sampled under constraints

                This formulation allows the masking operation to guide the generation process by zeroing out probabilities of invalid tokens according to the finite state machine states. But instead of checking the entire vocabulary (size N) at each generation step (O(N) complexity) to enforce output constraints, they convert constraints (regex/grammar) into FSM states and build an index mapping FSM states to valid vocabulary tokens. This achieves O(1) average complexity for token generation.

                -

                In summary, there are two stages in the Outlines framework [Tran-Thien, 2024]:

                +

                In summary, there are two stages in the Outlines framework [Tran-Thien, 2024]:

                1. Preprocessing Step: Outlines converts a character-level deterministic finite automaton (DFA) testing whether a string matches a regex into a token-level DFA testing whether a token sequence is decoded in a string matching the regex.

                2. Decoding Step: At decoding time, the DFA is used to determine, for each new token, which potential tokens are allowed. Starting from the initial state of the DFA, the allowed tokens are determined by the outgoing transitions from the current state. The corresponding mask is applied to the next token probabilities and these probabilities are renormalized. A new token can then be sampled and the state of the DFA updated.

                  3. @@ -812,7 +812,7 @@

                  -

                  3.4.5. Ollama

                  +

                  3.4.5. Ollama

                  Ollama is a popular tool that allows you to run large language models (LLMs) locally. It has recently added support for structured output generation. The current ollama implementation leverages llama.cpp GBNF (GGML BNF) grammars [Ggerganov, 2024] to enable structured output generation.

                  llama.cpp GBNF forces language models to generate output in specific, predefined formats by constraining their outputs to follow precise rules and patterns. The system accomplishes this through a formal grammar specification that defines exactly how valid outputs can be constructed. It’s essentially an extension of BNF (Backus-Naur Form) [Wikipedia contributors, 2024] with some modern regex-like features added. These rules carefully define what elements are allowed, how they can be combined, and what patterns of repetition and sequencing are valid. By enforcing these constraints during generation, GBNF ensures the model’s output strictly adheres to the desired format.

                  Ollama first introduced structured output generation in version 0.5.1 providing support for JSON output but highlighting additional formats are coming soon.

                  @@ -910,9 +910,9 @@

                  -

                  3.5. Discussion

                  +

                  3.5. Discussion

                  -

                  3.5.1. Comparing Solutions

                  +

                  3.5.1. Comparing Solutions

                  The choice of framework for structured LLM output depends heavily on specific constraints, requirements and use cases. LangChain is the most used LLM framework today with a large developer community base however its structured output support depends on the underlying LLM provider support. Ollama enables straightforward local deployment and experimentation democratizing access to LLMs while fostering privacy and control, however today it only offers JSON format with further formats to come. Outlines emerges as a solution with great flexibility and control over output structure while providing support for a wide range of LLMs. Table 3.1 provides a summary comparison of the different frameworks.

      @@ -958,7 +958,7 @@

      -

      3.5.2. Best Practices

      +

      3.5.2. Best Practices

      • Clear Schema Definition: Define the desired output structure clearly. This can be done in several ways including schemas, types, or Pydantic models as appropriate. This ensures the LLM knows exactly what format is expected.

      • Descriptive Naming: Use meaningful names for fields and elements in your schema. This makes the output more understandable and easier to work with.

        • @@ -967,7 +967,7 @@

        -

        3.5.3. Research and Ongoing Debate

        +

        3.5.3. Research and Ongoing Debate

        The use of structured output for Large Language Models (LLMs) is a developing area. While the ability to constrain LLM outputs offer clear benefits in parsing, robustness, and integration, there is growing debate on whether it also potentially comes at the cost of performance as well as reasoning abilities. Research in this area should be taken with a grain of salt since findings are mixed and often depend on the specific task and model family at hand furthermore model families are not always comparable and are getting updated by the day! Nonetheless, early findings provide some interesting insights as to why there is no one-size-fits-all solution when it comes to LLMs structured output.

        There is some evidence indicating that LLMs may have bias in their handling of different output formats [Long et al., 2024]. The study examined common output structures like multiple-choice answers, wrapped text, lists, and key-value mappings. The authors analyzed key LLM model families, namely Gemma, Mistral, and ChatGPT, uncovering bias across multiple tasks and formats. The researchers attributed these biases to the models’ underlying token distributions for different formats. An example of this format bias emerged in the comparison between JSON and YAML outputs. While models like Mistral and Gemma excelled at generating JSON structures, they performed notably worse with YAML. Their YAML outputs often contained extraneous information that degrades output quality. This disparity likely stems from JSON’s prevalence in training data, highlighting how a format’s popularity directly influences model performance. While the studied models can be probably considered outdated by now since models are getting updated on a rapidly fashion, it is important to remark that addressing format bias is critical for advancing LLMs and ensuring their reliable application in real-world scenarios.

        Recent research “Let Me Speak Freely? A Study on the Impact of Format Restrictions on Performance of Large Language Models” [Tam et al., 2024] suggests that imposing format restrictions on LLMs might impact their performance, particularly in reasoning-intensive tasks. Further evidence [Aider, 2024] suggests LLMs may produce lower quality code if they’re asked to return it as part of a structured JSON response, in particular:

        @@ -997,15 +997,15 @@

        -

        3.6. Conclusion

        +

        3.6. Conclusion

        Extracting structured output from LLMs is crucial for integrating them into real-world applications. By understanding the challenges and employing appropriate strategies and tools, developers can improve the reliability and usability of LLM-powered systems, unlocking their potential to automate complex tasks and generate valuable insights.

        -

        3.7. Acknowledgements

        +

        3.7. Acknowledgements

        We would like to thank Cameron Pfiffer from the .txt team for his insightful review and feedback.

        -

        3.8. References

        +

        3.8. References

        [Aid24] @@ -1038,7 +1038,7 @@

        [TT24] -

        Vivien Tran-Thien. Llm decoding with regex constraints. Blog post, 2024. URL: https://vivien000.github.io/blog/journal/llm-decoding-with-regex-constraints.html.

        +

        Vivien Tran-Thien. Fast, high-fidelity llm decoding with regex constraints. 2024. URL: https://vivien000.github.io/blog/journal/llm-decoding-with-regex-constraints.html.

        [WL23] diff --git a/tamingllms/_build/html/objects.inv b/tamingllms/_build/html/objects.inv index 2d912e9..9ee9c63 100644 Binary files a/tamingllms/_build/html/objects.inv and b/tamingllms/_build/html/objects.inv differ diff --git a/tamingllms/_build/html/searchindex.js b/tamingllms/_build/html/searchindex.js index 7ddb3ed..6e9a846 100644 --- a/tamingllms/_build/html/searchindex.js +++ b/tamingllms/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"docnames": ["markdown/intro", "markdown/toc", "notebooks/evals", "notebooks/output_size_limit", "notebooks/structured_output"], "filenames": ["markdown/intro.md", "markdown/toc.md", "notebooks/evals.ipynb", "notebooks/output_size_limit.ipynb", "notebooks/structured_output.ipynb"], "titles": ["1. Introduction", "Taming LLMs", "4. The Evals Gap", "2. Output Size Limitations", "3. Wrestling with Structured Output"], "terms": {"am": 0, "alwai": [0, 2, 4], "do": [0, 2, 3, 4], "which": [0, 2, 3, 4], "cannot": [0, 2], "order": [0, 2, 4], "mai": [0, 2, 3, 4], "learn": [0, 2], "how": [0, 2, 3, 4], "pablo": [0, 2], "picasso": 0, "In": [0, 2, 3, 4], "recent": [0, 2, 4], "year": [0, 2, 3, 4], "larg": [0, 1, 2, 3, 4], "languag": [0, 1, 2, 3, 4], "model": [0, 1, 4], "llm": [0, 3, 4], "have": [0, 2, 3, 4], "emerg": [0, 1, 4], "transform": [0, 2, 4], "forc": [0, 2, 4], "technologi": [0, 2, 3, 4], "promis": [0, 2], "revolution": 0, "build": [0, 1, 2, 3, 4], "product": [0, 1, 2, 4], "interact": [0, 2, 3, 4], "comput": [0, 2, 3, 4], "from": [0, 2, 3, 4], "chatgpt": [0, 4], "github": [0, 2, 4], "copilot": 0, "claud": [0, 2, 3], "artifact": 0, "system": [0, 2, 3, 4], "captur": [0, 2], "public": [0, 2], "imagin": 0, "spark": 0, "gold": [0, 2], "rush": 0, "ai": [0, 2, 4], "power": [0, 1, 2, 3, 4], "applic": [0, 1, 3, 4], "howev": [0, 2, 3, 4], "beneath": 0, "surfac": [0, 2], "technolog": [0, 2], "revolut": 0, "li": [0, 2], "complex": [0, 2, 3, 4], "landscap": [0, 2], "practition": [0, 2], "must": [0, 2, 3], "navig": [0, 1, 2], "focus": [0, 2, 3, 4], "bring": 0, "awar": [0, 2, 3], "limit": [0, 2, 4], "har": [0, 1, 3], "open": [0, 2, 3, 4], "sourc": [0, 2, 4], "solut": [0, 1, 2, 3], "overcom": [0, 3], "them": [0, 2, 3, 4], "robust": [0, 2, 3, 4], "It": [0, 2, 3, 4], "offer": [0, 2, 3, 4], "critic": [0, 1, 2, 3, 4], "implement": [0, 1, 2, 3, 4], "back": [0, 2, 4], "reproduc": [0, 1, 2], "exampl": [0, 1, 2, 4], "while": [0, 1, 2, 3, 4], "mani": [0, 2, 3, 4], "resourc": [0, 2, 3], "cover": [0, 2, 3], "capabl": [0, 1, 2, 3, 4], "specif": [0, 1, 2, 3], "hidden": 0, "pitfal": 0, "engin": [0, 1, 2, 4], "technic": [0, 1, 2, 3, 4], "manag": [0, 1, 2, 3, 4], "face": [0, 2], "when": [0, 1, 2, 3, 4], "comprehens": [0, 1, 2, 3, 4], "guid": [0, 2, 4], "leverag": [0, 2, 3, 4], "battl": [0, 1], "test": [0, 1, 4], "tool": [0, 3], "throughout": [0, 2, 3, 4], "tackl": [0, 2], "follow": [0, 2, 3, 4], "non": [0, 1, 4], "exhaust": 0, "list": [0, 2, 3, 4], "structur": [0, 2, 3], "un": 0, "reliabl": [0, 2, 4], "struggl": [0, 2, 4], "maintain": [0, 2, 3, 4], "consist": [0, 2, 3, 4], "output": [0, 2], "format": [0, 2, 3, 4], "complic": 0, "integr": [0, 2, 4], "larger": [0, 2, 3, 4], "make": [0, 2, 3, 4], "error": [0, 2, 4], "handl": [0, 1, 2, 3, 4], "more": [0, 2, 3, 4], "size": [0, 2, 4], "length": [0, 2, 4], "constraint": [0, 1, 2, 3, 4], "strict": [0, 4], "token": [0, 1, 2, 4], "both": [0, 2], "input": [0, 2, 3, 4], "requir": [0, 3, 4], "care": [0, 2, 4], "chunk": [0, 1], "strategi": [0, 1, 2, 3], "long": [0, 1, 2, 4], "form": [0, 1, 2, 4], "effect": [0, 2, 3, 4], "tradit": 0, "softwar": [0, 4], "methodologi": [0, 2, 4], "break": [0, 2, 3], "down": [0, 2, 3], "deal": 0, "determinist": [0, 1, 4], "gener": [0, 1, 4], "new": [0, 2, 3, 4], "hallucin": [0, 2, 4], "These": [0, 2, 3, 4], "can": [0, 2, 3, 4], "plausibl": 0, "sound": 0, "entir": [0, 2, 3, 4], "fabric": [0, 2], "inform": [0, 2, 3, 4], "creat": [0, 2, 3, 4], "signific": [0, 2, 3, 4], "risk": [0, 2, 3], "safeti": [0, 2, 4], "secur": [0, 2, 3, 4], "harm": [0, 2], "bias": [0, 2, 4], "inappropri": 0, "safeguard": [0, 2], "monitor": [0, 1, 2], "ensur": [0, 2, 3, 4], "safe": [0, 2, 4], "deploy": [0, 1, 2, 4], "cost": [0, 2, 4], "optim": [0, 1, 2, 3], "The": [0, 3, 4], "financi": [0, 2, 3, 4], "oper": [0, 2, 3, 4], "base": [0, 1, 4], "quickli": [0, 3], "becom": [0, 2, 4], "prohibit": [0, 2], "without": [0, 2, 3, 4], "observ": [0, 2, 4], "vendor": [0, 1, 2], "lock": [0, 1], "cloud": [0, 2, 4], "provid": [0, 2, 3], "depend": [0, 2, 4], "through": [0, 1, 2, 3, 4], "proprietari": [0, 4], "infrastructur": 0, "difficult": [0, 2], "switch": 0, "self": [0, 1, 2], "host": [0, 1, 2], "take": [0, 1, 2, 3, 4], "hand": [0, 3, 4], "concret": [0, 1], "you": [0, 2, 3, 4], "run": [0, 2, 4], "modifi": [0, 2], "real": [0, 2, 3, 4], "world": [0, 2, 4], "scenario": [0, 2, 4], "best": [0, 1, 2], "techniqu": [0, 1, 2, 3], "pattern": [0, 1, 2, 4], "anti": [0, 2], "look": [0, 1, 2], "our": [0, 2, 3, 4], "goal": [0, 2, 3], "discourag": 0, "us": [0, 3, 4], "enabl": [0, 2, 3, 4], "By": [0, 1, 2, 3, 4], "understand": [0, 1, 2, 3, 4], "upfront": [0, 1], "better": [0, 1, 2, 3], "equip": [0, 1, 2], "avoid": [0, 2, 4], "current": [0, 1, 2, 3, 4], "discours": [0, 1], "around": [0, 1, 2, 3, 4], "tend": [0, 1, 2], "toward": [0, 2, 4], "extrem": [0, 2], "either": [0, 2, 3], "uncrit": 0, "enthusiasm": 0, "wholesal": [0, 2], "dismiss": 0, "differ": [0, 2, 3, 4], "focu": [0, 1, 2, 3, 4], "rather": [0, 2], "than": [0, 2], "theoret": 0, "examin": [0, 2, 3, 4], "first": [0, 2, 3, 4], "everi": [0, 2], "concept": [0, 2], "illustr": [0, 2, 3, 4], "execut": [0, 2], "immedi": [0, 2], "analysi": [0, 1, 2, 3], "balanc": [0, 2, 3, 4], "help": [0, 2, 3, 4], "reader": [0, 1], "decis": [0, 2, 4], "intend": [0, 2], "develop": [0, 2, 3, 4], "step": [0, 1, 2, 4], "insight": [0, 2, 3, 4], "along": [0, 2], "guidanc": [0, 4], "framework": [0, 2], "could": [0, 2, 3, 4], "derail": 0, "project": [0, 2], "earli": [0, 2, 4], "befor": [0, 2, 4], "thei": [0, 2, 3, 4], "costli": [0, 2], "problem": [0, 1], "too": [0, 2, 3], "late": 0, "lifecycl": 0, "design": [0, 1, 3, 4], "lead": [0, 2, 3, 4], "genai": 0, "initi": [0, 2, 3, 4], "leader": [0, 2], "architectur": [0, 2, 3, 4], "advoc": 0, "anyon": 0, "seek": [0, 2], "work": [0, 1, 2, 3, 4], "typic": [0, 2, 3, 4], "job": [0, 2], "role": [0, 2, 3, 4], "platform": [0, 2, 3, 4], "backend": [0, 2], "exist": [0, 2], "ml": 0, "transit": [0, 2, 3, 4], "overse": 0, "motiv": [0, 2, 4], "need": [0, 2, 3], "readi": [0, 2], "desir": [0, 2, 4], "perform": [0, 1, 2, 3, 4], "after": [0, 2, 3, 4], "read": [0, 2, 3, 4], "implic": [0, 1, 2], "experi": [0, 2, 3, 4], "recommend": [0, 2, 3, 4], "abl": [0, 2, 3, 4], "deploi": [0, 2, 3], "proper": [0, 4], "realist": 0, "effort": [0, 2, 4], "estim": [0, 2], "impact": [0, 2, 3, 4], "timelin": 0, "To": [0, 2, 3, 4], "most": [0, 2, 3, 4], "should": [0, 2, 3, 4], "basic": [0, 2, 3], "program": [0, 2], "knowledg": [0, 2], "introductori": [0, 1], "langchain": [0, 1, 2, 3], "e": [0, 2, 3, 4], "g": [0, 2, 3, 4], "chat": [0, 2, 3, 4], "prompt": [0, 1, 2], "templat": [0, 1, 2], "access": [0, 2, 3, 4], "openai": [0, 2, 4], "anthrop": [0, 4], "similar": [0, 2, 4], "grade": 0, "dive": 0, "here": [0, 2, 3, 4], "get": [0, 2, 3, 4], "start": [0, 2, 4], "activ": [0, 2], "virtual": [0, 2], "m": [0, 2, 4], "venv": [0, 2], "env": [0, 2, 3, 4], "bin": 0, "On": [0, 2, 4], "window": [0, 1, 2], "script": 0, "instal": [0, 2, 4], "packag": [0, 2, 4], "pip": [0, 2, 4], "r": [0, 2, 3, 4], "txt": [0, 2, 3, 4], "file": [0, 2, 3, 4], "root": 0, "directori": [0, 2], "add": [0, 3], "other": [0, 2, 3, 4], "sensit": [0, 2], "openai_api_kei": 0, "your_openai_api_key_her": 0, "never": [0, 4], "share": [0, 2, 4], "commit": [0, 2], "version": [0, 2, 4], "control": [0, 2, 4], "contain": [0, 2, 3, 4], "kept": [0, 2], "privat": [0, 2], "clone": 0, "companion": 0, "git": 0, "http": [0, 2, 3, 4], "com": [0, 2, 3, 4], "souzatharsi": 0, "tamingllm": [0, 2], "cd": 0, "If": [0, 2, 4], "encount": [0, 1, 2], "rate": [0, 2], "consid": [0, 2, 3, 4], "smaller": [0, 2, 3, 4], "retri": [0, 4], "logic": [0, 2, 3], "conflict": [0, 2], "try": [0, 2, 4], "fresh": 0, "like": [0, 2, 3, 4], "poetri": 0, "check": [0, 2, 4], "page": [0, 2], "known": [0, 2, 4], "now": [0, 2, 3, 4], "let": [0, 2, 3, 4], "begin": [0, 2, 4], "explor": [0, 2, 4], "dr": 0, "tharsi": 0, "souza": 0, "scientist": 0, "special": [0, 2, 4], "he": [0, 2], "lectur": 0, "columbia": 0, "univers": [0, 2], "master": [0, 4], "scienc": [0, 2], "appli": [0, 2, 3, 4], "analyt": 0, "head": [0, 2, 3], "equiti": [0, 2], "citadel": 0, "former": [0, 2], "senior": [0, 2], "vp": 0, "two": [0, 2, 3, 4], "sigma": 0, "invest": [0, 2, 4], "With": [0, 2], "over": [0, 1, 2, 3, 4], "15": [0, 2, 4], "deliv": [0, 2], "across": [0, 2, 4], "startup": 0, "fortun": 0, "500": [0, 2], "compani": [0, 2, 3, 4], "global": [0, 2], "also": [0, 2, 3, 4], "an": [0, 1, 2, 3, 4], "numer": [0, 2], "scholarli": 0, "frequent": [0, 2, 4], "speaker": [0, 2], "academ": [0, 2], "busi": [0, 2], "confer": [0, 4], "ground": [0, 1, 2], "background": [0, 2, 3], "draw": [0, 2, 4], "scale": [0, 2, 4], "stage": [0, 4], "major": [0, 2, 4], "institut": [0, 2], "well": [0, 2, 4], "advis": 0, "profit": [0, 2, 3, 4], "organ": [0, 2, 3], "contribut": [0, 2, 3], "uniqu": [0, 2], "bridg": 0, "gap": 0, "between": [0, 2, 3, 4], "potenti": [0, 2, 3, 4], "next": [0, 2, 4], "hold": [0, 2], "ph": 0, "d": [0, 2, 4], "ucl": 0, "london": 0, "phil": 0, "sc": 0, "b": [0, 2, 4], "abstract": [1, 2, 4], "heavili": [1, 2, 4], "gloss": 1, "fundament": [1, 2, 4], "challeng": [1, 2, 3, 4], "convers": [1, 2, 3, 4], "thi": [1, 2, 3, 4], "book": [1, 2], "kei": [1, 4], "python": [1, 2, 3, 4], "proven": 1, "yet": [1, 2, 3], "i": [1, 2, 3, 4], "unstructur": [1, 4], "context": [1, 2, 3, 4], "code": [1, 2, 4], "sidestep": 1, "inher": [1, 2, 3, 4], "core": [1, 2], "we": [1, 2, 3, 4], "ll": [1, 2], "address": [1, 2, 3, 4], "approach": [1, 2, 3, 4], "note": [1, 2, 3, 4], "perspect": 1, "who": [1, 2, 3, 4], "For": [1, 2, 3, 4], "outcom": [1, 2, 4], "prerequisit": 1, "set": [1, 2, 3, 4], "up": [1, 2, 3, 4], "your": [1, 2, 3, 4], "environ": [1, 2, 3, 4], "setup": [1, 2, 4], "api": [1, 2], "configur": [1, 2], "repositori": [1, 2], "troubleshoot": 1, "common": [1, 2, 3, 4], "issu": [1, 2, 3, 4], "about": [1, 2, 3, 4], "author": [1, 2, 4], "": [1, 2, 3, 4], "statement": 1, "One": [1, 2], "shot": [1, 2], "json": [1, 2, 3], "mode": 1, "outlin": [1, 2], "multipl": [1, 2, 3, 4], "choic": [1, 2, 4], "pydant": [1, 2, 4], "discuss": [1, 2], "compar": [1, 2, 3], "research": [1, 2, 3], "ongo": [1, 2], "debat": 1, "conclus": [1, 2], "acknowledg": [1, 2], "refer": 1, "content": 1, "what": [1, 2, 4], "ar": [1, 2, 4], "contextu": [1, 2], "link": [1, 2], "write": [1, 2, 4], "construct": [1, 2, 4], "dynam": [1, 2], "paramet": [1, 2, 4], "report": [1, 2, 4], "usag": [1, 2, 4], "futur": [1, 2], "consider": [1, 4], "machin": [1, 4], "temperatur": [1, 3, 4], "sampl": [1, 3, 4], "spectrum": 1, "properti": 1, "conceptu": [1, 4], "overview": [1, 4], "compon": [1, 2], "metric": 1, "evalu": [1, 3, 4], "human": [1, 3, 4], "benchmark": 1, "leaderboard": 1, "type": [1, 2, 3, 4], "detect": [1, 2, 4], "retriev": [1, 2], "augment": [1, 2], "rag": 1, "select": [1, 2], "index": [1, 2, 3, 4], "vector": 1, "store": [1, 2, 3], "method": [1, 2, 3, 4], "pipelin": [1, 2, 4], "valid": [1, 2, 4], "guard": 1, "filter": [1, 2, 4], "sanit": 1, "alert": 1, "cach": [1, 2], "invalid": [1, 4], "predict": [1, 2, 4], "llama": [1, 2, 4], "llamafil": 1, "ollama": 1, "migrat": 1, "commun": [1, 2, 4], "doesn": [2, 3, 4], "t": [2, 3, 4], "matter": 2, "beauti": 2, "theori": 2, "smart": 2, "agre": 2, "wrong": 2, "richard": 2, "feynman": 2, "natur": [2, 3, 4], "unlik": 2, "where": [2, 3, 4], "same": [2, 3, 4], "produc": [2, 4], "novel": 2, "text": [2, 3, 4], "train": [2, 4], "data": [2, 3, 4], "respons": [2, 3, 4], "each": [2, 3, 4], "time": [2, 3, 4], "re": [2, 3, 4], "queri": 2, "even": [2, 3, 4], "ident": 2, "behavior": 2, "strength": 2, "ask": [2, 4], "question": [2, 4], "isn": 2, "bug": 2, "featur": [2, 4], "random": [2, 4], "allow": [2, 3, 4], "creativ": [2, 4], "divers": [2, 3, 4], "testabl": 2, "servic": [2, 3, 4], "advic": 2, "mean": [2, 3, 4], "yield": 2, "exceedingli": 2, "regulatori": 2, "complianc": [2, 4], "guarante": [2, 4], "user": [2, 3], "trust": [2, 4], "affect": 2, "inconsist": [2, 4], "primari": 2, "determin": [2, 3, 4], "come": [2, 3, 4], "dure": [2, 4], "calcul": 2, "probabl": [2, 4], "distribut": [2, 4], "nucleu": 2, "holtzman": 2, "et": [2, 4], "al": [2, 4], "2020": 2, "top": [2, 4], "k": [2, 3, 4], "coher": [2, 3], "0": [2, 3, 4], "repetit": [2, 3, 4], "1": [2, 4], "increas": [2, 3, 4], "incoher": 2, "dotenv": [2, 3, 4], "import": [2, 3, 4], "load_dotenv": [2, 3, 4], "o": [2, 3, 4], "load": [2, 3, 4], "variabl": [2, 3, 4], "panda": 2, "pd": 2, "def": [2, 3, 4], "generate_respons": 2, "model_nam": [2, 3], "str": [2, 3, 4], "float": [2, 3], "attempt": [2, 3], "int": [2, 3], "3": [2, 4], "datafram": 2, "demonstr": [2, 3, 4], "client": [2, 4], "result": [2, 3, 4], "temp": 2, "rang": [2, 3, 4], "complet": [2, 3, 4], "messag": [2, 4], "max_token": 2, "50": 2, "append": [2, 3, 4], "displai": [2, 4], "group": [2, 3], "df_result": 2, "print": [2, 3, 4], "f": [2, 3, 4], "ntemperatur": 2, "40": 2, "temp_respons": 2, "_": [2, 4], "row": 2, "iterrow": 2, "return": [2, 3, 4], "max_length": [2, 4], "10000": [2, 3, 4], "appl": [2, 3, 4], "sec_fil": [2, 4], "unit": [2, 3, 4], "state": [2, 3, 4], "nsecur": 2, "AND": [2, 4], "exchang": [2, 3, 4], "commiss": [2, 3, 4], "nwashington": 2, "c": [2, 4], "20549": 2, "n": [2, 3, 4], "nform": 2, "10": [2, 3, 4], "mark": 2, "annual": 2, "pursuant": 2, "TO": 2, "section": [2, 3, 4], "13": 2, "OR": 2, "OF": 2, "THE": 2, "act": 2, "1934": 2, "nfor": 2, "fiscal": [2, 3], "end": [2, 3, 4], "septemb": [2, 3], "28": [2, 3], "2024": [2, 3, 4], "nor": 2, "period": [2, 3], "ncommiss": 2, "number": [2, 3, 4], "001": 2, "36743": 2, "ng66145g66i43": 2, "jpg": 2, "nappl": 2, "inc": [2, 3, 4], "exact": 2, "name": [2, 3, 4], "registr": 2, "specifi": [2, 3, 4], "its": [2, 3, 4], "charter": 2, "ncalifornia": 2, "t94": 2, "2404110": 2, "jurisdict": 2, "nof": 2, "incorpor": 2, "employ": 2, "identif": 2, "No": [2, 4], "none": 2, "park": 2, "wai": [2, 3, 4], "ncupertino": 2, "california": [2, 4], "n95014": 2, "princip": 2, "offic": 2, "zip": 2, "408": 2, "996": 2, "1010": 2, "telephon": 2, "includ": [2, 3, 4], "area": [2, 4], "regist": 2, "12": [2, 3], "ntitl": 2, "class": [2, 3, 4], "ttrade": 2, "symbol": 2, "tname": 2, "ncommon": 2, "stock": [2, 4], "00001": 2, "par": 2, "valu": [2, 3, 4], "per": [2, 3], "naapl": 2, "tthe": 2, "nasdaq": [2, 4], "market": [2, 3, 4], "llc": [2, 4], "n0": 2, "000": [2, 4], "due": [2, 3], "2025": 2, "875": 2, "n1": 2, "625": 2, "2026": 2, "n2": 2, "2027": 2, "375": 2, "2029": 2, "n3": 2, "050": 2, "2031": 2, "600": 2, "2042": 2, "nindic": 2, "season": 2, "issuer": 2, "defin": [2, 3, 4], "rule": [2, 3, 4], "405": 2, "nye": 2, "whether": [2, 3, 4], "ha": [2, 4], "all": [2, 3, 4], "preced": 2, "month": 2, "shorter": 2, "wa": [2, 4], "2": [2, 4], "been": 2, "subject": 2, "past": 2, "90": 2, "dai": [2, 4], "submit": 2, "electron": 2, "regul": [2, 4], "232": 2, "chapter": 2, "acceler": 2, "filer": 2, "growth": 2, "see": [2, 4], "definit": [2, 4], "12b": 2, "nlarg": 2, "tacceler": 2, "nnon": 2, "tsmaller": 2, "nemerg": 2, "nif": 2, "indic": [2, 4], "elect": 2, "extend": [2, 4], "compli": [2, 4], "ani": [2, 3, 4], "revis": 2, "account": 2, "standard": 2, "attest": 2, "assess": [2, 3], "intern": 2, "under": [2, 4], "404": 2, "sarban": 2, "oxlei": 2, "u": [2, 4], "7262": 2, "firm": 2, "prepar": [2, 3], "audit": 2, "reflect": 2, "correct": [2, 4], "previous": [2, 3, 4], "those": [2, 3, 4], "restat": 2, "recoveri": 2, "incent": 2, "compens": 2, "receiv": [2, 3, 4], "relev": 2, "240": 2, "10d": 2, "shell": 2, "nthe": 2, "aggreg": 2, "vote": 2, "held": [2, 4], "affili": [2, 4], "march": [2, 4], "29": [2, 4], "last": [2, 3, 4], "second": [2, 3], "quarter": 2, "approxim": [2, 4], "628": [2, 4], "553": [2, 4], "sole": 2, "purpos": [2, 4], "disclosur": 2, "director": 2, "date": [2, 4], "exclud": 2, "becaus": 2, "person": [2, 4], "deem": 2, "necessarili": 2, "n15": 2, "115": [2, 4], "823": [2, 4], "were": [2, 4], "outstand": [2, 4], "octob": [2, 4], "18": [2, 4], "ndocument": 2, "BY": 2, "nportion": 2, "proxi": 2, "relat": 2, "meet": [2, 4], "sharehold": 2, "part": [2, 3, 4], "iii": 2, "within": [2, 3, 4], "120": 2, "ntabl": 2, "npage": 2, "npart": 2, "nitem": 2, "nbusi": 2, "1a": 2, "nrisk": 2, "factor": [2, 3, 4], "n5": 2, "1b": 2, "nunresolv": 2, "staff": 2, "comment": 2, "n17": 2, "1c": 2, "ncybersecur": 2, "nproperti": 2, "n18": 2, "nlegal": 2, "proceed": 2, "4": [2, 4], "nmine": 2, "ii": [2, 4], "5": [2, 3, 4], "nmarket": 2, "stockhold": 2, "purchas": 2, "n19": 2, "6": [2, 3, 4], "reserv": 2, "n20": 2, "7": [2, 3], "nmanag": 2, "condit": 2, "n21": 2, "7a": 2, "nquantit": 2, "qualit": 2, "n27": 2, "8": [2, 3], "nfinanci": 2, "supplementari": 2, "n28": 2, "9": 2, "nchang": 2, "disagr": 2, "n51": 2, "9a": 2, "ncontrol": 2, "procedur": 2, "9b": 2, "nother": 2, "n52": 2, "9c": 2, "ndisclosur": 2, "regard": 2, "foreign": 2, "prevent": [2, 4], "inspect": 2, "ndirector": 2, "corpor": 2, "govern": 2, "11": 2, "nexecut": 2, "ownership": 2, "certain": [2, 3, 4], "benefici": 2, "owner": 2, "ncertain": 2, "relationship": 2, "transact": 2, "independ": [2, 4], "14": [2, 4], "nprincip": 2, "fee": 2, "iv": 2, "nexhibit": 2, "schedul": 2, "n53": 2, "16": 2, "summari": [2, 4], "n56": 2, "nthi": 2, "forward": 2, "litig": 2, "reform": 2, "1995": 2, "involv": [2, 4], "uncertainti": 2, "locat": 2, "item": 2, "expect": [2, 3, 4], "event": 2, "assumpt": 2, "doe": [2, 3, 4], "directli": [2, 4], "histor": 2, "fact": 2, "macroeconom": 2, "identifi": [2, 3, 4], "word": [2, 3, 4], "anticip": 2, "believ": [2, 4], "plan": [2, 4], "would": [2, 3, 4], "term": [2, 3], "actual": [2, 3, 4], "significantli": [2, 3], "might": [2, 3, 4], "caus": 2, "assum": [2, 3], "oblig": [2, 3], "updat": [2, 3, 4], "reason": [2, 3, 4], "except": [2, 4], "law": 2, "nunless": 2, "otherwis": 2, "present": [2, 3, 4], "herein": 2, "calendar": 2, "particular": [2, 4], "associ": [2, 3, 4], "collect": [2, 3], "wholli": 2, "own": [2, 3], "subsidiari": 2, "unless": 2, "ncompani": 2, "manufactur": 2, "smartphon": 2, "tablet": 2, "wearabl": [2, 4], "accessori": 2, "sell": 2, "varieti": 2, "52": 2, "53": 2, "week": 2, "saturdai": 2, "nproduct": 2, "niphon": 2, "line": 2, "io": [2, 4], "iphon": [2, 4], "pro": [2, 3], "se": 2, "nmac": 2, "maco": 2, "mac": [2, 4], "laptop": 2, "macbook": 2, "air": 2, "desktop": 2, "imac": 2, "mini": [2, 3, 4], "studio": 2, "nipad": 2, "multipurpos": 2, "ipado": 2, "ipad": [2, 4], "nwearabl": 2, "home": 2, "smartwatch": 2, "wireless": 2, "headphon": 2, "spatial": 2, "watcho": 2, "watch": 2, "ultra": 2, "seri": 2, "airpod": 2, "max": 2, "beat": 2, "vision": 2, "visiono": 2, "nhome": 2, "tv": 2, "media": 2, "stream": [2, 4], "game": 2, "devic": [2, 4], "tvo": 2, "homepod": 2, "high": [2, 3], "fidel": 2, "naccessori": 2, "brand": 2, "third": 2, "parti": 2, "nservic": 2, "nadvertis": 2, "advertis": 2, "licens": 2, "arrang": 2, "napplecar": 2, "portfolio": [2, 4], "support": [2, 4], "applecar": 2, "prioriti": 2, "network": [2, 4], "repair": 2, "replac": 2, "case": [2, 3, 4], "addit": [2, 3, 4], "coverag": 2, "instanc": [2, 3], "accident": 2, "damag": 2, "theft": 2, "loss": 2, "countri": 2, "ncloud": 2, "keep": [2, 3], "custom": 2, "avail": [2, 3, 4], "ndigit": 2, "variou": [2, 3, 4], "app": 2, "discov": 2, "download": 2, "digit": 2, "music": 2, "video": 2, "podcast": 2, "subscript": 2, "arcad": 2, "fit": [2, 3, 4], "sm": 2, "curat": 2, "listen": 2, "demand": [2, 4], "radio": 2, "station": 2, "magazin": 2, "exclus": 2, "origin": [2, 3, 4], "live": 2, "sport": 2, "npayment": 2, "payment": 2, "card": 2, "co": 2, "credit": 2, "pai": 2, "cashless": 2, "nsegment": 2, "primarili": 2, "geograph": 2, "basi": 2, "segment": [2, 3, 4], "america": 2, "europ": 2, "greater": 2, "china": 2, "japan": 2, "rest": 2, "asia": 2, "pacif": 2, "north": 2, "south": 2, "european": 2, "india": 2, "middl": 2, "east": 2, "africa": 2, "mainland": 2, "hong": 2, "kong": 2, "taiwan": 2, "australia": 2, "asian": 2, "although": 2, "hardwar": 2, "one": [2, 3, 4], "separ": [2, 3], "align": [2, 3, 4], "partner": 2, "region": 2, "consum": [2, 4], "small": [2, 4], "mid": [2, 3], "educ": [2, 3], "enterpris": [2, 4], "resel": 2, "retail": 2, "onlin": 2, "direct": 2, "sale": 2, "emploi": [2, 4], "indirect": 2, "channel": 2, "cellular": 2, "carrier": 2, "net": [2, 4], "38": 2, "62": 2, "respect": 2, "total": [2, 3, 4], "ncompetit": 2, "highli": [2, 4], "competit": 2, "character": 2, "aggress": 2, "price": 2, "downward": 2, "pressur": 2, "gross": 2, "margin": [2, 4], "introduct": [2, 3], "short": [2, 3, 4], "life": 2, "cycl": 2, "evolv": [2, 3], "industri": [2, 4], "continu": [2, 3, 4], "improv": [2, 3, 4], "characterist": 2, "rapid": 2, "adopt": [2, 4], "advanc": [2, 3, 4], "competitor": 2, "compet": 2, "veri": 2, "low": [2, 4], "imit": 2, "infring": 2, "intellectu": 2, "abil": [2, 4], "successfulli": [2, 4], "innov": [2, 3], "marketplac": 2, "nearli": 2, "rel": 2, "qualiti": [2, 3, 4], "strong": [2, 4], "ecosystem": 2, "reput": 2, "expand": 2, "opportun": 2, "substanti": 2, "establish": 2, "some": [2, 3, 4], "broader": 2, "lower": [2, 4], "particularli": [2, 3, 4], "intens": [2, 4], "cut": [2, 3], "littl": 2, "free": 2, "illegitim": 2, "obtain": [2, 4], "collabor": 2, "nsuppli": 2, "nalthough": 2, "essenti": [2, 3, 4], "singl": [2, 3, 4], "particip": 2, "therefor": 2, "wide": [2, 3, 4], "shortag": 2, "commod": 2, "fluctuat": 2, "commonli": 2, "introduc": [2, 3, 4], "often": [2, 3, 4], "util": [2, 3], "onli": [2, 3, 4], "capac": 2, "until": [2, 4], "supplier": 2, "matur": 2, "accept": 2, "decid": [2, 3], "concentr": 2, "instead": [2, 3, 4], "enter": 2, "agreement": 2, "suppli": [2, 4], "renew": 2, "nresearch": 2, "nbecaus": 2, "upon": [2, 3], "flow": [2, 3], "enhanc": [2, 3, 4], "acquisit": 2, "nintellectu": 2, "broad": [2, 4], "right": 2, "aspect": [2, 3, 4], "patent": 2, "copyright": 2, "trademark": 2, "trade": [2, 4], "secret": 2, "differenti": 2, "success": [2, 4], "reli": 2, "skill": 2, "personnel": 2, "regularli": 2, "protect": 2, "aris": 2, "pursu": 2, "thousand": 2, "accumul": 2, "durat": 2, "adequ": 2, "nin": 2, "necessari": [2, 3], "process": [2, 3, 4], "commerci": [2, 4], "experienc": 2, "higher": 2, "holidai": 2, "addition": 2, "expens": 2, "fill": 2, "inventori": 2, "launch": 2, "older": 2, "declin": 2, "newer": 2, "distributor": 2, "nhuman": 2, "capit": [2, 3, 4], "peopl": 2, "plai": [2, 4], "strive": 2, "attract": 2, "retain": [2, 3], "talent": 2, "inclus": [2, 3, 4], "team": [2, 4], "member": 2, "so": [2, 4], "As": [2, 3, 4], "had": 2, "164": 2, "full": [2, 3, 4], "equival": 2, "employe": 2, "ncompens": 2, "benefit": [2, 4], "equit": 2, "recogn": 2, "thrive": [2, 4], "succe": 2, "profession": [2, 4], "health": 2, "awai": 2, "ngrowth": 2, "achiev": [2, 4], "career": 2, "leadership": 2, "influenc": [2, 4], "cultur": 2, "advantag": [2, 3, 4], "being": 2, "nworkplac": 2, "practic": [2, 3], "polici": 2, "equal": 2, "workplac": 2, "harass": 2, "discrimin": 2, "ninclus": 2, "sustain": 2, "workforc": 2, "repres": [2, 4], "serv": [2, 3, 4], "represent": [2, 3], "level": [2, 3, 4], "foster": [2, 4], "nengag": 2, "honest": 2, "among": 2, "everyon": 2, "grow": [2, 4], "encourag": [2, 4], "feedback": [2, 4], "concern": 2, "conduct": 2, "survei": [2, 4], "gaug": 2, "sentiment": [2, 4], "nhealth": 2, "everywher": 2, "measur": 2, "mitig": [2, 3, 4], "possibl": [2, 4], "hazard": 2, "crisi": 2, "put": 2, "place": [2, 4], "visitor": 2, "navail": 2, "quarterli": 2, "q": 2, "amend": 2, "sec": [2, 3, 4], "Such": 2, "charg": 2, "investor": [2, 4], "default": [2, 4], "aspx": 2, "websit": 2, "www": 2, "press": 2, "releas": [2, 4], "environment": 2, "social": 2, "detail": [2, 3, 4], "referenc": 2, "further": [2, 3, 4], "url": [2, 4], "inact": 2, "textual": 2, "unknown": 2, "describ": 2, "below": [2, 3, 4], "materi": [2, 4], "advers": 2, "trend": [2, 4], "conjunct": 2, "consolid": 2, "accompani": 2, "nmacroeconom": 2, "econom": 2, "outsid": 2, "chain": [2, 3], "facil": 2, "assembli": 2, "site": 2, "nadvers": 2, "slow": 2, "recess": 2, "unemploy": 2, "inflat": 2, "tighter": 2, "interest": [2, 3, 4], "currenc": 2, "confid": [2, 4], "spend": 2, "chang": 2, "monetari": 2, "volatil": 2, "incom": 2, "asset": 2, "contract": 2, "logist": 2, "instabl": 2, "inabl": 2, "financ": 2, "insolv": 2, "failur": 2, "deriv": 2, "counterparti": 2, "debt": 2, "reduc": [2, 3, 4], "liquid": [2, 3], "fair": 2, "instrument": 2, "polit": 2, "disput": 2, "geopolit": 2, "tension": 2, "terror": 2, "disast": 2, "accid": 2, "interrupt": 2, "npolit": 2, "whole": 2, "outsourc": 2, "korea": 2, "vietnam": 2, "restrict": [2, 4], "tariff": 2, "export": 2, "good": [2, 4], "portion": 2, "revenu": [2, 3, 4], "raw": [2, 4], "go": [2, 3, 4], "action": [2, 3], "restructur": 2, "ceas": 2, "accord": [2, 4], "disrupt": [2, 3], "announc": 2, "notic": [2, 4], "led": [2, 4], "escal": [2, 3], "sever": [2, 3, 4], "nmani": 2, "prone": 2, "earthquak": 2, "climat": 2, "weather": 2, "occur": 2, "fire": 2, "nuclear": 2, "plant": 2, "terrorist": 2, "attack": 2, "hostil": 2, "ransomwar": 2, "cybersecur": 2, "labor": 2, "beyond": 2, "nsuch": 2, "imposs": 2, "delai": 2, "ineffici": 2, "slowdown": 2, "outag": 2, "neg": [2, 4], "seriou": 2, "injuri": 2, "pandem": 2, "covid": 2, "19": 2, "economi": 2, "imposit": 2, "stringent": 2, "travel": 2, "freight": 2, "movement": 2, "ramp": 2, "nfollow": 2, "expenditur": 2, "resum": 2, "lose": 2, "exacerb": 2, "consequ": [2, 4], "insur": 2, "insuffici": 2, "nglobal": 2, "unabl": 2, "There": [2, 3, 4], "assur": 2, "contrast": 2, "minor": 2, "overal": [2, 3, 4], "naddition": 2, "intensifi": 2, "seamlessli": [2, 3], "function": [2, 3, 4], "nto": 2, "remain": [2, 3], "stimul": 2, "ndue": 2, "upgrad": 2, "appropri": [2, 3, 4], "quantiti": 2, "defect": 2, "defici": 2, "supersed": 2, "nsubstanti": 2, "much": 2, "transport": 2, "diminish": 2, "flexibl": [2, 3, 4], "respond": 2, "provis": 2, "reimburs": 2, "warranti": 2, "out": [2, 3, 4], "unanticip": 2, "liabil": 2, "adher": [2, 3, 4], "violat": 2, "final": [2, 3, 4], "finish": 2, "destin": 2, "man": 2, "made": [2, 3, 4], "prepay": 2, "termin": 2, "recover": 2, "exposur": 2, "nfutur": 2, "suffici": [2, 4], "semiconductor": 2, "suffer": 2, "poor": 2, "constrain": [2, 3, 4], "shipment": 2, "altern": [2, 3], "sophist": [2, 3], "unexpectedli": 2, "interfer": 2, "unsaf": 2, "artifici": 2, "intellig": 2, "expos": 2, "inaccur": [2, 4], "fix": [2, 3], "widespread": 2, "vulner": 2, "exploit": 2, "compromis": 2, "claim": 2, "recal": 2, "modif": 2, "off": [2, 3, 4], "intang": 2, "fine": [2, 4], "lost": [2, 3], "cancel": 2, "record": 2, "obsolet": 2, "exce": 2, "realiz": 2, "accru": 2, "excess": 2, "review": [2, 4], "impair": 2, "whenev": 2, "circumst": 2, "amount": [2, 3, 4], "carri": [2, 4], "incur": 2, "given": [2, 3, 4], "unpredict": [2, 4], "pace": 2, "obsolesc": 2, "forecast": 2, "150": 2, "incorrectli": [2, 4], "fulli": [2, 3], "extens": [2, 3, 4], "issuanc": 2, "unknowingli": 2, "notifi": 2, "preclud": 2, "choos": 2, "bui": 2, "percept": 2, "android": 2, "playstat": 2, "nintendo": 2, "xbox": 2, "posit": [2, 3, 4], "less": 2, "inclin": 2, "devot": 2, "compel": [2, 4], "fail": 2, "dissatisfi": 2, "vast": 2, "legal": 2, "storefront": 2, "mechan": [2, 4], "safari": 2, "union": 2, "eu": 2, "dma": 2, "interfac": 2, "reduct": 2, "narrow": 2, "scope": [2, 3], "elimin": 2, "nfailur": 2, "appeal": 2, "subscrib": 2, "nsome": 2, "manner": [2, 3, 4], "nurtur": 2, "distinct": 2, "nmuch": 2, "chief": 2, "especi": [2, 3, 4], "silicon": 2, "vallei": 2, "constantli": 2, "driver": 2, "recruit": 2, "subsidi": 2, "staf": 2, "contractor": 2, "placement": 2, "increment": 2, "weaken": 2, "stop": [2, 3], "telecommun": 2, "war": 2, "virus": 2, "physic": 2, "ins": 2, "incid": 2, "redund": 2, "ineffect": 2, "inadequ": 2, "eventu": 2, "thing": [2, 4], "interf": 2, "imped": 2, "ship": 2, "nloss": 2, "unauthor": 2, "confidenti": 2, "encrypt": 2, "But": [2, 4], "absolut": [2, 4], "malici": 2, "behalf": 2, "gain": 2, "regular": [2, 4], "normal": [2, 4], "investig": 2, "penalti": 2, "judgment": 2, "against": 2, "frequenc": [2, 3], "actor": 2, "circumv": [2, 3], "remov": 2, "obfusc": 2, "forens": 2, "evid": [2, 4], "hinder": [2, 4], "recov": 2, "perpetr": 2, "target": [2, 4], "profil": 2, "authent": 2, "hack": 2, "malfeas": 2, "faulti": 2, "password": 2, "irregular": 2, "fraudul": 2, "induc": 2, "disclos": [2, 3, 4], "usernam": 2, "turn": 2, "multifactor": 2, "unusu": 2, "freez": 2, "suspici": 2, "nwhile": 2, "ninvest": 2, "contempl": 2, "endeavor": 2, "distract": 2, "tangibl": 2, "approv": 2, "oner": 2, "ventur": 2, "riski": 2, "pose": [2, 3, 4], "leas": 2, "unfavor": 2, "arisen": 2, "ordinari": 2, "cours": 2, "resolv": 2, "sometim": [2, 4], "indemnif": 2, "indemnifi": 2, "alleg": 2, "magnitud": 2, "assert": 2, "royalti": 2, "vigor": 2, "defend": 2, "court": 2, "internation": 2, "plaintiff": 2, "injunct": 2, "relief": 2, "nregardless": 2, "merit": 2, "recognit": 2, "settl": 2, "uncertain": 2, "abov": 2, "disgorg": 2, "remedi": 2, "worldwid": 2, "antitrust": 2, "privaci": [2, 4], "local": [2, 3, 4], "bill": 2, "commerc": 2, "internet": 2, "mobil": [2, 4], "televis": 2, "film": 2, "anticorrupt": 2, "cash": [2, 3], "repatri": 2, "monei": 2, "launder": 2, "tax": 2, "wast": 2, "recycl": 2, "ncomplianc": 2, "impos": [2, 4], "interpret": 2, "ethic": 2, "agent": 2, "found": [2, 4], "nregulatori": 2, "satisfi": 2, "ban": 2, "nexpect": 2, "stakehold": 2, "increasingli": [2, 4], "greenhous": 2, "ga": 2, "emiss": 2, "civil": 2, "disagre": 2, "perceiv": 2, "feder": 2, "vari": 2, "scrutini": 2, "nfrom": 2, "taken": [2, 4], "engag": [2, 4], "noncompli": 2, "individu": [2, 3], "lawsuit": 2, "monopol": 2, "nfurther": 2, "earn": 2, "googl": [2, 4], "search": 2, "nthere": 2, "connect": [2, 4], "retent": 2, "transfer": 2, "pass": [2, 4], "pend": 2, "inquiri": 2, "government": 2, "entiti": [2, 4], "biometr": 2, "breach": 2, "notif": 2, "permit": [2, 4], "healthcar": 2, "liabl": 2, "investigatori": 2, "cardhold": 2, "compress": [2, 3], "acquir": 2, "shift": 2, "mix": [2, 4], "extent": 2, "unexpect": [2, 4], "dollar": 2, "denomin": 2, "rais": [2, 3], "offset": 2, "strengthen": 2, "nconvers": 2, "therebi": [2, 3], "thu": 2, "option": [2, 3, 4], "hedg": 2, "deterior": 2, "sovereign": 2, "heighten": 2, "worsen": 2, "A": [2, 3, 4], "collater": 2, "bank": 2, "unsecur": 2, "subassembli": 2, "assembl": 2, "few": [2, 3, 4], "legisl": 2, "ireland": 2, "singapor": 2, "organis": 2, "propos": 2, "modern": [2, 3, 4], "minimum": 2, "statutori": 2, "valuat": 2, "defer": 2, "bodi": 2, "likelihood": 2, "adequaci": 2, "ultim": 2, "ow": 2, "ngener": 2, "volum": [2, 3], "unrel": 2, "averag": [2, 4], "repurchas": 2, "point": [2, 3], "dividend": 2, "consumm": 2, "declar": 2, "board": 2, "unresolv": 2, "nnone": 2, "threat": 2, "dedic": [2, 4], "postur": 2, "25": 2, "sinc": [2, 3, 4], "2016": 2, "coordin": 2, "assist": [2, 4], "log": 2, "track": 2, "committe": 2, "oversight": 2, "counsel": 2, "chair": 2, "substanc": 2, "17": 2, "headquart": 2, "cupertino": [2, 4], "land": 2, "center": [2, 4], "suitabl": 2, "formal": [2, 4], "articl": [2, 3], "promot": 2, "conclud": 2, "uninstal": 2, "web": 2, "browser": 2, "screen": 2, "june": 2, "24": [2, 4], "preliminari": 2, "find": [2, 3, 4], "contractu": 2, "desist": 2, "stai": [2, 3], "grant": 2, "ndepart": 2, "justic": 2, "21": 2, "depart": 2, "doj": 2, "district": 2, "attornei": 2, "jersei": 2, "redress": 2, "anticompetit": 2, "nonmonetari": 2, "defens": 2, "itself": 2, "nepic": 2, "epic": 2, "northern": 2, "unfair": 2, "guidelin": 2, "enjoin": 2, "extern": 2, "januari": 2, "motion": 2, "enforc": [2, 4], "oppos": 2, "30": 2, "vacat": 2, "fourth": 2, "did": [2, 4], "mine": 2, "nnot": 2, "aapl": 2, "nholder": 2, "na": 2, "23": 2, "301": 2, "npurchas": 2, "nshare": 2, "three": 2, "million": 2, "nperiod": 2, "ttotal": 2, "taverag": 2, "npaid": 2, "publicli": [2, 4], "nannounc": 2, "napproxim": 2, "That": [2, 4], "Be": 2, "nunder": 2, "njune": 2, "august": 2, "nopen": 2, "negoti": 2, "t35": 2, "697": 2, "t224": 2, "naugust": 2, "31": 2, "t42": 2, "910": 2, "t221": 2, "39": 2, "nseptemb": 2, "t33": 2, "653": 2, "t222": 2, "86": 2, "ntotal": 2, "t112": 2, "260": 2, "t89": 2, "074": 2, "110": 2, "billion": 2, "20": [2, 4], "previou": [2, 3, 4], "2023": [2, 4], "10b5": 2, "graph": 2, "show": [2, 3, 4], "comparison": 2, "five": 2, "cumul": 2, "reinvest": 2, "p": [2, 4], "dow": 2, "jone": 2, "supersector": 2, "100": [2, 4], "close": 2, "27": 2, "2019": 2, "n2218": 2, "tseptemb": 2, "2021": 2, "2022": 2, "t100": 2, "t207": 2, "t273": 2, "t281": 2, "t322": 2, "t430": 2, "t113": 2, "t156": 2, "t131": 2, "t155": 2, "t210": 2, "ndow": 2, "t146": 2, "t216": 2, "t215": 2, "nfirst": 2, "nsecond": 2, "nthird": 2, "sequoia": 2, "nfourth": 2, "plu": 2, "nfiscal": 2, "six": 2, "realign": 2, "span": 2, "wherea": 2, "indirectli": 2, "tabl": [2, 3, 4], "n2024": 2, "tchang": 2, "t2023": 2, "t2022": 2, "namerica": 2, "t167": 2, "045": 2, "t3": 2, "t162": 2, "560": 2, "t169": 2, "658": 2, "neurop": 2, "t101": 2, "328": 2, "t7": 2, "294": 2, "t95": 2, "118": 2, "ngreater": 2, "t66": 2, "952": 2, "t72": 2, "559": 2, "t74": 2, "200": 2, "njapan": 2, "t25": 2, "052": 2, "t24": 2, "257": 2, "977": 2, "nrest": 2, "t30": 2, "t4": 2, "t29": 2, "615": 2, "t1": 2, "t391": 2, "035": 2, "t2": 2, "t383": 2, "285": 2, "t394": 2, "decreas": 2, "weak": 2, "renminbi": 2, "yen": [2, 4], "22": 2, "categori": 2, "t201": 2, "183": 2, "t200": 2, "583": 2, "t205": 2, "489": 2, "984": 2, "357": 2, "t40": 2, "177": 2, "t26": 2, "694": 2, "t28": 2, "300": [2, 3], "292": 2, "t37": 2, "005": 2, "t39": 2, "845": 2, "t41": 2, "241": 2, "n96": 2, "169": 2, "t13": 2, "t85": 2, "t9": 2, "t78": 2, "129": 2, "amort": 2, "bundl": 2, "flat": 2, "entri": 2, "partial": [2, 3], "ngross": 2, "percentag": 2, "t109": 2, "633": 2, "t108": 2, "803": 2, "t114": 2, "728": 2, "t71": 2, "t60": 2, "345": 2, "t56": 2, "054": 2, "t180": 2, "683": 2, "148": 2, "t170": 2, "782": 2, "t36": 2, "t73": 2, "t70": 2, "t46": 2, "t44": 2, "t43": 2, "save": [2, 3], "noper": 2, "t31": 2, "370": 2, "t5": 2, "915": 2, "t14": 2, "251": 2, "npercentag": 2, "t8": 2, "nsell": 2, "administr": 2, "097": 2, "932": 2, "094": 2, "t6": 2, "t57": 2, "467": 2, "t54": 2, "847": 2, "t51": 2, "t15": 2, "driven": 2, "headcount": 2, "nprovis": 2, "749": 2, "t16": 2, "741": 2, "t19": 2, "neffect": 2, "nstatutori": 2, "t21": 2, "aid": 2, "nliquid": 2, "unrestrict": 2, "140": 2, "ndebt": 2, "97": 2, "payabl": 2, "promissori": 2, "paper": [2, 4], "nleas": 2, "space": 2, "nmanufactur": 2, "noncancel": 2, "ndeem": 2, "2017": 2, "tcja": 2, "paid": 2, "nstate": 2, "fund": 2, "escrow": 2, "ncapit": 2, "95": 2, "nrecent": 2, "pronounc": 2, "nincom": 2, "decemb": 2, "fasb": 2, "asu": 2, "09": [2, 3], "topic": [2, 3, 4], "740": 2, "reconcili": 2, "reconcil": [2, 4], "quantit": 2, "threshold": 2, "disaggreg": 2, "prospect": 2, "novemb": 2, "07": [2, 3, 4], "280": 2, "maker": 2, "codm": 2, "titl": 2, "alloc": 2, "retrospect": 2, "ncritic": 2, "conform": [2, 4], "principl": 2, "gaap": 2, "nuncertain": 2, "domest": 2, "taxat": 2, "adjust": [2, 3, 4], "resolut": 2, "conting": 2, "26": 2, "still": 2, "ninterest": 2, "forth": 2, "hypothet": 2, "nsensit": 2, "nhypothet": 2, "nrate": 2, "npotenti": 2, "n100": 2, "tenor": 2, "ndeclin": 2, "755": 2, "089": 2, "nterm": 2, "nincreas": 2, "t139": 2, "t194": 2, "nforeign": 2, "express": [2, 4], "var": 2, "mont": 2, "carlo": 2, "simul": [2, 4], "maximum": [2, 3], "interv": 2, "538": 2, "669": 2, "underli": [2, 4], "nindex": 2, "tpage": 2, "nconsolid": 2, "n29": 2, "n30": 2, "sheet": 2, "n31": 2, "n32": 2, "n33": 2, "nnote": 2, "n34": 2, "nreport": 2, "n48": 2, "nall": 2, "omit": [2, 4], "submiss": 2, "nyear": 2, "n2023": 2, "n2022": 2, "nnet": 2, "t294": 2, "866": 2, "t298": 2, "085": 2, "t316": 2, "199": 2, "t96": 2, "ncost": 2, "t185": 2, "233": 2, "t189": 2, "282": 2, "471": 2, "119": 2, "855": 2, "t22": 2, "075": 2, "352": 2, "t214": 2, "137": 2, "t223": 2, "546": 2, "t123": 2, "216": 2, "t119": 2, "437": 2, "t269": 2, "565": 2, "334": 2, "485": 2, "736": 2, "103": 2, "t93": 2, "995": 2, "t99": 2, "nearn": 2, "nbasic": 2, "ndilut": 2, "08": [2, 4], "343": 2, "783": 2, "744": 2, "231": 2, "215": 2, "963": 2, "095": 2, "812": 2, "547": 2, "325": 2, "819": 2, "nsee": 2, "translat": 2, "t395": 2, "765": 2, "511": 2, "unreal": 2, "832": 2, "t323": 2, "212": 2, "nadjust": 2, "337": 2, "717": 2, "394": 2, "138": 2, "850": 2, "563": 2, "104": 2, "t204": 2, "t253": 2, "816": 2, "899": 2, "272": 2, "t98": 2, "016": 2, "652": 2, "t88": 2, "531": 2, "nasset": 2, "ncurrent": 2, "ncash": 2, "943": 2, "965": 2, "228": 2, "590": 2, "naccount": 2, "410": 2, "508": 2, "nvendor": 2, "t32": 2, "833": 2, "477": 2, "ninventori": 2, "286": 2, "331": 2, "287": 2, "695": 2, "t152": 2, "987": 2, "t143": 2, "566": 2, "t91": 2, "479": 2, "544": 2, "t45": 2, "680": 2, "715": 2, "834": 2, "t64": 2, "758": 2, "t211": 2, "993": 2, "t209": 2, "017": 2, "t364": 2, "980": 2, "t352": 2, "nliabil": 2, "t68": 2, "960": 2, "t62": 2, "611": 2, "304": 2, "t58": 2, "829": 2, "ndefer": 2, "249": 2, "061": 2, "ncommerci": 2, "967": 2, "985": 2, "t10": 2, "912": 2, "822": 2, "t176": 2, "392": 2, "t145": 2, "308": 2, "750": 2, "281": 2, "888": 2, "t49": 2, "848": 2, "638": 2, "t308": 2, "030": 2, "t290": 2, "ncommit": 2, "nsharehold": 2, "400": 2, "116": 2, "786": 2, "550": 2, "n83": 2, "276": 2, "naccumul": 2, "deficit": 2, "154": 2, "214": 2, "172": 2, "452": 2, "950": 2, "146": 2, "t50": 2, "672": 2, "t63": 2, "090": 2, "nbegin": 2, "849": 2, "365": 2, "423": 2, "346": 2, "175": 2, "withheld": 2, "settlement": 2, "award": 2, "521": 2, "971": 2, "t12": 2, "034": 2, "t11": 2, "nend": 2, "t83": 2, "nretain": 2, "068": 2, "562": 2, "ndividend": 2, "218": 2, "793": 2, "612": 2, "099": 2, "454": 2, "846": 2, "77": 2, "046": 2, "186": 2, "109": 2, "t163": 2, "rsu": 2, "t0": 2, "98": 2, "94": 2, "32": 2, "737": 2, "929": 2, "ndepreci": 2, "445": 2, "519": 2, "688": 2, "038": 2, "266": 2, "227": 2, "006": 2, "788": 2, "356": 2, "271": 2, "520": 2, "618": 2, "484": 2, "731": 2, "684": 2, "499": 2, "020": 2, "889": 2, "448": 2, "552": 2, "031": 2, "t118": 2, "254": 2, "t110": 2, "543": 2, "t122": 2, "151": 2, "48": 2, "656": 2, "513": 2, "76": 2, "923": 2, "nproce": 2, "211": 2, "686": 2, "917": 2, "135": 2, "828": 2, "446": 2, "447": 2, "959": 2, "708": 2, "086": 2, "935": 2, "705": 2, "354": 2, "nfinanc": 2, "441": 2, "431": 2, "223": 2, "234": 2, "025": 2, "841": 2, "nrepurchas": 2, "949": 2, "89": 2, "402": 2, "465": 2, "nrepay": 2, "958": 2, "repay": 2, "978": 2, "955": 2, "361": 2, "581": 2, "160": 2, "121": 2, "983": 2, "108": 2, "488": 2, "794": 2, "760": 2, "nsupplement": 2, "102": 2, "t18": 2, "679": 2, "573": 2, "33": 2, "nbasi": 2, "prior": 2, "reclassifi": 2, "nrevenu": 2, "remit": 2, "straight": 2, "vest": 2, "treat": 2, "sold": 2, "nderiv": 2, "combin": [2, 3, 4], "nonleas": 2, "34": 2, "entitl": 2, "reward": 2, "commenc": 2, "deliveri": 2, "stand": 2, "alon": 2, "ssp": 2, "object": [2, 4], "icloud": 2, "siri": 2, "map": [2, 4], "discount": 2, "lack": [2, 4], "undeliv": 2, "unbil": 2, "accordingli": 2, "n26": 2, "n37": 2, "35": 2, "proport": 2, "moder": 2, "64": 2, "dilut": 2, "nnumer": 2, "ndenomin": 2, "nweight": 2, "312": 2, "316": 2, "856": 2, "antidilut": 2, "tunreal": 2, "ngain": 2, "tfair": 2, "nvalu": 2, "tcash": 2, "nequival": 2, "tcurrent": 2, "tnon": 2, "t27": 2, "nlevel": 2, "nmonei": 2, "t778": 2, "nmutual": 2, "n515": 2, "t105": 2, "t617": 2, "nsubtot": 2, "293": 2, "395": 2, "nu": 2, "treasuri": 2, "516": 2, "t212": 2, "087": 2, "380": 2, "agenc": 2, "159": 2, "t703": 2, "t17": 2, "568": 2, "158": 2, "810": 2, "ncertif": 2, "deposit": 2, "t873": 2, "t387": 2, "t478": 2, "066": 2, "ncorpor": 2, "t65": 2, "622": 2, "t270": 2, "953": 2, "939": 2, "027": 2, "t47": 2, "886": 2, "nmunicip": 2, "t412": 2, "t405": 2, "t190": 2, "nmortgag": 2, "595": 2, "t175": 2, "403": 2, "t23": 2, "367": 2, "278": 2, "t132": 2, "t583": 2, "635": 2, "t128": 2, "056": 2, "966": 2, "t34": 2, "t160": 2, "t688": 2, "650": 2, "36": 2, "359": 2, "t481": 2, "n442": 2, "t428": 2, "t923": 2, "t909": 2, "406": 2, "114": 2, "468": 2, "136": 2, "t271": 2, "533": 2, "048": 2, "491": 2, "332": 2, "t320": 2, "t608": 2, "t76": 2, "840": 2, "956": 2, "890": 2, "t20": 2, "627": 2, "243": 2, "t628": 2, "t602": 2, "t192": 2, "t410": 2, "735": 2, "636": 2, "t344": 2, "t144": 2, "470": 2, "657": 2, "831": 2, "125": 2, "162": 2, "t173": 2, "752": 2, "quot": 2, "corrobor": 2, "mortgag": 2, "classifi": 2, "37": 2, "cross": 2, "swap": 2, "remeasur": 2, "notion": 2, "069": 2, "730": 2, "575": 2, "493": 2, "t104": 2, "777": 2, "nhedg": 2, "433": 2, "505": 2, "247": 2, "ntrade": 2, "41": 2, "44": 2, "depreci": 2, "nland": 2, "690": 2, "nmachineri": 2, "t80": 2, "205": 2, "314": 2, "nleasehold": 2, "839": 2, "128": 2, "599": 2, "73": 2, "70": 2, "884": 2, "852": 2, "t55": 2, "335": 2, "906": 2, "601": 2, "703": 2, "010": 2, "457": 2, "634": 2, "391": 2, "neuropean": 2, "opinion": 2, "1991": 2, "2007": 2, "irish": 2, "branch": 2, "2003": 2, "2014": 2, "2015": 2, "request": [2, 3, 4], "minist": 2, "juli": 2, "annul": 2, "ecj": 2, "hear": 2, "asid": 2, "confirm": 2, "via": [2, 4], "unrecogn": 2, "nfeder": 2, "571": 2, "080": 2, "644": 2, "265": 2, "801": 2, "726": 2, "570": 2, "298": 2, "49": 2, "t84": 2, "428": 2, "603": 2, "483": 2, "t347": 2, "t669": 2, "076": 2, "830": 2, "419": 2, "072": 2, "pretax": 2, "72": 2, "71": 2, "ncomput": 2, "885": 2, "012": 2, "124": 2, "518": 2, "nimpact": 2, "n10": 2, "246": 2, "311": 2, "366": 2, "397": 2, "153": 2, "nexcess": 2, "893": 2, "871": 2, "192": 2, "739": 2, "ntax": 2, "carryforward": 2, "302": 2, "naccru": 2, "413": 2, "421": 2, "nunreal": 2, "173": 2, "168": 2, "873": 2, "743": 2, "nless": 2, "374": 2, "007": 2, "369": 2, "551": 2, "998": 2, "nright": 2, "179": 2, "nminimum": 2, "674": 2, "940": 2, "t511": 2, "t455": 2, "t490": 2, "805": 2, "202": 2, "indefinit": 2, "temporari": 2, "727": 2, "044": 2, "284": 2, "ndecreas": 2, "386": 2, "463": 2, "982": 2, "542": 2, "936": 2, "070": 2, "expir": 2, "statut": 2, "229": 2, "494": 2, "closur": 2, "intercompani": 2, "exceed": 2, "multiyear": 2, "exercis": 2, "noncash": 2, "rou": 2, "tfinanci": 2, "t2024": 2, "tother": 2, "661": 2, "tproperti": 2, "015": 2, "303": 2, "676": 2, "t165": 2, "t752": 2, "t859": 2, "430": 2, "842": 2, "tfinanc": 2, "n2025": 2, "820": 2, "t171": 2, "991": 2, "n2026": 2, "914": 2, "n2027": 2, "t59": 2, "733": 2, "n2028": 2, "360": 2, "t38": 2, "398": 2, "n2029": 2, "187": 2, "nthereaft": 2, "t837": 2, "undiscount": 2, "790": 2, "imput": 2, "376": 2, "534": 2, "t896": 2, "weight": 2, "borrow": 2, "implicit": 2, "readili": 2, "42": 2, "proce": 2, "nine": 2, "00": 2, "nmatur": 2, "333": 2, "264": 2, "948": 2, "645": 2, "309": 2, "arrear": 2, "namount": 2, "n2013": 2, "nfix": 2, "2062": 2, "t97": 2, "341": 2, "03": 2, "65": 2, "t106": 2, "572": 2, "n97": 2, "nunamort": 2, "premium": 2, "321": 2, "358": 2, "113": 2, "662": 2, "convert": [2, 4], "930": 2, "342": 2, "800": 2, "180": 2, "43": 2, "88": 2, "ndure": 2, "425": 2, "426": 2, "372": 2, "589": 2, "055": 2, "appreci": 2, "four": 2, "holder": 2, "n2014": 2, "bonu": 2, "nrestrict": 2, "nnumber": 2, "nrsu": 2, "ngrant": 2, "naggreg": 2, "nfair": 2, "nbalanc": 2, "t240": 2, "427": 2, "t75": 2, "t150": 2, "861": 2, "501": 2, "768": 2, "87": 2, "101": 2, "878": 2, "144": 2, "t127": 2, "t135": 2, "91": 2, "456": 2, "78": 2, "59": 2, "t140": 2, "80": 2, "326": 2, "t158": 2, "204": 2, "350": 2, "002": [2, 3], "nuncondit": 2, "uncondit": 2, "206": 2, "440": 2, "156": 2, "t633": 2, "t670": 2, "226": 2, "45": 2, "nconting": 2, "least": 2, "accrual": 2, "nconcentr": 2, "attribut": [2, 4], "46": 2, "t67": 2, "098": 2, "082": 2, "062": 2, "569": 2, "895": 2, "458": 2, "207": 2, "nonrecur": 2, "t142": 2, "196": 2, "t138": 2, "t147": 2, "859": 2, "nchina": 2, "n66": 2, "t181": 2, "887": 2, "t172": 2, "269": 2, "nlong": 2, "664": 2, "n4": 2, "797": 2, "778": 2, "219": 2, "47": 2, "nopinion": 2, "nwe": 2, "fairli": 2, "pcaob": 2, "criteria": 2, "sponsor": 2, "treadwai": 2, "2013": 2, "unqualifi": 2, "thereon": 2, "nthese": 2, "misstat": 2, "fraud": 2, "alter": 2, "ndescript": 2, "naudit": 2, "nhow": 2, "nmatter": 2, "qualifi": 2, "letter": 2, "advisor": 2, "ernst": 2, "young": 2, "llp": 2, "auditor": 2, "2009": 2, "nsan": 2, "jose": 2, "nnovemb": 2, "coso": 2, "nour": 2, "ndefinit": 2, "pertain": 2, "mainten": 2, "accur": [2, 4], "disposit": 2, "receipt": 2, "degre": 2, "nevalu": 2, "nbase": 2, "supervis": 2, "13a": 2, "15d": 2, "summar": [2, 3], "ninher": 2, "met": 2, "appear": [2, 4], "paragraph": 2, "51": [2, 4], "ninsid": 2, "deirdr": 2, "brien": 2, "vice": 2, "presid": 2, "affirm": 2, "april": 2, "withhold": 2, "remitt": 2, "jeff": 2, "william": 2, "mr": 2, "insid": 2, "copi": [2, 3], "exhibit": 2, "solicit": 2, "document": [2, 3, 4], "id": 2, "00042": 2, "nincorpor": 2, "texhibit": 2, "descript": [2, 4], "tform": 2, "tfile": 2, "nrestat": 2, "n8": 2, "namend": 2, "bylaw": 2, "nindentur": 2, "york": [2, 4], "mellon": 2, "truste": 2, "noffic": 2, "certif": 2, "2018": 2, "85": 2, "2043": 2, "05": 2, "2044": 2, "februari": 2, "55": 2, "2045": 2, "900": 2, "700": 2, "60": 2, "250": 2, "2036": 2, "2046": 2, "450": 2, "2047": 2, "2049": 2, "2030": 2, "2050": 2, "2060": 2, "2028": 2, "2041": 2, "2051": 2, "2061": 2, "2032": 2, "2052": 2, "54": 2, "2033": 2, "2053": 2, "n9": 2, "ceo": 2, "n12": 2, "nsubsidiari": 2, "n23": 2, "nconsent": 2, "n24": 2, "npower": 2, "signatur": 2, "nrule": 2, "nsection": 2, "1350": 2, "n101": 2, "ninlin": 2, "xbrl": 2, "n104": 2, "inlin": 2, "compensatori": 2, "herewith": 2, "furnish": 2, "herebi": 2, "undertak": 2, "56": 2, "nsignatur": 2, "npursuant": 2, "duli": 2, "sign": 2, "undersign": 2, "thereunto": 2, "ndate": 2, "nby": 2, "luca": [2, 4], "maestri": 2, "nluca": 2, "nsenior": 2, "nchief": 2, "nknow": 2, "THESE": 2, "whose": 2, "constitut": 2, "appoint": 2, "timothi": 2, "cook": 2, "jointli": 2, "hi": [2, 4], "her": 2, "substitut": 2, "him": 2, "thereto": 2, "therewith": 2, "ratifi": 2, "said": 2, "done": [2, 4], "virtu": 2, "hereof": 2, "nname": 2, "ttitl": 2, "tdate": 2, "tchief": 2, "tnovemb": 2, "ntimothi": 2, "tsenior": 2, "chri": 2, "kondo": 2, "nchri": 2, "wanda": 2, "austin": 2, "nwanda": 2, "alex": 2, "gorski": 2, "tdirector": 2, "nalex": 2, "andrea": 2, "jung": 2, "nandrea": 2, "arthur": 2, "levinson": 2, "narthur": 2, "monica": 2, "lozano": 2, "nmonica": 2, "ronald": 2, "sugar": 2, "nronald": 2, "susan": 2, "l": 2, "wagner": 2, "nsusan": 2, "57": 2, "gpt": [2, 3, 4], "turbo": [2, 3, 4], "invdestacksmeticsisdict": 2, "setispect": 2, "20cyan": 2, "evaluationseld": 2, "anvis": 2, "droitent": 2, "discernminerv": 2, "versbobprefvers": 2, "vo\u8be5": 2, "option\u548c": 2, "meio": 2, "\u0432\u0440\u0435\u043ccisco": 2, "dellaischenpoihscap": 2, "geme": 2, "gettim": 2, "unscal": 2, "score": [2, 4], "vocabulari": [2, 4], "closer": 2, "sharpen": 2, "uniform": 2, "raschka": 2, "simpl": [2, 3, 4], "dramat": [2, 4], "systemat": [2, 4], "At": [2, 4], "rigid": 2, "wildli": 2, "radic": 2, "grappl": 2, "probabilist": 2, "seem": [2, 4], "safer": 2, "don": [2, 3, 4], "highlight": [2, 3, 4], "paradigm": 2, "anoth": 2, "fascin": 2, "spontan": 2, "answer": [2, 3, 4], "aren": 2, "explicitli": 2, "clear": [2, 4], "wei": 2, "fig": [2, 3, 4], "linear": 2, "absent": 2, "simpli": [2, 3, 4], "coax": 2, "onc": [2, 3], "reach": [2, 3, 4], "journei": 2, "suddenli": 2, "manifest": 2, "call": [2, 3, 4], "phase": 2, "stark": 2, "deliber": 2, "convent": 2, "stabl": 2, "suit": 2, "contend": 2, "7b": 2, "70b": 2, "rethink": 2, "math": 2, "tutor": 2, "children": 2, "verifi": [2, 4], "just": [2, 3, 4], "predefin": [2, 4], "adapt": [2, 3], "explan": [2, 4], "child": 2, "ag": 2, "bound": 2, "weren": 2, "accuraci": [2, 4], "kind": 2, "dimens": 2, "pre": 2, "explicit": [2, 4], "usual": 2, "precis": [2, 4], "resist": 2, "straightforward": [2, 3, 4], "quantif": 2, "contamin": 2, "carefulli": [2, 4], "craft": [2, 4], "massiv": 2, "alreadi": 2, "seen": 2, "memor": 2, "truli": 2, "unseen": 2, "rigor": 2, "evolut": 2, "longitudin": 2, "autom": [2, 4], "annot": 2, "mostli": [2, 4], "versu": 2, "latter": 2, "foundat": [2, 3], "tailor": 2, "solv": [2, 4], "great": [2, 4], "why": [2, 4], "misinform": 2, "factual": 2, "databas": [2, 4], "citat": 2, "tempor": 2, "scientif": 2, "fals": [2, 4], "manipul": 2, "medic": 2, "disclaim": 2, "referr": 2, "boundari": 2, "situat": [2, 3], "incorrect": 2, "expertis": 2, "bia": [2, 4], "gender": 2, "racial": 2, "demograph": 2, "stereotyp": 2, "reinforc": 2, "societ": 2, "pii": 2, "anonym": 2, "leakag": 2, "carryov": 2, "protocol": 2, "cognit": 2, "multi": [2, 4], "mathemat": 2, "fallaci": 2, "causal": 2, "edg": 2, "think": 2, "idiom": 2, "sarcasm": 2, "terminologi": 2, "lingual": 2, "misunderstand": 2, "syntax": 2, "scan": 2, "compat": [2, 4], "stabil": 2, "effici": [2, 3, 4], "scalabl": [2, 3], "meta": [2, 3], "overconfid": 2, "clariti": [2, 3, 4], "audienc": 2, "densiti": 2, "satisfact": [2, 4], "misus": 2, "moral": 2, "transpar": [2, 4], "co2": 2, "energi": 2, "consumpt": 2, "server": [2, 4], "batch": 2, "infer": 2, "imag": 2, "audio": 2, "etc": [2, 4], "truth": [2, 4], "layer": [2, 3, 4], "palm": 2, "shown": 2, "quantifi": 2, "rank": 2, "easi": [2, 3], "synthet": [2, 4], "post": [2, 4], "timeout": 2, "variat": 2, "maxim": 2, "inter": 2, "rater": 2, "priorit": 2, "ti": 2, "tier": 2, "holist": 2, "built": [2, 4], "mind": 2, "x": 2, "fast": 2, "experiment": [2, 4], "iter": [2, 3, 4], "vi": 2, "later": [2, 4], "categor": [2, 4], "intrins": 2, "extrins": 2, "sequenc": [2, 4], "perplex": 2, "downstream": [2, 4], "valuabl": [2, 4], "distinguish": 2, "classif": [2, 4], "true": [2, 3, 4], "synthesi": 2, "discret": 2, "f1": 2, "match": [2, 4], "prefix": 2, "roug": 2, "bleu": 2, "charact": [2, 3, 4], "gram": 2, "bilingu": 2, "understudi": 2, "overlap": [2, 3], "favor": [2, 4], "breviti": 2, "insensit": 2, "semant": [2, 3], "orient": 2, "gist": 2, "sentenc": [2, 3, 4], "ignor": 2, "meteor": 2, "synonym": 2, "stem": [2, 4], "paraphras": 2, "alongsid": 2, "computation": [2, 3], "cider": 2, "consensu": 2, "tf": 2, "idf": 2, "caption": 2, "reliant": 2, "corpu": 2, "statist": 2, "ter": 2, "edit": 2, "hypothesi": 2, "penal": 2, "bertscor": 2, "embed": [2, 3], "bert": 2, "spice": 2, "proposit": 2, "scene": 2, "emphasi": 2, "pure": 2, "analyst": [2, 3], "dictionari": [2, 4], "rouge_1": 2, "rouge_2": 2, "ideal": [2, 4], "expert": [2, 3, 4], "cheaper": 2, "4o": [2, 3, 4], "evaluate_summari": 2, "unigram": 2, "bigram": 2, "huggingfac": 2, "librari": [2, 3, 4], "absl": 2, "py": 2, "rouge_scor": 2, "generated_summari": 2, "reference_summari": 2, "arg": [2, 3, 4], "dict": [2, 3, 4], "google_bleu": 2, "bleu_scor": 2, "rouge1": 2, "rouge2": 2, "arbitrari": 2, "chosen": 2, "sentence1": 2, "cat": 2, "sat": 2, "mat": 2, "sentence2": 2, "ate": 2, "3333333333333333": 2, "7272727272727272": 2, "4444444444444445": 2, "generate_summari": 2, "summir": 2, "correspond": [2, 4], "liner": 2, "excerpt": 2, "evaluate_summary_model": 2, "model_benchmark": 2, "models_test": 2, "benchmark_summari": 2, "model_summari": 2, "evaluation_result": 2, "reveal": 2, "analyz": [2, 3, 4], "statu": 2, "concis": 2, "element": [2, 4], "Its": 2, "verbos": 2, "peripher": 2, "quit": [2, 4], "overli": [2, 4], "simplifi": [2, 4], "miss": 2, "convei": [2, 3], "breadth": 2, "Of": 2, "vibe": 2, "visualize_prompt_comparison": 2, "visual": 2, "matplotlib": 2, "radar": 2, "plot": 2, "radar_plot": 2, "tmp": 2, "ipykernel_1652501": 2, "940173201": 2, "userwarn": 2, "figurecanvasagg": 2, "closest": 2, "largest": 2, "deviat": [2, 4], "suggest": [2, 4], "mention": [2, 4], "nuanc": [2, 3, 4], "granular": [2, 3], "fall": 2, "judg": 2, "themselv": 2, "main": [2, 3, 4], "instruct": [2, 3, 4], "tune": [2, 4], "assign": [2, 4], "likert": 2, "style": 2, "pairwis": 2, "ensembl": 2, "repeatedli": 2, "domain": 2, "fluenci": 2, "refin": 2, "excel": [2, 4], "narr": 2, "mirror": 2, "similarli": 2, "notabl": [2, 4], "properli": [2, 4], "henc": 2, "worth": 2, "integ": 2, "rubric": 2, "hollist": 2, "judgeevalu": 2, "grammar": [2, 4], "evaluate_with_llm": 2, "candid": 2, "pars": [2, 4], "criterion": 2, "basemodel": [2, 4], "judge_model": 2, "candidate_summari": 2, "written": 2, "grammat": 2, "y": [2, 4], "z": 2, "w": [2, 3], "beta": [2, 4], "response_format": [2, 4], "Then": 2, "benchmark_model": 2, "test_model": 2, "input_text": [2, 3], "tupl": 2, "trillion": [2, 4], "evals_list": 2, "1775618912": 2, "variant": 2, "slightli": 2, "drift": 2, "lowest": 2, "drop": 2, "gradient": 2, "visibl": 2, "degrad": [2, 4], "firstli": 2, "overhead": 2, "neglect": 2, "prefer": [2, 4], "egocentr": 2, "tight": 2, "field": [2, 4], "aproach": 2, "workflow": [2, 4], "assessor": 2, "aplic": 2, "aim": [2, 3, 4], "clearli": [2, 4], "earlier": 2, "depict": [2, 4], "correl": 2, "multilingu": 2, "golden": 2, "languang": 2, "arena": 2, "blind": 2, "randomli": 2, "pair": 2, "loop": 2, "customiz": 2, "irrelev": 2, "unhelp": 2, "though": [2, 4], "occasion": 2, "rare": 2, "inaccuraci": 2, "perfectli": 2, "cater": 2, "critiqu": 2, "elo": 2, "democrat": [2, 4], "thought": [2, 4], "exam": 2, "probe": 2, "certifi": 2, "histori": 2, "move": [2, 3], "began": 2, "glue": 2, "wang": 2, "entail": 2, "baselin": 2, "superglu": 2, "deeper": [2, 3], "successor": 2, "grew": 2, "big": 2, "bench": 2, "srivastava": 2, "arithmet": 2, "truthfulqa": 2, "lin": [2, 4], "decept": 2, "multitask": 2, "hendryck": 2, "multidisciplinari": 2, "stanford": 2, "helm": 2, "liang": 2, "multidimension": 2, "surround": [2, 4], "emphas": [2, 4], "humanev": 2, "chen": [2, 4], "lmsy": 2, "brought": 2, "dialogu": 2, "len": [2, 3], "replic": [2, 4], "chatbot": 2, "chiang": 2, "gather": 2, "alpacaev": 2, "duboi": 2, "mt": 2, "zheng": 2, "Their": [2, 4], "render": 2, "crowdsourc": 2, "livebench": 2, "white": 2, "resili": 2, "meaningfulli": 2, "monthli": 2, "zebralog": 2, "grid": 2, "puzzl": 2, "brailsford": 2, "1999": 2, "lsat": 2, "hous": 2, "clue": 2, "strateg": [2, 4], "deduct": 2, "arriv": 2, "programmat": [2, 4], "2x2": 2, "6x6": 2, "reductio": 2, "ad": [2, 4], "absurdum": 2, "sonnet": [2, 3], "hard": 2, "10b": 2, "counterfactu": 2, "composit": 2, "came": 2, "arc": 2, "prize": 2, "chollet": 2, "mike": 2, "knoop": 2, "founder": 2, "zapier": 2, "fran\u00e7oi": 2, "creator": 2, "agi": 2, "kera": 2, "meaning": [2, 3, 4], "genuin": 2, "old": 2, "possess": 2, "count": [2, 3], "elementari": 2, "novelti": 2, "someth": 2, "wouldn": 2, "interpol": 2, "memori": [2, 3], "synthes": 2, "fly": 2, "brute": 2, "minim": [2, 4], "pixel": 2, "perfect": 2, "color": 2, "unbeaten": 2, "win": 2, "deep": [2, 4], "poorli": 2, "recombin": 2, "spur": 2, "art": 2, "takeawai": 2, "algorithm": 2, "fourrier": 2, "lightweight": [2, 4], "bespok": 2, "sdk": 2, "cli": 2, "extract": [2, 3, 4], "autoregress": 2, "sub": 2, "liter": 2, "disturb": 2, "zero": [2, 4], "varianc": 2, "yt": 2, "ut": 2, "suppos": [2, 4], "exactli": [2, 4], "ol": 2, "heteroscedast": 2, "regress": 2, "wish": 2, "lag": 2, "bivari": 2, "evaluation_track": 2, "evaluationtrack": 2, "model_config": 2, "basemodelconfig": 2, "parallelismmanag": 2, "pipelineparamet": 2, "envconfig": 2, "is_accelerate_avail": 2, "datetim": 2, "timedelta": 2, "initprocessgroupkwarg": 2, "create_evaluation_pipelin": 2, "output_dir": 2, "cache_dir": 2, "pretrain": 2, "dtype": 2, "float16": 2, "max_sampl": 2, "kwargs_handl": 2, "3000": 2, "els": [2, 3], "save_detail": 2, "push_to_hub": 2, "pipeline_param": 2, "launcher_typ": 2, "env_config": 2, "override_batch_s": 2, "use_chat_templ": 2, "trust_remote_cod": 2, "pipeline_paramet": 2, "schemat": [2, 3], "vllm": [2, 4], "tgi": 2, "instanti": 2, "storag": 2, "push": 2, "hub": 2, "parallel": 2, "num_few_shot": 2, "automat": 2, "string": [2, 4], "vertic": 2, "bar": 2, "binari": 2, "flag": 2, "bigbench": 2, "winogrand": 2, "hellaswag": 2, "nlp": 2, "save_and_push_result": 2, "show_result": 2, "model_arg": 2, "remot": 2, "send": [2, 4], "serverless": 2, "inference_server_address": 2, "inference_server_auth": 2, "model_id": 2, "null": 2, "bash": 2, "command": 2, "model_config_path": 2, "path": [2, 3], "endpoint_model": 2, "yaml": [2, 4], "llama3": [2, 3], "qwen2": [2, 4], "smollm2": 2, "3b": 2, "alibaba": [2, 4], "5b": [2, 4], "hui": 2, "yang": 2, "compact": 2, "360m": 2, "allal": 2, "cluster": 2, "noteworthi": 2, "superior": 2, "grain": [2, 4], "salt": [2, 4], "give": 2, "exponenti": 2, "hug": 2, "modular": 2, "visit": 2, "offici": 2, "revisit": 2, "rememb": 2, "api_kei": [2, 3], "trace": 2, "langchain_tracing_v2": 2, "langchain_api_kei": 2, "hf_evalu": 2, "langsmith_evalu": 2, "ls_client": 2, "tobia": 2, "src": 2, "lib": 2, "python3": 2, "tqdm": 2, "auto": 2, "tqdmwarn": 2, "iprogress": 2, "pleas": 2, "jupyt": 2, "ipywidget": 2, "readthedoc": 2, "en": [2, 4], "user_instal": 2, "html": [2, 3, 4], "autonotebook": 2, "notebook_tqdm": 2, "dataset_nam": 2, "create_dataset": 2, "create_exampl": 2, "dataset_id": 2, "calculate_scor": 2, "reference_output": 2, "oai_client": 2, "xp_model_nam": 2, "lastli": 2, "run_evalu": 2, "upload": 2, "And": 2, "upload_result": 2, "experiment_prefix": 2, "num_repetit": 2, "view": 2, "386a3620": 2, "smith": 2, "9e1cc3cb": 2, "9d6a": 2, "4356": 2, "ab34": 2, "138e0abe8be4": 2, "8741976e": 2, "5268": 2, "4b75": 2, "949f": 2, "99477dde5d64": 2, "selectedsess": 2, "b831dc1e": 2, "90bc": 2, "4ed8": 2, "8080": 2, "fb42444724d6": 2, "4it": 2, "latest": [2, 3, 4], "modul": [2, 4], "evaluate_modul": 2, "6fc70b7be0088120a372dfdd5d320b39b8bb3630cb8029b193941d9376e86bb0": 2, "tue": 2, "nov": 2, "couldn": 2, "5it": 2, "5053784e": 2, "64445871": 2, "a53c": 2, "44b1": 2, "a422": 2, "4f49b2f9656f": 2, "69": 2, "4b29f3c9": 2, "9ef7e39a": 2, "2add": 2, "410c": 2, "89f8": 2, "9f1a8b198cf1": 2, "61": 2, "df": 2, "to_panda": 2, "insert": 2, "combined_df": 2, "concat": 2, "ignore_index": 2, "execution_tim": 2, "example_id": 2, "333333": 2, "224388": 2, "feb10f92": 2, "3167": 2, "41f3": 2, "bb1c": 2, "d271153a31a8": 2, "5b196b22": 2, "9f4c": 2, "489c": 2, "b020": 2, "7823208b42d6": 2, "348101": 2, "722464": 2, "c310f159": 2, "064a": 2, "4035": 2, "97c3": 2, "a25bbf43abc2": 2, "386076": 2, "704104": 2, "f7f24899": 2, "dd50": 2, "409e": 2, "93cc": 2, "6fb1622b60bf": 2, "443038": 2, "725059": 2, "242856d6": 2, "efb5": 2, "4101": 2, "b1cf": 2, "5805532838ac": 2, "373418": 2, "795302": 2, "ce975169": 2, "a0ab": 2, "40ce": 2, "8e32": 2, "efa28d06079d": 2, "stat": 2, "groupbi": 2, "agg": 2, "std": 2, "round": 2, "sort": 2, "sort_valu": 2, "figur": [2, 4], "subplot": 2, "side": 2, "pyplot": 2, "plt": 2, "numpi": 2, "np": 2, "ax1": 2, "ax2": 2, "figsiz": 2, "2ecc71": 2, "3498db": 2, "e74c3c": 2, "bleu_mean": 2, "bleu_std": 2, "enumer": [2, 3], "errorbar": 2, "yerr": 2, "fmt": 2, "markers": 2, "capsiz": 2, "label": [2, 4], "alpha": [2, 4], "set_ylabel": 2, "set_titl": 2, "set_xtick": 2, "set_xticklabel": 2, "rotat": 2, "set_ylim": 2, "bottom": 2, "axi": 2, "legend": 2, "exec_mean": 2, "exec_std": 2, "tight_layout": 2, "ndetail": 2, "4038": 2, "0453": 2, "7815": 2, "0433": 2, "3768": 2, "0424": 2, "8343": 2, "2208": 2, "3519": 2, "0775": 2, "9122": 2, "1482": 2, "377": 2, "042": 2, "83": 2, "078": 2, "slower": 2, "fastest": 2, "04": [2, 3], "latenc": [2, 3], "speed": 2, "interestingli": 2, "longer": 2, "alb": 2, "loubna": 2, "ben": 2, "anton": 2, "lozhkov": 2, "eli": 2, "bakouch": 2, "gabriel": 2, "mart\u00edn": 2, "bl\u00e1zquez": 2, "lewi": 2, "tunstal": 2, "agust\u00edn": 2, "piquer": 2, "andr": 2, "marafioti": 2, "cyril": 2, "zakka": 2, "leandro": 2, "von": 2, "werra": 2, "thoma": 2, "wolf": 2, "are24": 2, "judgearena": 2, "bps99": 2, "salli": 2, "pott": 2, "barbara": 2, "journal": [2, 4], "557": 2, "sciencedirect": 2, "s0377221798003646": 2, "doi": [2, 4], "org": [2, 4], "1016": 2, "s0377": 2, "2217": 2, "00364": 2, "ctj": 2, "jerri": 2, "tworek": 2, "heewoo": 2, "jun": 2, "qime": 2, "yuan": 2, "henriqu": 2, "pond": 2, "de": 2, "oliveira": 2, "pinto": 2, "jare": 2, "kaplan": 2, "harri": 2, "edward": 2, "yuri": 2, "burda": 2, "nichola": 2, "joseph": 2, "greg": 2, "brockman": 2, "rai": 2, "raul": 2, "puri": 2, "gretchen": 2, "krueger": 2, "michael": [2, 4], "petrov": 2, "heidi": 2, "khlaaf": 2, "girish": 2, "sastri": 2, "pamela": 2, "mishkin": 2, "brook": 2, "chan": 2, "scott": 2, "grai": 2, "nick": 2, "ryder": 2, "mikhail": 2, "pavlov": 2, "alethea": 2, "lukasz": 2, "kaiser": 2, "mohammad": 2, "bavarian": 2, "clemen": 2, "winter": 2, "philipp": 2, "tillet": 2, "felip": 2, "petroski": 2, "dave": 2, "cum": 2, "matthia": 2, "plappert": 2, "fotio": 2, "chantzi": 2, "elizabeth": 2, "barn": 2, "ariel": 2, "herbert": 2, "voss": 2, "hebgen": 2, "guss": 2, "nichol": 2, "paino": 2, "nikola": 2, "tezak": 2, "jie": 2, "tang": 2, "igor": 2, "babuschkin": 2, "suchir": 2, "balaji": 2, "shantanu": 2, "jain": 2, "saunder": 2, "christoph": 2, "hess": 2, "andrew": 2, "carr": 2, "jan": 2, "leik": 2, "josh": 2, "achiam": 2, "vedant": 2, "misra": 2, "evan": 2, "morikawa": 2, "alec": 2, "radford": 2, "matthew": 2, "knight": 2, "mile": 2, "brundag": 2, "mira": 2, "murati": 2, "kati": 2, "mayer": 2, "peter": 2, "welind": 2, "bob": [2, 4], "mcgrew": 2, "dario": 2, "amodei": 2, "sam": 2, "mccandlish": 2, "ilya": 2, "sutskev": 2, "wojciech": 2, "zaremba": 2, "arxiv": [2, 4], "ab": [2, 4], "2107": 2, "03374": 2, "cz": 2, "lianmin": 2, "ying": 2, "sheng": 2, "anastasio": 2, "angelopoulo": 2, "tianl": 2, "dacheng": 2, "hao": 2, "zhang": 2, "banghua": 2, "zhu": 2, "jordan": 2, "gonzalez": 2, "ion": 2, "stoica": 2, "2403": 2, "04132": 2, "cho24a": 2, "francoi": 2, "arcpriz": 2, "cho24b": 2, "dglh24": 2, "yann": 2, "bal\u00e1z": 2, "galambosi": 2, "perci": 2, "tatsunori": 2, "hashimoto": 2, "debia": 2, "2404": 2, "04475": 2, "fac24a": 2, "wiki": [2, 4], "fac24b": 2, "fac24c": 2, "doc": [2, 3, 4], "model_doc": 2, "gpt2": 2, "fac24d": 2, "cookbook": 2, "llm_judg": 2, "fac24": 2, "fac24f": 2, "blog": [2, 4], "fhwt23": 2, "cl\u00e9mentin": 2, "nathan": 2, "habib": 2, "hbb": 2, "dan": 2, "collin": 2, "burn": 2, "steven": 2, "basart": 2, "andi": 2, "zou": 2, "manta": 2, "mazeika": 2, "dawn": 2, "song": 2, "jacob": 2, "steinhardt": 2, "03300": 2, "hbd": 2, "ari": 2, "du": 2, "maxwel": 2, "forb": 2, "yejin": 2, "choi": 2, "curiou": 2, "neural": [2, 4], "degener": 2, "1904": 2, "09751": 2, "hyc": 2, "binyuan": 2, "jian": 2, "zeyu": 2, "cui": 2, "jiaxi": 2, "dayiheng": 2, "liu": [2, 4], "lei": 2, "tianyu": 2, "jiajun": 2, "bowen": 2, "yu": 2, "kai": 2, "dang": 2, "coder": 2, "preprint": [2, 4], "2409": 2, "12186": 2, "lx": 2, "zhen": 2, "xiaohan": 2, "xu": 2, "tao": 2, "shen": 2, "jia": 2, "gu": 2, "yuxuan": 2, "lai": 2, "chongyang": 2, "shuai": 2, "ma": 2, "nlg": 2, "2401": 2, "07103": 2, "lbl": 2, "rishi": 2, "bommasani": 2, "toni": 2, "lee": [2, 4], "dimitri": 2, "tsipra": 2, "dilara": 2, "soylu": 2, "michihiro": 2, "yasunaga": 2, "yian": 2, "deepak": 2, "narayanan": 2, "yuhuai": 2, "wu": [2, 4], "ananya": 2, "kumar": 2, "benjamin": 2, "newman": 2, "binhang": 2, "bobbi": 2, "yan": 2, "ce": 2, "christian": 2, "cosgrov": 2, "r\u00e9": 2, "diana": 2, "acosta": 2, "nava": 2, "drew": 2, "hudson": 2, "eric": 2, "zelikman": 2, "esin": 2, "durmu": 2, "faisal": 2, "ladhak": 2, "frieda": 2, "rong": 2, "hongyu": 2, "ren": 2, "huaxiu": 2, "yao": 2, "jue": 2, "keshav": 2, "santhanam": 2, "laurel": 2, "orr": 2, "lucia": 2, "mert": 2, "yuksekgonul": 2, "mirac": 2, "suzgun": 2, "kim": 2, "neel": 2, "guha": 2, "niladri": 2, "chatterji": 2, "omar": 2, "khattab": 2, "henderson": 2, "qian": 2, "huang": 2, "ryan": 2, "chi": [2, 4], "sang": 2, "xie": 2, "shibani": 2, "santurkar": 2, "surya": 2, "ganguli": 2, "icard": 2, "tianyi": 2, "vishrav": 2, "chaudhari": 2, "xuechen": 2, "yifan": 2, "yuhui": 2, "yuta": 2, "koreeda": 2, "2211": 2, "09110": 2, "lbc24": 2, "yuchen": 2, "ronan": 2, "le": 2, "bra": 2, "allenai": 2, "lhe22": 2, "stephani": 2, "hilton": 2, "owain": 2, "mimic": 2, "falsehood": 2, "2109": 2, "07958": 2, "ras24": 2, "sebastian": 2, "scratch": 2, "isbn": 2, "1633437166": 2, "srr": 2, "aarohi": 2, "abhinav": 2, "rastogi": 2, "abhishek": 2, "rao": 2, "abu": 2, "awal": 2, "md": [2, 4], "shoeb": 2, "abubakar": 2, "abid": 2, "adam": 2, "fisch": 2, "brown": 2, "santoro": 2, "aditya": 2, "gupta": 2, "adri\u00e0": 2, "garriga": 2, "alonso": 2, "agnieszka": 2, "kluska": 2, "aitor": 2, "lewkowycz": 2, "akshat": 2, "agarw": 2, "warstadt": 2, "alexand": [2, 4], "kocurek": 2, "ali": 2, "safaya": 2, "tazarv": 2, "alic": [2, 4], "xiang": 2, "alicia": 2, "parrish": 2, "allen": 2, "nie": 2, "aman": 2, "hussain": 2, "amanda": 2, "askel": 2, "dsouza": 2, "ambros": 2, "slone": 2, "ameet": 2, "rahan": 2, "anantharaman": 2, "iyer": 2, "ander": 2, "andreassen": 2, "madotto": 2, "santilli": 2, "stuhlm\u00fcller": 2, "la": 2, "lampinen": 2, "angela": 2, "jiang": 2, "angelica": 2, "anh": 2, "vuong": 2, "animesh": 2, "anna": 2, "gottardi": 2, "antonio": 2, "norelli": 2, "anu": 2, "venkatesh": 2, "arash": 2, "gholamidavoodi": 2, "arfa": 2, "tabassum": 2, "arul": 2, "menez": 2, "arun": 2, "kirubarajan": 2, "asher": 2, "mullokandov": 2, "ashish": 2, "sabharw": 2, "herrick": 2, "avia": 2, "efrat": 2, "aykut": 2, "erdem": 2, "ayla": 2, "karaka\u015f": 2, "robert": 2, "bao": 2, "loe": 2, "barret": 2, "zoph": 2, "bart\u0142omiej": 2, "bojanowski": 2, "batuhan": 2, "\u00f6zyurt": 2, "behnam": 2, "hedayatnia": 2, "neyshabur": 2, "inden": 2, "benno": 2, "stein": 2, "berk": 2, "ekmekci": 2, "blake": 2, "howald": 2, "bryan": 2, "orinion": 2, "cameron": [2, 4], "diao": 2, "dour": 2, "catherin": 2, "stinson": 2, "cedrick": 2, "argueta": 2, "c\u00e9sar": 2, "ferri": 2, "ram\u00edrez": 2, "chandan": 2, "singh": 2, "charl": 2, "rathkopf": 2, "chenlin": 2, "meng": 2, "chitta": 2, "baral": 2, "chiyu": 2, "callison": 2, "burch": 2, "wait": 2, "voigt": 2, "cindi": 2, "ramirez": 2, "clara": 2, "rivera": 2, "clemencia": 2, "siro": 2, "colin": 2, "raffel": 2, "courtnei": 2, "ashcraft": 2, "cristina": 2, "garbacea": 2, "damien": 2, "sileo": 2, "garrett": 2, "kilman": 2, "roth": 2, "daniel": 2, "freeman": 2, "khashabi": 2, "levi": 2, "mosegu\u00ed": 2, "gonz\u00e1lez": 2, "perszyk": 2, "danni": 2, "hernandez": 2, "danqi": 2, "daphn": 2, "ippolito": 2, "dar": 2, "gilboa": 2, "david": 2, "dohan": 2, "drakard": 2, "jurgen": 2, "debajyoti": 2, "datta": 2, "deni": 2, "emelin": 2, "kleyko": 2, "deniz": 2, "yuret": 2, "derek": 2, "tam": [2, 4], "dieuwk": 2, "hupk": 2, "diganta": 2, "dilyar": 2, "buzan": 2, "coelho": 2, "mollo": 2, "diyi": 2, "dong": 2, "ho": 2, "dylan": 2, "schrader": 2, "ekaterina": 2, "shutova": 2, "ekin": 2, "dogu": 2, "cubuk": 2, "elad": 2, "segal": 2, "eleanor": 2, "hagerman": 2, "donowai": 2, "elli": 2, "pavlick": 2, "emanuel": 2, "rodola": 2, "emma": 2, "lam": 2, "chu": 2, "erkut": 2, "erni": 2, "ethan": 2, "dyer": 2, "jerzak": 2, "eunic": 2, "engefu": 2, "manyasi": 2, "evgenii": 2, "zheltonozhskii": 2, "fanyu": 2, "xia": 2, "fatemeh": 2, "siar": 2, "fernando": 2, "mart\u00ednez": 2, "plume": 2, "francesca": 2, "happ\u00e9": 2, "gaurav": 2, "mishra": 2, "genta": 2, "indra": 2, "winata": 2, "gerard": 2, "melo": 2, "germ\u00e1n": 2, "kruszewski": 2, "giambattista": 2, "parascandolo": 2, "giorgio": 2, "mariani": 2, "gloria": 2, "gonzalo": 2, "jaimovitch": 2, "l\u00f3pez": 2, "gregor": 2, "betz": 2, "gui": 2, "gur": 2, "hana": 2, "galijasev": 2, "hannah": 2, "rashkin": 2, "hannaneh": 2, "hajishirzi": 2, "harsh": 2, "mehta": 2, "hayden": 2, "bogar": 2, "henri": 2, "shevlin": 2, "hinrich": 2, "sch\u00fctze": 2, "hiromu": 2, "yakura": 2, "hongm": 2, "hugh": 2, "mee": 2, "wong": 2, "ian": 2, "ng": 2, "isaac": 2, "nobl": 2, "jaap": 2, "jumelet": 2, "jack": 2, "geissing": 2, "jackson": 2, "kernion": 2, "jaehoon": 2, "jaim": 2, "fern\u00e1ndez": 2, "fisac": 2, "jame": 2, "simon": 2, "koppel": 2, "koco\u0144": 2, "jana": 2, "thompson": 2, "janel": 2, "wingfield": 2, "jarema": 2, "radom": 2, "jascha": 2, "sohl": 2, "dickstein": 2, "jason": 2, "phang": 2, "yosinski": 2, "jekaterina": 2, "novikova": 2, "jell": 2, "bosscher": 2, "jennif": 2, "marsh": 2, "jeremi": 2, "jeroen": 2, "taal": 2, "jess": 2, "engel": 2, "jesujoba": 2, "alabi": 2, "jiacheng": 2, "jiam": 2, "jillian": 2, "joan": 2, "waweru": 2, "john": 2, "burden": 2, "miller": 2, "bali": 2, "jonathan": 2, "batcheld": 2, "berant": 2, "j\u00f6rg": 2, "frohberg": 2, "jo": 2, "rozen": 2, "orallo": 2, "boudeman": 2, "guerr": 2, "joshua": 2, "tenenbaum": 2, "joyc": 2, "chua": 2, "kamil": 2, "kanclerz": 2, "karen": 2, "livescu": 2, "karl": 2, "krauth": 2, "karthik": 2, "gopalakrishnan": 2, "katerina": 2, "ignatyeva": 2, "katja": 2, "markert": 2, "kaustubh": 2, "dhole": 2, "kevin": 2, "gimpel": 2, "omondi": 2, "kori": 2, "mathewson": 2, "kristen": 2, "chiafullo": 2, "ksenia": 2, "shkaruta": 2, "shridhar": 2, "kyle": 2, "mcdonel": 2, "richardson": 2, "laria": 2, "reynold": 2, "leo": 2, "gao": 2, "liam": 2, "dugan": 2, "lianhui": 2, "qin": 2, "lidia": 2, "contrera": 2, "ochando": 2, "loui": 2, "morenc": 2, "moschella": 2, "luci": 2, "ludwig": 2, "schmidt": 2, "luheng": 2, "lui": 2, "olivero": 2, "col\u00f3n": 2, "luke": 2, "metz": 2, "l\u00fctfi": 2, "kerem": 2, "\u015fenel": 2, "maarten": 2, "bosma": 2, "sap": 2, "maartj": 2, "hoev": 2, "maheen": 2, "farooqi": 2, "manaal": 2, "faruqui": 2, "marco": 2, "baturan": 2, "marelli": 2, "maru": 2, "maria": 2, "quintana": 2, "mari": 2, "tolkiehn": 2, "mario": 2, "giulianelli": 2, "martha": 2, "martin": 2, "potthast": 2, "leavitt": 2, "hagen": 2, "m\u00e1ty\u00e1": 2, "schubert": 2, "medina": 2, "orduna": 2, "baitemirova": 2, "melodi": 2, "arnaud": 2, "melvin": 2, "mcelrath": 2, "yee": 2, "cohen": 2, "ivanitskii": 2, "starritt": 2, "strube": 2, "micha\u0142": 2, "sw\u0119drowski": 2, "michel": 2, "bevilacqua": 2, "mihir": 2, "kale": 2, "cain": 2, "mime": 2, "mitch": 2, "walker": 2, "mo": 2, "tiwari": 2, "mohit": 2, "bansal": 2, "moin": 2, "aminnaseri": 2, "mor": 2, "geva": 2, "mozhdeh": 2, "gheini": 2, "mukund": 2, "varma": 2, "nanyun": 2, "peng": 2, "nayeon": 2, "neta": 2, "krakov": 2, "doiron": 2, "nicol": 2, "martinez": 2, "nikita": 2, "nangia": 2, "nikla": 2, "decker": 2, "muennighoff": 2, "nitish": 2, "shirish": 2, "keskar": 2, "niveditha": 2, "noah": 2, "constant": 2, "fiedel": 2, "nuan": 2, "wen": 2, "oliv": 2, "agha": 2, "elbaghdadi": 2, "omer": 2, "moreno": 2, "casar": 2, "parth": 2, "doshi": 2, "pascal": 2, "fung": 2, "paul": 2, "pu": 2, "vicol": 2, "pegah": 2, "alipoormolabashi": 2, "peiyuan": 2, "liao": 2, "eckerslei": 2, "phu": 2, "mon": 2, "htut": 2, "pinyu": 2, "hwang": 2, "piotr": 2, "mi\u0142kowski": 2, "piyush": 2, "patil": 2, "pouya": 2, "pezeshkpour": 2, "priti": 2, "oli": 2, "qiaozhu": 2, "mei": 2, "qing": 2, "lyu": 2, "qinlang": 2, "rabin": 2, "banjad": 2, "rachel": 2, "etta": 2, "rudolph": 2, "raefer": 2, "rahel": 2, "haback": 2, "ramon": 2, "risco": 2, "rapha\u00ebl": 2, "milli\u00e8r": 2, "rhythm": 2, "garg": 2, "rif": 2, "saurou": 2, "riku": 2, "arakawa": 2, "robb": 2, "raymaek": 2, "frank": 2, "rohan": 2, "sikand": 2, "roman": 2, "novak": 2, "sitelew": 2, "lebra": 2, "rosann": 2, "rowan": 2, "rui": [2, 4], "ruslan": 2, "salakhutdinov": 2, "stoval": 2, "teehan": 2, "rylan": 2, "sahib": 2, "saif": 2, "sajant": 2, "anand": 2, "dillav": 2, "shleifer": 2, "wiseman": 2, "samuel": 2, "gruetter": 2, "bowman": 2, "schoenholz": 2, "sanghyun": 2, "han": 2, "sanjeev": 2, "kwatra": 2, "sarah": 2, "sarik": 2, "ghazarian": 2, "sayan": 2, "ghosh": 2, "sean": 2, "casei": 2, "bischoff": 2, "gehrmann": 2, "schuster": 2, "sepideh": 2, "sadeghi": 2, "shadi": 2, "hamdan": 2, "sharon": 2, "zhou": 2, "shashank": 2, "sherri": 2, "shi": 2, "shikhar": 2, "shima": 2, "asaadi": 2, "shixiang": 2, "shane": 2, "shubh": 2, "pachchigar": 2, "shubham": 2, "toshniw": 2, "shyam": 2, "upadhyai": 2, "shyamolima": 2, "debnath": 2, "siamak": 2, "shakeri": 2, "thormey": 2, "melzi": 2, "siva": 2, "reddi": 2, "sneha": 2, "priscilla": 2, "makini": 2, "soo": 2, "hwan": 2, "spencer": 2, "toren": 2, "sriharsha": 2, "hatwar": 2, "stanisla": 2, "dehaen": 2, "stefan": 2, "divic": 2, "stefano": 2, "ermon": 2, "stella": 2, "biderman": 2, "stephen": 2, "prasad": 2, "piantadosi": 2, "stuart": 2, "shieber": 2, "summer": 2, "misherghi": 2, "svetlana": 2, "kiritchenko": 2, "swaroop": 2, "tal": 2, "linzen": 2, "tariq": 2, "tatsu": 2, "te": 2, "th\u00e9o": 2, "desbord": 2, "theodor": 2, "rothschild": 2, "phan": 2, "tiberiu": 2, "nkinyili": 2, "timo": 2, "schick": 2, "timofei": 2, "kornev": 2, "titu": 2, "tunduni": 2, "gerstenberg": 2, "trenton": 2, "trishala": 2, "neeraj": 2, "tushar": 2, "khot": 2, "tyler": 2, "shultz": 2, "uri": 2, "shaham": 2, "vera": 2, "demberg": 2, "victoria": 2, "nyamai": 2, "vika": 2, "raunak": 2, "vinai": 2, "ramasesh": 2, "udai": 2, "prabhu": 2, "vishakh": 2, "padmakumar": 2, "vivek": 2, "srikumar": 2, "fedu": 2, "wout": 2, "vossen": 2, "xiaoyu": 2, "tong": 2, "xinran": 2, "zhao": 2, "xinyi": 2, "xudong": 2, "yadollah": 2, "yaghoobzadeh": 2, "yair": 2, "lakretz": 2, "yangqiu": 2, "yasaman": 2, "bahri": 2, "yichi": 2, "yide": 2, "yifu": 2, "yonatan": 2, "belinkov": 2, "hou": 2, "yufang": 2, "yuntao": 2, "bai": 2, "zachari": 2, "seid": 2, "zhuoy": 2, "zijian": 2, "ziji": 2, "j": [2, 4], "zirui": 2, "ziyi": 2, "extrapol": 2, "2206": 2, "04615": 2, "wpn": 2, "yada": 2, "pruksachatkun": 2, "amanpreet": 2, "julian": 2, "felix": 2, "hill": 2, "stickier": 2, "wsm": 2, "1804": 2, "07461": 2, "wtb": 2, "yi": [2, 4], "tai": 2, "borgeaud": 2, "dani": 2, "yogatama": 2, "denni": 2, "donald": 2, "metzler": 2, "ed": 2, "h": 2, "oriol": 2, "vinyal": 2, "dean": 2, "07682": 2, "wdr": 2, "doolei": 2, "manlei": 2, "arka": 2, "pal": 2, "feuer": 2, "siddhartha": 2, "ravid": 2, "shwartz": 2, "ziv": 2, "khalid": 2, "saifullah": 2, "siddartha": 2, "naidu": 2, "chinmai": 2, "hegd": 2, "lecun": 2, "tom": 2, "goldstein": 2, "willi": 2, "neiswang": 2, "micah": 2, "goldblum": 2, "2406": 2, "19314": 2, "yyh": 2, "baosong": 2, "bo": 2, "chengpeng": 2, "chengyuan": 2, "fei": 2, "guant": 2, "haoran": 2, "huan": 2, "jialong": 2, "jialin": 2, "jianhong": 2, "tu": 2, "jianwei": 2, "jianxin": 2, "jin": 2, "jingren": 2, "jinz": 2, "jinzheng": 2, "junyang": 2, "keme": 2, "lu": 2, "keqin": 2, "kexin": 2, "mingfeng": 2, "xue": 2, "ni": 2, "pei": 2, "ru": 2, "men": 2, "ruiz": 2, "runji": 2, "shiji": 2, "sinan": 2, "tan": 2, "tianhang": 2, "tianhao": 2, "wenbin": 2, "ge": 2, "xiaodong": 2, "deng": 2, "xiaohuan": 2, "xingzhang": 2, "xinyu": 2, "xipin": 2, "xuancheng": 2, "fan": 2, "yichang": 2, "wan": 2, "yunfei": 2, "yuqiong": 2, "zhenru": 2, "zhihao": 2, "2407": 2, "10671": 2, "zc": 2, "siyuan": 2, "zhuang": 2, "zhanghao": 2, "yonghao": 2, "zi": 2, "zhuohan": 2, "xing": 2, "2306": 2, "05685": 2, "huggingface24": 2, "06": [2, 4], "metaai24": 2, "promptfoo24": 2, "toolkit": 2, "dev": 2, "far": 3, "possibli": 3, "eliot": 3, "english": 3, "thumb": 3, "\u00be": 3, "max_output_token": 3, "4096": 3, "16384": 3, "contrari": 3, "surpass": 3, "truncat": 3, "max_input_token": 3, "input_cost_per_token": 3, "output_cost_per_token": 3, "11b": 3, "v1": 3, "128000": 3, "5e": 3, "20241022": 3, "8192": 3, "200000": 3, "3e": 3, "0613": 3, "6e": 3, "1e": 3, "gemini": 3, "flash": 3, "1048576": 3, "2097152": 3, "05e": 3, "incomplet": 3, "abruptli": 3, "shallow": 3, "thorough": 3, "dissatisfact": 3, "frustrat": 3, "creation": 3, "feasibl": 3, "split": 3, "10k": 3, "diagram": 3, "charactertextsplitt": 3, "tiktoken": 3, "sequenti": 3, "newlin": 3, "broadli": [3, 4], "want": [3, 4], "sure": [3, 4], "cheap": 3, "speciali": 3, "naiv": 3, "nltk": 3, "spaci": 3, "recurs": 3, "divid": 3, "hierarch": 3, "talk": 3, "theme": 3, "splitter": 3, "markdown": 3, "get_chunk": 3, "chunk_siz": 3, "chunk_overlap": 3, "langchain_text_splitt": 3, "text_splitt": 3, "from_tiktoken_encod": 3, "split_text": 3, "persona": 3, "task": [3, 4], "langchain_cor": [3, 4], "prompttempl": 3, "get_base_prompt_templ": 3, "base_prompt": [3, 4], "from_templ": 3, "llmchain": 3, "togeth": 3, "parser": [3, 4], "output_pars": 3, "stroutputpars": 3, "langchain_commun": 3, "chat_model": 3, "chatlitellm": 3, "get_llm_chain": 3, "prompt_templ": [3, 4], "llm_chain": [3, 4], "api_key_label": 3, "upper": 3, "_api_kei": 3, "get_dynamic_prompt_templ": 3, "get_dynamic_prompt_param": 3, "prompt_param": 3, "part_idx": 3, "total_part": 3, "chat_context": 3, "param": 3, "dynamic_prompt_param": 3, "elif": 3, "merg": 3, "concaten": 3, "generate_report": 3, "input_cont": 3, "llm_model_nam": 3, "report_part": 3, "num_part": 3, "dinam": 3, "priovid": 3, "invok": [3, 4], "cummul": 3, "join": 3, "max_chunk_s": 3, "max_chunk_overlap": 3, "readabl": 3, "apple_report": 3, "luation": 3, "disciplin": 3, "smooth": 3, "subhead": 3, "despit": [3, 4], "depth": 3, "overlook": 3, "preserv": 3, "easier": [3, 4], "preprocess": [3, 4], "necessit": 3, "meticul": 3, "bottleneck": 3, "friendli": 3, "mustafa": 3, "suleyman": 3, "infinit": 3, "fewer": 3, "progress": 3, "condens": 3, "versatil": 3, "drive": [3, 4], "grace": 3, "fallback": 3, "empow": 3, "crucial": [3, 4], "langchain24": 3, "how_to": 3, "freedom": 4, "julia": 4, "easili": 4, "notebook": 4, "overrid": 4, "response_cont": 4, "wow": 4, "lot": 4, "breakdown": 4, "impress": 4, "huge": 4, "ye": 4, "serious": 4, "is_json": 4, "myjson": 4, "valueerror": 4, "trial": 4, "elicit": 4, "wrangl": 4, "hoc": 4, "streamlin": 4, "subsequ": 4, "dataset": 4, "unwant": 4, "ui": 4, "overflow": 4, "overwhelm": 4, "twitter": 4, "youtub": 4, "publish": 4, "schema": 4, "blueprint": 4, "nativ": 4, "json_format": 4, "person1": 4, "q1": 4, "person2": 4, "nest": 4, "todai": 4, "thellm": 4, "unend": 4, "whitespac": 4, "forget": 4, "throw": 4, "somewher": 4, "json_object": 4, "sheer": 4, "circul": 4, "vertex": 4, "worri": 4, "enum": 4, "refus": 4, "simpler": 4, "strongli": 4, "secextract": 4, "mentioned_ent": 4, "mentioned_plac": 4, "extract_from_sec_fil": 4, "sec_filing_text": 4, "hint": 4, "prompt_extract": 4, "sec_extract": 4, "washington": 4, "usabl": 4, "beg": 4, "with_structured_output": 4, "runnabl": 4, "typeddict": 4, "qu": 4, "langchain_openai": 4, "chatopenai": 4, "chatprompttempl": 4, "extract_from_sec_filing_langchain": 4, "structured_llm": 4, "from_messag": 4, "sec_extraction_langchain": 4, "hood": 4, "logit": 4, "willard": 4, "louf": 4, "reformul": 4, "finit": 4, "fsm": 4, "s_": 4, "sim": 4, "s_t": 4, "theta": 4, "s_1": 4, "v": 4, "mathbb": 4, "mask": 4, "tild": 4, "odot": 4, "rightarrow": 4, "boolean": 4, "wise": 4, "formul": 4, "regex": 4, "tran": 4, "thien": 4, "automaton": 4, "dfa": 4, "decod": 4, "outgo": 4, "renorm": 4, "yy": 4, "nn": 4, "ever": 4, "aa": 4, "lwai": 4, "prop": 4, "yynnaa": 4, "qwen": 4, "malform": 4, "sec_extraction_outlin": 4, "zsp": 4, "zicorp": 4, "phenomenon": 4, "popular": 4, "cpp": 4, "gbnf": 4, "ggml": 4, "bnf": 4, "ggerganov": 4, "accomplish": 4, "backu": 4, "naur": 4, "wikipedia": 4, "contributor": 4, "strictli": 4, "soon": 4, "curl": 4, "fssl": 4, "sh": 4, "extract_entities_from_sec_fil": 4, "suffix": 4, "ollama_structured_output_prompt_suffix": 4, "ollama_structured_output_temperatur": 4, "mistral": 4, "llama2": 4, "uncensor": 4, "model_json_schema": 4, "response_json": 4, "wrapper": 4, "exllama2": 4, "mlx": 4, "lm": 4, "medium": 4, "know": 4, "chanc": 4, "correctli": 4, "famili": 4, "furthermor": 4, "nonetheless": 4, "studi": 4, "wrap": 4, "gemma": 4, "uncov": 4, "wors": 4, "extran": 4, "dispar": 4, "preval": 4, "outdat": 4, "rapidli": 4, "fashion": 4, "remark": 4, "me": 4, "speak": 4, "freeli": 4, "aider": 4, "outweigh": 4, "rebutt": 4, "argu": 4, "reproduct": 4, "paint": 4, "pictur": 4, "verif": 4, "dottxt": 4, "flaw": 4, "uneven": 4, "didn": 4, "conflat": 4, "argument": 4, "drawback": 4, "unlock": 4, "wider": 4, "thank": 4, "pfiffer": 4, "aid24": 4, "dot24": 4, "sai": 4, "demo": 4, "tree": 4, "gge24": 4, "blob": 4, "readm": 4, "llf": 4, "xieyang": 4, "frederick": 4, "fiannaca": 4, "terri": 4, "koo": 4, "dixon": 4, "cai": 4, "ea": 4, "ny": 4, "usa": 4, "machineri": 4, "1145": 4, "3613905": 4, "3650756": 4, "ln": 4, "xuan": 4, "hai": 4, "nguyen": 4, "ngoc": 4, "tiviati": 4, "hieu": 4, "dao": 4, "shafiq": 4, "joti": 4, "kenji": 4, "kawaguchi": 4, "nanci": 4, "min": 4, "kan": 4, "2408": 4, "08656": 4, "out24": 4, "twt": 4, "zhi": 4, "cheng": 4, "kuang": 4, "tsai": 4, "chieh": 4, "hung": 4, "yun": 4, "nung": 4, "02442": 4, "tt24": 4, "vivien": 4, "vivien000": 4, "wl23": 4, "brandon": 4, "r\u00e9mi": 4, "2307": 4, "09702": 4, "wikipediacontributors24": 4, "wiktionari": 4, "naur_form": 4}, "objects": {}, "objtypes": {}, "objnames": {}, "titleterms": {"introduct": [0, 1, 4], "content": [0, 2, 3, 4], "core": 0, "challeng": 0, "we": 0, "ll": 0, "address": 0, "A": [0, 1], "practic": [0, 1, 4], "approach": 0, "note": 0, "perspect": 0, "who": 0, "thi": 0, "book": 0, "i": 0, "For": 0, "outcom": 0, "prerequisit": 0, "set": 0, "up": 0, "your": 0, "environ": 0, "python": 0, "setup": 0, "api": [0, 4], "kei": [0, 2, 3], "configur": 0, "code": 0, "repositori": 0, "troubleshoot": 0, "common": 0, "issu": 0, "about": 0, "author": 0, "": 0, "tame": 1, "llm": [1, 2], "guid": 1, "pitfal": 1, "open": 1, "sourc": 1, "softwar": [1, 2], "chapter": 1, "1": [1, 3], "2": [1, 3], "wrestl": [1, 4], "structur": [1, 4], "output": [1, 3, 4], "3": [1, 3], "input": 1, "size": [1, 3], "length": [1, 3], "limit": [1, 3], "4": [1, 3], "5": 1, "The": [1, 2], "eval": [1, 2], "gap": [1, 2], "6": 1, "hallucin": 1, "realiti": 1, "7": 1, "safeti": 1, "concern": 1, "8": 1, "cost": [1, 3], "factor": 1, "9": 1, "break": 1, "free": 1, "from": 1, "cloud": 1, "provid": [1, 4], "appendix": 1, "tool": [1, 2, 4], "resourc": 1, "non": 2, "determinist": 2, "gener": [2, 3], "machin": 2, "temperatur": 2, "sampl": 2, "spectrum": 2, "emerg": 2, "properti": 2, "problem": [2, 3, 4], "statement": [2, 3, 4], "tradit": 2, "v": 2, "design": 2, "applic": 2, "test": 2, "requir": 2, "matrix": 2, "conceptu": 2, "overview": 2, "consider": [2, 3], "metric": 2, "evalu": 2, "task": 2, "model": [2, 3], "base": [2, 3], "human": 2, "benchmark": 2, "leaderboard": 2, "lightev": 2, "mmlu": 2, "econometr": 2, "dataset": 2, "famili": 2, "us": 2, "langsmith": 2, "promptfoo": 2, "refer": [2, 3, 4], "what": 3, "ar": 3, "token": 3, "comparison": [3, 4], "across": 3, "chunk": 3, "contextu": 3, "link": 3, "long": 3, "form": 3, "step": 3, "write": 3, "prompt": [3, 4], "templat": 3, "construct": 3, "dynam": 3, "paramet": 3, "report": 3, "exampl": 3, "usag": 3, "discuss": [3, 4], "implic": 3, "futur": 3, "conclus": [3, 4], "user": 4, "need": 4, "solut": 4, "strategi": 4, "techniqu": 4, "One": 4, "shot": 4, "specif": 4, "json": 4, "mode": 4, "langchain": 4, "outlin": 4, "ollama": 4, "compar": 4, "framework": 4, "best": 4, "research": 4, "ongo": 4, "debat": 4, "acknowledg": 4}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.intersphinx": 1, "sphinxcontrib.bibtex": 9, "sphinx": 57}, "alltitles": {"Introduction": [[0, "introduction"], [4, "introduction"]], "Contents": [[0, "contents"], [2, "contents"], [3, "contents"], [4, "contents"]], "Core Challenges We\u2019ll Address": [[0, "core-challenges-we-ll-address"]], "A Practical Approach": [[0, "a-practical-approach"]], "A Note on Perspective": [[0, "a-note-on-perspective"]], "Who This Book Is For": [[0, "who-this-book-is-for"]], "Outcomes": [[0, "outcomes"]], "Prerequisites": [[0, "prerequisites"]], "Setting Up Your Environment": [[0, "setting-up-your-environment"]], "Python Environment Setup": [[0, "python-environment-setup"]], "API Keys Configuration": [[0, "api-keys-configuration"]], "Code Repository": [[0, "code-repository"]], "Troubleshooting Common Issues": [[0, "troubleshooting-common-issues"]], "About the Author(s)": [[0, "about-the-author-s"]], "Taming LLMs": [[1, "taming-llms"]], "A Practical Guide to LLM Pitfalls with Open Source Software": [[1, "a-practical-guide-to-llm-pitfalls-with-open-source-software"]], "Chapter 1: Introduction": [[1, "chapter-1-introduction"]], "Chapter 2: Wrestling with Structured Output": [[1, "chapter-2-wrestling-with-structured-output"]], "Chapter 3: Input Size and Length Limitations": [[1, "chapter-3-input-size-and-length-limitations"]], "Chapter 4: Output Size and Length Limitations": [[1, "chapter-4-output-size-and-length-limitations"]], "Chapter 5: The Evals Gap": [[1, "chapter-5-the-evals-gap"]], "Chapter 6: Hallucination: The Reality Gap": [[1, "chapter-6-hallucination-the-reality-gap"]], "Chapter 7: Safety Concerns": [[1, "chapter-7-safety-concerns"]], "Chapter 8: The Cost Factor": [[1, "chapter-8-the-cost-factor"]], "Chapter 9: Breaking Free from Cloud Providers": [[1, "chapter-9-breaking-free-from-cloud-providers"]], "Appendix A: Tools and Resources": [[1, "appendix-a-tools-and-resources"]], "The Evals Gap": [[2, "the-evals-gap"]], "Non-Deterministic Generative Machines": [[2, "non-deterministic-generative-machines"]], "Temperature and Sampling": [[2, "temperature-and-sampling"]], "The Temperature Spectrum": [[2, "the-temperature-spectrum"]], "Emerging Properties": [[2, "emerging-properties"]], "Problem Statement": [[2, "problem-statement"], [3, "problem-statement"], [4, "problem-statement"]], "Evals of Traditional Software vs LLMs": [[2, "evals-table"]], "Evals Design": [[2, "evals-design"]], "LLM Application Testing Requirements Matrix": [[2, "validation-requirements"]], "Conceptual Overview": [[2, "conceptual-overview"]], "Design Considerations": [[2, "design-considerations"]], "Metrics": [[2, "metrics"]], "Key Metrics for Evaluating Generative Tasks": [[2, "key-metrics"]], "Evaluators": [[2, "evaluators"]], "Model-Based Evaluation": [[2, "model-based-evaluation"]], "Human-Based Evaluation": [[2, "human-based-evaluation"]], "Evaluating Evaluators": [[2, "evaluating-evaluators"]], "Benchmarks and Leaderboards": [[2, "benchmarks-and-leaderboards"]], "Tools": [[2, "tools"]], "LightEval": [[2, "lighteval"]], "MMLU Econometrics Task Dataset sample": [[2, "mmlu-econometrics"]], "Model Families Evaluated Using LightEval": [[2, "model-families"]], "LangSmith": [[2, "langsmith"]], "PromptFoo": [[2, "promptfoo"]], "References": [[2, "references"], [3, "references"], [4, "references"]], "Output Size Limitations": [[3, "output-size-limitations"]], "What are Token Limits?": [[3, "what-are-token-limits"]], "Token Cost and Length Limitation Comparison Across Key Models": [[3, "token-cost-table"]], "Content Chunking with Contextual Linking": [[3, "content-chunking-with-contextual-linking"]], "Generating long-form content": [[3, "generating-long-form-content"]], "Step 1: Chunking the Content": [[3, "step-1-chunking-the-content"]], "Step 2: Writing the Base Prompt Template": [[3, "step-2-writing-the-base-prompt-template"]], "Step 3: Constructing Dynamic Prompt Parameters": [[3, "step-3-constructing-dynamic-prompt-parameters"]], "Step 4: Generating the Report": [[3, "step-4-generating-the-report"]], "Example Usage": [[3, "example-usage"]], "Discussion": [[3, "discussion"], [4, "discussion"]], "Implications": [[3, "implications"]], "Future Considerations": [[3, "future-considerations"]], "Conclusion": [[3, "conclusion"], [4, "conclusion"]], "Wrestling with Structured Output": [[4, "wrestling-with-structured-output"]], "User Needs": [[4, "user-needs"]], "Solutions": [[4, "solutions"]], "Strategies": [[4, "strategies"]], "Techniques and Tools": [[4, "techniques-and-tools"]], "One-Shot Prompts": [[4, "one-shot-prompts"]], "Structured Output with Provider-Specific APIs": [[4, "structured-output-with-provider-specific-apis"]], "JSON Mode": [[4, "json-mode"]], "LangChain": [[4, "langchain"]], "Outlines": [[4, "outlines"]], "Ollama": [[4, "ollama"]], "Comparing Solutions": [[4, "comparing-solutions"]], "Structured Output Frameworks Comparison": [[4, "structured-output-frameworks"]], "Best Practices": [[4, "best-practices"]], "Research and Ongoing Debate": [[4, "research-and-ongoing-debate"]], "Acknowledgements": [[4, "acknowledgements"]]}, "indexentries": {}}) \ No newline at end of file +Search.setIndex({"docnames": ["markdown/intro", "markdown/toc", "notebooks/evals", "notebooks/output_size_limit", "notebooks/structured_output"], "filenames": ["markdown/intro.md", "markdown/toc.md", "notebooks/evals.ipynb", "notebooks/output_size_limit.ipynb", "notebooks/structured_output.ipynb"], "titles": ["1. Introduction", "Taming LLMs", "4. The Evals Gap", "2. Output Size Limitations", "3. Wrestling with Structured Output"], "terms": {"am": 0, "alwai": [0, 2, 4], "do": [0, 2, 3, 4], "which": [0, 2, 3, 4], "cannot": [0, 2], "order": [0, 2, 4], "mai": [0, 2, 3, 4], "learn": [0, 2], "how": [0, 2, 3, 4], "pablo": [0, 2], "picasso": 0, "In": [0, 2, 3, 4], "recent": [0, 2, 4], "year": [0, 1, 2, 3, 4], "larg": [0, 1, 2, 3, 4], "languag": [0, 1, 2, 3, 4], "model": [0, 1, 4], "llm": [0, 3, 4], "have": [0, 2, 3, 4], "emerg": [0, 1, 4], "transform": [0, 2, 4], "forc": [0, 2, 4], "technologi": [0, 2, 3, 4], "promis": [0, 2], "revolution": 0, "build": [0, 1, 2, 3, 4], "product": [0, 1, 2, 4], "interact": [0, 2, 3, 4], "comput": [0, 2, 3, 4], "from": [0, 2, 3, 4], "chatgpt": [0, 4], "github": [0, 1, 2, 4], "copilot": 0, "claud": [0, 2, 3], "artifact": 0, "system": [0, 2, 3, 4], "captur": [0, 2], "public": [0, 2], "imagin": 0, "spark": 0, "gold": [0, 2], "rush": 0, "ai": [0, 2, 4], "power": [0, 1, 2, 3, 4], "applic": [0, 1, 3, 4], "howev": [0, 2, 3, 4], "beneath": 0, "surfac": [0, 2], "technolog": [0, 2], "revolut": 0, "li": [0, 2], "complex": [0, 2, 3, 4], "landscap": [0, 2], "practition": [0, 2], "must": [0, 2, 3], "navig": [0, 1, 2], "focus": [0, 2, 3, 4], "bring": 0, "awar": [0, 2, 3], "limit": [0, 2, 4], "har": [0, 1, 3], "open": [0, 2, 3, 4], "sourc": [0, 2, 4], "solut": [0, 1, 2, 3], "overcom": [0, 3], "them": [0, 2, 3, 4], "robust": [0, 2, 3, 4], "It": [0, 2, 3, 4], "offer": [0, 2, 3, 4], "critic": [0, 1, 2, 3, 4], "implement": [0, 1, 2, 3, 4], "back": [0, 2, 4], "reproduc": [0, 1, 2], "exampl": [0, 1, 2, 4], "while": [0, 1, 2, 3, 4], "mani": [0, 2, 3, 4], "resourc": [0, 2, 3], "cover": [0, 2, 3], "capabl": [0, 1, 2, 3, 4], "specif": [0, 1, 2, 3], "hidden": 0, "pitfal": 0, "engin": [0, 1, 2, 4], "technic": [0, 1, 2, 3, 4], "manag": [0, 1, 2, 3, 4], "face": [0, 2], "when": [0, 1, 2, 3, 4], "comprehens": [0, 1, 2, 3, 4], "guid": [0, 2, 4], "leverag": [0, 2, 3, 4], "battl": [0, 1], "test": [0, 1, 4], "tool": [0, 3], "throughout": [0, 2, 3, 4], "tackl": [0, 2], "follow": [0, 2, 3, 4], "non": [0, 1, 4], "exhaust": 0, "list": [0, 2, 3, 4], "structur": [0, 2, 3], "un": 0, "reliabl": [0, 2, 4], "struggl": [0, 2, 4], "maintain": [0, 2, 3, 4], "consist": [0, 2, 3, 4], "output": [0, 2], "format": [0, 2, 3, 4], "complic": 0, "integr": [0, 2, 4], "larger": [0, 2, 3, 4], "make": [0, 2, 3, 4], "error": [0, 2, 4], "handl": [0, 1, 2, 3, 4], "more": [0, 2, 3, 4], "size": [0, 2, 4], "length": [0, 2, 4], "constraint": [0, 1, 2, 3, 4], "strict": [0, 4], "token": [0, 1, 2, 4], "both": [0, 2], "input": [0, 2, 3, 4], "requir": [0, 3, 4], "care": [0, 2, 4], "chunk": [0, 1], "strategi": [0, 1, 2, 3], "long": [0, 1, 2, 4], "form": [0, 1, 2, 4], "effect": [0, 2, 3, 4], "tradit": 0, "softwar": [0, 4], "methodologi": [0, 2, 4], "break": [0, 2, 3], "down": [0, 2, 3], "deal": 0, "determinist": [0, 1, 4], "gener": [0, 1, 4], "new": [0, 2, 3, 4], "hallucin": [0, 2, 4], "These": [0, 2, 3, 4], "can": [0, 2, 3, 4], "plausibl": 0, "sound": 0, "entir": [0, 2, 3, 4], "fabric": [0, 2], "inform": [0, 2, 3, 4], "creat": [0, 2, 3, 4], "signific": [0, 2, 3, 4], "risk": [0, 2, 3], "safeti": [0, 2, 4], "secur": [0, 2, 3, 4], "harm": [0, 2], "bias": [0, 2, 4], "inappropri": 0, "safeguard": [0, 2], "monitor": [0, 1, 2], "ensur": [0, 2, 3, 4], "safe": [0, 2, 4], "deploy": [0, 1, 2, 4], "cost": [0, 2, 4], "optim": [0, 1, 2, 3], "The": [0, 3, 4], "financi": [0, 2, 3, 4], "oper": [0, 2, 3, 4], "base": [0, 1, 4], "quickli": [0, 3], "becom": [0, 2, 4], "prohibit": [0, 2], "without": [0, 2, 3, 4], "observ": [0, 2, 4], "vendor": [0, 1, 2], "lock": [0, 1], "cloud": [0, 2, 4], "provid": [0, 2, 3], "depend": [0, 2, 4], "through": [0, 1, 2, 3, 4], "proprietari": [0, 4], "infrastructur": 0, "difficult": [0, 2], "switch": 0, "self": [0, 1, 2], "host": [0, 1, 2], "take": [0, 1, 2, 3, 4], "hand": [0, 3, 4], "concret": [0, 1], "you": [0, 2, 3, 4], "run": [0, 2, 4], "modifi": [0, 2], "real": [0, 2, 3, 4], "world": [0, 2, 4], "scenario": [0, 2, 4], "best": [0, 1, 2], "techniqu": [0, 1, 2, 3], "pattern": [0, 1, 2, 4], "anti": [0, 2], "look": [0, 1, 2], "our": [0, 2, 3, 4], "goal": [0, 2, 3], "discourag": 0, "us": [0, 3, 4], "enabl": [0, 2, 3, 4], "By": [0, 1, 2, 3, 4], "understand": [0, 1, 2, 3, 4], "upfront": [0, 1], "better": [0, 1, 2, 3], "equip": [0, 1, 2], "avoid": [0, 2, 4], "current": [0, 1, 2, 3, 4], "discours": [0, 1], "around": [0, 1, 2, 3, 4], "tend": [0, 1, 2], "toward": [0, 2, 4], "extrem": [0, 2], "either": [0, 2, 3], "uncrit": 0, "enthusiasm": 0, "wholesal": [0, 2], "dismiss": 0, "differ": [0, 2, 3, 4], "focu": [0, 1, 2, 3, 4], "rather": [0, 2], "than": [0, 2], "theoret": 0, "examin": [0, 2, 3, 4], "first": [0, 2, 3, 4], "everi": [0, 2], "concept": [0, 2], "illustr": [0, 2, 3, 4], "execut": [0, 2], "immedi": [0, 2], "analysi": [0, 1, 2, 3], "balanc": [0, 2, 3, 4], "help": [0, 2, 3, 4], "reader": [0, 1], "decis": [0, 2, 4], "intend": [0, 2], "develop": [0, 2, 3, 4], "step": [0, 1, 2, 4], "insight": [0, 2, 3, 4], "along": [0, 2], "guidanc": [0, 4], "framework": [0, 2], "could": [0, 2, 3, 4], "derail": 0, "project": [0, 2], "earli": [0, 2, 4], "befor": [0, 2, 4], "thei": [0, 2, 3, 4], "costli": [0, 2], "problem": [0, 1], "too": [0, 2, 3], "late": 0, "lifecycl": 0, "design": [0, 1, 3, 4], "lead": [0, 2, 3, 4], "genai": 0, "initi": [0, 2, 3, 4], "leader": [0, 2], "architectur": [0, 2, 3, 4], "advoc": 0, "anyon": 0, "seek": [0, 2], "work": [0, 1, 2, 3, 4], "typic": [0, 2, 3, 4], "job": [0, 2], "role": [0, 2, 3, 4], "platform": [0, 2, 3, 4], "backend": [0, 2], "exist": [0, 2], "ml": 0, "transit": [0, 2, 3, 4], "overse": 0, "motiv": [0, 2, 4], "need": [0, 2, 3], "readi": [0, 2], "desir": [0, 2, 4], "perform": [0, 1, 2, 3, 4], "after": [0, 2, 3, 4], "read": [0, 2, 3, 4], "implic": [0, 1, 2], "experi": [0, 2, 3, 4], "recommend": [0, 2, 3, 4], "abl": [0, 2, 3, 4], "deploi": [0, 2, 3], "proper": [0, 4], "realist": 0, "effort": [0, 2, 4], "estim": [0, 2], "impact": [0, 2, 3, 4], "timelin": 0, "To": [0, 2, 3, 4], "most": [0, 2, 3, 4], "should": [0, 2, 3, 4], "basic": [0, 2, 3], "program": [0, 2], "knowledg": [0, 2], "introductori": [0, 1], "langchain": [0, 1, 2, 3], "e": [0, 2, 3, 4], "g": [0, 2, 3, 4], "chat": [0, 2, 3, 4], "prompt": [0, 1, 2], "templat": [0, 1, 2], "access": [0, 2, 3, 4], "openai": [0, 2, 4], "anthrop": [0, 4], "similar": [0, 2, 4], "grade": [0, 2], "dive": 0, "here": [0, 2, 3, 4], "get": [0, 2, 3, 4], "start": [0, 2, 4], "activ": [0, 2], "virtual": [0, 2], "m": [0, 2, 4], "venv": 0, "env": [0, 2, 3, 4], "bin": 0, "On": [0, 2, 4], "window": [0, 1, 2], "script": 0, "instal": [0, 2, 4], "packag": [0, 2, 4], "pip": [0, 2, 4], "r": [0, 2, 3, 4], "txt": [0, 2, 3, 4], "file": [0, 2, 3, 4], "root": 0, "directori": [0, 2], "add": [0, 2, 3], "other": [0, 2, 3, 4], "sensit": [0, 2], "openai_api_kei": 0, "your_openai_api_key_her": 0, "never": [0, 4], "share": [0, 2, 4], "commit": [0, 2], "version": [0, 2, 4], "control": [0, 2, 4], "contain": [0, 2, 3, 4], "kept": [0, 2], "privat": [0, 2], "clone": 0, "companion": 0, "git": 0, "http": [0, 1, 2, 3, 4], "com": [0, 1, 2, 3, 4], "souzatharsi": [0, 1], "tamingllm": [0, 1], "cd": 0, "If": [0, 2, 4], "encount": [0, 1, 2], "rate": [0, 2], "consid": [0, 2, 3, 4], "smaller": [0, 2, 3, 4], "retri": [0, 4], "logic": [0, 2, 3], "conflict": [0, 2], "try": [0, 2, 4], "fresh": 0, "like": [0, 2, 3, 4], "poetri": 0, "check": [0, 2, 4], "page": [0, 2], "known": [0, 2, 4], "now": [0, 2, 3, 4], "let": [0, 2, 3, 4], "begin": [0, 2, 4], "explor": [0, 2, 4], "dr": 0, "tharsi": [0, 1], "souza": [0, 1], "scientist": 0, "special": [0, 2, 4], "he": [0, 2], "lectur": 0, "columbia": 0, "univers": [0, 2], "master": [0, 4], "scienc": [0, 2], "appli": [0, 2, 3, 4], "analyt": 0, "head": [0, 2, 3], "equiti": [0, 2], "citadel": 0, "former": [0, 2], "senior": [0, 2], "vp": 0, "two": [0, 2, 3, 4], "sigma": 0, "invest": [0, 2, 4], "With": [0, 2], "over": [0, 1, 2, 3, 4], "15": [0, 2, 4], "deliv": [0, 2], "across": [0, 2, 4], "startup": 0, "fortun": 0, "500": [0, 2], "compani": [0, 2, 3, 4], "global": [0, 2], "also": [0, 2, 3, 4], "an": [0, 1, 2, 3, 4], "numer": [0, 2], "scholarli": 0, "frequent": [0, 2, 4], "speaker": [0, 2], "academ": [0, 2], "busi": [0, 2], "confer": [0, 4], "ground": [0, 1, 2], "background": [0, 2, 3], "draw": [0, 2, 4], "scale": [0, 2, 4], "stage": [0, 4], "major": [0, 2, 4], "institut": [0, 2], "well": [0, 2, 4], "advis": 0, "profit": [0, 2, 3, 4], "organ": [0, 2, 3], "contribut": [0, 2, 3], "uniqu": [0, 2], "bridg": 0, "gap": 0, "between": [0, 2, 3, 4], "potenti": [0, 2, 3, 4], "next": [0, 2, 4], "hold": [0, 2], "ph": 0, "d": [0, 2, 4], "ucl": 0, "london": 0, "phil": 0, "sc": 0, "b": [0, 2, 4], "abstract": [1, 2, 4], "heavili": [1, 2, 4], "gloss": 1, "fundament": [1, 2, 4], "challeng": [1, 2, 3, 4], "convers": [1, 2, 3, 4], "thi": [1, 2, 3, 4], "book": [1, 2], "kei": [1, 4], "python": [1, 2, 3, 4], "proven": 1, "yet": [1, 2, 3], "i": [1, 2, 3, 4], "unstructur": [1, 4], "context": [1, 2, 3, 4], "code": [1, 2, 4], "sidestep": 1, "inher": [1, 2, 3, 4], "core": [1, 2], "we": [1, 2, 3, 4], "ll": [1, 2], "address": [1, 2, 3, 4], "approach": [1, 2, 3, 4], "note": [1, 2, 3, 4], "perspect": 1, "who": [1, 2, 3, 4], "For": [1, 2, 3, 4], "outcom": [1, 2, 4], "prerequisit": 1, "set": [1, 2, 3, 4], "up": [1, 2, 3, 4], "your": [1, 2, 3, 4], "environ": [1, 2, 3, 4], "setup": [1, 2, 4], "api": [1, 2], "configur": [1, 2], "repositori": [1, 2], "troubleshoot": 1, "common": [1, 2, 3, 4], "issu": [1, 2, 3, 4], "about": [1, 2, 3, 4], "author": [1, 2, 4], "": [1, 2, 3, 4], "statement": 1, "One": [1, 2], "shot": [1, 2], "json": [1, 2, 3], "mode": 1, "outlin": [1, 2], "multipl": [1, 2, 3, 4], "choic": [1, 2, 4], "pydant": [1, 2, 4], "discuss": [1, 2], "compar": [1, 2, 3], "research": [1, 2, 3], "ongo": [1, 2], "debat": 1, "conclus": [1, 2], "acknowledg": [1, 2], "refer": 1, "content": 1, "what": [1, 2, 4], "ar": [1, 2, 4], "contextu": [1, 2], "link": [1, 2], "write": [1, 2, 4], "construct": [1, 2, 4], "dynam": [1, 2], "paramet": [1, 2, 4], "report": [1, 2, 4], "usag": [1, 2, 4], "futur": [1, 2], "consider": [1, 4], "machin": [1, 4], "temperatur": [1, 3, 4], "sampl": [1, 3, 4], "spectrum": 1, "properti": 1, "conceptu": [1, 4], "overview": [1, 4], "compon": [1, 2], "metric": 1, "evalu": [1, 3, 4], "human": [1, 3, 4], "benchmark": 1, "leaderboard": 1, "type": [1, 2, 3, 4], "detect": [1, 2, 4], "retriev": [1, 2], "augment": [1, 2], "rag": 1, "select": [1, 2], "index": [1, 2, 3, 4], "vector": 1, "store": [1, 2, 3], "method": [1, 2, 3, 4], "pipelin": [1, 2, 4], "valid": [1, 2, 4], "guard": 1, "filter": [1, 2, 4], "sanit": 1, "alert": 1, "cach": [1, 2], "invalid": [1, 4], "predict": [1, 2, 4], "llama": [1, 2, 4], "llamafil": 1, "ollama": 1, "migrat": 1, "commun": [1, 2, 4], "misc": 1, "tharsistpsouza2024tamingllm": 1, "t": [1, 2, 3, 4], "p": [1, 2, 4], "titl": [1, 2], "2024": [1, 2, 3, 4], "journal": [1, 2, 4], "url": [1, 2, 4], "doesn": [2, 3, 4], "matter": 2, "beauti": 2, "theori": 2, "smart": 2, "agre": 2, "wrong": 2, "richard": 2, "feynman": 2, "natur": [2, 3, 4], "unlik": 2, "where": [2, 3, 4], "same": [2, 3, 4], "produc": [2, 4], "novel": 2, "text": [2, 3, 4], "train": [2, 4], "data": [2, 3, 4], "respons": [2, 3, 4], "each": [2, 3, 4], "time": [2, 3, 4], "re": [2, 3, 4], "queri": 2, "even": [2, 3, 4], "ident": 2, "behavior": 2, "strength": 2, "ask": [2, 4], "question": [2, 4], "isn": 2, "bug": 2, "featur": [2, 4], "random": [2, 4], "allow": [2, 3, 4], "creativ": [2, 4], "divers": [2, 3, 4], "testabl": 2, "servic": [2, 3, 4], "advic": 2, "mean": [2, 3, 4], "yield": 2, "exceedingli": 2, "regulatori": 2, "complianc": [2, 4], "guarante": [2, 4], "user": [2, 3], "trust": [2, 4], "affect": 2, "inconsist": [2, 4], "primari": 2, "determin": [2, 3, 4], "come": [2, 3, 4], "dure": [2, 4], "calcul": 2, "probabl": [2, 4], "distribut": [2, 4], "nucleu": 2, "holtzman": 2, "et": [2, 4], "al": [2, 4], "2020": 2, "top": [2, 4], "k": [2, 3, 4], "coher": [2, 3], "0": [2, 3, 4], "repetit": [2, 3, 4], "1": [2, 4], "increas": [2, 3, 4], "incoher": 2, "A": [2, 3, 4], "repres": [2, 4], "unscal": 2, "score": [2, 4], "vocabulari": [2, 4], "decreas": 2, "closer": 2, "sharpen": 2, "so": [2, 4], "higher": 2, "uniform": 2, "raschka": 2, "simpl": [2, 3, 4], "singl": [2, 3, 4], "summari": [2, 4], "dramat": [2, 4], "alter": 2, "wai": [2, 3, 4], "systemat": [2, 4], "assess": [2, 3], "At": [2, 4], "rigid": 2, "vari": 2, "less": 2, "2": [2, 4], "wildli": 2, "often": [2, 3, 4], "inadequ": 2, "dotenv": [2, 3, 4], "import": [2, 3, 4], "load_dotenv": [2, 3, 4], "o": [2, 3, 4], "load": [2, 3, 4], "variabl": [2, 3, 4], "panda": 2, "pd": 2, "def": [2, 3, 4], "generate_respons": 2, "model_nam": [2, 3], "str": [2, 3, 4], "float": [2, 3], "attempt": [2, 3], "int": [2, 3], "3": [2, 4], "datafram": 2, "demonstr": [2, 3, 4], "client": [2, 4], "result": [2, 3, 4], "temp": 2, "rang": [2, 3, 4], "complet": [2, 3, 4], "messag": [2, 4], "max_token": 2, "50": 2, "append": [2, 3, 4], "displai": [2, 4], "group": [2, 3], "df_result": 2, "print": [2, 3, 4], "f": [2, 3, 4], "ntemperatur": 2, "40": 2, "temp_respons": 2, "_": [2, 4], "row": 2, "iterrow": 2, "return": [2, 3, 4], "max_length": [2, 4], "10000": [2, 3, 4], "appl": [2, 3, 4], "sec_fil": [2, 4], "unit": [2, 3, 4], "state": [2, 3, 4], "nsecur": 2, "AND": [2, 4], "exchang": [2, 3, 4], "commiss": [2, 3, 4], "nwashington": 2, "c": [2, 4], "20549": 2, "n": [2, 3, 4], "nform": 2, "10": [2, 3, 4], "mark": 2, "annual": 2, "pursuant": 2, "TO": 2, "section": [2, 3, 4], "13": 2, "OR": 2, "OF": 2, "THE": 2, "act": 2, "1934": 2, "nfor": 2, "fiscal": [2, 3], "end": [2, 3, 4], "septemb": [2, 3], "28": [2, 3], "nor": 2, "period": [2, 3], "ncommiss": 2, "number": [2, 3, 4], "001": 2, "36743": 2, "ng66145g66i43": 2, "jpg": 2, "nappl": 2, "inc": [2, 3, 4], "exact": 2, "name": [2, 3, 4], "registr": 2, "specifi": [2, 3, 4], "its": [2, 3, 4], "charter": 2, "ncalifornia": 2, "t94": 2, "2404110": 2, "jurisdict": 2, "nof": 2, "incorpor": 2, "employ": 2, "identif": 2, "No": [2, 4], "none": 2, "park": 2, "ncupertino": 2, "california": [2, 4], "n95014": 2, "princip": 2, "offic": 2, "zip": 2, "408": 2, "996": 2, "1010": 2, "telephon": 2, "includ": [2, 3, 4], "area": [2, 4], "regist": 2, "12": [2, 3], "ntitl": 2, "class": [2, 3, 4], "ttrade": 2, "symbol": 2, "tname": 2, "ncommon": 2, "stock": [2, 4], "00001": 2, "par": 2, "valu": [2, 3, 4], "per": [2, 3], "naapl": 2, "tthe": 2, "nasdaq": [2, 4], "market": [2, 3, 4], "llc": [2, 4], "n0": 2, "000": [2, 4], "due": [2, 3], "2025": 2, "875": 2, "n1": 2, "625": 2, "2026": 2, "n2": 2, "2027": 2, "375": 2, "2029": 2, "n3": 2, "050": 2, "2031": 2, "600": 2, "2042": 2, "nindic": 2, "season": 2, "issuer": 2, "defin": [2, 3, 4], "rule": [2, 3, 4], "405": 2, "nye": 2, "whether": [2, 3, 4], "ha": [2, 4], "all": [2, 3, 4], "preced": 2, "month": 2, "shorter": 2, "wa": [2, 4], "been": 2, "subject": 2, "past": 2, "90": 2, "dai": [2, 4], "submit": 2, "electron": 2, "regul": [2, 4], "232": 2, "chapter": 2, "acceler": 2, "filer": 2, "growth": 2, "see": [2, 4], "definit": [2, 4], "12b": 2, "nlarg": 2, "tacceler": 2, "nnon": 2, "tsmaller": 2, "nemerg": 2, "nif": 2, "indic": [2, 4], "elect": 2, "extend": [2, 4], "compli": [2, 4], "ani": [2, 3, 4], "revis": 2, "account": 2, "standard": 2, "attest": 2, "intern": 2, "under": [2, 4], "404": 2, "sarban": 2, "oxlei": 2, "u": [2, 4], "7262": 2, "firm": 2, "prepar": [2, 3], "audit": 2, "reflect": 2, "correct": [2, 4], "previous": [2, 3, 4], "those": [2, 3, 4], "restat": 2, "recoveri": 2, "incent": 2, "compens": 2, "receiv": [2, 3, 4], "relev": 2, "240": 2, "10d": 2, "shell": 2, "nthe": 2, "aggreg": 2, "vote": 2, "held": [2, 4], "affili": [2, 4], "march": [2, 4], "29": [2, 4], "last": [2, 3, 4], "second": [2, 3], "quarter": 2, "approxim": [2, 4], "628": [2, 4], "553": [2, 4], "sole": 2, "purpos": [2, 4], "disclosur": 2, "director": 2, "date": [2, 4], "exclud": 2, "becaus": 2, "person": [2, 4], "deem": 2, "necessarili": 2, "n15": 2, "115": [2, 4], "823": [2, 4], "were": [2, 4], "outstand": [2, 4], "octob": [2, 4], "18": [2, 4], "ndocument": 2, "BY": 2, "nportion": 2, "proxi": 2, "relat": 2, "meet": [2, 4], "sharehold": 2, "part": [2, 3, 4], "iii": 2, "within": [2, 3, 4], "120": 2, "ntabl": 2, "npage": 2, "npart": 2, "nitem": 2, "nbusi": 2, "1a": 2, "nrisk": 2, "factor": [2, 3, 4], "n5": 2, "1b": 2, "nunresolv": 2, "staff": 2, "comment": 2, "n17": 2, "1c": 2, "ncybersecur": 2, "nproperti": 2, "n18": 2, "nlegal": 2, "proceed": 2, "4": [2, 4], "nmine": 2, "ii": [2, 4], "5": [2, 3, 4], "nmarket": 2, "stockhold": 2, "purchas": 2, "n19": 2, "6": [2, 3, 4], "reserv": 2, "n20": 2, "7": [2, 3], "nmanag": 2, "condit": 2, "n21": 2, "7a": 2, "nquantit": 2, "qualit": 2, "n27": 2, "8": [2, 3], "nfinanci": 2, "supplementari": 2, "n28": 2, "9": 2, "nchang": 2, "disagr": 2, "n51": 2, "9a": 2, "ncontrol": 2, "procedur": 2, "9b": 2, "nother": 2, "n52": 2, "9c": 2, "ndisclosur": 2, "regard": 2, "foreign": 2, "prevent": [2, 4], "inspect": 2, "ndirector": 2, "corpor": 2, "govern": 2, "11": 2, "nexecut": 2, "ownership": 2, "certain": [2, 3, 4], "benefici": 2, "owner": 2, "ncertain": 2, "relationship": 2, "transact": 2, "independ": [2, 4], "14": [2, 4], "nprincip": 2, "fee": 2, "iv": 2, "nexhibit": 2, "schedul": 2, "n53": 2, "16": 2, "n56": 2, "nthi": 2, "forward": 2, "litig": 2, "reform": 2, "1995": 2, "involv": [2, 4], "uncertainti": 2, "locat": 2, "item": 2, "expect": [2, 3, 4], "event": 2, "assumpt": 2, "doe": [2, 3, 4], "directli": [2, 4], "histor": 2, "fact": 2, "macroeconom": 2, "identifi": [2, 3, 4], "word": [2, 3, 4], "anticip": 2, "believ": [2, 4], "plan": [2, 4], "would": [2, 3, 4], "term": [2, 3], "actual": [2, 3, 4], "significantli": [2, 3], "might": [2, 3, 4], "caus": 2, "assum": [2, 3], "oblig": [2, 3], "updat": [2, 3, 4], "reason": [2, 3, 4], "except": [2, 4], "law": 2, "nunless": 2, "otherwis": 2, "present": [2, 3, 4], "herein": 2, "calendar": 2, "particular": [2, 4], "associ": [2, 3, 4], "collect": [2, 3], "wholli": 2, "own": [2, 3], "subsidiari": 2, "unless": 2, "ncompani": 2, "manufactur": 2, "smartphon": 2, "tablet": 2, "wearabl": [2, 4], "accessori": 2, "sell": 2, "varieti": 2, "52": 2, "53": 2, "week": 2, "saturdai": 2, "nproduct": 2, "niphon": 2, "line": 2, "io": [2, 4], "iphon": [2, 4], "pro": [2, 3], "se": 2, "nmac": 2, "maco": 2, "mac": [2, 4], "laptop": 2, "macbook": 2, "air": 2, "desktop": 2, "imac": 2, "mini": [2, 3, 4], "studio": 2, "nipad": 2, "multipurpos": 2, "ipado": 2, "ipad": [2, 4], "nwearabl": 2, "home": 2, "smartwatch": 2, "wireless": 2, "headphon": 2, "spatial": 2, "watcho": 2, "watch": 2, "ultra": 2, "seri": 2, "airpod": 2, "max": 2, "beat": 2, "vision": 2, "visiono": 2, "nhome": 2, "tv": 2, "media": 2, "stream": [2, 4], "game": 2, "devic": [2, 4], "tvo": 2, "homepod": 2, "high": [2, 3, 4], "fidel": [2, 4], "naccessori": 2, "brand": 2, "third": 2, "parti": 2, "nservic": 2, "nadvertis": 2, "advertis": 2, "licens": 2, "arrang": 2, "napplecar": 2, "portfolio": [2, 4], "support": [2, 4], "applecar": 2, "prioriti": 2, "network": [2, 4], "repair": 2, "replac": 2, "case": [2, 3, 4], "addit": [2, 3, 4], "coverag": 2, "instanc": [2, 3], "accident": 2, "damag": 2, "theft": 2, "loss": 2, "countri": 2, "ncloud": 2, "keep": [2, 3], "custom": 2, "avail": [2, 3, 4], "ndigit": 2, "variou": [2, 3, 4], "app": 2, "discov": 2, "download": 2, "digit": 2, "music": 2, "video": 2, "podcast": 2, "subscript": 2, "arcad": 2, "fit": [2, 3, 4], "sm": 2, "curat": 2, "listen": 2, "demand": [2, 4], "radio": 2, "station": 2, "magazin": 2, "exclus": 2, "origin": [2, 3, 4], "live": 2, "sport": 2, "npayment": 2, "payment": 2, "card": 2, "co": 2, "credit": 2, "pai": 2, "cashless": 2, "nsegment": 2, "primarili": 2, "geograph": 2, "basi": 2, "segment": [2, 3, 4], "america": 2, "europ": 2, "greater": 2, "china": 2, "japan": 2, "rest": 2, "asia": 2, "pacif": 2, "north": 2, "south": 2, "european": 2, "india": 2, "middl": 2, "east": 2, "africa": 2, "mainland": 2, "hong": 2, "kong": 2, "taiwan": 2, "australia": 2, "asian": 2, "although": 2, "hardwar": 2, "one": [2, 3, 4], "separ": [2, 3], "align": [2, 3, 4], "partner": 2, "region": 2, "consum": [2, 4], "small": [2, 4], "mid": [2, 3], "educ": [2, 3], "enterpris": [2, 4], "resel": 2, "retail": 2, "onlin": 2, "direct": 2, "sale": 2, "emploi": [2, 4], "indirect": 2, "channel": 2, "cellular": 2, "carrier": 2, "net": [2, 4], "38": 2, "62": 2, "respect": 2, "total": [2, 3, 4], "ncompetit": 2, "highli": [2, 4], "competit": 2, "character": 2, "aggress": 2, "price": 2, "downward": 2, "pressur": 2, "gross": 2, "margin": [2, 4], "introduct": [2, 3], "short": [2, 3, 4], "life": 2, "cycl": 2, "evolv": [2, 3], "industri": [2, 4], "continu": [2, 3, 4], "improv": [2, 3, 4], "characterist": 2, "rapid": 2, "adopt": [2, 4], "advanc": [2, 3, 4], "competitor": 2, "compet": 2, "veri": 2, "low": [2, 4], "imit": 2, "infring": 2, "intellectu": 2, "abil": [2, 4], "successfulli": [2, 4], "innov": [2, 3], "marketplac": 2, "nearli": 2, "rel": 2, "qualiti": [2, 3, 4], "strong": [2, 4], "ecosystem": 2, "reput": 2, "expand": 2, "opportun": 2, "substanti": 2, "establish": 2, "some": [2, 3, 4], "broader": 2, "lower": [2, 4], "particularli": [2, 3, 4], "intens": [2, 4], "cut": [2, 3], "littl": 2, "free": 2, "illegitim": 2, "obtain": [2, 4], "collabor": 2, "nsuppli": 2, "nalthough": 2, "essenti": [2, 3, 4], "particip": 2, "therefor": 2, "wide": [2, 3, 4], "shortag": 2, "commod": 2, "fluctuat": 2, "commonli": 2, "introduc": [2, 3, 4], "util": [2, 3], "onli": [2, 3, 4], "capac": 2, "until": [2, 4], "supplier": 2, "matur": 2, "accept": 2, "decid": [2, 3], "concentr": 2, "instead": [2, 3, 4], "enter": 2, "agreement": 2, "suppli": [2, 4], "renew": 2, "nresearch": 2, "nbecaus": 2, "upon": [2, 3], "flow": [2, 3], "enhanc": [2, 3, 4], "acquisit": 2, "nintellectu": 2, "broad": [2, 4], "right": 2, "aspect": [2, 3, 4], "patent": 2, "copyright": 2, "trademark": 2, "trade": [2, 4], "secret": 2, "differenti": 2, "success": [2, 4], "reli": 2, "skill": 2, "personnel": 2, "regularli": 2, "protect": 2, "aris": 2, "pursu": 2, "thousand": 2, "accumul": 2, "durat": 2, "adequ": 2, "nin": 2, "necessari": [2, 3], "process": [2, 3, 4], "commerci": [2, 4], "experienc": 2, "holidai": 2, "addition": 2, "expens": 2, "fill": 2, "inventori": 2, "launch": 2, "older": 2, "declin": 2, "newer": 2, "distributor": 2, "nhuman": 2, "capit": [2, 3, 4], "peopl": 2, "plai": [2, 4], "strive": 2, "attract": 2, "retain": [2, 3], "talent": 2, "inclus": [2, 3, 4], "team": [2, 4], "member": 2, "As": [2, 3, 4], "had": 2, "164": 2, "full": [2, 3, 4], "equival": 2, "employe": 2, "ncompens": 2, "benefit": [2, 4], "equit": 2, "recogn": 2, "thrive": [2, 4], "succe": 2, "profession": [2, 4], "health": 2, "awai": 2, "ngrowth": 2, "achiev": [2, 4], "career": 2, "leadership": 2, "influenc": [2, 4], "cultur": 2, "advantag": [2, 3, 4], "being": 2, "nworkplac": 2, "practic": [2, 3], "polici": 2, "equal": 2, "workplac": 2, "harass": 2, "discrimin": 2, "ninclus": 2, "sustain": 2, "workforc": 2, "serv": [2, 3, 4], "represent": [2, 3], "level": [2, 3, 4], "foster": [2, 4], "nengag": 2, "honest": 2, "among": 2, "everyon": 2, "grow": [2, 4], "encourag": [2, 4], "feedback": [2, 4], "concern": 2, "conduct": 2, "survei": [2, 4], "gaug": 2, "sentiment": [2, 4], "nhealth": 2, "everywher": 2, "measur": 2, "mitig": [2, 3, 4], "possibl": [2, 4], "hazard": 2, "crisi": 2, "put": 2, "place": [2, 4], "visitor": 2, "navail": 2, "quarterli": 2, "q": 2, "amend": 2, "sec": [2, 3, 4], "Such": 2, "charg": 2, "investor": [2, 4], "default": [2, 4], "aspx": 2, "websit": 2, "www": 2, "press": 2, "releas": [2, 4], "environment": 2, "social": 2, "detail": [2, 3, 4], "referenc": 2, "further": [2, 3, 4], "inact": 2, "textual": 2, "unknown": 2, "describ": 2, "below": [2, 3, 4], "materi": [2, 4], "advers": 2, "trend": [2, 4], "conjunct": 2, "consolid": 2, "accompani": 2, "nmacroeconom": 2, "econom": 2, "outsid": 2, "chain": [2, 3], "facil": 2, "assembli": 2, "site": 2, "nadvers": 2, "slow": 2, "recess": 2, "unemploy": 2, "inflat": 2, "tighter": 2, "interest": [2, 3, 4], "currenc": 2, "confid": [2, 4], "spend": 2, "chang": 2, "monetari": 2, "volatil": 2, "incom": 2, "asset": 2, "contract": 2, "logist": 2, "instabl": 2, "inabl": 2, "financ": 2, "insolv": 2, "failur": 2, "deriv": 2, "counterparti": 2, "debt": 2, "reduc": [2, 3, 4], "liquid": [2, 3], "fair": 2, "instrument": 2, "polit": 2, "disput": 2, "geopolit": 2, "tension": 2, "terror": 2, "disast": 2, "accid": 2, "interrupt": 2, "npolit": 2, "whole": 2, "outsourc": 2, "korea": 2, "vietnam": 2, "restrict": [2, 4], "tariff": 2, "export": 2, "good": [2, 4], "portion": 2, "revenu": [2, 3, 4], "raw": [2, 4], "go": [2, 3, 4], "action": [2, 3], "restructur": 2, "ceas": 2, "accord": [2, 4], "disrupt": [2, 3], "announc": 2, "notic": [2, 4], "led": [2, 4], "escal": [2, 3], "sever": [2, 3, 4], "nmani": 2, "prone": 2, "earthquak": 2, "climat": 2, "weather": 2, "occur": 2, "fire": 2, "nuclear": 2, "plant": 2, "terrorist": 2, "attack": 2, "hostil": 2, "ransomwar": 2, "cybersecur": 2, "labor": 2, "beyond": 2, "nsuch": 2, "imposs": 2, "delai": 2, "ineffici": 2, "slowdown": 2, "outag": 2, "neg": [2, 4], "seriou": 2, "injuri": 2, "pandem": 2, "covid": 2, "19": 2, "economi": 2, "imposit": 2, "stringent": 2, "travel": 2, "freight": 2, "movement": 2, "ramp": 2, "nfollow": 2, "expenditur": 2, "resum": 2, "lose": 2, "exacerb": 2, "consequ": [2, 4], "insur": 2, "insuffici": 2, "nglobal": 2, "unabl": 2, "There": [2, 3, 4], "assur": 2, "contrast": 2, "minor": 2, "overal": [2, 3, 4], "naddition": 2, "intensifi": 2, "seamlessli": [2, 3], "function": [2, 3, 4], "nto": 2, "remain": [2, 3], "stimul": 2, "ndue": 2, "upgrad": 2, "appropri": [2, 3, 4], "quantiti": 2, "defect": 2, "defici": 2, "supersed": 2, "nsubstanti": 2, "much": 2, "transport": 2, "diminish": 2, "flexibl": [2, 3, 4], "respond": 2, "provis": 2, "reimburs": 2, "warranti": 2, "out": [2, 3, 4], "unanticip": 2, "liabil": 2, "adher": [2, 3, 4], "violat": 2, "final": [2, 3, 4], "finish": 2, "destin": 2, "man": 2, "made": [2, 3, 4], "prepay": 2, "termin": 2, "recover": 2, "exposur": 2, "nfutur": 2, "suffici": [2, 4], "semiconductor": 2, "suffer": 2, "poor": 2, "constrain": [2, 3, 4], "shipment": 2, "altern": [2, 3], "sophist": [2, 3], "unexpectedli": 2, "interfer": 2, "unsaf": 2, "artifici": 2, "intellig": 2, "expos": 2, "inaccur": [2, 4], "fix": [2, 3], "widespread": 2, "vulner": 2, "exploit": 2, "compromis": 2, "claim": 2, "recal": 2, "modif": 2, "off": [2, 3, 4], "intang": 2, "fine": [2, 4], "lost": [2, 3], "cancel": 2, "record": 2, "obsolet": 2, "exce": 2, "realiz": 2, "accru": 2, "excess": 2, "review": [2, 4], "impair": 2, "whenev": 2, "circumst": 2, "amount": [2, 3, 4], "carri": [2, 4], "incur": 2, "given": [2, 3, 4], "unpredict": [2, 4], "pace": 2, "obsolesc": 2, "forecast": 2, "150": 2, "incorrectli": [2, 4], "fulli": [2, 3], "extens": [2, 3, 4], "issuanc": 2, "unknowingli": 2, "notifi": 2, "preclud": 2, "choos": 2, "bui": 2, "percept": 2, "android": 2, "playstat": 2, "nintendo": 2, "xbox": 2, "posit": [2, 3, 4], "inclin": 2, "devot": 2, "compel": [2, 4], "fail": 2, "dissatisfi": 2, "vast": 2, "legal": 2, "storefront": 2, "mechan": [2, 4], "safari": 2, "union": 2, "eu": 2, "dma": 2, "interfac": 2, "reduct": 2, "narrow": 2, "scope": [2, 3], "elimin": 2, "nfailur": 2, "appeal": 2, "subscrib": 2, "nsome": 2, "manner": [2, 3, 4], "nurtur": 2, "distinct": 2, "nmuch": 2, "chief": 2, "especi": [2, 3, 4], "silicon": 2, "vallei": 2, "constantli": 2, "driver": 2, "recruit": 2, "subsidi": 2, "staf": 2, "contractor": 2, "placement": 2, "increment": 2, "weaken": 2, "stop": [2, 3], "telecommun": 2, "war": 2, "virus": 2, "physic": 2, "ins": 2, "incid": 2, "redund": 2, "ineffect": 2, "eventu": 2, "thing": [2, 4], "interf": 2, "imped": 2, "ship": 2, "nloss": 2, "unauthor": 2, "confidenti": 2, "encrypt": 2, "But": [2, 4], "absolut": [2, 4], "malici": 2, "behalf": 2, "gain": 2, "regular": [2, 4], "normal": [2, 4], "investig": 2, "penalti": 2, "judgment": 2, "against": 2, "frequenc": [2, 3], "actor": 2, "circumv": [2, 3], "remov": 2, "obfusc": 2, "forens": 2, "evid": [2, 4], "hinder": [2, 4], "recov": 2, "perpetr": 2, "target": [2, 4], "profil": 2, "authent": 2, "hack": 2, "malfeas": 2, "faulti": 2, "password": 2, "irregular": 2, "fraudul": 2, "induc": 2, "disclos": [2, 3, 4], "usernam": 2, "turn": 2, "multifactor": 2, "unusu": 2, "freez": 2, "suspici": 2, "nwhile": 2, "ninvest": 2, "contempl": 2, "endeavor": 2, "distract": 2, "tangibl": 2, "approv": 2, "oner": 2, "ventur": 2, "riski": 2, "pose": [2, 3, 4], "leas": 2, "unfavor": 2, "arisen": 2, "ordinari": 2, "cours": 2, "resolv": 2, "sometim": [2, 4], "indemnif": 2, "indemnifi": 2, "alleg": 2, "magnitud": 2, "assert": 2, "royalti": 2, "vigor": 2, "defend": 2, "court": 2, "internation": 2, "plaintiff": 2, "injunct": 2, "relief": 2, "nregardless": 2, "merit": 2, "recognit": 2, "settl": 2, "uncertain": 2, "abov": 2, "disgorg": 2, "remedi": 2, "worldwid": 2, "antitrust": 2, "privaci": [2, 4], "local": [2, 3, 4], "bill": 2, "commerc": 2, "internet": 2, "mobil": [2, 4], "televis": 2, "film": 2, "anticorrupt": 2, "cash": [2, 3], "repatri": 2, "monei": 2, "launder": 2, "tax": 2, "wast": 2, "recycl": 2, "ncomplianc": 2, "impos": [2, 4], "interpret": 2, "ethic": 2, "agent": 2, "found": [2, 4], "nregulatori": 2, "satisfi": 2, "ban": 2, "nexpect": 2, "stakehold": 2, "increasingli": [2, 4], "greenhous": 2, "ga": 2, "emiss": 2, "civil": 2, "disagre": 2, "perceiv": 2, "feder": 2, "scrutini": 2, "nfrom": 2, "taken": [2, 4], "engag": [2, 4], "noncompli": 2, "individu": [2, 3], "lawsuit": 2, "monopol": 2, "nfurther": 2, "earn": 2, "googl": [2, 4], "search": 2, "nthere": 2, "connect": [2, 4], "retent": 2, "transfer": 2, "pass": [2, 4], "pend": 2, "inquiri": 2, "government": 2, "entiti": [2, 4], "biometr": 2, "breach": 2, "notif": 2, "permit": [2, 4], "healthcar": 2, "liabl": 2, "investigatori": 2, "cardhold": 2, "compress": [2, 3], "acquir": 2, "shift": 2, "mix": [2, 4], "extent": 2, "unexpect": [2, 4], "dollar": 2, "denomin": 2, "rais": [2, 3], "offset": 2, "strengthen": 2, "nconvers": 2, "therebi": [2, 3], "thu": 2, "option": [2, 3, 4], "hedg": 2, "deterior": 2, "sovereign": 2, "heighten": 2, "worsen": 2, "collater": 2, "bank": 2, "unsecur": 2, "subassembli": 2, "assembl": 2, "few": [2, 3, 4], "legisl": 2, "ireland": 2, "singapor": 2, "organis": 2, "propos": 2, "modern": [2, 3, 4], "minimum": 2, "statutori": 2, "valuat": 2, "defer": 2, "bodi": 2, "likelihood": 2, "adequaci": 2, "ultim": 2, "ow": 2, "ngener": 2, "volum": [2, 3], "unrel": 2, "averag": [2, 4], "repurchas": 2, "point": [2, 3], "dividend": 2, "consumm": 2, "declar": 2, "board": 2, "unresolv": 2, "nnone": 2, "threat": 2, "dedic": [2, 4], "postur": 2, "25": 2, "sinc": [2, 3, 4], "2016": 2, "coordin": 2, "assist": [2, 4], "log": 2, "track": 2, "committe": 2, "oversight": 2, "counsel": 2, "chair": 2, "substanc": 2, "17": 2, "headquart": 2, "cupertino": [2, 4], "land": 2, "center": [2, 4], "suitabl": 2, "formal": [2, 4], "articl": [2, 3], "promot": 2, "conclud": 2, "uninstal": 2, "web": 2, "browser": 2, "screen": 2, "june": 2, "24": [2, 4], "preliminari": 2, "find": [2, 3, 4], "contractu": 2, "desist": 2, "stai": [2, 3], "grant": 2, "ndepart": 2, "justic": 2, "21": 2, "depart": 2, "doj": 2, "district": 2, "attornei": 2, "jersei": 2, "redress": 2, "anticompetit": 2, "nonmonetari": 2, "defens": 2, "itself": 2, "nepic": 2, "epic": 2, "northern": 2, "unfair": 2, "guidelin": 2, "enjoin": 2, "extern": 2, "januari": 2, "motion": 2, "enforc": [2, 4], "oppos": 2, "30": 2, "vacat": 2, "fourth": 2, "did": [2, 4], "mine": 2, "nnot": 2, "aapl": 2, "nholder": 2, "na": 2, "23": 2, "301": 2, "npurchas": 2, "nshare": 2, "three": 2, "million": 2, "nperiod": 2, "ttotal": 2, "taverag": 2, "npaid": 2, "publicli": [2, 4], "nannounc": 2, "napproxim": 2, "That": [2, 4], "Be": 2, "nunder": 2, "njune": 2, "august": 2, "nopen": 2, "negoti": 2, "t35": 2, "697": 2, "t224": 2, "naugust": 2, "31": 2, "t42": 2, "910": 2, "t221": 2, "39": 2, "nseptemb": 2, "t33": 2, "653": 2, "t222": 2, "86": 2, "ntotal": 2, "t112": 2, "260": 2, "t89": 2, "074": 2, "110": 2, "billion": 2, "20": [2, 4], "previou": [2, 3, 4], "2023": [2, 4], "10b5": 2, "graph": 2, "show": [2, 3, 4], "comparison": 2, "five": 2, "cumul": 2, "reinvest": 2, "dow": 2, "jone": 2, "supersector": 2, "100": [2, 4], "close": 2, "27": 2, "2019": 2, "n2218": 2, "tseptemb": 2, "2021": 2, "2022": 2, "t100": 2, "t207": 2, "t273": 2, "t281": 2, "t322": 2, "t430": 2, "t113": 2, "t156": 2, "t131": 2, "t155": 2, "t210": 2, "ndow": 2, "t146": 2, "t216": 2, "t215": 2, "nfirst": 2, "nsecond": 2, "nthird": 2, "sequoia": 2, "nfourth": 2, "plu": 2, "nfiscal": 2, "six": 2, "realign": 2, "span": 2, "wherea": 2, "indirectli": 2, "tabl": [2, 3, 4], "n2024": 2, "tchang": 2, "t2023": 2, "t2022": 2, "namerica": 2, "t167": 2, "045": 2, "t3": 2, "t162": 2, "560": 2, "t169": 2, "658": 2, "neurop": 2, "t101": 2, "328": 2, "t7": 2, "294": 2, "t95": 2, "118": 2, "ngreater": 2, "t66": 2, "952": 2, "t72": 2, "559": 2, "t74": 2, "200": 2, "njapan": 2, "t25": 2, "052": 2, "t24": 2, "257": 2, "977": 2, "nrest": 2, "t30": 2, "t4": 2, "t29": 2, "615": 2, "t1": 2, "t391": 2, "035": 2, "t2": 2, "t383": 2, "285": 2, "t394": 2, "weak": 2, "renminbi": 2, "yen": [2, 4], "22": 2, "categori": 2, "t201": 2, "183": 2, "t200": 2, "583": 2, "t205": 2, "489": 2, "984": 2, "357": 2, "t40": 2, "177": 2, "t26": 2, "694": 2, "t28": 2, "300": [2, 3], "292": 2, "t37": 2, "005": 2, "t39": 2, "845": 2, "t41": 2, "241": 2, "n96": 2, "169": 2, "t13": 2, "t85": 2, "t9": 2, "t78": 2, "129": 2, "amort": 2, "bundl": 2, "flat": 2, "entri": 2, "partial": [2, 3], "ngross": 2, "percentag": 2, "t109": 2, "633": 2, "t108": 2, "803": 2, "t114": 2, "728": 2, "t71": 2, "t60": 2, "345": 2, "t56": 2, "054": 2, "t180": 2, "683": 2, "148": 2, "t170": 2, "782": 2, "t36": 2, "t73": 2, "t70": 2, "t46": 2, "t44": 2, "t43": 2, "save": [2, 3], "noper": 2, "t31": 2, "370": 2, "t5": 2, "915": 2, "t14": 2, "251": 2, "npercentag": 2, "t8": 2, "nsell": 2, "administr": 2, "097": 2, "932": 2, "094": 2, "t6": 2, "t57": 2, "467": 2, "t54": 2, "847": 2, "t51": 2, "t15": 2, "driven": 2, "headcount": 2, "nprovis": 2, "749": 2, "t16": 2, "741": 2, "t19": 2, "neffect": 2, "nstatutori": 2, "t21": 2, "aid": 2, "nliquid": 2, "unrestrict": 2, "140": 2, "ndebt": 2, "97": 2, "payabl": 2, "promissori": 2, "paper": [2, 4], "nleas": 2, "space": 2, "nmanufactur": 2, "noncancel": 2, "ndeem": 2, "2017": 2, "tcja": 2, "paid": 2, "nstate": 2, "fund": 2, "escrow": 2, "ncapit": 2, "95": 2, "nrecent": 2, "pronounc": 2, "nincom": 2, "decemb": 2, "fasb": 2, "asu": 2, "09": [2, 3], "topic": [2, 3, 4], "740": 2, "reconcili": 2, "reconcil": [2, 4], "quantit": 2, "threshold": 2, "disaggreg": 2, "prospect": 2, "novemb": 2, "07": [2, 3, 4], "280": 2, "maker": 2, "codm": 2, "alloc": 2, "retrospect": 2, "ncritic": 2, "conform": [2, 4], "principl": 2, "gaap": 2, "nuncertain": 2, "domest": 2, "taxat": 2, "adjust": [2, 3, 4], "resolut": 2, "conting": 2, "26": 2, "still": 2, "ninterest": 2, "forth": 2, "hypothet": 2, "nsensit": 2, "nhypothet": 2, "nrate": 2, "npotenti": 2, "n100": 2, "tenor": 2, "ndeclin": 2, "755": 2, "089": 2, "nterm": 2, "nincreas": 2, "t139": 2, "t194": 2, "nforeign": 2, "express": [2, 4], "var": 2, "mont": 2, "carlo": 2, "simul": [2, 4], "maximum": [2, 3], "interv": 2, "538": 2, "669": 2, "underli": [2, 4], "nindex": 2, "tpage": 2, "nconsolid": 2, "n29": 2, "n30": 2, "sheet": 2, "n31": 2, "n32": 2, "n33": 2, "nnote": 2, "n34": 2, "nreport": 2, "n48": 2, "nall": 2, "omit": [2, 4], "submiss": 2, "nyear": 2, "n2023": 2, "n2022": 2, "nnet": 2, "t294": 2, "866": 2, "t298": 2, "085": 2, "t316": 2, "199": 2, "t96": 2, "ncost": 2, "t185": 2, "233": 2, "t189": 2, "282": 2, "471": 2, "119": 2, "855": 2, "t22": 2, "075": 2, "352": 2, "t214": 2, "137": 2, "t223": 2, "546": 2, "t123": 2, "216": 2, "t119": 2, "437": 2, "t269": 2, "565": 2, "334": 2, "485": 2, "736": 2, "103": 2, "t93": 2, "995": 2, "t99": 2, "nearn": 2, "nbasic": 2, "ndilut": 2, "08": [2, 4], "343": 2, "783": 2, "744": 2, "231": 2, "215": 2, "963": 2, "095": 2, "812": 2, "547": 2, "325": 2, "819": 2, "nsee": 2, "translat": 2, "t395": 2, "765": 2, "511": 2, "unreal": 2, "832": 2, "t323": 2, "212": 2, "nadjust": 2, "337": 2, "717": 2, "394": 2, "138": 2, "850": 2, "563": 2, "104": 2, "t204": 2, "t253": 2, "816": 2, "899": 2, "272": 2, "t98": 2, "016": 2, "652": 2, "t88": 2, "531": 2, "nasset": 2, "ncurrent": 2, "ncash": 2, "943": 2, "965": 2, "228": 2, "590": 2, "naccount": 2, "410": 2, "508": 2, "nvendor": 2, "t32": 2, "833": 2, "477": 2, "ninventori": 2, "286": 2, "331": 2, "287": 2, "695": 2, "t152": 2, "987": 2, "t143": 2, "566": 2, "t91": 2, "479": 2, "544": 2, "t45": 2, "680": 2, "715": 2, "834": 2, "t64": 2, "758": 2, "t211": 2, "993": 2, "t209": 2, "017": 2, "t364": 2, "980": 2, "t352": 2, "nliabil": 2, "t68": 2, "960": 2, "t62": 2, "611": 2, "304": 2, "t58": 2, "829": 2, "ndefer": 2, "249": 2, "061": 2, "ncommerci": 2, "967": 2, "985": 2, "t10": 2, "912": 2, "822": 2, "t176": 2, "392": 2, "t145": 2, "308": 2, "750": 2, "281": 2, "888": 2, "t49": 2, "848": 2, "638": 2, "t308": 2, "030": 2, "t290": 2, "ncommit": 2, "nsharehold": 2, "400": 2, "116": 2, "786": 2, "550": 2, "n83": 2, "276": 2, "naccumul": 2, "deficit": 2, "154": 2, "214": 2, "172": 2, "452": 2, "950": 2, "146": 2, "t50": 2, "672": 2, "t63": 2, "090": 2, "nbegin": 2, "849": 2, "365": 2, "423": 2, "346": 2, "175": 2, "withheld": 2, "settlement": 2, "award": 2, "521": 2, "971": 2, "t12": 2, "034": 2, "t11": 2, "nend": 2, "t83": 2, "nretain": 2, "068": 2, "562": 2, "ndividend": 2, "218": 2, "793": 2, "612": 2, "099": 2, "454": 2, "846": 2, "77": 2, "046": 2, "186": 2, "109": 2, "t163": 2, "rsu": 2, "t0": 2, "98": 2, "94": 2, "32": 2, "737": 2, "929": 2, "ndepreci": 2, "445": 2, "519": 2, "688": 2, "038": 2, "266": 2, "227": 2, "006": 2, "788": 2, "356": 2, "271": 2, "520": 2, "618": 2, "484": 2, "731": 2, "684": 2, "499": 2, "020": 2, "889": 2, "448": 2, "552": 2, "031": 2, "t118": 2, "254": 2, "t110": 2, "543": 2, "t122": 2, "151": 2, "48": 2, "656": 2, "513": 2, "76": 2, "923": 2, "nproce": 2, "211": 2, "686": 2, "917": 2, "135": 2, "828": 2, "446": 2, "447": 2, "959": 2, "708": 2, "086": 2, "935": 2, "705": 2, "354": 2, "nfinanc": 2, "441": 2, "431": 2, "223": 2, "234": 2, "025": 2, "841": 2, "nrepurchas": 2, "949": 2, "89": 2, "402": 2, "465": 2, "nrepay": 2, "958": 2, "repay": 2, "978": 2, "955": 2, "361": 2, "581": 2, "160": 2, "121": 2, "983": 2, "108": 2, "488": 2, "794": 2, "760": 2, "nsupplement": 2, "102": 2, "t18": 2, "679": 2, "573": 2, "33": 2, "nbasi": 2, "prior": 2, "reclassifi": 2, "nrevenu": 2, "remit": 2, "straight": 2, "vest": 2, "treat": 2, "sold": 2, "nderiv": 2, "combin": [2, 3, 4], "nonleas": 2, "34": 2, "entitl": 2, "reward": 2, "commenc": 2, "deliveri": 2, "stand": 2, "alon": 2, "ssp": 2, "object": [2, 4], "icloud": 2, "siri": 2, "map": [2, 4], "discount": 2, "lack": [2, 4], "undeliv": 2, "unbil": 2, "accordingli": 2, "n26": 2, "n37": 2, "35": 2, "proport": 2, "moder": 2, "64": 2, "dilut": 2, "nnumer": 2, "ndenomin": 2, "nweight": 2, "312": 2, "316": 2, "856": 2, "antidilut": 2, "tunreal": 2, "ngain": 2, "tfair": 2, "nvalu": 2, "tcash": 2, "nequival": 2, "tcurrent": 2, "tnon": 2, "t27": 2, "nlevel": 2, "nmonei": 2, "t778": 2, "nmutual": 2, "n515": 2, "t105": 2, "t617": 2, "nsubtot": 2, "293": 2, "395": 2, "nu": 2, "treasuri": 2, "516": 2, "t212": 2, "087": 2, "380": 2, "agenc": 2, "159": 2, "t703": 2, "t17": 2, "568": 2, "158": 2, "810": 2, "ncertif": 2, "deposit": 2, "t873": 2, "t387": 2, "t478": 2, "066": 2, "ncorpor": 2, "t65": 2, "622": 2, "t270": 2, "953": 2, "939": 2, "027": 2, "t47": 2, "886": 2, "nmunicip": 2, "t412": 2, "t405": 2, "t190": 2, "nmortgag": 2, "595": 2, "t175": 2, "403": 2, "t23": 2, "367": 2, "278": 2, "t132": 2, "t583": 2, "635": 2, "t128": 2, "056": 2, "966": 2, "t34": 2, "t160": 2, "t688": 2, "650": 2, "36": 2, "359": 2, "t481": 2, "n442": 2, "t428": 2, "t923": 2, "t909": 2, "406": 2, "114": 2, "468": 2, "136": 2, "t271": 2, "533": 2, "048": 2, "491": 2, "332": 2, "t320": 2, "t608": 2, "t76": 2, "840": 2, "956": 2, "890": 2, "t20": 2, "627": 2, "243": 2, "t628": 2, "t602": 2, "t192": 2, "t410": 2, "735": 2, "636": 2, "t344": 2, "t144": 2, "470": 2, "657": 2, "831": 2, "125": 2, "162": 2, "t173": 2, "752": 2, "quot": 2, "corrobor": 2, "mortgag": 2, "classifi": 2, "37": 2, "cross": 2, "swap": 2, "remeasur": 2, "notion": 2, "069": 2, "730": 2, "575": 2, "493": 2, "t104": 2, "777": 2, "nhedg": 2, "433": 2, "505": 2, "247": 2, "ntrade": 2, "41": 2, "44": 2, "depreci": 2, "nland": 2, "690": 2, "nmachineri": 2, "t80": 2, "205": 2, "314": 2, "nleasehold": 2, "839": 2, "128": 2, "599": 2, "73": 2, "70": 2, "884": 2, "852": 2, "t55": 2, "335": 2, "906": 2, "601": 2, "703": 2, "010": 2, "457": 2, "634": 2, "391": 2, "neuropean": 2, "opinion": 2, "1991": 2, "2007": 2, "irish": 2, "branch": 2, "2003": 2, "2014": 2, "2015": 2, "request": [2, 3, 4], "minist": 2, "juli": 2, "annul": 2, "ecj": 2, "hear": 2, "asid": 2, "confirm": 2, "via": [2, 4], "unrecogn": 2, "nfeder": 2, "571": 2, "080": 2, "644": 2, "265": 2, "801": 2, "726": 2, "570": 2, "298": 2, "49": 2, "t84": 2, "428": 2, "603": 2, "483": 2, "t347": 2, "t669": 2, "076": 2, "830": 2, "419": 2, "072": 2, "pretax": 2, "72": 2, "71": 2, "ncomput": 2, "885": 2, "012": 2, "124": 2, "518": 2, "nimpact": 2, "n10": 2, "246": 2, "311": 2, "366": 2, "397": 2, "153": 2, "nexcess": 2, "893": 2, "871": 2, "192": 2, "739": 2, "ntax": 2, "carryforward": 2, "302": 2, "naccru": 2, "413": 2, "421": 2, "nunreal": 2, "173": 2, "168": 2, "873": 2, "743": 2, "nless": 2, "374": 2, "007": 2, "369": 2, "551": 2, "998": 2, "nright": 2, "179": 2, "nminimum": 2, "674": 2, "940": 2, "t511": 2, "t455": 2, "t490": 2, "805": 2, "202": 2, "indefinit": 2, "temporari": 2, "727": 2, "044": 2, "284": 2, "ndecreas": 2, "386": 2, "463": 2, "982": 2, "542": 2, "936": 2, "070": 2, "expir": 2, "statut": 2, "229": 2, "494": 2, "closur": 2, "intercompani": 2, "exceed": 2, "multiyear": 2, "exercis": 2, "noncash": 2, "rou": 2, "tfinanci": 2, "t2024": 2, "tother": 2, "661": 2, "tproperti": 2, "015": 2, "303": 2, "676": 2, "t165": 2, "t752": 2, "t859": 2, "430": 2, "842": 2, "tfinanc": 2, "n2025": 2, "820": 2, "t171": 2, "991": 2, "n2026": 2, "914": 2, "n2027": 2, "t59": 2, "733": 2, "n2028": 2, "360": 2, "t38": 2, "398": 2, "n2029": 2, "187": 2, "nthereaft": 2, "t837": 2, "undiscount": 2, "790": 2, "imput": 2, "376": 2, "534": 2, "t896": 2, "weight": 2, "borrow": 2, "implicit": 2, "readili": 2, "42": 2, "proce": 2, "nine": 2, "00": 2, "nmatur": 2, "333": 2, "264": 2, "948": 2, "645": 2, "309": 2, "arrear": 2, "namount": 2, "n2013": 2, "nfix": 2, "2062": 2, "t97": 2, "341": 2, "03": 2, "65": 2, "t106": 2, "572": 2, "n97": 2, "nunamort": 2, "premium": 2, "321": 2, "358": 2, "113": 2, "662": 2, "convert": [2, 4], "930": 2, "342": 2, "800": 2, "180": 2, "43": 2, "88": 2, "ndure": 2, "425": 2, "426": 2, "372": 2, "589": 2, "055": 2, "appreci": 2, "four": 2, "holder": 2, "n2014": 2, "bonu": 2, "nrestrict": 2, "nnumber": 2, "nrsu": 2, "ngrant": 2, "naggreg": 2, "nfair": 2, "nbalanc": 2, "t240": 2, "427": 2, "t75": 2, "t150": 2, "861": 2, "501": 2, "768": 2, "87": 2, "101": 2, "878": 2, "144": 2, "t127": 2, "t135": 2, "91": 2, "456": 2, "78": 2, "59": 2, "t140": 2, "80": 2, "326": 2, "t158": 2, "204": 2, "350": 2, "002": [2, 3], "nuncondit": 2, "uncondit": 2, "206": 2, "440": 2, "156": 2, "t633": 2, "t670": 2, "226": 2, "45": 2, "nconting": 2, "least": 2, "accrual": 2, "nconcentr": 2, "attribut": [2, 4], "46": 2, "t67": 2, "098": 2, "082": 2, "062": 2, "569": 2, "895": 2, "458": 2, "207": 2, "nonrecur": 2, "t142": 2, "196": 2, "t138": 2, "t147": 2, "859": 2, "nchina": 2, "n66": 2, "t181": 2, "887": 2, "t172": 2, "269": 2, "nlong": 2, "664": 2, "n4": 2, "797": 2, "778": 2, "219": 2, "47": 2, "nopinion": 2, "nwe": 2, "fairli": 2, "pcaob": 2, "criteria": 2, "sponsor": 2, "treadwai": 2, "2013": 2, "unqualifi": 2, "thereon": 2, "nthese": 2, "misstat": 2, "fraud": 2, "ndescript": 2, "naudit": 2, "nhow": 2, "nmatter": 2, "qualifi": 2, "letter": 2, "advisor": 2, "ernst": 2, "young": 2, "llp": 2, "auditor": 2, "2009": 2, "nsan": 2, "jose": 2, "nnovemb": 2, "coso": 2, "nour": 2, "ndefinit": 2, "pertain": 2, "mainten": 2, "accur": [2, 4], "disposit": 2, "receipt": 2, "degre": 2, "nevalu": 2, "nbase": 2, "supervis": 2, "13a": 2, "15d": 2, "summar": [2, 3], "ninher": 2, "met": 2, "appear": [2, 4], "paragraph": 2, "51": [2, 4], "ninsid": 2, "deirdr": 2, "brien": 2, "vice": 2, "presid": 2, "affirm": 2, "april": 2, "withhold": 2, "remitt": 2, "jeff": 2, "william": 2, "mr": 2, "insid": 2, "copi": [2, 3], "exhibit": 2, "solicit": 2, "document": [2, 3, 4], "id": 2, "00042": 2, "nincorpor": 2, "texhibit": 2, "descript": [2, 4], "tform": 2, "tfile": 2, "nrestat": 2, "n8": 2, "namend": 2, "bylaw": 2, "nindentur": 2, "york": [2, 4], "mellon": 2, "truste": 2, "noffic": 2, "certif": 2, "2018": 2, "85": 2, "2043": 2, "05": 2, "2044": 2, "februari": 2, "55": 2, "2045": 2, "900": 2, "700": 2, "60": 2, "250": 2, "2036": 2, "2046": 2, "450": 2, "2047": 2, "2049": 2, "2030": 2, "2050": 2, "2060": 2, "2028": 2, "2041": 2, "2051": 2, "2061": 2, "2032": 2, "2052": 2, "54": 2, "2033": 2, "2053": 2, "n9": 2, "ceo": 2, "n12": 2, "nsubsidiari": 2, "n23": 2, "nconsent": 2, "n24": 2, "npower": 2, "signatur": 2, "nrule": 2, "nsection": 2, "1350": 2, "n101": 2, "ninlin": 2, "xbrl": 2, "n104": 2, "inlin": 2, "compensatori": 2, "herewith": 2, "furnish": 2, "herebi": 2, "undertak": 2, "56": 2, "nsignatur": 2, "npursuant": 2, "duli": 2, "sign": 2, "undersign": 2, "thereunto": 2, "ndate": 2, "nby": 2, "luca": [2, 4], "maestri": 2, "nluca": 2, "nsenior": 2, "nchief": 2, "nknow": 2, "THESE": 2, "whose": 2, "constitut": 2, "appoint": 2, "timothi": 2, "cook": 2, "jointli": 2, "hi": [2, 4], "her": 2, "substitut": 2, "him": 2, "thereto": 2, "therewith": 2, "ratifi": 2, "said": 2, "done": [2, 4], "virtu": 2, "hereof": 2, "nname": 2, "ttitl": 2, "tdate": 2, "tchief": 2, "tnovemb": 2, "ntimothi": 2, "tsenior": 2, "chri": 2, "kondo": 2, "nchri": 2, "wanda": 2, "austin": 2, "nwanda": 2, "alex": 2, "gorski": 2, "tdirector": 2, "nalex": 2, "andrea": 2, "jung": 2, "nandrea": 2, "arthur": 2, "levinson": 2, "narthur": 2, "monica": 2, "lozano": 2, "nmonica": 2, "ronald": 2, "sugar": 2, "nronald": 2, "susan": 2, "l": 2, "wagner": 2, "nsusan": 2, "57": 2, "gpt": [2, 3, 4], "turbo": [2, 3, 4], "invdestacksmeticsisdict": 2, "setispect": 2, "20cyan": 2, "evaluationseld": 2, "anvis": 2, "droitent": 2, "discernminerv": 2, "versbobprefvers": 2, "vo\u8be5": 2, "option\u548c": 2, "meio": 2, "\u0432\u0440\u0435\u043ccisco": 2, "dellaischenpoihscap": 2, "geme": 2, "gettim": 2, "radic": 2, "grappl": 2, "probabilist": 2, "seem": [2, 4], "safer": 2, "don": [2, 3, 4], "highlight": [2, 3, 4], "paradigm": 2, "anoth": 2, "fascin": 2, "spontan": 2, "answer": [2, 3, 4], "aren": 2, "explicitli": 2, "pre": 2, "fig": [2, 3, 4], "linear": 2, "absent": 2, "simpli": [2, 3, 4], "coax": 2, "onc": [2, 3], "reach": [2, 3, 4], "journei": 2, "suddenli": 2, "manifest": 2, "call": [2, 3, 4], "phase": 2, "stark": 2, "deliber": 2, "wei": 2, "convent": 2, "stabl": 2, "suit": 2, "contend": 2, "7b": 2, "70b": 2, "rethink": 2, "math": 2, "tutor": 2, "children": 2, "verifi": [2, 4], "just": [2, 3, 4], "predefin": [2, 4], "adapt": [2, 3], "explan": [2, 4], "child": 2, "ag": 2, "bound": 2, "weren": 2, "accuraci": [2, 4], "kind": 2, "dimens": 2, "explicit": [2, 4], "usual": 2, "precis": [2, 4], "resist": 2, "straightforward": [2, 3, 4], "quantif": 2, "contamin": 2, "carefulli": [2, 4], "craft": [2, 4], "massiv": 2, "alreadi": 2, "seen": 2, "memor": 2, "truli": 2, "unseen": 2, "rigor": 2, "evolut": 2, "longitudin": 2, "autom": [2, 4], "annot": 2, "mostli": [2, 4], "versu": 2, "latter": 2, "foundat": [2, 3], "tailor": 2, "solv": [2, 4], "great": [2, 4], "cognit": 2, "meta": [2, 3], "why": [2, 4], "misinform": 2, "factual": 2, "databas": [2, 4], "citat": 2, "tempor": 2, "scientif": 2, "fals": [2, 4], "manipul": 2, "medic": 2, "disclaim": 2, "referr": 2, "boundari": 2, "situat": [2, 3], "incorrect": 2, "expertis": 2, "bia": [2, 4], "gender": 2, "racial": 2, "demograph": 2, "stereotyp": 2, "reinforc": 2, "societ": 2, "pii": 2, "anonym": 2, "leakag": 2, "carryov": 2, "protocol": 2, "multi": [2, 4], "mathemat": 2, "fallaci": 2, "causal": 2, "edg": 2, "think": 2, "idiom": 2, "sarcasm": 2, "terminologi": 2, "lingual": 2, "misunderstand": 2, "syntax": 2, "scan": 2, "compat": [2, 4], "stabil": 2, "effici": [2, 3, 4], "scalabl": [2, 3], "overconfid": 2, "clariti": [2, 3, 4], "audienc": 2, "densiti": 2, "satisfact": [2, 4], "misus": 2, "moral": 2, "transpar": [2, 4], "co2": 2, "energi": 2, "consumpt": 2, "server": [2, 4], "batch": 2, "infer": 2, "imag": 2, "audio": 2, "etc": [2, 4], "truth": [2, 4], "layer": [2, 3, 4], "palm": 2, "shown": 2, "quantifi": 2, "rank": 2, "easi": [2, 3], "synthet": [2, 4], "post": [2, 4], "timeout": 2, "variat": 2, "maxim": 2, "inter": 2, "rater": 2, "priorit": 2, "ti": 2, "tier": 2, "holist": 2, "built": [2, 4], "mind": 2, "x": 2, "fast": [2, 4], "experiment": [2, 4], "iter": [2, 3, 4], "vi": 2, "later": [2, 4], "categor": [2, 4], "intrins": 2, "extrins": 2, "sequenc": [2, 4], "perplex": 2, "downstream": [2, 4], "valuabl": [2, 4], "distinguish": 2, "classif": [2, 4], "true": [2, 3, 4], "synthesi": 2, "discret": 2, "f1": 2, "match": [2, 4], "prefix": 2, "roug": 2, "bleu": 2, "charact": [2, 3, 4], "gram": 2, "bilingu": 2, "understudi": 2, "overlap": [2, 3], "favor": [2, 4], "breviti": 2, "insensit": 2, "semant": [2, 3], "orient": 2, "gist": 2, "sentenc": [2, 3, 4], "ignor": 2, "meteor": 2, "synonym": 2, "stem": [2, 4], "paraphras": 2, "alongsid": 2, "computation": [2, 3], "cider": 2, "consensu": 2, "tf": 2, "idf": 2, "caption": 2, "reliant": 2, "corpu": 2, "statist": 2, "ter": 2, "edit": 2, "hypothesi": 2, "penal": 2, "bertscor": 2, "embed": [2, 3], "bert": 2, "spice": 2, "proposit": 2, "scene": 2, "emphasi": 2, "pure": 2, "analyst": [2, 3], "dictionari": [2, 4], "rouge_1": 2, "rouge_2": 2, "ideal": [2, 4], "expert": [2, 3, 4], "cheaper": 2, "4o": [2, 3, 4], "evaluate_summari": 2, "unigram": 2, "bigram": 2, "huggingfac": 2, "librari": [2, 3, 4], "absl": 2, "py": 2, "rouge_scor": 2, "generated_summari": 2, "reference_summari": 2, "arg": [2, 3, 4], "dict": [2, 3, 4], "google_bleu": 2, "bleu_scor": 2, "rouge1": 2, "rouge2": 2, "arbitrari": 2, "chosen": 2, "sentence1": 2, "cat": 2, "sat": 2, "mat": 2, "sentence2": 2, "ate": 2, "3333333333333333": 2, "7272727272727272": 2, "4444444444444445": 2, "generate_summari": 2, "summir": 2, "correspond": [2, 4], "liner": 2, "excerpt": 2, "evaluate_summary_model": 2, "model_benchmark": 2, "models_test": 2, "benchmark_summari": 2, "model_summari": 2, "evaluation_result": 2, "reveal": 2, "analyz": [2, 3, 4], "statu": 2, "concis": 2, "element": [2, 4], "Its": 2, "verbos": 2, "peripher": 2, "quit": [2, 4], "overli": [2, 4], "simplifi": [2, 4], "miss": 2, "convei": [2, 3], "breadth": 2, "Of": 2, "vibe": 2, "visualize_prompt_comparison": 2, "visual": 2, "matplotlib": 2, "radar": 2, "plot": 2, "radar_plot": 2, "tmp": 2, "ipykernel_1652501": 2, "940173201": 2, "userwarn": 2, "figurecanvasagg": 2, "closest": 2, "largest": 2, "deviat": [2, 4], "suggest": [2, 4], "mention": [2, 4], "nuanc": [2, 3, 4], "granular": [2, 3], "fall": 2, "judg": 2, "themselv": 2, "main": [2, 3, 4], "instruct": [2, 3, 4], "tune": [2, 4], "assign": [2, 4], "likert": 2, "style": 2, "pairwis": 2, "ensembl": 2, "repeatedli": 2, "domain": 2, "clear": [2, 4], "fluenci": 2, "refin": 2, "excel": [2, 4], "narr": 2, "mirror": 2, "similarli": 2, "notabl": [2, 4], "properli": [2, 4], "henc": 2, "worth": 2, "integ": 2, "rubric": 2, "hollist": 2, "judgeevalu": 2, "grammar": [2, 4], "evaluate_with_llm": 2, "candid": 2, "pars": [2, 4], "criterion": 2, "basemodel": [2, 4], "judge_model": 2, "candidate_summari": 2, "written": 2, "grammat": 2, "y": [2, 4], "z": 2, "w": [2, 3], "beta": [2, 4], "response_format": [2, 4], "Then": 2, "benchmark_model": 2, "test_model": 2, "input_text": [2, 3], "tupl": 2, "trillion": [2, 4], "evals_list": 2, "1775618912": 2, "variant": 2, "slightli": 2, "drift": 2, "lowest": 2, "drop": 2, "gradient": 2, "visibl": 2, "degrad": [2, 4], "firstli": 2, "overhead": 2, "neglect": 2, "prefer": [2, 4], "egocentr": 2, "tight": 2, "field": [2, 4], "aproach": 2, "workflow": [2, 4], "assessor": 2, "aplic": 2, "aim": [2, 3, 4], "clearli": [2, 4], "earlier": 2, "depict": [2, 4], "correl": 2, "multilingu": 2, "golden": 2, "languang": 2, "arena": 2, "blind": 2, "randomli": 2, "pair": 2, "loop": 2, "customiz": 2, "irrelev": 2, "unhelp": 2, "though": [2, 4], "occasion": 2, "rare": 2, "inaccuraci": 2, "perfectli": 2, "cater": 2, "critiqu": 2, "elo": 2, "democrat": [2, 4], "thought": [2, 4], "exam": 2, "probe": 2, "certifi": 2, "histori": 2, "move": [2, 3], "began": 2, "glue": 2, "wang": 2, "entail": 2, "baselin": 2, "superglu": 2, "deeper": [2, 3], "successor": 2, "grew": 2, "big": 2, "bench": 2, "srivastava": 2, "arithmet": 2, "truthfulqa": 2, "lin": [2, 4], "decept": 2, "multitask": 2, "hendryck": 2, "multidisciplinari": 2, "stanford": 2, "helm": 2, "liang": 2, "multidimension": 2, "surround": [2, 4], "emphas": [2, 4], "humanev": 2, "chen": [2, 4], "lmsy": 2, "brought": 2, "dialogu": 2, "len": [2, 3], "replic": [2, 4], "chatbot": 2, "chiang": 2, "gather": 2, "alpacaev": 2, "duboi": 2, "mt": 2, "zheng": 2, "Their": [2, 4], "render": 2, "crowdsourc": 2, "livebench": 2, "white": 2, "resili": 2, "meaningfulli": 2, "monthli": 2, "zebralog": 2, "grid": 2, "puzzl": 2, "brailsford": 2, "1999": 2, "lsat": 2, "hous": 2, "clue": 2, "strateg": [2, 4], "deduct": 2, "arriv": 2, "programmat": [2, 4], "2x2": 2, "6x6": 2, "reductio": 2, "ad": [2, 4], "absurdum": 2, "sonnet": [2, 3], "hard": 2, "10b": 2, "counterfactu": 2, "composit": 2, "came": 2, "arc": 2, "prize": 2, "chollet": 2, "mike": 2, "knoop": 2, "founder": 2, "zapier": 2, "fran\u00e7oi": 2, "creator": 2, "agi": 2, "kera": 2, "meaning": [2, 3, 4], "genuin": 2, "old": 2, "possess": 2, "count": [2, 3], "elementari": 2, "novelti": 2, "someth": 2, "wouldn": 2, "interpol": 2, "memori": [2, 3], "synthes": 2, "fly": 2, "brute": 2, "minim": [2, 4], "pixel": 2, "perfect": 2, "color": 2, "unbeaten": 2, "win": 2, "deep": [2, 4], "poorli": 2, "recombin": 2, "spur": 2, "art": 2, "takeawai": 2, "algorithm": 2, "fourrier": 2, "lightweight": [2, 4], "bespok": 2, "sdk": 2, "cli": 2, "extract": [2, 3, 4], "autoregress": 2, "sub": 2, "liter": 2, "disturb": 2, "zero": [2, 4], "varianc": 2, "yt": 2, "ut": 2, "suppos": [2, 4], "exactli": [2, 4], "ol": 2, "heteroscedast": 2, "regress": 2, "wish": 2, "lag": 2, "bivari": 2, "evaluation_track": 2, "evaluationtrack": 2, "model_config": 2, "basemodelconfig": 2, "parallelismmanag": 2, "pipelineparamet": 2, "envconfig": 2, "is_accelerate_avail": 2, "datetim": 2, "timedelta": 2, "initprocessgroupkwarg": 2, "create_evaluation_pipelin": 2, "output_dir": 2, "cache_dir": 2, "pretrain": 2, "dtype": 2, "float16": 2, "max_sampl": 2, "kwargs_handl": 2, "3000": 2, "els": [2, 3], "save_detail": 2, "push_to_hub": 2, "pipeline_param": 2, "launcher_typ": 2, "env_config": 2, "override_batch_s": 2, "use_chat_templ": 2, "trust_remote_cod": 2, "pipeline_paramet": 2, "schemat": [2, 3], "vllm": [2, 4], "tgi": 2, "instanti": 2, "storag": 2, "push": 2, "hub": 2, "parallel": 2, "num_few_shot": 2, "automat": 2, "string": [2, 4], "vertic": 2, "bar": 2, "binari": 2, "flag": 2, "bigbench": 2, "winogrand": 2, "hellaswag": 2, "nlp": 2, "save_and_push_result": 2, "show_result": 2, "model_arg": 2, "remot": 2, "send": [2, 4], "serverless": 2, "inference_server_address": 2, "inference_server_auth": 2, "model_id": 2, "null": 2, "bash": 2, "command": 2, "model_config_path": 2, "path": [2, 3], "endpoint_model": 2, "yaml": [2, 4], "llama3": [2, 3], "qwen2": [2, 4], "smollm2": 2, "3b": 2, "alibaba": [2, 4], "5b": [2, 4], "hui": 2, "yang": 2, "compact": 2, "360m": 2, "allal": 2, "cluster": 2, "noteworthi": 2, "superior": 2, "grain": [2, 4], "salt": [2, 4], "give": 2, "exponenti": 2, "hug": 2, "modular": 2, "visit": 2, "offici": 2, "revisit": 2, "rememb": 2, "api_kei": [2, 3], "trace": 2, "langchain_tracing_v2": 2, "langchain_api_kei": 2, "hf_evalu": 2, "langsmith_evalu": 2, "ls_client": 2, "dataset_nam": 2, "create_dataset": 2, "create_exampl": 2, "dataset_id": 2, "calculate_scor": 2, "reference_output": 2, "oai_client": 2, "xp_model_nam": 2, "lastli": 2, "run_evalu": 2, "upload": 2, "And": 2, "upload_result": 2, "experiment_prefix": 2, "num_repetit": 2, "view": 2, "386a3620": 2, "smith": 2, "9e1cc3cb": 2, "9d6a": 2, "4356": 2, "ab34": 2, "138e0abe8be4": 2, "8741976e": 2, "5268": 2, "4b75": 2, "949f": 2, "99477dde5d64": 2, "selectedsess": 2, "b831dc1e": 2, "90bc": 2, "4ed8": 2, "8080": 2, "fb42444724d6": 2, "4it": 2, "latest": [2, 3, 4], "modul": [2, 4], "tobia": 2, "evaluate_modul": 2, "6fc70b7be0088120a372dfdd5d320b39b8bb3630cb8029b193941d9376e86bb0": 2, "tue": 2, "nov": 2, "couldn": 2, "5it": 2, "5053784e": 2, "64445871": 2, "a53c": 2, "44b1": 2, "a422": 2, "4f49b2f9656f": 2, "69": 2, "4b29f3c9": 2, "9ef7e39a": 2, "2add": 2, "410c": 2, "89f8": 2, "9f1a8b198cf1": 2, "61": 2, "df": 2, "to_panda": 2, "insert": 2, "combined_df": 2, "concat": 2, "ignore_index": 2, "execution_tim": 2, "example_id": 2, "333333": 2, "224388": 2, "feb10f92": 2, "3167": 2, "41f3": 2, "bb1c": 2, "d271153a31a8": 2, "5b196b22": 2, "9f4c": 2, "489c": 2, "b020": 2, "7823208b42d6": 2, "348101": 2, "722464": 2, "c310f159": 2, "064a": 2, "4035": 2, "97c3": 2, "a25bbf43abc2": 2, "386076": 2, "704104": 2, "f7f24899": 2, "dd50": 2, "409e": 2, "93cc": 2, "6fb1622b60bf": 2, "443038": 2, "725059": 2, "242856d6": 2, "efb5": 2, "4101": 2, "b1cf": 2, "5805532838ac": 2, "373418": 2, "795302": 2, "ce975169": 2, "a0ab": 2, "40ce": 2, "8e32": 2, "efa28d06079d": 2, "stat": 2, "groupbi": 2, "agg": 2, "std": 2, "round": 2, "sort": 2, "sort_valu": 2, "figur": [2, 4], "subplot": 2, "side": 2, "pyplot": 2, "plt": 2, "numpi": 2, "np": 2, "ax1": 2, "ax2": 2, "figsiz": 2, "2ecc71": 2, "3498db": 2, "e74c3c": 2, "bleu_mean": 2, "bleu_std": 2, "enumer": [2, 3], "errorbar": 2, "yerr": 2, "fmt": 2, "markers": 2, "capsiz": 2, "label": [2, 4], "alpha": [2, 4], "set_ylabel": 2, "set_titl": 2, "set_xtick": 2, "set_xticklabel": 2, "rotat": 2, "set_ylim": 2, "bottom": 2, "axi": 2, "legend": 2, "exec_mean": 2, "exec_std": 2, "tight_layout": 2, "ndetail": 2, "4038": 2, "0453": 2, "7815": 2, "0433": 2, "3768": 2, "0424": 2, "8343": 2, "2208": 2, "3519": 2, "0775": 2, "9122": 2, "1482": 2, "377": 2, "042": 2, "83": 2, "078": 2, "slower": 2, "fastest": 2, "04": [2, 3], "latenc": [2, 3], "speed": 2, "interestingli": 2, "longer": 2, "decoupl": 2, "friendli": [2, 3], "reload": 2, "facilit": 2, "pleas": 2, "promptfooconfig": 2, "model_comparison": 2, "config": 2, "safe_load": 2, "pretti": 2, "dump": 2, "default_flow_styl": 2, "sort_kei": 2, "prompt1": 2, "defaulttest": 2, "1000": 2, "ye": [2, 4], "csv": 2, "1000m": 2, "sure": [2, 3, 4], "millisecond": 2, "eval_data": 2, "latency_m": 2, "totallatencym": 2, "token_usag": 2, "tokenusag": 2, "assert_pass": 2, "assertpasscount": 2, "assert_fail": 2, "assertfailcount": 2, "prompt_token": 2, "num_request": 2, "numrequest": 2, "ipython": 2, "markdown": [2, 3], "2463": 2, "000035": 2, "3773": 2, "004620": 2, "1669": 2, "000091": 2, "1669m": 2, "highest": 2, "3773m": 2, "00462": 2, "promptfool": 2, "manual": 2, "redefin": 2, "prompt_comparison": 2, "prompt2": 2, "prompt3": 2, "Not": 2, "prompt_fil": 2, "prompt_cont": 2, "strip": 2, "join": [2, 3], "BE": 2, "again": 2, "prompt_id": 2, "promptid": 2, "gradingresult": 2, "split": [2, 3], "df_raw": 2, "column": 2, "pivot": 2, "reset_index": 2, "alb": 2, "loubna": 2, "ben": 2, "anton": 2, "lozhkov": 2, "eli": 2, "bakouch": 2, "gabriel": 2, "mart\u00edn": 2, "bl\u00e1zquez": 2, "lewi": 2, "tunstal": 2, "agust\u00edn": 2, "piquer": 2, "andr": 2, "marafioti": 2, "cyril": 2, "zakka": 2, "leandro": 2, "von": 2, "werra": 2, "thoma": 2, "wolf": 2, "are24": 2, "judgearena": 2, "bps99": 2, "salli": 2, "pott": 2, "barbara": 2, "557": 2, "sciencedirect": 2, "s0377221798003646": 2, "doi": [2, 4], "org": [2, 4], "1016": 2, "s0377": 2, "2217": 2, "00364": 2, "ctj": 2, "jerri": 2, "tworek": 2, "heewoo": 2, "jun": 2, "qime": 2, "yuan": 2, "henriqu": 2, "pond": 2, "de": 2, "oliveira": 2, "pinto": 2, "jare": 2, "kaplan": 2, "harri": 2, "edward": 2, "yuri": 2, "burda": 2, "nichola": 2, "joseph": 2, "greg": 2, "brockman": 2, "rai": 2, "raul": 2, "puri": 2, "gretchen": 2, "krueger": 2, "michael": [2, 4], "petrov": 2, "heidi": 2, "khlaaf": 2, "girish": 2, "sastri": 2, "pamela": 2, "mishkin": 2, "brook": 2, "chan": 2, "scott": 2, "grai": 2, "nick": 2, "ryder": 2, "mikhail": 2, "pavlov": 2, "alethea": 2, "lukasz": 2, "kaiser": 2, "mohammad": 2, "bavarian": 2, "clemen": 2, "winter": 2, "philipp": 2, "tillet": 2, "felip": 2, "petroski": 2, "dave": 2, "cum": 2, "matthia": 2, "plappert": 2, "fotio": 2, "chantzi": 2, "elizabeth": 2, "barn": 2, "ariel": 2, "herbert": 2, "voss": 2, "hebgen": 2, "guss": 2, "nichol": 2, "paino": 2, "nikola": 2, "tezak": 2, "jie": 2, "tang": 2, "igor": 2, "babuschkin": 2, "suchir": 2, "balaji": 2, "shantanu": 2, "jain": 2, "saunder": 2, "christoph": 2, "hess": 2, "andrew": 2, "carr": 2, "jan": 2, "leik": 2, "josh": 2, "achiam": 2, "vedant": 2, "misra": 2, "evan": 2, "morikawa": 2, "alec": 2, "radford": 2, "matthew": 2, "knight": 2, "mile": 2, "brundag": 2, "mira": 2, "murati": 2, "kati": 2, "mayer": 2, "peter": 2, "welind": 2, "bob": [2, 4], "mcgrew": 2, "dario": 2, "amodei": 2, "sam": 2, "mccandlish": 2, "ilya": 2, "sutskev": 2, "wojciech": 2, "zaremba": 2, "arxiv": [2, 4], "ab": [2, 4], "2107": 2, "03374": 2, "cz": 2, "lianmin": 2, "ying": 2, "sheng": 2, "anastasio": 2, "angelopoulo": 2, "tianl": 2, "dacheng": 2, "hao": 2, "zhang": 2, "banghua": 2, "zhu": 2, "jordan": 2, "gonzalez": 2, "ion": 2, "stoica": 2, "2403": 2, "04132": 2, "cho24a": 2, "francoi": 2, "arcpriz": 2, "cho24b": 2, "dglh24": 2, "yann": 2, "bal\u00e1z": 2, "galambosi": 2, "perci": 2, "tatsunori": 2, "hashimoto": 2, "debia": 2, "2404": 2, "04475": 2, "fac24a": 2, "wiki": [2, 4], "fac24b": 2, "fac24c": 2, "doc": [2, 3, 4], "model_doc": 2, "gpt2": 2, "fac24d": 2, "cookbook": 2, "en": [2, 4], "llm_judg": 2, "fac24": 2, "fac24f": 2, "blog": [2, 4], "fhwt23": 2, "cl\u00e9mentin": 2, "nathan": 2, "habib": 2, "hbb": 2, "dan": 2, "collin": 2, "burn": 2, "steven": 2, "basart": 2, "andi": 2, "zou": 2, "manta": 2, "mazeika": 2, "dawn": 2, "song": 2, "jacob": 2, "steinhardt": 2, "03300": 2, "hbd": 2, "ari": 2, "du": 2, "maxwel": 2, "forb": 2, "yejin": 2, "choi": 2, "curiou": 2, "neural": [2, 4], "degener": 2, "1904": 2, "09751": 2, "hyc": 2, "binyuan": 2, "jian": 2, "zeyu": 2, "cui": 2, "jiaxi": 2, "dayiheng": 2, "liu": [2, 4], "lei": 2, "tianyu": 2, "jiajun": 2, "bowen": 2, "yu": 2, "kai": 2, "dang": 2, "coder": 2, "preprint": [2, 4], "2409": 2, "12186": 2, "lx": 2, "zhen": 2, "xiaohan": 2, "xu": 2, "tao": 2, "shen": 2, "jia": 2, "gu": 2, "yuxuan": 2, "lai": 2, "chongyang": 2, "shuai": 2, "ma": 2, "nlg": 2, "2401": 2, "07103": 2, "lbl": 2, "rishi": 2, "bommasani": 2, "toni": 2, "lee": [2, 4], "dimitri": 2, "tsipra": 2, "dilara": 2, "soylu": 2, "michihiro": 2, "yasunaga": 2, "yian": 2, "deepak": 2, "narayanan": 2, "yuhuai": 2, "wu": [2, 4], "ananya": 2, "kumar": 2, "benjamin": 2, "newman": 2, "binhang": 2, "bobbi": 2, "yan": 2, "ce": 2, "christian": 2, "cosgrov": 2, "r\u00e9": 2, "diana": 2, "acosta": 2, "nava": 2, "drew": 2, "hudson": 2, "eric": 2, "zelikman": 2, "esin": 2, "durmu": 2, "faisal": 2, "ladhak": 2, "frieda": 2, "rong": 2, "hongyu": 2, "ren": 2, "huaxiu": 2, "yao": 2, "jue": 2, "keshav": 2, "santhanam": 2, "laurel": 2, "orr": 2, "lucia": 2, "mert": 2, "yuksekgonul": 2, "mirac": 2, "suzgun": 2, "kim": 2, "neel": 2, "guha": 2, "niladri": 2, "chatterji": 2, "omar": 2, "khattab": 2, "henderson": 2, "qian": 2, "huang": 2, "ryan": 2, "chi": [2, 4], "sang": 2, "xie": 2, "shibani": 2, "santurkar": 2, "surya": 2, "ganguli": 2, "icard": 2, "tianyi": 2, "vishrav": 2, "chaudhari": 2, "xuechen": 2, "yifan": 2, "yuhui": 2, "yuta": 2, "koreeda": 2, "2211": 2, "09110": 2, "lbc24": 2, "yuchen": 2, "ronan": 2, "le": 2, "bra": 2, "allenai": 2, "lhe22": 2, "stephani": 2, "hilton": 2, "owain": 2, "mimic": 2, "falsehood": 2, "2109": 2, "07958": 2, "pro24": 2, "dev": 2, "ras24": 2, "sebastian": 2, "scratch": 2, "isbn": 2, "1633437166": 2, "srr": 2, "aarohi": 2, "abhinav": 2, "rastogi": 2, "abhishek": 2, "rao": 2, "abu": 2, "awal": 2, "md": [2, 4], "shoeb": 2, "abubakar": 2, "abid": 2, "adam": 2, "fisch": 2, "brown": 2, "santoro": 2, "aditya": 2, "gupta": 2, "adri\u00e0": 2, "garriga": 2, "alonso": 2, "agnieszka": 2, "kluska": 2, "aitor": 2, "lewkowycz": 2, "akshat": 2, "agarw": 2, "warstadt": 2, "alexand": [2, 4], "kocurek": 2, "ali": 2, "safaya": 2, "tazarv": 2, "alic": [2, 4], "xiang": 2, "alicia": 2, "parrish": 2, "allen": 2, "nie": 2, "aman": 2, "hussain": 2, "amanda": 2, "askel": 2, "dsouza": 2, "ambros": 2, "slone": 2, "ameet": 2, "rahan": 2, "anantharaman": 2, "iyer": 2, "ander": 2, "andreassen": 2, "madotto": 2, "santilli": 2, "stuhlm\u00fcller": 2, "la": 2, "lampinen": 2, "angela": 2, "jiang": 2, "angelica": 2, "anh": 2, "vuong": 2, "animesh": 2, "anna": 2, "gottardi": 2, "antonio": 2, "norelli": 2, "anu": 2, "venkatesh": 2, "arash": 2, "gholamidavoodi": 2, "arfa": 2, "tabassum": 2, "arul": 2, "menez": 2, "arun": 2, "kirubarajan": 2, "asher": 2, "mullokandov": 2, "ashish": 2, "sabharw": 2, "herrick": 2, "avia": 2, "efrat": 2, "aykut": 2, "erdem": 2, "ayla": 2, "karaka\u015f": 2, "robert": 2, "bao": 2, "loe": 2, "barret": 2, "zoph": 2, "bart\u0142omiej": 2, "bojanowski": 2, "batuhan": 2, "\u00f6zyurt": 2, "behnam": 2, "hedayatnia": 2, "neyshabur": 2, "inden": 2, "benno": 2, "stein": 2, "berk": 2, "ekmekci": 2, "blake": 2, "howald": 2, "bryan": 2, "orinion": 2, "cameron": [2, 4], "diao": 2, "dour": 2, "catherin": 2, "stinson": 2, "cedrick": 2, "argueta": 2, "c\u00e9sar": 2, "ferri": 2, "ram\u00edrez": 2, "chandan": 2, "singh": 2, "charl": 2, "rathkopf": 2, "chenlin": 2, "meng": 2, "chitta": 2, "baral": 2, "chiyu": 2, "callison": 2, "burch": 2, "wait": 2, "voigt": 2, "cindi": 2, "ramirez": 2, "clara": 2, "rivera": 2, "clemencia": 2, "siro": 2, "colin": 2, "raffel": 2, "courtnei": 2, "ashcraft": 2, "cristina": 2, "garbacea": 2, "damien": 2, "sileo": 2, "garrett": 2, "kilman": 2, "roth": 2, "daniel": 2, "freeman": 2, "khashabi": 2, "levi": 2, "mosegu\u00ed": 2, "gonz\u00e1lez": 2, "perszyk": 2, "danni": 2, "hernandez": 2, "danqi": 2, "daphn": 2, "ippolito": 2, "dar": 2, "gilboa": 2, "david": 2, "dohan": 2, "drakard": 2, "jurgen": 2, "debajyoti": 2, "datta": 2, "deni": 2, "emelin": 2, "kleyko": 2, "deniz": 2, "yuret": 2, "derek": 2, "tam": [2, 4], "dieuwk": 2, "hupk": 2, "diganta": 2, "dilyar": 2, "buzan": 2, "coelho": 2, "mollo": 2, "diyi": 2, "dong": 2, "ho": 2, "dylan": 2, "schrader": 2, "ekaterina": 2, "shutova": 2, "ekin": 2, "dogu": 2, "cubuk": 2, "elad": 2, "segal": 2, "eleanor": 2, "hagerman": 2, "donowai": 2, "elli": 2, "pavlick": 2, "emanuel": 2, "rodola": 2, "emma": 2, "lam": 2, "chu": 2, "erkut": 2, "erni": 2, "ethan": 2, "dyer": 2, "jerzak": 2, "eunic": 2, "engefu": 2, "manyasi": 2, "evgenii": 2, "zheltonozhskii": 2, "fanyu": 2, "xia": 2, "fatemeh": 2, "siar": 2, "fernando": 2, "mart\u00ednez": 2, "plume": 2, "francesca": 2, "happ\u00e9": 2, "gaurav": 2, "mishra": 2, "genta": 2, "indra": 2, "winata": 2, "gerard": 2, "melo": 2, "germ\u00e1n": 2, "kruszewski": 2, "giambattista": 2, "parascandolo": 2, "giorgio": 2, "mariani": 2, "gloria": 2, "gonzalo": 2, "jaimovitch": 2, "l\u00f3pez": 2, "gregor": 2, "betz": 2, "gui": 2, "gur": 2, "hana": 2, "galijasev": 2, "hannah": 2, "rashkin": 2, "hannaneh": 2, "hajishirzi": 2, "harsh": 2, "mehta": 2, "hayden": 2, "bogar": 2, "henri": 2, "shevlin": 2, "hinrich": 2, "sch\u00fctze": 2, "hiromu": 2, "yakura": 2, "hongm": 2, "hugh": 2, "mee": 2, "wong": 2, "ian": 2, "ng": 2, "isaac": 2, "nobl": 2, "jaap": 2, "jumelet": 2, "jack": 2, "geissing": 2, "jackson": 2, "kernion": 2, "jaehoon": 2, "jaim": 2, "fern\u00e1ndez": 2, "fisac": 2, "jame": 2, "simon": 2, "koppel": 2, "koco\u0144": 2, "jana": 2, "thompson": 2, "janel": 2, "wingfield": 2, "jarema": 2, "radom": 2, "jascha": 2, "sohl": 2, "dickstein": 2, "jason": 2, "phang": 2, "yosinski": 2, "jekaterina": 2, "novikova": 2, "jell": 2, "bosscher": 2, "jennif": 2, "marsh": 2, "jeremi": 2, "jeroen": 2, "taal": 2, "jess": 2, "engel": 2, "jesujoba": 2, "alabi": 2, "jiacheng": 2, "jiam": 2, "jillian": 2, "joan": 2, "waweru": 2, "john": 2, "burden": 2, "miller": 2, "bali": 2, "jonathan": 2, "batcheld": 2, "berant": 2, "j\u00f6rg": 2, "frohberg": 2, "jo": 2, "rozen": 2, "orallo": 2, "boudeman": 2, "guerr": 2, "joshua": 2, "tenenbaum": 2, "joyc": 2, "chua": 2, "kamil": 2, "kanclerz": 2, "karen": 2, "livescu": 2, "karl": 2, "krauth": 2, "karthik": 2, "gopalakrishnan": 2, "katerina": 2, "ignatyeva": 2, "katja": 2, "markert": 2, "kaustubh": 2, "dhole": 2, "kevin": 2, "gimpel": 2, "omondi": 2, "kori": 2, "mathewson": 2, "kristen": 2, "chiafullo": 2, "ksenia": 2, "shkaruta": 2, "shridhar": 2, "kyle": 2, "mcdonel": 2, "richardson": 2, "laria": 2, "reynold": 2, "leo": 2, "gao": 2, "liam": 2, "dugan": 2, "lianhui": 2, "qin": 2, "lidia": 2, "contrera": 2, "ochando": 2, "loui": 2, "morenc": 2, "moschella": 2, "luci": 2, "ludwig": 2, "schmidt": 2, "luheng": 2, "lui": 2, "olivero": 2, "col\u00f3n": 2, "luke": 2, "metz": 2, "l\u00fctfi": 2, "kerem": 2, "\u015fenel": 2, "maarten": 2, "bosma": 2, "sap": 2, "maartj": 2, "hoev": 2, "maheen": 2, "farooqi": 2, "manaal": 2, "faruqui": 2, "marco": 2, "baturan": 2, "marelli": 2, "maru": 2, "maria": 2, "quintana": 2, "mari": 2, "tolkiehn": 2, "mario": 2, "giulianelli": 2, "martha": 2, "martin": 2, "potthast": 2, "leavitt": 2, "hagen": 2, "m\u00e1ty\u00e1": 2, "schubert": 2, "medina": 2, "orduna": 2, "baitemirova": 2, "melodi": 2, "arnaud": 2, "melvin": 2, "mcelrath": 2, "yee": 2, "cohen": 2, "ivanitskii": 2, "starritt": 2, "strube": 2, "micha\u0142": 2, "sw\u0119drowski": 2, "michel": 2, "bevilacqua": 2, "mihir": 2, "kale": 2, "cain": 2, "mime": 2, "mitch": 2, "walker": 2, "mo": 2, "tiwari": 2, "mohit": 2, "bansal": 2, "moin": 2, "aminnaseri": 2, "mor": 2, "geva": 2, "mozhdeh": 2, "gheini": 2, "mukund": 2, "varma": 2, "nanyun": 2, "peng": 2, "nayeon": 2, "neta": 2, "krakov": 2, "doiron": 2, "nicol": 2, "martinez": 2, "nikita": 2, "nangia": 2, "nikla": 2, "decker": 2, "muennighoff": 2, "nitish": 2, "shirish": 2, "keskar": 2, "niveditha": 2, "noah": 2, "constant": 2, "fiedel": 2, "nuan": 2, "wen": 2, "oliv": 2, "agha": 2, "elbaghdadi": 2, "omer": 2, "moreno": 2, "casar": 2, "parth": 2, "doshi": 2, "pascal": 2, "fung": 2, "paul": 2, "pu": 2, "vicol": 2, "pegah": 2, "alipoormolabashi": 2, "peiyuan": 2, "liao": 2, "eckerslei": 2, "phu": 2, "mon": 2, "htut": 2, "pinyu": 2, "hwang": 2, "piotr": 2, "mi\u0142kowski": 2, "piyush": 2, "patil": 2, "pouya": 2, "pezeshkpour": 2, "priti": 2, "oli": 2, "qiaozhu": 2, "mei": 2, "qing": 2, "lyu": 2, "qinlang": 2, "rabin": 2, "banjad": 2, "rachel": 2, "etta": 2, "rudolph": 2, "raefer": 2, "rahel": 2, "haback": 2, "ramon": 2, "risco": 2, "rapha\u00ebl": 2, "milli\u00e8r": 2, "rhythm": 2, "garg": 2, "rif": 2, "saurou": 2, "riku": 2, "arakawa": 2, "robb": 2, "raymaek": 2, "frank": 2, "rohan": 2, "sikand": 2, "roman": 2, "novak": 2, "sitelew": 2, "lebra": 2, "rosann": 2, "rowan": 2, "rui": [2, 4], "ruslan": 2, "salakhutdinov": 2, "stoval": 2, "teehan": 2, "rylan": 2, "sahib": 2, "saif": 2, "sajant": 2, "anand": 2, "dillav": 2, "shleifer": 2, "wiseman": 2, "samuel": 2, "gruetter": 2, "bowman": 2, "schoenholz": 2, "sanghyun": 2, "han": 2, "sanjeev": 2, "kwatra": 2, "sarah": 2, "sarik": 2, "ghazarian": 2, "sayan": 2, "ghosh": 2, "sean": 2, "casei": 2, "bischoff": 2, "gehrmann": 2, "schuster": 2, "sepideh": 2, "sadeghi": 2, "shadi": 2, "hamdan": 2, "sharon": 2, "zhou": 2, "shashank": 2, "sherri": 2, "shi": 2, "shikhar": 2, "shima": 2, "asaadi": 2, "shixiang": 2, "shane": 2, "shubh": 2, "pachchigar": 2, "shubham": 2, "toshniw": 2, "shyam": 2, "upadhyai": 2, "shyamolima": 2, "debnath": 2, "siamak": 2, "shakeri": 2, "thormey": 2, "melzi": 2, "siva": 2, "reddi": 2, "sneha": 2, "priscilla": 2, "makini": 2, "soo": 2, "hwan": 2, "spencer": 2, "toren": 2, "sriharsha": 2, "hatwar": 2, "stanisla": 2, "dehaen": 2, "stefan": 2, "divic": 2, "stefano": 2, "ermon": 2, "stella": 2, "biderman": 2, "stephen": 2, "prasad": 2, "piantadosi": 2, "stuart": 2, "shieber": 2, "summer": 2, "misherghi": 2, "svetlana": 2, "kiritchenko": 2, "swaroop": 2, "tal": 2, "linzen": 2, "tariq": 2, "tatsu": 2, "te": 2, "th\u00e9o": 2, "desbord": 2, "theodor": 2, "rothschild": 2, "phan": 2, "tiberiu": 2, "nkinyili": 2, "timo": 2, "schick": 2, "timofei": 2, "kornev": 2, "titu": 2, "tunduni": 2, "gerstenberg": 2, "trenton": 2, "trishala": 2, "neeraj": 2, "tushar": 2, "khot": 2, "tyler": 2, "shultz": 2, "uri": 2, "shaham": 2, "vera": 2, "demberg": 2, "victoria": 2, "nyamai": 2, "vika": 2, "raunak": 2, "vinai": 2, "ramasesh": 2, "udai": 2, "prabhu": 2, "vishakh": 2, "padmakumar": 2, "vivek": 2, "srikumar": 2, "fedu": 2, "wout": 2, "vossen": 2, "xiaoyu": 2, "tong": 2, "xinran": 2, "zhao": 2, "xinyi": 2, "xudong": 2, "yadollah": 2, "yaghoobzadeh": 2, "yair": 2, "lakretz": 2, "yangqiu": 2, "yasaman": 2, "bahri": 2, "yichi": 2, "yide": 2, "yifu": 2, "yonatan": 2, "belinkov": 2, "hou": 2, "yufang": 2, "yuntao": 2, "bai": 2, "zachari": 2, "seid": 2, "zhuoy": 2, "zijian": 2, "ziji": 2, "j": [2, 4], "zirui": 2, "ziyi": 2, "extrapol": 2, "2206": 2, "04615": 2, "wpn": 2, "yada": 2, "pruksachatkun": 2, "amanpreet": 2, "julian": 2, "felix": 2, "hill": 2, "stickier": 2, "wsm": 2, "1804": 2, "07461": 2, "wtb": 2, "yi": [2, 4], "tai": 2, "borgeaud": 2, "dani": 2, "yogatama": 2, "denni": 2, "donald": 2, "metzler": 2, "ed": 2, "h": 2, "oriol": 2, "vinyal": 2, "dean": 2, "07682": 2, "wdr": 2, "doolei": 2, "manlei": 2, "arka": 2, "pal": 2, "feuer": 2, "siddhartha": 2, "ravid": 2, "shwartz": 2, "ziv": 2, "khalid": 2, "saifullah": 2, "siddartha": 2, "naidu": 2, "chinmai": 2, "hegd": 2, "lecun": 2, "tom": 2, "goldstein": 2, "willi": 2, "neiswang": 2, "micah": 2, "goldblum": 2, "2406": 2, "19314": 2, "yyh": 2, "baosong": 2, "bo": 2, "chengpeng": 2, "chengyuan": 2, "fei": 2, "guant": 2, "haoran": 2, "huan": 2, "jialong": 2, "jialin": 2, "jianhong": 2, "tu": 2, "jianwei": 2, "jianxin": 2, "jin": 2, "jingren": 2, "jinz": 2, "jinzheng": 2, "junyang": 2, "keme": 2, "lu": 2, "keqin": 2, "kexin": 2, "mingfeng": 2, "xue": 2, "ni": 2, "pei": 2, "ru": 2, "men": 2, "ruiz": 2, "runji": 2, "shiji": 2, "sinan": 2, "tan": 2, "tianhang": 2, "tianhao": 2, "wenbin": 2, "ge": 2, "xiaodong": 2, "deng": 2, "xiaohuan": 2, "xingzhang": 2, "xinyu": 2, "xipin": 2, "xuancheng": 2, "fan": 2, "yichang": 2, "wan": 2, "yunfei": 2, "yuqiong": 2, "zhenru": 2, "zhihao": 2, "2407": 2, "10671": 2, "zc": 2, "siyuan": 2, "zhuang": 2, "zhanghao": 2, "yonghao": 2, "zi": 2, "zhuohan": 2, "xing": 2, "2306": 2, "05685": 2, "huggingface24": 2, "06": [2, 4], "metaai24": 2, "far": 3, "possibli": 3, "eliot": 3, "english": 3, "thumb": 3, "\u00be": 3, "max_output_token": 3, "4096": 3, "16384": 3, "contrari": 3, "surpass": 3, "truncat": 3, "max_input_token": 3, "input_cost_per_token": 3, "output_cost_per_token": 3, "11b": 3, "v1": 3, "128000": 3, "5e": 3, "20241022": 3, "8192": 3, "200000": 3, "3e": 3, "0613": 3, "6e": 3, "1e": 3, "gemini": 3, "flash": 3, "1048576": 3, "2097152": 3, "05e": 3, "incomplet": 3, "abruptli": 3, "shallow": 3, "thorough": 3, "dissatisfact": 3, "frustrat": 3, "creation": 3, "feasibl": 3, "10k": 3, "diagram": 3, "charactertextsplitt": 3, "tiktoken": 3, "sequenti": 3, "newlin": 3, "broadli": [3, 4], "want": [3, 4], "cheap": 3, "speciali": 3, "naiv": 3, "nltk": 3, "spaci": 3, "recurs": 3, "divid": 3, "hierarch": 3, "talk": 3, "theme": 3, "splitter": 3, "html": [3, 4], "get_chunk": 3, "chunk_siz": 3, "chunk_overlap": 3, "langchain_text_splitt": 3, "text_splitt": 3, "from_tiktoken_encod": 3, "split_text": 3, "persona": 3, "task": [3, 4], "langchain_cor": [3, 4], "prompttempl": 3, "get_base_prompt_templ": 3, "base_prompt": [3, 4], "from_templ": 3, "llmchain": 3, "togeth": 3, "parser": [3, 4], "output_pars": 3, "stroutputpars": 3, "langchain_commun": 3, "chat_model": 3, "chatlitellm": 3, "get_llm_chain": 3, "prompt_templ": [3, 4], "llm_chain": [3, 4], "api_key_label": 3, "upper": 3, "_api_kei": 3, "get_dynamic_prompt_templ": 3, "get_dynamic_prompt_param": 3, "prompt_param": 3, "part_idx": 3, "total_part": 3, "chat_context": 3, "param": 3, "dynamic_prompt_param": 3, "elif": 3, "merg": 3, "concaten": 3, "generate_report": 3, "input_cont": 3, "llm_model_nam": 3, "report_part": 3, "num_part": 3, "dinam": 3, "priovid": 3, "invok": [3, 4], "cummul": 3, "max_chunk_s": 3, "max_chunk_overlap": 3, "readabl": 3, "apple_report": 3, "luation": 3, "disciplin": 3, "smooth": 3, "subhead": 3, "despit": [3, 4], "depth": 3, "overlook": 3, "preserv": 3, "easier": [3, 4], "preprocess": [3, 4], "necessit": 3, "meticul": 3, "bottleneck": 3, "mustafa": 3, "suleyman": 3, "infinit": 3, "fewer": 3, "progress": 3, "condens": 3, "versatil": 3, "drive": [3, 4], "grace": 3, "fallback": 3, "empow": 3, "crucial": [3, 4], "langchain24": 3, "how_to": 3, "freedom": 4, "julia": 4, "easili": 4, "notebook": 4, "overrid": 4, "response_cont": 4, "wow": 4, "lot": 4, "breakdown": 4, "impress": 4, "huge": 4, "serious": 4, "is_json": 4, "myjson": 4, "valueerror": 4, "trial": 4, "elicit": 4, "wrangl": 4, "hoc": 4, "streamlin": 4, "subsequ": 4, "dataset": 4, "unwant": 4, "ui": 4, "overflow": 4, "overwhelm": 4, "twitter": 4, "youtub": 4, "publish": 4, "schema": 4, "blueprint": 4, "nativ": 4, "json_format": 4, "person1": 4, "q1": 4, "person2": 4, "nest": 4, "todai": 4, "thellm": 4, "unend": 4, "whitespac": 4, "forget": 4, "throw": 4, "somewher": 4, "json_object": 4, "sheer": 4, "circul": 4, "vertex": 4, "worri": 4, "enum": 4, "refus": 4, "simpler": 4, "strongli": 4, "secextract": 4, "mentioned_ent": 4, "mentioned_plac": 4, "extract_from_sec_fil": 4, "sec_filing_text": 4, "hint": 4, "prompt_extract": 4, "sec_extract": 4, "washington": 4, "usabl": 4, "beg": 4, "with_structured_output": 4, "runnabl": 4, "typeddict": 4, "qu": 4, "langchain_openai": 4, "chatopenai": 4, "chatprompttempl": 4, "extract_from_sec_filing_langchain": 4, "structured_llm": 4, "from_messag": 4, "sec_extraction_langchain": 4, "hood": 4, "logit": 4, "willard": 4, "louf": 4, "reformul": 4, "finit": 4, "fsm": 4, "s_": 4, "sim": 4, "s_t": 4, "theta": 4, "s_1": 4, "v": 4, "mathbb": 4, "mask": 4, "tild": 4, "odot": 4, "rightarrow": 4, "boolean": 4, "wise": 4, "formul": 4, "regex": 4, "tran": 4, "thien": 4, "automaton": 4, "dfa": 4, "decod": 4, "outgo": 4, "renorm": 4, "yy": 4, "nn": 4, "ever": 4, "aa": 4, "lwai": 4, "prop": 4, "yynnaa": 4, "qwen": 4, "malform": 4, "sec_extraction_outlin": 4, "zsp": 4, "zicorp": 4, "phenomenon": 4, "popular": 4, "cpp": 4, "gbnf": 4, "ggml": 4, "bnf": 4, "ggerganov": 4, "accomplish": 4, "backu": 4, "naur": 4, "wikipedia": 4, "contributor": 4, "strictli": 4, "soon": 4, "curl": 4, "fssl": 4, "sh": 4, "extract_entities_from_sec_fil": 4, "suffix": 4, "ollama_structured_output_prompt_suffix": 4, "ollama_structured_output_temperatur": 4, "mistral": 4, "llama2": 4, "uncensor": 4, "model_json_schema": 4, "response_json": 4, "wrapper": 4, "exllama2": 4, "mlx": 4, "lm": 4, "medium": 4, "know": 4, "chanc": 4, "correctli": 4, "famili": 4, "furthermor": 4, "nonetheless": 4, "studi": 4, "wrap": 4, "gemma": 4, "uncov": 4, "wors": 4, "extran": 4, "dispar": 4, "preval": 4, "outdat": 4, "rapidli": 4, "fashion": 4, "remark": 4, "me": 4, "speak": 4, "freeli": 4, "aider": 4, "outweigh": 4, "rebutt": 4, "argu": 4, "reproduct": 4, "paint": 4, "pictur": 4, "verif": 4, "dottxt": 4, "flaw": 4, "uneven": 4, "didn": 4, "conflat": 4, "argument": 4, "drawback": 4, "unlock": 4, "wider": 4, "thank": 4, "pfiffer": 4, "aid24": 4, "dot24": 4, "sai": 4, "demo": 4, "tree": 4, "gge24": 4, "blob": 4, "readm": 4, "llf": 4, "xieyang": 4, "frederick": 4, "fiannaca": 4, "terri": 4, "koo": 4, "dixon": 4, "cai": 4, "ea": 4, "ny": 4, "usa": 4, "machineri": 4, "1145": 4, "3613905": 4, "3650756": 4, "ln": 4, "xuan": 4, "hai": 4, "nguyen": 4, "ngoc": 4, "tiviati": 4, "hieu": 4, "dao": 4, "shafiq": 4, "joti": 4, "kenji": 4, "kawaguchi": 4, "nanci": 4, "min": 4, "kan": 4, "2408": 4, "08656": 4, "out24": 4, "twt": 4, "zhi": 4, "cheng": 4, "kuang": 4, "tsai": 4, "chieh": 4, "hung": 4, "yun": 4, "nung": 4, "02442": 4, "tt24": 4, "vivien": 4, "vivien000": 4, "wl23": 4, "brandon": 4, "r\u00e9mi": 4, "2307": 4, "09702": 4, "wikipediacontributors24": 4, "wiktionari": 4, "naur_form": 4}, "objects": {}, "objtypes": {}, "objnames": {}, "titleterms": {"introduct": [0, 1, 4], "content": [0, 2, 3, 4], "core": 0, "challeng": 0, "we": 0, "ll": 0, "address": 0, "A": [0, 1], "practic": [0, 1, 4], "approach": 0, "note": 0, "perspect": 0, "who": 0, "thi": 0, "book": 0, "i": 0, "For": 0, "outcom": 0, "prerequisit": 0, "set": 0, "up": 0, "your": 0, "environ": 0, "python": 0, "setup": 0, "api": [0, 4], "kei": [0, 2, 3], "configur": 0, "code": 0, "repositori": 0, "troubleshoot": 0, "common": 0, "issu": 0, "about": 0, "author": 0, "": 0, "tame": 1, "llm": [1, 2], "guid": 1, "pitfal": 1, "open": 1, "sourc": 1, "softwar": [1, 2], "chapter": 1, "1": [1, 3], "2": [1, 3], "wrestl": [1, 4], "structur": [1, 4], "output": [1, 3, 4], "3": [1, 3], "input": 1, "size": [1, 3], "length": [1, 3], "limit": [1, 3], "4": [1, 3], "5": 1, "The": [1, 2], "eval": [1, 2], "gap": [1, 2], "6": 1, "hallucin": 1, "realiti": 1, "7": 1, "safeti": 1, "concern": 1, "8": 1, "cost": [1, 3], "factor": 1, "9": 1, "break": 1, "free": 1, "from": 1, "cloud": 1, "provid": [1, 4], "appendix": 1, "tool": [1, 2, 4], "resourc": 1, "citat": 1, "non": 2, "determinist": 2, "gener": [2, 3], "machin": 2, "temperatur": 2, "sampl": 2, "spectrum": 2, "emerg": 2, "properti": 2, "problem": [2, 3, 4], "statement": [2, 3, 4], "tradit": 2, "v": 2, "design": 2, "applic": 2, "test": 2, "requir": 2, "matrix": 2, "conceptu": 2, "overview": 2, "consider": [2, 3], "metric": 2, "evalu": 2, "task": 2, "model": [2, 3], "base": [2, 3], "human": 2, "benchmark": 2, "leaderboard": 2, "lightev": 2, "mmlu": 2, "econometr": 2, "dataset": 2, "famili": 2, "us": 2, "langsmith": 2, "promptfoo": 2, "refer": [2, 3, 4], "what": 3, "ar": 3, "token": 3, "comparison": [3, 4], "across": 3, "chunk": 3, "contextu": 3, "link": 3, "long": 3, "form": 3, "step": 3, "write": 3, "prompt": [3, 4], "templat": 3, "construct": 3, "dynam": 3, "paramet": 3, "report": 3, "exampl": 3, "usag": 3, "discuss": [3, 4], "implic": 3, "futur": 3, "conclus": [3, 4], "user": 4, "need": 4, "solut": 4, "strategi": 4, "techniqu": 4, "One": 4, "shot": 4, "specif": 4, "json": 4, "mode": 4, "langchain": 4, "outlin": 4, "ollama": 4, "compar": 4, "framework": 4, "best": 4, "research": 4, "ongo": 4, "debat": 4, "acknowledg": 4}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.intersphinx": 1, "sphinxcontrib.bibtex": 9, "sphinx": 57}, "alltitles": {"Introduction": [[0, "introduction"], [4, "introduction"]], "Contents": [[0, "contents"], [2, "contents"], [3, "contents"], [4, "contents"]], "Core Challenges We\u2019ll Address": [[0, "core-challenges-we-ll-address"]], "A Practical Approach": [[0, "a-practical-approach"]], "A Note on Perspective": [[0, "a-note-on-perspective"]], "Who This Book Is For": [[0, "who-this-book-is-for"]], "Outcomes": [[0, "outcomes"]], "Prerequisites": [[0, "prerequisites"]], "Setting Up Your Environment": [[0, "setting-up-your-environment"]], "Python Environment Setup": [[0, "python-environment-setup"]], "API Keys Configuration": [[0, "api-keys-configuration"]], "Code Repository": [[0, "code-repository"]], "Troubleshooting Common Issues": [[0, "troubleshooting-common-issues"]], "About the Author(s)": [[0, "about-the-author-s"]], "Taming LLMs": [[1, "taming-llms"]], "A Practical Guide to LLM Pitfalls with Open Source Software": [[1, "a-practical-guide-to-llm-pitfalls-with-open-source-software"]], "Chapter 1: Introduction": [[1, "chapter-1-introduction"]], "Chapter 2: Wrestling with Structured Output": [[1, "chapter-2-wrestling-with-structured-output"]], "Chapter 3: Input Size and Length Limitations": [[1, "chapter-3-input-size-and-length-limitations"]], "Chapter 4: Output Size and Length Limitations": [[1, "chapter-4-output-size-and-length-limitations"]], "Chapter 5: The Evals Gap": [[1, "chapter-5-the-evals-gap"]], "Chapter 6: Hallucination: The Reality Gap": [[1, "chapter-6-hallucination-the-reality-gap"]], "Chapter 7: Safety Concerns": [[1, "chapter-7-safety-concerns"]], "Chapter 8: The Cost Factor": [[1, "chapter-8-the-cost-factor"]], "Chapter 9: Breaking Free from Cloud Providers": [[1, "chapter-9-breaking-free-from-cloud-providers"]], "Appendix A: Tools and Resources": [[1, "appendix-a-tools-and-resources"]], "Citation": [[1, "citation"]], "The Evals Gap": [[2, "the-evals-gap"]], "Non-Deterministic Generative Machines": [[2, "non-deterministic-generative-machines"]], "Temperature and Sampling": [[2, "temperature-and-sampling"]], "The Temperature Spectrum": [[2, "the-temperature-spectrum"]], "Emerging Properties": [[2, "emerging-properties"]], "Problem Statement": [[2, "problem-statement"], [3, "problem-statement"], [4, "problem-statement"]], "Evals of Traditional Software vs LLMs": [[2, "evals-table"]], "Evals Design": [[2, "evals-design"]], "LLM Application Testing Requirements Matrix": [[2, "validation-requirements"]], "Conceptual Overview": [[2, "conceptual-overview"]], "Design Considerations": [[2, "design-considerations"]], "Metrics": [[2, "metrics"]], "Key Metrics for Evaluating Generative Tasks": [[2, "key-metrics"]], "Evaluators": [[2, "evaluators"]], "Model-Based Evaluation": [[2, "model-based-evaluation"]], "Human-Based Evaluation": [[2, "human-based-evaluation"]], "Evaluating Evaluators": [[2, "evaluating-evaluators"]], "Benchmarks and Leaderboards": [[2, "benchmarks-and-leaderboards"]], "Tools": [[2, "tools"]], "LightEval": [[2, "lighteval"]], "MMLU Econometrics Task Dataset sample": [[2, "mmlu-econometrics"]], "Model Families Evaluated Using LightEval": [[2, "model-families"]], "LangSmith": [[2, "langsmith"]], "PromptFoo": [[2, "promptfoo"]], "References": [[2, "references"], [3, "references"], [4, "references"]], "Output Size Limitations": [[3, "output-size-limitations"]], "What are Token Limits?": [[3, "what-are-token-limits"]], "Token Cost and Length Limitation Comparison Across Key Models": [[3, "token-cost-table"]], "Content Chunking with Contextual Linking": [[3, "content-chunking-with-contextual-linking"]], "Generating long-form content": [[3, "generating-long-form-content"]], "Step 1: Chunking the Content": [[3, "step-1-chunking-the-content"]], "Step 2: Writing the Base Prompt Template": [[3, "step-2-writing-the-base-prompt-template"]], "Step 3: Constructing Dynamic Prompt Parameters": [[3, "step-3-constructing-dynamic-prompt-parameters"]], "Step 4: Generating the Report": [[3, "step-4-generating-the-report"]], "Example Usage": [[3, "example-usage"]], "Discussion": [[3, "discussion"], [4, "discussion"]], "Implications": [[3, "implications"]], "Future Considerations": [[3, "future-considerations"]], "Conclusion": [[3, "conclusion"], [4, "conclusion"]], "Wrestling with Structured Output": [[4, "wrestling-with-structured-output"]], "User Needs": [[4, "user-needs"]], "Solutions": [[4, "solutions"]], "Strategies": [[4, "strategies"]], "Techniques and Tools": [[4, "techniques-and-tools"]], "One-Shot Prompts": [[4, "one-shot-prompts"]], "Structured Output with Provider-Specific APIs": [[4, "structured-output-with-provider-specific-apis"]], "JSON Mode": [[4, "json-mode"]], "LangChain": [[4, "langchain"]], "Outlines": [[4, "outlines"]], "Ollama": [[4, "ollama"]], "Comparing Solutions": [[4, "comparing-solutions"]], "Structured Output Frameworks Comparison": [[4, "structured-output-frameworks"]], "Best Practices": [[4, "best-practices"]], "Research and Ongoing Debate": [[4, "research-and-ongoing-debate"]], "Acknowledgements": [[4, "acknowledgements"]]}, "indexentries": {}}) \ No newline at end of file diff --git a/tamingllms/_build/jupyter_execute/markdown/intro.ipynb b/tamingllms/_build/jupyter_execute/markdown/intro.ipynb index c759f80..f80237a 100644 --- a/tamingllms/_build/jupyter_execute/markdown/intro.ipynb +++ b/tamingllms/_build/jupyter_execute/markdown/intro.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "5a67fb7d", + "id": "60c9196f", "metadata": {}, "source": [ "(intro)=\n", diff --git a/tamingllms/_build/jupyter_execute/notebooks/evals.ipynb b/tamingllms/_build/jupyter_execute/notebooks/evals.ipynb index 390ab38..860c760 100644 --- a/tamingllms/_build/jupyter_execute/notebooks/evals.ipynb +++ b/tamingllms/_build/jupyter_execute/notebooks/evals.ipynb @@ -37,7 +37,11 @@ "\n", "- Temperature = 0: Most deterministic, but potentially repetitive\n", "- Temperature = 1: Balanced creativity and coherence\n", - "- Temperature > 1: Increased randomness, potentially incoherent" + "- Temperature > 1: Increased randomness, potentially incoherent\n", + "\n", + "A temperature of 1 represents the unscaled probability scores for each token in the vocabulary. Decreasing the temperature closer to 0 sharpens the distribution, so the most likely token will have an even higher probability score. Conversely, increasing the temperature makes the distribution more uniform {cite}`build-llms-from-scratch-book`.\n", + "\n", + "In this simple experiment, we use an LLM to write a single-statement executive summary of an input financial filing. We observe that even a simple parameter like temperature can dramatically alter model behavior in ways that are difficult to systematically assess. At temperature 0.0, responses are consistent but potentially too rigid. At 1.0, outputs become more varied but less predictable. At 2.0, responses can be wildly different and often incoherent. This non-deterministic behavior makes traditional software testing approaches inadequate." ] }, { @@ -171,13 +175,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "A temperature of 1 represents the unscaled probability scores for each token in the vocabulary. Decreasing the temperature closer to 0 sharpens the distribution, so the most likely token will have an even higher probability score. Conversely, increasing the temperature makes the distribution more uniform {cite}`build-llms-from-scratch-book`.\n", - "\n", - "In this simple experiment, we use an LLM to write a single-statement executive summary of an input financial filing. We observe that even a simple parameter like temperature can dramatically alter model behavior in ways that are difficult to systematically assess. At temperature 0.0, responses are consistent but potentially too rigid. At 1.0, outputs become more varied but less predictable. At 2.0, responses can be wildly different and often incoherent. This non-deterministic behavior makes traditional software testing approaches inadequate.\n", - "\n", - "\n", - "\n", - "The implications for evaluation are critical. How can one effectively test an LLM-powered system when the same prompt can yield radically different outputs based on a single parameter? Traditional testing relies on predictable inputs and outputs, but LLMs force us to grapple with probabilistic behavior. While lower temperatures may seem safer for critical applications, they don't necessarily eliminate the underlying uncertainty. This highlights the need for new evaluation paradigms that can handle both deterministic and probabilistic aspects of LLM behavior." + "How can one effectively test an LLM-powered system when the same prompt can yield radically different outputs based on a single parameter? Traditional testing relies on predictable inputs and outputs, but LLMs force us to grapple with probabilistic behavior. While lower temperatures may seem safer for critical applications, they don't necessarily eliminate the underlying uncertainty. This highlights the need for new evaluation paradigms that can handle both deterministic and probabilistic aspects of LLM behavior." ] }, { @@ -187,7 +185,9 @@ "\n", "## Emerging Properties\n", "\n", - "Beyond their non-deterministic nature, LLMs present another fascinating challenge: emergent abilities that spontaneously arise as models scale up in size. These abilities - from basic question answering to complex reasoning - aren't explicitly programmed but rather emerge \"naturally\" as the models grow larger and are trained on more data. This makes evaluation fundamentally different from traditional software testing, where capabilities are explicitly coded and can be tested against clear specifications.\n", + "Beyond their non-deterministic nature, LLMs present another fascinating characteristic: emergent abilities that spontaneously arise as models scale up in size. These abilities - from basic question answering to complex reasoning - aren't explicitly programmed but rather emerge \"naturally\" as the models grow larger and are trained on more data. This makes evaluation fundamentally different from traditional software testing, where capabilities are explicitly coded and can be tested against pre-defined specifications.\n", + "\n", + "{numref}`emerging-properties` provides a list of emergent abilities of large language models and the scale. The relationship between model scale and emergent abilities follows a fascinating non-linear pattern. Below certain size thresholds, specific abilities may be completely absent from the model - it simply cannot perform certain tasks, no matter how much you try to coax them out. However, once the model reaches critical points in its scaling journey, these abilities can suddenly manifest in what researchers call a phase transition - a dramatic shift from inability to capability. This unpredictable emergence of capabilities stands in stark contrast to traditional software development, where features are deliberately implemented and can be systematically tested.\n", "\n", "```{figure} ../_static/evals/emerging.png\n", "---\n", @@ -200,9 +200,7 @@ "Emergent abilities of large language models and the scale {cite}`wei2022emergentabilitieslargelanguage`.\n", "```\n", "\n", - " {numref}`emerging-properties` provides a list of emergent abilities of large language models and the scale. The relationship between model scale and emergent abilities follows a fascinating non-linear pattern. Below certain size thresholds, specific abilities may be completely absent from the model - it simply cannot perform certain tasks, no matter how much you try to coax them out. However, once the model reaches critical points in its scaling journey, these abilities can suddenly manifest in what researchers call a phase transition - a dramatic shift from inability to capability. This unpredictable emergence of capabilities stands in stark contrast to traditional software development, where features are deliberately implemented and can be systematically tested.\n", - "\n", - "The implications for evaluation are pressing. While conventional software testing relies on stable test suites and well-defined acceptance criteria, LLM evaluation must contend with a constantly shifting landscape of capabilities. What worked to evaluate a 7B parameter model may be completely inadequate for a 70B parameter model that has developed new emergent abilities. This dynamic nature of LLM capabilities forces us to fundamentally rethink our approach to testing and evaluation.\n", + "The implications for evaluation are critical. While conventional software testing relies on stable test suites and well-defined acceptance criteria, LLM evaluation must contend with a constantly shifting landscape of capabilities. What worked to evaluate a 7B parameter model may be completely inadequate for a 70B parameter model that has developed new emergent abilities. This dynamic nature of LLM capabilities forces us to fundamentally rethink our approach to testing and evaluation.\n", "\n", "## Problem Statement\n", "\n", @@ -216,11 +214,11 @@ "The challenge becomes even more complex when we consider that traditional software evaluation methods simply weren't designed for these kinds of systems - There is an **Evals Gap** between traditional software testing and LLM evaluation. We need new frameworks that can account for both the deterministic aspects we're used to testing and the emergent properties that make LLMs unique. \n", "\n", "{numref}`evals-table` explores how LLM evaluation differs from traditional software testing across several key dimensions:\n", - "- **Capability Assessment vs Functional Testing**: Traditional software testing validates specific functionality against predefined requirements. LLM evaluation must assess not necessarily pre-defined \"emergent properties\" like reasoning, creativity, and language understanding that extend beyond explicit programming.\n", + "- **Capability Assessment vs Functional Testing**: Traditional software testing validates specific functionality against predefined requirements. LLM evaluation must assess not necessarily pre-defined behavior but also \"emergent properties\" like reasoning, creativity, and language understanding that extend beyond explicit programming.\n", "\n", - "- **Metrics and Measurement Challenges**: While traditional software metrics can usually be precisely defined and measured, LLM evaluation often involves subjective qualities like \"helpfulness\" or \"naturalness\" that resist straightforward quantification. Even when we try to break these down into numeric scores, the underlying judgment remains inherently human and context-dependent.\n", + "- **Metrics and Measurement Challenges**: While traditional software metrics can usually be precisely defined and measured, LLM evaluation often involves subjective qualities like \"helpfulness\" or \"naturalness\" that resist straightforward quantification. Even when we try to break these down into numeric scores, the underlying judgment often remains inherently human and context-dependent.\n", "\n", - "- **Dataset Contamination**: Traditional software testing uses carefully crafted test cases with known inputs and expected outputs (e.g., unit tests, integration tests). In contrast, LLMs trained on massive internet-scale datasets risk having already seen and memorized evaluation examples during training, which can lead to artificially inflated performance scores. This requires careful dataset curation to ensure test sets are truly unseen by the model and rigorous cross-validation approaches.\n", + "- **Dataset Contamination**: Traditional software testing uses carefully crafted test cases with known inputs and expected outputs (e.g., unit tests). In contrast, LLMs trained on massive internet-scale datasets risk having already seen and memorized evaluation examples during training, which can lead to artificially inflated performance scores. This requires careful dataset curation to ensure test sets are truly unseen by the model and rigorous cross-validation approaches.\n", "\n", "- **Benchmark Evolution**: Traditional software maintains stable test suites over time. LLM benchmarks continuously evolve as capabilities advance, making longitudinal performance comparisons difficult and potentially obsoleting older evaluation methods.\n", "\n", @@ -252,7 +250,7 @@ "- Evaluation must align with business objectives\n", "- A great LLM doesn't guarantee a great application!\n", "\n", - "Examples of key requirements for validation are listed in {numref}`validation-requirements`.\n", + "Examples of key requirements for validation are listed in {numref}`validation-requirements` ranging from Safety, Cognitive, Technical, Meta-Cognitive, to Ethical aspects.\n", "\n", "```{table} LLM Application Testing Requirements Matrix\n", ":name: validation-requirements\n", @@ -1524,18 +1522,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/tobias/src/tamingLLMs/tamingllms/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], + "outputs": [], "source": [ "import evaluate as hf_evaluate # HuggingFace's evaluate\n", "from langsmith import evaluate as langsmith_evaluate # LangSmith's evaluate\n", @@ -2098,8 +2087,428 @@ "source": [ "### PromptFoo\n", "\n", - "PromptFoo {cite}`promptfoo` is a framework for evaluating the quality of prompts for LLMs.\n", - "\n" + "Promptfoo {cite}`promptfoo2024` is an open-source framework designed for evaluating applications that utilize large language models (LLMs). Key features include:\n", + "\n", + "1. **Automated Testing**: Promptfoo provides automated testing capabilities, allowing developers to run custom evaluations tailored to their applications.\n", + "\n", + "2. **Custom Probes**: Developers can create custom probes to focus on specific use cases for instance decoupling prompts from tests cases.\n", + "\n", + "3. **User-Friendly CLI**: The framework features a command-line interface that supports live reloads and caching, facilitating rapid testing and iteration.\n", + "\n", + "We will use promptfoo's command line interface in the following examples. Please follow installation instructions [here](https://www.promptfoo.dev/docs/installation/#for-command-line-usage).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Evals are defined in a configuration file `promptfooconfig.yaml`, which defines elements such as providers, prompts, test cases, and assertions." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the following example, we will perform a two-step evaluation:\n", + "\n", + "1. Evaluate the performance of different LLM models given a set of constraints.\n", + "2. Evaluate the quality of different prompts for the best performing model from 1." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "description: Best model eval\n", + "prompts:\n", + "- file://prompt1.txt\n", + "providers:\n", + "- openai:gpt-4o-mini\n", + "- openai:gpt-4\n", + "- openai:gpt-3.5-turbo\n", + "defaultTest:\n", + " assert:\n", + " - type: cost\n", + " threshold: 0.001\n", + " - type: latency\n", + " threshold: 1000\n", + " - type: python\n", + " value: len(output) < 200\n", + " - type: llm-rubric\n", + " value: Does the summary look like it was written by an expert analyst [Yes/No]?\n", + "tests: file://tests.csv\n", + "\n" + ] + } + ], + "source": [ + "import yaml\n", + "\n", + "# Read the YAML file\n", + "with open('promptfoo/model_comparison/promptfooconfig.yaml', 'r') as f:\n", + " config = yaml.safe_load(f)\n", + "\n", + "# Pretty print the YAML content\n", + "print(yaml.dump(config, default_flow_style=False, sort_keys=False))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The configuration file shows how PromptFoo can be used to evaluate different LLM models. The YAML configuration defines three providers (GPT-4o-mini, GPT-4, and GPT-3.5-turbo) and sets up test assertions to validate their outputs. These assertions check important constraints:\n", + "\n", + "1. Cost efficiency: Each inference must cost less than $0.001\n", + "2. Latency requirements: Response time must be under 1000ms \n", + "3. Output length: Generated text must be less than 200 characters\n", + "4. Output quality: An LLM-based rubric evaluates if the output appears to be written by an expert (uses openai's gpt-4o model)\n", + "\n", + "The prompts are loaded from an external file (prompt1.txt) and test cases are defined in tests.csv. This structured approach enables systematic evaluation of model performance across multiple decoupled dimensions, namely prompts and test cases.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```bash\n", + "promptfoo eval --no-cache --output eval.json\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This command will run the evaluation and store the results in eval.json while making sure that the evaluation is not cached so we are measuring actual latency of the LLMs. The code below processes the PromptFoo evaluation results stored in eval.json. It reads the evaluation data from the JSON file and extracts key metrics including:\n", + "\n", + "- Provider name (e.g. gpt-4, gpt-3.5-turbo)\n", + "- Latency in milliseconds \n", + "- Token usage statistics\n", + "- Cost per request\n", + "- Number of passed/failed assertions\n", + "- Prompt token count\n", + "- Total number of API requests" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import pandas as pd\n", + "\n", + "# Read the JSON file\n", + "with open('promptfoo/model_comparison/eval.json', 'r') as f:\n", + " eval_data = json.load(f)\n", + "\n", + "# Extract results into a list of dictionaries\n", + "results = []\n", + "for prompt in eval_data['results']['prompts']:\n", + " result = {\n", + " 'provider': prompt['provider'],\n", + " 'latency_ms': prompt['metrics']['totalLatencyMs'],\n", + " 'token_usage': prompt['metrics']['tokenUsage']['total'],\n", + " 'cost': prompt['metrics']['cost'],\n", + " 'assert_pass': prompt['metrics']['assertPassCount'], \n", + " 'assert_fail': prompt['metrics']['assertFailCount'],\n", + " 'prompt_tokens': prompt['metrics']['tokenUsage']['prompt'],\n", + " 'num_requests': prompt['metrics']['tokenUsage']['numRequests']\n", + " }\n", + " results.append(result)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import display, Markdown" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "### PromptFoo Evaluation Results" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " provider latency_ms token_usage cost assert_pass \\\n", + "0 openai:gpt-4o-mini 2463 97 0.000035 6 \n", + "1 openai:gpt-4 3773 103 0.004620 4 \n", + "2 openai:gpt-3.5-turbo 1669 95 0.000091 7 \n", + "\n", + " assert_fail prompt_tokens num_requests \n", + "0 2 52 2 \n", + "1 4 52 2 \n", + "2 1 52 2 \n" + ] + } + ], + "source": [ + "# Convert to DataFrame\n", + "df = pd.DataFrame(results)\n", + "display(Markdown(\"### PromptFoo Evaluation Results\"))\n", + "print(df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The evaluation results reveal interesting performance characteristics across different OpenAI models. GPT-3.5-turbo demonstrates the best overall performance given our criteria with the lowest latency (1669ms), lowest token usage (95), and highest number of passed assertions (7). While GPT-4 shows higher token usage (103) and latency (3773ms), it also has the highest cost per request ($0.00462). The GPT-4-mini variant offers a middle ground, with moderate latency and token usage, while maintaining relatively good assertion performance (6 passes). These results suggest that for this particular evaluation task, GPT-3.5-turbo provides the best balance of performance, reliability, and cost-effectiveness.\n", + "\n", + "Promptfool also offers a web interface for visualizing the evaluation results as shown in {numref}`promptfoo1`. \n", + "\n", + "```bash\n", + "promptfoo view\n", + "```\n", + "\n", + "We can observe results per test case (i.e. section of the SEC filing) and per provider. Humans can also manually review the results and provide feedback as well as generate new test cases." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```{figure} ../_static/evals/promptfoo1.png\n", + "---\n", + "name: promptfoo1\n", + "alt: PromptFoo Evaluation Results\n", + "scale: 30%\n", + "align: center\n", + "---\n", + "PromptFoo evaluation results showing performance metrics across different models.\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we have established `GPT-3.5-turbo` as our model of choice given the minimum required criteria based on cost, latency and basic qualitative evaluation, we can compare the performance of different prompts as a next evaluation step. Can we improve the quality of the summaries by using different prompts?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, we redefine our evaluation criteria. We now would like to select the prompt that delivers the most \"detailed\" summaries. Our updated promptfoo configuration file is shown below." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "description: Best model eval\n", + "prompts:\n", + "- file://prompt1.txt\n", + "- file://prompt2.txt\n", + "- file://prompt3.txt\n", + "providers:\n", + "- openai:gpt-3.5-turbo\n", + "defaultTest:\n", + " assert:\n", + " - type: llm-rubric\n", + " value: 'Evaluate the output based on how detailed it is. Grade it on a scale\n", + " of 0.0 to 1.0, where:\n", + "\n", + " Score of 0.1: Not much detail.\n", + "\n", + " Score of 0.5: Some detail.\n", + "\n", + " Score of 1.0: Very detailed.\n", + "\n", + " '\n", + "tests: file://tests.csv\n", + "\n" + ] + } + ], + "source": [ + "# Read the YAML file\n", + "with open('promptfoo/prompt_comparison/promptfooconfig.yaml', 'r') as f:\n", + " config = yaml.safe_load(f)\n", + "\n", + "# Pretty print the YAML content\n", + "print(yaml.dump(config, default_flow_style=False, sort_keys=False))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that we are now passing 3 different prompts. And we have updated our assertions to check if the output is detailed. Now, let's define 3 prompt variations we would like to test aiming at improving the quality/detail of the summaries." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "### prompt1.txt\n", + "---\n", + "'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'\n", + "\n", + "\n", + "### prompt2.txt\n", + "---\n", + "'ROLE: You are a financial analyst. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'\n", + "\n", + "\n", + "### prompt3.txt\n", + "---\n", + "'ROLE: You are a financial analyst. REQUIREMENTS: BE DETAILED. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Display the prompt variations\n", + "from IPython.display import display, Markdown\n", + "\n", + "prompt_files = ['prompt1.txt', 'prompt2.txt', 'prompt3.txt']\n", + "prompt_content = []\n", + "\n", + "for file in prompt_files:\n", + " with open(f'promptfoo/prompt_comparison/{file}', 'r') as f:\n", + " content = f.read().strip()\n", + " prompt_content.append(f\"### {file}\\n---\\n{content}\\n\")\n", + "\n", + "display(Markdown(\"\\n\\n\".join(prompt_content)))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The first prompt matches our previous prompt. The second prompt adds a \"financial analyst\" role to the prompt. The third prompt expands on second prompt and add a requirement \"BE DETAILED\"." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now run the evaluation again." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```bash\n", + "promptfoo eval --output eval.json\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "### Prompt Comparison Results by Section" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Prompt Section prompt1.txt prompt2.txt prompt3.txt\n", + "0 Legal Proceedings 0.1 0.5 1.0\n", + "1 Risk Factors 0.1 0.5 0.5\n" + ] + } + ], + "source": [ + "# Read the evaluation results from JSON file\n", + "import json\n", + "with open('promptfoo/prompt_comparison/eval.json', 'r') as f:\n", + " eval_data = json.load(f)\n", + "\n", + "# Create a list to store the data\n", + "data = []\n", + "\n", + "# Extract results for each test case\n", + "for result in eval_data['results']['results']:\n", + " section = result['vars']['section']\n", + " prompt_id = result['promptId']\n", + " score = result['gradingResult']['score'] if 'gradingResult' in result else 0.0\n", + " \n", + " # Find the corresponding prompt file\n", + " for prompt in eval_data['results']['prompts']:\n", + " if prompt['id'] == prompt_id:\n", + " prompt_file = prompt['label'].split(':')[0]\n", + " break\n", + " \n", + " # Add to data list\n", + " data.append([section, prompt_file, score])\n", + "\n", + "# Convert to DataFrame\n", + "df_raw = pd.DataFrame(data, columns=['Section', 'Prompt', 'Score'])\n", + "\n", + "# Pivot to get desired format\n", + "df = df_raw.pivot(index='Section', columns='Prompt', values='Score').reset_index()\n", + "df = df[['Section', 'prompt1.txt', 'prompt2.txt', 'prompt3.txt']]\n", + "\n", + "display(Markdown(\"### Prompt Comparison Results by Section\"))\n", + "print(df)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The results show that prompt3.txt performs best for Legal Proceedings sections, achieving a perfect score of 1.0 compared to 0.5 for prompt2.txt and 0.1 for prompt1.txt. For Risk Factors sections, both prompt2.txt and prompt3.txt achieve moderate scores of 0.5, while prompt1.txt scores poorly at 0.1. This suggests that prompt3.txt is generally more effective at extracting detailed information, particularly for legal content. In summary, defining a Role and a requirement for the output to be detailed is a good way to improve the quality of the summaries at least for this specific task, model and criteria.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In conclusion, Promptfoo can serve as an effective LLM application evaluation tool particularly for its ability to decouple several components of the evaluation process. Hence enabling the user to focus on the most important aspects of the evaluation given the particular application and criteria making it a valuable and flexible tool for LLM application development." ] }, { diff --git a/tamingllms/markdown/toc.md b/tamingllms/markdown/toc.md index b06c890..e66ebb8 100644 --- a/tamingllms/markdown/toc.md +++ b/tamingllms/markdown/toc.md @@ -139,3 +139,21 @@ Abstract: *The current discourse around Large Language Models (LLMs) tends to fo - A.2 Monitoring Solutions - A.3 Open Source Models - A.4 Community Resources + + +## Citation +[![CC BY-NC-SA 4.0][cc-by-nc-sa-image]][cc-by-nc-sa] + +[cc-by-nc-sa]: http://creativecommons.org/licenses/by-nc-sa/4.0/ +[cc-by-nc-sa-image]: https://licensebuttons.net/l/by-nc-sa/4.0/88x31.png +[cc-by-nc-sa-shield]: https://img.shields.io/badge/License-CC-BY--NC--SA-4.0-lightgrey.svg + +``` +@misc{tharsistpsouza2024tamingllms, + author = {Tharsis T. P. Souza}, + title = {Taming LLMs: A Practical Guide to LLM Pitfalls with Open Source Software}, + year = {2024}, + journal = {GitHub repository}, + url = {https://github.com/souzatharsis/tamingLLMs) +} +``` \ No newline at end of file diff --git a/tamingllms/notebooks/evals.ipynb b/tamingllms/notebooks/evals.ipynb index 6b5b1ca..75041e9 100644 --- a/tamingllms/notebooks/evals.ipynb +++ b/tamingllms/notebooks/evals.ipynb @@ -37,7 +37,11 @@ "\n", "- Temperature = 0: Most deterministic, but potentially repetitive\n", "- Temperature = 1: Balanced creativity and coherence\n", - "- Temperature > 1: Increased randomness, potentially incoherent" + "- Temperature > 1: Increased randomness, potentially incoherent\n", + "\n", + "A temperature of 1 represents the unscaled probability scores for each token in the vocabulary. Decreasing the temperature closer to 0 sharpens the distribution, so the most likely token will have an even higher probability score. Conversely, increasing the temperature makes the distribution more uniform {cite}`build-llms-from-scratch-book`.\n", + "\n", + "In this simple experiment, we use an LLM to write a single-statement executive summary of an input financial filing. We observe that even a simple parameter like temperature can dramatically alter model behavior in ways that are difficult to systematically assess. At temperature 0.0, responses are consistent but potentially too rigid. At 1.0, outputs become more varied but less predictable. At 2.0, responses can be wildly different and often incoherent. This non-deterministic behavior makes traditional software testing approaches inadequate." ] }, { @@ -171,13 +175,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "A temperature of 1 represents the unscaled probability scores for each token in the vocabulary. Decreasing the temperature closer to 0 sharpens the distribution, so the most likely token will have an even higher probability score. Conversely, increasing the temperature makes the distribution more uniform {cite}`build-llms-from-scratch-book`.\n", - "\n", - "In this simple experiment, we use an LLM to write a single-statement executive summary of an input financial filing. We observe that even a simple parameter like temperature can dramatically alter model behavior in ways that are difficult to systematically assess. At temperature 0.0, responses are consistent but potentially too rigid. At 1.0, outputs become more varied but less predictable. At 2.0, responses can be wildly different and often incoherent. This non-deterministic behavior makes traditional software testing approaches inadequate.\n", - "\n", - "\n", - "\n", - "The implications for evaluation are critical. How can one effectively test an LLM-powered system when the same prompt can yield radically different outputs based on a single parameter? Traditional testing relies on predictable inputs and outputs, but LLMs force us to grapple with probabilistic behavior. While lower temperatures may seem safer for critical applications, they don't necessarily eliminate the underlying uncertainty. This highlights the need for new evaluation paradigms that can handle both deterministic and probabilistic aspects of LLM behavior." + "How can one effectively test an LLM-powered system when the same prompt can yield radically different outputs based on a single parameter? Traditional testing relies on predictable inputs and outputs, but LLMs force us to grapple with probabilistic behavior. While lower temperatures may seem safer for critical applications, they don't necessarily eliminate the underlying uncertainty. This highlights the need for new evaluation paradigms that can handle both deterministic and probabilistic aspects of LLM behavior." ] }, { @@ -187,7 +185,9 @@ "\n", "## Emerging Properties\n", "\n", - "Beyond their non-deterministic nature, LLMs present another fascinating challenge: emergent abilities that spontaneously arise as models scale up in size. These abilities - from basic question answering to complex reasoning - aren't explicitly programmed but rather emerge \"naturally\" as the models grow larger and are trained on more data. This makes evaluation fundamentally different from traditional software testing, where capabilities are explicitly coded and can be tested against clear specifications.\n", + "Beyond their non-deterministic nature, LLMs present another fascinating characteristic: emergent abilities that spontaneously arise as models scale up in size. These abilities - from basic question answering to complex reasoning - aren't explicitly programmed but rather emerge \"naturally\" as the models grow larger and are trained on more data. This makes evaluation fundamentally different from traditional software testing, where capabilities are explicitly coded and can be tested against pre-defined specifications.\n", + "\n", + "{numref}`emerging-properties` provides a list of emergent abilities of large language models and the scale. The relationship between model scale and emergent abilities follows a fascinating non-linear pattern. Below certain size thresholds, specific abilities may be completely absent from the model - it simply cannot perform certain tasks, no matter how much you try to coax them out. However, once the model reaches critical points in its scaling journey, these abilities can suddenly manifest in what researchers call a phase transition - a dramatic shift from inability to capability. This unpredictable emergence of capabilities stands in stark contrast to traditional software development, where features are deliberately implemented and can be systematically tested.\n", "\n", "```{figure} ../_static/evals/emerging.png\n", "---\n", @@ -200,9 +200,7 @@ "Emergent abilities of large language models and the scale {cite}`wei2022emergentabilitieslargelanguage`.\n", "```\n", "\n", - " {numref}`emerging-properties` provides a list of emergent abilities of large language models and the scale. The relationship between model scale and emergent abilities follows a fascinating non-linear pattern. Below certain size thresholds, specific abilities may be completely absent from the model - it simply cannot perform certain tasks, no matter how much you try to coax them out. However, once the model reaches critical points in its scaling journey, these abilities can suddenly manifest in what researchers call a phase transition - a dramatic shift from inability to capability. This unpredictable emergence of capabilities stands in stark contrast to traditional software development, where features are deliberately implemented and can be systematically tested.\n", - "\n", - "The implications for evaluation are pressing. While conventional software testing relies on stable test suites and well-defined acceptance criteria, LLM evaluation must contend with a constantly shifting landscape of capabilities. What worked to evaluate a 7B parameter model may be completely inadequate for a 70B parameter model that has developed new emergent abilities. This dynamic nature of LLM capabilities forces us to fundamentally rethink our approach to testing and evaluation.\n", + "The implications for evaluation are critical. While conventional software testing relies on stable test suites and well-defined acceptance criteria, LLM evaluation must contend with a constantly shifting landscape of capabilities. What worked to evaluate a 7B parameter model may be completely inadequate for a 70B parameter model that has developed new emergent abilities. This dynamic nature of LLM capabilities forces us to fundamentally rethink our approach to testing and evaluation.\n", "\n", "## Problem Statement\n", "\n", @@ -216,11 +214,11 @@ "The challenge becomes even more complex when we consider that traditional software evaluation methods simply weren't designed for these kinds of systems - There is an **Evals Gap** between traditional software testing and LLM evaluation. We need new frameworks that can account for both the deterministic aspects we're used to testing and the emergent properties that make LLMs unique. \n", "\n", "{numref}`evals-table` explores how LLM evaluation differs from traditional software testing across several key dimensions:\n", - "- **Capability Assessment vs Functional Testing**: Traditional software testing validates specific functionality against predefined requirements. LLM evaluation must assess not necessarily pre-defined \"emergent properties\" like reasoning, creativity, and language understanding that extend beyond explicit programming.\n", + "- **Capability Assessment vs Functional Testing**: Traditional software testing validates specific functionality against predefined requirements. LLM evaluation must assess not necessarily pre-defined behavior but also \"emergent properties\" like reasoning, creativity, and language understanding that extend beyond explicit programming.\n", "\n", - "- **Metrics and Measurement Challenges**: While traditional software metrics can usually be precisely defined and measured, LLM evaluation often involves subjective qualities like \"helpfulness\" or \"naturalness\" that resist straightforward quantification. Even when we try to break these down into numeric scores, the underlying judgment remains inherently human and context-dependent.\n", + "- **Metrics and Measurement Challenges**: While traditional software metrics can usually be precisely defined and measured, LLM evaluation often involves subjective qualities like \"helpfulness\" or \"naturalness\" that resist straightforward quantification. Even when we try to break these down into numeric scores, the underlying judgment often remains inherently human and context-dependent.\n", "\n", - "- **Dataset Contamination**: Traditional software testing uses carefully crafted test cases with known inputs and expected outputs (e.g., unit tests, integration tests). In contrast, LLMs trained on massive internet-scale datasets risk having already seen and memorized evaluation examples during training, which can lead to artificially inflated performance scores. This requires careful dataset curation to ensure test sets are truly unseen by the model and rigorous cross-validation approaches.\n", + "- **Dataset Contamination**: Traditional software testing uses carefully crafted test cases with known inputs and expected outputs (e.g., unit tests). In contrast, LLMs trained on massive internet-scale datasets risk having already seen and memorized evaluation examples during training, which can lead to artificially inflated performance scores. This requires careful dataset curation to ensure test sets are truly unseen by the model and rigorous cross-validation approaches.\n", "\n", "- **Benchmark Evolution**: Traditional software maintains stable test suites over time. LLM benchmarks continuously evolve as capabilities advance, making longitudinal performance comparisons difficult and potentially obsoleting older evaluation methods.\n", "\n", @@ -252,7 +250,7 @@ "- Evaluation must align with business objectives\n", "- A great LLM doesn't guarantee a great application!\n", "\n", - "Examples of key requirements for validation are listed in {numref}`validation-requirements`.\n", + "Examples of key requirements for validation are listed in {numref}`validation-requirements` ranging from Safety, Cognitive, Technical, Meta-Cognitive, to Ethical aspects.\n", "\n", "```{table} LLM Application Testing Requirements Matrix\n", ":name: validation-requirements\n", @@ -1524,18 +1522,9 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/tobias/src/tamingLLMs/tamingllms/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n" - ] - } - ], + "outputs": [], "source": [ "import evaluate as hf_evaluate # HuggingFace's evaluate\n", "from langsmith import evaluate as langsmith_evaluate # LangSmith's evaluate\n", @@ -2098,8 +2087,428 @@ "source": [ "### PromptFoo\n", "\n", - "PromptFoo {cite}`promptfoo` is a framework for evaluating the quality of prompts for LLMs.\n", - "\n" + "Promptfoo {cite}`promptfoo2024` is an open-source framework designed for evaluating applications that utilize large language models (LLMs). Key features include:\n", + "\n", + "1. **Automated Testing**: Promptfoo provides automated testing capabilities, allowing developers to run custom evaluations tailored to their applications.\n", + "\n", + "2. **Custom Probes**: Developers can create custom probes to focus on specific use cases for instance decoupling prompts from tests cases.\n", + "\n", + "3. **User-Friendly CLI**: The framework features a command-line interface that supports live reloads and caching, facilitating rapid testing and iteration.\n", + "\n", + "We will use promptfoo's command line interface in the following examples. Please follow installation instructions [here](https://www.promptfoo.dev/docs/installation/#for-command-line-usage).\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Evals are defined in a configuration file `promptfooconfig.yaml`, which defines elements such as providers, prompts, test cases, and assertions." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the following example, we will perform a two-step evaluation:\n", + "\n", + "1. Evaluate the performance of different LLM models given a set of constraints.\n", + "2. Evaluate the quality of different prompts for the best performing model from 1." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "description: Best model eval\n", + "prompts:\n", + "- file://prompt1.txt\n", + "providers:\n", + "- openai:gpt-4o-mini\n", + "- openai:gpt-4\n", + "- openai:gpt-3.5-turbo\n", + "defaultTest:\n", + " assert:\n", + " - type: cost\n", + " threshold: 0.001\n", + " - type: latency\n", + " threshold: 1000\n", + " - type: python\n", + " value: len(output) < 200\n", + " - type: llm-rubric\n", + " value: Does the summary look like it was written by an expert analyst [Yes/No]?\n", + "tests: file://tests.csv\n", + "\n" + ] + } + ], + "source": [ + "import yaml\n", + "\n", + "# Read the YAML file\n", + "with open('promptfoo/model_comparison/promptfooconfig.yaml', 'r') as f:\n", + " config = yaml.safe_load(f)\n", + "\n", + "# Pretty print the YAML content\n", + "print(yaml.dump(config, default_flow_style=False, sort_keys=False))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The configuration file shows how PromptFoo can be used to evaluate different LLM models. The YAML configuration defines three providers (GPT-4o-mini, GPT-4, and GPT-3.5-turbo) and sets up test assertions to validate their outputs. These assertions check important constraints:\n", + "\n", + "1. Cost efficiency: Each inference must cost less than $0.001\n", + "2. Latency requirements: Response time must be under 1000ms \n", + "3. Output length: Generated text must be less than 200 characters\n", + "4. Output quality: An LLM-based rubric evaluates if the output appears to be written by an expert (uses openai's gpt-4o model)\n", + "\n", + "The prompts are loaded from an external file (prompt1.txt) and test cases are defined in tests.csv. This structured approach enables systematic evaluation of model performance across multiple decoupled dimensions, namely prompts and test cases.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```bash\n", + "promptfoo eval --no-cache --output eval.json\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This command will run the evaluation and store the results in eval.json while making sure that the evaluation is not cached so we are measuring actual latency of the LLMs. The code below processes the PromptFoo evaluation results stored in eval.json. It reads the evaluation data from the JSON file and extracts key metrics including:\n", + "\n", + "- Provider name (e.g. gpt-4, gpt-3.5-turbo)\n", + "- Latency in milliseconds \n", + "- Token usage statistics\n", + "- Cost per request\n", + "- Number of passed/failed assertions\n", + "- Prompt token count\n", + "- Total number of API requests" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import pandas as pd\n", + "\n", + "# Read the JSON file\n", + "with open('promptfoo/model_comparison/eval.json', 'r') as f:\n", + " eval_data = json.load(f)\n", + "\n", + "# Extract results into a list of dictionaries\n", + "results = []\n", + "for prompt in eval_data['results']['prompts']:\n", + " result = {\n", + " 'provider': prompt['provider'],\n", + " 'latency_ms': prompt['metrics']['totalLatencyMs'],\n", + " 'token_usage': prompt['metrics']['tokenUsage']['total'],\n", + " 'cost': prompt['metrics']['cost'],\n", + " 'assert_pass': prompt['metrics']['assertPassCount'], \n", + " 'assert_fail': prompt['metrics']['assertFailCount'],\n", + " 'prompt_tokens': prompt['metrics']['tokenUsage']['prompt'],\n", + " 'num_requests': prompt['metrics']['tokenUsage']['numRequests']\n", + " }\n", + " results.append(result)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import display, Markdown" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "### PromptFoo Evaluation Results" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " provider latency_ms token_usage cost assert_pass \\\n", + "0 openai:gpt-4o-mini 2463 97 0.000035 6 \n", + "1 openai:gpt-4 3773 103 0.004620 4 \n", + "2 openai:gpt-3.5-turbo 1669 95 0.000091 7 \n", + "\n", + " assert_fail prompt_tokens num_requests \n", + "0 2 52 2 \n", + "1 4 52 2 \n", + "2 1 52 2 \n" + ] + } + ], + "source": [ + "# Convert to DataFrame\n", + "df = pd.DataFrame(results)\n", + "display(Markdown(\"### PromptFoo Evaluation Results\"))\n", + "print(df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The evaluation results reveal interesting performance characteristics across different OpenAI models. GPT-3.5-turbo demonstrates the best overall performance given our criteria with the lowest latency (1669ms), lowest token usage (95), and highest number of passed assertions (7). While GPT-4 shows higher token usage (103) and latency (3773ms), it also has the highest cost per request ($0.00462). The GPT-4-mini variant offers a middle ground, with moderate latency and token usage, while maintaining relatively good assertion performance (6 passes). These results suggest that for this particular evaluation task, GPT-3.5-turbo provides the best balance of performance, reliability, and cost-effectiveness.\n", + "\n", + "Promptfool also offers a web interface for visualizing the evaluation results as shown in {numref}`promptfoo1`. \n", + "\n", + "```bash\n", + "promptfoo view\n", + "```\n", + "\n", + "We can observe results per test case (i.e. section of the SEC filing) and per provider. Humans can also manually review the results and provide feedback as well as generate new test cases." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```{figure} ../_static/evals/promptfoo1.png\n", + "---\n", + "name: promptfoo1\n", + "alt: PromptFoo Evaluation Results\n", + "scale: 30%\n", + "align: center\n", + "---\n", + "PromptFoo evaluation results showing performance metrics across different models.\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we have established `GPT-3.5-turbo` as our model of choice given the minimum required criteria based on cost, latency and basic qualitative evaluation, we can compare the performance of different prompts as a next evaluation step. Can we improve the quality of the summaries by using different prompts?" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, we redefine our evaluation criteria. We now would like to select the prompt that delivers the most \"detailed\" summaries. Our updated promptfoo configuration file is shown below." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "description: Best model eval\n", + "prompts:\n", + "- file://prompt1.txt\n", + "- file://prompt2.txt\n", + "- file://prompt3.txt\n", + "providers:\n", + "- openai:gpt-3.5-turbo\n", + "defaultTest:\n", + " assert:\n", + " - type: llm-rubric\n", + " value: 'Evaluate the output based on how detailed it is. Grade it on a scale\n", + " of 0.0 to 1.0, where:\n", + "\n", + " Score of 0.1: Not much detail.\n", + "\n", + " Score of 0.5: Some detail.\n", + "\n", + " Score of 1.0: Very detailed.\n", + "\n", + " '\n", + "tests: file://tests.csv\n", + "\n" + ] + } + ], + "source": [ + "# Read the YAML file\n", + "with open('promptfoo/prompt_comparison/promptfooconfig.yaml', 'r') as f:\n", + " config = yaml.safe_load(f)\n", + "\n", + "# Pretty print the YAML content\n", + "print(yaml.dump(config, default_flow_style=False, sort_keys=False))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that we are now passing 3 different prompts. And we have updated our assertions to check if the output is detailed. Now, let's define 3 prompt variations we would like to test aiming at improving the quality/detail of the summaries." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "### prompt1.txt\n", + "---\n", + "'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'\n", + "\n", + "\n", + "### prompt2.txt\n", + "---\n", + "'ROLE: You are a financial analyst. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'\n", + "\n", + "\n", + "### prompt3.txt\n", + "---\n", + "'ROLE: You are a financial analyst. REQUIREMENTS: BE DETAILED. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Display the prompt variations\n", + "from IPython.display import display, Markdown\n", + "\n", + "prompt_files = ['prompt1.txt', 'prompt2.txt', 'prompt3.txt']\n", + "prompt_content = []\n", + "\n", + "for file in prompt_files:\n", + " with open(f'promptfoo/prompt_comparison/{file}', 'r') as f:\n", + " content = f.read().strip()\n", + " prompt_content.append(f\"### {file}\\n---\\n{content}\\n\")\n", + "\n", + "display(Markdown(\"\\n\\n\".join(prompt_content)))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The first prompt matches our previous prompt. The second prompt adds a \"financial analyst\" role to the prompt. The third prompt expands on second prompt and add a requirement \"BE DETAILED\"." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now run the evaluation again." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```bash\n", + "promptfoo eval --output eval.json\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/markdown": [ + "### Prompt Comparison Results by Section" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Prompt Section prompt1.txt prompt2.txt prompt3.txt\n", + "0 Legal Proceedings 0.1 0.5 1.0\n", + "1 Risk Factors 0.1 0.5 0.5\n" + ] + } + ], + "source": [ + "# Read the evaluation results from JSON file\n", + "import json\n", + "with open('promptfoo/prompt_comparison/eval.json', 'r') as f:\n", + " eval_data = json.load(f)\n", + "\n", + "# Create a list to store the data\n", + "data = []\n", + "\n", + "# Extract results for each test case\n", + "for result in eval_data['results']['results']:\n", + " section = result['vars']['section']\n", + " prompt_id = result['promptId']\n", + " score = result['gradingResult']['score'] if 'gradingResult' in result else 0.0\n", + " \n", + " # Find the corresponding prompt file\n", + " for prompt in eval_data['results']['prompts']:\n", + " if prompt['id'] == prompt_id:\n", + " prompt_file = prompt['label'].split(':')[0]\n", + " break\n", + " \n", + " # Add to data list\n", + " data.append([section, prompt_file, score])\n", + "\n", + "# Convert to DataFrame\n", + "df_raw = pd.DataFrame(data, columns=['Section', 'Prompt', 'Score'])\n", + "\n", + "# Pivot to get desired format\n", + "df = df_raw.pivot(index='Section', columns='Prompt', values='Score').reset_index()\n", + "df = df[['Section', 'prompt1.txt', 'prompt2.txt', 'prompt3.txt']]\n", + "\n", + "display(Markdown(\"### Prompt Comparison Results by Section\"))\n", + "print(df)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The results show that prompt3.txt performs best for Legal Proceedings sections, achieving a perfect score of 1.0 compared to 0.5 for prompt2.txt and 0.1 for prompt1.txt. For Risk Factors sections, both prompt2.txt and prompt3.txt achieve moderate scores of 0.5, while prompt1.txt scores poorly at 0.1. This suggests that prompt3.txt is generally more effective at extracting detailed information, particularly for legal content. In summary, defining a Role and a requirement for the output to be detailed is a good way to improve the quality of the summaries at least for this specific task, model and criteria.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In conclusion, Promptfoo can serve as an effective LLM application evaluation tool particularly for its ability to decouple several components of the evaluation process. Hence enabling the user to focus on the most important aspects of the evaluation given the particular application and criteria making it a valuable and flexible tool for LLM application development." ] }, { diff --git a/tamingllms/notebooks/promptfoo/README.md b/tamingllms/notebooks/promptfoo/README.md new file mode 100644 index 0000000..a128bdf --- /dev/null +++ b/tamingllms/notebooks/promptfoo/README.md @@ -0,0 +1,10 @@ +To get started, set your OPENAI_API_KEY environment variable, or other required keys for the providers you selected. + +Next, edit promptfooconfig.yaml. + +Then run: +``` +promptfoo eval +``` + +Afterwards, you can view the results by running `promptfoo view` diff --git a/tamingllms/notebooks/promptfoo/model_comparison/eval.json b/tamingllms/notebooks/promptfoo/model_comparison/eval.json new file mode 100644 index 0000000..742577e --- /dev/null +++ b/tamingllms/notebooks/promptfoo/model_comparison/eval.json @@ -0,0 +1,845 @@ +{ + "evalId": "eval-FDA-2024-12-10T15:35:24", + "results": { + "version": 3, + "timestamp": "2024-12-10T15:35:24.707Z", + "prompts": [ + { + "raw": "'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'", + "label": "prompt1.txt: 'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'", + "id": "1cd4a63a09ec6bcce588ef0486fa8be9ffa6dbbb4637fdf551a3dc72ac9a6462", + "provider": "openai:gpt-4o-mini", + "metrics": { + "score": 1.5, + "testPassCount": 0, + "testFailCount": 2, + "assertPassCount": 6, + "assertFailCount": 2, + "totalLatencyMs": 2463, + "tokenUsage": { + "total": 97, + "prompt": 52, + "completion": 45, + "cached": 0, + "numRequests": 2 + }, + "namedScores": {}, + "namedScoresCount": {}, + "cost": 0.0000348 + } + }, + { + "raw": "'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'", + "label": "prompt1.txt: 'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'", + "id": "1cd4a63a09ec6bcce588ef0486fa8be9ffa6dbbb4637fdf551a3dc72ac9a6462", + "provider": "openai:gpt-4", + "metrics": { + "score": 1, + "testPassCount": 0, + "testFailCount": 2, + "assertPassCount": 4, + "assertFailCount": 4, + "totalLatencyMs": 3773, + "tokenUsage": { + "total": 103, + "prompt": 52, + "completion": 51, + "cached": 0, + "numRequests": 2 + }, + "namedScores": {}, + "namedScoresCount": {}, + "cost": 0.00462 + } + }, + { + "raw": "'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'", + "label": "prompt1.txt: 'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'", + "id": "1cd4a63a09ec6bcce588ef0486fa8be9ffa6dbbb4637fdf551a3dc72ac9a6462", + "provider": "openai:gpt-3.5-turbo", + "metrics": { + "score": 1.75, + "testPassCount": 1, + "testFailCount": 1, + "assertPassCount": 7, + "assertFailCount": 1, + "totalLatencyMs": 1669, + "tokenUsage": { + "total": 95, + "prompt": 52, + "completion": 43, + "cached": 0, + "numRequests": 2 + }, + "namedScores": {}, + "namedScoresCount": {}, + "cost": 0.0000905 + } + } + ], + "results": [ + { + "cost": 0.0000415, + "error": "Latency 1016ms is greater than threshold 1000ms", + "gradingResult": { + "pass": false, + "score": 0.75, + "reason": "Latency 1016ms is greater than threshold 1000ms", + "namedScores": {}, + "tokensUsed": { + "total": 266, + "prompt": 215, + "completion": 51, + "cached": 0 + }, + "componentResults": [ + { + "pass": true, + "score": 1, + "reason": "Assertion passed", + "assertion": { + "type": "cost", + "threshold": 0.001 + } + }, + { + "pass": false, + "score": 0, + "reason": "Latency 1016ms is greater than threshold 1000ms", + "assertion": { + "type": "latency", + "threshold": 1000 + } + }, + { + "pass": true, + "score": 1, + "reason": "Assertion passed", + "assertion": { + "type": "python", + "value": "len(output) < 200" + } + }, + { + "assertion": { + "type": "llm-rubric", + "value": "Does the summary look like it was written by an expert analyst [Yes/No]?" + }, + "pass": true, + "score": 1, + "reason": "The summary uses professional language and clearly identifies potential threats and uncertainties related to financial performance and operations, which suggests it was written by someone with expertise.", + "tokensUsed": { + "total": 266, + "prompt": 215, + "completion": 51, + "cached": 0 + } + } + ], + "assertion": null + }, + "id": "013dd17b-5296-4cdd-bdd1-6c02b9d4180a", + "latencyMs": 1016, + "namedScores": {}, + "prompt": { + "raw": "'Generate a 1-liner summary of the Section Risk Factors from an SEC filing: '", + "label": "prompt1.txt: 'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'" + }, + "promptId": "1cd4a63a09ec6bcce588ef0486fa8be9ffa6dbbb4637fdf551a3dc72ac9a6462", + "promptIdx": 2, + "provider": { + "id": "openai:gpt-3.5-turbo", + "label": "" + }, + "response": { + "output": "This section highlights potential threats and uncertainties that could negatively impact the company's financial performance and operations.", + "tokenUsage": { + "total": 45, + "prompt": 26, + "completion": 19 + }, + "cached": false, + "cost": 0.0000415 + }, + "score": 0.75, + "success": false, + "testCase": { + "vars": { + "section": "Risk Factors", + " content": "Item 1A. Macroeconomic and Industry Risks The Company’s operations and performance depend significantly on global and regional economic conditions and adverse economic conditions can materially adversely affect the Company’s business, results of operations and financial condition. The Company has international operations with sales outside the U.S. representing a majority of the Company’s total net sales. In addition, the Company’s global supply chain is large and complex and a majority of the Company’s supplier facilities, including manufacturing and assembly sites, are located outside the U.S. As a result, the Company’s operations and performance depend significantly on global and regional economic conditions. Adverse macroeconomic conditions, including slow growth or recession, high unemployment, inflation, tighter credit, higher interest rates, and currency fluctuations, can adversely impact consumer confidence and spending and materially adversely affect demand for the Company’s products and services. In addition, consumer confidence and spending can be materially adversely affected in response to changes in fiscal and monetary policy, financial market volatility, declines in income or asset values, and other economic factors. In addition to an adverse impact on demand for the Company’s products and services, uncertainty about, or a decline in, global or regional economic conditions can have a significant impact on the Company’s suppliers, contract manufacturers, logistics providers, distributors, cellular network carriers and other channel partners, and developers. Potential outcomes include financial instability; inability to obtain credit to finance business operations; and insolvency. Adverse economic conditions can also lead to increased credit and collectibility risk on the Company’s trade receivables; the failure of derivative counterparties and other financial institutions; limitations on the Company’s ability to issue new debt; reduced liquidity; and declines in the fair values of the Company’s financial instruments. These and other impacts can materially adversely affect the Company’s business, results of operations, financial condition and stock price." + }, + "assert": [ + { + "type": "cost", + "threshold": 0.001 + }, + { + "type": "latency", + "threshold": 1000 + }, + { + "type": "python", + "value": "len(output) < 200" + }, + { + "type": "llm-rubric", + "value": "Does the summary look like it was written by an expert analyst [Yes/No]?" + } + ], + "options": {}, + "description": "Row #1", + "metadata": {} + }, + "testIdx": 0, + "vars": { + "section": "Risk Factors", + " content": "Item 1A. Macroeconomic and Industry Risks The Company’s operations and performance depend significantly on global and regional economic conditions and adverse economic conditions can materially adversely affect the Company’s business, results of operations and financial condition. The Company has international operations with sales outside the U.S. representing a majority of the Company’s total net sales. In addition, the Company’s global supply chain is large and complex and a majority of the Company’s supplier facilities, including manufacturing and assembly sites, are located outside the U.S. As a result, the Company’s operations and performance depend significantly on global and regional economic conditions. Adverse macroeconomic conditions, including slow growth or recession, high unemployment, inflation, tighter credit, higher interest rates, and currency fluctuations, can adversely impact consumer confidence and spending and materially adversely affect demand for the Company’s products and services. In addition, consumer confidence and spending can be materially adversely affected in response to changes in fiscal and monetary policy, financial market volatility, declines in income or asset values, and other economic factors. In addition to an adverse impact on demand for the Company’s products and services, uncertainty about, or a decline in, global or regional economic conditions can have a significant impact on the Company’s suppliers, contract manufacturers, logistics providers, distributors, cellular network carriers and other channel partners, and developers. Potential outcomes include financial instability; inability to obtain credit to finance business operations; and insolvency. Adverse economic conditions can also lead to increased credit and collectibility risk on the Company’s trade receivables; the failure of derivative counterparties and other financial institutions; limitations on the Company’s ability to issue new debt; reduced liquidity; and declines in the fair values of the Company’s financial instruments. These and other impacts can materially adversely affect the Company’s business, results of operations, financial condition and stock price." + }, + "metadata": {} + }, + { + "cost": 0.0000171, + "error": "Latency 1131ms is greater than threshold 1000ms", + "gradingResult": { + "pass": false, + "score": 0.75, + "reason": "Latency 1131ms is greater than threshold 1000ms", + "namedScores": {}, + "tokensUsed": { + "total": 265, + "prompt": 219, + "completion": 46, + "cached": 0 + }, + "componentResults": [ + { + "pass": true, + "score": 1, + "reason": "Assertion passed", + "assertion": { + "type": "cost", + "threshold": 0.001 + } + }, + { + "pass": false, + "score": 0, + "reason": "Latency 1131ms is greater than threshold 1000ms", + "assertion": { + "type": "latency", + "threshold": 1000 + } + }, + { + "pass": true, + "score": 1, + "reason": "Assertion passed", + "assertion": { + "type": "python", + "value": "len(output) < 200" + } + }, + { + "assertion": { + "type": "llm-rubric", + "value": "Does the summary look like it was written by an expert analyst [Yes/No]?" + }, + "pass": true, + "score": 1, + "reason": "The summary uses professional language and addresses potential challenges and uncertainties in a detailed manner, indicative of an expert analyst's perspective.", + "tokensUsed": { + "total": 265, + "prompt": 219, + "completion": 46, + "cached": 0 + } + } + ], + "assertion": null + }, + "id": "0013241c-196a-4005-955d-ed5f6444b5e8", + "latencyMs": 1131, + "namedScores": {}, + "prompt": { + "raw": "'Generate a 1-liner summary of the Section Risk Factors from an SEC filing: '", + "label": "prompt1.txt: 'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'" + }, + "promptId": "1cd4a63a09ec6bcce588ef0486fa8be9ffa6dbbb4637fdf551a3dc72ac9a6462", + "promptIdx": 0, + "provider": { + "id": "openai:gpt-4o-mini", + "label": "" + }, + "response": { + "output": "The Section on Risk Factors outlines potential challenges and uncertainties that could negatively impact the company's financial performance and operational stability.", + "tokenUsage": { + "total": 48, + "prompt": 26, + "completion": 22 + }, + "cached": false, + "cost": 0.0000171 + }, + "score": 0.75, + "success": false, + "testCase": { + "vars": { + "section": "Risk Factors", + " content": "Item 1A. Macroeconomic and Industry Risks The Company’s operations and performance depend significantly on global and regional economic conditions and adverse economic conditions can materially adversely affect the Company’s business, results of operations and financial condition. The Company has international operations with sales outside the U.S. representing a majority of the Company’s total net sales. In addition, the Company’s global supply chain is large and complex and a majority of the Company’s supplier facilities, including manufacturing and assembly sites, are located outside the U.S. As a result, the Company’s operations and performance depend significantly on global and regional economic conditions. Adverse macroeconomic conditions, including slow growth or recession, high unemployment, inflation, tighter credit, higher interest rates, and currency fluctuations, can adversely impact consumer confidence and spending and materially adversely affect demand for the Company’s products and services. In addition, consumer confidence and spending can be materially adversely affected in response to changes in fiscal and monetary policy, financial market volatility, declines in income or asset values, and other economic factors. In addition to an adverse impact on demand for the Company’s products and services, uncertainty about, or a decline in, global or regional economic conditions can have a significant impact on the Company’s suppliers, contract manufacturers, logistics providers, distributors, cellular network carriers and other channel partners, and developers. Potential outcomes include financial instability; inability to obtain credit to finance business operations; and insolvency. Adverse economic conditions can also lead to increased credit and collectibility risk on the Company’s trade receivables; the failure of derivative counterparties and other financial institutions; limitations on the Company’s ability to issue new debt; reduced liquidity; and declines in the fair values of the Company’s financial instruments. These and other impacts can materially adversely affect the Company’s business, results of operations, financial condition and stock price." + }, + "assert": [ + { + "type": "cost", + "threshold": 0.001 + }, + { + "type": "latency", + "threshold": 1000 + }, + { + "type": "python", + "value": "len(output) < 200" + }, + { + "type": "llm-rubric", + "value": "Does the summary look like it was written by an expert analyst [Yes/No]?" + } + ], + "options": {}, + "description": "Row #1", + "metadata": {} + }, + "testIdx": 0, + "vars": { + "section": "Risk Factors", + " content": "Item 1A. Macroeconomic and Industry Risks The Company’s operations and performance depend significantly on global and regional economic conditions and adverse economic conditions can materially adversely affect the Company’s business, results of operations and financial condition. The Company has international operations with sales outside the U.S. representing a majority of the Company’s total net sales. In addition, the Company’s global supply chain is large and complex and a majority of the Company’s supplier facilities, including manufacturing and assembly sites, are located outside the U.S. As a result, the Company’s operations and performance depend significantly on global and regional economic conditions. Adverse macroeconomic conditions, including slow growth or recession, high unemployment, inflation, tighter credit, higher interest rates, and currency fluctuations, can adversely impact consumer confidence and spending and materially adversely affect demand for the Company’s products and services. In addition, consumer confidence and spending can be materially adversely affected in response to changes in fiscal and monetary policy, financial market volatility, declines in income or asset values, and other economic factors. In addition to an adverse impact on demand for the Company’s products and services, uncertainty about, or a decline in, global or regional economic conditions can have a significant impact on the Company’s suppliers, contract manufacturers, logistics providers, distributors, cellular network carriers and other channel partners, and developers. Potential outcomes include financial instability; inability to obtain credit to finance business operations; and insolvency. Adverse economic conditions can also lead to increased credit and collectibility risk on the Company’s trade receivables; the failure of derivative counterparties and other financial institutions; limitations on the Company’s ability to issue new debt; reduced liquidity; and declines in the fair values of the Company’s financial instruments. These and other impacts can materially adversely affect the Company’s business, results of operations, financial condition and stock price." + }, + "metadata": {} + }, + { + "cost": 0.0000177, + "error": "Latency 1332ms is greater than threshold 1000ms", + "gradingResult": { + "pass": false, + "score": 0.75, + "reason": "Latency 1332ms is greater than threshold 1000ms", + "namedScores": {}, + "tokensUsed": { + "total": 275, + "prompt": 220, + "completion": 55, + "cached": 0 + }, + "componentResults": [ + { + "pass": true, + "score": 1, + "reason": "Assertion passed", + "assertion": { + "type": "cost", + "threshold": 0.001 + } + }, + { + "pass": false, + "score": 0, + "reason": "Latency 1332ms is greater than threshold 1000ms", + "assertion": { + "type": "latency", + "threshold": 1000 + } + }, + { + "pass": true, + "score": 1, + "reason": "Assertion passed", + "assertion": { + "type": "python", + "value": "len(output) < 200" + } + }, + { + "assertion": { + "type": "llm-rubric", + "value": "Does the summary look like it was written by an expert analyst [Yes/No]?" + }, + "pass": true, + "score": 1, + "reason": "The summary is clearly written, uses precise language, and covers the essential details regarding legal proceedings and their potential impact, which suggests it was written by an expert analyst.", + "tokensUsed": { + "total": 275, + "prompt": 220, + "completion": 55, + "cached": 0 + } + } + ], + "assertion": null + }, + "id": "82ea399d-2d17-4d5c-bfe7-41d7d0a12b13", + "latencyMs": 1332, + "namedScores": {}, + "prompt": { + "raw": "'Generate a 1-liner summary of the Section Legal Proceedings from an SEC filing: '", + "label": "prompt1.txt: 'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'" + }, + "promptId": "1cd4a63a09ec6bcce588ef0486fa8be9ffa6dbbb4637fdf551a3dc72ac9a6462", + "promptIdx": 0, + "provider": { + "id": "openai:gpt-4o-mini", + "label": "" + }, + "response": { + "output": "The section on Legal Proceedings outlines any significant ongoing litigation or regulatory matters that may impact the company's financial condition or operations.", + "tokenUsage": { + "total": 49, + "prompt": 26, + "completion": 23 + }, + "cached": false, + "cost": 0.0000177 + }, + "score": 0.75, + "success": false, + "testCase": { + "vars": { + "section": "Legal Proceedings", + " content": "Item 3. Legal Proceedings Digital Markets Act Investigations On March 25, 2024, the Commission announced that it had opened two formal noncompliance investigations against the Company under the DMA. The Commission’s investigations concern (1) Article 5(4) of the DMA, which relates to how developers may communicate and promote offers to end users for apps distributed through the App Store as well as how developers may conclude contracts with those end users; and (2) Article 6(3) of the DMA, which relates to default settings, uninstallation of apps, and a web browser choice screen on iOS. On June 24, 2024, the Commission announced its preliminary findings in the Article 5(4) investigation alleging that the Company’s App Store rules are in breach of the DMA and announced that it had opened a third formal investigation against the Company regarding whether the Company’s new contractual requirements for third-party app developers and app marketplaces may violate the DMA. If the Commission makes a final determination that there has been a violation, it can issue a cease and desist order and may impose fines up to 10% of the Company’s annual worldwide net sales. Although any decision by the Commission can be appealed to the General Court of the EU, the effectiveness of the Commission’s order would apply immediately while the appeal is pending, unless a stay of the order is granted. The Company believes that it complies with the DMA and has continued to make changes to its compliance plan in response to feedback and engagement with the Commission. Department of Justice Lawsuit On March 21, 2024, the U.S. Department of Justice (the “DOJ”) and a number of state and district attorneys general filed a civil antitrust lawsuit in the U.S. District Court for the District of New Jersey against the Company alleging monopolization or attempted monopolization in the markets for “performance smartphones” and “smartphones” in violation of U.S. antitrust laws. The DOJ is seeking equitable relief to redress the alleged anticompetitive behavior. In addition, various civil litigation matters have been filed in state and federal courts in the U.S. alleging similar violations of U.S. antitrust laws and seeking monetary damages and other nonmonetary relief. The Company believes it has substantial defenses and intends to vigorously defend itself. Epic Games Epic Games, Inc. (“Epic”) filed a lawsuit in the U.S. District Court for the Northern District of California (the “California District Court”) against the Company alleging violations of federal and state antitrust laws and California’s unfair competition law based upon the Company’s operation of its App Store. The California District Court found that certain provisions of the Company’s App Store Review Guidelines violate California’s unfair competition law and issued an injunction enjoining the Company from prohibiting developers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app purchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS App Store. On January 16, 2024, the Company implemented a plan to comply with the injunction and filed a statement of compliance with the California District Court. A motion by Epic disputing the Company’s compliance plan and seeking to enforce the injunction, which the Company has opposed, is pending before the California District Court. On September 30, 2024, the Company filed a motion with the California District Court to narrow or vacate the injunction. The Company believes it has substantial defenses and intends to vigorously defend itself. Other Legal Proceedings The Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the ordinary course of business. The Company settled certain matters during the fourth quarter of 2024 that did not individually or in the aggregate have a material impact on the Company’s financial condition or operating results. The outcome of litigation is inherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above management’s expectations, the Company’s financial condition and operating results for that reporting period could be materially adversely affected." + }, + "assert": [ + { + "type": "cost", + "threshold": 0.001 + }, + { + "type": "latency", + "threshold": 1000 + }, + { + "type": "python", + "value": "len(output) < 200" + }, + { + "type": "llm-rubric", + "value": "Does the summary look like it was written by an expert analyst [Yes/No]?" + } + ], + "options": {}, + "description": "Row #2", + "metadata": {} + }, + "testIdx": 1, + "vars": { + "section": "Legal Proceedings", + " content": "Item 3. Legal Proceedings Digital Markets Act Investigations On March 25, 2024, the Commission announced that it had opened two formal noncompliance investigations against the Company under the DMA. The Commission’s investigations concern (1) Article 5(4) of the DMA, which relates to how developers may communicate and promote offers to end users for apps distributed through the App Store as well as how developers may conclude contracts with those end users; and (2) Article 6(3) of the DMA, which relates to default settings, uninstallation of apps, and a web browser choice screen on iOS. On June 24, 2024, the Commission announced its preliminary findings in the Article 5(4) investigation alleging that the Company’s App Store rules are in breach of the DMA and announced that it had opened a third formal investigation against the Company regarding whether the Company’s new contractual requirements for third-party app developers and app marketplaces may violate the DMA. If the Commission makes a final determination that there has been a violation, it can issue a cease and desist order and may impose fines up to 10% of the Company’s annual worldwide net sales. Although any decision by the Commission can be appealed to the General Court of the EU, the effectiveness of the Commission’s order would apply immediately while the appeal is pending, unless a stay of the order is granted. The Company believes that it complies with the DMA and has continued to make changes to its compliance plan in response to feedback and engagement with the Commission. Department of Justice Lawsuit On March 21, 2024, the U.S. Department of Justice (the “DOJ”) and a number of state and district attorneys general filed a civil antitrust lawsuit in the U.S. District Court for the District of New Jersey against the Company alleging monopolization or attempted monopolization in the markets for “performance smartphones” and “smartphones” in violation of U.S. antitrust laws. The DOJ is seeking equitable relief to redress the alleged anticompetitive behavior. In addition, various civil litigation matters have been filed in state and federal courts in the U.S. alleging similar violations of U.S. antitrust laws and seeking monetary damages and other nonmonetary relief. The Company believes it has substantial defenses and intends to vigorously defend itself. Epic Games Epic Games, Inc. (“Epic”) filed a lawsuit in the U.S. District Court for the Northern District of California (the “California District Court”) against the Company alleging violations of federal and state antitrust laws and California’s unfair competition law based upon the Company’s operation of its App Store. The California District Court found that certain provisions of the Company’s App Store Review Guidelines violate California’s unfair competition law and issued an injunction enjoining the Company from prohibiting developers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app purchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS App Store. On January 16, 2024, the Company implemented a plan to comply with the injunction and filed a statement of compliance with the California District Court. A motion by Epic disputing the Company’s compliance plan and seeking to enforce the injunction, which the Company has opposed, is pending before the California District Court. On September 30, 2024, the Company filed a motion with the California District Court to narrow or vacate the injunction. The Company believes it has substantial defenses and intends to vigorously defend itself. Other Legal Proceedings The Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the ordinary course of business. The Company settled certain matters during the fourth quarter of 2024 that did not individually or in the aggregate have a material impact on the Company’s financial condition or operating results. The outcome of litigation is inherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above management’s expectations, the Company’s financial condition and operating results for that reporting period could be materially adversely affected." + }, + "metadata": {} + }, + { + "cost": 0.0027, + "error": "Latency 2251ms is greater than threshold 1000ms", + "gradingResult": { + "pass": false, + "score": 0.5, + "reason": "Latency 2251ms is greater than threshold 1000ms", + "namedScores": {}, + "tokensUsed": { + "total": 267, + "prompt": 228, + "completion": 39, + "cached": 0 + }, + "componentResults": [ + { + "pass": false, + "score": 0, + "reason": "Cost 0.0027 is greater than threshold 0.001", + "assertion": { + "type": "cost", + "threshold": 0.001 + } + }, + { + "pass": false, + "score": 0, + "reason": "Latency 2251ms is greater than threshold 1000ms", + "assertion": { + "type": "latency", + "threshold": 1000 + } + }, + { + "pass": true, + "score": 1, + "reason": "Assertion passed", + "assertion": { + "type": "python", + "value": "len(output) < 200" + } + }, + { + "assertion": { + "type": "llm-rubric", + "value": "Does the summary look like it was written by an expert analyst [Yes/No]?" + }, + "pass": true, + "score": 1, + "reason": "The summary contains professional language and details specific factors that are typically covered by expert analysts.", + "tokensUsed": { + "total": 267, + "prompt": 228, + "completion": 39, + "cached": 0 + } + } + ], + "assertion": null + }, + "id": "68b0bb93-d8af-4b19-b2ad-b47a739d7cd4", + "latencyMs": 2251, + "namedScores": {}, + "prompt": { + "raw": "'Generate a 1-liner summary of the Section Risk Factors from an SEC filing: '", + "label": "prompt1.txt: 'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'" + }, + "promptId": "1cd4a63a09ec6bcce588ef0486fa8be9ffa6dbbb4637fdf551a3dc72ac9a6462", + "promptIdx": 1, + "provider": { + "id": "openai:gpt-4", + "label": "" + }, + "response": { + "output": "The Risk Factors section details potential risks, uncertainties, and challenges that could materially affect the company's business, financial condition, results of operations, and future growth.", + "tokenUsage": { + "total": 58, + "prompt": 26, + "completion": 32 + }, + "cached": false, + "cost": 0.0027 + }, + "score": 0.5, + "success": false, + "testCase": { + "vars": { + "section": "Risk Factors", + " content": "Item 1A. Macroeconomic and Industry Risks The Company’s operations and performance depend significantly on global and regional economic conditions and adverse economic conditions can materially adversely affect the Company’s business, results of operations and financial condition. The Company has international operations with sales outside the U.S. representing a majority of the Company’s total net sales. In addition, the Company’s global supply chain is large and complex and a majority of the Company’s supplier facilities, including manufacturing and assembly sites, are located outside the U.S. As a result, the Company’s operations and performance depend significantly on global and regional economic conditions. Adverse macroeconomic conditions, including slow growth or recession, high unemployment, inflation, tighter credit, higher interest rates, and currency fluctuations, can adversely impact consumer confidence and spending and materially adversely affect demand for the Company’s products and services. In addition, consumer confidence and spending can be materially adversely affected in response to changes in fiscal and monetary policy, financial market volatility, declines in income or asset values, and other economic factors. In addition to an adverse impact on demand for the Company’s products and services, uncertainty about, or a decline in, global or regional economic conditions can have a significant impact on the Company’s suppliers, contract manufacturers, logistics providers, distributors, cellular network carriers and other channel partners, and developers. Potential outcomes include financial instability; inability to obtain credit to finance business operations; and insolvency. Adverse economic conditions can also lead to increased credit and collectibility risk on the Company’s trade receivables; the failure of derivative counterparties and other financial institutions; limitations on the Company’s ability to issue new debt; reduced liquidity; and declines in the fair values of the Company’s financial instruments. These and other impacts can materially adversely affect the Company’s business, results of operations, financial condition and stock price." + }, + "assert": [ + { + "type": "cost", + "threshold": 0.001 + }, + { + "type": "latency", + "threshold": 1000 + }, + { + "type": "python", + "value": "len(output) < 200" + }, + { + "type": "llm-rubric", + "value": "Does the summary look like it was written by an expert analyst [Yes/No]?" + } + ], + "options": {}, + "description": "Row #1", + "metadata": {} + }, + "testIdx": 0, + "vars": { + "section": "Risk Factors", + " content": "Item 1A. Macroeconomic and Industry Risks The Company’s operations and performance depend significantly on global and regional economic conditions and adverse economic conditions can materially adversely affect the Company’s business, results of operations and financial condition. The Company has international operations with sales outside the U.S. representing a majority of the Company’s total net sales. In addition, the Company’s global supply chain is large and complex and a majority of the Company’s supplier facilities, including manufacturing and assembly sites, are located outside the U.S. As a result, the Company’s operations and performance depend significantly on global and regional economic conditions. Adverse macroeconomic conditions, including slow growth or recession, high unemployment, inflation, tighter credit, higher interest rates, and currency fluctuations, can adversely impact consumer confidence and spending and materially adversely affect demand for the Company’s products and services. In addition, consumer confidence and spending can be materially adversely affected in response to changes in fiscal and monetary policy, financial market volatility, declines in income or asset values, and other economic factors. In addition to an adverse impact on demand for the Company’s products and services, uncertainty about, or a decline in, global or regional economic conditions can have a significant impact on the Company’s suppliers, contract manufacturers, logistics providers, distributors, cellular network carriers and other channel partners, and developers. Potential outcomes include financial instability; inability to obtain credit to finance business operations; and insolvency. Adverse economic conditions can also lead to increased credit and collectibility risk on the Company’s trade receivables; the failure of derivative counterparties and other financial institutions; limitations on the Company’s ability to issue new debt; reduced liquidity; and declines in the fair values of the Company’s financial instruments. These and other impacts can materially adversely affect the Company’s business, results of operations, financial condition and stock price." + }, + "metadata": {} + }, + { + "cost": 0.000049, + "gradingResult": { + "pass": true, + "score": 1, + "reason": "All assertions passed", + "namedScores": {}, + "tokensUsed": { + "total": 267, + "prompt": 221, + "completion": 46, + "cached": 0 + }, + "componentResults": [ + { + "pass": true, + "score": 1, + "reason": "Assertion passed", + "assertion": { + "type": "cost", + "threshold": 0.001 + } + }, + { + "pass": true, + "score": 1, + "reason": "Assertion passed", + "assertion": { + "type": "latency", + "threshold": 1000 + } + }, + { + "pass": true, + "score": 1, + "reason": "Assertion passed", + "assertion": { + "type": "python", + "value": "len(output) < 200" + } + }, + { + "assertion": { + "type": "llm-rubric", + "value": "Does the summary look like it was written by an expert analyst [Yes/No]?" + }, + "pass": true, + "score": 1, + "reason": "the summary provides detailed and specific information about legal actions, which indicates it was written by someone with expertise in the field", + "tokensUsed": { + "total": 267, + "prompt": 221, + "completion": 46, + "cached": 0 + } + } + ], + "assertion": null + }, + "id": "8385a18e-cef6-42d4-a060-25149244a05d", + "latencyMs": 653, + "namedScores": {}, + "prompt": { + "raw": "'Generate a 1-liner summary of the Section Legal Proceedings from an SEC filing: '", + "label": "prompt1.txt: 'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'" + }, + "promptId": "1cd4a63a09ec6bcce588ef0486fa8be9ffa6dbbb4637fdf551a3dc72ac9a6462", + "promptIdx": 2, + "provider": { + "id": "openai:gpt-3.5-turbo", + "label": "" + }, + "response": { + "output": "This section provides information on any ongoing or potential legal actions involving the company, including lawsuits, investigations, or regulatory inquiries.", + "tokenUsage": { + "total": 50, + "prompt": 26, + "completion": 24 + }, + "cached": false, + "cost": 0.000049 + }, + "score": 1, + "success": true, + "testCase": { + "vars": { + "section": "Legal Proceedings", + " content": "Item 3. Legal Proceedings Digital Markets Act Investigations On March 25, 2024, the Commission announced that it had opened two formal noncompliance investigations against the Company under the DMA. The Commission’s investigations concern (1) Article 5(4) of the DMA, which relates to how developers may communicate and promote offers to end users for apps distributed through the App Store as well as how developers may conclude contracts with those end users; and (2) Article 6(3) of the DMA, which relates to default settings, uninstallation of apps, and a web browser choice screen on iOS. On June 24, 2024, the Commission announced its preliminary findings in the Article 5(4) investigation alleging that the Company’s App Store rules are in breach of the DMA and announced that it had opened a third formal investigation against the Company regarding whether the Company’s new contractual requirements for third-party app developers and app marketplaces may violate the DMA. If the Commission makes a final determination that there has been a violation, it can issue a cease and desist order and may impose fines up to 10% of the Company’s annual worldwide net sales. Although any decision by the Commission can be appealed to the General Court of the EU, the effectiveness of the Commission’s order would apply immediately while the appeal is pending, unless a stay of the order is granted. The Company believes that it complies with the DMA and has continued to make changes to its compliance plan in response to feedback and engagement with the Commission. Department of Justice Lawsuit On March 21, 2024, the U.S. Department of Justice (the “DOJ”) and a number of state and district attorneys general filed a civil antitrust lawsuit in the U.S. District Court for the District of New Jersey against the Company alleging monopolization or attempted monopolization in the markets for “performance smartphones” and “smartphones” in violation of U.S. antitrust laws. The DOJ is seeking equitable relief to redress the alleged anticompetitive behavior. In addition, various civil litigation matters have been filed in state and federal courts in the U.S. alleging similar violations of U.S. antitrust laws and seeking monetary damages and other nonmonetary relief. The Company believes it has substantial defenses and intends to vigorously defend itself. Epic Games Epic Games, Inc. (“Epic”) filed a lawsuit in the U.S. District Court for the Northern District of California (the “California District Court”) against the Company alleging violations of federal and state antitrust laws and California’s unfair competition law based upon the Company’s operation of its App Store. The California District Court found that certain provisions of the Company’s App Store Review Guidelines violate California’s unfair competition law and issued an injunction enjoining the Company from prohibiting developers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app purchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS App Store. On January 16, 2024, the Company implemented a plan to comply with the injunction and filed a statement of compliance with the California District Court. A motion by Epic disputing the Company’s compliance plan and seeking to enforce the injunction, which the Company has opposed, is pending before the California District Court. On September 30, 2024, the Company filed a motion with the California District Court to narrow or vacate the injunction. The Company believes it has substantial defenses and intends to vigorously defend itself. Other Legal Proceedings The Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the ordinary course of business. The Company settled certain matters during the fourth quarter of 2024 that did not individually or in the aggregate have a material impact on the Company’s financial condition or operating results. The outcome of litigation is inherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above management’s expectations, the Company’s financial condition and operating results for that reporting period could be materially adversely affected." + }, + "assert": [ + { + "type": "cost", + "threshold": 0.001 + }, + { + "type": "latency", + "threshold": 1000 + }, + { + "type": "python", + "value": "len(output) < 200" + }, + { + "type": "llm-rubric", + "value": "Does the summary look like it was written by an expert analyst [Yes/No]?" + } + ], + "options": {}, + "description": "Row #2", + "metadata": {} + }, + "testIdx": 1, + "vars": { + "section": "Legal Proceedings", + " content": "Item 3. Legal Proceedings Digital Markets Act Investigations On March 25, 2024, the Commission announced that it had opened two formal noncompliance investigations against the Company under the DMA. The Commission’s investigations concern (1) Article 5(4) of the DMA, which relates to how developers may communicate and promote offers to end users for apps distributed through the App Store as well as how developers may conclude contracts with those end users; and (2) Article 6(3) of the DMA, which relates to default settings, uninstallation of apps, and a web browser choice screen on iOS. On June 24, 2024, the Commission announced its preliminary findings in the Article 5(4) investigation alleging that the Company’s App Store rules are in breach of the DMA and announced that it had opened a third formal investigation against the Company regarding whether the Company’s new contractual requirements for third-party app developers and app marketplaces may violate the DMA. If the Commission makes a final determination that there has been a violation, it can issue a cease and desist order and may impose fines up to 10% of the Company’s annual worldwide net sales. Although any decision by the Commission can be appealed to the General Court of the EU, the effectiveness of the Commission’s order would apply immediately while the appeal is pending, unless a stay of the order is granted. The Company believes that it complies with the DMA and has continued to make changes to its compliance plan in response to feedback and engagement with the Commission. Department of Justice Lawsuit On March 21, 2024, the U.S. Department of Justice (the “DOJ”) and a number of state and district attorneys general filed a civil antitrust lawsuit in the U.S. District Court for the District of New Jersey against the Company alleging monopolization or attempted monopolization in the markets for “performance smartphones” and “smartphones” in violation of U.S. antitrust laws. The DOJ is seeking equitable relief to redress the alleged anticompetitive behavior. In addition, various civil litigation matters have been filed in state and federal courts in the U.S. alleging similar violations of U.S. antitrust laws and seeking monetary damages and other nonmonetary relief. The Company believes it has substantial defenses and intends to vigorously defend itself. Epic Games Epic Games, Inc. (“Epic”) filed a lawsuit in the U.S. District Court for the Northern District of California (the “California District Court”) against the Company alleging violations of federal and state antitrust laws and California’s unfair competition law based upon the Company’s operation of its App Store. The California District Court found that certain provisions of the Company’s App Store Review Guidelines violate California’s unfair competition law and issued an injunction enjoining the Company from prohibiting developers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app purchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS App Store. On January 16, 2024, the Company implemented a plan to comply with the injunction and filed a statement of compliance with the California District Court. A motion by Epic disputing the Company’s compliance plan and seeking to enforce the injunction, which the Company has opposed, is pending before the California District Court. On September 30, 2024, the Company filed a motion with the California District Court to narrow or vacate the injunction. The Company believes it has substantial defenses and intends to vigorously defend itself. Other Legal Proceedings The Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the ordinary course of business. The Company settled certain matters during the fourth quarter of 2024 that did not individually or in the aggregate have a material impact on the Company’s financial condition or operating results. The outcome of litigation is inherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above management’s expectations, the Company’s financial condition and operating results for that reporting period could be materially adversely affected." + }, + "metadata": {} + }, + { + "cost": 0.0019199999999999998, + "error": "Latency 1522ms is greater than threshold 1000ms", + "gradingResult": { + "pass": false, + "score": 0.5, + "reason": "Latency 1522ms is greater than threshold 1000ms", + "namedScores": {}, + "tokensUsed": { + "total": 257, + "prompt": 216, + "completion": 41, + "cached": 0 + }, + "componentResults": [ + { + "pass": false, + "score": 0, + "reason": "Cost 0.0019 is greater than threshold 0.001", + "assertion": { + "type": "cost", + "threshold": 0.001 + } + }, + { + "pass": false, + "score": 0, + "reason": "Latency 1522ms is greater than threshold 1000ms", + "assertion": { + "type": "latency", + "threshold": 1000 + } + }, + { + "pass": true, + "score": 1, + "reason": "Assertion passed", + "assertion": { + "type": "python", + "value": "len(output) < 200" + } + }, + { + "assertion": { + "type": "llm-rubric", + "value": "Does the summary look like it was written by an expert analyst [Yes/No]?" + }, + "pass": true, + "score": 1, + "reason": "The summary provides a clear and professional overview of the Legal Proceedings section, characteristic of expert analysis.", + "tokensUsed": { + "total": 257, + "prompt": 216, + "completion": 41, + "cached": 0 + } + } + ], + "assertion": null + }, + "id": "30d8a10b-ef8b-464d-a4bd-cc0f9b6d527e", + "latencyMs": 1522, + "namedScores": {}, + "prompt": { + "raw": "'Generate a 1-liner summary of the Section Legal Proceedings from an SEC filing: '", + "label": "prompt1.txt: 'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'" + }, + "promptId": "1cd4a63a09ec6bcce588ef0486fa8be9ffa6dbbb4637fdf551a3dc72ac9a6462", + "promptIdx": 1, + "provider": { + "id": "openai:gpt-4", + "label": "" + }, + "response": { + "output": "The Legal Proceedings section details ongoing or potential legal issues, lawsuits, or disputes involving the company.", + "tokenUsage": { + "total": 45, + "prompt": 26, + "completion": 19 + }, + "cached": false, + "cost": 0.0019199999999999998 + }, + "score": 0.5, + "success": false, + "testCase": { + "vars": { + "section": "Legal Proceedings", + " content": "Item 3. Legal Proceedings Digital Markets Act Investigations On March 25, 2024, the Commission announced that it had opened two formal noncompliance investigations against the Company under the DMA. The Commission’s investigations concern (1) Article 5(4) of the DMA, which relates to how developers may communicate and promote offers to end users for apps distributed through the App Store as well as how developers may conclude contracts with those end users; and (2) Article 6(3) of the DMA, which relates to default settings, uninstallation of apps, and a web browser choice screen on iOS. On June 24, 2024, the Commission announced its preliminary findings in the Article 5(4) investigation alleging that the Company’s App Store rules are in breach of the DMA and announced that it had opened a third formal investigation against the Company regarding whether the Company’s new contractual requirements for third-party app developers and app marketplaces may violate the DMA. If the Commission makes a final determination that there has been a violation, it can issue a cease and desist order and may impose fines up to 10% of the Company’s annual worldwide net sales. Although any decision by the Commission can be appealed to the General Court of the EU, the effectiveness of the Commission’s order would apply immediately while the appeal is pending, unless a stay of the order is granted. The Company believes that it complies with the DMA and has continued to make changes to its compliance plan in response to feedback and engagement with the Commission. Department of Justice Lawsuit On March 21, 2024, the U.S. Department of Justice (the “DOJ”) and a number of state and district attorneys general filed a civil antitrust lawsuit in the U.S. District Court for the District of New Jersey against the Company alleging monopolization or attempted monopolization in the markets for “performance smartphones” and “smartphones” in violation of U.S. antitrust laws. The DOJ is seeking equitable relief to redress the alleged anticompetitive behavior. In addition, various civil litigation matters have been filed in state and federal courts in the U.S. alleging similar violations of U.S. antitrust laws and seeking monetary damages and other nonmonetary relief. The Company believes it has substantial defenses and intends to vigorously defend itself. Epic Games Epic Games, Inc. (“Epic”) filed a lawsuit in the U.S. District Court for the Northern District of California (the “California District Court”) against the Company alleging violations of federal and state antitrust laws and California’s unfair competition law based upon the Company’s operation of its App Store. The California District Court found that certain provisions of the Company’s App Store Review Guidelines violate California’s unfair competition law and issued an injunction enjoining the Company from prohibiting developers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app purchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS App Store. On January 16, 2024, the Company implemented a plan to comply with the injunction and filed a statement of compliance with the California District Court. A motion by Epic disputing the Company’s compliance plan and seeking to enforce the injunction, which the Company has opposed, is pending before the California District Court. On September 30, 2024, the Company filed a motion with the California District Court to narrow or vacate the injunction. The Company believes it has substantial defenses and intends to vigorously defend itself. Other Legal Proceedings The Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the ordinary course of business. The Company settled certain matters during the fourth quarter of 2024 that did not individually or in the aggregate have a material impact on the Company’s financial condition or operating results. The outcome of litigation is inherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above management’s expectations, the Company’s financial condition and operating results for that reporting period could be materially adversely affected." + }, + "assert": [ + { + "type": "cost", + "threshold": 0.001 + }, + { + "type": "latency", + "threshold": 1000 + }, + { + "type": "python", + "value": "len(output) < 200" + }, + { + "type": "llm-rubric", + "value": "Does the summary look like it was written by an expert analyst [Yes/No]?" + } + ], + "options": {}, + "description": "Row #2", + "metadata": {} + }, + "testIdx": 1, + "vars": { + "section": "Legal Proceedings", + " content": "Item 3. Legal Proceedings Digital Markets Act Investigations On March 25, 2024, the Commission announced that it had opened two formal noncompliance investigations against the Company under the DMA. The Commission’s investigations concern (1) Article 5(4) of the DMA, which relates to how developers may communicate and promote offers to end users for apps distributed through the App Store as well as how developers may conclude contracts with those end users; and (2) Article 6(3) of the DMA, which relates to default settings, uninstallation of apps, and a web browser choice screen on iOS. On June 24, 2024, the Commission announced its preliminary findings in the Article 5(4) investigation alleging that the Company’s App Store rules are in breach of the DMA and announced that it had opened a third formal investigation against the Company regarding whether the Company’s new contractual requirements for third-party app developers and app marketplaces may violate the DMA. If the Commission makes a final determination that there has been a violation, it can issue a cease and desist order and may impose fines up to 10% of the Company’s annual worldwide net sales. Although any decision by the Commission can be appealed to the General Court of the EU, the effectiveness of the Commission’s order would apply immediately while the appeal is pending, unless a stay of the order is granted. The Company believes that it complies with the DMA and has continued to make changes to its compliance plan in response to feedback and engagement with the Commission. Department of Justice Lawsuit On March 21, 2024, the U.S. Department of Justice (the “DOJ”) and a number of state and district attorneys general filed a civil antitrust lawsuit in the U.S. District Court for the District of New Jersey against the Company alleging monopolization or attempted monopolization in the markets for “performance smartphones” and “smartphones” in violation of U.S. antitrust laws. The DOJ is seeking equitable relief to redress the alleged anticompetitive behavior. In addition, various civil litigation matters have been filed in state and federal courts in the U.S. alleging similar violations of U.S. antitrust laws and seeking monetary damages and other nonmonetary relief. The Company believes it has substantial defenses and intends to vigorously defend itself. Epic Games Epic Games, Inc. (“Epic”) filed a lawsuit in the U.S. District Court for the Northern District of California (the “California District Court”) against the Company alleging violations of federal and state antitrust laws and California’s unfair competition law based upon the Company’s operation of its App Store. The California District Court found that certain provisions of the Company’s App Store Review Guidelines violate California’s unfair competition law and issued an injunction enjoining the Company from prohibiting developers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app purchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS App Store. On January 16, 2024, the Company implemented a plan to comply with the injunction and filed a statement of compliance with the California District Court. A motion by Epic disputing the Company’s compliance plan and seeking to enforce the injunction, which the Company has opposed, is pending before the California District Court. On September 30, 2024, the Company filed a motion with the California District Court to narrow or vacate the injunction. The Company believes it has substantial defenses and intends to vigorously defend itself. Other Legal Proceedings The Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the ordinary course of business. The Company settled certain matters during the fourth quarter of 2024 that did not individually or in the aggregate have a material impact on the Company’s financial condition or operating results. The outcome of litigation is inherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above management’s expectations, the Company’s financial condition and operating results for that reporting period could be materially adversely affected." + }, + "metadata": {} + } + ], + "stats": { + "successes": 1, + "failures": 5, + "tokenUsage": { + "cached": 0, + "completion": 139, + "prompt": 156, + "total": 295, + "numRequests": 6 + } + } + }, + "config": { + "description": "Best model eval", + "prompts": [ + "file://prompt1.txt" + ], + "providers": [ + "openai:gpt-4o-mini", + "openai:gpt-4", + "openai:gpt-3.5-turbo" + ], + "tests": "file://tests.csv", + "sharing": true, + "defaultTest": { + "assert": [ + { + "type": "cost", + "threshold": 0.001 + }, + { + "type": "latency", + "threshold": 1000 + }, + { + "type": "python", + "value": "len(output) < 200" + }, + { + "type": "llm-rubric", + "value": "Does the summary look like it was written by an expert analyst [Yes/No]?" + } + ] + }, + "outputPath": [ + "eval.json" + ], + "extensions": [] + }, + "shareableUrl": null +} \ No newline at end of file diff --git a/tamingllms/notebooks/promptfoo/model_comparison/prompt1.txt b/tamingllms/notebooks/promptfoo/model_comparison/prompt1.txt new file mode 100644 index 0000000..7b560db --- /dev/null +++ b/tamingllms/notebooks/promptfoo/model_comparison/prompt1.txt @@ -0,0 +1 @@ +'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}' diff --git a/tamingllms/notebooks/promptfoo/model_comparison/promptfooconfig.yaml b/tamingllms/notebooks/promptfoo/model_comparison/promptfooconfig.yaml new file mode 100644 index 0000000..7bf17d8 --- /dev/null +++ b/tamingllms/notebooks/promptfoo/model_comparison/promptfooconfig.yaml @@ -0,0 +1,35 @@ +# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json + +# Learn more about building a configuration: https://promptfoo.dev/docs/configuration/guide + +description: "Best model eval" + +prompts: + - file://prompt1.txt + + +providers: + - openai:gpt-4o-mini + - openai:gpt-4 + - openai:gpt-3.5-turbo + +# These test properties are applied to every test +defaultTest: + assert: + # Inference should always cost less than this (USD) + - type: cost + threshold: 0.001 + + # Inference should always be faster than this (milliseconds) + - type: latency + threshold: 1000 + + # Length should be less than 200 characters + - type: python + value: len(output) < 200 + + # Verify that the output looks like it was written by an expert analyst + - type: llm-rubric + value: Does the summary look like it was written by an expert analyst [Yes/No]? + +tests: file://tests.csv diff --git a/tamingllms/notebooks/promptfoo/model_comparison/tests.csv b/tamingllms/notebooks/promptfoo/model_comparison/tests.csv new file mode 100644 index 0000000..4d94f54 --- /dev/null +++ b/tamingllms/notebooks/promptfoo/model_comparison/tests.csv @@ -0,0 +1,3 @@ +section, content +Risk Factors,"Item 1A. Macroeconomic and Industry Risks The Company’s operations and performance depend significantly on global and regional economic conditions and adverse economic conditions can materially adversely affect the Company’s business, results of operations and financial condition. The Company has international operations with sales outside the U.S. representing a majority of the Company’s total net sales. In addition, the Company’s global supply chain is large and complex and a majority of the Company’s supplier facilities, including manufacturing and assembly sites, are located outside the U.S. As a result, the Company’s operations and performance depend significantly on global and regional economic conditions. Adverse macroeconomic conditions, including slow growth or recession, high unemployment, inflation, tighter credit, higher interest rates, and currency fluctuations, can adversely impact consumer confidence and spending and materially adversely affect demand for the Company’s products and services. In addition, consumer confidence and spending can be materially adversely affected in response to changes in fiscal and monetary policy, financial market volatility, declines in income or asset values, and other economic factors. In addition to an adverse impact on demand for the Company’s products and services, uncertainty about, or a decline in, global or regional economic conditions can have a significant impact on the Company’s suppliers, contract manufacturers, logistics providers, distributors, cellular network carriers and other channel partners, and developers. Potential outcomes include financial instability; inability to obtain credit to finance business operations; and insolvency. Adverse economic conditions can also lead to increased credit and collectibility risk on the Company’s trade receivables; the failure of derivative counterparties and other financial institutions; limitations on the Company’s ability to issue new debt; reduced liquidity; and declines in the fair values of the Company’s financial instruments. These and other impacts can materially adversely affect the Company’s business, results of operations, financial condition and stock price." +Legal Proceedings,"Item 3. Legal Proceedings Digital Markets Act Investigations On March 25, 2024, the Commission announced that it had opened two formal noncompliance investigations against the Company under the DMA. The Commission’s investigations concern (1) Article 5(4) of the DMA, which relates to how developers may communicate and promote offers to end users for apps distributed through the App Store as well as how developers may conclude contracts with those end users; and (2) Article 6(3) of the DMA, which relates to default settings, uninstallation of apps, and a web browser choice screen on iOS. On June 24, 2024, the Commission announced its preliminary findings in the Article 5(4) investigation alleging that the Company’s App Store rules are in breach of the DMA and announced that it had opened a third formal investigation against the Company regarding whether the Company’s new contractual requirements for third-party app developers and app marketplaces may violate the DMA. If the Commission makes a final determination that there has been a violation, it can issue a cease and desist order and may impose fines up to 10% of the Company’s annual worldwide net sales. Although any decision by the Commission can be appealed to the General Court of the EU, the effectiveness of the Commission’s order would apply immediately while the appeal is pending, unless a stay of the order is granted. The Company believes that it complies with the DMA and has continued to make changes to its compliance plan in response to feedback and engagement with the Commission. Department of Justice Lawsuit On March 21, 2024, the U.S. Department of Justice (the “DOJ”) and a number of state and district attorneys general filed a civil antitrust lawsuit in the U.S. District Court for the District of New Jersey against the Company alleging monopolization or attempted monopolization in the markets for “performance smartphones” and “smartphones” in violation of U.S. antitrust laws. The DOJ is seeking equitable relief to redress the alleged anticompetitive behavior. In addition, various civil litigation matters have been filed in state and federal courts in the U.S. alleging similar violations of U.S. antitrust laws and seeking monetary damages and other nonmonetary relief. The Company believes it has substantial defenses and intends to vigorously defend itself. Epic Games Epic Games, Inc. (“Epic”) filed a lawsuit in the U.S. District Court for the Northern District of California (the “California District Court”) against the Company alleging violations of federal and state antitrust laws and California’s unfair competition law based upon the Company’s operation of its App Store. The California District Court found that certain provisions of the Company’s App Store Review Guidelines violate California’s unfair competition law and issued an injunction enjoining the Company from prohibiting developers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app purchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS App Store. On January 16, 2024, the Company implemented a plan to comply with the injunction and filed a statement of compliance with the California District Court. A motion by Epic disputing the Company’s compliance plan and seeking to enforce the injunction, which the Company has opposed, is pending before the California District Court. On September 30, 2024, the Company filed a motion with the California District Court to narrow or vacate the injunction. The Company believes it has substantial defenses and intends to vigorously defend itself. Other Legal Proceedings The Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the ordinary course of business. The Company settled certain matters during the fourth quarter of 2024 that did not individually or in the aggregate have a material impact on the Company’s financial condition or operating results. The outcome of litigation is inherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above management’s expectations, the Company’s financial condition and operating results for that reporting period could be materially adversely affected." \ No newline at end of file diff --git a/tamingllms/notebooks/promptfoo/prompt_comparison/eval.json b/tamingllms/notebooks/promptfoo/prompt_comparison/eval.json new file mode 100644 index 0000000..25e99bf --- /dev/null +++ b/tamingllms/notebooks/promptfoo/prompt_comparison/eval.json @@ -0,0 +1,596 @@ +{ + "evalId": "eval-bvf-2024-12-10T16:33:47", + "results": { + "version": 3, + "timestamp": "2024-12-10T16:33:47.191Z", + "prompts": [ + { + "raw": "'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'", + "label": "prompt1.txt: 'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'", + "id": "1cd4a63a09ec6bcce588ef0486fa8be9ffa6dbbb4637fdf551a3dc72ac9a6462", + "provider": "openai:gpt-3.5-turbo", + "metrics": { + "score": 0.2, + "testPassCount": 0, + "testFailCount": 2, + "assertPassCount": 0, + "assertFailCount": 2, + "totalLatencyMs": 1621, + "tokenUsage": { + "total": 84, + "prompt": 52, + "completion": 32, + "cached": 0, + "numRequests": 2 + }, + "namedScores": {}, + "namedScoresCount": {}, + "cost": 0.00007400000000000001 + } + }, + { + "raw": "'ROLE: You are a financial analyst. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'", + "label": "prompt2.txt: 'ROLE: You are a financial analyst. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'", + "id": "67b9d227a9b9ffc025b0f3503a0461da85fe5184f253af38cb2315159ffe9b02", + "provider": "openai:gpt-3.5-turbo", + "metrics": { + "score": 1, + "testPassCount": 2, + "testFailCount": 0, + "assertPassCount": 2, + "assertFailCount": 0, + "totalLatencyMs": 1506, + "tokenUsage": { + "total": 121, + "prompt": 72, + "completion": 49, + "cached": 0, + "numRequests": 2 + }, + "namedScores": {}, + "namedScoresCount": {}, + "cost": 0.0001095 + } + }, + { + "raw": "'ROLE: You are a financial analyst. REQUIREMENTS: BE DETAILED. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'", + "label": "prompt3.txt: 'ROLE: You are a financial analyst. REQUIREMENTS: BE DETAILED. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'", + "id": "4e36379853105286bc2b2bba84330e0ea2402b1c965acc338605ac84d2c35dee", + "provider": "openai:gpt-3.5-turbo", + "metrics": { + "score": 1.5, + "testPassCount": 2, + "testFailCount": 0, + "assertPassCount": 2, + "assertFailCount": 0, + "totalLatencyMs": 1610, + "tokenUsage": { + "total": 159, + "prompt": 88, + "completion": 71, + "cached": 0, + "numRequests": 2 + }, + "namedScores": {}, + "namedScoresCount": {}, + "cost": 0.0001505 + } + } + ], + "results": [ + { + "cost": 0.0000615, + "gradingResult": { + "pass": true, + "score": 0.5, + "reason": "All assertions passed", + "namedScores": {}, + "tokensUsed": { + "total": 307, + "prompt": 267, + "completion": 40, + "cached": 0 + }, + "componentResults": [ + { + "assertion": { + "type": "llm-rubric", + "value": "Evaluate the output based on how detailed it is. Grade it on a scale of 0.0 to 1.0, where:\nScore of 0.1: Not much detail.\nScore of 0.5: Some detail.\nScore of 1.0: Very detailed.\n" + }, + "pass": true, + "score": 0.5, + "reason": "the output provides an overview of the Risk Factors section without describing specific risks or their implications", + "tokensUsed": { + "total": 307, + "prompt": 267, + "completion": 40, + "cached": 0 + } + } + ], + "assertion": null + }, + "id": "9985421c-09b8-47a9-8732-8cc8f4e1fcea", + "latencyMs": 821, + "namedScores": {}, + "prompt": { + "raw": "'ROLE: You are a financial analyst. TASK: Generate a 1-liner summary of the Section Risk Factors from an SEC filing: '", + "label": "prompt2.txt: 'ROLE: You are a financial analyst. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'" + }, + "promptId": "67b9d227a9b9ffc025b0f3503a0461da85fe5184f253af38cb2315159ffe9b02", + "promptIdx": 1, + "provider": { + "id": "openai:gpt-3.5-turbo", + "label": "" + }, + "response": { + "output": "The risks outlined in the Risk Factors section of the SEC filing highlight potential challenges and uncertainties that may impact the company's financial performance and overall success.", + "tokenUsage": { + "total": 65, + "prompt": 36, + "completion": 29 + }, + "cached": false, + "cost": 0.0000615 + }, + "score": 0.5, + "success": true, + "testCase": { + "vars": { + "section": "Risk Factors", + " content": "Item 1A. Macroeconomic and Industry Risks The Company’s operations and performance depend significantly on global and regional economic conditions and adverse economic conditions can materially adversely affect the Company’s business, results of operations and financial condition. The Company has international operations with sales outside the U.S. representing a majority of the Company’s total net sales. In addition, the Company’s global supply chain is large and complex and a majority of the Company’s supplier facilities, including manufacturing and assembly sites, are located outside the U.S. As a result, the Company’s operations and performance depend significantly on global and regional economic conditions. Adverse macroeconomic conditions, including slow growth or recession, high unemployment, inflation, tighter credit, higher interest rates, and currency fluctuations, can adversely impact consumer confidence and spending and materially adversely affect demand for the Company’s products and services. In addition, consumer confidence and spending can be materially adversely affected in response to changes in fiscal and monetary policy, financial market volatility, declines in income or asset values, and other economic factors. In addition to an adverse impact on demand for the Company’s products and services, uncertainty about, or a decline in, global or regional economic conditions can have a significant impact on the Company’s suppliers, contract manufacturers, logistics providers, distributors, cellular network carriers and other channel partners, and developers. Potential outcomes include financial instability; inability to obtain credit to finance business operations; and insolvency. Adverse economic conditions can also lead to increased credit and collectibility risk on the Company’s trade receivables; the failure of derivative counterparties and other financial institutions; limitations on the Company’s ability to issue new debt; reduced liquidity; and declines in the fair values of the Company’s financial instruments. These and other impacts can materially adversely affect the Company’s business, results of operations, financial condition and stock price." + }, + "assert": [ + { + "type": "llm-rubric", + "value": "Evaluate the output based on how detailed it is. Grade it on a scale of 0.0 to 1.0, where:\nScore of 0.1: Not much detail.\nScore of 0.5: Some detail.\nScore of 1.0: Very detailed.\n" + } + ], + "options": {}, + "description": "Row #1", + "metadata": {} + }, + "testIdx": 0, + "vars": { + "section": "Risk Factors", + " content": "Item 1A. Macroeconomic and Industry Risks The Company’s operations and performance depend significantly on global and regional economic conditions and adverse economic conditions can materially adversely affect the Company’s business, results of operations and financial condition. The Company has international operations with sales outside the U.S. representing a majority of the Company’s total net sales. In addition, the Company’s global supply chain is large and complex and a majority of the Company’s supplier facilities, including manufacturing and assembly sites, are located outside the U.S. As a result, the Company’s operations and performance depend significantly on global and regional economic conditions. Adverse macroeconomic conditions, including slow growth or recession, high unemployment, inflation, tighter credit, higher interest rates, and currency fluctuations, can adversely impact consumer confidence and spending and materially adversely affect demand for the Company’s products and services. In addition, consumer confidence and spending can be materially adversely affected in response to changes in fiscal and monetary policy, financial market volatility, declines in income or asset values, and other economic factors. In addition to an adverse impact on demand for the Company’s products and services, uncertainty about, or a decline in, global or regional economic conditions can have a significant impact on the Company’s suppliers, contract manufacturers, logistics providers, distributors, cellular network carriers and other channel partners, and developers. Potential outcomes include financial instability; inability to obtain credit to finance business operations; and insolvency. Adverse economic conditions can also lead to increased credit and collectibility risk on the Company’s trade receivables; the failure of derivative counterparties and other financial institutions; limitations on the Company’s ability to issue new debt; reduced liquidity; and declines in the fair values of the Company’s financial instruments. These and other impacts can materially adversely affect the Company’s business, results of operations, financial condition and stock price." + }, + "metadata": {} + }, + { + "cost": 0.0000355, + "error": "The output gives a general statement but does not provide any specific details about the legal actions or the company's involvement.", + "gradingResult": { + "pass": false, + "score": 0.1, + "reason": "The output gives a general statement but does not provide any specific details about the legal actions or the company's involvement.", + "namedScores": {}, + "tokensUsed": { + "total": 298, + "prompt": 254, + "completion": 44, + "cached": 0 + }, + "componentResults": [ + { + "assertion": { + "type": "llm-rubric", + "value": "Evaluate the output based on how detailed it is. Grade it on a scale of 0.0 to 1.0, where:\nScore of 0.1: Not much detail.\nScore of 0.5: Some detail.\nScore of 1.0: Very detailed.\n" + }, + "pass": false, + "score": 0.1, + "reason": "The output gives a general statement but does not provide any specific details about the legal actions or the company's involvement.", + "tokensUsed": { + "total": 298, + "prompt": 254, + "completion": 44, + "cached": 0 + } + } + ], + "assertion": null + }, + "id": "8ac8c4e4-90fe-4f8f-947d-666b8b9a1338", + "latencyMs": 805, + "namedScores": {}, + "prompt": { + "raw": "'Generate a 1-liner summary of the Section Legal Proceedings from an SEC filing: '", + "label": "prompt1.txt: 'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'" + }, + "promptId": "1cd4a63a09ec6bcce588ef0486fa8be9ffa6dbbb4637fdf551a3dc72ac9a6462", + "promptIdx": 0, + "provider": { + "id": "openai:gpt-3.5-turbo", + "label": "" + }, + "response": { + "output": "This section provides information on any current or pending legal actions involving the company.", + "tokenUsage": { + "total": 41, + "prompt": 26, + "completion": 15 + }, + "cached": false, + "cost": 0.0000355 + }, + "score": 0.1, + "success": false, + "testCase": { + "vars": { + "section": "Legal Proceedings", + " content": "Item 3. Legal Proceedings Digital Markets Act Investigations On March 25, 2024, the Commission announced that it had opened two formal noncompliance investigations against the Company under the DMA. The Commission’s investigations concern (1) Article 5(4) of the DMA, which relates to how developers may communicate and promote offers to end users for apps distributed through the App Store as well as how developers may conclude contracts with those end users; and (2) Article 6(3) of the DMA, which relates to default settings, uninstallation of apps, and a web browser choice screen on iOS. On June 24, 2024, the Commission announced its preliminary findings in the Article 5(4) investigation alleging that the Company’s App Store rules are in breach of the DMA and announced that it had opened a third formal investigation against the Company regarding whether the Company’s new contractual requirements for third-party app developers and app marketplaces may violate the DMA. If the Commission makes a final determination that there has been a violation, it can issue a cease and desist order and may impose fines up to 10% of the Company’s annual worldwide net sales. Although any decision by the Commission can be appealed to the General Court of the EU, the effectiveness of the Commission’s order would apply immediately while the appeal is pending, unless a stay of the order is granted. The Company believes that it complies with the DMA and has continued to make changes to its compliance plan in response to feedback and engagement with the Commission. Department of Justice Lawsuit On March 21, 2024, the U.S. Department of Justice (the “DOJ”) and a number of state and district attorneys general filed a civil antitrust lawsuit in the U.S. District Court for the District of New Jersey against the Company alleging monopolization or attempted monopolization in the markets for “performance smartphones” and “smartphones” in violation of U.S. antitrust laws. The DOJ is seeking equitable relief to redress the alleged anticompetitive behavior. In addition, various civil litigation matters have been filed in state and federal courts in the U.S. alleging similar violations of U.S. antitrust laws and seeking monetary damages and other nonmonetary relief. The Company believes it has substantial defenses and intends to vigorously defend itself. Epic Games Epic Games, Inc. (“Epic”) filed a lawsuit in the U.S. District Court for the Northern District of California (the “California District Court”) against the Company alleging violations of federal and state antitrust laws and California’s unfair competition law based upon the Company’s operation of its App Store. The California District Court found that certain provisions of the Company’s App Store Review Guidelines violate California’s unfair competition law and issued an injunction enjoining the Company from prohibiting developers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app purchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS App Store. On January 16, 2024, the Company implemented a plan to comply with the injunction and filed a statement of compliance with the California District Court. A motion by Epic disputing the Company’s compliance plan and seeking to enforce the injunction, which the Company has opposed, is pending before the California District Court. On September 30, 2024, the Company filed a motion with the California District Court to narrow or vacate the injunction. The Company believes it has substantial defenses and intends to vigorously defend itself. Other Legal Proceedings The Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the ordinary course of business. The Company settled certain matters during the fourth quarter of 2024 that did not individually or in the aggregate have a material impact on the Company’s financial condition or operating results. The outcome of litigation is inherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above management’s expectations, the Company’s financial condition and operating results for that reporting period could be materially adversely affected." + }, + "assert": [ + { + "type": "llm-rubric", + "value": "Evaluate the output based on how detailed it is. Grade it on a scale of 0.0 to 1.0, where:\nScore of 0.1: Not much detail.\nScore of 0.5: Some detail.\nScore of 1.0: Very detailed.\n" + } + ], + "options": {}, + "description": "Row #2", + "metadata": {} + }, + "testIdx": 1, + "vars": { + "section": "Legal Proceedings", + " content": "Item 3. Legal Proceedings Digital Markets Act Investigations On March 25, 2024, the Commission announced that it had opened two formal noncompliance investigations against the Company under the DMA. The Commission’s investigations concern (1) Article 5(4) of the DMA, which relates to how developers may communicate and promote offers to end users for apps distributed through the App Store as well as how developers may conclude contracts with those end users; and (2) Article 6(3) of the DMA, which relates to default settings, uninstallation of apps, and a web browser choice screen on iOS. On June 24, 2024, the Commission announced its preliminary findings in the Article 5(4) investigation alleging that the Company’s App Store rules are in breach of the DMA and announced that it had opened a third formal investigation against the Company regarding whether the Company’s new contractual requirements for third-party app developers and app marketplaces may violate the DMA. If the Commission makes a final determination that there has been a violation, it can issue a cease and desist order and may impose fines up to 10% of the Company’s annual worldwide net sales. Although any decision by the Commission can be appealed to the General Court of the EU, the effectiveness of the Commission’s order would apply immediately while the appeal is pending, unless a stay of the order is granted. The Company believes that it complies with the DMA and has continued to make changes to its compliance plan in response to feedback and engagement with the Commission. Department of Justice Lawsuit On March 21, 2024, the U.S. Department of Justice (the “DOJ”) and a number of state and district attorneys general filed a civil antitrust lawsuit in the U.S. District Court for the District of New Jersey against the Company alleging monopolization or attempted monopolization in the markets for “performance smartphones” and “smartphones” in violation of U.S. antitrust laws. The DOJ is seeking equitable relief to redress the alleged anticompetitive behavior. In addition, various civil litigation matters have been filed in state and federal courts in the U.S. alleging similar violations of U.S. antitrust laws and seeking monetary damages and other nonmonetary relief. The Company believes it has substantial defenses and intends to vigorously defend itself. Epic Games Epic Games, Inc. (“Epic”) filed a lawsuit in the U.S. District Court for the Northern District of California (the “California District Court”) against the Company alleging violations of federal and state antitrust laws and California’s unfair competition law based upon the Company’s operation of its App Store. The California District Court found that certain provisions of the Company’s App Store Review Guidelines violate California’s unfair competition law and issued an injunction enjoining the Company from prohibiting developers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app purchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS App Store. On January 16, 2024, the Company implemented a plan to comply with the injunction and filed a statement of compliance with the California District Court. A motion by Epic disputing the Company’s compliance plan and seeking to enforce the injunction, which the Company has opposed, is pending before the California District Court. On September 30, 2024, the Company filed a motion with the California District Court to narrow or vacate the injunction. The Company believes it has substantial defenses and intends to vigorously defend itself. Other Legal Proceedings The Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the ordinary course of business. The Company settled certain matters during the fourth quarter of 2024 that did not individually or in the aggregate have a material impact on the Company’s financial condition or operating results. The outcome of litigation is inherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above management’s expectations, the Company’s financial condition and operating results for that reporting period could be materially adversely affected." + }, + "metadata": {} + }, + { + "cost": 0.00006549999999999999, + "gradingResult": { + "pass": true, + "score": 0.5, + "reason": "All assertions passed", + "namedScores": {}, + "tokensUsed": { + "total": 311, + "prompt": 267, + "completion": 44, + "cached": 0 + }, + "componentResults": [ + { + "assertion": { + "type": "llm-rubric", + "value": "Evaluate the output based on how detailed it is. Grade it on a scale of 0.0 to 1.0, where:\nScore of 0.1: Not much detail.\nScore of 0.5: Some detail.\nScore of 1.0: Very detailed.\n" + }, + "pass": true, + "score": 0.5, + "reason": "The output provides a general overview of the Risk Factors section but lacks specific details or examples of threats and uncertainties.", + "tokensUsed": { + "total": 311, + "prompt": 267, + "completion": 44, + "cached": 0 + } + } + ], + "assertion": null + }, + "id": "6024f883-6d72-4838-b969-cc692ba5b0e4", + "latencyMs": 818, + "namedScores": {}, + "prompt": { + "raw": "'ROLE: You are a financial analyst. REQUIREMENTS: BE DETAILED. TASK: Generate a 1-liner summary of the Section Risk Factors from an SEC filing: '", + "label": "prompt3.txt: 'ROLE: You are a financial analyst. REQUIREMENTS: BE DETAILED. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'" + }, + "promptId": "4e36379853105286bc2b2bba84330e0ea2402b1c965acc338605ac84d2c35dee", + "promptIdx": 2, + "provider": { + "id": "openai:gpt-3.5-turbo", + "label": "" + }, + "response": { + "output": "The section on Risk Factors in the SEC filing highlights the potential threats and uncertainties that could negatively impact the company's financial performance and overall business operations.", + "tokenUsage": { + "total": 73, + "prompt": 44, + "completion": 29 + }, + "cached": false, + "cost": 0.00006549999999999999 + }, + "score": 0.5, + "success": true, + "testCase": { + "vars": { + "section": "Risk Factors", + " content": "Item 1A. Macroeconomic and Industry Risks The Company’s operations and performance depend significantly on global and regional economic conditions and adverse economic conditions can materially adversely affect the Company’s business, results of operations and financial condition. The Company has international operations with sales outside the U.S. representing a majority of the Company’s total net sales. In addition, the Company’s global supply chain is large and complex and a majority of the Company’s supplier facilities, including manufacturing and assembly sites, are located outside the U.S. As a result, the Company’s operations and performance depend significantly on global and regional economic conditions. Adverse macroeconomic conditions, including slow growth or recession, high unemployment, inflation, tighter credit, higher interest rates, and currency fluctuations, can adversely impact consumer confidence and spending and materially adversely affect demand for the Company’s products and services. In addition, consumer confidence and spending can be materially adversely affected in response to changes in fiscal and monetary policy, financial market volatility, declines in income or asset values, and other economic factors. In addition to an adverse impact on demand for the Company’s products and services, uncertainty about, or a decline in, global or regional economic conditions can have a significant impact on the Company’s suppliers, contract manufacturers, logistics providers, distributors, cellular network carriers and other channel partners, and developers. Potential outcomes include financial instability; inability to obtain credit to finance business operations; and insolvency. Adverse economic conditions can also lead to increased credit and collectibility risk on the Company’s trade receivables; the failure of derivative counterparties and other financial institutions; limitations on the Company’s ability to issue new debt; reduced liquidity; and declines in the fair values of the Company’s financial instruments. These and other impacts can materially adversely affect the Company’s business, results of operations, financial condition and stock price." + }, + "assert": [ + { + "type": "llm-rubric", + "value": "Evaluate the output based on how detailed it is. Grade it on a scale of 0.0 to 1.0, where:\nScore of 0.1: Not much detail.\nScore of 0.5: Some detail.\nScore of 1.0: Very detailed.\n" + } + ], + "options": {}, + "description": "Row #1", + "metadata": {} + }, + "testIdx": 0, + "vars": { + "section": "Risk Factors", + " content": "Item 1A. Macroeconomic and Industry Risks The Company’s operations and performance depend significantly on global and regional economic conditions and adverse economic conditions can materially adversely affect the Company’s business, results of operations and financial condition. The Company has international operations with sales outside the U.S. representing a majority of the Company’s total net sales. In addition, the Company’s global supply chain is large and complex and a majority of the Company’s supplier facilities, including manufacturing and assembly sites, are located outside the U.S. As a result, the Company’s operations and performance depend significantly on global and regional economic conditions. Adverse macroeconomic conditions, including slow growth or recession, high unemployment, inflation, tighter credit, higher interest rates, and currency fluctuations, can adversely impact consumer confidence and spending and materially adversely affect demand for the Company’s products and services. In addition, consumer confidence and spending can be materially adversely affected in response to changes in fiscal and monetary policy, financial market volatility, declines in income or asset values, and other economic factors. In addition to an adverse impact on demand for the Company’s products and services, uncertainty about, or a decline in, global or regional economic conditions can have a significant impact on the Company’s suppliers, contract manufacturers, logistics providers, distributors, cellular network carriers and other channel partners, and developers. Potential outcomes include financial instability; inability to obtain credit to finance business operations; and insolvency. Adverse economic conditions can also lead to increased credit and collectibility risk on the Company’s trade receivables; the failure of derivative counterparties and other financial institutions; limitations on the Company’s ability to issue new debt; reduced liquidity; and declines in the fair values of the Company’s financial instruments. These and other impacts can materially adversely affect the Company’s business, results of operations, financial condition and stock price." + }, + "metadata": {} + }, + { + "cost": 0.0000385, + "error": "The output identifies potential risks but does not elaborate on specific risks or provide examples. It is lacking detail and specificity.", + "gradingResult": { + "pass": false, + "score": 0.1, + "reason": "The output identifies potential risks but does not elaborate on specific risks or provide examples. It is lacking detail and specificity.", + "namedScores": {}, + "tokensUsed": { + "total": 300, + "prompt": 255, + "completion": 45, + "cached": 0 + }, + "componentResults": [ + { + "assertion": { + "type": "llm-rubric", + "value": "Evaluate the output based on how detailed it is. Grade it on a scale of 0.0 to 1.0, where:\nScore of 0.1: Not much detail.\nScore of 0.5: Some detail.\nScore of 1.0: Very detailed.\n" + }, + "pass": false, + "score": 0.1, + "reason": "The output identifies potential risks but does not elaborate on specific risks or provide examples. It is lacking detail and specificity.", + "tokensUsed": { + "total": 300, + "prompt": 255, + "completion": 45, + "cached": 0 + } + } + ], + "assertion": null + }, + "id": "3a20385e-6135-48fe-aef0-a719937c031a", + "latencyMs": 816, + "namedScores": {}, + "prompt": { + "raw": "'Generate a 1-liner summary of the Section Risk Factors from an SEC filing: '", + "label": "prompt1.txt: 'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'" + }, + "promptId": "1cd4a63a09ec6bcce588ef0486fa8be9ffa6dbbb4637fdf551a3dc72ac9a6462", + "promptIdx": 0, + "provider": { + "id": "openai:gpt-3.5-turbo", + "label": "" + }, + "response": { + "output": "Identifies potential risks that could negatively impact the company's financial performance or future prospects.", + "tokenUsage": { + "total": 43, + "prompt": 26, + "completion": 17 + }, + "cached": false, + "cost": 0.0000385 + }, + "score": 0.1, + "success": false, + "testCase": { + "vars": { + "section": "Risk Factors", + " content": "Item 1A. Macroeconomic and Industry Risks The Company’s operations and performance depend significantly on global and regional economic conditions and adverse economic conditions can materially adversely affect the Company’s business, results of operations and financial condition. The Company has international operations with sales outside the U.S. representing a majority of the Company’s total net sales. In addition, the Company’s global supply chain is large and complex and a majority of the Company’s supplier facilities, including manufacturing and assembly sites, are located outside the U.S. As a result, the Company’s operations and performance depend significantly on global and regional economic conditions. Adverse macroeconomic conditions, including slow growth or recession, high unemployment, inflation, tighter credit, higher interest rates, and currency fluctuations, can adversely impact consumer confidence and spending and materially adversely affect demand for the Company’s products and services. In addition, consumer confidence and spending can be materially adversely affected in response to changes in fiscal and monetary policy, financial market volatility, declines in income or asset values, and other economic factors. In addition to an adverse impact on demand for the Company’s products and services, uncertainty about, or a decline in, global or regional economic conditions can have a significant impact on the Company’s suppliers, contract manufacturers, logistics providers, distributors, cellular network carriers and other channel partners, and developers. Potential outcomes include financial instability; inability to obtain credit to finance business operations; and insolvency. Adverse economic conditions can also lead to increased credit and collectibility risk on the Company’s trade receivables; the failure of derivative counterparties and other financial institutions; limitations on the Company’s ability to issue new debt; reduced liquidity; and declines in the fair values of the Company’s financial instruments. These and other impacts can materially adversely affect the Company’s business, results of operations, financial condition and stock price." + }, + "assert": [ + { + "type": "llm-rubric", + "value": "Evaluate the output based on how detailed it is. Grade it on a scale of 0.0 to 1.0, where:\nScore of 0.1: Not much detail.\nScore of 0.5: Some detail.\nScore of 1.0: Very detailed.\n" + } + ], + "options": {}, + "description": "Row #1", + "metadata": {} + }, + "testIdx": 0, + "vars": { + "section": "Risk Factors", + " content": "Item 1A. Macroeconomic and Industry Risks The Company’s operations and performance depend significantly on global and regional economic conditions and adverse economic conditions can materially adversely affect the Company’s business, results of operations and financial condition. The Company has international operations with sales outside the U.S. representing a majority of the Company’s total net sales. In addition, the Company’s global supply chain is large and complex and a majority of the Company’s supplier facilities, including manufacturing and assembly sites, are located outside the U.S. As a result, the Company’s operations and performance depend significantly on global and regional economic conditions. Adverse macroeconomic conditions, including slow growth or recession, high unemployment, inflation, tighter credit, higher interest rates, and currency fluctuations, can adversely impact consumer confidence and spending and materially adversely affect demand for the Company’s products and services. In addition, consumer confidence and spending can be materially adversely affected in response to changes in fiscal and monetary policy, financial market volatility, declines in income or asset values, and other economic factors. In addition to an adverse impact on demand for the Company’s products and services, uncertainty about, or a decline in, global or regional economic conditions can have a significant impact on the Company’s suppliers, contract manufacturers, logistics providers, distributors, cellular network carriers and other channel partners, and developers. Potential outcomes include financial instability; inability to obtain credit to finance business operations; and insolvency. Adverse economic conditions can also lead to increased credit and collectibility risk on the Company’s trade receivables; the failure of derivative counterparties and other financial institutions; limitations on the Company’s ability to issue new debt; reduced liquidity; and declines in the fair values of the Company’s financial instruments. These and other impacts can materially adversely affect the Company’s business, results of operations, financial condition and stock price." + }, + "metadata": {} + }, + { + "cost": 0.000085, + "gradingResult": { + "pass": true, + "score": 1, + "reason": "All assertions passed", + "namedScores": {}, + "tokensUsed": { + "total": 342, + "prompt": 281, + "completion": 61, + "cached": 0 + }, + "componentResults": [ + { + "assertion": { + "type": "llm-rubric", + "value": "Evaluate the output based on how detailed it is. Grade it on a scale of 0.0 to 1.0, where:\nScore of 0.1: Not much detail.\nScore of 0.5: Some detail.\nScore of 1.0: Very detailed.\n" + }, + "pass": true, + "score": 1, + "reason": "The output provides a clear and comprehensive overview, covering pending or potential legal actions, disputes, regulatory matters, risks, outcomes, and potential impacts on financial performance, without leaving major elements unmentioned.", + "tokensUsed": { + "total": 342, + "prompt": 281, + "completion": 61, + "cached": 0 + } + } + ], + "assertion": null + }, + "id": "cdb6b482-8cba-4501-8da1-f8f9d84d2661", + "latencyMs": 792, + "namedScores": {}, + "prompt": { + "raw": "'ROLE: You are a financial analyst. REQUIREMENTS: BE DETAILED. TASK: Generate a 1-liner summary of the Section Legal Proceedings from an SEC filing: '", + "label": "prompt3.txt: 'ROLE: You are a financial analyst. REQUIREMENTS: BE DETAILED. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'" + }, + "promptId": "4e36379853105286bc2b2bba84330e0ea2402b1c965acc338605ac84d2c35dee", + "promptIdx": 2, + "provider": { + "id": "openai:gpt-3.5-turbo", + "label": "" + }, + "response": { + "output": "The Section on Legal Proceedings in an SEC filing provides a comprehensive overview of any pending or potential legal actions, disputes, or regulatory matters involving the company, including risks, outcomes, and potential impacts on financial performance.", + "tokenUsage": { + "total": 86, + "prompt": 44, + "completion": 42 + }, + "cached": false, + "cost": 0.000085 + }, + "score": 1, + "success": true, + "testCase": { + "vars": { + "section": "Legal Proceedings", + " content": "Item 3. Legal Proceedings Digital Markets Act Investigations On March 25, 2024, the Commission announced that it had opened two formal noncompliance investigations against the Company under the DMA. The Commission’s investigations concern (1) Article 5(4) of the DMA, which relates to how developers may communicate and promote offers to end users for apps distributed through the App Store as well as how developers may conclude contracts with those end users; and (2) Article 6(3) of the DMA, which relates to default settings, uninstallation of apps, and a web browser choice screen on iOS. On June 24, 2024, the Commission announced its preliminary findings in the Article 5(4) investigation alleging that the Company’s App Store rules are in breach of the DMA and announced that it had opened a third formal investigation against the Company regarding whether the Company’s new contractual requirements for third-party app developers and app marketplaces may violate the DMA. If the Commission makes a final determination that there has been a violation, it can issue a cease and desist order and may impose fines up to 10% of the Company’s annual worldwide net sales. Although any decision by the Commission can be appealed to the General Court of the EU, the effectiveness of the Commission’s order would apply immediately while the appeal is pending, unless a stay of the order is granted. The Company believes that it complies with the DMA and has continued to make changes to its compliance plan in response to feedback and engagement with the Commission. Department of Justice Lawsuit On March 21, 2024, the U.S. Department of Justice (the “DOJ”) and a number of state and district attorneys general filed a civil antitrust lawsuit in the U.S. District Court for the District of New Jersey against the Company alleging monopolization or attempted monopolization in the markets for “performance smartphones” and “smartphones” in violation of U.S. antitrust laws. The DOJ is seeking equitable relief to redress the alleged anticompetitive behavior. In addition, various civil litigation matters have been filed in state and federal courts in the U.S. alleging similar violations of U.S. antitrust laws and seeking monetary damages and other nonmonetary relief. The Company believes it has substantial defenses and intends to vigorously defend itself. Epic Games Epic Games, Inc. (“Epic”) filed a lawsuit in the U.S. District Court for the Northern District of California (the “California District Court”) against the Company alleging violations of federal and state antitrust laws and California’s unfair competition law based upon the Company’s operation of its App Store. The California District Court found that certain provisions of the Company’s App Store Review Guidelines violate California’s unfair competition law and issued an injunction enjoining the Company from prohibiting developers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app purchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS App Store. On January 16, 2024, the Company implemented a plan to comply with the injunction and filed a statement of compliance with the California District Court. A motion by Epic disputing the Company’s compliance plan and seeking to enforce the injunction, which the Company has opposed, is pending before the California District Court. On September 30, 2024, the Company filed a motion with the California District Court to narrow or vacate the injunction. The Company believes it has substantial defenses and intends to vigorously defend itself. Other Legal Proceedings The Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the ordinary course of business. The Company settled certain matters during the fourth quarter of 2024 that did not individually or in the aggregate have a material impact on the Company’s financial condition or operating results. The outcome of litigation is inherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above management’s expectations, the Company’s financial condition and operating results for that reporting period could be materially adversely affected." + }, + "assert": [ + { + "type": "llm-rubric", + "value": "Evaluate the output based on how detailed it is. Grade it on a scale of 0.0 to 1.0, where:\nScore of 0.1: Not much detail.\nScore of 0.5: Some detail.\nScore of 1.0: Very detailed.\n" + } + ], + "options": {}, + "description": "Row #2", + "metadata": {} + }, + "testIdx": 1, + "vars": { + "section": "Legal Proceedings", + " content": "Item 3. Legal Proceedings Digital Markets Act Investigations On March 25, 2024, the Commission announced that it had opened two formal noncompliance investigations against the Company under the DMA. The Commission’s investigations concern (1) Article 5(4) of the DMA, which relates to how developers may communicate and promote offers to end users for apps distributed through the App Store as well as how developers may conclude contracts with those end users; and (2) Article 6(3) of the DMA, which relates to default settings, uninstallation of apps, and a web browser choice screen on iOS. On June 24, 2024, the Commission announced its preliminary findings in the Article 5(4) investigation alleging that the Company’s App Store rules are in breach of the DMA and announced that it had opened a third formal investigation against the Company regarding whether the Company’s new contractual requirements for third-party app developers and app marketplaces may violate the DMA. If the Commission makes a final determination that there has been a violation, it can issue a cease and desist order and may impose fines up to 10% of the Company’s annual worldwide net sales. Although any decision by the Commission can be appealed to the General Court of the EU, the effectiveness of the Commission’s order would apply immediately while the appeal is pending, unless a stay of the order is granted. The Company believes that it complies with the DMA and has continued to make changes to its compliance plan in response to feedback and engagement with the Commission. Department of Justice Lawsuit On March 21, 2024, the U.S. Department of Justice (the “DOJ”) and a number of state and district attorneys general filed a civil antitrust lawsuit in the U.S. District Court for the District of New Jersey against the Company alleging monopolization or attempted monopolization in the markets for “performance smartphones” and “smartphones” in violation of U.S. antitrust laws. The DOJ is seeking equitable relief to redress the alleged anticompetitive behavior. In addition, various civil litigation matters have been filed in state and federal courts in the U.S. alleging similar violations of U.S. antitrust laws and seeking monetary damages and other nonmonetary relief. The Company believes it has substantial defenses and intends to vigorously defend itself. Epic Games Epic Games, Inc. (“Epic”) filed a lawsuit in the U.S. District Court for the Northern District of California (the “California District Court”) against the Company alleging violations of federal and state antitrust laws and California’s unfair competition law based upon the Company’s operation of its App Store. The California District Court found that certain provisions of the Company’s App Store Review Guidelines violate California’s unfair competition law and issued an injunction enjoining the Company from prohibiting developers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app purchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS App Store. On January 16, 2024, the Company implemented a plan to comply with the injunction and filed a statement of compliance with the California District Court. A motion by Epic disputing the Company’s compliance plan and seeking to enforce the injunction, which the Company has opposed, is pending before the California District Court. On September 30, 2024, the Company filed a motion with the California District Court to narrow or vacate the injunction. The Company believes it has substantial defenses and intends to vigorously defend itself. Other Legal Proceedings The Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the ordinary course of business. The Company settled certain matters during the fourth quarter of 2024 that did not individually or in the aggregate have a material impact on the Company’s financial condition or operating results. The outcome of litigation is inherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above management’s expectations, the Company’s financial condition and operating results for that reporting period could be materially adversely affected." + }, + "metadata": {} + }, + { + "cost": 0.000048, + "gradingResult": { + "pass": true, + "score": 0.5, + "reason": "All assertions passed", + "namedScores": {}, + "tokensUsed": { + "total": 320, + "prompt": 259, + "completion": 61, + "cached": 0 + }, + "componentResults": [ + { + "assertion": { + "type": "llm-rubric", + "value": "Evaluate the output based on how detailed it is. Grade it on a scale of 0.0 to 1.0, where:\nScore of 0.1: Not much detail.\nScore of 0.5: Some detail.\nScore of 1.0: Very detailed.\n" + }, + "pass": true, + "score": 0.5, + "reason": "The output states that the company is involved in ongoing legal proceedings that may impact its financial results but lacks further specifics such as the nature of the legal proceedings, parties involved, or potential financial impact.", + "tokensUsed": { + "total": 320, + "prompt": 259, + "completion": 61, + "cached": 0 + } + } + ], + "assertion": null + }, + "id": "12f8c180-0cec-40bb-8d4a-3fc09fb9c936", + "latencyMs": 685, + "namedScores": {}, + "prompt": { + "raw": "'ROLE: You are a financial analyst. TASK: Generate a 1-liner summary of the Section Legal Proceedings from an SEC filing: '", + "label": "prompt2.txt: 'ROLE: You are a financial analyst. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}'" + }, + "promptId": "67b9d227a9b9ffc025b0f3503a0461da85fe5184f253af38cb2315159ffe9b02", + "promptIdx": 1, + "provider": { + "id": "openai:gpt-3.5-turbo", + "label": "" + }, + "response": { + "output": "The company is currently involved in ongoing legal proceedings that may have a material impact on its financial results.", + "tokenUsage": { + "total": 56, + "prompt": 36, + "completion": 20 + }, + "cached": false, + "cost": 0.000048 + }, + "score": 0.5, + "success": true, + "testCase": { + "vars": { + "section": "Legal Proceedings", + " content": "Item 3. Legal Proceedings Digital Markets Act Investigations On March 25, 2024, the Commission announced that it had opened two formal noncompliance investigations against the Company under the DMA. The Commission’s investigations concern (1) Article 5(4) of the DMA, which relates to how developers may communicate and promote offers to end users for apps distributed through the App Store as well as how developers may conclude contracts with those end users; and (2) Article 6(3) of the DMA, which relates to default settings, uninstallation of apps, and a web browser choice screen on iOS. On June 24, 2024, the Commission announced its preliminary findings in the Article 5(4) investigation alleging that the Company’s App Store rules are in breach of the DMA and announced that it had opened a third formal investigation against the Company regarding whether the Company’s new contractual requirements for third-party app developers and app marketplaces may violate the DMA. If the Commission makes a final determination that there has been a violation, it can issue a cease and desist order and may impose fines up to 10% of the Company’s annual worldwide net sales. Although any decision by the Commission can be appealed to the General Court of the EU, the effectiveness of the Commission’s order would apply immediately while the appeal is pending, unless a stay of the order is granted. The Company believes that it complies with the DMA and has continued to make changes to its compliance plan in response to feedback and engagement with the Commission. Department of Justice Lawsuit On March 21, 2024, the U.S. Department of Justice (the “DOJ”) and a number of state and district attorneys general filed a civil antitrust lawsuit in the U.S. District Court for the District of New Jersey against the Company alleging monopolization or attempted monopolization in the markets for “performance smartphones” and “smartphones” in violation of U.S. antitrust laws. The DOJ is seeking equitable relief to redress the alleged anticompetitive behavior. In addition, various civil litigation matters have been filed in state and federal courts in the U.S. alleging similar violations of U.S. antitrust laws and seeking monetary damages and other nonmonetary relief. The Company believes it has substantial defenses and intends to vigorously defend itself. Epic Games Epic Games, Inc. (“Epic”) filed a lawsuit in the U.S. District Court for the Northern District of California (the “California District Court”) against the Company alleging violations of federal and state antitrust laws and California’s unfair competition law based upon the Company’s operation of its App Store. The California District Court found that certain provisions of the Company’s App Store Review Guidelines violate California’s unfair competition law and issued an injunction enjoining the Company from prohibiting developers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app purchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS App Store. On January 16, 2024, the Company implemented a plan to comply with the injunction and filed a statement of compliance with the California District Court. A motion by Epic disputing the Company’s compliance plan and seeking to enforce the injunction, which the Company has opposed, is pending before the California District Court. On September 30, 2024, the Company filed a motion with the California District Court to narrow or vacate the injunction. The Company believes it has substantial defenses and intends to vigorously defend itself. Other Legal Proceedings The Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the ordinary course of business. The Company settled certain matters during the fourth quarter of 2024 that did not individually or in the aggregate have a material impact on the Company’s financial condition or operating results. The outcome of litigation is inherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above management’s expectations, the Company’s financial condition and operating results for that reporting period could be materially adversely affected." + }, + "assert": [ + { + "type": "llm-rubric", + "value": "Evaluate the output based on how detailed it is. Grade it on a scale of 0.0 to 1.0, where:\nScore of 0.1: Not much detail.\nScore of 0.5: Some detail.\nScore of 1.0: Very detailed.\n" + } + ], + "options": {}, + "description": "Row #2", + "metadata": {} + }, + "testIdx": 1, + "vars": { + "section": "Legal Proceedings", + " content": "Item 3. Legal Proceedings Digital Markets Act Investigations On March 25, 2024, the Commission announced that it had opened two formal noncompliance investigations against the Company under the DMA. The Commission’s investigations concern (1) Article 5(4) of the DMA, which relates to how developers may communicate and promote offers to end users for apps distributed through the App Store as well as how developers may conclude contracts with those end users; and (2) Article 6(3) of the DMA, which relates to default settings, uninstallation of apps, and a web browser choice screen on iOS. On June 24, 2024, the Commission announced its preliminary findings in the Article 5(4) investigation alleging that the Company’s App Store rules are in breach of the DMA and announced that it had opened a third formal investigation against the Company regarding whether the Company’s new contractual requirements for third-party app developers and app marketplaces may violate the DMA. If the Commission makes a final determination that there has been a violation, it can issue a cease and desist order and may impose fines up to 10% of the Company’s annual worldwide net sales. Although any decision by the Commission can be appealed to the General Court of the EU, the effectiveness of the Commission’s order would apply immediately while the appeal is pending, unless a stay of the order is granted. The Company believes that it complies with the DMA and has continued to make changes to its compliance plan in response to feedback and engagement with the Commission. Department of Justice Lawsuit On March 21, 2024, the U.S. Department of Justice (the “DOJ”) and a number of state and district attorneys general filed a civil antitrust lawsuit in the U.S. District Court for the District of New Jersey against the Company alleging monopolization or attempted monopolization in the markets for “performance smartphones” and “smartphones” in violation of U.S. antitrust laws. The DOJ is seeking equitable relief to redress the alleged anticompetitive behavior. In addition, various civil litigation matters have been filed in state and federal courts in the U.S. alleging similar violations of U.S. antitrust laws and seeking monetary damages and other nonmonetary relief. The Company believes it has substantial defenses and intends to vigorously defend itself. Epic Games Epic Games, Inc. (“Epic”) filed a lawsuit in the U.S. District Court for the Northern District of California (the “California District Court”) against the Company alleging violations of federal and state antitrust laws and California’s unfair competition law based upon the Company’s operation of its App Store. The California District Court found that certain provisions of the Company’s App Store Review Guidelines violate California’s unfair competition law and issued an injunction enjoining the Company from prohibiting developers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app purchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS App Store. On January 16, 2024, the Company implemented a plan to comply with the injunction and filed a statement of compliance with the California District Court. A motion by Epic disputing the Company’s compliance plan and seeking to enforce the injunction, which the Company has opposed, is pending before the California District Court. On September 30, 2024, the Company filed a motion with the California District Court to narrow or vacate the injunction. The Company believes it has substantial defenses and intends to vigorously defend itself. Other Legal Proceedings The Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the ordinary course of business. The Company settled certain matters during the fourth quarter of 2024 that did not individually or in the aggregate have a material impact on the Company’s financial condition or operating results. The outcome of litigation is inherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above management’s expectations, the Company’s financial condition and operating results for that reporting period could be materially adversely affected." + }, + "metadata": {} + } + ], + "stats": { + "successes": 4, + "failures": 2, + "tokenUsage": { + "cached": 0, + "completion": 152, + "prompt": 212, + "total": 364, + "numRequests": 6 + } + } + }, + "config": { + "description": "Best model eval", + "prompts": [ + "file://prompt1.txt", + "file://prompt2.txt", + "file://prompt3.txt" + ], + "providers": [ + "openai:gpt-3.5-turbo" + ], + "tests": "file://tests.csv", + "sharing": true, + "defaultTest": { + "assert": [ + { + "type": "llm-rubric", + "value": "Evaluate the output based on how detailed it is. Grade it on a scale of 0.0 to 1.0, where:\nScore of 0.1: Not much detail.\nScore of 0.5: Some detail.\nScore of 1.0: Very detailed.\n" + } + ] + }, + "outputPath": [ + "eval.json" + ], + "extensions": [] + }, + "shareableUrl": null +} \ No newline at end of file diff --git a/tamingllms/notebooks/promptfoo/prompt_comparison/prompt1.txt b/tamingllms/notebooks/promptfoo/prompt_comparison/prompt1.txt new file mode 100644 index 0000000..7b560db --- /dev/null +++ b/tamingllms/notebooks/promptfoo/prompt_comparison/prompt1.txt @@ -0,0 +1 @@ +'Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}' diff --git a/tamingllms/notebooks/promptfoo/prompt_comparison/prompt2.txt b/tamingllms/notebooks/promptfoo/prompt_comparison/prompt2.txt new file mode 100644 index 0000000..3d3125d --- /dev/null +++ b/tamingllms/notebooks/promptfoo/prompt_comparison/prompt2.txt @@ -0,0 +1 @@ +'ROLE: You are a financial analyst. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}' diff --git a/tamingllms/notebooks/promptfoo/prompt_comparison/prompt3.txt b/tamingllms/notebooks/promptfoo/prompt_comparison/prompt3.txt new file mode 100644 index 0000000..64e50a8 --- /dev/null +++ b/tamingllms/notebooks/promptfoo/prompt_comparison/prompt3.txt @@ -0,0 +1 @@ +'ROLE: You are a financial analyst. REQUIREMENTS: BE DETAILED. TASK: Generate a 1-liner summary of the Section {{section}} from an SEC filing: {{content}}' \ No newline at end of file diff --git a/tamingllms/notebooks/promptfoo/prompt_comparison/promptfooconfig.yaml b/tamingllms/notebooks/promptfoo/prompt_comparison/promptfooconfig.yaml new file mode 100644 index 0000000..14acdea --- /dev/null +++ b/tamingllms/notebooks/promptfoo/prompt_comparison/promptfooconfig.yaml @@ -0,0 +1,26 @@ +# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json + +# Learn more about building a configuration: https://promptfoo.dev/docs/configuration/guide + +description: "Best model eval" + +prompts: + - file://prompt1.txt + - file://prompt2.txt + - file://prompt3.txt + + +providers: + - openai:gpt-3.5-turbo + +# These test properties are applied to every test +defaultTest: + assert: + - type: llm-rubric + value: | + Evaluate the output based on how detailed it is. Grade it on a scale of 0.0 to 1.0, where: + Score of 0.1: Not much detail. + Score of 0.5: Some detail. + Score of 1.0: Very detailed. + +tests: file://tests.csv diff --git a/tamingllms/notebooks/promptfoo/prompt_comparison/tests.csv b/tamingllms/notebooks/promptfoo/prompt_comparison/tests.csv new file mode 100644 index 0000000..4d94f54 --- /dev/null +++ b/tamingllms/notebooks/promptfoo/prompt_comparison/tests.csv @@ -0,0 +1,3 @@ +section, content +Risk Factors,"Item 1A. Macroeconomic and Industry Risks The Company’s operations and performance depend significantly on global and regional economic conditions and adverse economic conditions can materially adversely affect the Company’s business, results of operations and financial condition. The Company has international operations with sales outside the U.S. representing a majority of the Company’s total net sales. In addition, the Company’s global supply chain is large and complex and a majority of the Company’s supplier facilities, including manufacturing and assembly sites, are located outside the U.S. As a result, the Company’s operations and performance depend significantly on global and regional economic conditions. Adverse macroeconomic conditions, including slow growth or recession, high unemployment, inflation, tighter credit, higher interest rates, and currency fluctuations, can adversely impact consumer confidence and spending and materially adversely affect demand for the Company’s products and services. In addition, consumer confidence and spending can be materially adversely affected in response to changes in fiscal and monetary policy, financial market volatility, declines in income or asset values, and other economic factors. In addition to an adverse impact on demand for the Company’s products and services, uncertainty about, or a decline in, global or regional economic conditions can have a significant impact on the Company’s suppliers, contract manufacturers, logistics providers, distributors, cellular network carriers and other channel partners, and developers. Potential outcomes include financial instability; inability to obtain credit to finance business operations; and insolvency. Adverse economic conditions can also lead to increased credit and collectibility risk on the Company’s trade receivables; the failure of derivative counterparties and other financial institutions; limitations on the Company’s ability to issue new debt; reduced liquidity; and declines in the fair values of the Company’s financial instruments. These and other impacts can materially adversely affect the Company’s business, results of operations, financial condition and stock price." +Legal Proceedings,"Item 3. Legal Proceedings Digital Markets Act Investigations On March 25, 2024, the Commission announced that it had opened two formal noncompliance investigations against the Company under the DMA. The Commission’s investigations concern (1) Article 5(4) of the DMA, which relates to how developers may communicate and promote offers to end users for apps distributed through the App Store as well as how developers may conclude contracts with those end users; and (2) Article 6(3) of the DMA, which relates to default settings, uninstallation of apps, and a web browser choice screen on iOS. On June 24, 2024, the Commission announced its preliminary findings in the Article 5(4) investigation alleging that the Company’s App Store rules are in breach of the DMA and announced that it had opened a third formal investigation against the Company regarding whether the Company’s new contractual requirements for third-party app developers and app marketplaces may violate the DMA. If the Commission makes a final determination that there has been a violation, it can issue a cease and desist order and may impose fines up to 10% of the Company’s annual worldwide net sales. Although any decision by the Commission can be appealed to the General Court of the EU, the effectiveness of the Commission’s order would apply immediately while the appeal is pending, unless a stay of the order is granted. The Company believes that it complies with the DMA and has continued to make changes to its compliance plan in response to feedback and engagement with the Commission. Department of Justice Lawsuit On March 21, 2024, the U.S. Department of Justice (the “DOJ”) and a number of state and district attorneys general filed a civil antitrust lawsuit in the U.S. District Court for the District of New Jersey against the Company alleging monopolization or attempted monopolization in the markets for “performance smartphones” and “smartphones” in violation of U.S. antitrust laws. The DOJ is seeking equitable relief to redress the alleged anticompetitive behavior. In addition, various civil litigation matters have been filed in state and federal courts in the U.S. alleging similar violations of U.S. antitrust laws and seeking monetary damages and other nonmonetary relief. The Company believes it has substantial defenses and intends to vigorously defend itself. Epic Games Epic Games, Inc. (“Epic”) filed a lawsuit in the U.S. District Court for the Northern District of California (the “California District Court”) against the Company alleging violations of federal and state antitrust laws and California’s unfair competition law based upon the Company’s operation of its App Store. The California District Court found that certain provisions of the Company’s App Store Review Guidelines violate California’s unfair competition law and issued an injunction enjoining the Company from prohibiting developers from including in their apps external links that direct customers to purchasing mechanisms other than Apple in-app purchasing. The injunction applies to apps on the U.S. storefront of the iOS and iPadOS App Store. On January 16, 2024, the Company implemented a plan to comply with the injunction and filed a statement of compliance with the California District Court. A motion by Epic disputing the Company’s compliance plan and seeking to enforce the injunction, which the Company has opposed, is pending before the California District Court. On September 30, 2024, the Company filed a motion with the California District Court to narrow or vacate the injunction. The Company believes it has substantial defenses and intends to vigorously defend itself. Other Legal Proceedings The Company is subject to other legal proceedings and claims that have not been fully resolved and that have arisen in the ordinary course of business. The Company settled certain matters during the fourth quarter of 2024 that did not individually or in the aggregate have a material impact on the Company’s financial condition or operating results. The outcome of litigation is inherently uncertain. If one or more legal matters were resolved against the Company in a reporting period for amounts above management’s expectations, the Company’s financial condition and operating results for that reporting period could be materially adversely affected." \ No newline at end of file diff --git a/tamingllms/references.bib b/tamingllms/references.bib index 86c4761..fca567b 100644 --- a/tamingllms/references.bib +++ b/tamingllms/references.bib @@ -414,11 +414,11 @@ @article{brailsford1999constraint } @misc{vivien2024regex, - title={LLM Decoding with Regex Constraints}, - author={Vivien Tran-Thien}, - year={2024}, - howpublished={Blog post}, - url={https://vivien000.github.io/blog/journal/llm-decoding-with-regex-constraints.html} +title={Fast, High-Fidelity LLM Decoding with Regex Constraints}, +url={https://vivien000.github.io/blog/journal/llm-decoding-with-regex-constraints.html}, +journal={Unsupervised Thoughts (blog)}, +author={Tran-Thien, Vivien}, +year={2024} } @misc{willard2023efficientguidedgenerationlarge, @@ -430,3 +430,11 @@ @misc{willard2023efficientguidedgenerationlarge primaryClass={cs.CL}, url={https://arxiv.org/abs/2307.09702}, } + +@misc{promptfoo2024, + title={promptfoo: LLM Testing and Evaluation Framework}, + author={promptfoo}, + year={2024}, + url={https://www.promptfoo.dev/}, + note={Open source framework for testing and evaluating LLM prompts} +}
      Table 3.1 Structured Output Frameworks Comparison