From 4869afc4d3e858344d7548405ce94dc432f07c93 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 28 Sep 2023 09:36:25 +0700 Subject: [PATCH 1/8] Fix 500 server error on project dashboard (#1779) Large projects were returning an HTTP 500 on the project dashboard, because PHP was running out of memory trying to return every entry for stats to be calculated client-side. This changes the stats to be calculated server-side instead. We get rid of the "number of entries with audio" stat because it's not possible to calculate in a simple Mongo query and would require serializing the entire entry database server-side, causing the very memory error we're trying to get rid of. --- .../projects/[project_code]/+page.svelte | 6 ---- .../projects/[project_code]/meta/+server.ts | 26 +++++----------- .../Languageforge/Lexicon/Dto/LexStatsDto.php | 25 +++++++++++++++ src/Api/Model/Shared/Mapper/MongoQueries.php | 31 +++++++++++++++++++ src/Api/Service/Sf.php | 3 +- 5 files changed, 65 insertions(+), 26 deletions(-) create mode 100644 src/Api/Model/Languageforge/Lexicon/Dto/LexStatsDto.php create mode 100644 src/Api/Model/Shared/Mapper/MongoQueries.php diff --git a/next-app/src/routes/projects/[project_code]/+page.svelte b/next-app/src/routes/projects/[project_code]/+page.svelte index 00eb8d52e4..130254988d 100644 --- a/next-app/src/routes/projects/[project_code]/+page.svelte +++ b/next-app/src/routes/projects/[project_code]/+page.svelte @@ -33,12 +33,6 @@ icon: NotesIcon, url: `/app/lexicon/${ project.id }`, }, - { - title: 'Entries with audio', - value: project.num_entries_with_audio, - icon: VoiceIcon, - url: `/app/lexicon/${ project.id }#!/editor/entry/000000?filterBy=Audio`, - }, { title: 'Entries with pictures', value: project.num_entries_with_pictures, diff --git a/next-app/src/routes/projects/[project_code]/meta/+server.ts b/next-app/src/routes/projects/[project_code]/meta/+server.ts index 35bc918f3d..c01e3a1dc9 100644 --- a/next-app/src/routes/projects/[project_code]/meta/+server.ts +++ b/next-app/src/routes/projects/[project_code]/meta/+server.ts @@ -9,8 +9,9 @@ type LegacyProjectDetails = { } type LegacyStats = { - entries: object[], - comments: Comment[], + num_entries, + num_entries_with_pictures, + num_unresolved_comments, } type Comment = { @@ -23,40 +24,27 @@ export type ProjectDetails = { name: string, num_users: number, num_entries: number, - num_entries_with_audio: number, num_entries_with_pictures: number, num_unresolved_comments?: number, } export async function fetch_project_details({ project_code, cookie }) { const { id, projectName: name, users }: LegacyProjectDetails = await sf({ name: 'set_project', args: [ project_code ], cookie }) - const { entries, comments }: LegacyStats = await sf({ name: 'lex_stats', cookie }) + const stats: LegacyStats = await sf({ name: 'lex_stats', cookie }) const details: ProjectDetails = { id, code: project_code, name, num_users: Object.keys(users).length, - num_entries: entries.length, - num_entries_with_audio: entries.filter(has_audio).length, - num_entries_with_pictures: entries.filter(has_picture).length, + num_entries: stats.num_entries, + num_entries_with_pictures: stats.num_entries_with_pictures, } const { role } = await fetch_current_user(cookie) if (can_view_comments(role)) { - const unresolved_comments = comments.filter(({ status }) => status !== 'resolved') - - details.num_unresolved_comments = unresolved_comments.length + details.num_unresolved_comments = stats.num_unresolved_comments } return details } - -function has_picture(entry: object) { - return JSON.stringify(entry).includes('"pictures":') -} - -// audio can be found in lots of places other than lexeme, ref impl used: https://github.com/sillsdev/web-languageforge/blob/develop/src/angular-app/bellows/core/offline/editor-data.service.ts#L523 -function has_audio(entry: object) { - return JSON.stringify(entry).includes('-audio":') // naming convention imposed by src/angular-app/languageforge/lexicon/settings/configuration/input-system-view.model.ts L81 -} diff --git a/src/Api/Model/Languageforge/Lexicon/Dto/LexStatsDto.php b/src/Api/Model/Languageforge/Lexicon/Dto/LexStatsDto.php new file mode 100644 index 0000000000..55a385797b --- /dev/null +++ b/src/Api/Model/Languageforge/Lexicon/Dto/LexStatsDto.php @@ -0,0 +1,25 @@ +databaseName()); + $num_entries = MongoQueries::countEntries($db, "lexicon"); + $num_entries_with_pictures = MongoQueries::countEntriesWithPictures($db, "lexicon"); + $num_unresolved_comments = MongoQueries::countUnresolvedComments($db, "lexiconComments"); + return [ + "num_entries" => $num_entries, + "num_entries_with_pictures" => $num_entries_with_pictures, + "num_unresolved_comments" => $num_unresolved_comments, + ]; + } +} diff --git a/src/Api/Model/Shared/Mapper/MongoQueries.php b/src/Api/Model/Shared/Mapper/MongoQueries.php new file mode 100644 index 0000000000..7f59f1d9cf --- /dev/null +++ b/src/Api/Model/Shared/Mapper/MongoQueries.php @@ -0,0 +1,31 @@ +selectCollection($collectionName); + return $coll->count(); + } + + public static function countEntriesWithPictures($db, $collectionName) + { + $coll = $db->selectCollection($collectionName); + $query = [ + "senses" => ['$exists' => true, '$ne' => []], + "senses.pictures" => ['$exists' => true, '$ne' => []], + ]; + return $coll->count($query); + } + + public static function countUnresolvedComments($db, $collectionName) + { + $coll = $db->selectCollection($collectionName); + $query = [ + "status" => ['$exists' => true, '$ne' => "resolved"], + ]; + return $coll->count($query); + } +} diff --git a/src/Api/Service/Sf.php b/src/Api/Service/Sf.php index d3ae5bc22c..97a51cdefe 100644 --- a/src/Api/Service/Sf.php +++ b/src/Api/Service/Sf.php @@ -14,6 +14,7 @@ use Api\Model\Languageforge\Lexicon\Dto\LexBaseViewDto; use Api\Model\Languageforge\Lexicon\Dto\LexDbeDto; use Api\Model\Languageforge\Lexicon\Dto\LexProjectDto; +use Api\Model\Languageforge\Lexicon\Dto\LexStatsDto; use Api\Model\Shared\Command\ProjectCommands; use Api\Model\Shared\Command\SessionCommands; use Api\Model\Shared\Command\UserCommands; @@ -518,7 +519,7 @@ public function lex_stats() $user = new UserModel($this->userId); if ($user->isMemberOfProject($this->projectId)) { - return LexDbeDto::encode($projectModel->id->asString(), $this->userId, 1); + return LexStatsDto::encode($projectModel); } throw new UserUnauthorizedException("User $this->userId is not a member of project $projectModel->projectCode"); From 79521beb4968f2e7a7ff2d89a3314d1d0babc7b8 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 28 Sep 2023 09:43:40 +0700 Subject: [PATCH 2/8] Fix missing PHP use statement in lex_stats --- src/Api/Model/Languageforge/Lexicon/Dto/LexStatsDto.php | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Api/Model/Languageforge/Lexicon/Dto/LexStatsDto.php b/src/Api/Model/Languageforge/Lexicon/Dto/LexStatsDto.php index 55a385797b..bee3c9f4df 100644 --- a/src/Api/Model/Languageforge/Lexicon/Dto/LexStatsDto.php +++ b/src/Api/Model/Languageforge/Lexicon/Dto/LexStatsDto.php @@ -2,6 +2,7 @@ namespace Api\Model\Languageforge\Lexicon\Dto; use Api\Model\Shared\Mapper\MongoQueries; +use Api\Model\Shared\Mapper\MongoStore; class LexStatsDto { From a33383741406cb1823c05a3f6e00d20f35d1ec4d Mon Sep 17 00:00:00 2001 From: Christopher Hirt Date: Mon, 2 Oct 2023 11:33:47 -0400 Subject: [PATCH 3/8] Update prettier to 3.0.3 and fix warnings (#1778) --- .github/workflows/pull-request.yml | 4 +- package-lock.json | 147 +++++++++--------- package.json | 4 +- .../languageforge/lexicon/editor/_editor.scss | 13 +- .../editor/field/_dc-multiparagraph.scss | 4 +- .../lexicon/editor/field/_dc-multitext.scss | 4 +- 6 files changed, 96 insertions(+), 80 deletions(-) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 2f5b7feda5..b0798c194a 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -69,4 +69,6 @@ jobs: cache: "npm" - name: Run prettier check - run: npx prettier --check . + run: | + npx prettier -v + npx prettier --check . diff --git a/package-lock.json b/package-lock.json index 451748e745..34e293a01e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -31,7 +31,7 @@ }, "devDependencies": { "@playwright/test": "^1.29", - "@prettier/plugin-php": "^0.19.1", + "@prettier/plugin-php": "^0.20.1", "@types/angular": "1.8.1", "@types/angular-mocks": "^1.7.0", "@types/angular-route": "^1.7.1", @@ -64,7 +64,7 @@ "ng-annotate-loader": "^0.7.0", "ngtemplate-loader": "^2.1.0", "npm-run-all": "^4.1.5", - "prettier": "2.7.1", + "prettier": "3.0.3", "sass": "^1.39.0", "sass-loader": "^8.0.2", "style-loader": "^1.2.1", @@ -1926,42 +1926,17 @@ "dev": true }, "node_modules/@prettier/plugin-php": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@prettier/plugin-php/-/plugin-php-0.19.1.tgz", - "integrity": "sha512-faoQX6QF4JZ0xJT+FRkMU7JnUTqVFRSwz7oKYuKgjY6u8VHSURlRXVkP4UyXxsTPAGjVJpNh7JseKkpK1DZziw==", + "version": "0.20.1", + "resolved": "https://registry.npmjs.org/@prettier/plugin-php/-/plugin-php-0.20.1.tgz", + "integrity": "sha512-CFn+NH44jb0buvg8D/DvLjeetadmNBMRq3yrgL8I+tXioDZlhQ+eBLLRILPEBIQ6jBmOjzm/Q7Qin8KD7raiFw==", "dev": true, "dependencies": { "linguist-languages": "^7.21.0", - "mem": "^8.0.0", - "php-parser": "^3.1.1" + "mem": "^9.0.2", + "php-parser": "^3.1.5" }, "peerDependencies": { - "prettier": "^1.15.0 || ^2.0.0" - } - }, - "node_modules/@prettier/plugin-php/node_modules/mem": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/mem/-/mem-8.1.1.tgz", - "integrity": "sha512-qFCFUDs7U3b8mBDPyz5EToEKoAkgCzqquIgi9nkkR9bixxOVOre+09lbuH7+9Kn2NFpm56M3GUWVbU2hQgdACA==", - "dev": true, - "dependencies": { - "map-age-cleaner": "^0.1.3", - "mimic-fn": "^3.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sindresorhus/mem?sponsor=1" - } - }, - "node_modules/@prettier/plugin-php/node_modules/mimic-fn": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-3.1.0.tgz", - "integrity": "sha512-Ysbi9uYW9hFyfrThdDEQuykN4Ey6BuwPD2kpI5ES/nFTDn/98yxYNLZJcgUAKPT/mcrLLKaGzJR9YVxJrIdASQ==", - "dev": true, - "engines": { - "node": ">=8" + "prettier": "^3.0.0" } }, "node_modules/@rollup/plugin-babel": { @@ -6265,6 +6240,34 @@ "node": ">= 0.6" } }, + "node_modules/mem": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/mem/-/mem-9.0.2.tgz", + "integrity": "sha512-F2t4YIv9XQUBHt6AOJ0y7lSmP1+cY7Fm1DRh9GClTGzKST7UWLMx6ly9WZdLH/G/ppM5RL4MlQfRT71ri9t19A==", + "dev": true, + "dependencies": { + "map-age-cleaner": "^0.1.3", + "mimic-fn": "^4.0.0" + }, + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sindresorhus/mem?sponsor=1" + } + }, + "node_modules/mem/node_modules/mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/memfs": { "version": "3.4.1", "resolved": "https://registry.npmjs.org/memfs/-/memfs-3.4.1.tgz", @@ -6938,7 +6941,7 @@ "node_modules/p-defer": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", - "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=", + "integrity": "sha512-wB3wfAxZpk2AzOfUMJNL+d36xothRSyj8EXOa4f6GMqYDN9BJaaSISbsk+wS9abmnebVw95C2Kb5t85UmpCxuw==", "dev": true, "engines": { "node": ">=4" @@ -7094,9 +7097,9 @@ } }, "node_modules/php-parser": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/php-parser/-/php-parser-3.1.1.tgz", - "integrity": "sha512-HUxWIWpJoGhnSVzM6nPI1O2RePd7eJKzJoL3VZr6/KUUdcHKBex2Cp7p6pWOV1WxgKI/oYgRPMywwLCP789OYA==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/php-parser/-/php-parser-3.1.5.tgz", + "integrity": "sha512-jEY2DcbgCm5aclzBdfW86GM6VEIWcSlhTBSHN1qhJguVePlYe28GhwS0yoeLYXpM2K8y6wzLwrbq814n2PHSoQ==", "dev": true }, "node_modules/picocolors": { @@ -7294,15 +7297,15 @@ } }, "node_modules/prettier": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.7.1.tgz", - "integrity": "sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.0.3.tgz", + "integrity": "sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg==", "dev": true, "bin": { - "prettier": "bin-prettier.js" + "prettier": "bin/prettier.cjs" }, "engines": { - "node": ">=10.13.0" + "node": ">=14" }, "funding": { "url": "https://github.com/prettier/prettier?sponsor=1" @@ -11743,32 +11746,14 @@ "dev": true }, "@prettier/plugin-php": { - "version": "0.19.1", - "resolved": "https://registry.npmjs.org/@prettier/plugin-php/-/plugin-php-0.19.1.tgz", - "integrity": "sha512-faoQX6QF4JZ0xJT+FRkMU7JnUTqVFRSwz7oKYuKgjY6u8VHSURlRXVkP4UyXxsTPAGjVJpNh7JseKkpK1DZziw==", + "version": "0.20.1", + "resolved": "https://registry.npmjs.org/@prettier/plugin-php/-/plugin-php-0.20.1.tgz", + "integrity": "sha512-CFn+NH44jb0buvg8D/DvLjeetadmNBMRq3yrgL8I+tXioDZlhQ+eBLLRILPEBIQ6jBmOjzm/Q7Qin8KD7raiFw==", "dev": true, "requires": { "linguist-languages": "^7.21.0", - "mem": "^8.0.0", - "php-parser": "^3.1.1" - }, - "dependencies": { - "mem": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/mem/-/mem-8.1.1.tgz", - "integrity": "sha512-qFCFUDs7U3b8mBDPyz5EToEKoAkgCzqquIgi9nkkR9bixxOVOre+09lbuH7+9Kn2NFpm56M3GUWVbU2hQgdACA==", - "dev": true, - "requires": { - "map-age-cleaner": "^0.1.3", - "mimic-fn": "^3.1.0" - } - }, - "mimic-fn": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-3.1.0.tgz", - "integrity": "sha512-Ysbi9uYW9hFyfrThdDEQuykN4Ey6BuwPD2kpI5ES/nFTDn/98yxYNLZJcgUAKPT/mcrLLKaGzJR9YVxJrIdASQ==", - "dev": true - } + "mem": "^9.0.2", + "php-parser": "^3.1.5" } }, "@rollup/plugin-babel": { @@ -15098,6 +15083,24 @@ "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", "dev": true }, + "mem": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/mem/-/mem-9.0.2.tgz", + "integrity": "sha512-F2t4YIv9XQUBHt6AOJ0y7lSmP1+cY7Fm1DRh9GClTGzKST7UWLMx6ly9WZdLH/G/ppM5RL4MlQfRT71ri9t19A==", + "dev": true, + "requires": { + "map-age-cleaner": "^0.1.3", + "mimic-fn": "^4.0.0" + }, + "dependencies": { + "mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "dev": true + } + } + }, "memfs": { "version": "3.4.1", "resolved": "https://registry.npmjs.org/memfs/-/memfs-3.4.1.tgz", @@ -15628,7 +15631,7 @@ "p-defer": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", - "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=", + "integrity": "sha512-wB3wfAxZpk2AzOfUMJNL+d36xothRSyj8EXOa4f6GMqYDN9BJaaSISbsk+wS9abmnebVw95C2Kb5t85UmpCxuw==", "dev": true }, "p-limit": { @@ -15741,9 +15744,9 @@ "dev": true }, "php-parser": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/php-parser/-/php-parser-3.1.1.tgz", - "integrity": "sha512-HUxWIWpJoGhnSVzM6nPI1O2RePd7eJKzJoL3VZr6/KUUdcHKBex2Cp7p6pWOV1WxgKI/oYgRPMywwLCP789OYA==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/php-parser/-/php-parser-3.1.5.tgz", + "integrity": "sha512-jEY2DcbgCm5aclzBdfW86GM6VEIWcSlhTBSHN1qhJguVePlYe28GhwS0yoeLYXpM2K8y6wzLwrbq814n2PHSoQ==", "dev": true }, "picocolors": { @@ -15894,9 +15897,9 @@ "dev": true }, "prettier": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.7.1.tgz", - "integrity": "sha512-ujppO+MkdPqoVINuDFDRLClm7D78qbDt0/NR+wp5FqEZOoTNAjPHWj17QRhu7geIHJfcNhRk1XVQmF8Bp3ye+g==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.0.3.tgz", + "integrity": "sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg==", "dev": true }, "pretty-bytes": { diff --git a/package.json b/package.json index 8bbbbf0ae5..d86a7f9a2e 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ }, "devDependencies": { "@playwright/test": "^1.29", - "@prettier/plugin-php": "^0.19.1", + "@prettier/plugin-php": "^0.20.1", "@types/angular": "1.8.1", "@types/angular-mocks": "^1.7.0", "@types/angular-route": "^1.7.1", @@ -80,7 +80,7 @@ "ng-annotate-loader": "^0.7.0", "ngtemplate-loader": "^2.1.0", "npm-run-all": "^4.1.5", - "prettier": "2.7.1", + "prettier": "3.0.3", "sass": "^1.39.0", "sass-loader": "^8.0.2", "style-loader": "^1.2.1", diff --git a/src/angular-app/languageforge/lexicon/editor/_editor.scss b/src/angular-app/languageforge/lexicon/editor/_editor.scss index 7e6669ecc2..e261d4d470 100644 --- a/src/angular-app/languageforge/lexicon/editor/_editor.scss +++ b/src/angular-app/languageforge/lexicon/editor/_editor.scss @@ -468,7 +468,9 @@ dc-entry .card { } .right-panel-list { width: 0; - transition: width 0.5s ease-in-out, margin-left 0.5s ease-in-out; + transition: + width 0.5s ease-in-out, + margin-left 0.5s ease-in-out; margin-left: 0; position: relative; .right-panel { @@ -479,7 +481,10 @@ dc-entry .card { top: 0; width: 100%; &.panel-closing { - transition: width 0.5s ease-in-out 0.5s, margin-left 0.5s ease-in-out 0.5s, opacity 0.5s; + transition: + width 0.5s ease-in-out 0.5s, + margin-left 0.5s ease-in-out 0.5s, + opacity 0.5s; overflow: visible !important; height: auto !important; } @@ -507,7 +512,9 @@ dc-entry .card { } &.panel-closing { .right-panel-list { - transition: width 0.5s ease-in-out 0.5s, margin-left 0.5s ease-in-out 0.5s; + transition: + width 0.5s ease-in-out 0.5s, + margin-left 0.5s ease-in-out 0.5s; overflow: visible !important; height: auto !important; } diff --git a/src/angular-app/languageforge/lexicon/editor/field/_dc-multiparagraph.scss b/src/angular-app/languageforge/lexicon/editor/field/_dc-multiparagraph.scss index f049edf433..2008c4b010 100644 --- a/src/angular-app/languageforge/lexicon/editor/field/_dc-multiparagraph.scss +++ b/src/angular-app/languageforge/lexicon/editor/field/_dc-multiparagraph.scss @@ -34,7 +34,9 @@ form.dc-multiparagraph textarea { form.dc-multiparagraph textarea:focus { border-color: rgba(82, 168, 236, 0.8); outline: 0; - box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6); + box-shadow: + inset 0 1px 1px rgba(0, 0, 0, 0.075), + 0 0 8px rgba(82, 168, 236, 0.6); } /* input prepend sized to input and textarea */ diff --git a/src/angular-app/languageforge/lexicon/editor/field/_dc-multitext.scss b/src/angular-app/languageforge/lexicon/editor/field/_dc-multitext.scss index 3a1b660453..bb7306b3f3 100644 --- a/src/angular-app/languageforge/lexicon/editor/field/_dc-multitext.scss +++ b/src/angular-app/languageforge/lexicon/editor/field/_dc-multitext.scss @@ -28,7 +28,9 @@ form.dc-multitext .uneditable-input:hover { form.dc-multitext textarea:focus { border-color: rgba(82, 168, 236, 0.8); outline: 0; - box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px rgba(82, 168, 236, 0.6); + box-shadow: + inset 0 1px 1px rgba(0, 0, 0, 0.075), + 0 0 8px rgba(82, 168, 236, 0.6); } form.dc-multitext .wsid { From d7420341abbd1798f25514a2cbd5d81885bd3ee9 Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Wed, 8 Nov 2023 11:58:48 +0700 Subject: [PATCH 4/8] Add liveness probe for MongoDB container (#1784) This will allow k8s to reboot the pod if MongoDB becomes unresponsive. --- docker/deployment/db-deployment.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docker/deployment/db-deployment.yaml b/docker/deployment/db-deployment.yaml index 606803960f..71d945541b 100644 --- a/docker/deployment/db-deployment.yaml +++ b/docker/deployment/db-deployment.yaml @@ -51,6 +51,18 @@ spec: - name: db image: mongo:6 # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers + livenessProbe: + exec: + command: + - mongosh + - '--quiet' + - '--eval' + - db.runCommand('ping') + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 resources: requests: memory: 260Mi From 4463f0ada0631efe97394652c2045c0b183fefaa Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Wed, 8 Nov 2023 02:16:30 -0700 Subject: [PATCH 5/8] use kustomize instead of sed for environment configuration (#1785) * add a forward-db make command * move k8s resources into base folder * refactor k8s resources to use kustomize. --- docker/deployment/Makefile | 36 ++++-------------- docker/deployment/base/app-config.yaml | 7 ++++ .../deployment/{ => base}/app-deployment.yaml | 37 +++---------------- .../deployment/{ => base}/db-deployment.yaml | 0 docker/deployment/base/ingress-config.yaml | 9 +++++ docker/deployment/base/kustomization.yaml | 13 +++++++ .../{ => base}/lfmerge-deployment.yaml | 37 +++---------------- docker/deployment/base/lfmerge-pvcs.yaml | 28 ++++++++++++++ .../{ => base}/mail-deployment.yaml | 5 ++- .../{ => base}/next-app-deployment.yaml | 4 +- .../{ => base}/next-proxy-deployment.yaml | 8 ++-- docker/deployment/{ => base}/secrets.yaml | 0 docker/deployment/prod/app-config-patch.yaml | 8 ++++ .../deployment/prod/ingress-config-patch.yaml | 9 +++++ docker/deployment/prod/kustomization.yaml | 30 +++++++++++++++ .../deployment/prod/lfmerge-pvcs-patch.yaml | 8 ++++ .../deployment/staging/app-config-patch.yaml | 7 ++++ .../staging/ingress-config-patch.yaml | 9 +++++ docker/deployment/staging/kustomization.yaml | 30 +++++++++++++++ .../deployment/staging/mongo-pvc-patch.yaml | 6 +++ docs/RELEASE.md | 8 +++- 21 files changed, 202 insertions(+), 97 deletions(-) create mode 100644 docker/deployment/base/app-config.yaml rename docker/deployment/{ => base}/app-deployment.yaml (83%) rename docker/deployment/{ => base}/db-deployment.yaml (100%) create mode 100644 docker/deployment/base/ingress-config.yaml create mode 100644 docker/deployment/base/kustomization.yaml rename docker/deployment/{ => base}/lfmerge-deployment.yaml (82%) create mode 100644 docker/deployment/base/lfmerge-pvcs.yaml rename docker/deployment/{ => base}/mail-deployment.yaml (92%) rename docker/deployment/{ => base}/next-app-deployment.yaml (79%) rename docker/deployment/{ => base}/next-proxy-deployment.yaml (85%) rename docker/deployment/{ => base}/secrets.yaml (100%) create mode 100644 docker/deployment/prod/app-config-patch.yaml create mode 100644 docker/deployment/prod/ingress-config-patch.yaml create mode 100644 docker/deployment/prod/kustomization.yaml create mode 100644 docker/deployment/prod/lfmerge-pvcs-patch.yaml create mode 100644 docker/deployment/staging/app-config-patch.yaml create mode 100644 docker/deployment/staging/ingress-config-patch.yaml create mode 100644 docker/deployment/staging/kustomization.yaml create mode 100644 docker/deployment/staging/mongo-pvc-patch.yaml diff --git a/docker/deployment/Makefile b/docker/deployment/Makefile index e5fb2c9115..7e2b49a0a9 100644 --- a/docker/deployment/Makefile +++ b/docker/deployment/Makefile @@ -35,40 +35,20 @@ shell-next-app: shell-next-proxy: kubectl exec -it deploy/next-proxy -- sh +forward-db: + kubectl port-forward pod/db-6b494cd89f-whfr7 27018:27017 -n languageforge + init-secrets: kubectl apply -f secrets.yaml create-new-deployment-mail: kubectl create deployment mail --image=juanluisbaptiste/postfix:1.0.0 --dry-run=client -o yaml > mail-deployment-new.yaml -deploy-staging: deploy-db deploy-mail-staging deploy-app-staging deploy-lfmerge-staging deploy-next-proxy-staging deploy-next-app-staging -deploy-mail-staging: - sed -e s/{{SERVER_HOSTNAME}}/staging.languageforge.org/ mail-deployment.yaml | kubectl apply -f - -deploy-app-staging: - sed -e s/{{WEBSITE}}/staging.languageforge.org/ app-deployment.yaml \ - | sed -e s/{{VERSION}}/$(VERSION_APP)/ | kubectl apply -f - -deploy-lfmerge-staging: - sed -e s/{{VERSION_LFMERGE}}/$(VERSION_LFMERGE)/ lfmerge-deployment.yaml | kubectl apply -f - -deploy-next-proxy-staging: - sed -e s/{{WEBSITE}}/staging.languageforge.org/ next-proxy-deployment.yaml \ - | sed -e s/{{VERSION}}/$(VERSION_PROXY)/ | kubectl apply -f - -deploy-next-app-staging: - sed -e s/{{VERSION}}/$(VERSION_NEXT_APP)/ next-app-deployment.yaml | kubectl apply -f - -deploy-prod: deploy-db deploy-mail-prod deploy-app-prod deploy-lfmerge-prod deploy-next-proxy-prod deploy-next-app-prod -deploy-mail-prod: - sed -e s/{{SERVER_HOSTNAME}}/languageforge.org/ mail-deployment.yaml | kubectl apply -f - -deploy-app-prod: - sed -e s/{{WEBSITE}}/languageforge.org/ app-deployment.yaml \ - | sed -e s/{{VERSION}}/$(VERSION_APP)/ | kubectl apply -f - -deploy-lfmerge-prod: - sed -e s/{{VERSION_LFMERGE}}/$(VERSION_LFMERGE)/ lfmerge-deployment.yaml | kubectl apply -f - -deploy-next-proxy-prod: - sed -e s/{{WEBSITE}}/languageforge.org/ next-proxy-deployment.yaml \ - | sed -e s/{{VERSION}}/$(VERSION_PROXY)/ | kubectl apply -f - -deploy-next-app-prod: - sed -e s/{{VERSION}}/$(VERSION_NEXT_APP)/ next-app-deployment.yaml | kubectl apply -f - -deploy-db: - kubectl apply -f db-deployment.yaml +deploy-staging: + kubectl --context dallas-rke apply -k staging/ +deploy-prod: + kubectl --context aws-rke apply -k prod/ + delete: delete-app delete-lfmerge delete-mail delete-db delete-next-proxy delete-next-app delete-db: # does NOT delete the volume, i.e., the data in the database diff --git a/docker/deployment/base/app-config.yaml b/docker/deployment/base/app-config.yaml new file mode 100644 index 0000000000..52628a97db --- /dev/null +++ b/docker/deployment/base/app-config.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-config +data: + website: "localhost" + hg-domain: "hg-staging.languagforge.org" diff --git a/docker/deployment/app-deployment.yaml b/docker/deployment/base/app-deployment.yaml similarity index 83% rename from docker/deployment/app-deployment.yaml rename to docker/deployment/base/app-deployment.yaml index 9b48d58185..7c743d15e5 100644 --- a/docker/deployment/app-deployment.yaml +++ b/docker/deployment/base/app-deployment.yaml @@ -18,35 +18,6 @@ spec: --- -# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: lf-project-assets -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 10Gi - storageClassName: weekly-snapshots-retain-4 # provided by LTOps - ---- - -# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: lfmerge-sendreceive-data -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 50Gi - storageClassName: weekly-snapshots-retain-4 # provided by LTOps - ---- # https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#writing-a-deployment-spec apiVersion: apps/v1 @@ -93,7 +64,7 @@ spec: name: sendreceive-data containers: - name: app - image: sillsdev/web-languageforge:{{VERSION}} + image: sillsdev/web-languageforge:latest imagePullPolicy: Always # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers resources: @@ -113,7 +84,10 @@ spec: - name: ENVIRONMENT value: production - name: WEBSITE - value: {{WEBSITE}} + valueFrom: + configMapKeyRef: + name: app-config + key: website - name: MAIL_HOST value: mail - name: LFMERGE_LOGGING_DEST @@ -143,6 +117,7 @@ spec: secretKeyRef: key: LEX_BOX_HOST name: ld-api + optional: true - name: FACEBOOK_CLIENT_ID valueFrom: secretKeyRef: diff --git a/docker/deployment/db-deployment.yaml b/docker/deployment/base/db-deployment.yaml similarity index 100% rename from docker/deployment/db-deployment.yaml rename to docker/deployment/base/db-deployment.yaml diff --git a/docker/deployment/base/ingress-config.yaml b/docker/deployment/base/ingress-config.yaml new file mode 100644 index 0000000000..b538b6b6ce --- /dev/null +++ b/docker/deployment/base/ingress-config.yaml @@ -0,0 +1,9 @@ +# template, copy into env folder and change the values, then add as a patch to the kustomization.yaml file + +- op: replace + path: /spec/rules/0/host + value: localhost +- op: replace + path: /spec/tls/0/hosts + value: + - localhost diff --git a/docker/deployment/base/kustomization.yaml b/docker/deployment/base/kustomization.yaml new file mode 100644 index 0000000000..4dd8371403 --- /dev/null +++ b/docker/deployment/base/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: languageforge + +resources: + - app-deployment.yaml + - db-deployment.yaml + - lfmerge-deployment.yaml + - lfmerge-pvcs.yaml + - mail-deployment.yaml + - next-app-deployment.yaml + - next-proxy-deployment.yaml + - app-config.yaml diff --git a/docker/deployment/lfmerge-deployment.yaml b/docker/deployment/base/lfmerge-deployment.yaml similarity index 82% rename from docker/deployment/lfmerge-deployment.yaml rename to docker/deployment/base/lfmerge-deployment.yaml index 7d9ed1d4f7..389da7e859 100644 --- a/docker/deployment/lfmerge-deployment.yaml +++ b/docker/deployment/base/lfmerge-deployment.yaml @@ -18,36 +18,6 @@ spec: --- -# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: lf-project-assets -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 10Gi - storageClassName: weekly-snapshots-retain-4 # provided by LTOps - ---- - -# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: lfmerge-sendreceive-data -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 50Gi - storageClassName: weekly-snapshots-retain-4 # provided by LTOps - ---- - # https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#writing-a-deployment-spec apiVersion: apps/v1 kind: Deployment @@ -100,7 +70,7 @@ spec: name: sendreceive-data containers: - name: lfmerge - image: ghcr.io/sillsdev/lfmerge:{{VERSION_LFMERGE}} + image: ghcr.io/sillsdev/lfmerge:latest imagePullPolicy: Always # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers resources: @@ -133,6 +103,11 @@ spec: value: sf_ - name: LFMERGE_VERBOSE_PROGRESS value: "true" + - name: LFMERGE_LANGUAGE_DEPOT_HG_PUBLIC_HOSTNAME + valueFrom: + configMapKeyRef: + name: app-config + key: hg-domain - name: LANGUAGE_DEPOT_TRUST_TOKEN valueFrom: secretKeyRef: diff --git a/docker/deployment/base/lfmerge-pvcs.yaml b/docker/deployment/base/lfmerge-pvcs.yaml new file mode 100644 index 0000000000..d814952b50 --- /dev/null +++ b/docker/deployment/base/lfmerge-pvcs.yaml @@ -0,0 +1,28 @@ + +# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: lf-project-assets +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi + storageClassName: weekly-snapshots-retain-4 # provided by LTOps + +--- + +# https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: lfmerge-sendreceive-data +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 50Gi + storageClassName: weekly-snapshots-retain-4 # provided by LTOps diff --git a/docker/deployment/mail-deployment.yaml b/docker/deployment/base/mail-deployment.yaml similarity index 92% rename from docker/deployment/mail-deployment.yaml rename to docker/deployment/base/mail-deployment.yaml index 2474834c21..405294e579 100644 --- a/docker/deployment/mail-deployment.yaml +++ b/docker/deployment/base/mail-deployment.yaml @@ -41,7 +41,10 @@ spec: memory: 100Mi env: - name: SERVER_HOSTNAME - value: {{SERVER_HOSTNAME}} + valueFrom: + configMapKeyRef: + name: app-config + key: website - name: SMTP_SERVER valueFrom: secretKeyRef: diff --git a/docker/deployment/next-app-deployment.yaml b/docker/deployment/base/next-app-deployment.yaml similarity index 79% rename from docker/deployment/next-app-deployment.yaml rename to docker/deployment/base/next-app-deployment.yaml index 312759286b..23025cc854 100644 --- a/docker/deployment/next-app-deployment.yaml +++ b/docker/deployment/base/next-app-deployment.yaml @@ -37,7 +37,9 @@ spec: spec: containers: - name: next-app - image: sillsdev/web-languageforge:{{VERSION}} + # this image doesn't actually exist as for some reason tags were used for different image types instead of just version number. + # this is a workaround because kustomize is designed to work with image names + image: sillsdev/web-languageforge-next-app:latest imagePullPolicy: Always # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers resources: diff --git a/docker/deployment/next-proxy-deployment.yaml b/docker/deployment/base/next-proxy-deployment.yaml similarity index 85% rename from docker/deployment/next-proxy-deployment.yaml rename to docker/deployment/base/next-proxy-deployment.yaml index 6978b0bbba..30d1fa7130 100644 --- a/docker/deployment/next-proxy-deployment.yaml +++ b/docker/deployment/base/next-proxy-deployment.yaml @@ -9,7 +9,7 @@ metadata: nginx.ingress.kubernetes.io/proxy-body-size: 60M spec: rules: - - host: {{WEBSITE}} + - host: localhost http: paths: - path: / @@ -21,7 +21,7 @@ spec: number: 80 tls: - hosts: - - {{WEBSITE}} + - localhost secretName: languageforge-tls --- @@ -64,7 +64,9 @@ spec: spec: containers: - name: next-proxy - image: sillsdev/web-languageforge:{{VERSION}} + # this image doesn't actually exist as for some reason tags were used for different image types instead of just version number. + # this is a workaround because kustomize is designed to work with image names + image: sillsdev/web-languageforge-next-proxy:latest # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers imagePullPolicy: Always resources: diff --git a/docker/deployment/secrets.yaml b/docker/deployment/base/secrets.yaml similarity index 100% rename from docker/deployment/secrets.yaml rename to docker/deployment/base/secrets.yaml diff --git a/docker/deployment/prod/app-config-patch.yaml b/docker/deployment/prod/app-config-patch.yaml new file mode 100644 index 0000000000..0632fb7327 --- /dev/null +++ b/docker/deployment/prod/app-config-patch.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-config +data: + website: "languageforge.org" + hg-domain: "hg-public.languagforge.org" + diff --git a/docker/deployment/prod/ingress-config-patch.yaml b/docker/deployment/prod/ingress-config-patch.yaml new file mode 100644 index 0000000000..e4d0ba374e --- /dev/null +++ b/docker/deployment/prod/ingress-config-patch.yaml @@ -0,0 +1,9 @@ +# template, copy into env folder and change the values, then add as a patch to the kustomization.yaml file + +- op: replace + path: /spec/rules/0/host + value: languageforge.org +- op: replace + path: /spec/tls/0/hosts + value: + - languageforge.org diff --git a/docker/deployment/prod/kustomization.yaml b/docker/deployment/prod/kustomization.yaml new file mode 100644 index 0000000000..1cb87c21c9 --- /dev/null +++ b/docker/deployment/prod/kustomization.yaml @@ -0,0 +1,30 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: languageforge + +resources: + - ../base + +# both next app and next proxy require their own images, but the image name web-languageforge was reused for them +# with a different tag to select the correct image instead of making a new image name +# because of that we have to do a bit of a workaround to allow setting the image tag for the correct container +images: + - name: sillsdev/web-languageforge + newTag: 2023-09-21 + - name: sillsdev/web-languageforge-next-proxy + newName: sillsdev/web-languageforge + newTag: next-proxy-2023-09-21 + - name: sillsdev/web-languageforge-next-app + newName: sillsdev/web-languageforge + newTag: next-app-2023-09-21 + - name: ghcr.io/sillsdev/lfmerge + newTag: 2.0.135 + +patches: + - path: app-config-patch.yaml + - path: lfmerge-pvcs-patch.yaml + - path: ingress-config-patch.yaml + target: + kind: Ingress + name: languageforge-app + namespace: languageforge diff --git a/docker/deployment/prod/lfmerge-pvcs-patch.yaml b/docker/deployment/prod/lfmerge-pvcs-patch.yaml new file mode 100644 index 0000000000..ba5fbcb10c --- /dev/null +++ b/docker/deployment/prod/lfmerge-pvcs-patch.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: lfmerge-sendreceive-data +spec: + resources: + requests: + storage: 70Gi diff --git a/docker/deployment/staging/app-config-patch.yaml b/docker/deployment/staging/app-config-patch.yaml new file mode 100644 index 0000000000..8137b6be86 --- /dev/null +++ b/docker/deployment/staging/app-config-patch.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: app-config +data: + website: "staging.languageforge.org" + diff --git a/docker/deployment/staging/ingress-config-patch.yaml b/docker/deployment/staging/ingress-config-patch.yaml new file mode 100644 index 0000000000..4bde103633 --- /dev/null +++ b/docker/deployment/staging/ingress-config-patch.yaml @@ -0,0 +1,9 @@ +# template, copy into env folder and change the values, then add as a patch to the kustomization.yaml file + +- op: replace + path: /spec/rules/0/host + value: staging.languageforge.org +- op: replace + path: /spec/tls/0/hosts + value: + - staging.languageforge.org diff --git a/docker/deployment/staging/kustomization.yaml b/docker/deployment/staging/kustomization.yaml new file mode 100644 index 0000000000..6af7b0242a --- /dev/null +++ b/docker/deployment/staging/kustomization.yaml @@ -0,0 +1,30 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: languageforge + +resources: + - ../base + +# both next app and next proxy require their own images, but the image name web-languageforge was reused for them +# with a different tag to select the correct image instead of making a new image name +# because of that we have to do a bit of a workaround to allow setting the image tag for the correct container +images: + - name: sillsdev/web-languageforge + newTag: develop-20231002-a33383741406cb1823c05a3f6e00d20f35d1ec4d + - name: sillsdev/web-languageforge-next-proxy + newName: sillsdev/web-languageforge + newTag: develop-next-proxy-20231002-a33383741406cb1823c05a3f6e00d20f35d1ec4d + - name: sillsdev/web-languageforge-next-app + newName: sillsdev/web-languageforge + newTag: develop-next-app-20231002-a33383741406cb1823c05a3f6e00d20f35d1ec4d + - name: ghcr.io/sillsdev/lfmerge + newTag: 2.0.135 + +patches: + - path: app-config-patch.yaml + - path: mongo-pvc-patch.yaml + - path: ingress-config-patch.yaml + target: + kind: Ingress + name: languageforge-app + namespace: languageforge diff --git a/docker/deployment/staging/mongo-pvc-patch.yaml b/docker/deployment/staging/mongo-pvc-patch.yaml new file mode 100644 index 0000000000..9f55986f6d --- /dev/null +++ b/docker/deployment/staging/mongo-pvc-patch.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: lf-mongo-data +spec: + storageClassName: xfs-weekly-snapshots-retain-4 # provided by LTOps diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 2c0cc81f42..d303754d32 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -19,7 +19,9 @@ Current workflow: 1. merge PR into or make commits on `develop` branch 1. this will kick off the GHA (`.github/workflows/staging.yml`) to build, test and publish the necessary images to Docker Hub (https://hub.docker.com/r/sillsdev/web-languageforge/tags) and deploy this code to the staging environment. -Staging deployments can be manually run with `VERSION_APP= VERSION_PROXY= VERSION_NEXT_APP= VERSION_LFMERGE= make deploy-staging`. +Update the image tags in `staging/kustomization.yaml` + +Staging deployments can be manually run with `make deploy-staging`. ### Production @@ -32,7 +34,9 @@ Current workflow: 1. "Publish" the new release 1. this will kick off the GHA (`.github/workflows/production.yml`) to build, test and publish the necessary images to Docker Hub (https://hub.docker.com/r/sillsdev/web-languageforge/tags) and deploy this code to the production environment at https://languageforge.org -Production deployments can be manually run with `VERSION_APP= VERSION_PROXY= VERSION_NEXT_APP= VERSION_LFMERGE= make deploy-prod`. +Update the image tags in `prod/kustomization.yaml` + +Production deployments can be manually run with `make deploy-prod`. ### Revert From 7256a1d48b45b6ea05afc954344911e786e4f61d Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Wed, 8 Nov 2023 16:30:21 +0700 Subject: [PATCH 6/8] update RELEASE.md to mention expected context names --- docs/RELEASE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/RELEASE.md b/docs/RELEASE.md index d303754d32..23d7ae5848 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -22,6 +22,7 @@ Current workflow: Update the image tags in `staging/kustomization.yaml` Staging deployments can be manually run with `make deploy-staging`. +Note, this command assumes that the staging k8s context is named dallas-rke ### Production @@ -37,6 +38,7 @@ Current workflow: Update the image tags in `prod/kustomization.yaml` Production deployments can be manually run with `make deploy-prod`. +Note, this command assumes that the staging k8s context is named aws-rke ### Revert From 1b8ddeb5e8ff4defd1aba6849b2b27c6a972520e Mon Sep 17 00:00:00 2001 From: Robin Munn Date: Thu, 9 Nov 2023 15:00:47 +0700 Subject: [PATCH 7/8] Add reset cache button to advanced project configuration tab (#1781) * Cache service can now clear entire cache * Add UI for clearing browser storage cache Added a button to the Advanced Configuration section to clear the storage cache, and a short paragraph explaining what it does and why and when one might want to do that. * Make cursor be a pointer on cache reload button * Refresh page after cache reload button clicked --- .../offline/comments-offline-cache.service.ts | 4 ++++ .../offline/editor-offline-cache.service.ts | 4 ++++ .../core/offline/offline-cache.service.ts | 4 ++++ ...onfiguration-advanced-options.component.html | 11 +++++++++++ .../configuration-advanced-options.component.ts | 17 +++++++++++++++-- 5 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/angular-app/bellows/core/offline/comments-offline-cache.service.ts b/src/angular-app/bellows/core/offline/comments-offline-cache.service.ts index dbdb7b35c2..2573317bd7 100644 --- a/src/angular-app/bellows/core/offline/comments-offline-cache.service.ts +++ b/src/angular-app/bellows/core/offline/comments-offline-cache.service.ts @@ -13,6 +13,10 @@ export class CommentsOfflineCacheService { return this.offlineCache.deleteObjectInStore('comments', id); } + deleteAllComments(): angular.IPromise { + return this.offlineCache.clearEntireStore('comments'); + } + getAllComments(): angular.IPromise { return this.offlineCache.getAllFromStore('comments', this.sessionService.projectId()); } diff --git a/src/angular-app/bellows/core/offline/editor-offline-cache.service.ts b/src/angular-app/bellows/core/offline/editor-offline-cache.service.ts index 6f1f9a23f8..a896b1693c 100644 --- a/src/angular-app/bellows/core/offline/editor-offline-cache.service.ts +++ b/src/angular-app/bellows/core/offline/editor-offline-cache.service.ts @@ -15,6 +15,10 @@ export class EditorOfflineCacheService { return this.offlineCache.deleteObjectInStore('entries', id); } + deleteAllEntries(): angular.IPromise { + return this.offlineCache.clearEntireStore('entries'); + } + getAllEntries(): angular.IPromise { return this.offlineCache.getAllFromStore('entries', this.sessionService.projectId()); } diff --git a/src/angular-app/bellows/core/offline/offline-cache.service.ts b/src/angular-app/bellows/core/offline/offline-cache.service.ts index 9c86d249cf..1655e5d3a6 100644 --- a/src/angular-app/bellows/core/offline/offline-cache.service.ts +++ b/src/angular-app/bellows/core/offline/offline-cache.service.ts @@ -20,6 +20,10 @@ export class OfflineCacheService { return this.$q.when(this.getStore(storeName).removeItem(key)); } + clearEntireStore(storeName: string): angular.IPromise { + return this.$q.when(this.getStore(storeName).clear()); + } + getAllFromStore(storeName: string, projectId?: string): angular.IPromise { const results: any[] = []; return this.$q.when(this.getStore(storeName).iterate(value => { diff --git a/src/angular-app/languageforge/lexicon/settings/configuration/configuration-advanced-options.component.html b/src/angular-app/languageforge/lexicon/settings/configuration/configuration-advanced-options.component.html index 9ae47aa4a3..7db003224e 100644 --- a/src/angular-app/languageforge/lexicon/settings/configuration/configuration-advanced-options.component.html +++ b/src/angular-app/languageforge/lexicon/settings/configuration/configuration-advanced-options.component.html @@ -4,5 +4,16 @@

Caution!

Please do not change these settings unless asked to by tech support.

+
+

If something went wrong with downloading the dictionary and it only loaded a partial copy (i.e. some dictionary entries are missing), the button below can let you reset the browser storage, removing the partial copy of the dictionary stored in your browser so that you can download it again.

+

CAUTION: Make sure you are online before doing this, otherwise you won't be able to redownload the dictionary and you'll see an empty project until you can go online again.

+ diff --git a/src/angular-app/languageforge/lexicon/settings/configuration/configuration-advanced-options.component.ts b/src/angular-app/languageforge/lexicon/settings/configuration/configuration-advanced-options.component.ts index 69e697d578..009442cc44 100644 --- a/src/angular-app/languageforge/lexicon/settings/configuration/configuration-advanced-options.component.ts +++ b/src/angular-app/languageforge/lexicon/settings/configuration/configuration-advanced-options.component.ts @@ -1,12 +1,18 @@ import * as angular from 'angular'; import {LexiconConfig} from '../../shared/model/lexicon-config.model'; +import {CommentsOfflineCacheService} from '../../../../bellows/core/offline/comments-offline-cache.service'; +import {EditorOfflineCacheService} from '../../../../bellows/core/offline/editor-offline-cache.service'; export class AdvancedOptionsConfigurationController implements angular.IController { accPollUpdateTimerSecondsDirty: number; accOnUpdate: (params: { $event: { pollUpdateTimerSecondsDirty: number } }) => void; - static $inject: string[] = ['$scope']; - constructor(private $scope: angular.IScope) { + static $inject: string[] = ['$scope', 'editorOfflineCache', 'commentsOfflineCache',]; + constructor( + private $scope: angular.IScope, + private editorOfflineCache: EditorOfflineCacheService, + private commentsOfflineCache: CommentsOfflineCacheService, + ) { $scope.$watch( () => this.accPollUpdateTimerSecondsDirty, (newVal: number, oldVal: number) => { @@ -18,6 +24,13 @@ export class AdvancedOptionsConfigurationController implements angular.IControll ); } + async resetLocalStorage() { + await this.editorOfflineCache.deleteAllEntries(); + await this.commentsOfflineCache.deleteAllComments(); + window.location.hash = '#!/'; + window.location.reload(); // To force the redownload + } + $onChanges(changes: any) { const configChange = changes.accConfigPristine as angular.IChangesObject; if (configChange != null && configChange.currentValue != null) { From baade5f86ffc1e0aeedd92eb58d30be3e91b953b Mon Sep 17 00:00:00 2001 From: Kevin Hahn Date: Thu, 9 Nov 2023 16:06:45 +0700 Subject: [PATCH 8/8] update lf merge version, configure staging to use lexbox --- docker/deployment/base/app-config.yaml | 4 +++- docker/deployment/base/lfmerge-deployment.yaml | 11 +++++++++++ docker/deployment/staging/app-config-patch.yaml | 3 ++- docker/deployment/staging/kustomization.yaml | 6 +++--- docker/lfmerge/Dockerfile | 2 +- 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/docker/deployment/base/app-config.yaml b/docker/deployment/base/app-config.yaml index 52628a97db..7bf0400334 100644 --- a/docker/deployment/base/app-config.yaml +++ b/docker/deployment/base/app-config.yaml @@ -4,4 +4,6 @@ metadata: name: app-config data: website: "localhost" - hg-domain: "hg-staging.languagforge.org" + hg-domain: "hg-staging.languageforge.org" + hg-protocol: "https" + hg-username: "lf-merge" diff --git a/docker/deployment/base/lfmerge-deployment.yaml b/docker/deployment/base/lfmerge-deployment.yaml index 389da7e859..70412cf477 100644 --- a/docker/deployment/base/lfmerge-deployment.yaml +++ b/docker/deployment/base/lfmerge-deployment.yaml @@ -108,8 +108,19 @@ spec: configMapKeyRef: name: app-config key: hg-domain + - name: LFMERGE_LANGUAGE_DEPOT_HG_PROTOCOL + valueFrom: + configMapKeyRef: + name: app-config + key: hg-protocol - name: LANGUAGE_DEPOT_TRUST_TOKEN valueFrom: secretKeyRef: key: LD_TRUST_TOKEN name: ld-trust-token + - name: LANGUAGE_DEPOT_HG_USERNAME + valueFrom: + configMapKeyRef: + name: app-config + key: hg-username + diff --git a/docker/deployment/staging/app-config-patch.yaml b/docker/deployment/staging/app-config-patch.yaml index 8137b6be86..03880ce6bb 100644 --- a/docker/deployment/staging/app-config-patch.yaml +++ b/docker/deployment/staging/app-config-patch.yaml @@ -4,4 +4,5 @@ metadata: name: app-config data: website: "staging.languageforge.org" - + hg-domain: "lexbox.languagedepot:5158" #reference lexbox service in the same cluster + hg-protocol: "http" diff --git a/docker/deployment/staging/kustomization.yaml b/docker/deployment/staging/kustomization.yaml index 6af7b0242a..094f6f726b 100644 --- a/docker/deployment/staging/kustomization.yaml +++ b/docker/deployment/staging/kustomization.yaml @@ -10,13 +10,13 @@ resources: # because of that we have to do a bit of a workaround to allow setting the image tag for the correct container images: - name: sillsdev/web-languageforge - newTag: develop-20231002-a33383741406cb1823c05a3f6e00d20f35d1ec4d + newTag: develop-20231108-7256a1d48b45b6ea05afc954344911e786e4f61d - name: sillsdev/web-languageforge-next-proxy newName: sillsdev/web-languageforge - newTag: develop-next-proxy-20231002-a33383741406cb1823c05a3f6e00d20f35d1ec4d + newTag: develop-next-proxy-20231108-7256a1d48b45b6ea05afc954344911e786e4f61d - name: sillsdev/web-languageforge-next-app newName: sillsdev/web-languageforge - newTag: develop-next-app-20231002-a33383741406cb1823c05a3f6e00d20f35d1ec4d + newTag: develop-next-app-20231108-7256a1d48b45b6ea05afc954344911e786e4f61d - name: ghcr.io/sillsdev/lfmerge newTag: 2.0.135 diff --git a/docker/lfmerge/Dockerfile b/docker/lfmerge/Dockerfile index f4564a920a..8b3309ebd7 100644 --- a/docker/lfmerge/Dockerfile +++ b/docker/lfmerge/Dockerfile @@ -1,2 +1,2 @@ -FROM ghcr.io/sillsdev/lfmerge:2.0.135 +FROM ghcr.io/sillsdev/lfmerge:2.0.136 # Do not add anything to this Dockerfile, it should stay empty