-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1180 from France-ioi/results_propagation_recomput…
…e_by_chunks Speed up the results propagation and make it less locking + Introduce a command recomputing all the results of chapters/skills
- Loading branch information
Showing
7 changed files
with
196 additions
and
60 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| package cmd | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "log" | ||
| "os" | ||
|
|
||
| _ "github.com/go-sql-driver/mysql" // use to force database/sql to use mysql | ||
| "github.com/spf13/cobra" | ||
|
|
||
| "github.com/France-ioi/AlgoreaBackend/v2/app" | ||
| "github.com/France-ioi/AlgoreaBackend/v2/app/appenv" | ||
| "github.com/France-ioi/AlgoreaBackend/v2/app/database" | ||
| ) | ||
|
|
||
| func init() { //nolint:gochecknoinits | ||
| recomputeResultsCmd := &cobra.Command{ | ||
| Use: "recompute-results [environment]", | ||
| Short: "recompute results for chapters and skills", | ||
| Long: `for each chapter/skill marks all results linked to it as to_be_recomputed and runs the results propagation`, | ||
| Args: cobra.MaximumNArgs(1), | ||
| Run: func(cmd *cobra.Command, args []string) { | ||
| var err error | ||
|
|
||
| // Set the environment. | ||
| if len(args) > 0 { | ||
| appenv.SetEnv(args[0]) | ||
| } | ||
|
|
||
| var application *app.Application | ||
| application, err = app.New() | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
|
|
||
| store := database.NewDataStore(application.Database) | ||
| itemNumber := 0 | ||
| err = store.Items().Where("type = 'Chapter' OR type = 'Skill'").Select("id"). | ||
| ScanAndHandleMaps(func(item map[string]interface{}) error { | ||
| itemNumber++ | ||
| return store.InTransaction(func(store *database.DataStore) error { | ||
| log.Printf("Recomputing results for item %s (#%d)\n", item["id"], itemNumber) | ||
| err = store.Exec("INSERT IGNORE INTO results_recompute_for_items (item_id) values (?)", item["id"]).Error() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| store.ScheduleResultsPropagation() | ||
| return nil | ||
| }) | ||
| }).Error() | ||
| if err != nil { | ||
| fmt.Println("Error while recomputing results: ", err) | ||
| os.Exit(1) | ||
| } | ||
|
|
||
| fmt.Println("Done.") | ||
| }, | ||
| } | ||
|
|
||
| rootCmd.AddCommand(recomputeResultsCmd) | ||
| } |
7 changes: 7 additions & 0 deletions
7
db/migrations/2409290723_add_column_results_recomputing_state.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| -- +migrate Up | ||
| ALTER TABLE `results` | ||
| ADD COLUMN `recomputing_state` ENUM('recomputing', 'modified', 'unchanged') NOT NULL DEFAULT 'unchanged' | ||
| COMMENT 'State of the result, used during recomputing' AFTER `help_requested`; | ||
|
|
||
| -- +migrate Down | ||
| ALTER TABLE `results` DROP COLUMN `recomputing_state`; |
27 changes: 27 additions & 0 deletions
27
db/migrations/2409290724_create_trigger_before_update_results.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| -- +migrate Up | ||
| DROP TRIGGER IF EXISTS `before_update_results`; | ||
| -- +migrate StatementBegin | ||
| CREATE TRIGGER `before_update_results` | ||
| BEFORE UPDATE | ||
| ON `results` | ||
| FOR EACH ROW | ||
| BEGIN | ||
| IF NEW.recomputing_state = 'recomputing' THEN | ||
| SET NEW.recomputing_state = IF( | ||
| NEW.latest_activity_at <=> OLD.latest_activity_at AND | ||
| NEW.tasks_tried <=> OLD.tasks_tried AND | ||
| NEW.tasks_with_help <=> OLD.tasks_with_help AND | ||
| NEW.validated_at <=> OLD.validated_at AND | ||
| NEW.score_computed <=> OLD.score_computed AND | ||
| -- We always consider results with the default latest_activity_at as changed | ||
| -- because they look like a newly inserted result for a chapter/skill. | ||
| -- This way we make sure that a newly inserted result is propagated. | ||
| NEW.latest_activity_at <> '1000-01-01 00:00:00', | ||
| 'unchanged', | ||
| 'modified'); | ||
| END IF; | ||
| END; | ||
| -- +migrate StatementEnd | ||
|
|
||
| -- +migrate Down | ||
| DROP TRIGGER IF EXISTS `before_update_results`; |
5 changes: 5 additions & 0 deletions
5
db/migrations/2409300018_add_enum_value_recomputing_into_results_propagate_state.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| -- +migrate Up | ||
| ALTER TABLE `results_propagate` MODIFY `state` ENUM('to_be_propagated','to_be_recomputed','propagating', 'recomputing') NOT NULL COMMENT '"to_be_propagated" means that ancestors should be recomputed'; | ||
|
|
||
| -- +migrate Down | ||
| ALTER TABLE `results_propagate` MODIFY `state` ENUM('to_be_propagated','to_be_recomputed','propagating') NOT NULL COMMENT '"to_be_propagated" means that ancestors should be recomputed'; |