Feature: add option to delay new assignments after review has been submitted

.circleci/config.yml

@@ -8,7 +8,7 @@ parameters:
     default: "master"
   openreview-api-v2-branch:
     type: string
-    default: "main"
+    default: "feature/postprocesses"
   python-version:
     type: string
     default: "3.13.1"

openreview/journal/invitation.py

@@ -3667,8 +3667,9 @@ def set_review_invitation(self):
         invitation_content = {
             'process_script': {
                 'value': self.get_process_content('process/review_process.py')
-            }                
+            }
         }
+
         edit_content = {
             'noteId': { 
                 'value': {
@@ -3812,7 +3813,21 @@ def set_review_invitation(self):
 
         if self.journal.get_review_additional_fields():
             for key, value in self.journal.get_review_additional_fields().items():
-                invitation['edit']['note']['content'][key] = value if value else { "delete": True }         
+                invitation['edit']['note']['content'][key] = value if value else { "delete": True }
+
+        if self.journal.get_assignment_delay_after_submitted_review() > 0:
+            weeks = datetime.timedelta(weeks=self.journal.get_assignment_delay_after_submitted_review())
+            milliseconds = int(weeks.total_seconds() * 1000)
+            invitation['postprocesses'] = [
+                {
+                    'script': self.get_super_process_content('post_process_script'),
+                    'delay': milliseconds
+                }
+            ]
+
+            invitation_content['post_process_script'] = {
+                'value': self.get_process_content('process/review_post_process.py')
+            }
 
         self.save_super_invitation(self.journal.get_review_id(), invitation_content, edit_content, invitation)
 

openreview/journal/journal.py

@@ -384,6 +384,9 @@ def get_review_period_length(self, note=None):
     def get_expert_reviewer_certification(self):
         return "Expert Certification"
 
+    def get_assignment_delay_after_submitted_review(self):
+        return self.settings.get('assignment_delay_after_submitted_review', 0)
+
     def is_active_submission(self, submission):
         venue_id = submission.content.get('venueid', {}).get('value')
         return venue_id in [self.submitted_venue_id, self.under_review_venue_id, self.assigning_AE_venue_id, self.assigned_AE_venue_id]

openreview/journal/process/review_post_process.py

@@ -0,0 +1,19 @@
+def process(client, edit, invitation):
+
+    journal = openreview.journal.Journal()
+
+    review_note=client.get_note(edit.note.id)
+    print('Review id:', review_note.id)
+
+    ## Decrease pending reviews counter
+    signature_group = client.get_group(id=review_note.signatures[0])
+    reviewer_profile = openreview.tools.get_profile(client, signature_group.members[0])
+    print('reviewer profile:', reviewer_profile.id)
+    edges = client.get_edges(invitation=journal.get_reviewer_pending_review_id(), tail=(reviewer_profile.id if reviewer_profile else signature_group.members[0]))
+
+    if edges and edges[0].weight > 0:
+        pending_review_edge = edges[0]
+        if not review_note.ddate and journal.get_assignment_delay_after_submitted_review() > 0:
+            print('Decreasing pending review count!')
+            pending_review_edge.weight -= 1
+            client.post_edge(pending_review_edge)

openreview/journal/process/review_process.py

@@ -8,17 +8,18 @@ def process(client, edit, invitation):
     ## Notify readers
     journal.notify_readers(edit)
 
-    ## Decrease pending reviews counter
+    ## Decrease pending reviews counter if there is no assignment delay after submitted review
     signature_group = client.get_group(id=review_note.signatures[0])
     reviewer_profile = openreview.tools.get_profile(client, signature_group.members[0])
     edges = client.get_edges(invitation=journal.get_reviewer_pending_review_id(), tail=(reviewer_profile.id if reviewer_profile else signature_group.members[0]))
     if edges and edges[0].weight > 0:
         pending_review_edge = edges[0]
         if review_note.ddate:
             pending_review_edge.weight += 1
-        else:
+            client.post_edge(pending_review_edge)
+        elif journal.get_assignment_delay_after_submitted_review() == 0:
             pending_review_edge.weight -= 1
-        client.post_edge(pending_review_edge)
+            client.post_edge(pending_review_edge)
 
     ## On update or delete return
     if review_note.tcdate != review_note.tmdate:

tests/test_journal.py

@@ -183,7 +183,8 @@ def test_setup(self, openreview_client, request_page, selenium, helpers, journal
                                     },
                                     'readers': ['TMLR', 'TMLR/Paper${7/content/noteNumber/value}/Action_Editors']
                                 }                                
-                            }                            
+                            },
+                            'assignment_delay_after_submitted_review': 0.0001   # ~ 1 minute
                         }
                     }
                 }
@@ -1260,6 +1261,10 @@ def test_submission(self, journal, openreview_client, test_client, helpers):
         david_anon_groups=david_client.get_groups(prefix=f'{venue_id}/Paper1/Reviewer_.*', signatory='~David_Belanger1')
         assert len(david_anon_groups) == 1
 
+        edges = david_client.get_grouped_edges(invitation='TMLR/Reviewers/-/Pending_Reviews', groupby='weight')
+        assert len(edges) == 1
+        assert edges[0]['values'][0]['weight'] == 1
+
         ## Post a review edit
         david_review_note = david_client.post_note_edit(invitation=f'{venue_id}/Paper1/-/Review',
             signatures=[david_anon_groups[0].id],
@@ -1277,6 +1282,13 @@ def test_submission(self, journal, openreview_client, test_client, helpers):
 
         helpers.await_queue_edit(openreview_client, edit_id=david_review_note['id'])
 
+        edges = david_client.get_grouped_edges(invitation='TMLR/Reviewers/-/Pending_Reviews', groupby='weight')
+        assert len(edges) == 1
+        assert edges[0]['values'][0]['weight'] == 0
+
+        logs = openreview_client.get_process_logs(invitation='TMLR/Paper1/-/Review', status='ok')
+        assert logs and len(logs) == 2
+
         ## Check invitations as a reviewer
         invitations = david_client.get_invitations(replyForum=note_id_1)
         assert len(invitations) == 2

tests/test_melba_journal.py

@@ -313,6 +313,10 @@ def test_ae_assignment(self, journal, openreview_client, test_client, helpers):
         ## Post a review edit
         reviewer_one_client = OpenReviewClient(username='[email protected]', password=helpers.strong_password)
         reviewer_one_anon_groups=reviewer_one_client.get_groups(prefix=f'{venue_id}/Paper1/Reviewer_.*', signatory='~MELBARev_One1')
+
+        edges = reviewer_one_client.get_grouped_edges(invitation=f'{venue_id}/Reviewers/-/Pending_Reviews', groupby='weight')
+        assert len(edges) == 1
+        assert edges[0]['values'][0]['weight'] == 1
 
         review_note = reviewer_one_client.post_note_edit(invitation=f'{venue_id}/Paper1/-/Review',
             signatures=[reviewer_one_anon_groups[0].id],
@@ -331,6 +335,13 @@ def test_ae_assignment(self, journal, openreview_client, test_client, helpers):
 
         helpers.await_queue_edit(openreview_client, edit_id=review_note['id'])
 
+        edges = reviewer_one_client.get_grouped_edges(invitation=f'{venue_id}/Reviewers/-/Pending_Reviews', groupby='weight')
+        assert len(edges) == 1
+        assert edges[0]['values'][0]['weight'] == 0
+
+        logs = openreview_client.get_process_logs(invitation=f'{venue_id}/Paper1/-/Review', status='ok')
+        assert logs and len(logs) == 1
+
         reviewer_two_client = OpenReviewClient(username='[email protected]', password=helpers.strong_password)
         reviewer_two_anon_groups=reviewer_two_client.get_groups(prefix=f'{venue_id}/Paper1/Reviewer_.*', signatory='~MELBARev_Two1')