From 88720f8fe2fd644d0cbb43ee77111d0b83b2622a Mon Sep 17 00:00:00 2001 From: Celeste Martinez Date: Wed, 8 Jan 2025 12:53:21 -0500 Subject: [PATCH 1/8] add option to delay new assignments after review has been submitted --- openreview/journal/invitation.py | 19 +++++++++++++++++-- openreview/journal/journal.py | 3 +++ .../journal/process/review_post_process.py | 19 +++++++++++++++++++ openreview/journal/process/review_process.py | 7 ++++--- tests/test_journal.py | 14 +++++++++++++- tests/test_melba_journal.py | 11 +++++++++++ 6 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 openreview/journal/process/review_post_process.py diff --git a/openreview/journal/invitation.py b/openreview/journal/invitation.py index 95d88b2e0..3fe44a6f0 100644 --- a/openreview/journal/invitation.py +++ b/openreview/journal/invitation.py @@ -3667,8 +3667,9 @@ def set_review_invitation(self): invitation_content = { 'process_script': { 'value': self.get_process_content('process/review_process.py') - } + } } + edit_content = { 'noteId': { 'value': { @@ -3812,7 +3813,21 @@ def set_review_invitation(self): if self.journal.get_review_additional_fields(): for key, value in self.journal.get_review_additional_fields().items(): - invitation['edit']['note']['content'][key] = value if value else { "delete": True } + invitation['edit']['note']['content'][key] = value if value else { "delete": True } + + if self.journal.get_assignment_delay_after_submitted_review() > 0: + weeks = datetime.timedelta(weeks=self.journal.get_assignment_delay_after_submitted_review()) + milliseconds = int(weeks.total_seconds() * 1000) + invitation['postprocesses'] = [ + { + 'script': self.get_super_process_content('post_process_script'), + 'delay': milliseconds + } + ] + + invitation_content['post_process_script'] = { + 'value': self.get_process_content('process/review_post_process.py') + } self.save_super_invitation(self.journal.get_review_id(), invitation_content, edit_content, invitation) diff --git a/openreview/journal/journal.py b/openreview/journal/journal.py index e5c9b457c..c9feeb278 100644 --- a/openreview/journal/journal.py +++ b/openreview/journal/journal.py @@ -384,6 +384,9 @@ def get_review_period_length(self, note=None): def get_expert_reviewer_certification(self): return "Expert Certification" + def get_assignment_delay_after_submitted_review(self): + return self.settings.get('assignment_delay_after_submitted_review', 0) + def is_active_submission(self, submission): venue_id = submission.content.get('venueid', {}).get('value') return venue_id in [self.submitted_venue_id, self.under_review_venue_id, self.assigning_AE_venue_id, self.assigned_AE_venue_id] diff --git a/openreview/journal/process/review_post_process.py b/openreview/journal/process/review_post_process.py new file mode 100644 index 000000000..de66ac5bf --- /dev/null +++ b/openreview/journal/process/review_post_process.py @@ -0,0 +1,19 @@ +def process(client, edit, invitation): + + journal = openreview.journal.Journal() + + review_note=client.get_note(edit.note.id) + print('Review id:', review_note.id) + + ## Decrease pending reviews counter + signature_group = client.get_group(id=review_note.signatures[0]) + reviewer_profile = openreview.tools.get_profile(client, signature_group.members[0]) + print('reviewer profile:', reviewer_profile.id) + edges = client.get_edges(invitation=journal.get_reviewer_pending_review_id(), tail=(reviewer_profile.id if reviewer_profile else signature_group.members[0])) + + if edges and edges[0].weight > 0: + pending_review_edge = edges[0] + if not review_note.ddate and journal.get_assignment_delay_after_submitted_review() > 0: + print('Decreasing pending review count!') + pending_review_edge.weight -= 1 + client.post_edge(pending_review_edge) \ No newline at end of file diff --git a/openreview/journal/process/review_process.py b/openreview/journal/process/review_process.py index a55ff8a3b..49991c207 100644 --- a/openreview/journal/process/review_process.py +++ b/openreview/journal/process/review_process.py @@ -8,7 +8,7 @@ def process(client, edit, invitation): ## Notify readers journal.notify_readers(edit) - ## Decrease pending reviews counter + ## Decrease pending reviews counter if there is no assignment delay after submitted review signature_group = client.get_group(id=review_note.signatures[0]) reviewer_profile = openreview.tools.get_profile(client, signature_group.members[0]) edges = client.get_edges(invitation=journal.get_reviewer_pending_review_id(), tail=(reviewer_profile.id if reviewer_profile else signature_group.members[0])) @@ -16,9 +16,10 @@ def process(client, edit, invitation): pending_review_edge = edges[0] if review_note.ddate: pending_review_edge.weight += 1 - else: + client.post_edge(pending_review_edge) + elif journal.get_assignment_delay_after_submitted_review() == 0: pending_review_edge.weight -= 1 - client.post_edge(pending_review_edge) + client.post_edge(pending_review_edge) ## On update or delete return if review_note.tcdate != review_note.tmdate: diff --git a/tests/test_journal.py b/tests/test_journal.py index 8e5583bdc..36f1f838b 100644 --- a/tests/test_journal.py +++ b/tests/test_journal.py @@ -183,7 +183,8 @@ def test_setup(self, openreview_client, request_page, selenium, helpers, journal }, 'readers': ['TMLR', 'TMLR/Paper${7/content/noteNumber/value}/Action_Editors'] } - } + }, + 'assignment_delay_after_submitted_review': 0.0001 # ~ 1 minute } } } @@ -1260,6 +1261,10 @@ def test_submission(self, journal, openreview_client, test_client, helpers): david_anon_groups=david_client.get_groups(prefix=f'{venue_id}/Paper1/Reviewer_.*', signatory='~David_Belanger1') assert len(david_anon_groups) == 1 + edges = david_client.get_grouped_edges(invitation='TMLR/Reviewers/-/Pending_Reviews', groupby='weight') + assert len(edges) == 1 + assert edges[0]['values'][0]['weight'] == 1 + ## Post a review edit david_review_note = david_client.post_note_edit(invitation=f'{venue_id}/Paper1/-/Review', signatures=[david_anon_groups[0].id], @@ -1277,6 +1282,13 @@ def test_submission(self, journal, openreview_client, test_client, helpers): helpers.await_queue_edit(openreview_client, edit_id=david_review_note['id']) + edges = david_client.get_grouped_edges(invitation='TMLR/Reviewers/-/Pending_Reviews', groupby='weight') + assert len(edges) == 1 + assert edges[0]['values'][0]['weight'] == 0 + + logs = openreview_client.get_process_logs(invitation='TMLR/Paper1/-/Review', status='ok') + assert logs and len(logs) == 2 + ## Check invitations as a reviewer invitations = david_client.get_invitations(replyForum=note_id_1) assert len(invitations) == 2 diff --git a/tests/test_melba_journal.py b/tests/test_melba_journal.py index 6e3f61d27..fc2713a60 100644 --- a/tests/test_melba_journal.py +++ b/tests/test_melba_journal.py @@ -313,6 +313,10 @@ def test_ae_assignment(self, journal, openreview_client, test_client, helpers): ## Post a review edit reviewer_one_client = OpenReviewClient(username='rev1@mailone.com', password=helpers.strong_password) reviewer_one_anon_groups=reviewer_one_client.get_groups(prefix=f'{venue_id}/Paper1/Reviewer_.*', signatory='~MELBARev_One1') + + edges = reviewer_one_client.get_grouped_edges(invitation=f'{venue_id}/Reviewers/-/Pending_Reviews', groupby='weight') + assert len(edges) == 1 + assert edges[0]['values'][0]['weight'] == 1 review_note = reviewer_one_client.post_note_edit(invitation=f'{venue_id}/Paper1/-/Review', signatures=[reviewer_one_anon_groups[0].id], @@ -331,6 +335,13 @@ def test_ae_assignment(self, journal, openreview_client, test_client, helpers): helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + edges = reviewer_one_client.get_grouped_edges(invitation=f'{venue_id}/Reviewers/-/Pending_Reviews', groupby='weight') + assert len(edges) == 1 + assert edges[0]['values'][0]['weight'] == 0 + + logs = openreview_client.get_process_logs(invitation=f'{venue_id}/Paper1/-/Review', status='ok') + assert logs and len(logs) == 1 + reviewer_two_client = OpenReviewClient(username='rev2@mailtwo.com', password=helpers.strong_password) reviewer_two_anon_groups=reviewer_two_client.get_groups(prefix=f'{venue_id}/Paper1/Reviewer_.*', signatory='~MELBARev_Two1') From 871b1835ac9432d790fb27fb54a820f128783c42 Mon Sep 17 00:00:00 2001 From: Celeste Martinez Date: Wed, 8 Jan 2025 12:57:59 -0500 Subject: [PATCH 2/8] temporarily pull from postprocesses branch --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f320a141..bcb15aec1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ parameters: default: "master" openreview-api-v2-branch: type: string - default: "main" + default: "feature/postprocesses" jobs: build: From 18b36a000adcd2ce21462acddd4d11e2177ce6ca Mon Sep 17 00:00:00 2001 From: Celeste Martinez Date: Mon, 13 Jan 2025 10:31:54 -0500 Subject: [PATCH 3/8] always add postprocess and edit pending edge here --- openreview/journal/invitation.py | 28 +++++++++---------- .../journal/process/review_post_process.py | 5 +++- openreview/journal/process/review_process.py | 5 +--- tests/test_melba_journal.py | 2 +- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/openreview/journal/invitation.py b/openreview/journal/invitation.py index 58400ca0a..2a1bdcb4c 100644 --- a/openreview/journal/invitation.py +++ b/openreview/journal/invitation.py @@ -3664,9 +3664,15 @@ def set_review_invitation(self): editors_in_chief_id = self.journal.get_editors_in_chief_id() review_invitation_id = self.journal.get_review_id() + weeks = datetime.timedelta(weeks=self.journal.get_assignment_delay_after_submitted_review()) + milliseconds = int(weeks.total_seconds() * 1000) + invitation_content = { 'process_script': { 'value': self.get_process_content('process/review_process.py') + }, + 'post_process_script': { + 'value': self.get_process_content('process/review_post_process.py') } } @@ -3692,7 +3698,7 @@ def set_review_invitation(self): } } } - } + } invitation = { 'id': self.journal.get_review_id(number='${2/content/noteNumber/value}'), @@ -3705,6 +3711,12 @@ def set_review_invitation(self): 'duedate': '${2/content/duedate/value}', 'process': self.process_script, 'dateprocesses': [self.reviewer_reminder_process_with_EIC, self.review_reminder_process_with_no_ACK], + 'postprocesses': [ + { + 'script': self.get_super_process_content('post_process_script'), + 'delay': milliseconds + } + ], 'edit': { 'signatures': { 'param': { @@ -3815,20 +3827,6 @@ def set_review_invitation(self): for key, value in self.journal.get_review_additional_fields().items(): invitation['edit']['note']['content'][key] = value if value else { "delete": True } - if self.journal.get_assignment_delay_after_submitted_review() > 0: - weeks = datetime.timedelta(weeks=self.journal.get_assignment_delay_after_submitted_review()) - milliseconds = int(weeks.total_seconds() * 1000) - invitation['postprocesses'] = [ - { - 'script': self.get_super_process_content('post_process_script'), - 'delay': milliseconds - } - ] - - invitation_content['post_process_script'] = { - 'value': self.get_process_content('process/review_post_process.py') - } - self.save_super_invitation(self.journal.get_review_id(), invitation_content, edit_content, invitation) invitation = { diff --git a/openreview/journal/process/review_post_process.py b/openreview/journal/process/review_post_process.py index de66ac5bf..ebc0c6f00 100644 --- a/openreview/journal/process/review_post_process.py +++ b/openreview/journal/process/review_post_process.py @@ -5,6 +5,9 @@ def process(client, edit, invitation): review_note=client.get_note(edit.note.id) print('Review id:', review_note.id) + if review_note.ddate: + return + ## Decrease pending reviews counter signature_group = client.get_group(id=review_note.signatures[0]) reviewer_profile = openreview.tools.get_profile(client, signature_group.members[0]) @@ -13,7 +16,7 @@ def process(client, edit, invitation): if edges and edges[0].weight > 0: pending_review_edge = edges[0] - if not review_note.ddate and journal.get_assignment_delay_after_submitted_review() > 0: + if not review_note.ddate: print('Decreasing pending review count!') pending_review_edge.weight -= 1 client.post_edge(pending_review_edge) \ No newline at end of file diff --git a/openreview/journal/process/review_process.py b/openreview/journal/process/review_process.py index 49991c207..939e2d995 100644 --- a/openreview/journal/process/review_process.py +++ b/openreview/journal/process/review_process.py @@ -8,7 +8,7 @@ def process(client, edit, invitation): ## Notify readers journal.notify_readers(edit) - ## Decrease pending reviews counter if there is no assignment delay after submitted review + ## increase pending review if review is deleted signature_group = client.get_group(id=review_note.signatures[0]) reviewer_profile = openreview.tools.get_profile(client, signature_group.members[0]) edges = client.get_edges(invitation=journal.get_reviewer_pending_review_id(), tail=(reviewer_profile.id if reviewer_profile else signature_group.members[0])) @@ -17,9 +17,6 @@ def process(client, edit, invitation): if review_note.ddate: pending_review_edge.weight += 1 client.post_edge(pending_review_edge) - elif journal.get_assignment_delay_after_submitted_review() == 0: - pending_review_edge.weight -= 1 - client.post_edge(pending_review_edge) ## On update or delete return if review_note.tcdate != review_note.tmdate: diff --git a/tests/test_melba_journal.py b/tests/test_melba_journal.py index fc2713a60..bec7c483a 100644 --- a/tests/test_melba_journal.py +++ b/tests/test_melba_journal.py @@ -340,7 +340,7 @@ def test_ae_assignment(self, journal, openreview_client, test_client, helpers): assert edges[0]['values'][0]['weight'] == 0 logs = openreview_client.get_process_logs(invitation=f'{venue_id}/Paper1/-/Review', status='ok') - assert logs and len(logs) == 1 + assert logs and len(logs) == 2 reviewer_two_client = OpenReviewClient(username='rev2@mailtwo.com', password=helpers.strong_password) reviewer_two_anon_groups=reviewer_two_client.get_groups(prefix=f'{venue_id}/Paper1/Reviewer_.*', signatory='~MELBARev_Two1') From 520ccca2b8b3b4a4c58ad12fb7d0f9ddd8488fd7 Mon Sep 17 00:00:00 2001 From: Celeste Martinez Date: Tue, 14 Jan 2025 10:15:41 -0500 Subject: [PATCH 4/8] return on edit; change api2 branch back to main --- .circleci/config.yml | 2 +- openreview/journal/process/review_post_process.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f7e1ea788..f3dbf0177 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ parameters: default: "master" openreview-api-v2-branch: type: string - default: "feature/postprocesses" + default: "main" python-version: type: string default: "3.13.1" diff --git a/openreview/journal/process/review_post_process.py b/openreview/journal/process/review_post_process.py index ebc0c6f00..272bbc110 100644 --- a/openreview/journal/process/review_post_process.py +++ b/openreview/journal/process/review_post_process.py @@ -5,7 +5,9 @@ def process(client, edit, invitation): review_note=client.get_note(edit.note.id) print('Review id:', review_note.id) - if review_note.ddate: + ## On update or delete return + if review_note.tcdate != review_note.tmdate: + print('Review edited, exit') return ## Decrease pending reviews counter From d5d6553e0050969f2048e58b260e1ef46dcc89a3 Mon Sep 17 00:00:00 2001 From: Celeste Martinez Date: Thu, 16 Jan 2025 15:06:48 -0500 Subject: [PATCH 5/8] run post process for first edit only --- openreview/journal/process/review_post_process.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openreview/journal/process/review_post_process.py b/openreview/journal/process/review_post_process.py index 272bbc110..c4a0fdaa2 100644 --- a/openreview/journal/process/review_post_process.py +++ b/openreview/journal/process/review_post_process.py @@ -3,10 +3,11 @@ def process(client, edit, invitation): journal = openreview.journal.Journal() review_note=client.get_note(edit.note.id) + review_edits = client.get_note_edits(note_id=review_note.id, sort='tcdate:asc') print('Review id:', review_note.id) - ## On update or delete return - if review_note.tcdate != review_note.tmdate: + # run post-process only if this is the first edit of the review + if edit.id != review_edits[0].id: print('Review edited, exit') return From dadfcbf2aab4e05cec5d58ddd64e618b48e7c4d5 Mon Sep 17 00:00:00 2001 From: Melisa Bok <545506+melisabok@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:46:03 -0500 Subject: [PATCH 6/8] fix delayed process tests --- tests/conftest.py | 4 +- tests/test_journal.py | 87 +++++++++++++++++++++++++++---------------- 2 files changed, 56 insertions(+), 35 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3ca8cdcc5..03c84a6e2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -96,8 +96,8 @@ def await_queue_edit(super_client, edit_id=None, invitation=None, count=1, error wait_time = 0.5 cycles = 60 * 1 / wait_time # print every 1 minutes while True: - process_logs = super_client.get_process_logs(id=edit_id, invitation=invitation) - if len(process_logs) >= count and all(process_log['status'] in finished_status for process_log in process_logs): + process_logs = super_client.get_process_logs(id=edit_id, invitation=invitation)[:count] + if len(process_logs) == count and all(process_log['status'] in finished_status for process_log in process_logs): for process_log in process_logs: assert process_log['status'] == (expected_status), process_log.get('log', 'No log available') return diff --git a/tests/test_journal.py b/tests/test_journal.py index e7c820568..d80e1f7e6 100644 --- a/tests/test_journal.py +++ b/tests/test_journal.py @@ -1284,10 +1284,13 @@ def test_submission(self, journal, openreview_client, test_client, helpers): edges = david_client.get_grouped_edges(invitation='TMLR/Reviewers/-/Pending_Reviews', groupby='weight') assert len(edges) == 1 - assert edges[0]['values'][0]['weight'] == 0 + assert edges[0]['values'][0]['weight'] == 1 - logs = openreview_client.get_process_logs(invitation='TMLR/Paper1/-/Review', status='ok') - assert logs and len(logs) == 2 + helpers.await_queue_edit(openreview_client, edit_id=david_review_note['id'], count=2) + + edges = david_client.get_grouped_edges(invitation='TMLR/Reviewers/-/Pending_Reviews', groupby='weight') + assert len(edges) == 1 + assert edges[0]['values'][0]['weight'] == 0 ## Check invitations as a reviewer invitations = david_client.get_invitations(replyForum=note_id_1) @@ -1482,7 +1485,7 @@ def test_submission(self, journal, openreview_client, test_client, helpers): assert len(javier_anon_groups) == 1 ## Post a review edit - review_note = javier_client.post_note_edit(invitation=f'{venue_id}/Paper1/-/Review', + javier_review_note = javier_client.post_note_edit(invitation=f'{venue_id}/Paper1/-/Review', signatures=[javier_anon_groups[0].id], note=Note( content={ @@ -1495,7 +1498,7 @@ def test_submission(self, journal, openreview_client, test_client, helpers): ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=javier_review_note['id']) ## Check invitations invitations = openreview_client.get_invitations(replyForum=note_id_1) @@ -1859,7 +1862,7 @@ def test_submission(self, journal, openreview_client, test_client, helpers): ## Post a review edit - review_note = carlos_client.post_note_edit(invitation=f'{venue_id}/Paper1/-/Review', + carlos_review_note = carlos_client.post_note_edit(invitation=f'{venue_id}/Paper1/-/Review', signatures=[carlos_anon_groups[0].id], note=Note( content={ @@ -1872,7 +1875,7 @@ def test_submission(self, journal, openreview_client, test_client, helpers): ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=carlos_review_note['id']) ## Check invitations invitations = openreview_client.get_invitations(replyForum=note_id_1) @@ -2045,7 +2048,7 @@ def test_submission(self, journal, openreview_client, test_client, helpers): assert len(hugo_anon_groups) == 1 ## Post a review edit - review_note = hugo_client.post_note_edit(invitation=f'{venue_id}/Paper1/-/Review', + hugo_review_note = hugo_client.post_note_edit(invitation=f'{venue_id}/Paper1/-/Review', signatures=[hugo_anon_groups[0].id], note=Note( content={ @@ -2058,13 +2061,13 @@ def test_submission(self, journal, openreview_client, test_client, helpers): ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=hugo_review_note['id']) antony_anon_groups=antony_client.get_groups(prefix=f'{venue_id}/Paper1/Reviewer_.*', signatory='~Antony_Bal1') assert len(antony_anon_groups) == 1 ## Post a review edit - review_note = antony_client.post_note_edit(invitation=f'{venue_id}/Paper1/-/Review', + antony_review_note = antony_client.post_note_edit(invitation=f'{venue_id}/Paper1/-/Review', signatures=[antony_anon_groups[0].id], note=Note( content={ @@ -2077,7 +2080,7 @@ def test_submission(self, journal, openreview_client, test_client, helpers): ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=antony_review_note['id']) ## All the reviews should be public now reviews=openreview_client.get_notes(forum=note_id_1, invitation=f'{venue_id}/Paper1/-/Review', sort= 'number:asc') @@ -2798,6 +2801,11 @@ def test_submission(self, journal, openreview_client, test_client, helpers): note={Retracted after acceptance} }''' + helpers.await_queue_edit(openreview_client, edit_id=carlos_review_note['id'], count=2) + helpers.await_queue_edit(openreview_client, edit_id=hugo_review_note['id'], count=2) + helpers.await_queue_edit(openreview_client, edit_id=antony_review_note['id'], count=2) + helpers.await_queue_edit(openreview_client, edit_id=javier_review_note['id'], count=2) + helpers.await_queue_edit(openreview_client, edit_id=david_review_note['id'], count=2) def test_rejected_submission(self, journal, openreview_client, test_client, helpers): @@ -3186,7 +3194,7 @@ def test_rejected_submission(self, journal, openreview_client, test_client, help david_anon_groups=david_client.get_groups(prefix=f'{venue_id}/Paper4/Reviewer_.*', signatory='~David_Belanger1') assert len(david_anon_groups) == 1 - review_note = david_client.post_note_edit(invitation=f'{venue_id}/Paper4/-/Review', + david_review_note = david_client.post_note_edit(invitation=f'{venue_id}/Paper4/-/Review', signatures=[david_anon_groups[0].id], note=Note( content={ @@ -3200,14 +3208,14 @@ def test_rejected_submission(self, journal, openreview_client, test_client, help ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=david_review_note['id']) messages = journal.client.get_messages(subject = '[TMLR] Review posted on TMLR submission 4: Paper title 4') ## Post a review edit javier_anon_groups=javier_client.get_groups(prefix=f'{venue_id}/Paper4/Reviewer_.*', signatory='~Javier_Burroni1') assert len(javier_anon_groups) == 1 - review_note = javier_client.post_note_edit(invitation=f'{venue_id}/Paper4/-/Review', + javier_review_note = javier_client.post_note_edit(invitation=f'{venue_id}/Paper4/-/Review', signatures=[javier_anon_groups[0].id], note=Note( content={ @@ -3220,12 +3228,12 @@ def test_rejected_submission(self, journal, openreview_client, test_client, help ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=javier_review_note['id']) ## Post a review edit carlos_anon_groups=carlos_client.get_groups(prefix=f'{venue_id}/Paper4/Reviewer_.*', signatory='~Carlos_Mondragon1') assert len(carlos_anon_groups) == 1 - review_note = carlos_client.post_note_edit(invitation=f'{venue_id}/Paper4/-/Review', + carlos_review_note = carlos_client.post_note_edit(invitation=f'{venue_id}/Paper4/-/Review', signatures=[carlos_anon_groups[0].id], note=Note( content={ @@ -3238,7 +3246,7 @@ def test_rejected_submission(self, journal, openreview_client, test_client, help ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=carlos_review_note['id']) ## Assign a 4th reviewer paper_assignment_edge = joelle_client.post_edge(openreview.api.Edge(invitation='TMLR/Reviewers/-/Assignment', @@ -3253,6 +3261,11 @@ def test_rejected_submission(self, journal, openreview_client, test_client, help helpers.await_queue_edit(openreview_client, edit_id=paper_assignment_edge.id) + + helpers.await_queue_edit(openreview_client, edit_id=carlos_review_note['id'], count=2) + helpers.await_queue_edit(openreview_client, edit_id=javier_review_note['id'], count=2) + helpers.await_queue_edit(openreview_client, edit_id=david_review_note['id'], count=2) + ## Check pending review edges edges = joelle_client.get_edges_count(invitation='TMLR/Reviewers/-/Pending_Reviews') assert edges == 5 @@ -3645,7 +3658,7 @@ def test_eic_submission(self, journal, openreview_client, test_client, helpers): david_anon_groups=david_client.get_groups(prefix=f'{venue_id}/Paper5/Reviewer_.*', signatory='~David_Belanger1') assert len(david_anon_groups) == 1 - review_note = david_client.post_note_edit(invitation=f'{venue_id}/Paper5/-/Review', + david_review_note = david_client.post_note_edit(invitation=f'{venue_id}/Paper5/-/Review', signatures=[david_anon_groups[0].id], note=Note( content={ @@ -3659,12 +3672,12 @@ def test_eic_submission(self, journal, openreview_client, test_client, helpers): ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=david_review_note['id']) ## Post a review edit javier_anon_groups=javier_client.get_groups(prefix=f'{venue_id}/Paper5/Reviewer_.*', signatory='~Javier_Burroni1') assert len(javier_anon_groups) == 1 - review_note = javier_client.post_note_edit(invitation=f'{venue_id}/Paper5/-/Review', + javier_review_note = javier_client.post_note_edit(invitation=f'{venue_id}/Paper5/-/Review', signatures=[javier_anon_groups[0].id], note=Note( content={ @@ -3677,12 +3690,12 @@ def test_eic_submission(self, journal, openreview_client, test_client, helpers): ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=javier_review_note['id']) ## Post a review edit carlos_anon_groups=carlos_client.get_groups(prefix=f'{venue_id}/Paper5/Reviewer_.*', signatory='~Carlos_Mondragon1') assert len(carlos_anon_groups) == 1 - review_note = carlos_client.post_note_edit(invitation=f'{venue_id}/Paper5/-/Review', + carlos_review_note = carlos_client.post_note_edit(invitation=f'{venue_id}/Paper5/-/Review', signatures=[carlos_anon_groups[0].id], note=Note( content={ @@ -3695,7 +3708,7 @@ def test_eic_submission(self, journal, openreview_client, test_client, helpers): ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=carlos_review_note['id']) invitation = cho_client.get_invitation(f'{venue_id}/Paper5/-/Official_Recommendation') @@ -3845,6 +3858,10 @@ def test_eic_submission(self, journal, openreview_client, test_client, helpers): 'withdrawal_confirmation': { 'value': 'I have read and agree with the venue\'s withdrawal policy on behalf of myself and my co-authors.' }, } )) + + helpers.await_queue_edit(openreview_client, edit_id=carlos_review_note['id'], count=2) + helpers.await_queue_edit(openreview_client, edit_id=javier_review_note['id'], count=2) + helpers.await_queue_edit(openreview_client, edit_id=david_review_note['id'], count=2) def test_withdraw_submission(self, journal, openreview_client, helpers): @@ -3962,7 +3979,7 @@ def test_withdraw_submission(self, journal, openreview_client, helpers): david_anon_groups=david_client.get_groups(prefix=f'{venue_id}/Paper6/Reviewer_.*', signatory='~David_Belanger1') assert len(david_anon_groups) == 1 - review_note = david_client.post_note_edit(invitation=f'{venue_id}/Paper6/-/Review', + david_review_note = david_client.post_note_edit(invitation=f'{venue_id}/Paper6/-/Review', signatures=[david_anon_groups[0].id], note=Note( content={ @@ -3976,12 +3993,12 @@ def test_withdraw_submission(self, journal, openreview_client, helpers): ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=david_review_note['id']) ## Post a review edit javier_anon_groups=javier_client.get_groups(prefix=f'{venue_id}/Paper6/Reviewer_.*', signatory='~Javier_Burroni1') assert len(javier_anon_groups) == 1 - review_note = javier_client.post_note_edit(invitation=f'{venue_id}/Paper6/-/Review', + javier_review_note = javier_client.post_note_edit(invitation=f'{venue_id}/Paper6/-/Review', signatures=[javier_anon_groups[0].id], note=Note( content={ @@ -3994,12 +4011,12 @@ def test_withdraw_submission(self, journal, openreview_client, helpers): ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=javier_review_note['id']) ## Post a review edit carlos_anon_groups=carlos_client.get_groups(prefix=f'{venue_id}/Paper6/Reviewer_.*', signatory='~Carlos_Mondragon1') assert len(carlos_anon_groups) == 1 - review_note = carlos_client.post_note_edit(invitation=f'{venue_id}/Paper6/-/Review', + carlos_review_note = carlos_client.post_note_edit(invitation=f'{venue_id}/Paper6/-/Review', signatures=[carlos_anon_groups[0].id], note=Note( content={ @@ -4012,7 +4029,7 @@ def test_withdraw_submission(self, journal, openreview_client, helpers): ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=carlos_review_note['id']) invitation = cho_client.get_invitation(f'{venue_id}/Paper6/-/Official_Recommendation') @@ -4164,6 +4181,10 @@ def test_withdraw_submission(self, journal, openreview_client, helpers): note={Withdrawn} }''' + helpers.await_queue_edit(openreview_client, edit_id=carlos_review_note['id'], count=2) + helpers.await_queue_edit(openreview_client, edit_id=javier_review_note['id'], count=2) + helpers.await_queue_edit(openreview_client, edit_id=david_review_note['id'], count=2) + def test_submitted_submission(self, journal, openreview_client, helpers): @@ -5055,7 +5076,7 @@ def test_archived_action_editor(self, journal, openreview_client, test_client, h assert len(carlos_anon_groups) == 1 ## Post a review edit - review_note = carlos_client.post_note_edit(invitation=f'{venue_id}/Paper13/-/Review', + carlos_review_note = carlos_client.post_note_edit(invitation=f'{venue_id}/Paper13/-/Review', signatures=[carlos_anon_groups[0].id], note=Note( content={ @@ -5068,13 +5089,13 @@ def test_archived_action_editor(self, journal, openreview_client, test_client, h ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=carlos_review_note['id']) javier_anon_groups=javier_client.get_groups(prefix=f'{venue_id}/Paper13/Reviewer_.*', signatory='~Javier_Burroni1') assert len(javier_anon_groups) == 1 ## Post a review edit - review_note = javier_client.post_note_edit(invitation=f'{venue_id}/Paper13/-/Review', + javier_review_note = javier_client.post_note_edit(invitation=f'{venue_id}/Paper13/-/Review', signatures=[javier_anon_groups[0].id], note=Note( content={ @@ -5087,7 +5108,7 @@ def test_archived_action_editor(self, journal, openreview_client, test_client, h ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=javier_review_note['id']) raia_client.post_invitation_edit( invitations='TMLR/-/Edit', From 1718ae5c542859888b54e9e0a42da6030f8aab4c Mon Sep 17 00:00:00 2001 From: Melisa Bok <545506+melisabok@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:54:38 -0500 Subject: [PATCH 7/8] fix more tests --- tests/test_melba_journal.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_melba_journal.py b/tests/test_melba_journal.py index bec7c483a..91f57eaf9 100644 --- a/tests/test_melba_journal.py +++ b/tests/test_melba_journal.py @@ -333,7 +333,7 @@ def test_ae_assignment(self, journal, openreview_client, test_client, helpers): ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=review_note['id'], count=2) edges = reviewer_one_client.get_grouped_edges(invitation=f'{venue_id}/Reviewers/-/Pending_Reviews', groupby='weight') assert len(edges) == 1 @@ -360,7 +360,7 @@ def test_ae_assignment(self, journal, openreview_client, test_client, helpers): ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=review_note['id'], count=2) reviewer_three_client = OpenReviewClient(username='rev3@mailthree.com', password=helpers.strong_password) reviewer_three_anon_groups=reviewer_two_client.get_groups(prefix=f'{venue_id}/Paper1/Reviewer_.*', signatory='~MELBARev_Three1') @@ -380,7 +380,7 @@ def test_ae_assignment(self, journal, openreview_client, test_client, helpers): ) ) - helpers.await_queue_edit(openreview_client, edit_id=review_note['id']) + helpers.await_queue_edit(openreview_client, edit_id=review_note['id'], count=2) reviews=openreview_client.get_notes(forum=note_id_1, invitation=f'{venue_id}/Paper1/-/Review', sort='number:desc') assert len(reviews) == 3 From 98e1ba208627408f70aaeea879d86189caa74d35 Mon Sep 17 00:00:00 2001 From: Melisa Bok <545506+melisabok@users.noreply.github.com> Date: Fri, 24 Jan 2025 16:28:27 -0500 Subject: [PATCH 8/8] fix log count --- tests/test_icml_conference.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_icml_conference.py b/tests/test_icml_conference.py index 78ca41cdf..dd509d9a7 100644 --- a/tests/test_icml_conference.py +++ b/tests/test_icml_conference.py @@ -4293,7 +4293,7 @@ def test_rebuttal_stage(self, client, openreview_client, helpers): venue.create_review_stage() - helpers.await_queue_edit(openreview_client, 'ICML.cc/2023/Conference/-/Official_Review-0-1', count=4) + helpers.await_queue_edit(openreview_client, 'ICML.cc/2023/Conference/-/Official_Review-0-1', count=5) pc_client_v2=openreview.api.OpenReviewClient(username='pc@icml.cc', password=helpers.strong_password) @@ -4334,7 +4334,7 @@ def test_rebuttal_stage(self, client, openreview_client, helpers): venue.create_review_stage() - helpers.await_queue_edit(openreview_client, 'ICML.cc/2023/Conference/-/Position_Paper_Review-0-1', count=3) + helpers.await_queue_edit(openreview_client, 'ICML.cc/2023/Conference/-/Position_Paper_Review-0-1', count=4) # check reviews of a non-flagged paper is not visible to ethics reviewers but it visible to authors reviews = pc_client_v2.get_notes(invitation='ICML.cc/2023/Conference/Submission2/-/Official_Review')