Skip to content

Commit

Permalink
ARR: Change Seniority Condition (#2329)
Browse files Browse the repository at this point in the history 
* Check seniority with publication count

* Change flag criteria

* Revert publication threshold

---------

Co-authored-by: celestemartinez <[email protected]>
  • Loading branch information
@haroldrubio @celestemartinez
haroldrubio and celestemartinez authored Oct 10, 2024
1 parent 105ded8 commit 1fb147f
Show file tree
Hide file tree
Showing 3 changed files with 150 additions and 10 deletions.
1 change: 1 addition & 0 deletions openreview/arr/arr.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

SHORT_BUFFER_MIN = 30
LONG_BUFFER_DAYS = 10
SENIORITY_PUBLICATION_COUNT = 8

class ARR(object):

Expand Down
75 changes: 68 additions & 7 deletions openreview/arr/management/setup_reassignment_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,75 @@ def process(client, invitation):

from openreview.venue import matching
from openreview.arr.helpers import get_resubmissions
from openreview.arr.arr import SENIORITY_PUBLICATION_COUNT
from collections import defaultdict

def get_title(profile):
d = profile.content.get('history', [{}])
if len(d) > 0:
return d[0].get('position', 'Student')
def is_main_venue(link, pub):
venueid = pub.content.get('venueid')
if venueid is not None:
if not isinstance(venueid, str):
venueid = venueid.get("value")
if venueid.startswith("dblp.org/conf/"):
parts = venueid.split("/")
conf = parts[2]
if conf.lower() in [ "aacl", "acl", "cl", "conll", "eacl", "emnlp", "findings", "naacl", "tacl", "coling", "ijcnlp"]:
return True
venue = pub.content.get('venue')
if venue is not None:
if not isinstance(venue, str):
venue = venue.get("value")
if 'withdraw' in venue.lower() or 'desk reject' in venue.lower():
return False
for token in venue.lower().split():
if token in ["aacl", "acl", "cl", "conll", "eacl", "emnlp", "findings", "naacl", "tacl", "coling", "main", "ijcnlp", "short", "papers", "hlt", 'sem']:
return True

if link.startswith("https://transacl.org"):
return True
ending = ''
if "://aclanthology.org/" in link:
ending = link.split('/')[3]
elif "://aclweb.org/anthology/" in link:
ending = link.split("/")[4]
elif "://aclanthology.info/papers/" in link:
ending = link.split("/")[4]
else:
return ''
return False

if ending[0] in ["C", "D", "E", "I", "J", "K", "N", "P", "Q"]:
return True
elif '.' in ending and ending.split('.')[1].split("-")[0] in [ "aacl", "acl", "cl", "conll", "eacl", "emnlp", "findings", "naacl", "tacl", "coling", "ijcnlp"]:
return True
return False

def is_recent(pub, recent=5):
venueid = pub.content.get('venueid')
if venueid is not None and (not isinstance(venueid, str)):
venueid = venueid.get("value")
venue = pub.content.get('venue')
if venue is not None and (not isinstance(venue, str)):
venue = venue.get("value")
current = datetime.date.today().year
for year in range(current - recent + 1, current + 1):
if venue is not None and str(year) in venue:
return True
if venueid is not None and str(year) in venueid:
return True
return False

def collect_pub_stats(publications):
# Here 'publications' comes from the user profile:
# profile.content.get("publications", [])
acl_main_recent = 0
for pub in publications:
link = pub.content.get('pdf', '')
if link == '' or isinstance(link, dict):
link = pub.content.get('html', '')
if isinstance(link, dict):
link = link.get('value', '')
if is_recent(pub) and is_main_venue(link, pub) and 'everyone' in pub.readers:
acl_main_recent += 1
return acl_main_recent

def replace_edge(existing_edge=None, edge_inv=None, new_weight=None, submission_id=None, profile_id=None, edge_readers=None):
if existing_edge:
Expand Down Expand Up @@ -103,7 +164,7 @@ def replace_edge(existing_edge=None, edge_inv=None, new_weight=None, submission_
all_profiles = []
name_to_id = {}
for role_id in [reviewers_id, area_chairs_id, senior_area_chairs_id]:
profiles = openreview.tools.get_profiles(client, client.get_group(role_id).members)
profiles = openreview.tools.get_profiles(client, client.get_group(role_id).members, with_publications=True)
if role_id == reviewers_id:
reviewer_profiles.extend(profiles) ## Cache reviewer profiles for seniority
all_profiles.extend(profiles)
Expand Down Expand Up @@ -429,7 +490,7 @@ def replace_edge(existing_edge=None, edge_inv=None, new_weight=None, submission_
seniority_edges = []
seniority_inv = f"{reviewers_id}/-/{seniority_name}"
for profile in reviewer_profiles:
if 'student' not in get_title(profile).lower():
if collect_pub_stats(profile.content.get('publications', [])) >= SENIORITY_PUBLICATION_COUNT:
seniority_edges.append(
openreview.api.Edge(
invitation=seniority_inv,
Expand Down
84 changes: 81 additions & 3 deletions tests/test_arr_venue_v2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,14 @@ def test_august_cycle(self, client, openreview_client, helpers, test_client, req
helpers.create_user('[email protected]', 'AC', 'ARROne')
helpers.create_user('[email protected]', 'AC', 'ARRTwo')
helpers.create_user('[email protected]', 'AC', 'ARRThree')
helpers.create_user('[email protected]', 'Reviewer', 'ARRTwo')
#helpers.create_user('[email protected]', 'Reviewer', 'ARRTwo')
helpers.create_user('[email protected]', 'Reviewer', 'ARRThree')
helpers.create_user('[email protected]', 'Reviewer', 'ARRFour')
helpers.create_user('[email protected]', 'Reviewer', 'ARRFive')
helpers.create_user('[email protected]', 'Reviewer', 'ARRSix')
helpers.create_user('[email protected]', 'EthicsReviewer', 'ARROne')

# Manually create Reviewer ARROne as a professor
# Manually create Reviewer ARROne as having more than 5 *CL main publications
fullname = f'Reviewer ARROne'
res = openreview_client.register_user(email = '[email protected]', fullname = fullname, password = helpers.strong_password)
username = res.get('id')
Expand All @@ -70,7 +70,7 @@ def test_august_cycle(self, client, openreview_client, helpers, test_client, req
'preferredEmail': '[email protected]'
}
profile_content['history'] = [{
'position': 'Full Professor',
'position': 'Student',
'start': 2017,
'end': None,
'institution': {
Expand All @@ -85,6 +85,84 @@ def test_august_cycle(self, client, openreview_client, helpers, test_client, req
assert profile.content['names'][0]['username'] == '~Reviewer_ARROne1'
assert profile.content['names'][1]['username'] == '~Reviewer_Alternate_ARROne1'

for i in range(1, 9):
edit = rev_client.post_note_edit(
invitation='openreview.net/Archive/-/Direct_Upload',
signatures=['~Reviewer_ARROne1'],
note = openreview.api.Note(
pdate = openreview.tools.datetime_millis(datetime.datetime(2019, 4, 30)),
content = {
'title': { 'value': f'Paper title {i}' },
'abstract': { 'value': f'Paper abstract {i}' },
'authors': { 'value': ['Reviewer ARROne', 'Test2 Client'] },
'authorids': { 'value': ['~Reviewer_ARROne1', '[email protected]'] },
'venue': { 'value': 'EMNLP 2024 Main' }
},
license = 'CC BY-SA 4.0'
))
openreview_client.post_note_edit(
invitation='openreview.net/-/Edit',
readers=['openreview.net'],
writers=['openreview.net'],
signatures=['openreview.net'],
note=openreview.api.Note(
id = edit['note']['id'],
content = {
'venueid': { 'value': 'EMNLP/2024/Conference' }
}
)
)

# Manually create Reviewer ARRTwo as having more than 5 non-*CL main publications
fullname = f'Reviewer ARRTwo'
res = openreview_client.register_user(email = '[email protected]', fullname = fullname, password = helpers.strong_password)
username = res.get('id')
profile_content={
'names': [
{
'fullname': fullname,
'username': username,
'preferred': False
},
{
'fullname': 'Reviewer Alternate ARRTwo',
'preferred': True
}
],
'emails': ['[email protected]'],
'preferredEmail': '[email protected]'
}
profile_content['history'] = [{
'position': 'Full Professor',
'start': 2017,
'end': None,
'institution': {
'country': 'US',
'domain': 'aclrollingreview.com',
}
}]
rev_client = openreview.api.OpenReviewClient(baseurl = 'http://localhost:3001')
rev_client.activate_user('[email protected]', profile_content)

profile = rev_client.get_profile('~Reviewer_ARRTwo1')
assert profile.content['names'][0]['username'] == '~Reviewer_ARRTwo1'
assert profile.content['names'][1]['username'] == '~Reviewer_Alternate_ARRTwo1'

for i in range(1, 9):
rev_client.post_note_edit(
invitation='openreview.net/Archive/-/Direct_Upload',
signatures=['~Reviewer_ARRTwo1'],
note = openreview.api.Note(
pdate = openreview.tools.datetime_millis(datetime.datetime(2019, 4, 30)),
content = {
'title': { 'value': f'Paper title {i}' },
'abstract': { 'value': f'Paper abstract {i}' },
'authors': { 'value': ['Reviewer ARRTwo', 'Test2 Client'] },
'authorids': { 'value': ['~Reviewer_ARRTwo1', '[email protected]'] },
'venue': { 'value': 'Arxiv' }
},
license = 'CC BY-SA 4.0'
))

request_form_note = pc_client.post_note(openreview.Note(
invitation='openreview.net/Support/-/Request_Form',
Expand Down

0 comments on commit 1fb147f

Please sign in to comment.