Merge remote-tracking branch 'upstream/develop' into feature/preprint…

…s-affiliations-test-merge-issue-3-updated

.github/workflows/test-build.yml

@@ -21,7 +21,7 @@ jobs:
       uses: actions/cache@v2
       with:
         path: ~/.cache
-        key: reqs_${{ hashFiles('**/requirements.txt') }}
+        key: reqs_${{ hashFiles('**/pyproject.toml') }}
         restore-keys: reqs
     - run: |
         mkdir -p ~/.cache/downloads

addons/base/views.py

@@ -696,6 +696,10 @@ def osfstoragefile_mark_viewed(self, auth, fileversion, file_node):
 @file_signals.file_viewed.connect
 def osfstoragefile_update_view_analytics(self, auth, fileversion, file_node):
     resource = file_node.target
+    user = getattr(auth, 'user', None)
+    if hasattr(resource, 'is_contributor_or_group_member') and resource.is_contributor_or_group_member(user):
+        # Don't record views by contributors
+        return
     enqueue_update_analytics(
         resource,
         file_node,
@@ -707,6 +711,10 @@ def osfstoragefile_update_view_analytics(self, auth, fileversion, file_node):
 @file_signals.file_viewed.connect
 def osfstoragefile_viewed_update_metrics(self, auth, fileversion, file_node):
     resource = file_node.target
+    user = getattr(auth, 'user', None)
+    if hasattr(resource, 'is_contributor_or_group_member') and resource.is_contributor_or_group_member(user):
+        # Don't record views by contributors
+        return
     if waffle.switch_is_active(features.ELASTICSEARCH_METRICS) and isinstance(resource, Preprint):
         try:
             PreprintView.record_for_preprint(
@@ -730,6 +738,10 @@ def osfstoragefile_downloaded_update_analytics(self, auth, fileversion, file_nod
 @file_signals.file_downloaded.connect
 def osfstoragefile_downloaded_update_metrics(self, auth, fileversion, file_node):
     resource = file_node.target
+    user = getattr(auth, 'user', None)
+    if hasattr(resource, 'is_contributor_or_group_member') and resource.is_contributor_or_group_member(user):
+        # Don't record downloads by contributors
+        return
     if waffle.switch_is_active(features.ELASTICSEARCH_METRICS) and isinstance(resource, Preprint):
         try:
             PreprintDownload.record_for_preprint(

admin/management/views.py

@@ -100,7 +100,6 @@ def post(self, request, *args, **kwargs):
 class DailyReportersGo(ManagementCommandPermissionView):
 
     def post(self, request, *args, **kwargs):
-        also_keen = bool(request.POST.get('also_send_to_keen', False))
         report_date = request.POST.get('report_date', None)
         if report_date:
             report_date = isoparse(report_date).date()
@@ -109,7 +108,6 @@ def post(self, request, *args, **kwargs):
 
         daily_reporters_go.apply_async(kwargs={
             'report_date': report_date,
-            'also_send_to_keen': also_keen
         })
         messages.success(request, 'Daily reporters going!')
         return redirect(reverse('management:commands'))

admin/templates/management/commands.html

@@ -89,11 +89,6 @@ <h4><u>Daily Reporters, Go!</u></h4>
                     <input type="date" name="report_date" id="report_date"/>
                     (default: yesterday)
                     <br>
-                    <label for="also_send_to_keen">
-                        Also send to keen?
-                    </label>
-                    <input type="checkbox" name="also_send_to_keen" id="also_send_to_keen"/>
-                    (may result in duplicates)
                     <nav>
                         <input class="btn btn-success" type="submit" value="Run" />
                     </nav>

api/base/settings/defaults.py

@@ -317,7 +317,7 @@
 # django-elasticsearch-metrics
 ELASTICSEARCH_DSL = {
     'default': {
-        'hosts': os.environ.get('ELASTIC6_URI', '127.0.0.1:9201'),
+        'hosts': osf_settings.ELASTIC6_URI,
         'retry_on_timeout': True,
     },
 }

api/base/utils.py

@@ -1,5 +1,5 @@
 from furl import furl
-from urllib.parse import urlunsplit, urlsplit, parse_qs, urlencode, quote
+from urllib.parse import urlunsplit, urlsplit, parse_qs, urlencode
 from packaging.version import Version
 from hashids import Hashids
 
@@ -236,7 +236,7 @@ def waterbutler_api_url_for(node_id, provider, path='/', _internal=False, base_u
     # NOTE: furl encoding to be verified later
     url = furl(website_settings.WATERBUTLER_INTERNAL_URL if _internal else (base_url or website_settings.WATERBUTLER_URL))
     segments = ['v1', 'resources', node_id, 'providers', provider] + path.split('/')[1:]
-    url.add(path=[quote(x) for x in segments])
+    url.add(path=segments)
     url.args.update(kwargs)
     return url.url
 

conftest.py

@@ -124,7 +124,7 @@ def _test_speedups_disable(request, settings, _test_speedups):
 
 @pytest.fixture(scope='session')
 def setup_connections():
-    connections.create_connection(hosts=['http://localhost:9201'])
+    connections.create_connection(hosts=[website_settings.ELASTIC6_URI])
 
 
 @pytest.fixture(scope='function')

docker-compose-dist-arm64.override.yml

@@ -6,43 +6,6 @@ services:
   # OSF #
   #######
 
-  requirements:
-    image: quay.io/centerforopenscience/osf:develop-arm64
+  elasticsearch6:
+    image: quay.io/centerforopenscience/elasticsearch:es6-arm-6.3.1
     platform: linux/arm64
-
-  assets:
-    image: quay.io/centerforopenscience/osf:develop-arm64
-    platform: linux/arm64
-    # Need to allocate tty to be able to call invoke for requirements task
-    tty: true
-
-  admin_assets:
-    image: quay.io/centerforopenscience/osf:develop-arm64
-    platform: linux/arm64
-    # Need to allocate tty to be able to call invoke for requirements task
-    tty: true
-
-  worker:
-    image: quay.io/centerforopenscience/osf:develop-arm64
-    platform: linux/arm64
-    # Need to allocate tty to be able to call invoke for requirements task
-    tty: true
-
-  admin:
-    image: quay.io/centerforopenscience/osf:develop-arm64
-    platform: linux/arm64
-    # Need to allocate tty to be able to call invoke for requirements task
-    tty: true
-
-  api:
-    image: quay.io/centerforopenscience/osf:develop-arm64
-    platform: linux/arm64
-    # Need to allocate tty to be able to call invoke for requirements task
-    tty: true
-
-  web:
-    image: quay.io/centerforopenscience/osf:develop-arm64
-    platform: linux/arm64
-    # Need to allocate tty to be able to call invoke for requirements task
-    tty: true
-

docker-compose.yml

@@ -358,15 +358,16 @@ services:
       - /bin/bash
       - -c
       - python -m venv /tmp/venv
-        && /tmp/venv/bin/pip install poetry==1.8.0 &&
+        && /tmp/venv/bin/pip install poetry==1.8.3 &&
         /tmp/venv/bin/poetry install --no-root --without release --compile --sync &&
         rm -rf /python3.12/* &&
         cp -Rf -p /usr/local/lib/python3.12 /
     restart: 'no'
     environment:
       DJANGO_SETTINGS_MODULE: api.base.settings
     volumes:
-      - ./:/code:cached
+      - ./pyproject.toml:/code/pyproject.toml
+      - ./poetry.lock:/code/poetry.lock
       - osf_requirements_3_12_vol:/python3.12
 
   assets:

osf/management/commands/daily_reporters_go.py

@@ -2,56 +2,51 @@
 import logging
 
 from django.core.management.base import BaseCommand
+from django.db.utils import OperationalError
 from django.utils import timezone
 
-from framework import sentry
 from framework.celery_tasks import app as celery_app
-from osf.metrics.reporters import DAILY_REPORTERS
+from osf.metrics.reporters import AllDailyReporters
 from website.app import init_app
 
 
 logger = logging.getLogger(__name__)
 
 
 @celery_app.task(name='management.commands.daily_reporters_go')
-def daily_reporters_go(also_send_to_keen=False, report_date=None, reporter_filter=None):
+def daily_reporters_go(report_date=None, reporter_filter=None, **kwargs):
     init_app()  # OSF-specific setup
 
     if report_date is None:  # default to yesterday
         report_date = (timezone.now() - datetime.timedelta(days=1)).date()
 
-    errors = {}
-    for reporter_class in DAILY_REPORTERS:
-        if reporter_filter and (reporter_filter.lower() not in reporter_class.__name__.lower()):
+    for _reporter_key, _reporter_class in AllDailyReporters.__members__.items():
+        if reporter_filter and (reporter_filter.lower() not in _reporter_class.__name__.lower()):
             continue
-        try:
-            reporter_class().run_and_record_for_date(
-                report_date=report_date,
-                also_send_to_keen=also_send_to_keen,
-            )
-        except Exception as e:
-            errors[reporter_class.__name__] = repr(e)
-            logger.exception(e)
-            sentry.log_exception(e)
-            # continue with the next reporter
-    return errors
+        daily_reporter_go.apply_async(kwargs={
+            'reporter_key': _reporter_key,
+            'report_date': report_date.isoformat(),
+        })
 
 
-def date_fromisoformat(date_str):
-    return datetime.datetime.strptime(date_str, '%Y-%m-%d').date()
+@celery_app.task(
+    name='management.commands.daily_reporter_go',
+    autoretry_for=(OperationalError,),
+    max_retries=5,
+    retry_backoff=True,
+    bind=True,
+)
+def daily_reporter_go(task, reporter_key: str, report_date: str):
+    _reporter_class = AllDailyReporters[reporter_key].value
+    _parsed_date = datetime.date.fromisoformat(report_date)
+    _reporter_class().run_and_record_for_date(report_date=_parsed_date)
 
 
 class Command(BaseCommand):
     def add_arguments(self, parser):
-        parser.add_argument(
-            '--keen',
-            type=bool,
-            default=False,
-            help='also send reports to keen',
-        )
         parser.add_argument(
             '--date',
-            type=date_fromisoformat,  # in python 3.7+, could pass datetime.date.fromisoformat
+            type=datetime.date.fromisoformat,
             help='run for a specific date (default: yesterday)',
         )
         parser.add_argument(
@@ -62,7 +57,6 @@ def add_arguments(self, parser):
     def handle(self, *args, **options):
         errors = daily_reporters_go(
             report_date=options.get('date'),
-            also_send_to_keen=options['keen'],
             reporter_filter=options.get('filter'),
         )
         for error_key, error_val in errors.items():

osf/management/commands/force_archive.py

@@ -58,10 +58,13 @@
 # Ignorable NodeLogs
 LOG_WHITELIST = {
     'affiliated_institution_added',
+    'category_updated',
     'comment_added',
     'comment_removed',
     'comment_restored',
     'comment_updated',
+    'confirm_ham',
+    'confirm_spam',
     'contributor_added',
     'contributor_removed',
     'contributors_reordered',
@@ -72,14 +75,19 @@
     'embargo_completed',
     'embargo_initiated',
     'embargo_terminated',
+    'external_ids_added',
     'file_tag_added',
+    'flag_spam',
+    'guid_metadata_updated',
     'license_changed',
     'made_contributor_invisible',
     'made_private',
     'made_public',
     'made_wiki_private',
     'made_wiki_public',
     'node_removed',
+    'node_access_requests_disabled',
+    'node_access_requests_enabled',
     'permissions_updated',
     'pointer_created',
     'pointer_removed',
@@ -92,6 +100,7 @@
     'registration_initiated',
     'retraction_approved',
     'retraction_initiated',
+    'subjects_updated',
     'tag_added',
     'tag_removed',
     'wiki_deleted',

osf/management/commands/monthly_reporters_go.py

@@ -1,11 +1,11 @@
 import logging
 
 from django.core.management.base import BaseCommand
+from django.db.utils import OperationalError
 from django.utils import timezone
 
-from framework import sentry
 from framework.celery_tasks import app as celery_app
-from osf.metrics.reporters import MONTHLY_REPORTERS
+from osf.metrics.reporters import AllMonthlyReporters
 from osf.metrics.utils import YearMonth
 from website.app import init_app
 
@@ -28,17 +28,24 @@ def monthly_reporters_go(report_year=None, report_month=None):
             year=today.year if today.month > 1 else today.year - 1,
             month=today.month - 1 or MAXMONTH,
         )
+    for _reporter_key in AllMonthlyReporters.__members__.keys():
+        monthly_reporter_go.apply_async(kwargs={
+            'reporter_key': _reporter_key,
+            'yearmonth': str(report_yearmonth),
+        })
 
-    errors = {}
-    for reporter_class in MONTHLY_REPORTERS:
-        try:
-            reporter_class().run_and_record_for_month(report_yearmonth)
-        except Exception as e:
-            errors[reporter_class.__name__] = str(e)
-            logger.exception(e)
-            sentry.log_exception(e)
-            # continue with the next reporter
-    return errors
+
+@celery_app.task(
+    name='management.commands.monthly_reporter_go',
+    autoretry_for=(OperationalError,),
+    max_retries=5,
+    retry_backoff=True,
+    bind=True,
+)
+def monthly_reporter_go(task, reporter_key: str, yearmonth: str):
+    _reporter_class = AllMonthlyReporters[reporter_key].value
+    _parsed_yearmonth = YearMonth.from_str(yearmonth)
+    _reporter_class().run_and_record_for_month(_parsed_yearmonth)
 
 
 class Command(BaseCommand):