Merge branch 'api_fix_branch_move' of github.com:codecov/codecov-api …

…into api_fix_branch_move

pytest.ini

@@ -1,3 +1,3 @@
 [pytest]
 DJANGO_SETTINGS_MODULE = codecov.settings_dev
-addopts = -p no:warnings --ignore=shared
+addopts = -p no:warnings --ignore=shared --ignore-glob=**/test_results*

reports/migrations/0013_test_testinstance.py

@@ -0,0 +1,94 @@
+# Generated by Django 4.2.7 on 2024-01-17 20:41
+
+import uuid
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    """
+    BEGIN;
+    --
+    -- Create model Test
+    --
+    CREATE TABLE "reports_test" ("id" text NOT NULL PRIMARY KEY, "external_id" uuid NOT NULL, "created_at" timestamp with time zone NOT NULL, "updated_at" timestamp with time zone NOT NULL, "name" text NOT NULL, "testsuite" text NOT NULL, "env" text NOT NULL, "repoid" integer NOT NULL);
+    --
+    -- Create model TestInstance
+    --
+    CREATE TABLE "reports_testinstance" ("id" bigint NOT NULL PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, "external_id" uuid NOT NULL, "created_at" timestamp with time zone NOT NULL, "updated_at" timestamp with time zone NOT NULL, "duration_seconds" double precision NOT NULL, "outcome" integer NOT NULL, "failure_message" text NULL, "test_id" text NOT NULL, "upload_id" bigint NOT NULL);
+    ALTER TABLE "reports_test" ADD CONSTRAINT "reports_test_repoid_445c33d7_fk_repos_repoid" FOREIGN KEY ("repoid") REFERENCES "repos" ("repoid") DEFERRABLE INITIALLY DEFERRED;
+    CREATE INDEX "reports_test_id_5c60c58c_like" ON "reports_test" ("id" text_pattern_ops);
+    CREATE INDEX "reports_test_repoid_445c33d7" ON "reports_test" ("repoid");
+    ALTER TABLE "reports_testinstance" ADD CONSTRAINT "reports_testinstance_test_id_9c8dd6c1_fk_reports_test_id" FOREIGN KEY ("test_id") REFERENCES "reports_test" ("id") DEFERRABLE INITIALLY DEFERRED;
+    ALTER TABLE "reports_testinstance" ADD CONSTRAINT "reports_testinstance_upload_id_7350520f_fk_reports_upload_id" FOREIGN KEY ("upload_id") REFERENCES "reports_upload" ("id") DEFERRABLE INITIALLY DEFERRED;
+    CREATE INDEX "reports_testinstance_test_id_9c8dd6c1" ON "reports_testinstance" ("test_id");
+    CREATE INDEX "reports_testinstance_test_id_9c8dd6c1_like" ON "reports_testinstance" ("test_id" text_pattern_ops);
+    CREATE INDEX "reports_testinstance_upload_id_7350520f" ON "reports_testinstance" ("upload_id");
+    COMMIT;
+    """
+
+    dependencies = [
+        ("core", "0045_repository_languages_last_updated"),
+        ("reports", "0012_alter_repositoryflag_flag_name"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Test",
+            fields=[
+                ("id", models.TextField(primary_key=True, serialize=False)),
+                ("external_id", models.UUIDField(default=uuid.uuid4, editable=False)),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("name", models.TextField()),
+                ("testsuite", models.TextField()),
+                ("env", models.TextField()),
+                (
+                    "repository",
+                    models.ForeignKey(
+                        db_column="repoid",
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="tests",
+                        to="core.repository",
+                    ),
+                ),
+            ],
+            options={
+                "db_table": "reports_test",
+            },
+        ),
+        migrations.CreateModel(
+            name="TestInstance",
+            fields=[
+                ("id", models.BigAutoField(primary_key=True, serialize=False)),
+                ("external_id", models.UUIDField(default=uuid.uuid4, editable=False)),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("duration_seconds", models.FloatField()),
+                ("outcome", models.IntegerField()),
+                ("failure_message", models.TextField(null=True)),
+                (
+                    "test",
+                    models.ForeignKey(
+                        db_column="test_id",
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="testinstances",
+                        to="reports.test",
+                    ),
+                ),
+                (
+                    "upload",
+                    models.ForeignKey(
+                        db_column="upload_id",
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="testinstances",
+                        to="reports.reportsession",
+                    ),
+                ),
+            ],
+            options={
+                "db_table": "reports_testinstance",
+            },
+        ),
+    ]

reports/models.py

@@ -1,4 +1,5 @@
 import logging
+import uuid
 
 from django.contrib.postgres.fields import ArrayField
 from django.db import models
@@ -216,3 +217,49 @@ class UploadLevelTotals(AbstractTotals):
 
     class Meta:
         db_table = "reports_uploadleveltotals"
+
+
+class Test(models.Model):
+    # the reason we aren't using the regular primary key
+    # in this case is because we want to be able to compute/predict
+    # the primary key of a Test object ourselves in the processor
+    # so we can easily do concurrent writes to the database
+    id = models.TextField(primary_key=True)
+
+    external_id = models.UUIDField(default=uuid.uuid4, editable=False)
+    created_at = models.DateTimeField(auto_now_add=True)
+    updated_at = models.DateTimeField(auto_now=True)
+
+    repository = models.ForeignKey(
+        "core.Repository",
+        db_column="repoid",
+        related_name="tests",
+        on_delete=models.CASCADE,
+    )
+    name = models.TextField()
+    testsuite = models.TextField()
+    env = models.TextField()
+
+    class Meta:
+        db_table = "reports_test"
+
+
+class TestInstance(BaseCodecovModel):
+    test = models.ForeignKey(
+        "Test",
+        db_column="test_id",
+        related_name="testinstances",
+        on_delete=models.CASCADE,
+    )
+    duration_seconds = models.FloatField()
+    outcome = models.IntegerField()
+    upload = models.ForeignKey(
+        "ReportSession",
+        db_column="upload_id",
+        related_name="testinstances",
+        on_delete=models.CASCADE,
+    )
+    failure_message = models.TextField(null=True)
+
+    class Meta:
+        db_table = "reports_testinstance"

services/archive.py

@@ -33,6 +33,8 @@ class MinioEndpoints(Enum):
         "{version}/repos/{repo_hash}/static_analysis/files/{location}"
     )
 
+    test_results = "test_results/v1/raw/{date}/{repo_hash}/{commit_sha}/{uploadid}.txt"
+
     def get_path(self, **kwaargs):
         return self.value.format(**kwaargs)
 

upload/helpers.py

@@ -596,7 +596,6 @@ def validate_upload(upload_params, repository, redis):
         and not repository.activated
         and not bool(get_config("setup", "enterprise_license", default=False))
     ):
-
         owner = _determine_responsible_owner(repository)
 
         # If author is on per repo billing, check their repo credits
@@ -698,7 +697,10 @@ def dispatch_upload_task(
     countdown = 0
     if task_arguments.get("version") == "v4":
         countdown = 4
-    if report_type == CommitReport.ReportType.BUNDLE_ANALYSIS:
+    if (
+        report_type == CommitReport.ReportType.BUNDLE_ANALYSIS
+        or CommitReport.ReportType.TEST_RESULTS
+    ):
         countdown = 4
 
     redis.rpush(repo_queue_key, dumps(task_arguments))

upload/serializers.py

@@ -1,3 +1,5 @@
+from typing import Dict
+
 from django.conf import settings
 from rest_framework import serializers
 

upload/tests/views/test_test_results.py

@@ -0,0 +1,181 @@
+import json
+import re
+from unittest.mock import ANY
+
+from django.urls import reverse
+from rest_framework.test import APIClient
+
+from codecov_auth.tests.factories import OrganizationLevelTokenFactory
+from core.models import Commit
+from core.tests.factories import CommitFactory, OwnerFactory, RepositoryFactory
+from services.redis_configuration import get_redis_connection
+from services.task import TaskService
+
+
+def test_upload_test_results(db, client, mocker, mock_redis):
+    upload = mocker.patch.object(TaskService, "upload")
+    create_presigned_put = mocker.patch(
+        "services.archive.StorageService.create_presigned_put",
+        return_value="test-presigned-put",
+    )
+
+    owner = OwnerFactory(service="github", username="codecov")
+    repository = RepositoryFactory.create(author=owner)
+    commit_sha = "6fd5b89357fc8cdf34d6197549ac7c6d7e5977ef"
+
+    client = APIClient()
+    client.credentials(HTTP_AUTHORIZATION=f"token {repository.upload_token}")
+
+    res = client.post(
+        reverse("upload-test-results"),
+        {
+            "commit": commit_sha,
+            "slug": f"{repository.author.username}::::{repository.name}",
+            "build": "test-build",
+            "buildURL": "test-build-url",
+            "job": "test-job",
+            "service": "test-service",
+        },
+        format="json",
+    )
+    assert res.status_code == 201
+
+    # returns presigned storage URL
+    assert res.json() == {"raw_upload_location": "test-presigned-put"}
+
+    create_presigned_put.assert_called_once_with("archive", ANY, 10)
+    call = create_presigned_put.mock_calls[0]
+    _, storage_path, _ = call.args
+    match = re.match(
+        r"test_results/v1/raw/([\d\w\-]+)/([\d\w\-]+)/([\d\w\-]+)/([\d\w\-]+)\.txt",
+        storage_path,
+    )
+    assert match
+    (
+        date,
+        repo_hash,
+        commit_sha,
+        reportid,
+    ) = match.groups()
+
+    # creates commit
+    commit = Commit.objects.get(commitid=commit_sha)
+    assert commit
+
+    # saves args in Redis
+    redis = get_redis_connection()
+    args = redis.rpop(f"uploads/{repository.repoid}/{commit_sha}/test_results")
+    assert json.loads(args) == {
+        "reportid": reportid,
+        "build": "test-build",
+        "build_url": "test-build-url",
+        "job": "test-job",
+        "service": "test-service",
+        "url": f"test_results/v1/raw/{date}/{repo_hash}/{commit_sha}/{reportid}.txt",
+        "commit": commit_sha,
+        "report_code": None,
+        "flags": None,
+    }
+
+    # sets latest upload timestamp
+    ts = redis.get(f"latest_upload/{repository.repoid}/{commit_sha}/test_results")
+    assert ts
+
+    # triggers upload task
+    upload.assert_called_with(
+        commitid=commit_sha,
+        repoid=repository.repoid,
+        countdown=4,
+        report_code=None,
+        report_type="test_results",
+    )
+
+
+def test_test_results_org_token(db, client, mocker, mock_redis):
+    upload = mocker.patch.object(TaskService, "upload")
+    create_presigned_put = mocker.patch(
+        "services.archive.StorageService.create_presigned_put",
+        return_value="test-presigned-put",
+    )
+
+    owner = OwnerFactory(service="github", username="codecov")
+    repository = RepositoryFactory.create(author=owner)
+    org_token = OrganizationLevelTokenFactory.create(owner=repository.author)
+
+    client = APIClient()
+    client.credentials(HTTP_AUTHORIZATION=f"token {org_token.token}")
+
+    res = client.post(
+        reverse("upload-test-results"),
+        {
+            "commit": "6fd5b89357fc8cdf34d6197549ac7c6d7e5977ef",
+            "slug": f"{repository.author.username}::::{repository.name}",
+        },
+        format="json",
+    )
+    assert res.status_code == 201
+
+
+def test_upload_test_results_missing_args(db, client, mocker, mock_redis):
+    upload = mocker.patch.object(TaskService, "upload")
+    create_presigned_put = mocker.patch(
+        "services.archive.StorageService.create_presigned_put",
+        return_value="test-presigned-put",
+    )
+
+    repository = RepositoryFactory.create()
+    commit = CommitFactory.create(repository=repository)
+
+    client = APIClient()
+    client.credentials(HTTP_AUTHORIZATION=f"token {repository.upload_token}")
+
+    res = client.post(
+        reverse("upload-test-results"),
+        {
+            "commit": commit.commitid,
+        },
+        format="json",
+    )
+    assert res.status_code == 400
+    assert res.json() == {"slug": ["This field is required."]}
+    assert not upload.called
+
+    res = client.post(
+        reverse("upload-test-results"),
+        {
+            "slug": f"{repository.author.username}::::{repository.name}",
+        },
+        format="json",
+    )
+    assert res.status_code == 400
+    assert res.json() == {"commit": ["This field is required."]}
+    assert not upload.called
+
+
+def test_upload_test_results_rollout_fails(db, client, mocker, mock_redis):
+    upload = mocker.patch.object(TaskService, "upload")
+    create_presigned_put = mocker.patch(
+        "services.archive.StorageService.create_presigned_put",
+        return_value="test-presigned-put",
+    )
+
+    owner = OwnerFactory(service="github", username="not-codecov")
+    repository = RepositoryFactory.create(author=owner)
+    commit_sha = "6fd5b89357fc8cdf34d6197549ac7c6d7e5977ef"
+
+    client = APIClient()
+    client.credentials(HTTP_AUTHORIZATION=f"token {repository.upload_token}")
+
+    res = client.post(
+        reverse("upload-test-results"),
+        {
+            "commit": commit_sha,
+            "slug": f"{repository.author.username}::::{repository.name}",
+            "build": "test-build",
+            "buildURL": "test-build-url",
+            "job": "test-job",
+            "service": "test-service",
+        },
+        format="json",
+    )
+    assert res.status_code == 403

upload/urls.py

@@ -5,10 +5,16 @@
 from upload.views.empty_upload import EmptyUploadView
 from upload.views.legacy import UploadDownloadHandler, UploadHandler
 from upload.views.reports import ReportResultsView, ReportViews
+from upload.views.test_results import TestResultsView
 from upload.views.upload_completion import UploadCompletionView
 from upload.views.uploads import UploadViews
 
 urlpatterns = [
+    path(
+        "test_results/v1",
+        TestResultsView.as_view(),
+        name="upload-test-results",
+    ),
     path(
         "bundle_analysis/v1",
         BundleAnalysisView.as_view(),