From 51da428ea1e478893f533375d4a3c42a9d09b803 Mon Sep 17 00:00:00 2001 From: Arpad Borsos Date: Fri, 21 Mar 2025 14:28:40 +0100 Subject: [PATCH] Turn `labelanalysis` and `staticanalysis` views into noops As part of deprecating ATS, this turns all the views into noops that return empty objects. Checking in with the requests that the CLI sends, returning an empty `external_id` and `absent_labels` for the labelanalysis will result in the CLI falling back to using the clientside requested labels. Similarly, returning an empty list of `filepaths` means that the CLI will not do any further uploads. --- labelanalysis/serializers.py | 89 ---- labelanalysis/tests/integration/test_views.py | 415 +----------------- labelanalysis/urls.py | 9 +- labelanalysis/views.py | 71 ++- staticanalysis/serializers.py | 156 ------- staticanalysis/tests/test_views.py | 60 +-- staticanalysis/urls.py | 19 +- staticanalysis/views.py | 47 +- 8 files changed, 68 insertions(+), 798 deletions(-) delete mode 100644 labelanalysis/serializers.py delete mode 100644 staticanalysis/serializers.py diff --git a/labelanalysis/serializers.py b/labelanalysis/serializers.py deleted file mode 100644 index a75237a1c5..0000000000 --- a/labelanalysis/serializers.py +++ /dev/null @@ -1,89 +0,0 @@ -from rest_framework import exceptions, serializers - -from core.models import Commit -from labelanalysis.models import ( - LabelAnalysisProcessingError, - LabelAnalysisRequest, - LabelAnalysisRequestState, -) - - -class CommitFromShaSerializerField(serializers.Field): - def __init__(self, *args, **kwargs): - self.accepts_fallback = kwargs.pop("accepts_fallback", False) - super().__init__(*args, **kwargs) - - def to_representation(self, commit): - return commit.commitid - - def to_internal_value(self, commit_sha): - commit = Commit.objects.filter( - repository__in=self.context["request"].auth.get_repositories(), - commitid=commit_sha, - ).first() - if commit is None: - raise exceptions.NotFound(f"Commit {commit_sha[:7]} not found.") - if commit.staticanalysissuite_set.exists(): - return commit - if not self.accepts_fallback: - raise serializers.ValidationError("No static analysis found") - attempted_commits = [] - for _ in range(10): - attempted_commits.append(commit.commitid) - commit = commit.parent_commit - if commit is None: - raise serializers.ValidationError( - f"No possible commits have static analysis sent. Attempted commits: {','.join(attempted_commits)}" - ) - if commit.staticanalysissuite_set.exists(): - return commit - raise serializers.ValidationError( - f"No possible commits have static analysis sent. Attempted too many commits: {','.join(attempted_commits)}" - ) - - -class LabelAnalysisProcessingErrorSerializer(serializers.ModelSerializer): - class Meta: - model = LabelAnalysisProcessingError - fields = ("error_code", "error_params") - read_only_fields = ("error_code", "error_params") - - -class ProcessingErrorList(serializers.ListField): - child = LabelAnalysisProcessingErrorSerializer() - - def to_representation(self, data): - data = data.select_related( - "label_analysis_request", - ).all() - return super().to_representation(data) - - -class LabelAnalysisRequestSerializer(serializers.ModelSerializer): - base_commit = CommitFromShaSerializerField(required=True, accepts_fallback=True) - head_commit = CommitFromShaSerializerField(required=True, accepts_fallback=False) - state = serializers.SerializerMethodField() - errors = ProcessingErrorList(required=False) - - def validate(self, data): - if data["base_commit"] == data["head_commit"]: - raise serializers.ValidationError( - {"base_commit": "Base and head must be different commits"} - ) - return data - - class Meta: - model = LabelAnalysisRequest - fields = ( - "base_commit", - "head_commit", - "requested_labels", - "result", - "state", - "external_id", - "errors", - ) - read_only_fields = ("result", "external_id", "errors") - - def get_state(self, obj): - return LabelAnalysisRequestState.enum_from_int(obj.state_id).name.lower() diff --git a/labelanalysis/tests/integration/test_views.py b/labelanalysis/tests/integration/test_views.py index 6f153abc79..cd755e720f 100644 --- a/labelanalysis/tests/integration/test_views.py +++ b/labelanalysis/tests/integration/test_views.py @@ -1,30 +1,16 @@ -from uuid import uuid4 - from django.urls import reverse from rest_framework.test import APIClient -from shared.celery_config import label_analysis_task_name from shared.django_apps.core.tests.factories import ( CommitFactory, - RepositoryFactory, RepositoryTokenFactory, ) -from labelanalysis.models import ( - LabelAnalysisProcessingError, - LabelAnalysisRequest, - LabelAnalysisRequestState, -) -from labelanalysis.tests.factories import LabelAnalysisRequestFactory -from services.task import TaskService -from staticanalysis.tests.factories import StaticAnalysisSuiteFactory +from labelanalysis.views import EMPTY_RESPONSE -def test_simple_label_analysis_call_flow(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") +def test_simple_label_analysis_call_flow(db): commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) base_commit = CommitFactory.create(repository=commit.repository) - StaticAnalysisSuiteFactory.create(commit=base_commit) token = RepositoryTokenFactory.create( repository=commit.repository, token_type="static_analysis" ) @@ -42,365 +28,25 @@ def test_simple_label_analysis_call_flow(db, mocker): format="json", ) assert response.status_code == 201 - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 1 - produced_object = LabelAnalysisRequest.objects.get(head_commit=commit) - assert produced_object - assert produced_object.base_commit == base_commit - assert produced_object.head_commit == commit - assert produced_object.requested_labels is None - assert produced_object.state_id == LabelAnalysisRequestState.CREATED.db_id - assert produced_object.result is None - response_json = response.json() - expected_response_json = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - "result": None, - "state": "created", - "external_id": str(produced_object.external_id), - "errors": [], - } - assert response_json == expected_response_json - mocked_task_service.assert_called_with( - label_analysis_task_name, - kwargs={"request_id": produced_object.id}, - apply_async_kwargs={}, - ) - get_url = reverse( - "view_label_analysis", kwargs=dict(external_id=produced_object.external_id) - ) - response = client.get( - get_url, - format="json", - ) - assert response.status_code == 200 - assert response.json() == expected_response_json - - -def test_simple_label_analysis_call_flow_same_commit_error(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - client = APIClient() - url = reverse("create_label_analysis") - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - payload = { - "base_commit": commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - } - response = client.post( - url, - payload, - format="json", - ) - assert response.status_code == 400 - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 0 - response_json = response.json() - expected_response_json = { - "base_commit": ["Base and head must be different commits"] - } - assert response_json == expected_response_json - assert not mocked_task_service.called + assert response.json() == EMPTY_RESPONSE - -def test_simple_label_analysis_call_flow_with_fallback_on_base(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) - base_commit_parent_parent = CommitFactory.create(repository=commit.repository) - base_commit_parent = CommitFactory.create( - parent_commit_id=base_commit_parent_parent.commitid, - repository=commit.repository, - ) - base_commit = CommitFactory.create( - parent_commit_id=base_commit_parent.commitid, repository=commit.repository - ) - StaticAnalysisSuiteFactory.create(commit=base_commit_parent_parent) - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - client = APIClient() - url = reverse("create_label_analysis") - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - payload = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - } - response = client.post( - url, - payload, - format="json", - ) - assert response.status_code == 201 - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 1 - produced_object = LabelAnalysisRequest.objects.get(head_commit=commit) - assert produced_object - assert produced_object.base_commit == base_commit_parent_parent - assert produced_object.head_commit == commit - assert produced_object.requested_labels is None - assert produced_object.state_id == LabelAnalysisRequestState.CREATED.db_id - assert produced_object.result is None - response_json = response.json() - expected_response_json = { - "base_commit": base_commit_parent_parent.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - "result": None, - "state": "created", - "external_id": str(produced_object.external_id), - "errors": [], - } - assert response_json == expected_response_json - mocked_task_service.assert_called_with( - label_analysis_task_name, - kwargs={"request_id": produced_object.id}, - apply_async_kwargs={}, - ) - get_url = reverse( - "view_label_analysis", kwargs=dict(external_id=produced_object.external_id) - ) - response = client.get( - get_url, - format="json", - ) + get_url = reverse("view_label_analysis", kwargs={"external_id": "doesnotmatter"}) + response = client.get(get_url, format="json") assert response.status_code == 200 - assert response.json() == expected_response_json - - -def test_simple_label_analysis_call_flow_with_fallback_on_base_error_too_long( - db, mocker -): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - repository = RepositoryFactory.create(active=True) - commit = CommitFactory.create(repository=repository) - StaticAnalysisSuiteFactory.create(commit=commit) - base_commit_root = CommitFactory.create(repository=repository) - StaticAnalysisSuiteFactory.create(commit=base_commit_root) - current = base_commit_root - attempted_commit_list = [base_commit_root.commitid] - for i in range(12): - current = CommitFactory.create( - parent_commit_id=current.commitid, repository=repository - ) - attempted_commit_list.append(current.commitid) - base_commit = current - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - client = APIClient() - url = reverse("create_label_analysis") - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - payload = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - } - response = client.post( - url, - payload, - format="json", - ) - assert response.status_code == 400 - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 0 - response_json = response.json() - # reverse and get 10 first elements, thats how far we look - attempted_commit_list = ",".join(list(reversed(attempted_commit_list))[:10]) - expected_response_json = { - "base_commit": [ - f"No possible commits have static analysis sent. Attempted too many commits: {attempted_commit_list}" - ] - } - assert response_json == expected_response_json - assert not mocked_task_service.called - - -def test_simple_label_analysis_call_flow_with_fallback_on_base_error(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) - base_commit_parent_parent = CommitFactory.create(repository=commit.repository) - base_commit_parent = CommitFactory.create( - parent_commit_id=base_commit_parent_parent.commitid, - repository=commit.repository, - ) - base_commit = CommitFactory.create( - parent_commit_id=base_commit_parent.commitid, repository=commit.repository - ) - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - client = APIClient() - url = reverse("create_label_analysis") - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - payload = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - } - response = client.post( - url, - payload, - format="json", - ) - assert response.status_code == 400 - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 0 - response_json = response.json() - attempted_commit_list = ",".join( - [ - base_commit.commitid, - base_commit_parent.commitid, - base_commit_parent_parent.commitid, - ] - ) - expected_response_json = { - "base_commit": [ - f"No possible commits have static analysis sent. Attempted commits: {attempted_commit_list}" - ] - } - assert response_json == expected_response_json - assert not mocked_task_service.called - - -def test_simple_label_analysis_call_flow_with_fallback_on_head_error(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - repository = RepositoryFactory.create(active=True) - head_commit_parent = CommitFactory.create(repository=repository) - head_commit = CommitFactory.create( - parent_commit_id=head_commit_parent.commitid, repository=repository - ) - base_commit = CommitFactory.create(repository=repository) - StaticAnalysisSuiteFactory.create(commit=base_commit) - StaticAnalysisSuiteFactory.create(commit=head_commit_parent) - token = RepositoryTokenFactory.create( - repository=head_commit.repository, token_type="static_analysis" - ) - client = APIClient() - url = reverse("create_label_analysis") - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - payload = { - "base_commit": base_commit.commitid, - "head_commit": head_commit.commitid, - "requested_labels": None, - } - response = client.post( - url, - payload, - format="json", - ) - assert response.status_code == 400 - assert LabelAnalysisRequest.objects.filter(head_commit=head_commit).count() == 0 - assert ( - LabelAnalysisRequest.objects.filter(head_commit=head_commit_parent).count() == 0 - ) - response_json = response.json() - expected_response_json = {"head_commit": ["No static analysis found"]} - assert response_json == expected_response_json - assert not mocked_task_service.called + assert response.json() == EMPTY_RESPONSE -def test_simple_label_analysis_only_get(db, mocker): +def test_simple_label_analysis_put_labels(db): commit = CommitFactory.create(repository__active=True) base_commit = CommitFactory.create(repository=commit.repository) token = RepositoryTokenFactory.create( repository=commit.repository, token_type="static_analysis" ) - label_analysis = LabelAnalysisRequestFactory.create( - head_commit=commit, - base_commit=base_commit, - state_id=LabelAnalysisRequestState.FINISHED.db_id, - result={"some": ["result"]}, - ) - larq_processing_error = LabelAnalysisProcessingError( - label_analysis_request=label_analysis, - error_code="Missing FileSnapshot", - error_params={"message": "Something is wrong"}, - ) - label_analysis.save() - larq_processing_error.save() - client = APIClient() - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 1 - produced_object = LabelAnalysisRequest.objects.get(head_commit=commit) - assert produced_object == label_analysis - expected_response_json = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - "result": {"some": ["result"]}, - "state": "finished", - "external_id": str(produced_object.external_id), - "errors": [ - { - "error_code": "Missing FileSnapshot", - "error_params": {"message": "Something is wrong"}, - } - ], - } - get_url = reverse( - "view_label_analysis", kwargs=dict(external_id=produced_object.external_id) - ) - response = client.get( - get_url, - format="json", - ) - assert response.status_code == 200 - assert response.json() == expected_response_json - -def test_simple_label_analysis_get_does_not_exist(db, mocker): - token = RepositoryTokenFactory.create( - repository__active=True, token_type="static_analysis" - ) client = APIClient() client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - get_url = reverse("view_label_analysis", kwargs=dict(external_id=uuid4())) - response = client.get( - get_url, - format="json", - ) - assert response.status_code == 404 - assert response.json() == {"detail": "No such Label Analysis exists"} - -def test_simple_label_analysis_put_labels(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) - base_commit = CommitFactory.create(repository=commit.repository) - StaticAnalysisSuiteFactory.create(commit=base_commit) - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - label_analysis = LabelAnalysisRequestFactory.create( - head_commit=commit, - base_commit=base_commit, - state_id=LabelAnalysisRequestState.CREATED.db_id, - result=None, - ) - label_analysis.save() - client = APIClient() - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 1 - produced_object = LabelAnalysisRequest.objects.get(head_commit=commit) - assert produced_object == label_analysis - assert produced_object.requested_labels is None - expected_response_json = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": ["label_1", "label_2", "label_3"], - "result": None, - "state": "created", - "external_id": str(produced_object.external_id), - "errors": [], - } - patch_url = reverse( - "view_label_analysis", kwargs=dict(external_id=produced_object.external_id) - ) + patch_url = reverse("view_label_analysis", kwargs={"external_id": "doesnotmatter"}) response = client.patch( patch_url, format="json", @@ -411,47 +57,4 @@ def test_simple_label_analysis_put_labels(db, mocker): }, ) assert response.status_code == 200 - assert response.json() == expected_response_json - mocked_task_service.assert_called_with( - label_analysis_task_name, - kwargs=dict(request_id=label_analysis.id), - apply_async_kwargs=dict(), - ) - - -def test_simple_label_analysis_put_labels_wrong_base_return_404(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) - base_commit = CommitFactory.create(repository=commit.repository) - StaticAnalysisSuiteFactory.create(commit=base_commit) - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - label_analysis = LabelAnalysisRequestFactory.create( - head_commit=commit, - base_commit=base_commit, - state_id=LabelAnalysisRequestState.CREATED.db_id, - result=None, - ) - label_analysis.save() - client = APIClient() - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 1 - produced_object = LabelAnalysisRequest.objects.get(head_commit=commit) - assert produced_object == label_analysis - assert produced_object.requested_labels is None - patch_url = reverse( - "view_label_analysis", kwargs=dict(external_id=produced_object.external_id) - ) - response = client.patch( - patch_url, - format="json", - data={ - "requested_labels": ["label_1", "label_2", "label_3"], - "base_commit": "not_base_commit", - "head_commit": commit.commitid, - }, - ) - assert response.status_code == 404 - mocked_task_service.assert_not_called() + assert response.json() == EMPTY_RESPONSE diff --git a/labelanalysis/urls.py b/labelanalysis/urls.py index 2ebda9b54e..4deff7507f 100644 --- a/labelanalysis/urls.py +++ b/labelanalysis/urls.py @@ -1,19 +1,16 @@ from django.urls import path -from labelanalysis.views import ( - LabelAnalysisRequestCreateView, - LabelAnalysisRequestDetailView, -) +from labelanalysis.views import LabelAnalysisRequestView urlpatterns = [ path( "labels-analysis", - LabelAnalysisRequestCreateView.as_view(), + LabelAnalysisRequestView.as_view(), name="create_label_analysis", ), path( "labels-analysis/", - LabelAnalysisRequestDetailView.as_view(), + LabelAnalysisRequestView.as_view(), name="view_label_analysis", ), ] diff --git a/labelanalysis/views.py b/labelanalysis/views.py index 222ea6be3a..88dbf38c89 100644 --- a/labelanalysis/views.py +++ b/labelanalysis/views.py @@ -1,58 +1,37 @@ -from rest_framework.exceptions import NotFound -from rest_framework.generics import CreateAPIView, RetrieveAPIView, UpdateAPIView -from shared.celery_config import label_analysis_task_name +from rest_framework import status +from rest_framework.response import Response +from rest_framework.views import APIView from codecov_auth.authentication.repo_auth import RepositoryTokenAuthentication from codecov_auth.permissions import SpecificScopePermission -from labelanalysis.models import LabelAnalysisRequest, LabelAnalysisRequestState -from labelanalysis.serializers import LabelAnalysisRequestSerializer -from services.task import TaskService - -class LabelAnalysisRequestCreateView(CreateAPIView): - serializer_class = LabelAnalysisRequestSerializer +EMPTY_RESPONSE = { + "external_id": None, + "state": "finished", + "errors": [], + "requested_labels": [], + "base_commit": "", + "head_commit": "", + "result": { + "absent_labels": [], + "present_diff_labels": [], + "present_report_labels": [], + "global_level_labels": [], + }, +} + + +class LabelAnalysisRequestView(APIView): authentication_classes = [RepositoryTokenAuthentication] permission_classes = [SpecificScopePermission] # TODO Consider using a different permission scope required_scopes = ["static_analysis"] - def perform_create(self, serializer): - instance = serializer.save(state_id=LabelAnalysisRequestState.CREATED.db_id) - TaskService().schedule_task( - label_analysis_task_name, - kwargs=dict(request_id=instance.id), - apply_async_kwargs=dict(), - ) - return instance + def post(self, request, *args, **kwargs): + return Response(EMPTY_RESPONSE, status=status.HTTP_201_CREATED) - -class LabelAnalysisRequestDetailView(RetrieveAPIView, UpdateAPIView): - serializer_class = LabelAnalysisRequestSerializer - authentication_classes = [RepositoryTokenAuthentication] - permission_classes = [SpecificScopePermission] - # TODO Consider using a different permission scope - required_scopes = ["static_analysis"] + def get(self, request, *args, **kwargs): + return Response(EMPTY_RESPONSE) def patch(self, request, *args, **kwargs): - # This is called by the CLI to patch the request_labels information after it's collected - # First we let rest_framework validate and update the larq object - response = super().patch(request, *args, **kwargs) - if response.status_code == 200: - # IF the larq update was successful - # we trigger the task again for the same larq to update the result saved - # The result saved is what we use to get metrics - uid = self.kwargs.get("external_id") - larq = LabelAnalysisRequest.objects.get(external_id=uid) - TaskService().schedule_task( - label_analysis_task_name, - kwargs=dict(request_id=larq.id), - apply_async_kwargs=dict(), - ) - return response - - def get_object(self): - uid = self.kwargs.get("external_id") - try: - return LabelAnalysisRequest.objects.get(external_id=uid) - except LabelAnalysisRequest.DoesNotExist: - raise NotFound("No such Label Analysis exists") + return Response(EMPTY_RESPONSE) diff --git a/staticanalysis/serializers.py b/staticanalysis/serializers.py deleted file mode 100644 index d7056ebaee..0000000000 --- a/staticanalysis/serializers.py +++ /dev/null @@ -1,156 +0,0 @@ -import logging -import math - -from rest_framework import exceptions, serializers -from shared.api_archive.archive import ArchiveService, MinioEndpoints - -from core.models import Commit -from staticanalysis.models import ( - StaticAnalysisSingleFileSnapshot, - StaticAnalysisSingleFileSnapshotState, - StaticAnalysisSuite, - StaticAnalysisSuiteFilepath, -) - -log = logging.getLogger(__name__) - - -class CommitFromShaSerializerField(serializers.Field): - def to_representation(self, commit): - return commit.commitid - - def to_internal_value(self, commit_sha): - # TODO: Change this query when we change how we fetch URLs - commit = Commit.objects.filter( - repository__in=self.context["request"].auth.get_repositories(), - commitid=commit_sha, - ).first() - if commit is None: - raise exceptions.NotFound("Commit not found.") - return commit - - -def _dict_to_suite_filepath( - analysis_suite, - repository, - archive_service, - existing_file_snapshots_mapping, - file_dict, -): - if file_dict["file_hash"] in existing_file_snapshots_mapping: - db_element = existing_file_snapshots_mapping[file_dict["file_hash"]] - was_created = False - else: - path = MinioEndpoints.static_analysis_single_file.get_path( - version="v4", - repo_hash=archive_service.storage_hash, - location=f"{file_dict['file_hash']}.json", - ) - # Using get or create in the case the object was already - # created somewhere else first, but also because get_or_create - # is internally get_or_create_or_get, so Django handles the conflicts - # that can arise on race conditions on the create step - # We might choose to change it if the number of extra GETs become too much - ( - db_element, - was_created, - ) = StaticAnalysisSingleFileSnapshot.objects.get_or_create( - file_hash=file_dict["file_hash"], - repository=repository, - defaults=dict( - state_id=StaticAnalysisSingleFileSnapshotState.CREATED.db_id, - content_location=path, - ), - ) - if was_created: - log.debug( - "Created new snapshot for repository", - extra=dict(repoid=repository.repoid, snapshot_id=db_element.id), - ) - return StaticAnalysisSuiteFilepath( - filepath=file_dict["filepath"], - file_snapshot=db_element, - analysis_suite=analysis_suite, - ) - - -class StaticAnalysisSuiteFilepathField(serializers.ModelSerializer): - file_hash = serializers.UUIDField() - raw_upload_location = serializers.SerializerMethodField() - state = serializers.SerializerMethodField() - - class Meta: - model = StaticAnalysisSuiteFilepath - fields = [ - "filepath", - "file_hash", - "raw_upload_location", - "state", - ] - - def get_state(self, obj): - return StaticAnalysisSingleFileSnapshotState.enum_from_int( - obj.file_snapshot.state_id - ).name - - def get_raw_upload_location(self, obj): - # TODO: This has a built-in ttl of 10 seconds. - # We have to consider changing it in case customers are doing a few - # thousand uploads on the first time - return self.context["archive_service"].create_presigned_put( - obj.file_snapshot.content_location - ) - - -class FilepathListField(serializers.ListField): - child = StaticAnalysisSuiteFilepathField() - - def to_representation(self, data): - data = data.select_related( - "file_snapshot", - ).all() - return super().to_representation(data) - - -class StaticAnalysisSuiteSerializer(serializers.ModelSerializer): - commit = CommitFromShaSerializerField(required=True) - filepaths = FilepathListField() - - class Meta: - model = StaticAnalysisSuite - fields = ["external_id", "commit", "filepaths"] - read_only_fields = ["raw_upload_location", "external_id"] - - def create(self, validated_data): - file_metadata_array = validated_data.pop("filepaths") - # `validated_data` only contains `commit` after pop - obj = StaticAnalysisSuite.objects.create(**validated_data) - request = self.context["request"] - repository = request.auth.get_repositories()[0] - archive_service = ArchiveService(repository) - # allow 1s per 10 uploads - ttl = max(math.ceil(len(file_metadata_array) / 10) + 5, 10) - self.context["archive_service"] = ArchiveService(repository, ttl=ttl) - all_hashes = [val["file_hash"] for val in file_metadata_array] - existing_values = StaticAnalysisSingleFileSnapshot.objects.filter( - repository=repository, file_hash__in=all_hashes - ) - existing_values_mapping = {val.file_hash: val for val in existing_values} - created_filepaths = [ - _dict_to_suite_filepath( - obj, - repository, - archive_service, - existing_values_mapping, - file_dict, - ) - for file_dict in file_metadata_array - ] - StaticAnalysisSuiteFilepath.objects.bulk_create(created_filepaths) - log.info( - "Created static analysis filepaths", - extra=dict( - created_ids=[f.id for f in created_filepaths], repoid=repository.repoid - ), - ) - return obj diff --git a/staticanalysis/tests/test_views.py b/staticanalysis/tests/test_views.py index 7e04738df1..e6037450c0 100644 --- a/staticanalysis/tests/test_views.py +++ b/staticanalysis/tests/test_views.py @@ -2,23 +2,15 @@ from django.urls import reverse from rest_framework.test import APIClient -from shared.celery_config import static_analysis_task_name from shared.django_apps.core.tests.factories import ( CommitFactory, RepositoryTokenFactory, ) -from services.task import TaskService -from staticanalysis.models import StaticAnalysisSuite -from staticanalysis.tests.factories import StaticAnalysisSuiteFactory +from staticanalysis.views import EMPTY_RESPONSE -def test_simple_static_analysis_call_no_uploads_yet(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - mocked_presigned_put = mocker.patch( - "shared.storage.MinioStorageService.create_presigned_put", - return_value="banana.txt", - ) +def test_simple_static_analysis_call_no_uploads_yet(db): commit = CommitFactory.create(repository__active=True) token = RepositoryTokenFactory.create( repository=commit.repository, token_type="static_analysis" @@ -45,57 +37,17 @@ def test_simple_static_analysis_call_no_uploads_yet(db, mocker): format="json", ) assert response.status_code == 201 - assert StaticAnalysisSuite.objects.filter(commit=commit).count() == 1 - produced_object = StaticAnalysisSuite.objects.filter(commit=commit).get() - response_json = response.json() - assert "filepaths" in response_json - # Popping and sorting because the order doesn't matter, as long as all are there - assert sorted(response_json.pop("filepaths"), key=lambda x: x["filepath"]) == [ - { - "filepath": "banana.cpp", - "file_hash": str(second_uuid), - "raw_upload_location": "banana.txt", - "state": "CREATED", - }, - { - "filepath": "path/to/a.py", - "file_hash": str(some_uuid), - "raw_upload_location": "banana.txt", - "state": "CREATED", - }, - ] - # Now asserting the remaining of the response - assert response_json == { - "external_id": str(produced_object.external_id), - "commit": commit.commitid, - } - mocked_task_service.assert_called_with( - static_analysis_task_name, - kwargs={"suite_id": produced_object.id}, - apply_async_kwargs={"countdown": 10}, - ) - mocked_presigned_put.assert_called_with( - "archive", - mocker.ANY, - 10, - ) + assert response.json() == EMPTY_RESPONSE -def test_static_analysis_finish(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") +def test_static_analysis_finish(db): commit = CommitFactory.create(repository__active=True) - suite = StaticAnalysisSuiteFactory(commit=commit) token = RepositoryTokenFactory.create( repository=commit.repository, token_type="static_analysis" ) client = APIClient() client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) response = client.post( - reverse("staticanalyses-finish", kwargs={"external_id": suite.external_id}) - ) - assert response.status_code == 204 - mocked_task_service.assert_called_with( - static_analysis_task_name, - kwargs={"suite_id": suite.id}, - apply_async_kwargs={}, + reverse("staticanalyses-finish", kwargs={"external_id": "doesnotmatter"}) ) + assert response.status_code == 201 diff --git a/staticanalysis/urls.py b/staticanalysis/urls.py index 6a2a5510b8..87bf090921 100644 --- a/staticanalysis/urls.py +++ b/staticanalysis/urls.py @@ -1,7 +1,16 @@ -from staticanalysis.views import StaticAnalysisSuiteViewSet -from utils.routers import OptionalTrailingSlashRouter +from django.urls import path -router = OptionalTrailingSlashRouter() -router.register("analyses", StaticAnalysisSuiteViewSet, basename="staticanalyses") +from staticanalysis.views import StaticAnalysisSuiteView -urlpatterns = router.urls +urlpatterns = [ + path( + "staticanalyses", + StaticAnalysisSuiteView.as_view(), + name="staticanalyses-list", + ), + path( + "staticanalyses//finish", + StaticAnalysisSuiteView.as_view(), + name="staticanalyses-finish", + ), +] diff --git a/staticanalysis/views.py b/staticanalysis/views.py index 7045e63020..b557d333ab 100644 --- a/staticanalysis/views.py +++ b/staticanalysis/views.py @@ -1,46 +1,21 @@ -import logging - -from django.http import HttpResponse -from rest_framework import mixins, viewsets -from rest_framework.decorators import action -from shared.celery_config import static_analysis_task_name +from rest_framework import status +from rest_framework.response import Response +from rest_framework.views import APIView from codecov_auth.authentication.repo_auth import RepositoryTokenAuthentication from codecov_auth.permissions import SpecificScopePermission -from services.task import TaskService -from staticanalysis.models import StaticAnalysisSuite -from staticanalysis.serializers import StaticAnalysisSuiteSerializer -log = logging.getLogger(__name__) +EMPTY_RESPONSE = { + "external_id": "0000", + "filepaths": [], + "commit": "", +} -class StaticAnalysisSuiteViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet): - serializer_class = StaticAnalysisSuiteSerializer +class StaticAnalysisSuiteView(APIView): authentication_classes = [RepositoryTokenAuthentication] permission_classes = [SpecificScopePermission] required_scopes = ["static_analysis"] - lookup_field = "external_id" - - def get_queryset(self): - repository = self.request.auth.get_repositories()[0] - return StaticAnalysisSuite.objects.filter(commit__repository=repository) - - def perform_create(self, serializer): - instance = serializer.save() - # TODO: remove this once the CLI is calling the `finish` endpoint - TaskService().schedule_task( - static_analysis_task_name, - kwargs=dict(suite_id=instance.id), - apply_async_kwargs=dict(countdown=10), - ) - return instance - @action(detail=True, methods=["post"]) - def finish(self, request, *args, **kwargs): - suite = self.get_object() - TaskService().schedule_task( - static_analysis_task_name, - kwargs=dict(suite_id=suite.pk), - apply_async_kwargs={}, - ) - return HttpResponse(status=204) + def post(self, request, *args, **kwargs): + return Response(EMPTY_RESPONSE, status=status.HTTP_201_CREATED)