diff --git a/.github/workflows/build_assets.yml b/.github/workflows/build_assets.yml index b6629bd9..73c1b1cb 100644 --- a/.github/workflows/build_assets.yml +++ b/.github/workflows/build_assets.yml @@ -21,12 +21,11 @@ jobs: TARGET: macos # currently, wrapt pulls the arm64 version instead of the universal one, so the below is a hack CMD_REQS: > - mkdir -p pip-packages && cd pip-packages && pip wheel --no-cache-dir --no-binary tree_sitter,ijson,charset_normalizer,PyYAML .. && + mkdir -p pip-packages && cd pip-packages && pip wheel --no-cache-dir --no-binary ijson,charset_normalizer,PyYAML .. && rm $(ls | grep wrapt) && pip download wrapt --platform=universal2 --only-binary=:all: && pip install $(ls | grep wrapt) --force-reinstall && cd .. && pip install --no-deps --no-index --find-links=pip-packages pip-packages/* CMD_BUILD: > - STATICCODECOV_LIB_PATH=$(find build/ -maxdepth 1 -type d -name 'lib.*' -print -quit | xargs -I {} sh -c "find {} -type f -name 'staticcodecov*' -print -quit | sed 's|^./||'") && - pyinstaller --add-binary ${STATICCODECOV_LIB_PATH}:. --copy-metadata codecov-cli --hidden-import staticcodecov_languages --target-arch universal2 -F codecov_cli/main.py && + pyinstaller --copy-metadata codecov-cli --target-arch universal2 -F codecov_cli/main.py && mv dist/main dist/codecovcli_macos && lipo -archs dist/codecovcli_macos | grep 'x86_64 arm64' OUT_FILE_NAME: codecovcli_macos @@ -37,8 +36,7 @@ jobs: CMD_REQS: > pip install -r requirements.txt && pip install . CMD_BUILD: > - STATICCODECOV_LIB_PATH=$(find build/ -maxdepth 1 -type d -name 'lib.*' -print -quit | xargs -I {} sh -c "find {} -type f -name 'staticcodecov*' -print -quit | sed 's|^./||'") && - pyinstaller --add-binary ${STATICCODECOV_LIB_PATH}:. --copy-metadata codecov-cli --hidden-import staticcodecov_languages -F codecov_cli/main.py && + pyinstaller --copy-metadata codecov-cli -F codecov_cli/main.py && cp ./dist/main ./dist/codecovcli_linux OUT_FILE_NAME: codecovcli_linux ASSET_MIME: application/octet-stream @@ -48,15 +46,13 @@ jobs: CMD_REQS: > pip install -r requirements.txt && pip install . CMD_BUILD: > - pyinstaller --add-binary "build\lib.win-amd64-cpython-311\staticcodecov_languages.cp311-win_amd64.pyd;." --copy-metadata codecov-cli --hidden-import staticcodecov_languages -F codecov_cli\main.py && + pyinstaller --copy-metadata codecov-cli -F codecov_cli\main.py && Copy-Item -Path ".\dist\main.exe" -Destination ".\dist\codecovcli_windows.exe" OUT_FILE_NAME: codecovcli_windows.exe ASSET_MIME: application/vnd.microsoft.portable-executable steps: - uses: actions/checkout@v4 - with: - submodules: true - name: Set up Python 3.11 uses: actions/setup-python@v3 @@ -119,8 +115,6 @@ jobs: steps: - uses: actions/checkout@v4 - with: - submodules: true - name: Set up QEMU uses: docker/setup-qemu-action@v1 diff --git a/.github/workflows/build_for_pypi.yml b/.github/workflows/build_for_pypi.yml index 3d82e104..967b63cf 100644 --- a/.github/workflows/build_for_pypi.yml +++ b/.github/workflows/build_for_pypi.yml @@ -16,7 +16,6 @@ jobs: - uses: actions/checkout@v4 with: persist-credentials: false - submodules: true - name: Install dependencies run: | diff --git a/.github/workflows/ci-job.yml b/.github/workflows/ci-job.yml index 496676b7..5a4c0b06 100644 --- a/.github/workflows/ci-job.yml +++ b/.github/workflows/ci-job.yml @@ -12,7 +12,6 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodules: true fetch-depth: 2 - name: Set up Python 3.12 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53992a46..e585027d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,8 +16,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - submodules: true - name: Check linting with ruff run: | @@ -27,8 +25,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - with: - submodules: true - name: Set up Python 3.13 uses: actions/setup-python@v5 @@ -44,7 +40,7 @@ jobs: - name: Run command_dump run: | - ./command_dump.py + python command_dump.py - name: Detect changes on commit run: | @@ -60,7 +56,6 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodules: true fetch-depth: 2 - uses: actions/setup-python@v5 @@ -90,7 +85,6 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodules: true fetch-depth: 2 - name: Set up Python ${{matrix.python-version}} diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index d87fd851..00000000 --- a/.gitmodules +++ /dev/null @@ -1,6 +0,0 @@ -[submodule "languages/treesitterpython"] - path = codecov-cli/languages/treesitterpython - url = git@github.com:tree-sitter/tree-sitter-python.git -[submodule "languages/treesitterjavascript"] - path = codecov-cli/languages/treesitterjavascript - url = git@github.com:tree-sitter/tree-sitter-javascript.git diff --git a/codecov-cli/MANIFEST.in b/codecov-cli/MANIFEST.in index e0bbc1dc..4f3a2eaa 100644 --- a/codecov-cli/MANIFEST.in +++ b/codecov-cli/MANIFEST.in @@ -1,5 +1,3 @@ -include languages/treesitterjavascript/src/tree_sitter/parser.h -include languages/treesitterpython/src/tree_sitter/parser.h include requirements.txt recursive-include codecov_cli * diff --git a/codecov-cli/codecov_cli/services/staticanalysis/__init__.py b/codecov-cli/codecov_cli/services/staticanalysis/__init__.py deleted file mode 100644 index 16cd15fd..00000000 --- a/codecov-cli/codecov_cli/services/staticanalysis/__init__.py +++ /dev/null @@ -1,296 +0,0 @@ -import asyncio -import json -import logging -import typing -from functools import partial -from multiprocessing import Pool -from pathlib import Path - -import click -import httpx -import requests - -from codecov_cli.helpers import request -from codecov_cli.helpers.config import CODECOV_API_URL -from codecov_cli.services.staticanalysis.analyzers import get_best_analyzer -from codecov_cli.services.staticanalysis.exceptions import AnalysisError -from codecov_cli.services.staticanalysis.finders import select_file_finder -from codecov_cli.services.staticanalysis.types import ( - FileAnalysisRequest, - FileAnalysisResult, -) - -logger = logging.getLogger("codecovcli") - - -async def run_analysis_entrypoint( - config: typing.Optional[typing.Dict], - folder: Path, - numberprocesses: typing.Optional[int], - pattern, - commit: str, - token: str, - should_force: bool, - folders_to_exclude: typing.List[Path], - enterprise_url: typing.Optional[str], - args: dict, -): - ff = select_file_finder(config) - files = list(ff.find_files(folder, pattern, folders_to_exclude)) - processing_results = await process_files(files, numberprocesses, config) - # Let users know if there were processing errors - # This is here and not in the function so we can add an option to ignore those (possibly) - # Also makes the function easier to test - processing_errors = processing_results["processing_errors"] - log_processing_errors(processing_errors) - # Upload results metadata to codecov to get list of files that we need to upload - file_metadata = processing_results["file_metadata"] - all_data = processing_results["all_data"] - try: - json_output = {"commit": commit, "filepaths": file_metadata} - logger.info( - "Sending files fingerprints to Codecov", - extra=dict( - extra_log_attributes=dict( - files_effectively_analyzed=len(json_output["filepaths"]) - ) - ), - ) - logger.debug( - "Data sent to Codecov", - extra=dict(extra_log_attributes=dict(json_payload=json_output)), - ) - upload_url = enterprise_url or CODECOV_API_URL - response = request.post( - f"{upload_url}/staticanalysis/analyses", - data=json_output, - headers={"Authorization": f"Repotoken {token}"}, - ) - response_json = response.json() - if response.status_code >= 500: - raise click.ClickException("Sorry. Codecov is having problems") - if response.status_code >= 400: - raise click.ClickException( - f"There is some problem with the submitted information.\n{response_json.get('detail')}" - ) - except requests.RequestException: - raise click.ClickException(click.style("Unable to reach Codecov", fg="red")) - logger.info( - "Received response from server", - extra=dict( - extra_log_attributes=dict(time_taken=response.elapsed.total_seconds()) - ), - ) - logger.debug( - "Response", - extra=dict( - extra_log_attributes=dict( - response_json=response_json, - ) - ), - ) - - valid_files_len = len( - [el for el in response_json["filepaths"] if el["state"].lower() == "valid"] - ) - created_files_len = len( - [el for el in response_json["filepaths"] if el["state"].lower() == "created"] - ) - logger.info( - f"{valid_files_len} files VALID; {created_files_len} files CREATED", - ) - - files_that_need_upload = [ - el - for el in response_json["filepaths"] - if (el["state"].lower() == "created" or should_force) - ] - - if files_that_need_upload: - uploaded_files = [] - failed_uploads = [] - with click.progressbar( - length=len(files_that_need_upload), - label="Upload info to storage", - ) as bar: - # It's better to have less files competing over CPU time when uploading - # Especially if we might have large files - limits = httpx.Limits(max_connections=20) - # Because there might be too many files to upload we will ignore most timeouts - timeout = httpx.Timeout(read=None, pool=None, connect=None, write=10.0) - async with httpx.AsyncClient(timeout=timeout, limits=limits) as client: - all_tasks = [] - for el in files_that_need_upload: - all_tasks.append(send_single_upload_put(client, all_data, el)) - try: - for task in asyncio.as_completed(all_tasks): - resp = await task - bar.update(1, el["filepath"]) - if resp["succeeded"]: - uploaded_files.append(resp["filepath"]) - else: - failed_uploads.append(resp["filepath"]) - except asyncio.CancelledError: - message = ( - "Unknown error cancelled the upload tasks.\n" - + f"Uploaded {len(uploaded_files)}/{len(files_that_need_upload)} files successfully." - ) - raise click.ClickException(message) - if failed_uploads: - logger.warning(f"{len(failed_uploads)} files failed to upload") - logger.debug( - "Failed files", - extra=dict(extra_log_attributes=dict(filenames=failed_uploads)), - ) - logger.info( - f"Uploaded {len(uploaded_files)} files", - ) - logger.debug( - "Uploaded files", - extra=dict(extra_log_attributes=dict(filenames=uploaded_files)), - ) - else: - logger.info("All files are already uploaded!") - try: - response = send_finish_signal(response_json, upload_url, token) - except requests.RequestException: - raise click.ClickException(click.style("Unable to reach Codecov", fg="red")) - logger.info( - "Received response with status code %s from server", - response.status_code, - extra=dict( - extra_log_attributes=dict(time_taken=response.elapsed.total_seconds()) - ), - ) - log_processing_errors(processing_errors) - - -def log_processing_errors(processing_errors: typing.Dict[str, str]) -> None: - if len(processing_errors) > 0: - logger.error( - f"{len(processing_errors)} files have processing errors and have been IGNORED." - ) - for file, error in processing_errors.items(): - logger.error(f"-> {file}: ERROR {error}") - - -async def process_files( - files_to_analyze: typing.List[FileAnalysisRequest], - numberprocesses: int, - config: typing.Optional[typing.Dict], -): - logger.info(f"Running the analyzer on {len(files_to_analyze)} files") - mapped_func = partial(analyze_file, config) - all_data = {} - file_metadata = [] - errors = {} - with click.progressbar( - length=len(files_to_analyze), - label="Analyzing files", - ) as bar: - # https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods - # from the link above, we want to use the default start methods - with Pool(processes=numberprocesses) as pool: - file_results = pool.imap_unordered(mapped_func, files_to_analyze) - for result in file_results: - bar.update(1, result) - if result is not None: - if result.result: - all_data[result.filename] = result.result - file_metadata.append( - { - "filepath": result.filename, - "file_hash": result.result["hash"], - } - ) - elif result.error: - errors[result.filename] = result.error - logger.info("All files have been processed") - return dict( - all_data=all_data, file_metadata=file_metadata, processing_errors=errors - ) - - -async def send_single_upload_put(client, all_data, el) -> typing.Dict: - retryable_statuses = (429,) - presigned_put = el["raw_upload_location"] - number_retries = 5 - try: - for current_retry in range(number_retries): - response = await client.put( - presigned_put, data=json.dumps(all_data[el["filepath"]]) - ) - if response.status_code < 300: - return { - "status_code": response.status_code, - "filepath": el["filepath"], - "succeeded": True, - } - if response.status_code in retryable_statuses: - await asyncio.sleep(2**current_retry) - status_code = response.status_code - message_to_warn = response.text - exception = None - except httpx.HTTPError as exp: - status_code = None - exception = type(exp) - message_to_warn = str(exp) - logger.warning( - "Unable to send single_upload_put", - extra=dict( - extra_log_attributes=dict( - message=message_to_warn, - exception=exception, - filepath=el["filepath"], - latest_status_code=status_code, - ) - ), - ) - return { - "status_code": status_code, - "exception": exception, - "filepath": el["filepath"], - "succeeded": False, - } - - -def send_finish_signal(response_json, upload_url: str, token: str): - external_id = response_json["external_id"] - logger.debug( - "Sending finish signal to let API know to schedule static analysis task", - extra=dict(extra_log_attributes=dict(external_id=external_id)), - ) - response = request.post( - f"{upload_url}/staticanalysis/analyses/{external_id}/finish", - headers={"Authorization": f"Repotoken {token}"}, - ) - if response.status_code >= 500: - raise click.ClickException("Sorry. Codecov is having problems") - if response.status_code >= 400: - raise click.ClickException( - f"There is some problem with the submitted information.\n{response_json.get('detail')}" - ) - return response - - -def analyze_file( - config, filename: FileAnalysisRequest -) -> typing.Optional[FileAnalysisResult]: - try: - with open(filename.actual_filepath, "rb") as file: - actual_code = file.read() - analyzer = get_best_analyzer(filename, actual_code) - if analyzer is None: - return None - output = analyzer.process() - if output is None: - return None - return FileAnalysisResult(filename=filename.result_filename, result=output) - except AnalysisError as e: - error_dict = { - "filename": str(filename.result_filename), - "error": str(e), - } - return FileAnalysisResult( - filename=str(filename.result_filename), error=error_dict - ) diff --git a/codecov-cli/codecov_cli/services/staticanalysis/analyzers/__init__.py b/codecov-cli/codecov_cli/services/staticanalysis/analyzers/__init__.py deleted file mode 100644 index 077cbc5f..00000000 --- a/codecov-cli/codecov_cli/services/staticanalysis/analyzers/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -from codecov_cli.services.staticanalysis.analyzers.general import BaseAnalyzer -from codecov_cli.services.staticanalysis.analyzers.javascript_es6 import ES6Analyzer -from codecov_cli.services.staticanalysis.analyzers.python import PythonAnalyzer -from codecov_cli.services.staticanalysis.types import FileAnalysisRequest - - -def get_best_analyzer( - filename: FileAnalysisRequest, actual_code: bytes -) -> BaseAnalyzer: - if filename.actual_filepath.suffix == ".py": - return PythonAnalyzer(filename, actual_code) - if filename.actual_filepath.suffix == ".js": - return ES6Analyzer(filename, actual_code) - return None diff --git a/codecov-cli/codecov_cli/services/staticanalysis/analyzers/general.py b/codecov-cli/codecov_cli/services/staticanalysis/analyzers/general.py deleted file mode 100644 index c0554f73..00000000 --- a/codecov-cli/codecov_cli/services/staticanalysis/analyzers/general.py +++ /dev/null @@ -1,124 +0,0 @@ -import hashlib -from collections import deque - - -class BaseAnalyzer(object): - def __init__(self, filename, actual_code): - pass - - def process(self): - return {} - - def _count_elements(self, node, types): - count = 0 - for c in node.children: - count += self._count_elements(c, types) - if node.type in types: - count += 1 - return count - - def _get_max_nested_conditional(self, head): - """Iterates over all nodes in a function body and returns the max nested conditional depth. - Uses BFS to avoid recursion calls (so we don't throw RecursionError) - """ - nodes_to_visit = deque() - nodes_to_visit.append([head, int(head.type in self.condition_statements)]) - max_nested_depth = 0 - - while nodes_to_visit: - curr_node, curr_depth = nodes_to_visit.popleft() - max_nested_depth = max(max_nested_depth, curr_depth) - # Here is where the depth might change - # If the current node is a conditional - is_curr_conditional = curr_node.type in self.condition_statements - - # Enqueue all child nodes of the curr_node - for child in curr_node.children: - nodes_to_visit.append([child, curr_depth + is_curr_conditional]) - - return max_nested_depth - - def _get_complexity_metrics(self, body_node): - number_conditions = self._count_elements( - body_node, - self.condition_statements, - ) - return { - "conditions": number_conditions, - "mccabe_cyclomatic_complexity": number_conditions + 1, - "returns": self._count_elements(body_node, ["return_statement"]), - "max_nested_conditional": self._get_max_nested_conditional(body_node), - } - - def _get_name(self, node): - name_node = node.child_by_field_name("name") - body_node = node.child_by_field_name("body") - actual_name = ( - self.actual_code[name_node.start_byte : name_node.end_byte].decode() - if name_node - else f"Anonymous_{body_node.start_point[0] + 1}_{body_node.end_point[0] - body_node.start_point[0]}" - ) - wrapping_classes = [ - x for x in self._get_parent_chain(node) if x.type in self.wrappers - ] - wrapping_classes.reverse() - if wrapping_classes: - parents_actual_names = "" - - for x in wrapping_classes: - name = x.child_by_field_name("name") - body = x.child_by_field_name("body") - class_name = ( - self.actual_code[name.start_byte : name.end_byte].decode() - if name - else f"Anonymous_{body.start_point[0] + 1}_{body.end_point[0] - body.start_point[0]}" - ) - parents_actual_names = parents_actual_names + class_name + "::" - return f"{parents_actual_names}{actual_name}" - return actual_name - - def _get_parent_chain(self, node): - cur = node.parent - while cur: - yield cur - cur = cur.parent - - def get_import_lines(self, root_node, imports_query): - import_lines = set() - for a, _ in imports_query.captures(root_node): - import_lines.add((a.start_point[0] + 1, a.end_point[0] - a.start_point[0])) - return import_lines - - def get_definition_lines(self, root_node, definitions_query): - definition_lines = set() - for a, _ in definitions_query.captures(root_node): - definition_lines.add( - (a.start_point[0] + 1, a.end_point[0] - a.start_point[0]) - ) - return definition_lines - - def _get_code_hash(self, start_byte, end_byte): - j = hashlib.md5() - j.update(self.actual_code[start_byte:end_byte].strip()) - return j.hexdigest() - - def get_statements(self): - return sorted( - ( - ( - x["current_line"], - { - "line_surety_ancestorship": self.line_surety_ancestorship.get( - x["current_line"], None - ), - **dict( - (k, v) - for (k, v) in x.items() - if k not in ["line_surety_ancestorship", "current_line"] - ), - }, - ) - for x in self.statements - ), - key=lambda x: (x[0], x[1]["start_column"]), - ) diff --git a/codecov-cli/codecov_cli/services/staticanalysis/analyzers/javascript_es6/__init__.py b/codecov-cli/codecov_cli/services/staticanalysis/analyzers/javascript_es6/__init__.py deleted file mode 100644 index 107a34b2..00000000 --- a/codecov-cli/codecov_cli/services/staticanalysis/analyzers/javascript_es6/__init__.py +++ /dev/null @@ -1,122 +0,0 @@ -import hashlib - -from tree_sitter import Language, Parser - -import staticcodecov_languages -from codecov_cli.services.staticanalysis.analyzers.general import BaseAnalyzer -from codecov_cli.services.staticanalysis.analyzers.javascript_es6.node_wrappers import ( - NodeVisitor, -) - -function_query_str = """ -(function_declaration) @elemen -(generator_function_declaration) @elemen2 -(function) @elemen3 -(generator_function) @elemen4 -(arrow_function) @elemen5 -""" - -method_query_str = """ -(method_definition) @elemen -""" - -imports_query_str = """ -(import_statement) @elemen -(import) @elemen -""" - -definitions_query_str = """ -(function_declaration) @elemen -(generator_function_declaration) @elemen2 -(function) @elemen3 -(generator_function) @elemen4 -(arrow_function) @elemen5 -(method_definition) @elemen6 -(class_declaration) @elemen7 -""" - - -class ES6Analyzer(BaseAnalyzer): - condition_statements = [ - "if_statement", - "switch_statement", - "for_statement", - "for_in_statement", - "while_statement", - "do_statement", - ] - - wrappers = [ - "class_declaration", - "function_declaration", - "generator_function_declaration", - "function", - "generator_function", - "arrow_function", - ] - - def __init__(self, path, actual_code, **options): - self.actual_code = actual_code - self.lines = self.actual_code.split(b"\n") - self.executable_lines = set() - self.functions = [] - self.path = path.result_filename - self.JS_LANGUAGE = Language(staticcodecov_languages.__file__, "javascript") - self.parser = Parser() - self.parser.set_language(self.JS_LANGUAGE) - self.import_lines = set() - self.definitions_lines = set() - self.line_surety_ancestorship = {} - self.statements = [] - - def get_code_hash(self, start_byte, end_byte): - j = hashlib.md5() - j.update(self.actual_code[start_byte:end_byte].strip()) - return j.hexdigest() - - def process(self): - tree = self.parser.parse(self.actual_code) - root_node = tree.root_node - function_query = self.JS_LANGUAGE.query(function_query_str) - method_query = self.JS_LANGUAGE.query(method_query_str) - imports_query = self.JS_LANGUAGE.query(imports_query_str) - definitions_query = self.JS_LANGUAGE.query(definitions_query_str) - combined_results = function_query.captures(root_node) + method_query.captures( - root_node - ) - for func_node, _ in combined_results: - body_node = func_node.child_by_field_name("body") - self.functions.append( - { - "identifier": self._get_name(func_node), - "start_line": func_node.start_point[0] + 1, - "end_line": func_node.end_point[0] + 1, - "code_hash": self.get_code_hash( - body_node.start_byte, body_node.end_byte - ), - "complexity_metrics": self._get_complexity_metrics(body_node), - } - ) - self.functions = sorted(self.functions, key=lambda x: x["start_line"]) - - self.import_lines = self.get_import_lines(root_node, imports_query) - self.definition_lines = self.get_definition_lines(root_node, definitions_query) - - visitor = NodeVisitor(self) - visitor.start_visit(tree.root_node) - statements = self.get_statements() - - h = hashlib.md5() - h.update(self.actual_code) - return { - "empty_lines": [i + 1 for (i, n) in enumerate(self.lines) if not n.strip()], - "executable_lines": sorted(self.executable_lines), - "functions": self.functions, - "number_lines": len(self.lines), - "hash": h.hexdigest(), - "filename": str(self.path), - "language": "javascript", - "import_lines": sorted(self.import_lines), - "definition_lines": sorted(self.definition_lines), - "statements": statements, - } diff --git a/codecov-cli/codecov_cli/services/staticanalysis/analyzers/javascript_es6/node_wrappers.py b/codecov-cli/codecov_cli/services/staticanalysis/analyzers/javascript_es6/node_wrappers.py deleted file mode 100644 index 9a364a47..00000000 --- a/codecov-cli/codecov_cli/services/staticanalysis/analyzers/javascript_es6/node_wrappers.py +++ /dev/null @@ -1,71 +0,0 @@ -class NodeVisitor(object): - def __init__(self, analyzer): - self.analyzer = analyzer - - def start_visit(self, node): - self.visit(node) - - def visit(self, node): - self.do_visit(node) - for c in node.children: - self.visit(c) - - def do_visit(self, node): - if node.is_named: - current_line_number = node.start_point[0] + 1 - if node.type in ( - "expression_statement", - "variable_declaration", - "lexical_declaration", - "return_statement", - "if_statement", - "for_statement", - "for_in_statement", - "while_statement", - "do_statement", - "switch_statement", - ): - if node.prev_named_sibling: - self.analyzer.line_surety_ancestorship[current_line_number] = ( - node.prev_named_sibling.start_point[0] + 1 - ) - self.analyzer.statements.append( - { - "current_line": current_line_number, - "start_column": node.start_point[1], - "line_hash": self.analyzer._get_code_hash( - node.start_byte, node.end_byte - ), - "len": node.end_point[0] + 1 - current_line_number, - "extra_connected_lines": tuple(), - } - ) - if node.type in ("if_statement",): - first_if_statement = node.child_by_field_name("consequence") - if first_if_statement.type == "statement_block": - first_if_statement = first_if_statement.children[1] - if first_if_statement.type == "expression_statement": - first_if_statement = first_if_statement.children[0] - self.analyzer.line_surety_ancestorship[ - first_if_statement.start_point[0] + 1 - ] = current_line_number - - if node.type in ("for_statement", "while_statement", "for_in_statement"): - first_statement = node.child_by_field_name("body") - if first_statement.type == "statement_block": - first_statement = first_statement.children[1] - if first_statement.type == "expression_statement": - first_statement = first_statement.children[0] - self.analyzer.line_surety_ancestorship[ - first_statement.start_point[0] + 1 - ] = current_line_number - - if node.type == "do_statement": - do_statement_body = node.child_by_field_name("body") - if do_statement_body.type == "statement_block": - do_statement_body = do_statement_body.children[1] - elif do_statement_body.type == "expression_statement": - do_statement_body = do_statement_body.children[0] - self.analyzer.line_surety_ancestorship[ - do_statement_body.start_point[0] + 1 - ] = current_line_number diff --git a/codecov-cli/codecov_cli/services/staticanalysis/analyzers/python/__init__.py b/codecov-cli/codecov_cli/services/staticanalysis/analyzers/python/__init__.py deleted file mode 100644 index d5e6db0c..00000000 --- a/codecov-cli/codecov_cli/services/staticanalysis/analyzers/python/__init__.py +++ /dev/null @@ -1,110 +0,0 @@ -import hashlib - -from tree_sitter import Language, Parser - -import staticcodecov_languages -from codecov_cli.services.staticanalysis.analyzers.general import BaseAnalyzer -from codecov_cli.services.staticanalysis.analyzers.python.node_wrappers import ( - NodeVisitor, -) -from codecov_cli.services.staticanalysis.types import FileAnalysisRequest - -_function_query_str = """ -(function_definition - name: (identifier) - parameters: (parameters) -) @elemen -""" - -_unreachable_code_query_str = """ -(function_definition - body: (block (return_statement) @return_stmt . (_)) -) -""" - -_executable_lines_query_str = """ -(block (_) @elem) -(expression_statement) @elemf -""" - -_definitions_query_str = """ -(function_definition) @elemc -(class_definition) @elemd -(decorated_definition) @eleme -""" - -_imports_query_str = """ -(import_statement) @elema -(import_from_statement) @elemb -""" - -_wildcard_import_query_str = """ -(wildcard_import) @elema -""" - - -class PythonAnalyzer(BaseAnalyzer): - condition_statements = [ - "if_statement", - "while_statement", - "for_statement", - "conditional_expression", - ] - wrappers = ["class_definition", "function_definition"] - - def __init__( - self, file_analysis_request: FileAnalysisRequest, actual_code: bytes, **options - ): - self.actual_code = actual_code - self.lines = self.actual_code.split(b"\n") - self.statements = [] - self.import_lines = set() - self.definitions_lines = set() - self.functions = [] - self.path = file_analysis_request.result_filename - self.PY_LANGUAGE = Language(staticcodecov_languages.__file__, "python") - self.parser = Parser() - self.parser.set_language(self.PY_LANGUAGE) - self.line_surety_ancestorship = {} - - def process(self): - function_query = self.PY_LANGUAGE.query(_function_query_str) - definitions_query = self.PY_LANGUAGE.query(_definitions_query_str) - imports_query = self.PY_LANGUAGE.query(_imports_query_str) - tree = self.parser.parse(self.actual_code) - root_node = tree.root_node - captures = function_query.captures(root_node) - for node, _ in captures: - actual_name = self._get_name(node) - body_node = node.child_by_field_name("body") - self.functions.append( - { - "identifier": actual_name, - "start_line": node.start_point[0] + 1, - "end_line": node.end_point[0] + 1, - "code_hash": self._get_code_hash( - body_node.start_byte, body_node.end_byte - ), - "complexity_metrics": self._get_complexity_metrics(body_node), - } - ) - visitor = NodeVisitor(self) - visitor.start_visit(tree.root_node) - self.functions = sorted(self.functions, key=lambda x: x["start_line"]) - - self.import_lines = self.get_import_lines(root_node, imports_query) - self.definitions_lines = self.get_definition_lines(root_node, definitions_query) - - h = hashlib.md5() - h.update(self.actual_code) - statements = self.get_statements() - return { - "language": "python", - "empty_lines": [i + 1 for (i, n) in enumerate(self.lines) if not n.strip()], - "functions": self.functions, - "hash": h.hexdigest(), - "number_lines": len(self.lines), - "statements": statements, - "definition_lines": sorted(self.definitions_lines), - "import_lines": sorted(self.import_lines), - } diff --git a/codecov-cli/codecov_cli/services/staticanalysis/analyzers/python/node_wrappers.py b/codecov-cli/codecov_cli/services/staticanalysis/analyzers/python/node_wrappers.py deleted file mode 100644 index 681eeb1a..00000000 --- a/codecov-cli/codecov_cli/services/staticanalysis/analyzers/python/node_wrappers.py +++ /dev/null @@ -1,117 +0,0 @@ -from tree_sitter import Node - -from codecov_cli.services.staticanalysis.exceptions import AnalysisError - - -class NodeVisitor(object): - def __init__(self, analyzer): - self.analyzer = analyzer - - def start_visit(self, node): - self.visit(node) - - def visit(self, node: Node): - self.do_visit(node) - for c in node.children: - self.visit(c) - - def _is_function_docstring(self, node: Node): - """Skips docstrings for functions, such as this one. - Pytest doesn't include them in the report, so I don't think we should either, - at least for now. - """ - # Docstrings have type 'expression_statement - if node.type != "expression_statement": - return False - # Docstrings for a module are OK - they show up in pytest result - # Docstrings for a class are OK - they show up in pytest result - # Docstrings for functions are NOT OK - they DONT show up in pytest result - # Check if it's docstring - has_single_child = len(node.children) == 1 - only_child_is_string = node.children[0].type == "string" - # Check if is the first line of a function - parent_is_block = node.parent.type == "block" - first_exp_in_block = node.prev_named_sibling is None - is_in_function_context = ( - parent_is_block and node.parent.parent.type == "function_definition" - ) - - return ( - has_single_child - and only_child_is_string - and parent_is_block - and first_exp_in_block - and is_in_function_context - ) - - def _get_previous_sibling_that_is_not_comment_not_func_docstring(self, node: Node): - curr = node.prev_named_sibling - while curr is not None and ( - curr.type == "comment" or self._is_function_docstring(curr) - ): - curr = curr.prev_named_sibling - return curr - - def do_visit(self, node: Node): - if node.is_named: - current_line_number = node.start_point[0] + 1 - if node.type in ( - "expression_statement", - "return_statement", - "if_statement", - "for_statement", - "while_statement", - ): - if self._is_function_docstring(node): - # We ignore these - return - closest_named_sibling_not_comment_that_is_in_statements = ( - self._get_previous_sibling_that_is_not_comment_not_func_docstring( - node - ) - ) - if closest_named_sibling_not_comment_that_is_in_statements: - self.analyzer.line_surety_ancestorship[current_line_number] = ( - closest_named_sibling_not_comment_that_is_in_statements.start_point[ - 0 - ] - + 1 - ) - self.analyzer.statements.append( - { - "current_line": current_line_number, - "start_column": node.start_point[1], - "line_hash": self.analyzer._get_code_hash( - node.start_byte, node.end_byte - ), - "len": node.end_point[0] + 1 - current_line_number, - "extra_connected_lines": tuple(), - } - ) - if node.type in ("if_statement", "elif_clause"): - # Some of the children of a node have a field_name associated to them - # In the case of an if and elif, "consequence" is the code that is executed in that branch of code - first_if_statement = node.child_by_field_name("consequence") - try: - if first_if_statement.type == "block": - first_if_statement = first_if_statement.children[0] # BUG - except IndexError: - raise AnalysisError( - f"if_statement consequence is empty block @ {self.analyzer.path}:{first_if_statement.start_point[0] + 1}, column {first_if_statement.start_point[1]}" - ) - self.analyzer.line_surety_ancestorship[ - first_if_statement.start_point[0] + 1 - ] = current_line_number - if node.type in ("for_statement", "while_statement"): - first_loop_statement = node.child_by_field_name("body") - try: - if first_loop_statement.type == "block": - first_loop_statement = first_loop_statement.children[0] - except IndexError: - raise AnalysisError( - f"loop_statement body is empty block @ {self.analyzer.path}:{first_loop_statement.start_point[0] + 1}, column {first_loop_statement.start_point[1]}" - ) - self.analyzer.line_surety_ancestorship[ - first_loop_statement.start_point[0] + 1 - ] = current_line_number - pass diff --git a/codecov-cli/codecov_cli/services/staticanalysis/exceptions.py b/codecov-cli/codecov_cli/services/staticanalysis/exceptions.py deleted file mode 100644 index 0e6e67dc..00000000 --- a/codecov-cli/codecov_cli/services/staticanalysis/exceptions.py +++ /dev/null @@ -1,2 +0,0 @@ -class AnalysisError(Exception): - pass diff --git a/codecov-cli/codecov_cli/services/staticanalysis/finders.py b/codecov-cli/codecov_cli/services/staticanalysis/finders.py deleted file mode 100644 index e3c11eee..00000000 --- a/codecov-cli/codecov_cli/services/staticanalysis/finders.py +++ /dev/null @@ -1,47 +0,0 @@ -import subprocess -from pathlib import Path - -from codecov_cli.helpers.folder_searcher import globs_to_regex, search_files -from codecov_cli.services.staticanalysis.types import FileAnalysisRequest - - -class FileFinder(object): - def find_files(self, root_folder, pattern, exclude_folders): - regex_patterns_to_include = globs_to_regex( - [ - pattern, - ] - ) - exclude_folders = list(map(str, exclude_folders)) - files_paths = search_files( - folder_to_search=root_folder, - folders_to_ignore=exclude_folders, - filename_include_regex=regex_patterns_to_include, - ) - - return [ - FileAnalysisRequest( - actual_filepath=p, result_filename=str(p.relative_to(root_folder)) - ) - for p in files_paths - ] - - -class GitFileFinder(object): - def find_files(self, folder_name, pattern, exclude_folders): - res = subprocess.run( - ["git", "-C", str(folder_name), "ls-files"], capture_output=True - ) - return [ - FileAnalysisRequest( - actual_filepath=f"{Path(folder_name) / x}", result_filename=x - ) - for x in res.stdout.decode().split() - ] - - def find_configuration_file(self, folder_name): - return None - - -def select_file_finder(config): - return FileFinder() diff --git a/codecov-cli/codecov_cli/services/staticanalysis/types.py b/codecov-cli/codecov_cli/services/staticanalysis/types.py deleted file mode 100644 index f189dbfe..00000000 --- a/codecov-cli/codecov_cli/services/staticanalysis/types.py +++ /dev/null @@ -1,19 +0,0 @@ -import pathlib -from dataclasses import dataclass -from typing import Optional - - -@dataclass -class FileAnalysisRequest(object): - result_filename: str - actual_filepath: pathlib.Path - - -@dataclass -class FileAnalysisResult(object): - filename: str - result: Optional[dict] = None - error: Optional[dict] = None - - def asdict(self): - return {"result": self.result, "error": self.error} diff --git a/codecov-cli/languages/languages.c b/codecov-cli/languages/languages.c deleted file mode 100644 index 7a2aac1a..00000000 --- a/codecov-cli/languages/languages.c +++ /dev/null @@ -1,21 +0,0 @@ -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#include - -static PyMethodDef module_methods[] = { - NULL -}; - -static struct PyModuleDef module_definition = { - .m_base = PyModuleDef_HEAD_INIT, - .m_name = "binding", - .m_doc = NULL, - .m_size = -1, - .m_methods = NULL, -}; - -PyMODINIT_FUNC PyInit_staticcodecov_languages(void) { - PyObject *module = PyModule_Create(&module_definition); - if (module == NULL) return NULL; - return module; -} \ No newline at end of file diff --git a/codecov-cli/languages/treesitterjavascript b/codecov-cli/languages/treesitterjavascript deleted file mode 160000 index 936d976a..00000000 --- a/codecov-cli/languages/treesitterjavascript +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 936d976a782e75395d9b1c8c7c7bf4ba6fe0d86b diff --git a/codecov-cli/languages/treesitterpython b/codecov-cli/languages/treesitterpython deleted file mode 160000 index de221ecc..00000000 --- a/codecov-cli/languages/treesitterpython +++ /dev/null @@ -1 +0,0 @@ -Subproject commit de221eccf9a221f5b85474a553474a69b4b5784d diff --git a/codecov-cli/pyproject.toml b/codecov-cli/pyproject.toml index 0593e675..53076bc4 100644 --- a/codecov-cli/pyproject.toml +++ b/codecov-cli/pyproject.toml @@ -19,14 +19,13 @@ dependencies = [ "responses==0.21.*", "sentry-sdk>=2.20.0", "test-results-parser==0.5.4", - "tree-sitter==0.20.*", "wrapt>=1.17.2", ] license = {file = "LICENSE"} name = "codecov-cli" readme = "README.md" requires-python = ">= 3.9" -version = "10.4.1" +version = "10.4.0" [project.scripts] codecov = "codecov_cli.main:run" diff --git a/codecov-cli/requirements.txt b/codecov-cli/requirements.txt index 3cd4e69d..30128dbc 100644 --- a/codecov-cli/requirements.txt +++ b/codecov-cli/requirements.txt @@ -29,8 +29,6 @@ idna==3.10 # requests ijson==3.3.0 # via codecov-cli (pyproject.toml) -packaging==24.2 - # via setuptools-scm pyyaml==6.0.2 # via codecov-cli (pyproject.toml) regex==2024.11.6 @@ -41,16 +39,12 @@ responses==0.21.0 # via codecov-cli (pyproject.toml) sentry-sdk==2.20.0 # via codecov-cli (pyproject.toml) -setuptools-scm==8.1.0 - # via codecov-cli (pyproject.toml::build-system.requires) sniffio==1.3.1 # via # anyio # httpx test-results-parser==0.5.4 # via codecov-cli (pyproject.toml) -tree-sitter==0.20.4 - # via codecov-cli (pyproject.toml) typing-extensions==4.12.2 # via anyio urllib3==2.3.0 diff --git a/codecov-cli/scripts/build_alpine_arm.sh b/codecov-cli/scripts/build_alpine_arm.sh index 20f72770..a18691eb 100755 --- a/codecov-cli/scripts/build_alpine_arm.sh +++ b/codecov-cli/scripts/build_alpine_arm.sh @@ -4,7 +4,6 @@ apk add musl-dev build-base pip install -r requirements.txt pip install . python setup.py build -STATICCODECOV_LIB_PATH=$(find build/ -maxdepth 1 -type d -name 'lib.*' -print -quit | xargs -I {} sh -c "find {} -type f -name 'staticcodecov*' -print -quit | sed 's|^./||'") pip install pyinstaller -pyinstaller --add-binary ${STATICCODECOV_LIB_PATH}:. --copy-metadata codecov-cli --hidden-import staticcodecov_languages -F codecov_cli/main.py +pyinstaller --copy-metadata codecov-cli -F codecov_cli/main.py cp ./dist/main ./dist/codecovcli_$1 diff --git a/codecov-cli/scripts/build_linux_arm.sh b/codecov-cli/scripts/build_linux_arm.sh index 3dab760b..b389584c 100755 --- a/codecov-cli/scripts/build_linux_arm.sh +++ b/codecov-cli/scripts/build_linux_arm.sh @@ -4,7 +4,6 @@ apt install build-essential pip install -r requirements.txt pip install . python setup.py build -STATICCODECOV_LIB_PATH=$(find build/ -maxdepth 1 -type d -name 'lib.*' -print -quit | xargs -I {} sh -c "find {} -type f -name 'staticcodecov*' -print -quit | sed 's|^./||'") pip install pyinstaller -pyinstaller --add-binary ${STATICCODECOV_LIB_PATH}:. --copy-metadata codecov-cli --hidden-import staticcodecov_languages -F codecov_cli/main.py +pyinstaller --copy-metadata codecov-cli -F codecov_cli/main.py cp ./dist/main ./dist/codecovcli_$1 diff --git a/codecov-cli/setup.py b/codecov-cli/setup.py index 8234a4a5..60684932 100644 --- a/codecov-cli/setup.py +++ b/codecov-cli/setup.py @@ -1,27 +1,3 @@ -from platform import system +from setuptools import setup -from setuptools import Extension, setup - -setup( - ext_modules=[ - Extension( - "staticcodecov_languages", - [ - "languages/languages.c", - "languages/treesitterpython/src/parser.c", - "languages/treesitterjavascript/src/parser.c", - "languages/treesitterpython/src/scanner.cc", - "languages/treesitterjavascript/src/scanner.c", - ], - include_dirs=[ - "languages/treesitterpython/src", - "languages/treesitterjavascript/src", - "languages/treesitterjavascript/src/tree_sitter", - "languages/treesitterpython/src/tree_sitter", - ], - extra_compile_args=( - ["-Wno-unused-variable"] if system() != "Windows" else None - ), - ) - ], -) +setup() diff --git a/codecov-cli/tests/services/static_analysis/languages/python/test_node_wrappers_malformed_code.py b/codecov-cli/tests/services/static_analysis/languages/python/test_node_wrappers_malformed_code.py deleted file mode 100644 index aa7f7fd1..00000000 --- a/codecov-cli/tests/services/static_analysis/languages/python/test_node_wrappers_malformed_code.py +++ /dev/null @@ -1,65 +0,0 @@ -import pathlib - -import pytest -from tree_sitter import Node - -from codecov_cli.services.staticanalysis.analyzers.python import PythonAnalyzer -from codecov_cli.services.staticanalysis.analyzers.python.node_wrappers import ( - NodeVisitor, -) -from codecov_cli.services.staticanalysis.exceptions import AnalysisError -from codecov_cli.services.staticanalysis.types import FileAnalysisRequest - - -class TestMalformedIfStatements(object): - def test_if_empty_block_raises_analysis_error(self): - analysis_request = FileAnalysisRequest( - actual_filepath=pathlib.Path("test_file"), result_filename="test_file" - ) - # Code for an empty IF. NOT valid python code - actual_code = b'x = 10\nif x == "batata":\n\n' - python_analyser = PythonAnalyzer(analysis_request, actual_code=actual_code) - # Parse the code snippet and get the if_statement node - tree = python_analyser.parser.parse(actual_code) - root = tree.root_node - assert root.type == "module" - assert root.child_count == 2 - if_statement_node = root.children[1] - assert if_statement_node.type == "if_statement" - # Make sure it is indeed an empty if_statement - if_body = if_statement_node.child_by_field_name("consequence") - assert if_body.type == "block" - assert if_body.child_count == 0 - visitor = NodeVisitor(python_analyser) - with pytest.raises(AnalysisError) as exp: - visitor.do_visit(if_statement_node) - assert ( - str(exp.value) - == "if_statement consequence is empty block @ test_file:2, column 17" - ) - - def test_for_empty_block_raises_analysis_error(self): - analysis_request = FileAnalysisRequest( - actual_filepath=pathlib.Path("test_file"), result_filename="test_file" - ) - # Code for an empty IF. NOT valid python code - actual_code = b"for x in range(10):\n\n" - python_analyser = PythonAnalyzer(analysis_request, actual_code=actual_code) - # Parse the code snippet and get the if_statement node - tree = python_analyser.parser.parse(actual_code) - root = tree.root_node - assert root.type == "module" - assert root.child_count == 1 - for_statement_node = root.children[0] - assert for_statement_node.type == "for_statement" - # Make sure it is indeed an empty if_statement - if_body = for_statement_node.child_by_field_name("body") - assert if_body.type == "block" - assert if_body.child_count == 0 - visitor = NodeVisitor(python_analyser) - with pytest.raises(AnalysisError) as exp: - visitor.do_visit(for_statement_node) - assert ( - str(exp.value) - == "loop_statement body is empty block @ test_file:1, column 19" - ) diff --git a/codecov-cli/tests/services/static_analysis/test_analyse_file.py b/codecov-cli/tests/services/static_analysis/test_analyse_file.py deleted file mode 100644 index 9d4ad749..00000000 --- a/codecov-cli/tests/services/static_analysis/test_analyse_file.py +++ /dev/null @@ -1,62 +0,0 @@ -import json -import sys -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest - -from codecov_cli.services.staticanalysis import analyze_file -from codecov_cli.services.staticanalysis.types import FileAnalysisRequest - -here = Path(__file__) -here_parent = here.parent - - -@pytest.mark.parametrize( - "input_filename,output_filename", - [ - ("samples/inputs/sample_001.py", "samples/outputs/sample_001.json"), - ("samples/inputs/sample_002.py", "samples/outputs/sample_002.json"), - ("samples/inputs/sample_003.js", "samples/outputs/sample_003.json"), - ("samples/inputs/sample_004.js", "samples/outputs/sample_004.json"), - ("samples/inputs/sample_005.py", "samples/outputs/sample_005.json"), - ], -) -@pytest.mark.skipif( - sys.platform == "win32", reason="windows is producing different `code_hash` values" -) -def test_sample_analysis(input_filename, output_filename): - config = {} - res = analyze_file( - config, FileAnalysisRequest(input_filename, Path(input_filename)) - ) - with open(output_filename, "r") as file: - expected_result = json.load(file) - json_res = json.dumps(res.asdict()) - res_dict = json.loads(json_res) - assert sorted(res_dict["result"].keys()) == sorted(expected_result["result"].keys()) - res_dict["result"]["functions"] = sorted( - res_dict["result"]["functions"], key=lambda x: x["start_line"] - ) - expected_result["result"]["functions"] = sorted( - expected_result["result"]["functions"], key=lambda x: x["start_line"] - ) - assert res_dict["result"]["functions"] == expected_result["result"]["functions"] - assert res_dict["result"].get("statements") == expected_result["result"].get( - "statements" - ) - assert res_dict["result"] == expected_result["result"] - assert res_dict == expected_result - - -@patch("builtins.open") -@patch("codecov_cli.services.staticanalysis.get_best_analyzer", return_value=None) -def test_analyse_file_no_analyzer(mock_get_analyzer, mock_open): - fake_contents = MagicMock(name="fake_file_contents") - file_name = MagicMock(actual_filepath="filepath") - mock_open.return_value.__enter__.return_value.read.return_value = fake_contents - config = {} - res = analyze_file(config, file_name) - assert res is None - mock_open.assert_called_with("filepath", "rb") - mock_get_analyzer.assert_called_with(file_name, fake_contents) diff --git a/codecov-cli/tests/services/static_analysis/test_static_analysis_service.py b/codecov-cli/tests/services/static_analysis/test_static_analysis_service.py deleted file mode 100644 index 461ed684..00000000 --- a/codecov-cli/tests/services/static_analysis/test_static_analysis_service.py +++ /dev/null @@ -1,617 +0,0 @@ -from asyncio import CancelledError -from pathlib import Path -from unittest.mock import MagicMock - -import click -import httpx -import pytest -import requests -import responses -from responses import matchers - -from codecov_cli.services.staticanalysis import ( - process_files, - run_analysis_entrypoint, - send_single_upload_put, -) -from codecov_cli.services.staticanalysis.types import ( - FileAnalysisRequest, - FileAnalysisResult, -) - - -class TestStaticAnalysisService: - @pytest.mark.asyncio - async def test_process_files_with_error(self, mocker): - files_found = list( - map( - lambda filename: FileAnalysisRequest(str(filename), Path(filename)), - [ - "correct_file.py", - "error_file.py", - ], - ) - ) - mock_pool = mocker.patch("codecov_cli.services.staticanalysis.Pool") - - def side_effect(config, filename: FileAnalysisRequest): - if filename.result_filename == "correct_file.py": - return FileAnalysisResult( - filename=filename.result_filename, result={"hash": "abc123"} - ) - if filename.result_filename == "error_file.py": - return FileAnalysisResult( - filename=filename.result_filename, error="some error @ line 12" - ) - # Should not get here, so fail test - assert False - - mock_analyze_function = mocker.patch( - "codecov_cli.services.staticanalysis.analyze_file" - ) - mock_analyze_function.side_effect = side_effect - - def imap_side_effect(mapped_func, files): - results = [] - for file in files: - results.append(mapped_func(file)) - return results - - mock_pool.return_value.__enter__.return_value.imap_unordered.side_effect = ( - imap_side_effect - ) - - results = await process_files(files_found, 1, {}) - mock_pool.return_value.__enter__.return_value.imap_unordered.assert_called() - assert mock_analyze_function.call_count == 2 - assert results == dict( - all_data={"correct_file.py": {"hash": "abc123"}}, - file_metadata=[{"file_hash": "abc123", "filepath": "correct_file.py"}], - processing_errors={"error_file.py": "some error @ line 12"}, - ) - - @pytest.mark.asyncio - async def test_static_analysis_service_success(self, mocker): - mock_file_finder = mocker.patch( - "codecov_cli.services.staticanalysis.select_file_finder" - ) - mock_send_upload_put = mocker.patch( - "codecov_cli.services.staticanalysis.send_single_upload_put" - ) - - # Doing it this way to support Python 3.7 - async def side_effect(*args, **kwargs): - return MagicMock() - - mock_send_upload_put.side_effect = side_effect - - files_found = map( - lambda filename: FileAnalysisRequest(str(filename), Path(filename)), - [ - "samples/inputs/sample_001.py", - "samples/inputs/sample_002.py", - ], - ) - mock_file_finder.return_value.find_files = MagicMock(return_value=files_found) - with responses.RequestsMock() as rsps: - rsps.add( - responses.POST, - "https://api.codecov.io/staticanalysis/analyses", - json={ - "external_id": "externalid", - "filepaths": [ - { - "state": "created", - "filepath": "samples/inputs/sample_001.py", - "raw_upload_location": "http://storage-url", - }, - { - "state": "valid", - "filepath": "samples/inputs/sample_002.py", - "raw_upload_location": "http://storage-url", - }, - ], - }, - status=200, - match=[ - matchers.header_matcher({"Authorization": "Repotoken STATIC_TOKEN"}) - ], - ) - rsps.add( - responses.POST, - "https://api.codecov.io/staticanalysis/analyses/externalid/finish", - status=204, - match=[ - matchers.header_matcher({"Authorization": "Repotoken STATIC_TOKEN"}) - ], - ) - - await run_analysis_entrypoint( - config={}, - folder=".", - numberprocesses=1, - pattern="*.py", - token="STATIC_TOKEN", - commit="COMMIT", - should_force=False, - folders_to_exclude=[], - enterprise_url=None, - args=None, - ) - mock_file_finder.assert_called_with({}) - mock_file_finder.return_value.find_files.assert_called() - assert mock_send_upload_put.call_count == 1 - args, _ = mock_send_upload_put.call_args - assert args[2] == { - "state": "created", - "filepath": "samples/inputs/sample_001.py", - "raw_upload_location": "http://storage-url", - } - - @pytest.mark.asyncio - async def test_static_analysis_service_CancelledError(self, mocker): - mock_file_finder = mocker.patch( - "codecov_cli.services.staticanalysis.select_file_finder" - ) - mock_send_upload_put = mocker.patch( - "codecov_cli.services.staticanalysis.send_single_upload_put" - ) - - async def side_effect(client, all_data, el): - if el["filepath"] == "samples/inputs/sample_001.py": - return { - "status_code": 204, - "filepath": el["filepath"], - "succeeded": True, - } - raise CancelledError("Pretending something cancelled this task") - - mock_send_upload_put.side_effect = side_effect - - files_found = map( - lambda filename: FileAnalysisRequest(str(filename), Path(filename)), - [ - "samples/inputs/sample_001.py", - "samples/inputs/sample_002.py", - ], - ) - mock_file_finder.return_value.find_files = MagicMock(return_value=files_found) - with responses.RequestsMock() as rsps: - rsps.add( - responses.POST, - "https://api.codecov.io/staticanalysis/analyses", - json={ - "external_id": "externalid", - "filepaths": [ - { - "state": "created", - "filepath": "samples/inputs/sample_001.py", - "raw_upload_location": "http://storage-url-001", - }, - { - "state": "created", - "filepath": "samples/inputs/sample_002.py", - "raw_upload_location": "http://storage-url-002", - }, - ], - }, - status=200, - match=[ - matchers.header_matcher({"Authorization": "Repotoken STATIC_TOKEN"}) - ], - ) - - with pytest.raises(click.ClickException) as exp: - await run_analysis_entrypoint( - config={}, - folder=".", - numberprocesses=1, - pattern="*.py", - token="STATIC_TOKEN", - commit="COMMIT", - should_force=False, - folders_to_exclude=[], - enterprise_url=None, - args=None, - ) - assert "Unknown error cancelled the upload tasks." in str(exp.value) - mock_file_finder.assert_called_with({}) - mock_file_finder.return_value.find_files.assert_called() - assert mock_send_upload_put.call_count == 2 - - @pytest.mark.asyncio - async def test_send_single_upload_put_success(self, mocker): - mock_client = MagicMock() - - async def side_effect(presigned_put, data): - if presigned_put == "http://storage-url-001": - return httpx.Response(status_code=204) - - mock_client.put.side_effect = side_effect - - all_data = { - "file-001": {"some": "data", "id": "1"}, - "file-002": {"some": "data", "id": "2"}, - "file-003": {"some": "data", "id": "3"}, - } - - success_response = await send_single_upload_put( - mock_client, - all_data=all_data, - el={ - "filepath": "file-001", - "raw_upload_location": "http://storage-url-001", - }, - ) - assert success_response == { - "status_code": 204, - "filepath": "file-001", - "succeeded": True, - } - - @pytest.mark.asyncio - async def test_send_single_upload_put_fail_401(self, mocker): - mock_client = MagicMock() - - async def side_effect(presigned_put, data): - if presigned_put == "http://storage-url-002": - return httpx.Response(status_code=401) - - mock_client.put.side_effect = side_effect - - all_data = { - "file-001": {"some": "data", "id": "1"}, - "file-002": {"some": "data", "id": "2"}, - "file-003": {"some": "data", "id": "3"}, - } - - fail_401_response = await send_single_upload_put( - mock_client, - all_data=all_data, - el={ - "filepath": "file-002", - "raw_upload_location": "http://storage-url-002", - }, - ) - assert fail_401_response == { - "status_code": 401, - "exception": None, - "filepath": "file-002", - "succeeded": False, - } - - @pytest.mark.asyncio - async def test_send_single_upload_put_fail_exception(self, mocker): - mock_client = MagicMock() - - async def side_effect(presigned_put, data): - if presigned_put == "http://storage-url-003": - raise httpx.HTTPError("Some error occurred in the request") - - mock_client.put.side_effect = side_effect - - all_data = { - "file-001": {"some": "data", "id": "1"}, - "file-002": {"some": "data", "id": "2"}, - "file-003": {"some": "data", "id": "3"}, - } - - fail_401_response = await send_single_upload_put( - mock_client, - all_data=all_data, - el={ - "filepath": "file-003", - "raw_upload_location": "http://storage-url-003", - }, - ) - assert fail_401_response == { - "status_code": None, - "exception": httpx.HTTPError, - "filepath": "file-003", - "succeeded": False, - } - - @pytest.mark.asyncio - @pytest.mark.parametrize( - "finish_endpoint_response,expected", - [ - (500, "Codecov is having problems"), - (400, "some problem with the submitted information"), - ], - ) - async def test_static_analysis_service_finish_fails_status_code( - self, mocker, finish_endpoint_response, expected - ): - mock_file_finder = mocker.patch( - "codecov_cli.services.staticanalysis.select_file_finder" - ) - mock_send_upload_put = mocker.patch( - "codecov_cli.services.staticanalysis.send_single_upload_put" - ) - - # Doing it this way to support Python 3.7 - async def side_effect(*args, **kwargs): - return MagicMock() - - mock_send_upload_put.side_effect = side_effect - - files_found = map( - lambda filename: FileAnalysisRequest(str(filename), Path(filename)), - [ - "samples/inputs/sample_001.py", - "samples/inputs/sample_002.py", - ], - ) - mock_file_finder.return_value.find_files = MagicMock(return_value=files_found) - with responses.RequestsMock() as rsps: - rsps.add( - responses.POST, - "https://api.codecov.io/staticanalysis/analyses", - json={ - "external_id": "externalid", - "filepaths": [ - { - "state": "created", - "filepath": "samples/inputs/sample_001.py", - "raw_upload_location": "http://storage-url", - }, - { - "state": "valid", - "filepath": "samples/inputs/sample_002.py", - "raw_upload_location": "http://storage-url", - }, - ], - }, - status=200, - match=[ - matchers.header_matcher({"Authorization": "Repotoken STATIC_TOKEN"}) - ], - ) - rsps.add( - responses.POST, - "https://api.codecov.io/staticanalysis/analyses/externalid/finish", - status=finish_endpoint_response, - match=[ - matchers.header_matcher({"Authorization": "Repotoken STATIC_TOKEN"}) - ], - ) - with pytest.raises(click.ClickException, match=expected): - await run_analysis_entrypoint( - config={}, - folder=".", - numberprocesses=1, - pattern="*.py", - token="STATIC_TOKEN", - commit="COMMIT", - should_force=False, - folders_to_exclude=[], - enterprise_url=None, - args=None, - ) - mock_file_finder.assert_called_with({}) - mock_file_finder.return_value.find_files.assert_called() - assert mock_send_upload_put.call_count == 1 - args, _ = mock_send_upload_put.call_args - assert args[2] == { - "state": "created", - "filepath": "samples/inputs/sample_001.py", - "raw_upload_location": "http://storage-url", - } - - @pytest.mark.asyncio - async def test_static_analysis_service_finish_fails_request_exception(self, mocker): - mock_file_finder = mocker.patch( - "codecov_cli.services.staticanalysis.select_file_finder" - ) - mock_send_upload_put = mocker.patch( - "codecov_cli.services.staticanalysis.send_single_upload_put" - ) - - # Doing it this way to support Python 3.7 - async def side_effect(*args, **kwargs): - return MagicMock() - - mock_send_upload_put.side_effect = side_effect - - files_found = map( - lambda filename: FileAnalysisRequest(str(filename), Path(filename)), - [ - "samples/inputs/sample_001.py", - "samples/inputs/sample_002.py", - ], - ) - mock_file_finder.return_value.find_files = MagicMock(return_value=files_found) - with responses.RequestsMock() as rsps: - rsps.add( - responses.POST, - "https://api.codecov.io/staticanalysis/analyses", - json={ - "external_id": "externalid", - "filepaths": [ - { - "state": "created", - "filepath": "samples/inputs/sample_001.py", - "raw_upload_location": "http://storage-url", - }, - { - "state": "valid", - "filepath": "samples/inputs/sample_002.py", - "raw_upload_location": "http://storage-url", - }, - ], - }, - status=200, - match=[ - matchers.header_matcher({"Authorization": "Repotoken STATIC_TOKEN"}) - ], - ) - rsps.add( - responses.POST, - "https://api.codecov.io/staticanalysis/analyses/externalid/finish", - body=requests.RequestException(), - ) - with pytest.raises(click.ClickException, match="Unable to reach Codecov"): - await run_analysis_entrypoint( - config={}, - folder=".", - numberprocesses=1, - pattern="*.py", - token="STATIC_TOKEN", - commit="COMMIT", - should_force=False, - folders_to_exclude=[], - enterprise_url=None, - args=None, - ) - mock_file_finder.assert_called_with({}) - mock_file_finder.return_value.find_files.assert_called() - assert mock_send_upload_put.call_count == 1 - args, _ = mock_send_upload_put.call_args - assert args[2] == { - "state": "created", - "filepath": "samples/inputs/sample_001.py", - "raw_upload_location": "http://storage-url", - } - - @pytest.mark.asyncio - async def test_static_analysis_service_should_force_option(self, mocker): - mock_file_finder = mocker.patch( - "codecov_cli.services.staticanalysis.select_file_finder" - ) - mock_send_upload_put = mocker.patch( - "codecov_cli.services.staticanalysis.send_single_upload_put" - ) - - # Doing it this way to support Python 3.7 - async def side_effect(*args, **kwargs): - return MagicMock() - - mock_send_upload_put.side_effect = side_effect - - files_found = map( - lambda filename: FileAnalysisRequest(str(filename), Path(filename)), - [ - "samples/inputs/sample_001.py", - "samples/inputs/sample_002.py", - ], - ) - mock_file_finder.return_value.find_files = MagicMock(return_value=files_found) - with responses.RequestsMock() as rsps: - rsps.add( - responses.POST, - "https://api.codecov.io/staticanalysis/analyses", - json={ - "external_id": "externalid", - "filepaths": [ - { - "state": "created", - "filepath": "samples/inputs/sample_001.py", - "raw_upload_location": "http://storage-url", - }, - { - "state": "valid", - "filepath": "samples/inputs/sample_002.py", - "raw_upload_location": "http://storage-url", - }, - ], - }, - status=200, - match=[ - matchers.header_matcher({"Authorization": "Repotoken STATIC_TOKEN"}) - ], - ) - rsps.add( - responses.POST, - "https://api.codecov.io/staticanalysis/analyses/externalid/finish", - status=204, - match=[ - matchers.header_matcher({"Authorization": "Repotoken STATIC_TOKEN"}) - ], - ) - await run_analysis_entrypoint( - config={}, - folder=".", - numberprocesses=1, - pattern="*.py", - token="STATIC_TOKEN", - commit="COMMIT", - should_force=True, - folders_to_exclude=[], - enterprise_url=None, - args=None, - ) - mock_file_finder.assert_called_with({}) - mock_file_finder.return_value.find_files.assert_called() - assert mock_send_upload_put.call_count == 2 - - @pytest.mark.asyncio - async def test_static_analysis_service_no_upload(self, mocker): - mock_file_finder = mocker.patch( - "codecov_cli.services.staticanalysis.select_file_finder" - ) - mock_send_upload_put = mocker.patch( - "codecov_cli.services.staticanalysis.send_single_upload_put" - ) - - # Doing it this way to support Python 3.7 - async def side_effect(*args, **kwargs): - return MagicMock() - - mock_send_upload_put.side_effect = side_effect - - files_found = map( - lambda filename: FileAnalysisRequest(str(filename), Path(filename)), - [ - "samples/inputs/sample_001.py", - "samples/inputs/sample_002.py", - ], - ) - mock_file_finder.return_value.find_files = MagicMock(return_value=files_found) - with responses.RequestsMock() as rsps: - rsps.add( - responses.POST, - "https://api.codecov.io/staticanalysis/analyses", - json={ - "external_id": "externalid", - "filepaths": [ - { - "state": "valid", - "filepath": "samples/inputs/sample_001.py", - "raw_upload_location": "http://storage-url", - }, - { - "state": "valid", - "filepath": "samples/inputs/sample_002.py", - "raw_upload_location": "http://storage-url", - }, - ], - }, - status=200, - match=[ - matchers.header_matcher({"Authorization": "Repotoken STATIC_TOKEN"}) - ], - ) - rsps.add( - responses.POST, - "https://api.codecov.io/staticanalysis/analyses/externalid/finish", - status=204, - match=[ - matchers.header_matcher({"Authorization": "Repotoken STATIC_TOKEN"}) - ], - ) - - await run_analysis_entrypoint( - config={}, - folder=".", - numberprocesses=1, - pattern="*.py", - token="STATIC_TOKEN", - commit="COMMIT", - should_force=False, - folders_to_exclude=[], - enterprise_url=None, - args=None, - ) - mock_file_finder.assert_called_with({}) - mock_file_finder.return_value.find_files.assert_called() - assert mock_send_upload_put.call_count == 0 diff --git a/prevent-cli/pyproject.toml b/prevent-cli/pyproject.toml index 1e1a9413..ee10e20f 100644 --- a/prevent-cli/pyproject.toml +++ b/prevent-cli/pyproject.toml @@ -9,7 +9,7 @@ authors = [ requires-python = ">=3.9" dependencies = [ "click==8.*", - "codecov-cli>=10.3.0", + "codecov-cli>=10.4.0", "httpx==0.27.*", "ijson==3.*", "pyyaml==6.*", @@ -17,11 +17,9 @@ dependencies = [ "responses==0.21.*", "sentry-sdk>=2.20.0", "test-results-parser==0.5.4", - "tree-sitter==0.20.*", "wrapt>=1.17.2", ] - [project.scripts] sentry-prevent-cli = "prevent_cli.main:run"