From d4782755d083d796ac6a882fec5de8dc2f9fceb9 Mon Sep 17 00:00:00 2001 From: uttam282005 Date: Thu, 8 Jan 2026 02:15:28 +0530 Subject: [PATCH 01/11] support replace directive Signed-off-by: uttam282005 --- src/packagedcode/go_mod.py | 75 ++++++++++++++++++++++++++++++++++++++ src/packagedcode/golang.py | 39 ++++++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/src/packagedcode/go_mod.py b/src/packagedcode/go_mod.py index 527931c2dcc..93ae0a3607e 100644 --- a/src/packagedcode/go_mod.py +++ b/src/packagedcode/go_mod.py @@ -21,6 +21,7 @@ class GoModule(object): module = attr.ib(default=None) require = attr.ib(default=None) exclude = attr.ib(default=None) + local_path_replacement = attr.ib(default=None) def purl(self, include_version=True): version = None @@ -50,6 +51,13 @@ def purl(self, include_version=True): r'(?P(.*))' ).match +parse_replace = re.compile( + r"(?P\S+)" + r"(?:\s+(?P\S+))?" + r"\s*=>\s*" + r"(?P\S+)" + r"(?:\s+(?P\S+))?" +).match def preprocess(line): """ @@ -120,6 +128,7 @@ def parse_gomod(location): gomods = GoModule() require = [] exclude = [] + local_path_replacement = [] for i, line in enumerate(lines): line = preprocess(line) @@ -157,6 +166,69 @@ def parse_gomod(location): ) ) continue + + if 'replace' in line and '(' in line: + for rep in lines[i + 1:]: + rep = preprocess(rep) + if ')' in rep: + break + parsed_replace = parse_replace(rep) + ns_name = parsed_replace.group('ns_name') + replacement_ns_name = parsed_replace.group('replacement_ns_name') + namespace, _, name = ns_name.rpartition('/') + replacement_namespace, _, replacement_name = replacement_ns_name.rpartition('/') + + exclude.append( + GoModule( + namespace=namespace, + name=name, + version=parsed_replace.group('version') + ) + ) + + if is_local_file(replacement_ns_name): + local_path_replacement.append({ + 'replaces': ns_name, + 'local_path': replacement_ns_name + }) + else: + require.append( + GoModule( + namespace=replacement_namespace, + name=replacement_name, + version=parsed_replace.group('replacement_version') + ) + ) + + if "replace" and "=>" in line: + line = line.lstrip("replace").strip() + parsed_replace = parse_replace(line) + ns_name = parsed_replace.group("ns_name") + replacement_ns_name = parsed_replace.group("replacement_ns_name") + namespace, _, name = ns_name.rpartition("/") + replacement_namespace, _, replacement_name = replacement_ns_name.rpartition("/") + + exclude.append( + GoModule( + namespace=namespace, + name=name, + version=parsed_rep_link.group("version"), + ) + ) + + if is_local_file(replacement_ns_name): + local_path_replacement.append({ + 'replaces': ns_name, + 'local_path': replacement_ns_name + }) + else: + require.append( + GoModule( + namespace=replacement_namespace, + name=replacement_name, + version=parsed_replace.group('replacement_version') + ) + ) parsed_module_name = parse_module(line) if parsed_module_name: @@ -188,6 +260,7 @@ def parse_gomod(location): gomods.require = require gomods.exclude = exclude + gomods.local_path_replacement = local_path_replacement return gomods @@ -202,6 +275,8 @@ def parse_gomod(location): r'h1:(?P[^\s]*)' ).match +def is_local_file(file_name): + return file_name.startswith("./") or file_name.startswith("../") def parse_gosum(location): """ diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index c45be7a5e3a..1d48d8e4fa4 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -10,6 +10,8 @@ from packagedcode import go_mod from packagedcode import models +import os + """ Handle Go packages including go.mod and go.sum files. """ @@ -32,6 +34,38 @@ def assemble(cls, package_data, resource, codebase, package_adder): """ Always use go.mod first then go.sum """ + base_dir = resource.parent(codebase) + local_path_replacement = package_data.extra_data.get('local_path_replacement') + for lpr in local_path_replacement: + local_path = lpr.local_path + full_path = os.path.join(base_dir.path, local_path) + local_resource = codebase.get_resource(full_path) + if not local_resource or not local_resource.is_dir: + continue + local_gomod = None + for child in local_resource.children(codebase): + if child.name == 'go.mod': + local_gomod = child + break + + if not local_gomod: + continue + + if not local_gomod.package_data: + continue + + local_pkg_data = models.PackageData.from_dict(local_gomod.package_data[0]) + package_data.dependencies.append( + models.DependentPackage( + purl=local_pkg_data.purl(include_version=True), + extracted_requirement=local_pkg_data.version, + scope='require', + is_runtime=True, + is_optional=False, + is_pinned=False, + ) + ) + yield from cls.assemble_from_many_datafiles( datafile_name_patterns=('go.mod', 'go.sum',), directory=resource.parent(codebase), @@ -78,6 +112,10 @@ def parse(cls, location, package_only=False): is_pinned=False, ) ) + + extra_data = { + 'local_path_replacement': gomods.local_path_replacement or [] + } name = gomods.name namespace = gomods.namespace @@ -98,6 +136,7 @@ def parse(cls, location, package_only=False): homepage_url=homepage_url, repository_homepage_url=repository_homepage_url, dependencies=dependencies, + extra_data=extra_data, primary_language=cls.default_primary_language, ) yield models.PackageData.from_data(package_data, package_only) From 27c918492825bcd678da072e2bfb7423194adf00 Mon Sep 17 00:00:00 2001 From: uttam282005 Date: Thu, 8 Jan 2026 16:45:56 +0530 Subject: [PATCH 02/11] refactor Signed-off-by: uttam282005 --- src/packagedcode/go_mod.py | 123 +++++++++++++++++++------------------ src/packagedcode/golang.py | 89 ++++++++++++++++++--------- 2 files changed, 122 insertions(+), 90 deletions(-) diff --git a/src/packagedcode/go_mod.py b/src/packagedcode/go_mod.py index 93ae0a3607e..786d2ebe64b 100644 --- a/src/packagedcode/go_mod.py +++ b/src/packagedcode/go_mod.py @@ -21,7 +21,7 @@ class GoModule(object): module = attr.ib(default=None) require = attr.ib(default=None) exclude = attr.ib(default=None) - local_path_replacement = attr.ib(default=None) + local_replacements = attr.ib(default=None) def purl(self, include_version=True): version = None @@ -51,7 +51,7 @@ def purl(self, include_version=True): r'(?P(.*))' ).match -parse_replace = re.compile( +parse_rep_link = re.compile( r"(?P\S+)" r"(?:\s+(?P\S+))?" r"\s*=>\s*" @@ -68,6 +68,60 @@ def preprocess(line): line = line.strip() return line +def parse_replace_directive(line): + parsed_replace = parse_rep_link(line) + ns_name = parsed_replace.group("ns_name") + version = parsed_replace.group("version") + namespace, _, name = ns_name.rpartition("/") + original_module = { + "namespace": namespace, + "name": name, + "version": version + } + + replacement_ns_name = parsed_replace.group("replacement_ns_name") + replacement_version = parsed_replace.group("replacement_version") + is_local = replacement_ns_name.startswith("./") or replacement_ns_name.startswith("../") + + if is_local: + replacement_namespace = None, + replacement_name = replacement_ns_name + else: + replacement_namespace, _, replacement_name = replacement_ns_name.rpartition("/") + + replacement_module = { + "namespace": replacement_namespace, + "name": replacement_name, + "version": replacement_version, + "is_local": is_local, + "local_path": replacement_ns_name if is_local else None + } + + return original_module, replacement_module + +def handle_repalce_directive(line, require, exclude, local_replacements): + original, replacement = parse_replace_directive(line) + exclude.append( + GoModule( + namespace=original.get('namespace'), + name=original.get('name'), + version=original.get('version'), + ) + ) + + if replacement.get('is_local'): + local_replacements.append({ + 'replaces': f"{original.get('namespace')}/{original.get('name')}", + 'local_path': replacement.get('local_path') + }) + else: + require.append( + GoModule( + namespace=replacement.get('namespace'), + name=replacement.get('name'), + version=replacement.get('version') + ) + ) def parse_gomod(location): """ @@ -128,7 +182,7 @@ def parse_gomod(location): gomods = GoModule() require = [] exclude = [] - local_path_replacement = [] + local_replacements = [] for i, line in enumerate(lines): line = preprocess(line) @@ -172,63 +226,13 @@ def parse_gomod(location): rep = preprocess(rep) if ')' in rep: break - parsed_replace = parse_replace(rep) - ns_name = parsed_replace.group('ns_name') - replacement_ns_name = parsed_replace.group('replacement_ns_name') - namespace, _, name = ns_name.rpartition('/') - replacement_namespace, _, replacement_name = replacement_ns_name.rpartition('/') - - exclude.append( - GoModule( - namespace=namespace, - name=name, - version=parsed_replace.group('version') - ) - ) - - if is_local_file(replacement_ns_name): - local_path_replacement.append({ - 'replaces': ns_name, - 'local_path': replacement_ns_name - }) - else: - require.append( - GoModule( - namespace=replacement_namespace, - name=replacement_name, - version=parsed_replace.group('replacement_version') - ) - ) + handle_repalce_directive(rep, require, exclude, local_replacements) + continue if "replace" and "=>" in line: line = line.lstrip("replace").strip() - parsed_replace = parse_replace(line) - ns_name = parsed_replace.group("ns_name") - replacement_ns_name = parsed_replace.group("replacement_ns_name") - namespace, _, name = ns_name.rpartition("/") - replacement_namespace, _, replacement_name = replacement_ns_name.rpartition("/") - - exclude.append( - GoModule( - namespace=namespace, - name=name, - version=parsed_rep_link.group("version"), - ) - ) - - if is_local_file(replacement_ns_name): - local_path_replacement.append({ - 'replaces': ns_name, - 'local_path': replacement_ns_name - }) - else: - require.append( - GoModule( - namespace=replacement_namespace, - name=replacement_name, - version=parsed_replace.group('replacement_version') - ) - ) + handle_repalce_directive(line, require, exclude, local_replacements) + continue parsed_module_name = parse_module(line) if parsed_module_name: @@ -260,7 +264,7 @@ def parse_gomod(location): gomods.require = require gomods.exclude = exclude - gomods.local_path_replacement = local_path_replacement + gomods.local_replacements = local_replacements return gomods @@ -275,9 +279,6 @@ def parse_gomod(location): r'h1:(?P[^\s]*)' ).match -def is_local_file(file_name): - return file_name.startswith("./") or file_name.startswith("../") - def parse_gosum(location): """ Return a list of GoSum from parsing the go.sum file at `location`. diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index 1d48d8e4fa4..4f9e0a7b947 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import posixpath from packagedcode import go_mod from packagedcode import models @@ -34,45 +35,75 @@ def assemble(cls, package_data, resource, codebase, package_adder): """ Always use go.mod first then go.sum """ + yield from cls.assemble_from_many_datafiles( + datafile_name_patterns=('go.mod', 'go.sum',), + directory=resource.parent(codebase), + codebase=codebase, + package_adder=package_adder, + ) + + if not codebase.has_single_resource: + cls.resolve_local_replacements( + package_data=package_data, + resource=resource, + codebase=codebase, + ) + + @classmethod + def resolve_local_replacements(cls, package_data, resource, codebase): + local_replacements = package_data.extra_data.get('local_replacements', []) + if not local_replacements: + return base_dir = resource.parent(codebase) - local_path_replacement = package_data.extra_data.get('local_path_replacement') - for lpr in local_path_replacement: - local_path = lpr.local_path - full_path = os.path.join(base_dir.path, local_path) + base_path = base_dir.path + + for replacement in local_replacements: + local_path = replacement.get('local_path') + if not local_path: + continue + + full_path = posixpath.normpath( + posixpath.join(base_path, local_path) + ) + local_resource = codebase.get_resource(full_path) - if not local_resource or not local_resource.is_dir: + if not local_resource: continue + local_gomod = None for child in local_resource.children(codebase): if child.name == 'go.mod': local_gomod = child break - - if not local_gomod: + if not local_gomod or not local_gomod.package_data: continue - - if not local_gomod.package_data: + + try: + local_pkg_dict = local_gomod.package_data[0] + local_pkg_data = models.PackageData.from_dict(local_pkg_dict) + except (IndexError, KeyError, TypeError): continue - - local_pkg_data = models.PackageData.from_dict(local_gomod.package_data[0]) - package_data.dependencies.append( - models.DependentPackage( - purl=local_pkg_data.purl(include_version=True), - extracted_requirement=local_pkg_data.version, - scope='require', - is_runtime=True, - is_optional=False, - is_pinned=False, - ) + + if not local_pkg_data.purl: + continue + + resolved_dependency = models.DependentPackage( + purl=local_pkg_data.purl, + extracted_requirement=local_pkg_data.version or '', + scope='require', + is_runtime=True, + is_optional=False, + is_resolved=True, + extra_data={ + 'replaces': replacement.get('replaces'), + 'resolved_from_local': True, + 'local_path': local_path, + 'local_resolved_path': full_path, + } ) - - yield from cls.assemble_from_many_datafiles( - datafile_name_patterns=('go.mod', 'go.sum',), - directory=resource.parent(codebase), - codebase=codebase, - package_adder=package_adder, - ) + if not any(dep.purl == resolved_dependency.purl for dep in package_data.dependencies): + package_data.dependencies.append(resolved_dependency) class GoModHandler(BaseGoModuleHandler): datasource_id = 'go_mod' @@ -112,9 +143,9 @@ def parse(cls, location, package_only=False): is_pinned=False, ) ) - + extra_data = { - 'local_path_replacement': gomods.local_path_replacement or [] + 'local_replacements': gomods.local_replacements or [] } name = gomods.name From ba1a48ef7fd4992b8a672176b1d89cc23787ec10 Mon Sep 17 00:00:00 2001 From: uttam282005 Date: Thu, 8 Jan 2026 17:46:54 +0530 Subject: [PATCH 03/11] add tests and regen test fixtures Signed-off-by: uttam282005 --- src/packagedcode/go_mod.py | 2 +- .../golang/gomod/kingpin/output.expected.json | 4 +- .../data/golang/gomod/milvus/go.mod | 24 ++ .../golang/gomod/milvus/output.expected.json | 219 ++++++++++++++++++ .../opencensus-service/output.expected.json | 26 ++- .../gomod/participle/output.expected.json | 4 +- .../golang/gomod/sample/output.expected.json | 4 +- .../golang/gomod/uap-go/output.expected.json | 4 +- .../gomod/user_agent/output.expected.json | 4 +- tests/packagedcode/test_golang.py | 6 + 10 files changed, 290 insertions(+), 7 deletions(-) create mode 100644 tests/packagedcode/data/golang/gomod/milvus/go.mod create mode 100644 tests/packagedcode/data/golang/gomod/milvus/output.expected.json diff --git a/src/packagedcode/go_mod.py b/src/packagedcode/go_mod.py index 786d2ebe64b..668b1d3dfff 100644 --- a/src/packagedcode/go_mod.py +++ b/src/packagedcode/go_mod.py @@ -229,7 +229,7 @@ def parse_gomod(location): handle_repalce_directive(rep, require, exclude, local_replacements) continue - if "replace" and "=>" in line: + if "replace" in line and "=>" in line: line = line.lstrip("replace").strip() handle_repalce_directive(line, require, exclude, local_replacements) continue diff --git a/tests/packagedcode/data/golang/gomod/kingpin/output.expected.json b/tests/packagedcode/data/golang/gomod/kingpin/output.expected.json index 656ba79d622..8396d386755 100644 --- a/tests/packagedcode/data/golang/gomod/kingpin/output.expected.json +++ b/tests/packagedcode/data/golang/gomod/kingpin/output.expected.json @@ -35,7 +35,9 @@ "file_references": [], "is_private": false, "is_virtual": false, - "extra_data": {}, + "extra_data": { + "local_replacements": [] + }, "dependencies": [ { "purl": "pkg:golang/github.com/alecthomas/template@v0.0.0-20160405071501-a0175ee3bccc", diff --git a/tests/packagedcode/data/golang/gomod/milvus/go.mod b/tests/packagedcode/data/golang/gomod/milvus/go.mod new file mode 100644 index 00000000000..d4be3913a9a --- /dev/null +++ b/tests/packagedcode/data/golang/gomod/milvus/go.mod @@ -0,0 +1,24 @@ +module github.com/milvus-io/milvus + +go 1.20 + +require github.com/apache/arrow/go/v12 v12.0.1 + +require github.com/milvus-io/milvus-storage/go v0.0.0-20231227072638-ebd0b8e56d70 + +require ( + github.com/go-playground/validator/v10 v10.14.0 + github.com/quasilyte/go-ruleguard/dsl v0.3.22 + golang.org/x/net v0.19.0 +) + +replace ( + github.com/apache/pulsar-client-go => github.com/milvus-io/pulsar-client-go v0.6.10 + github.com/bketelsen/crypt v0.0.3 => github.com/bketelsen/crypt v0.0.4 // Fix security alert for core-os/etcd + github.com/expr-lang/expr => github.com/SimFG/expr v0.0.0-20231218130003-94d085776dc5 + github.com/milvus-io/milvus/pkg => ./pkg +) + +replace github.com/streamnative/pulsarctl => github.com/xiaofan-luan/pulsarctl v0.5.1 + +exclude github.com/apache/pulsar-client-go/oauth2 v0.0.0-20211108044248-fe3b7c4e445b diff --git a/tests/packagedcode/data/golang/gomod/milvus/output.expected.json b/tests/packagedcode/data/golang/gomod/milvus/output.expected.json new file mode 100644 index 00000000000..1c4073967f9 --- /dev/null +++ b/tests/packagedcode/data/golang/gomod/milvus/output.expected.json @@ -0,0 +1,219 @@ +[ + { + "type": "golang", + "namespace": "github.com/milvus-io", + "name": "milvus", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Go", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://pkg.go.dev/github.com/milvus-io/milvus", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://github.com/milvus-io/milvus.git", + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "local_replacements": [ + { + "replaces": "github.com/milvus-io/milvus/pkg", + "local_path": "./pkg" + } + ] + }, + "dependencies": [ + { + "purl": "pkg:golang/github.com/apache/arrow/go/v12@v12.0.1", + "extracted_requirement": "v12.0.1", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/milvus-io/milvus-storage/go@v0.0.0-20231227072638-ebd0b8e56d70", + "extracted_requirement": "v0.0.0-20231227072638-ebd0b8e56d70", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/go-playground/validator/v10@v10.14.0", + "extracted_requirement": "v10.14.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/quasilyte/go-ruleguard/dsl@v0.3.22", + "extracted_requirement": "v0.3.22", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/golang.org/x/net@v0.19.0", + "extracted_requirement": "v0.19.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/milvus-io/pulsar-client-go@v0.6.10", + "extracted_requirement": "v0.6.10", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/bketelsen/crypt@v0.0.4", + "extracted_requirement": "v0.0.4", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/SimFG/expr@v0.0.0-20231218130003-94d085776dc5", + "extracted_requirement": "v0.0.0-20231218130003-94d085776dc5", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/xiaofan-luan/pulsarctl@v0.5.1", + "extracted_requirement": "v0.5.1", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/apache/pulsar-client-go", + "extracted_requirement": null, + "scope": "exclude", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/bketelsen/crypt@v0.0.3", + "extracted_requirement": "v0.0.3", + "scope": "exclude", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/expr-lang/expr", + "extracted_requirement": null, + "scope": "exclude", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/milvus-io/milvus/pkg", + "extracted_requirement": null, + "scope": "exclude", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/streamnative/pulsarctl", + "extracted_requirement": null, + "scope": "exclude", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/apache/pulsar-client-go/oauth2@v0.0.0-20211108044248-fe3b7c4e445b", + "extracted_requirement": "v0.0.0-20211108044248-fe3b7c4e445b", + "scope": "exclude", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://pkg.go.dev/github.com/milvus-io/milvus", + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "go_mod", + "purl": "pkg:golang/github.com/milvus-io/milvus" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/golang/gomod/opencensus-service/output.expected.json b/tests/packagedcode/data/golang/gomod/opencensus-service/output.expected.json index df05ab713ce..668b5c7a0dc 100644 --- a/tests/packagedcode/data/golang/gomod/opencensus-service/output.expected.json +++ b/tests/packagedcode/data/golang/gomod/opencensus-service/output.expected.json @@ -35,7 +35,9 @@ "file_references": [], "is_private": false, "is_virtual": false, - "extra_data": {}, + "extra_data": { + "local_replacements": [] + }, "dependencies": [ { "purl": "pkg:golang/contrib.go.opencensus.io@v0.0.0-20181029163544-2befc13012d0", @@ -58,6 +60,28 @@ "is_direct": true, "resolved_package": {}, "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/apache/thrift@v0.12.0", + "extracted_requirement": "v0.12.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/git.apache.org/thrift.git", + "extracted_requirement": null, + "scope": "exclude", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} } ], "repository_homepage_url": "https://pkg.go.dev/github.com/census-instrumentation/opencensus-service", diff --git a/tests/packagedcode/data/golang/gomod/participle/output.expected.json b/tests/packagedcode/data/golang/gomod/participle/output.expected.json index ab2a17feca8..e36650877c2 100644 --- a/tests/packagedcode/data/golang/gomod/participle/output.expected.json +++ b/tests/packagedcode/data/golang/gomod/participle/output.expected.json @@ -35,7 +35,9 @@ "file_references": [], "is_private": false, "is_virtual": false, - "extra_data": {}, + "extra_data": { + "local_replacements": [] + }, "dependencies": [ { "purl": "pkg:golang/github.com/alecthomas/repr@v0.0.0-20181024024818-d37bc2a10ba1", diff --git a/tests/packagedcode/data/golang/gomod/sample/output.expected.json b/tests/packagedcode/data/golang/gomod/sample/output.expected.json index a6348340df5..62cbc9404c6 100644 --- a/tests/packagedcode/data/golang/gomod/sample/output.expected.json +++ b/tests/packagedcode/data/golang/gomod/sample/output.expected.json @@ -35,7 +35,9 @@ "file_references": [], "is_private": false, "is_virtual": false, - "extra_data": {}, + "extra_data": { + "local_replacements": [] + }, "dependencies": [ { "purl": "pkg:golang/github.com/davecgh/go-spew@v1.1.1", diff --git a/tests/packagedcode/data/golang/gomod/uap-go/output.expected.json b/tests/packagedcode/data/golang/gomod/uap-go/output.expected.json index 77b89adb54d..9fc3cb6dcd2 100644 --- a/tests/packagedcode/data/golang/gomod/uap-go/output.expected.json +++ b/tests/packagedcode/data/golang/gomod/uap-go/output.expected.json @@ -35,7 +35,9 @@ "file_references": [], "is_private": false, "is_virtual": false, - "extra_data": {}, + "extra_data": { + "local_replacements": [] + }, "dependencies": [ { "purl": "pkg:golang/gopkg.in/yaml.v2@v2.2.1", diff --git a/tests/packagedcode/data/golang/gomod/user_agent/output.expected.json b/tests/packagedcode/data/golang/gomod/user_agent/output.expected.json index 408fad06d37..b11d9c77b02 100644 --- a/tests/packagedcode/data/golang/gomod/user_agent/output.expected.json +++ b/tests/packagedcode/data/golang/gomod/user_agent/output.expected.json @@ -35,7 +35,9 @@ "file_references": [], "is_private": false, "is_virtual": false, - "extra_data": {}, + "extra_data": { + "local_replacements": [] + }, "dependencies": [], "repository_homepage_url": "https://pkg.go.dev/github.com/mssola/user_agent", "repository_download_url": null, diff --git a/tests/packagedcode/test_golang.py b/tests/packagedcode/test_golang.py index 58feea2def0..f36cd86523b 100644 --- a/tests/packagedcode/test_golang.py +++ b/tests/packagedcode/test_golang.py @@ -48,6 +48,12 @@ def test_parse_gomod_sample(self): package = golang.GoModHandler.parse(test_file) self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + def test_parse_gomod_milvus(self): + test_file = self.get_test_loc('golang/gomod/milvus/go.mod') + expected_loc = self.get_test_loc('golang/gomod/milvus/output.expected.json') + package = golang.GoModHandler.parse(test_file) + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + def test_parse_gomod_uap_go(self): test_file = self.get_test_loc('golang/gomod/uap-go/go.mod') expected_loc = self.get_test_loc('golang/gomod/uap-go/output.expected.json') From d52b3011c8c774fee7beb445407e1f65f07f9874 Mon Sep 17 00:00:00 2001 From: uttam282005 Date: Thu, 8 Jan 2026 18:04:10 +0530 Subject: [PATCH 04/11] include local_replacements in extra_data only if exist Signed-off-by: uttam282005 --- src/packagedcode/golang.py | 4 +- .../data/golang/gomod/gopls/go.mod | 30 ++ .../golang/gomod/gopls/output.expected.json | 274 ++++++++++++++++++ .../golang/gomod/kingpin/output.expected.json | 4 +- .../opencensus-service/output.expected.json | 4 +- .../gomod/participle/output.expected.json | 4 +- .../golang/gomod/sample/output.expected.json | 4 +- .../golang/gomod/uap-go/output.expected.json | 4 +- .../gomod/user_agent/output.expected.json | 4 +- tests/packagedcode/test_golang.py | 6 + 10 files changed, 318 insertions(+), 20 deletions(-) create mode 100644 tests/packagedcode/data/golang/gomod/gopls/go.mod create mode 100644 tests/packagedcode/data/golang/gomod/gopls/output.expected.json diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index 4f9e0a7b947..67d058cb36c 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -145,7 +145,7 @@ def parse(cls, location, package_only=False): ) extra_data = { - 'local_replacements': gomods.local_replacements or [] + 'local_replacements': gomods.local_replacements } name = gomods.name @@ -167,7 +167,7 @@ def parse(cls, location, package_only=False): homepage_url=homepage_url, repository_homepage_url=repository_homepage_url, dependencies=dependencies, - extra_data=extra_data, + extra_data=extra_data if gomods.local_replacements else {}, primary_language=cls.default_primary_language, ) yield models.PackageData.from_data(package_data, package_only) diff --git a/tests/packagedcode/data/golang/gomod/gopls/go.mod b/tests/packagedcode/data/golang/gomod/gopls/go.mod new file mode 100644 index 00000000000..1d6fdd3d6b7 --- /dev/null +++ b/tests/packagedcode/data/golang/gomod/gopls/go.mod @@ -0,0 +1,30 @@ +module golang.org/x/tools/gopls + +go 1.18 + +require ( + github.com/google/go-cmp v0.5.9 + github.com/jba/printsrc v0.2.2 + github.com/jba/templatecheck v0.6.0 + github.com/sergi/go-diff v1.1.0 + golang.org/x/mod v0.12.0 + golang.org/x/sync v0.3.0 + golang.org/x/sys v0.11.0 + golang.org/x/telemetry v0.0.0-20230808152233-a65b40c0fdb0 + golang.org/x/text v0.12.0 + golang.org/x/tools v0.6.0 + golang.org/x/vuln v0.0.0-20230110180137-6ad3e3d07815 + gopkg.in/yaml.v3 v3.0.1 + honnef.co/go/tools v0.4.2 + mvdan.cc/gofumpt v0.4.0 + mvdan.cc/xurls/v2 v2.4.0 +) + +require ( + github.com/BurntSushi/toml v1.2.1 // indirect + github.com/google/safehtml v0.1.0 // indirect + golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect + golang.org/x/exp/typeparams v0.0.0-20221212164502-fae10dda9338 // indirect +) + +replace golang.org/x/tools => ../ diff --git a/tests/packagedcode/data/golang/gomod/gopls/output.expected.json b/tests/packagedcode/data/golang/gomod/gopls/output.expected.json new file mode 100644 index 00000000000..e6caecd5f96 --- /dev/null +++ b/tests/packagedcode/data/golang/gomod/gopls/output.expected.json @@ -0,0 +1,274 @@ +[ + { + "type": "golang", + "namespace": "golang.org/x/tools", + "name": "gopls", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Go", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://pkg.go.dev/golang.org/x/tools/gopls", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://golang.org/x/tools/gopls.git", + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "local_replacements": [ + { + "replaces": "golang.org/x/tools", + "local_path": "../" + } + ] + }, + "dependencies": [ + { + "purl": "pkg:golang/github.com/google/go-cmp@v0.5.9", + "extracted_requirement": "v0.5.9", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/jba/printsrc@v0.2.2", + "extracted_requirement": "v0.2.2", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/jba/templatecheck@v0.6.0", + "extracted_requirement": "v0.6.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/sergi/go-diff@v1.1.0", + "extracted_requirement": "v1.1.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/golang.org/x/mod@v0.12.0", + "extracted_requirement": "v0.12.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/golang.org/x/sync@v0.3.0", + "extracted_requirement": "v0.3.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/golang.org/x/sys@v0.11.0", + "extracted_requirement": "v0.11.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/golang.org/x/telemetry@v0.0.0-20230808152233-a65b40c0fdb0", + "extracted_requirement": "v0.0.0-20230808152233-a65b40c0fdb0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/golang.org/x/text@v0.12.0", + "extracted_requirement": "v0.12.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/golang.org/x/tools@v0.6.0", + "extracted_requirement": "v0.6.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/golang.org/x/vuln@v0.0.0-20230110180137-6ad3e3d07815", + "extracted_requirement": "v0.0.0-20230110180137-6ad3e3d07815", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/gopkg.in/yaml.v3@v3.0.1", + "extracted_requirement": "v3.0.1", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/honnef.co/go/tools@v0.4.2", + "extracted_requirement": "v0.4.2", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/mvdan.cc/gofumpt@v0.4.0", + "extracted_requirement": "v0.4.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/mvdan.cc/xurls/v2@v2.4.0", + "extracted_requirement": "v2.4.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/BurntSushi/toml@v1.2.1", + "extracted_requirement": "v1.2.1", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/github.com/google/safehtml@v0.1.0", + "extracted_requirement": "v0.1.0", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/golang.org/x/exp@v0.0.0-20220722155223-a9213eeb770e", + "extracted_requirement": "v0.0.0-20220722155223-a9213eeb770e", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/golang.org/x/exp/typeparams@v0.0.0-20221212164502-fae10dda9338", + "extracted_requirement": "v0.0.0-20221212164502-fae10dda9338", + "scope": "require", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:golang/golang.org/x/tools", + "extracted_requirement": null, + "scope": "exclude", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://pkg.go.dev/golang.org/x/tools/gopls", + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "go_mod", + "purl": "pkg:golang/golang.org/x/tools/gopls" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/golang/gomod/kingpin/output.expected.json b/tests/packagedcode/data/golang/gomod/kingpin/output.expected.json index 8396d386755..656ba79d622 100644 --- a/tests/packagedcode/data/golang/gomod/kingpin/output.expected.json +++ b/tests/packagedcode/data/golang/gomod/kingpin/output.expected.json @@ -35,9 +35,7 @@ "file_references": [], "is_private": false, "is_virtual": false, - "extra_data": { - "local_replacements": [] - }, + "extra_data": {}, "dependencies": [ { "purl": "pkg:golang/github.com/alecthomas/template@v0.0.0-20160405071501-a0175ee3bccc", diff --git a/tests/packagedcode/data/golang/gomod/opencensus-service/output.expected.json b/tests/packagedcode/data/golang/gomod/opencensus-service/output.expected.json index 668b5c7a0dc..8ad089103a9 100644 --- a/tests/packagedcode/data/golang/gomod/opencensus-service/output.expected.json +++ b/tests/packagedcode/data/golang/gomod/opencensus-service/output.expected.json @@ -35,9 +35,7 @@ "file_references": [], "is_private": false, "is_virtual": false, - "extra_data": { - "local_replacements": [] - }, + "extra_data": {}, "dependencies": [ { "purl": "pkg:golang/contrib.go.opencensus.io@v0.0.0-20181029163544-2befc13012d0", diff --git a/tests/packagedcode/data/golang/gomod/participle/output.expected.json b/tests/packagedcode/data/golang/gomod/participle/output.expected.json index e36650877c2..ab2a17feca8 100644 --- a/tests/packagedcode/data/golang/gomod/participle/output.expected.json +++ b/tests/packagedcode/data/golang/gomod/participle/output.expected.json @@ -35,9 +35,7 @@ "file_references": [], "is_private": false, "is_virtual": false, - "extra_data": { - "local_replacements": [] - }, + "extra_data": {}, "dependencies": [ { "purl": "pkg:golang/github.com/alecthomas/repr@v0.0.0-20181024024818-d37bc2a10ba1", diff --git a/tests/packagedcode/data/golang/gomod/sample/output.expected.json b/tests/packagedcode/data/golang/gomod/sample/output.expected.json index 62cbc9404c6..a6348340df5 100644 --- a/tests/packagedcode/data/golang/gomod/sample/output.expected.json +++ b/tests/packagedcode/data/golang/gomod/sample/output.expected.json @@ -35,9 +35,7 @@ "file_references": [], "is_private": false, "is_virtual": false, - "extra_data": { - "local_replacements": [] - }, + "extra_data": {}, "dependencies": [ { "purl": "pkg:golang/github.com/davecgh/go-spew@v1.1.1", diff --git a/tests/packagedcode/data/golang/gomod/uap-go/output.expected.json b/tests/packagedcode/data/golang/gomod/uap-go/output.expected.json index 9fc3cb6dcd2..77b89adb54d 100644 --- a/tests/packagedcode/data/golang/gomod/uap-go/output.expected.json +++ b/tests/packagedcode/data/golang/gomod/uap-go/output.expected.json @@ -35,9 +35,7 @@ "file_references": [], "is_private": false, "is_virtual": false, - "extra_data": { - "local_replacements": [] - }, + "extra_data": {}, "dependencies": [ { "purl": "pkg:golang/gopkg.in/yaml.v2@v2.2.1", diff --git a/tests/packagedcode/data/golang/gomod/user_agent/output.expected.json b/tests/packagedcode/data/golang/gomod/user_agent/output.expected.json index b11d9c77b02..408fad06d37 100644 --- a/tests/packagedcode/data/golang/gomod/user_agent/output.expected.json +++ b/tests/packagedcode/data/golang/gomod/user_agent/output.expected.json @@ -35,9 +35,7 @@ "file_references": [], "is_private": false, "is_virtual": false, - "extra_data": { - "local_replacements": [] - }, + "extra_data": {}, "dependencies": [], "repository_homepage_url": "https://pkg.go.dev/github.com/mssola/user_agent", "repository_download_url": null, diff --git a/tests/packagedcode/test_golang.py b/tests/packagedcode/test_golang.py index f36cd86523b..6480db939e0 100644 --- a/tests/packagedcode/test_golang.py +++ b/tests/packagedcode/test_golang.py @@ -54,6 +54,12 @@ def test_parse_gomod_milvus(self): package = golang.GoModHandler.parse(test_file) self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + def test_parse_gomod_gopls(self): + test_file = self.get_test_loc('golang/gomod/gopls/go.mod') + expected_loc = self.get_test_loc('golang/gomod/gopls/output.expected.json') + package = golang.GoModHandler.parse(test_file) + + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) def test_parse_gomod_uap_go(self): test_file = self.get_test_loc('golang/gomod/uap-go/go.mod') expected_loc = self.get_test_loc('golang/gomod/uap-go/output.expected.json') From 21d109143a40442245c7d0e1590d8c787104bcc8 Mon Sep 17 00:00:00 2001 From: uttam282005 Date: Thu, 8 Jan 2026 18:05:51 +0530 Subject: [PATCH 05/11] minor Signed-off-by: uttam282005 --- tests/packagedcode/test_golang.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/packagedcode/test_golang.py b/tests/packagedcode/test_golang.py index 6480db939e0..b5d64170682 100644 --- a/tests/packagedcode/test_golang.py +++ b/tests/packagedcode/test_golang.py @@ -58,8 +58,8 @@ def test_parse_gomod_gopls(self): test_file = self.get_test_loc('golang/gomod/gopls/go.mod') expected_loc = self.get_test_loc('golang/gomod/gopls/output.expected.json') package = golang.GoModHandler.parse(test_file) - self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + def test_parse_gomod_uap_go(self): test_file = self.get_test_loc('golang/gomod/uap-go/go.mod') expected_loc = self.get_test_loc('golang/gomod/uap-go/output.expected.json') From 2108013d195be6bbb3cfd5173259299b0570e613 Mon Sep 17 00:00:00 2001 From: uttam282005 Date: Fri, 9 Jan 2026 13:52:59 +0530 Subject: [PATCH 06/11] debug with traces Signed-off-by: uttam282005 --- src/packagedcode/go_mod.py | 10 +++++----- src/packagedcode/golang.py | 11 +++++++++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/packagedcode/go_mod.py b/src/packagedcode/go_mod.py index 668b1d3dfff..51b2478497e 100644 --- a/src/packagedcode/go_mod.py +++ b/src/packagedcode/go_mod.py @@ -84,7 +84,7 @@ def parse_replace_directive(line): is_local = replacement_ns_name.startswith("./") or replacement_ns_name.startswith("../") if is_local: - replacement_namespace = None, + replacement_namespace = None replacement_name = replacement_ns_name else: replacement_namespace, _, replacement_name = replacement_ns_name.rpartition("/") @@ -99,7 +99,7 @@ def parse_replace_directive(line): return original_module, replacement_module -def handle_repalce_directive(line, require, exclude, local_replacements): +def handle_replace_directive(line, require, exclude, local_replacements): original, replacement = parse_replace_directive(line) exclude.append( GoModule( @@ -226,12 +226,12 @@ def parse_gomod(location): rep = preprocess(rep) if ')' in rep: break - handle_repalce_directive(rep, require, exclude, local_replacements) + handle_replace_directive(rep, require, exclude, local_replacements) continue - if "replace" in line and "=>" in line: + if 'replace' in line and '=>' in line: line = line.lstrip("replace").strip() - handle_repalce_directive(line, require, exclude, local_replacements) + handle_replace_directive(line, require, exclude, local_replacements) continue parsed_module_name = parse_module(line) diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index 67d058cb36c..3d3e9208a16 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -13,6 +13,8 @@ import os +from packagedcode.npm import TRACE + """ Handle Go packages including go.mod and go.sum files. """ @@ -51,6 +53,10 @@ def assemble(cls, package_data, resource, codebase, package_adder): @classmethod def resolve_local_replacements(cls, package_data, resource, codebase): + """ + Resolve local paths present in replace directives + """ + local_replacements = package_data.extra_data.get('local_replacements', []) if not local_replacements: return @@ -68,6 +74,8 @@ def resolve_local_replacements(cls, package_data, resource, codebase): local_resource = codebase.get_resource(full_path) if not local_resource: + if TRACE: + print(full_path) continue local_gomod = None @@ -102,8 +110,7 @@ def resolve_local_replacements(cls, package_data, resource, codebase): } ) - if not any(dep.purl == resolved_dependency.purl for dep in package_data.dependencies): - package_data.dependencies.append(resolved_dependency) + package_data.dependencies.append(resolved_dependency) class GoModHandler(BaseGoModuleHandler): datasource_id = 'go_mod' From c7192a5690f6b3d14970403ff67c808ad835fb70 Mon Sep 17 00:00:00 2001 From: uttam282005 Date: Sat, 10 Jan 2026 20:37:58 +0530 Subject: [PATCH 07/11] change package_data in resource Signed-off-by: uttam282005 --- src/packagedcode/go_mod.py | 2 +- src/packagedcode/golang.py | 35 +++++++++++++++++------------------ 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/packagedcode/go_mod.py b/src/packagedcode/go_mod.py index 51b2478497e..151d0f92407 100644 --- a/src/packagedcode/go_mod.py +++ b/src/packagedcode/go_mod.py @@ -220,7 +220,7 @@ def parse_gomod(location): ) ) continue - + if 'replace' in line and '(' in line: for rep in lines[i + 1:]: rep = preprocess(rep) diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index 3d3e9208a16..ee3283edb6d 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -11,10 +11,6 @@ from packagedcode import go_mod from packagedcode import models -import os - -from packagedcode.npm import TRACE - """ Handle Go packages including go.mod and go.sum files. """ @@ -37,20 +33,23 @@ def assemble(cls, package_data, resource, codebase, package_adder): """ Always use go.mod first then go.sum """ - yield from cls.assemble_from_many_datafiles( - datafile_name_patterns=('go.mod', 'go.sum',), - directory=resource.parent(codebase), - codebase=codebase, - package_adder=package_adder, - ) if not codebase.has_single_resource: cls.resolve_local_replacements( - package_data=package_data, + package_data=resource.package_data, resource=resource, codebase=codebase, ) + resource.package_data[0] = package_data.to_dict() + + yield from cls.assemble_from_many_datafiles( + datafile_name_patterns=('go.mod', 'go.sum',), + directory=resource.parent(codebase), + codebase=codebase, + package_adder=package_adder, + ) + @classmethod def resolve_local_replacements(cls, package_data, resource, codebase): """ @@ -74,8 +73,6 @@ def resolve_local_replacements(cls, package_data, resource, codebase): local_resource = codebase.get_resource(full_path) if not local_resource: - if TRACE: - print(full_path) continue local_gomod = None @@ -86,6 +83,7 @@ def resolve_local_replacements(cls, package_data, resource, codebase): if not local_gomod or not local_gomod.package_data: continue + try: local_pkg_dict = local_gomod.package_data[0] local_pkg_data = models.PackageData.from_dict(local_pkg_dict) @@ -97,11 +95,11 @@ def resolve_local_replacements(cls, package_data, resource, codebase): resolved_dependency = models.DependentPackage( purl=local_pkg_data.purl, - extracted_requirement=local_pkg_data.version or '', + extracted_requirement=local_pkg_data.version or None, + resolved_package=local_pkg_data, scope='require', is_runtime=True, is_optional=False, - is_resolved=True, extra_data={ 'replaces': replacement.get('replaces'), 'resolved_from_local': True, @@ -109,8 +107,9 @@ def resolve_local_replacements(cls, package_data, resource, codebase): 'local_resolved_path': full_path, } ) - - package_data.dependencies.append(resolved_dependency) + + if not any(dep.purl == resolved_dependency.purl for dep in package_data.dependencies): + package_data.dependencies.append(resolved_dependency) class GoModHandler(BaseGoModuleHandler): datasource_id = 'go_mod' @@ -123,7 +122,7 @@ class GoModHandler(BaseGoModuleHandler): @classmethod def parse(cls, location, package_only=False): gomods = go_mod.parse_gomod(location) - + dependencies = [] require = gomods.require or [] for gomod in require: From 9889074a5d3df44822b620d4eaca3afdc8a32d69 Mon Sep 17 00:00:00 2001 From: uttam282005 Date: Sat, 10 Jan 2026 21:14:55 +0530 Subject: [PATCH 08/11] pass pacakage_data Signed-off-by: uttam282005 --- src/packagedcode/golang.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index ee3283edb6d..2e85aa11ef6 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -36,7 +36,7 @@ def assemble(cls, package_data, resource, codebase, package_adder): if not codebase.has_single_resource: cls.resolve_local_replacements( - package_data=resource.package_data, + package_data=package_data, resource=resource, codebase=codebase, ) @@ -83,7 +83,6 @@ def resolve_local_replacements(cls, package_data, resource, codebase): if not local_gomod or not local_gomod.package_data: continue - try: local_pkg_dict = local_gomod.package_data[0] local_pkg_data = models.PackageData.from_dict(local_pkg_dict) From 01498c0669ca7442db9f2e9ef6dc21cfaa956a28 Mon Sep 17 00:00:00 2001 From: uttam282005 Date: Tue, 13 Jan 2026 00:01:24 +0530 Subject: [PATCH 09/11] add logs for debugging Signed-off-by: uttam282005 --- src/packagedcode/golang.py | 53 +++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index 2e85aa11ef6..86f73ecaee2 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -7,6 +7,7 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import os import posixpath from packagedcode import go_mod from packagedcode import models @@ -25,6 +26,27 @@ # TODO: use the LICENSE file convention! # TODO: support "vendor" and "workspace" layouts +# Tracing flags +TRACE = False or os.environ.get("SCANCODE_DEBUG_PACKAGE", False) + + +# Tracing flags +def logger_debug(*args): + pass + + +if TRACE: + import logging + import sys + + logger = logging.getLogger(__name__) + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) + + def logger_debug(*args): + return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args)) + + class BaseGoModuleHandler(models.DatafileHandler): @@ -57,14 +79,17 @@ def resolve_local_replacements(cls, package_data, resource, codebase): """ local_replacements = package_data.extra_data.get('local_replacements', []) - if not local_replacements: + if not local_replacements: + logger_debug(f"resolve_local_replacements: No local replacements found") return + base_dir = resource.parent(codebase) base_path = base_dir.path - for replacement in local_replacements: - local_path = replacement.get('local_path') + for idx, replacement in enumerate(local_replacements): + local_path = replacement. get('local_path') if not local_path: + logger_debug(f"resolve_local_replacements: Skipping replacement {idx + 1} - no local_path found") continue full_path = posixpath.normpath( @@ -73,23 +98,28 @@ def resolve_local_replacements(cls, package_data, resource, codebase): local_resource = codebase.get_resource(full_path) if not local_resource: + logger_debug(f"resolve_local_replacements: Resource not found at {full_path}") continue local_gomod = None - for child in local_resource.children(codebase): + for child in local_resource. children(codebase): if child.name == 'go.mod': local_gomod = child break - if not local_gomod or not local_gomod.package_data: + + if not local_gomod or not local_gomod. package_data: + logger_debug(f"resolve_local_replacements: No go.mod or package_data found in {full_path}") continue try: local_pkg_dict = local_gomod.package_data[0] local_pkg_data = models.PackageData.from_dict(local_pkg_dict) - except (IndexError, KeyError, TypeError): + except (IndexError, KeyError, TypeError) as e: + logger_debug(f"resolve_local_replacements: Failed to parse package data: {e}") continue - if not local_pkg_data.purl: + if not local_pkg_data. purl: + logger_debug(f"resolve_local_replacements: No purl found in local package data") continue resolved_dependency = models.DependentPackage( @@ -100,15 +130,18 @@ def resolve_local_replacements(cls, package_data, resource, codebase): is_runtime=True, is_optional=False, extra_data={ - 'replaces': replacement.get('replaces'), - 'resolved_from_local': True, + 'replaces': replacement.get('replaces'), + 'resolved_from_local': True, 'local_path': local_path, 'local_resolved_path': full_path, } ) - + if not any(dep.purl == resolved_dependency.purl for dep in package_data.dependencies): package_data.dependencies.append(resolved_dependency) + logger_debug(f"resolve_local_replacements: Added dependency: {resolved_dependency.purl}") + else: + logger_debug(f"resolve_local_replacements: Dependency already exists, skipping: {resolved_dependency. purl}") class GoModHandler(BaseGoModuleHandler): datasource_id = 'go_mod' From 4d40657d9b2b84c66294c83f0b10b956eb975600 Mon Sep 17 00:00:00 2001 From: Uttam Raj Date: Tue, 13 Jan 2026 00:08:25 +0530 Subject: [PATCH 10/11] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: uttam282005 --- src/packagedcode/golang.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index 86f73ecaee2..6eb3f9e1bdc 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -66,11 +66,11 @@ def assemble(cls, package_data, resource, codebase, package_adder): resource.package_data[0] = package_data.to_dict() yield from cls.assemble_from_many_datafiles( - datafile_name_patterns=('go.mod', 'go.sum',), - directory=resource.parent(codebase), - codebase=codebase, - package_adder=package_adder, - ) + datafile_name_patterns=('go.mod', 'go.sum',), + directory=resource.parent(codebase), + codebase=codebase, + package_adder=package_adder, + ) @classmethod def resolve_local_replacements(cls, package_data, resource, codebase): @@ -154,7 +154,7 @@ class GoModHandler(BaseGoModuleHandler): @classmethod def parse(cls, location, package_only=False): gomods = go_mod.parse_gomod(location) - + dependencies = [] require = gomods.require or [] for gomod in require: From d6a025290f594a2f73e29747cc992f70b1bc2313 Mon Sep 17 00:00:00 2001 From: uttam282005 Date: Tue, 13 Jan 2026 19:38:51 +0530 Subject: [PATCH 11/11] convert package data to dict Signed-off-by: uttam282005 --- src/packagedcode/golang.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py index 6eb3f9e1bdc..dacd1427cd8 100644 --- a/src/packagedcode/golang.py +++ b/src/packagedcode/golang.py @@ -125,7 +125,7 @@ def resolve_local_replacements(cls, package_data, resource, codebase): resolved_dependency = models.DependentPackage( purl=local_pkg_data.purl, extracted_requirement=local_pkg_data.version or None, - resolved_package=local_pkg_data, + resolved_package=local_pkg_data.to_dict(), scope='require', is_runtime=True, is_optional=False,