- fix functionality to not fail if dataset files are not downloaded (reverse dependencies and pypi stats)

- prepare aura for release on pypi to work as standalone package out of the box if installed via pip
- change default cache behaviour from "ask" to "auto"
- add tox-build.ini that tests the sdist package of aura for automated pypi package testing before release

Author: RootLUG

aura/analyzers/package_enrichment.py

@@ -3,6 +3,7 @@
 from .detections import Detection
 from ..utils import Analyzer
 from ..type_definitions import AnalyzerReturnType
+from ..exceptions import MissingFile
 
 
 @Analyzer.ID("package_enrichment")
@@ -15,13 +16,18 @@ def analyze(*, location: ScanLocation) -> AnalyzerReturnType:
 
     pkg_score = pkg.score
 
+    try:
+        rev_dependencies = package.get_reverse_dependencies(pkg_name=pkg_name)
+    except MissingFile:
+        rev_dependencies = None
+
     extra = {
         "source_url": pkg.source_url,
         "homepage_url": pkg.homepage_url,
         "documentation_url": pkg.documentation_url,
         "latest_release": pkg.get_latest_release(),
         "score": pkg_score.get_score_matrix(),
-        "reverse_dependencies": package.get_reverse_dependencies(pkg_name=pkg_name),
+        "reverse_dependencies": rev_dependencies,
         "classifiers": pkg.info["info"].get("classifiers", []),
         "version": pkg.get_latest_release()
     }

aura/analyzers/typosquatting.py

@@ -4,14 +4,19 @@
 from .. import typos
 from ..utils import Analyzer
 from ..type_definitions import AnalyzerReturnType
+from ..exceptions import MissingFile
 
 
 @Analyzer.ID("typosquatting")
 def analyze(*, location: ScanLocation) -> AnalyzerReturnType:
     if not (pkg_name:=location.metadata.get("package_name")):
         return
 
-    typo_names = typos.check_name(pkg_name, full_list=True)
+    try:
+        typo_names = typos.check_name(pkg_name, full_list=True)
+    except MissingFile:
+        return
+
 
     for x in typo_names:
         yield Detection(

aura/cache.py

@@ -495,7 +495,7 @@ def purge(standard: bool=False):
     if Cache.DISABLE_CACHE:
         return
 
-    mode = config.CFG.get("cache", {}).get("mode", "ask")
+    mode = config.get_cache_mode()
     if mode not in ("ask", "auto", "always"):
         raise ValueError(f"Cache mode has invalid value in the configuration: '{mode}'")
 

aura/config.py

@@ -20,7 +20,7 @@
 except ImportError:
     import json
 
-from .exceptions import InvalidConfiguration
+from .exceptions import InvalidConfiguration, MissingFile
 
 
 CFG: Optional[dict] = None
@@ -154,7 +154,15 @@ def find_configuration() -> Path:  # TODO: add tests
     return Path(DEFAULT_CFG_PATH)
 
 
-def get_file_location(location: str, base_path: Optional[str]=None) -> str:
+def get_file_location(location: str, base_path: Optional[str]=None, exc: bool=True) -> str:
+    """
+    Lookup a location of a file
+
+    :param location: relative or absolute file location or a filename
+    :param base_path: base path used for resolving relative paths or filenames
+    :param exc: Flag, raise an exception if the file could not be found/does not exists
+    :return: resolved path to the given file location
+    """
     if location.startswith("aura.data."):  # Load file as a resource from aura package
         return location
 
@@ -165,9 +173,14 @@ def get_file_location(location: str, base_path: Optional[str]=None) -> str:
         pth = Path(base_path) / location
         if pth.is_file():
             return str(pth)
+    else:
+        pth = location
 
-    # TODO: use custom exception here so we can log as fatal and sys.exit(1)
-    raise InvalidConfiguration(f"Can't find configuration file `{location}` using base path `{base_path}`")
+    if exc:
+        # TODO: add location and base path directoly to the exception as variables for easy extraction
+        raise MissingFile(f"Can't find configuration file `{location}` using base path `{base_path}`")
+    else:
+        return pth
 
 
 def get_file_content(location: str, base_path: Optional[str]=None) -> str:
@@ -236,14 +249,14 @@ def load_config():
         sys.setrecursionlimit(int(rec_limit))
 
 
-def get_pypi_stats_path() -> Path:
+def get_pypi_stats_path(exc=True) -> Path:
     pth = os.environ.get("AURA_PYPI_STATS", None) or CFG["aura"]["pypi_stats"]
-    return Path(get_file_location(pth, CFG_PATH))
+    return Path(get_file_location(pth, CFG_PATH, exc=exc))
 
 
-def get_reverse_dependencies_path() -> Path:
+def get_reverse_dependencies_path(exc=True) -> Path:
     pth = os.environ.get("AURA_REVERSE_DEPENDENCIES", None) or CFG["aura"]["reverse_dependencies"]
-    return Path(get_file_location(pth, CFG_PATH))
+    return Path(get_file_location(pth, CFG_PATH, exc=exc))
 
 
 def iter_pypi_stats() -> Generator[dict, None, None]:
@@ -253,6 +266,11 @@ def iter_pypi_stats() -> Generator[dict, None, None]:
             yield json.loads(line)
 
 
+def get_cache_mode() -> str:
+    fallback = CFG.get("cache", {}).get("mode", "auto")
+    return os.environ.get("AURA_CACHE_MODE", fallback)
+
+
 def get_maximum_archive_size() ->typing.Optional[int] :
     """
     Get settings for a maximum archive file size that can be extracted

aura/data/aura_config.yaml

@@ -216,7 +216,7 @@ cache: &cache
   # - "ask": ask if cache should be purged when running some commands (aura info, aura update ...)
   # - "auto": same as ask but it would purge the cage automatically without asking on specific commands
   # - "always": purge cache after every operation. This is same as "auto" + other non-standard operations including `aura scan`, `aura diff`
-  mode: ask
+  mode: auto
   max-size: 1G
   expiration:
     default: 72  # 3 days

aura/exceptions.py

@@ -30,6 +30,10 @@ class InvalidConfiguration(AuraException, ValueError):
     pass
 
 
+class MissingFile(InvalidLocation):
+    pass
+
+
 class ASTNodeRewrite(AuraException):
     pass
 

aura/info.py

@@ -13,7 +13,7 @@
     jsonschema = None
 
 from . import __version__ as version
-from .exceptions import InvalidConfiguration
+from .exceptions import MissingFile
 from .uri_handlers.base import URIHandler
 from . import github
 from . import plugins
@@ -35,7 +35,7 @@ def check_pypi_stats() -> dict:
                 "enabled": True,
                 "description": "PyPI typosquatting protection enabled"
             }
-    except InvalidConfiguration:
+    except MissingFile:
         pass
 
     return {
@@ -51,7 +51,7 @@ def check_reverse_dependencies() -> dict:
                 "enabled": True,
                 "description": "Reverse dependencies dataset present. Package scoring feature is fully enabled"
             }
-    except InvalidConfiguration:
+    except MissingFile:
         pass
 
     return {

aura/package.py

@@ -30,7 +30,7 @@
 from .output import table
 from .mirror import LocalMirror
 from .type_definitions import ReleaseInfo
-from .exceptions import NoSuchPackage
+from .exceptions import NoSuchPackage, MissingFile
 
 
 LOGGER = config.get_logger(__name__)
@@ -78,10 +78,10 @@ def list_packages(cls):
         )
         return list(repo.list_packages())
 
-    def __contains__(self, item):
+    def __contains__(self, item: str):
         return item in self.info["releases"].keys()
 
-    def __getitem__(self, item):
+    def __getitem__(self, item: str):
         return self.info[item]
 
     @property
@@ -370,10 +370,17 @@ def score_pypi_downloads(self) -> Union[Value, NA]:
                     return self.Value(downloads, normalized, "PyPI downloads", explanation)
         except ValueError:
             pass
+        except MissingFile:
+            pass
+
         return PackageScore.NA("PyPI downloads")
 
     def score_reverse_dependencies(self) -> Union[Value, NA]:
-        dependencies = get_reverse_dependencies(self.package_name)
+        try:
+            dependencies = get_reverse_dependencies(self.package_name)
+        except MissingFile:
+            return PackageScore.NA("Reverse dependencies")
+
         if not dependencies:
             return PackageScore.NA("Reverse dependencies")
 
@@ -542,7 +549,7 @@ def md5_filter(release: ReleaseInfo, md5: Optional[str]=None) -> bool:
 
 def get_reverse_dependencies(pkg_name: str) -> List[str]:
     pkg_name = canonicalize_name(pkg_name)
-    dataset_path = Path("reverse_dependencies.json")
+    dataset_path = config.get_reverse_dependencies_path()
     with dataset_path.open("r") as fd:
         dataset = loads(fd.read())
 

aura/update.py

@@ -6,6 +6,7 @@
 
 from click import secho
 
+from . import config
 from . import utils
 
 
@@ -30,7 +31,12 @@ def update_dataset():
 
         archive = tarfile.open(fd.name, "r:*")
 
-        for f in ("reverse_dependencies.json", "pypi_stats.json"):
+        data_files = (
+            config.get_reverse_dependencies_path(exc=False).name,
+            config.get_pypi_stats_path(exc=False).name
+        )
+
+        for f in data_files:
             backup_file(cwd/f)
             archive.extract(f, path=cwd)