From 2c2adc0cb6f2e1e98ac5c8c066241c5173feb879 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 18 Feb 2025 11:23:57 -0800 Subject: [PATCH 1/8] Setup module for interacting with conda_meta --- dof/_src/conda_meta/__init__.py | 16 ++++++++++++++ dof/_src/conda_meta/conda.py | 3 +++ dof/_src/conda_meta/conda_meta.py | 36 +++++++++++++++++++++++++++++++ dof/_src/conda_meta/pixi.py | 3 +++ 4 files changed, 58 insertions(+) create mode 100644 dof/_src/conda_meta/__init__.py create mode 100644 dof/_src/conda_meta/conda.py create mode 100644 dof/_src/conda_meta/conda_meta.py create mode 100644 dof/_src/conda_meta/pixi.py diff --git a/dof/_src/conda_meta/__init__.py b/dof/_src/conda_meta/__init__.py new file mode 100644 index 0000000..e5478a2 --- /dev/null +++ b/dof/_src/conda_meta/__init__.py @@ -0,0 +1,16 @@ +# This module provides an interface for reading from the +# conda-meta directory of an environment. Tools like conda +# and pixi use conda-meta to keep important metadata about +# the environment and it's history. + + +# interacts with pixi and conda specific +# details in order to understand what the specs requested +# from the user have been. We'll call these requested_specs. +# These are different from dependency_specs which are specs +# that are installed because they are dependencies of the +# requested_specs (and not because the user specifically asked +# for them). +# There is a case for refactoring this into a pluggable or hook +# based setup. For the purpose of exploring this approach we +# won't set that up here. diff --git a/dof/_src/conda_meta/conda.py b/dof/_src/conda_meta/conda.py new file mode 100644 index 0000000..cd4db50 --- /dev/null +++ b/dof/_src/conda_meta/conda.py @@ -0,0 +1,3 @@ +class CondaMeta: + def __init__(self): + pass \ No newline at end of file diff --git a/dof/_src/conda_meta/conda_meta.py b/dof/_src/conda_meta/conda_meta.py new file mode 100644 index 0000000..c75cf6b --- /dev/null +++ b/dof/_src/conda_meta/conda_meta.py @@ -0,0 +1,36 @@ +# NOTE: +# There is a case for refactoring this into a pluggable or hook +# based setup. For the purpose of exploring this approach we +# won't set that up here. + +class CondaMeta(): + def __init__(self, prefix): + """CondaMeta provides a way of interacting with the + conda-meta directory of an environment. Tools like conda + and pixi use conda-meta to keep important metadata about + the environment and it's history. + + Parameters + ---------- + prefix: str + The path to the environment + """ + self.prefix = prefix + + def get_requested_specs(self) -> list[str]: + """Return a list of all the specs a user requested to be installed. + + A user_requested_spec is one that the user explicitly asked to be + installed. These are different from dependency_specs which are specs + that are installed because they are dependencies of the + requested_specs. + + For example, when a user runs `conda install flask`, the user requested + spec is flask. And all the other installed packages are dependency_specs + + Returns + ------- + specs: list[str] + A list of all the specs a user requested to be installed. + """ + return \ No newline at end of file diff --git a/dof/_src/conda_meta/pixi.py b/dof/_src/conda_meta/pixi.py new file mode 100644 index 0000000..cd4db50 --- /dev/null +++ b/dof/_src/conda_meta/pixi.py @@ -0,0 +1,3 @@ +class CondaMeta: + def __init__(self): + pass \ No newline at end of file From f9286b574aaa552244b29d7dfb5aa226bfbeef77 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 18 Feb 2025 11:29:16 -0800 Subject: [PATCH 2/8] Add user_specs command for demoing exptracting user requested specs from an env --- dof/_src/conda_meta/conda_meta.py | 2 +- dof/cli/root.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/dof/_src/conda_meta/conda_meta.py b/dof/_src/conda_meta/conda_meta.py index c75cf6b..06532b1 100644 --- a/dof/_src/conda_meta/conda_meta.py +++ b/dof/_src/conda_meta/conda_meta.py @@ -33,4 +33,4 @@ def get_requested_specs(self) -> list[str]: specs: list[str] A list of all the specs a user requested to be installed. """ - return \ No newline at end of file + return [] diff --git a/dof/cli/root.py b/dof/cli/root.py index b41f30f..0357ad3 100644 --- a/dof/cli/root.py +++ b/dof/cli/root.py @@ -7,6 +7,7 @@ from dof._src.checkpoint import Checkpoint from dof._src.park.park import Park from dof.cli.checkpoint import checkpoint_command +from dof._src.conda_meta.conda_meta import CondaMeta app = typer.Typer( @@ -45,6 +46,20 @@ def lock( yaml.dump(solved_env.model_dump(), env_file) +@app.command() +def user_specs( + rev: str = typer.Option( + None, + help="uuid of the revision to inspect for user_specs" + ), +): + """Demo command: output the list of user requested specs for a revision""" + prefix = os.environ.get("CONDA_PREFIX") + meta = CondaMeta(prefix=prefix) + specs = meta.get_requested_specs() + print(specs) + + @app.command() def push( target: Annotated[str, typer.Option( From 55abd7e36b80478795c478f901309f3d6243e754 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 18 Feb 2025 11:31:53 -0800 Subject: [PATCH 3/8] Add some notes --- dof/_src/conda_meta/__init__.py | 16 ---------------- dof/_src/data/local.py | 1 + dof/_src/lock.py | 1 + dof/_src/models/environment.py | 1 + dof/cli/root.py | 1 + 5 files changed, 4 insertions(+), 16 deletions(-) diff --git a/dof/_src/conda_meta/__init__.py b/dof/_src/conda_meta/__init__.py index e5478a2..e69de29 100644 --- a/dof/_src/conda_meta/__init__.py +++ b/dof/_src/conda_meta/__init__.py @@ -1,16 +0,0 @@ -# This module provides an interface for reading from the -# conda-meta directory of an environment. Tools like conda -# and pixi use conda-meta to keep important metadata about -# the environment and it's history. - - -# interacts with pixi and conda specific -# details in order to understand what the specs requested -# from the user have been. We'll call these requested_specs. -# These are different from dependency_specs which are specs -# that are installed because they are dependencies of the -# requested_specs (and not because the user specifically asked -# for them). -# There is a case for refactoring this into a pluggable or hook -# based setup. For the purpose of exploring this approach we -# won't set that up here. diff --git a/dof/_src/data/local.py b/dof/_src/data/local.py index f573580..9cd89e9 100644 --- a/dof/_src/data/local.py +++ b/dof/_src/data/local.py @@ -1,3 +1,4 @@ +# TODO: rename this to `data_dir` and move up one module - this doesn't need it's whole own module from pathlib import Path from typing import List import os diff --git a/dof/_src/lock.py b/dof/_src/lock.py index 372da91..966b651 100644 --- a/dof/_src/lock.py +++ b/dof/_src/lock.py @@ -1,3 +1,4 @@ +# TODO: delete this whole thing import asyncio import yaml diff --git a/dof/_src/models/environment.py b/dof/_src/models/environment.py index 330bec0..cbf2182 100644 --- a/dof/_src/models/environment.py +++ b/dof/_src/models/environment.py @@ -5,6 +5,7 @@ from dof._src.models import package +# TODO: delete this class CondaEnvironmentSpec(BaseModel): """Input conda environment.yaml spec""" name: Optional[str] diff --git a/dof/cli/root.py b/dof/cli/root.py index 0357ad3..3f0041a 100644 --- a/dof/cli/root.py +++ b/dof/cli/root.py @@ -25,6 +25,7 @@ ) +# TODO: Delete @app.command() def lock( env_file: str = typer.Option( From f4e2ee217081797c763fa4f8c4cceb3d16a7ef84 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 18 Feb 2025 11:51:53 -0800 Subject: [PATCH 4/8] Setup detecting conda-meta --- dof/_src/conda_meta/conda.py | 14 +++++++++++--- dof/_src/conda_meta/conda_meta.py | 14 ++++++++++++++ dof/_src/conda_meta/pixi.py | 14 +++++++++++--- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/dof/_src/conda_meta/conda.py b/dof/_src/conda_meta/conda.py index cd4db50..f93e975 100644 --- a/dof/_src/conda_meta/conda.py +++ b/dof/_src/conda_meta/conda.py @@ -1,3 +1,11 @@ -class CondaMeta: - def __init__(self): - pass \ No newline at end of file +class CondaCondaMeta: + @classmethod + def detect(cls, prefix): + """Detect if the given prefix is a conda based conda meta. + If it is, it will return an instance of CondaCondaMeta + """ + # TODO: detect conda-meta + return cls(prefix) + + def __init__(self, prefix): + self.prefix = prefix \ No newline at end of file diff --git a/dof/_src/conda_meta/conda_meta.py b/dof/_src/conda_meta/conda_meta.py index 06532b1..03e7d9a 100644 --- a/dof/_src/conda_meta/conda_meta.py +++ b/dof/_src/conda_meta/conda_meta.py @@ -3,6 +3,10 @@ # based setup. For the purpose of exploring this approach we # won't set that up here. +from dof._src.conda_meta.conda import CondaCondaMeta +from dof._src.conda_meta.pixi import PixiCondaMeta + + class CondaMeta(): def __init__(self, prefix): """CondaMeta provides a way of interacting with the @@ -17,6 +21,16 @@ def __init__(self, prefix): """ self.prefix = prefix + # detect which conda-meta flavour is used by the environment + for impl in [CondaCondaMeta, PixiCondaMeta]: + self.conda_meta = impl.detect(prefix) + if self.conda_meta is not None: + break + + # if none is detected raise an exception + if self.conda_meta is None: + raise Exception("Could not detect conda or pixi based conda meta") + def get_requested_specs(self) -> list[str]: """Return a list of all the specs a user requested to be installed. diff --git a/dof/_src/conda_meta/pixi.py b/dof/_src/conda_meta/pixi.py index cd4db50..e815c10 100644 --- a/dof/_src/conda_meta/pixi.py +++ b/dof/_src/conda_meta/pixi.py @@ -1,3 +1,11 @@ -class CondaMeta: - def __init__(self): - pass \ No newline at end of file +class PixiCondaMeta: + @classmethod + def detect(cls, prefix): + """Detect if the given prefix is a pixi based conda meta. + If it is, it will return an instance of PixiCondaMeta + """ + # TODO: detect conda-meta + return None + + def __init__(self, prefix): + self.prefix = prefix \ No newline at end of file From 9d6846cbf7859e36be297bcc58700f7644e5732f Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 18 Feb 2025 14:17:56 -0800 Subject: [PATCH 5/8] Detect conda or pixi env --- dof/_src/conda_meta/conda.py | 19 ++++++++++++++++--- dof/_src/conda_meta/conda_meta.py | 10 +++++++++- dof/_src/conda_meta/pixi.py | 20 ++++++++++++++++++-- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/dof/_src/conda_meta/conda.py b/dof/_src/conda_meta/conda.py index f93e975..9ad3503 100644 --- a/dof/_src/conda_meta/conda.py +++ b/dof/_src/conda_meta/conda.py @@ -1,11 +1,24 @@ +from conda.core import envs_manager + class CondaCondaMeta: @classmethod def detect(cls, prefix): """Detect if the given prefix is a conda based conda meta. If it is, it will return an instance of CondaCondaMeta """ - # TODO: detect conda-meta - return cls(prefix) + known_prefixes = envs_manager.list_all_known_prefixes() + if prefix in known_prefixes: + return cls(prefix) + return None def __init__(self, prefix): - self.prefix = prefix \ No newline at end of file + self.prefix = prefix + + def get_requested_specs(self) -> list[str]: + """Return a list of all the specs a user requested to be installed. + Returns + ------- + specs: list[str] + A list of all the specs a user requested to be installed. + """ + return [] \ No newline at end of file diff --git a/dof/_src/conda_meta/conda_meta.py b/dof/_src/conda_meta/conda_meta.py index 03e7d9a..007f004 100644 --- a/dof/_src/conda_meta/conda_meta.py +++ b/dof/_src/conda_meta/conda_meta.py @@ -3,6 +3,8 @@ # based setup. For the purpose of exploring this approach we # won't set that up here. +import os + from dof._src.conda_meta.conda import CondaCondaMeta from dof._src.conda_meta.pixi import PixiCondaMeta @@ -21,6 +23,12 @@ def __init__(self, prefix): """ self.prefix = prefix + if not os.path.exists(prefix): + raise Exception(f"prefix {prefix} does not exist") + + if not os.path.exists(f"{prefix}/conda-meta"): + raise Exception(f"invalid environment at {prefix}, conda-meta dir does not exist") + # detect which conda-meta flavour is used by the environment for impl in [CondaCondaMeta, PixiCondaMeta]: self.conda_meta = impl.detect(prefix) @@ -47,4 +55,4 @@ def get_requested_specs(self) -> list[str]: specs: list[str] A list of all the specs a user requested to be installed. """ - return [] + return self.conda_meta.get_requested_specs() diff --git a/dof/_src/conda_meta/pixi.py b/dof/_src/conda_meta/pixi.py index e815c10..8d651f0 100644 --- a/dof/_src/conda_meta/pixi.py +++ b/dof/_src/conda_meta/pixi.py @@ -1,11 +1,27 @@ +import os + class PixiCondaMeta: @classmethod def detect(cls, prefix): """Detect if the given prefix is a pixi based conda meta. If it is, it will return an instance of PixiCondaMeta """ - # TODO: detect conda-meta + conda_meta_path = f"{prefix}/conda-meta" + # if the {prefix}/conda-meta/pixi path exists, then this is + # a pixi based conda meta environment + if os.path.exists(f"{conda_meta_path}/pixi"): + return cls(prefix) return None def __init__(self, prefix): - self.prefix = prefix \ No newline at end of file + self.prefix = prefix + + # TODO + def get_requested_specs(self) -> list[str]: + """Return a list of all the specs a user requested to be installed. + Returns + ------- + specs: list[str] + A list of all the specs a user requested to be installed. + """ + return [] \ No newline at end of file From 6462634a1582a269325f6ae5b2982c2fd3818810 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 18 Feb 2025 14:46:04 -0800 Subject: [PATCH 6/8] Get conda user requested specs --- dof/_src/conda_meta/conda.py | 9 ++++++++- dof/cli/root.py | 13 +++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/dof/_src/conda_meta/conda.py b/dof/_src/conda_meta/conda.py index 9ad3503..ac5a93c 100644 --- a/dof/_src/conda_meta/conda.py +++ b/dof/_src/conda_meta/conda.py @@ -1,4 +1,6 @@ +import os from conda.core import envs_manager +from conda.history import History class CondaCondaMeta: @classmethod @@ -13,6 +15,10 @@ def detect(cls, prefix): def __init__(self, prefix): self.prefix = prefix + history_file = f"{prefix}/conda-meta/history" + if not os.path.exists(history_file): + raise Exception(f"history file for prefix '{prefix}' does not exist") + self.history = History(prefix) def get_requested_specs(self) -> list[str]: """Return a list of all the specs a user requested to be installed. @@ -21,4 +27,5 @@ def get_requested_specs(self) -> list[str]: specs: list[str] A list of all the specs a user requested to be installed. """ - return [] \ No newline at end of file + requested_specs = self.history.get_requested_specs_map() + return [spec.dist_str() for spec in requested_specs.values()] \ No newline at end of file diff --git a/dof/cli/root.py b/dof/cli/root.py index 3f0041a..8993a6d 100644 --- a/dof/cli/root.py +++ b/dof/cli/root.py @@ -55,10 +55,15 @@ def user_specs( ), ): """Demo command: output the list of user requested specs for a revision""" - prefix = os.environ.get("CONDA_PREFIX") - meta = CondaMeta(prefix=prefix) - specs = meta.get_requested_specs() - print(specs) + if rev is None: + prefix = os.environ.get("CONDA_PREFIX") + meta = CondaMeta(prefix=prefix) + specs = meta.get_requested_specs() + print("the user requested specs in this environment are:") + for spec in specs: + print(f" {spec}") + else: + print("I don't know how to do this yet") @app.command() From b2d08f4f039653ce31fb9ac329d25a01bafc19d4 Mon Sep 17 00:00:00 2001 From: sophia Date: Tue, 18 Feb 2025 15:51:10 -0800 Subject: [PATCH 7/8] Mark packages as user requested --- dof/_src/checkpoint.py | 10 +++++++--- dof/_src/conda_meta/conda.py | 19 ++++++++++++++++--- dof/_src/conda_meta/conda_meta.py | 15 +++++++++++++-- dof/_src/conda_meta/pixi.py | 14 +++++++++++++- dof/_src/models/package.py | 6 +++++- dof/cli/root.py | 13 ++++++++++--- 6 files changed, 64 insertions(+), 13 deletions(-) diff --git a/dof/_src/checkpoint.py b/dof/_src/checkpoint.py index 484ff9a..1c0e836 100644 --- a/dof/_src/checkpoint.py +++ b/dof/_src/checkpoint.py @@ -8,6 +8,7 @@ from dof._src.models import package, environment from dof._src.utils import hash_string from dof._src.data.local import LocalData +from dof._src.conda_meta.conda_meta import CondaMeta class Checkpoint(): @@ -15,6 +16,9 @@ class Checkpoint(): def from_prefix(cls, prefix: str, uuid: str, tags: List[str] = []): packages = [] channels = set() + meta = CondaMeta(prefix=prefix) + user_requested_specs_map = meta.get_requested_specs_map() + for prefix_record in PrefixData(prefix, pip_interop_enabled=True).iter_records_sorted(): if prefix_record.subdir == "pypi": packages.append( @@ -36,9 +40,9 @@ def from_prefix(cls, prefix: str, uuid: str, tags: List[str] = []): conda_channel=prefix_record.channel.url(), # TODO arch="", - # not sure here - platform="linux-64", - url=prefix_record.url + platform=prefix_record.channel.platform, + url=prefix_record.url, + user_requested_spec=user_requested_specs_map.get(prefix_record.name, None) ) ) diff --git a/dof/_src/conda_meta/conda.py b/dof/_src/conda_meta/conda.py index ac5a93c..8c3cae4 100644 --- a/dof/_src/conda_meta/conda.py +++ b/dof/_src/conda_meta/conda.py @@ -21,11 +21,24 @@ def __init__(self, prefix): self.history = History(prefix) def get_requested_specs(self) -> list[str]: - """Return a list of all the specs a user requested to be installed. + """Return a list of all the MatchSpecs a user requested to be installed + Returns ------- specs: list[str] - A list of all the specs a user requested to be installed. + A list of all the MatchSpecs a user requested to be installed + """ + requested_specs = self.history.get_requested_specs_map() + return [spec.spec for spec in requested_specs.values()] + + def get_requested_specs_map(self) -> dict[str, str]: + """Return a dict of all the package name to MatchSpecs user requested + specs to be installed. + + Returns + ------- + specs: dict[str, str] + A list of all the package names to MatchSpecs a user requested to be installed """ requested_specs = self.history.get_requested_specs_map() - return [spec.dist_str() for spec in requested_specs.values()] \ No newline at end of file + return {k: v.spec for k,v in requested_specs.items()} diff --git a/dof/_src/conda_meta/conda_meta.py b/dof/_src/conda_meta/conda_meta.py index 007f004..3993cbe 100644 --- a/dof/_src/conda_meta/conda_meta.py +++ b/dof/_src/conda_meta/conda_meta.py @@ -40,7 +40,7 @@ def __init__(self, prefix): raise Exception("Could not detect conda or pixi based conda meta") def get_requested_specs(self) -> list[str]: - """Return a list of all the specs a user requested to be installed. + """Return a list of all the MatchSpecs a user requested to be installed. A user_requested_spec is one that the user explicitly asked to be installed. These are different from dependency_specs which are specs @@ -53,6 +53,17 @@ def get_requested_specs(self) -> list[str]: Returns ------- specs: list[str] - A list of all the specs a user requested to be installed. + A list of all the MatchSpecs a user requested to be installed """ return self.conda_meta.get_requested_specs() + + def get_requested_specs_map(self) -> dict[str, str]: + """Return a dict of all the package name to MatchSpecs user requested + specs to be installed. + + Returns + ------- + specs: dict[str, str] + A list of all the package names to MatchSpecs a user requested to be installed + """ + return self.conda_meta.get_requested_specs_map() diff --git a/dof/_src/conda_meta/pixi.py b/dof/_src/conda_meta/pixi.py index 8d651f0..78aba81 100644 --- a/dof/_src/conda_meta/pixi.py +++ b/dof/_src/conda_meta/pixi.py @@ -24,4 +24,16 @@ def get_requested_specs(self) -> list[str]: specs: list[str] A list of all the specs a user requested to be installed. """ - return [] \ No newline at end of file + return [] + + # TODO + def get_requested_specs_map(self) -> dict[str, str]: + """Return a dict of all the package name to MatchSpecs user requested + specs to be installed. + + Returns + ------- + specs: dict[str, str] + A list of all the package names to MatchSpecs a user requested to be installed + """ + return {} diff --git a/dof/_src/models/package.py b/dof/_src/models/package.py index 97db516..2617adc 100644 --- a/dof/_src/models/package.py +++ b/dof/_src/models/package.py @@ -14,6 +14,10 @@ class CondaPackage(BaseModel): arch: str platform: str url: str + # the string representation of the matchspec that the user + # used to request the package. If this was not a package + # the user explicitly added, this will be none. + user_requested_spec: Optional[str] = None def to_repodata_record(self): """Converts a url package into a rattler compatible repodata record.""" @@ -28,7 +32,6 @@ def to_repodata_record(self): channel=self.conda_channel, url=self.url ) - def __str__(self): return f"conda: {self.name} - {self.version}" @@ -59,6 +62,7 @@ def to_repodata_record(self): pass +# TODO: probably remove? class UrlCondaPackage(BaseModel): url: str diff --git a/dof/cli/root.py b/dof/cli/root.py index 8993a6d..49aa460 100644 --- a/dof/cli/root.py +++ b/dof/cli/root.py @@ -55,15 +55,22 @@ def user_specs( ), ): """Demo command: output the list of user requested specs for a revision""" + prefix = os.environ.get("CONDA_PREFIX") if rev is None: - prefix = os.environ.get("CONDA_PREFIX") meta = CondaMeta(prefix=prefix) specs = meta.get_requested_specs() print("the user requested specs in this environment are:") - for spec in specs: + # sort alphabetically for readability + for spec in sorted(specs): print(f" {spec}") else: - print("I don't know how to do this yet") + chck = Checkpoint.from_uuid(prefix=prefix, uuid=rev) + pkgs = chck.list_packages() + print(f"the user requested specs rev {rev}:") + # sort alphabetically for readability + for spec in sorted(pkgs, key=lambda p: p.name): + if spec.user_requested_spec is not None: + print(f" {spec.user_requested_spec}") @app.command() From 229290a10e4cc52c0bbd4866145ed7ad90260629 Mon Sep 17 00:00:00 2001 From: sophia Date: Wed, 19 Feb 2025 16:49:26 -0800 Subject: [PATCH 8/8] Support --prefix --- dof/cli/root.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dof/cli/root.py b/dof/cli/root.py index 49aa460..dd3e355 100644 --- a/dof/cli/root.py +++ b/dof/cli/root.py @@ -53,9 +53,17 @@ def user_specs( None, help="uuid of the revision to inspect for user_specs" ), + prefix: str = typer.Option( + None, + help="prefix to save" + ), ): """Demo command: output the list of user requested specs for a revision""" - prefix = os.environ.get("CONDA_PREFIX") + if prefix is None: + prefix = os.environ.get("CONDA_PREFIX") + else: + prefix = os.path.abspath(prefix) + if rev is None: meta = CondaMeta(prefix=prefix) specs = meta.get_requested_specs()