diff --git a/CHANGELOG.md b/CHANGELOG.md index bd0cb4c..1aa2428 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,30 @@ +## 5.2.0 + +* Adds `all-cabal-nixes` argument to stacklock2nix. This gives an + improvement on initial build time when using stacklock2nix with a shared + cache. + + The `all-cabal-nixes` argument can be used like the following: + + ```nix + stacklock2nix { + stackYaml = ./stack.yaml; + all-cabal-nixes = fetchFromGitHub { + owner = "all-cabal-nixes"; + repo = "all-cabal-nixes"; + rev = "c37e66df270014a18ed11527db55928a4a2ae5d4"; + sha256 = "sha256-QcAhW8yFGwj7L1LWbvjdSQ2bTDeQ+1DVeC+/jS4gJA4="; + }; + } + ``` + + This greatly reduces the amount of IFD needed for building packages from + Hackage. + + See the PR adding this feature for more information: + [#65](https://github.com/cdepillabout/stacklock2nix/pull/65) + ## 5.1.0 * Adds `os-string` and `ghc-internal` to list of GHC boot libraries. This is diff --git a/nix/build-support/stacklock2nix/default.nix b/nix/build-support/stacklock2nix/default.nix index f648660..ca981d2 100644 --- a/nix/build-support/stacklock2nix/default.nix +++ b/nix/build-support/stacklock2nix/default.nix @@ -140,6 +140,9 @@ # }; # ``` # + # An alternative to specifying this option is to instead just override + # the top-level all-cabal-hashes attribute in Nixpkgs. + # # This is not used if `baseHaskellPkgSet` is `null`. # # (NOTE: You likely want to fetch all-cabal-hashes with `fetchFromGitHub` @@ -147,6 +150,55 @@ # and untarred. stacklock2nix can still work with an all-cabal-hashes that is # a tarball, but building will be faster with a plain directory.) all-cabal-hashes ? null +, # This option is similar to the above `all-cabal-hashes` option. + # This option allows you to specify a checkout of the + # https://github.com/all-cabal-nixes/all-cabal-nixes repo. + # This is mainly used to speed up the internal IFD used by stacklock2nix. + # + # By default, stacklock2nix internally uses the `hfinal.callHackage` function to + # generate the `.nix` files for Haskell packages that come from Hackage. + # All packages from the Stackage resolver fall into this category, as well + # as packages added to extra-deps in stack.yaml. + # + # `hfinal.callHackage` will internally call the `cabal2nix` executable on the + # `.cabal` file from `all-cabal-hashes`. While this is simple, one big + # downside is that if you have many, many dependencies, you end up with eval + # time in Nix taking quite a long time, since `cabal2nix` has to be called + # sequentially on each of your transitive dependencies. + # + # `all-cabal-nixes` presents an alternative approach, since it already + # contains the output of running `cabal2nix` on all the packages on Hackage. + # + # If you specify `all-cabal-nixes`, then instead of needing to use + # `hfinal.callHackage` to generate the package, stacklock2nix will instead + # just do `hfinal.callPackage` on the correct package/version from + # `all-cabal-nixes`. + # + # all-cabal-nixes :: Drv + # + # Example: + # ``` + # final.fetchFromGitHub { + # owner = "all-cabal-nixes"; + # repo = "all-cabal-nixes"; + # rev = "fabab20db25b8e90686c18b56ced346306ecf29b"; + # sha256 = "sha256-dPM+jMBU9rsGL6oiPRsQi+lxAv/n6rDNoHFSakboljU="; + # }; + # ``` + # + # Using `all-cabal-nixes` can considerably speed up an initial build, since + # it doesn't have to use IFD to call `cabal2nix` for all transitive + # dependencies from Hackage. However, once you've done a full build and have + # all required files in your Nix store, `all-cabal-nixes` doesn't provide any + # benefit for subsequent builds. + # + # Specifying `all-cabal-nixes` is often the most helpful when paired with a + # shared Nix cache used by all developers and CI. + # + # Make sure you read the FAQ in the README of + # https://github.com/all-cabal-nixes/all-cabal-nixes so that you understand + # the downsides before using this. + all-cabal-nixes ? null , callPackage ? topargs.callPackage , # Path to Nixpkgs. # @@ -221,10 +273,10 @@ let readYAML = callPackage ./read-yaml.nix {}; # The `stack.yaml` file read in as a Nix value. - stackYamlParsed = readYAML stackYamlReal; + stackYamlParsed = readYAML "stack.yaml" stackYamlReal; # The `stack.yaml.lock` file read in as a Nix value. - stackYamlLockParsed = readYAML stackYamlLockReal; + stackYamlLockParsed = readYAML "stack.yaml.lock" stackYamlLockReal; # The URL and sha256 for the snapshot from the `stack.yaml.lock` file. # @@ -247,7 +299,7 @@ let # The Nix value for the Stackage snapshot specified in the `stack.yaml.lock` # file. - resolverParsed = readYAML resolverRawYaml; + resolverParsed = readYAML "stackage-resolver" resolverRawYaml; # Fetch cabal revisions from the stackage content addressable store. fetchCabalFileRevision = {name, version, hash}: @@ -398,6 +450,104 @@ let else {}; + # This is a derivation that contains only the cabal2nix-generated .nix files + # from all-cabal-nixes for exactly the packages from the Stackage resolver, + # as well as those packages passed in the stack.yaml extra-deps list. + # + # all-cabal-nixes contains ALL the cabal2nix-generated .nix files from + # all-cabal-hashes/Hackage, while some-cabal-nixes contains just the .nix + # files relevant to your project. + # + # some-cabal-nixes :: Drv + # + # This is mainly useful as an optimization for CI. By using this, in + # combination with a shared Nix cachix, you will only need to download the + # small some-cabal-nixes derivation in order to build your project, instead + # of needing to download the huge all-cabal-nixes repo. + some-cabal-nixes = + if all-cabal-nixes == null then + null + else + let + # Figure out the Haskell package name a version from a Hackage package lock string. + # + # Example: + # ``` + # > getHackagePkgNameVersion "cassava-0.5.3.0@sha256:06e6dbc0f3467f3d9823321171adc08d6edc639491cadb0ad33b2dca1e530d29,6083" + # { name = "cassava"; version = "0.5.3.0"; } + # ``` + getHackagePkgNameVersion = haskPkgLock: + { inherit (parseHackageStr haskPkgLock.hackage) name version; }; + + # A list of package names and versions from the Stackage resolver. + resolverPkgs = map getHackagePkgNameVersion resolverParsed.packages; + + # A list of package names and versions from the extra-deps in the stack.yaml. + extraPkgs = + lib.pipe + stackYamlLockParsed.packages + [ + # pull out the .completed attribute + (map (pkg: pkg.completed)) + + # filter out everything that doesn't have a .hackage attribute + (builtins.filter (pkg: pkg ? "hackage")) + + # get the name and version for the hackage package + (map getHackagePkgNameVersion) + ]; + + pkgs = resolverPkgs ++ extraPkgs; + + # Shell command to copy a default.nix for a given package/version out + # of all-cabal-nixes. + # + # copyForPkg :: { name :: String, version :: String } -> String + # + # Only copy the default.nix file if it doesn't already exist. It can + # sometimes already exist if the same file was specified both in the + # Stackage resolver and in the extra-deps from stack.yaml. + copyForPkg = pkg: + '' + mkdir -p "./${pkg.name}/${pkg.version}" + [ ! -f "./${pkg.name}/${pkg.version}/default.nix" ] && cp "${all-cabal-nixes}/${pkg.name}/${pkg.version}/default.nix" "./${pkg.name}/${pkg.version}/" + ''; + in + runCommand + "some-cabal-nixes" + {} + '' + mkdir -p "$out" + cd "$out" + ${lib.concatMapStrings copyForPkg pkgs} + ''; + + # Similar to callHackage, but instead of internally running cabal2nix, it + # just does hfinal.callPackage on the generated .nix file from + # all-cabal-nixes. + # + # callAllCabalNixes :: HaskPkgSet -> String -> String -> AttrSet -> HaskellPkgDrv + # + # Example: + # ``` + # callAllCabalNixes hfinal "lens" "5.2.3" { doctest = final.my-cool-doctest; } + # ``` + callAllCabalNixes = + if all-cabal-nixes == null then + null + else + hfinal: name: version: args: + hfinal.callPackage "${all-cabal-nixes}/${name}/${version}/default.nix" args; + + # Similar to callAllCabalNixes, but uses some-cabal-nixes instead of + # all-cabal-nixes. + callSomeCabalNixes = + if some-cabal-nixes == null then + null + else + hfinal: name: version: args: + hfinal.callPackage "${some-cabal-nixes}/${name}/${version}/default.nix" args; + # Take the Hackage lock info for a given package, and turn it into an actual # Haskell package derivation. # @@ -424,14 +574,29 @@ let # hfinal # ``` # - # This takes care of replaces the `.cabal` file from Hackage with the correct revision + # This takes care of replacing the `.cabal` file from Hackage with the correct revision # specified in the Hackage lock info. pkgHackageInfoToNixHaskPkg = isExtraDep: pkgHackageInfo: hfinal: let additionalArgs = getAdditionalCabal2nixArgs pkgHackageInfo.name pkgHackageInfo.version; baseDrv = - hfinal.callHackage pkgHackageInfo.name pkgHackageInfo.version additionalArgs; + # TODO: Could we be more clever about this fallback logic? + # + # For instanced, we could actually check within the some-cabal-nixes + # directory to figure out if the requested package/version exists or + # not. And if it doesn't exist, then we could try all-cabal-nixes. + # And only if that doesn't exist, try callHackage. + # + # Just checking whether or not they are null is somewhat too big of a + # hammer. + if some-cabal-nixes != null then + callSomeCabalNixes hfinal pkgHackageInfo.name pkgHackageInfo.version additionalArgs + else if all-cabal-nixes != null then + callAllCabalNixes hfinal pkgHackageInfo.name pkgHackageInfo.version additionalArgs + else + hfinal.callHackage pkgHackageInfo.name pkgHackageInfo.version additionalArgs; + baseDrvWithCorrectRev = overrideCabalFileRevision pkgHackageInfo.name @@ -449,7 +614,8 @@ let # Return a derivation for a Haskell package for the given Haskell package # lock info. # - # extraDepCreateNixHaskPkg :: Bool -> HaskellPkgSet -> HaskellPkgLock -> HaskellPkgDrv + # extraDepCreateNixHaskPkg :: + # Bool -> HaskellPkgSet -> HaskellPkgLock -> { name :: String, value :: HaskellPkgDrv } # # The first `Bool` is `true` if the passed-in HaskellPkgLock is an # `extra-deps` from `stack.yaml`. Otherwise, `false` if the passed-in @@ -719,7 +885,7 @@ let # Example: `"my-cool-pkg"` pkgNameFromPackageYaml = let - packageYaml = readYAML (justCabalFilePath + "/package.yaml"); + packageYaml = readYAML (baseNameOf rawPkgPath + "-package.yaml") (justCabalFilePath + "/package.yaml"); in if packageYaml ? name then packageYaml.name @@ -1109,6 +1275,10 @@ in newPkgSet newPkgSetDevShell all-cabal-hashes + all-cabal-nixes + some-cabal-nixes + callAllCabalNixes + callSomeCabalNixes ; # These are a bunch of internal attributes, used for testing. diff --git a/nix/build-support/stacklock2nix/read-yaml.nix b/nix/build-support/stacklock2nix/read-yaml.nix index a4f3b63..00ccdc8 100644 --- a/nix/build-support/stacklock2nix/read-yaml.nix +++ b/nix/build-support/stacklock2nix/read-yaml.nix @@ -9,10 +9,14 @@ # # but takes an input file in YAML instead of JSON. # -# readYAML :: Path -> a -# -# where `a` is the Nixified version of the input file. -path: +# readYAML :: +# -- | The name to use in the output derivation. Helpful for debugging. +# String -> +# -- | The path of the actual YAML file to turn into JSON. +# Path -> +# -- | The nixified version of the input YAML file. +# a +drvName: path: let # Starting in remarshal-0.17.0, it added a new `--stringify` CLI flag: @@ -28,7 +32,7 @@ let jsonOutputDrv = runCommand - "from-yaml" + "from-yaml-${drvName}" { nativeBuildInputs = [ remarshal ]; } "remarshal ${remarshal-stringify-arg} -if yaml -i \"${path}\" -of json -o \"$out\""; in diff --git a/test/nixpkgs.nix b/test/nixpkgs.nix index 6f43ade..16ee50d 100644 --- a/test/nixpkgs.nix +++ b/test/nixpkgs.nix @@ -28,6 +28,9 @@ let # This also tests that our stacklock2nix-specific passthru values are working. all-cabal-hashes-is-dir = final.callPackage ./test-all-cabal-hashes-is-dir.nix {}; + # This tests that the all-cabal-nixes argument works correctly. + all-cabal-nixes = final.callPackage ./test-all-cabal-nixes.nix {}; + # This test that a stack.yaml with no local packages defined is still # able to be used by stacklock2nix, and produces a reasonable package set. no-local-packages = final.callPackage ./test-no-local-packages {}; diff --git a/test/test-all-cabal-nixes.nix b/test/test-all-cabal-nixes.nix new file mode 100644 index 0000000..0f45b48 --- /dev/null +++ b/test/test-all-cabal-nixes.nix @@ -0,0 +1,46 @@ +{ lib +, stacklock2nix +, haskell +, cabal-install +, fetchFromGitHub +}: + +let + hasklib = haskell.lib.compose; + + stacklock = stacklock2nix { + stackYaml = ../my-example-haskell-lib-easy/stack.yaml; + baseHaskellPkgSet = haskell.packages.ghc984; + additionalHaskellPkgSetOverrides = hfinal: hprev: { + # The servant-cassava.cabal file is malformed on GitHub: + # https://github.com/haskell-servant/servant-cassava/pull/29 + servant-cassava = + hasklib.overrideCabal + { editedCabalFile = null; revision = null; } + hprev.servant-cassava; + + amazonka = hasklib.dontCheck hprev.amazonka; + amazonka-core = hasklib.dontCheck hprev.amazonka-core; + amazonka-sso = hasklib.dontCheck hprev.amazonka-sso; + amazonka-sts = hasklib.dontCheck hprev.amazonka-sts; + }; + cabal2nixArgsOverrides = args: args // { + amazonka-sso = ver: { amazonka-test = null; }; + amazonka-sts = ver: { amazonka-test = null; }; + }; + all-cabal-hashes = fetchFromGitHub { + owner = "commercialhaskell"; + repo = "all-cabal-hashes"; + rev = "578b09df5072f21768cfe13edfc3e4c3e41428fc"; + sha256 = "sha256-fmf4LukOJ2c0bCmNfuN+n2R6bxGhJqag9CBvZQEl3kA="; + }; + # This test is to confirm that using `all-cabal-nixes` works. + all-cabal-nixes = fetchFromGitHub { + owner = "all-cabal-nixes"; + repo = "all-cabal-nixes"; + rev = "c37e66df270014a18ed11527db55928a4a2ae5d4"; + sha256 = "sha256-QcAhW8yFGwj7L1LWbvjdSQ2bTDeQ+1DVeC+/jS4gJA4="; + }; + }; +in +stacklock.newPkgSet.my-example-haskell-app