From 94e7ec4f6280a80e2e929db9aa3c00fe8e066684 Mon Sep 17 00:00:00 2001 From: Ben Sima Date: Wed, 10 Apr 2024 20:31:27 -0400 Subject: Explicitly pass Bild.nix to nix builds & expose references to stable This change was motivated by my testing of tabbyAPI. I kept doing like `nix-build -A pkgs.tabbyAPI` and I thought, can't bild just do this? So I wrote a file called TabbyAPI.nix with the following contents:: { bild }: bild.pkgs.tabbyAPI and it worked, I just needed this change to Bild.hs to supply the `bild` argument. The benefit of using bild here is that I can get the logging, concurrency settings, and linking to _/nix etc all by default. Plus, using a standalone nix file like TabbyAPI.nix might be a good way to pin some package in the build system and make sure it continues to build, test, and so on. Also, thie means I don't sprinkle relative paths to the Bild.nix library throughout the repo, which is bad practice anyway. Re: explicitly exposing refernces to stable: This keeps things a bit more tidy and less confusing when working on the nix library. --- Biz/Bild.hs | 10 +- Biz/Bild.nix | 333 +++++++++++++++++++++++++---------------------- Biz/Bild/Builder.nix | 25 ++-- Biz/Cloud.nix | 2 +- Biz/Dev/Beryllium.nix | 2 +- Biz/Dev/Lithium.nix | 2 +- Biz/Dragons/Analysis.nix | 2 +- Biz/Que.nix | 2 +- 8 files changed, 201 insertions(+), 177 deletions(-) (limited to 'Biz') diff --git a/Biz/Bild.hs b/Biz/Bild.hs index 2ec0c7f..f8385c8 100644 --- a/Biz/Bild.hs +++ b/Biz/Bild.hs @@ -680,7 +680,10 @@ analyze hmap ns = case Map.lookup ns hmap of "--out-link", root nixdir Namespace.toPath namespace, "--builders", - toNixFlag builder + toNixFlag builder, + "--arg", + "bild", + str <| "import " <> root "Biz/Bild.nix {}" ] |> map Text.pack, out = Meta.None, @@ -1059,8 +1062,6 @@ nixBuild loud maxJobs cores target@(Target {..}) = >> run symlink x -> pure x where - argstr :: Text -> Text -> [Text] - argstr n v = ["--argstr", n, v] instantiate root = Proc { loud = loud, @@ -1070,7 +1071,8 @@ nixBuild loud maxJobs cores target@(Target {..}) = -- is tightly coupled with the code in the nix builder and there's no -- way around that, methinks. args = - [ argstr "analysisJSON" <| str <| Aeson.encode <| (Map.singleton namespace target :: Analysis), + [ ["--argstr", "analysisJSON", str <| Aeson.encode <| (Map.singleton namespace target :: Analysis)], + ["--arg", "bild", str <| "import " <> root "Biz/Bild.nix {}"], [str <| root "Biz/Bild/Builder.nix"] ] |> mconcat diff --git a/Biz/Bild.nix b/Biz/Bild.nix index 329726a..077f364 100644 --- a/Biz/Bild.nix +++ b/Biz/Bild.nix @@ -2,172 +2,195 @@ let constants = import ./Bild/Constants.nix; - stable = nixpkgs.nixos-23_05; -in rec { - # provided by .envrc - root = builtins.getEnv "CODEROOT"; - - inherit (stable) sources lib makeWrapper stdenv; - - haskell = rec { - inherit (constants) ghcCompiler; - - # all available packages - deps = import ./Bild/Deps/Haskell.nix; - packages = lib.attrsets.getAttrs deps stable.haskellPackages; - - # make a ghc with dependencies - ghcWith = stable.haskell.packages.${ghcCompiler}.ghcWithHoogle; - - # ghc with all packages, used for generating bild's package database - ghcPackageSetFull = ghcWith (p: lib.attrsets.attrVals deps p); - - # bild's dependencies, needs to be hand-written - ghcPackageSetBild = ghcWith (hpkgs: - with hpkgs; [ - aeson - async - base - bytestring - conduit - conduit-extra - containers - directory - docopt - filepath - process - protolude - rainbow - regex-applicative - split - tasty - tasty-hunit - tasty-quickcheck - text - hostname - wai # can remove when removed from Biz.Log - ]); + + # expose some attrs from stable, keep this minimal and simple + stable = { + inherit (nixpkgs.nixos-23_05) + sources lib makeWrapper ccacheStdenv haskellPackages haskell + lispPackages_new python3 nixos mkShell dockerTools stdenv; }; - lisp = { sbclWith = stable.lispPackages_new.sbclWithPackages; }; + # for exposing packages as bild.pkgs + stable-pkgs = nixpkgs.nixos-23_05.pkgs; + + # this is the main library definitions, recursive references can be made with + # `self.thing`, like in Python objects + self = { + # provided by .envrc + root = builtins.getEnv "CODEROOT"; + + inherit (stable) sources lib makeWrapper stdenv; + + haskell = rec { + inherit (constants) ghcCompiler; + + # all available packages + deps = import ./Bild/Deps/Haskell.nix; + packages = self.lib.attrsets.getAttrs self.deps stable.haskellPackages; + + # make a ghc with dependencies + ghcWith = stable.haskell.packages.${ghcCompiler}.ghcWithHoogle; + + # ghc with all packages, used for generating bild's package database + ghcPackageSetFull = ghcWith (p: self.lib.attrsets.attrVals deps p); + + # bild's dependencies, needs to be hand-written + ghcPackageSetBild = ghcWith (hpkgs: + with hpkgs; [ + aeson + async + base + bytestring + conduit + conduit-extra + containers + directory + docopt + filepath + process + protolude + rainbow + regex-applicative + split + tasty + tasty-hunit + tasty-quickcheck + text + hostname + wai # can remove when removed from Biz.Log + ]); + }; - python = { - packages = stable.python3.pkgs; - pythonWith = stable.python3.withPackages; - buildPythonApplication = stable.python3.pkgs.buildPythonApplication; - }; + lisp = { sbclWith = stable.lispPackages_new.sbclWithPackages; }; - # c packages are just stable, filtered to just the list of deps i want - c.packages = lib.attrsets.getAttrs (import ./Bild/Deps/C.nix) stable.pkgs; + python = { + packages = stable.python3.pkgs; + pythonWith = stable.python3.withPackages; + buildPythonApplication = stable.python3.pkgs.buildPythonApplication; + }; - # exposed packages for inclusion in builds - pkgs = with stable.pkgs; { - inherit deadnix git hlint indent ormolu shellcheck nixfmt mypy pkg-config - rustc cmark; - ruff = nixpkgs.nixos-unstable-small.ruff; - }; + # c packages are just stable, filtered to just the list of deps i want + c.packages = + self.lib.attrsets.getAttrs (import ./Bild/Deps/C.nix) stable-pkgs; - # a standard nix build for bild, for bootstrapping. this should be the only - # hand-written builder we need - bild = stable.stdenv.mkDerivation { - name = "bild"; - src = ../.; - nativeBuildInputs = [ haskell.ghcPackageSetBild ]; - buildInputs = [ stable.makeWrapper ]; - propagatedBuildInputs = with stable; [ - pkg-config - git - # this is just to get access to ghc-pkg in bild - (haskell.ghcWith (_: [ ])) - - # lisp deps, remove this when i implement nix builds for lisp - guile - (lisp.sbclWith - (p: with p; [ asdf alexandria ])) # just enough to build Example.lisp - ]; - strictDeps = true; - buildPhase = '' - mkdir -p $out/bin $out/lib/ghc-${haskell.ghcPackageSetFull.version} - cp -r \ - ${haskell.ghcPackageSetFull}/lib/ghc-${haskell.ghcPackageSetFull.version}/package.conf.d \ - $out/lib/ghc-${haskell.ghcPackageSetFull.version} - ghc \ - -threaded \ - -Werror \ - -i. \ - --make Biz/Bild.hs \ - -main-is Biz.Bild \ - -o $out/bin/bild - ''; - installPhase = '' - wrapProgram $out/bin/bild \ - --prefix PATH : ${ - lib.makeBinPath [ haskell.ghcPackageSetBild stable.pkgs.git ] - } \ - --set GHC_PACKAGE_PATH \ - $out/lib/ghc-${haskell.ghcPackageSetFull.version}/package.conf.d - ''; - }; + # exposed packages for inclusion in builds + pkgs = with stable-pkgs; { + inherit bat bc cmark ctags deadnix fd figlet fzf git git-branchless + gitlint guile hlint indent jq lolcat mypy nixfmt ormolu pkg-config + ripgrep rustc shellcheck tree wemux; + ruff = nixpkgs.nixos-unstable-small.ruff; + }; - # wrapper around bild - runBildAnalyze = target: - stable.stdenv.mkDerivation rec { - name = "bild-analysis"; + # a standard nix build for bild, for bootstrapping. this should be the only + # hand-written builder we need + bild = self.stdenv.mkDerivation { + name = "bild"; src = ../.; - USER = "nixbld"; - HOSTNAME = "nix-sandbox"; - # we need to remove the $src root because bild expects paths relative to the - # working directory: - TARGET = "." + lib.strings.removePrefix (toString src) (toString target); + nativeBuildInputs = [ self.haskell.ghcPackageSetBild ]; + buildInputs = [ self.makeWrapper ]; + propagatedBuildInputs = with self.pkgs; [ + pkg-config + git + # this is just to get access to ghc-pkg in bild + (self.haskell.ghcWith (_: [ ])) + + # lisp deps, remove this when i implement nix builds for lisp + guile + (self.lisp.sbclWith + (p: with p; [ asdf alexandria ])) # just enough to build Example.lisp + ]; + strictDeps = true; buildPhase = '' - export CODEROOT=$(pwd) - mkdir $out - ${bild}/bin/bild --plan "$TARGET" 1> $out/analysis.json \ - 2> >(tee -a $out/stderr >&2) + mkdir -p $out/bin $out/lib/ghc-${self.haskell.ghcPackageSetFull.version} + cp -r \ + ${self.haskell.ghcPackageSetFull}/lib/ghc-${self.haskell.ghcPackageSetFull.version}/package.conf.d \ + $out/lib/ghc-${self.haskell.ghcPackageSetFull.version} + ghc \ + -threaded \ + -Werror \ + -i. \ + --make Biz/Bild.hs \ + -main-is Biz.Bild \ + -o $out/bin/bild + ''; + installPhase = '' + wrapProgram $out/bin/bild \ + --prefix PATH : ${ + self.lib.makeBinPath [ + self.haskell.ghcPackageSetBild + self.pkgs.git + ] + } \ + --set GHC_PACKAGE_PATH \ + $out/lib/ghc-${self.haskell.ghcPackageSetFull.version}/package.conf.d ''; - installPhase = "exit 0"; }; - # gather data needed for compiling by analyzing the main module. returns the - # json object of the build - analyze = target: - builtins.readFile (runBildAnalyze target + "/analysis.json"); - - # this does a bild build for the given target, but entirely in nix. its kinda - # like IFD, but not as costly, i think - run = target: import ./Bild/Builder.nix { analysisJSON = analyze target; }; - - # the main development environment - env = stable.pkgs.mkShell { - name = "bizdev"; - # this should just be dev tools - buildInputs = with stable.pkgs; [ - bat - bc - bild - ctags - fd - figlet - fzf - git - git-branchless - gitlint - jq - lolcat - stable.haskell.packages.${constants.ghcCompiler}.fast-tags - ormolu - ripgrep - tree - wemux - ]; - shellHook = '' - export GHC_PACKAGE_PATH=${bild}/lib/ghc-${haskell.ghcPackageSetFull.version}/package.conf.d - ''; - }; + # wrapper around bild + runBildAnalyze = target: + self.stdenv.mkDerivation rec { + name = "bild-analysis"; + src = ../.; + USER = "nixbld"; + HOSTNAME = "nix-sandbox"; + # we need to remove the $src root because bild expects paths relative to the + # working directory: + TARGET = "." + + self.lib.strings.removePrefix (toString src) (toString target); + buildPhase = '' + export CODEROOT=$(pwd) + mkdir $out + ${self.bild}/bin/bild --plan "$TARGET" 1> $out/analysis.json \ + 2> >(tee -a $out/stderr >&2) + ''; + installPhase = "exit 0"; + }; + + # gather data needed for compiling by analyzing the main module. returns the + # json object of the build + analyze = target: + builtins.readFile (self.runBildAnalyze target + "/analysis.json"); + + # this does a bild build for the given target, but entirely in nix. its kinda + # like IFD, but not as costly, i think + run = target: + import ./Bild/Builder.nix { + analysisJSON = self.analyze target; + bild = self; + }; + + # the main development environment + env = stable.mkShell { + name = "bizdev"; + # this should just be dev tools + buildInputs = with self.pkgs; [ + bat + bc + self.bild + ctags + fd + figlet + fzf + git + git-branchless + gitlint + jq + lolcat + ormolu + ripgrep + stable.haskell.packages.${constants.ghcCompiler}.fast-tags + tree + wemux + ]; + shellHook = '' + export GHC_PACKAGE_PATH=${self.bild}/lib/ghc-${self.haskell.ghcPackageSetFull.version}/package.conf.d + ''; + }; - # build an operating system. 'cfg' is the NixOS config - os = cfg: (stable.nixos (_args: cfg)).toplevel; + # build an operating system. 'cfg' is the NixOS config + os = cfg: (stable.nixos (_args: cfg)).toplevel; - # build a docker image - image = stable.pkgs.dockerTools.buildImage; -} + # build a docker image + image = stable.dockerTools.buildImage; + }; +in self diff --git a/Biz/Bild/Builder.nix b/Biz/Bild/Builder.nix index d1fc48b..1a2460a 100644 --- a/Biz/Bild/Builder.nix +++ b/Biz/Bild/Builder.nix @@ -3,9 +3,9 @@ wouldn't you? - Try to reuse as much upstream Nix as possible. */ -{ analysisJSON, bild ? import ../Bild.nix { } }: +{ analysisJSON, bild }: +with bild; let - lib = bild.lib; analysis = builtins.fromJSON analysisJSON; build = _: target: let @@ -53,29 +53,28 @@ let sysdeps_ = if isEmpty target.sysdeps then [ ] else - lib.attrsets.attrVals target.sysdeps bild.pkgs; + lib.attrsets.attrVals target.sysdeps pkgs; rundeps_ = if isEmpty target.rundeps then [ ] else - lib.attrsets.attrVals target.rundeps bild.pkgs; + lib.attrsets.attrVals target.rundeps pkgs; CODEROOT = "."; builders = { - base = bild.stdenv.mkDerivation rec { + base = stdenv.mkDerivation rec { inherit name src CODEROOT preBuild; buildInputs = langdeps_ ++ sysdeps_; installPhase = "install -D ${name} $out/bin/${name}"; buildPhase = compileLine; }; - haskell = bild.stdenv.mkDerivation rec { + haskell = stdenv.mkDerivation rec { inherit name src CODEROOT preBuild; - nativeBuildInputs = [ bild.makeWrapper ]; + nativeBuildInputs = [ makeWrapper ]; buildInputs = sysdeps_ ++ [ - (bild.haskell.ghcWith - (p: (lib.attrsets.attrVals target.langdeps p))) + (haskell.ghcWith (p: (lib.attrsets.attrVals target.langdeps p))) ]; buildPhase = compileLine; installPhase = '' @@ -85,7 +84,7 @@ let ''; }; - c = bild.stdenv.mkDerivation rec { + c = stdenv.mkDerivation rec { inherit name src CODEROOT preBuild; buildInputs = langdeps_ ++ sysdeps_; installPhase = "install -D ${name} $out/bin/${name}"; @@ -106,11 +105,11 @@ let ]; }; - python = bild.python.buildPythonApplication rec { + python = python.buildPythonApplication rec { inherit name src CODEROOT; propagatedBuildInputs = langdeps_ ++ sysdeps_; buildInputs = sysdeps_; - nativeCheckInputs = lib.attrsets.attrVals [ "mypy" "ruff" ] bild.pkgs; + nativeCheckInputs = lib.attrsets.attrVals [ "mypy" "ruff" ] pkgs; checkPhase = '' check() { $@ || { echo "fail: $name: $3"; exit 1; } @@ -155,4 +154,4 @@ let # return a single drv, so just take the first one for now. ideally i would only # pass Target, one at a time, (perhaps parallelized in haskell land) and then i # wouldn't need all of this let nesting -in builtins.head (bild.lib.attrsets.mapAttrsToList build analysis) +in builtins.head (lib.attrsets.mapAttrsToList build analysis) diff --git a/Biz/Cloud.nix b/Biz/Cloud.nix index 4ef98eb..b2dbcfc 100644 --- a/Biz/Cloud.nix +++ b/Biz/Cloud.nix @@ -1,4 +1,4 @@ -{ bild ? import ./Bild.nix { } }: +{ bild }: # Cloud infrastructure, always online. Mostly for messaging-related stuff. bild.os { diff --git a/Biz/Dev/Beryllium.nix b/Biz/Dev/Beryllium.nix index b46126d..607e5c1 100644 --- a/Biz/Dev/Beryllium.nix +++ b/Biz/Dev/Beryllium.nix @@ -1,4 +1,4 @@ -{ bild ? import ../Bild.nix { } }: +{ bild }: bild.os { imports = [ ../OsBase.nix diff --git a/Biz/Dev/Lithium.nix b/Biz/Dev/Lithium.nix index fec8574..02488d9 100644 --- a/Biz/Dev/Lithium.nix +++ b/Biz/Dev/Lithium.nix @@ -1,4 +1,4 @@ -{ bild ? import ../Bild.nix { } }: +{ bild }: # Dev machine for work and building stuff. bild.os { diff --git a/Biz/Dragons/Analysis.nix b/Biz/Dragons/Analysis.nix index 2ff0246..9e59dee 100644 --- a/Biz/Dragons/Analysis.nix +++ b/Biz/Dragons/Analysis.nix @@ -1,4 +1,4 @@ -{ bild ? import ../Bild.nix { } }: +{ bild }: # Run this like so: # diff --git a/Biz/Que.nix b/Biz/Que.nix index 6971914..fc76758 100644 --- a/Biz/Que.nix +++ b/Biz/Que.nix @@ -1,4 +1,4 @@ -{ bild ? import ./Bild.nix { } }: +{ bild }: # The production server for que.run -- cgit v1.2.3