From e5a6175e044d69b8f598a2c2acb9bcfd77b9001c Mon Sep 17 00:00:00 2001 From: Ben Sima Date: Mon, 21 Aug 2023 20:36:12 -0400 Subject: Refactor the build system for readability Lots of changes here but the code is much improved. The nix code is clearer and structured better. The Haskell code improved in response to the nix changes. I needed to use a qualified path instead of the abspath because the BIZ_ROOT changes based on whether bild runs in nix or runs in the user environment. Rather than passing every argument into Builder.nix, now I just pass the json from bild and deconstruct it in nix. This is obviously a much better design and it only came to be after sleeping on it the other night. --- Biz/Bild.hs | 119 ++++++++++++++------------ Biz/Bild.nix | 156 ++++++++++++--------------------- Biz/Bild/Builder.nix | 214 ++++++++++++++++++++++++---------------------- Biz/Bild/Deps/C.nix | 3 + Biz/Bild/Deps/Haskell.nix | 144 +++++++++++++++---------------- Biz/Bild/Sources.json | 12 --- Biz/Cloud.nix | 4 +- Biz/Dev.nix | 6 +- Biz/Dev/Configuration.nix | 2 +- Biz/Dragons/Analysis.nix | 6 +- Biz/Que.nix | 11 ++- shell.nix | 2 +- 12 files changed, 319 insertions(+), 360 deletions(-) create mode 100644 Biz/Bild/Deps/C.nix diff --git a/Biz/Bild.hs b/Biz/Bild.hs index 2db5ccc..9c4f035 100644 --- a/Biz/Bild.hs +++ b/Biz/Bild.hs @@ -133,6 +133,7 @@ import qualified Data.Char as Char import Data.Conduit ((.|)) import qualified Data.Conduit.Combinators as Conduit import qualified Data.Conduit.Process as Conduit +import qualified Data.List as List import qualified Data.Map as Map import qualified Data.Set as Set import qualified Data.String as String @@ -271,6 +272,10 @@ instance Aeson.ToJSON Compiler where instance ToNixFlag Compiler where toNixFlag = compilerExe +-- | Type alias for making sure that the path is qualified, meaning it starts at +-- the root of the repo, and is not an absolute path nor a subpath +type QualifiedPath = FilePath + data Target = Target { -- | Output name out :: Meta.Out, @@ -278,8 +283,10 @@ data Target = Target outPath :: FilePath, -- | Fully qualified namespace partitioned by '.' namespace :: Namespace, - -- | Absolute path to file - path :: FilePath, + -- | Path to file, qualified based on the root of the git directory + quapath :: QualifiedPath, + -- | Main module name, formatted as the language expects + mainModule :: String, -- | Name of the packageset in Bild.nix to pull langdeps from packageSet :: Text, -- | Language-specific dependencies, required during compilation @@ -291,8 +298,12 @@ data Target = Target sysdeps :: Set Meta.Dep, -- | Which compiler should we use? compiler :: Compiler, - -- | Where is this machine being built? - builder :: Builder, + -- | Which nix build expression? + builder :: Text, + -- | Who is building this? + user :: Text, + -- | Where are they buildint it? + host :: Text, -- | Flags and arguments passed to 'Compiler' when building compilerFlags :: [Text], -- | Wrapper script (if necessary) @@ -419,12 +430,13 @@ analyze hmap ns = case Map.lookup ns hmap of analyzeOne namespace@(Namespace _ ext) = do let path = Namespace.toPath namespace root <- Env.getEnv "BIZ_ROOT" - let absPath = root path + let abspath = root path + let quapath = path user <- Env.getEnv "USER" /> Text.pack host <- HostName.getHostName /> Text.pack Log.info ["bild", "analyze", str path] contentLines <- - withFile absPath ReadMode <| \h -> + withFile abspath ReadMode <| \h -> IO.hSetEncoding h IO.utf8_bom >> Text.IO.hGetContents h /> Text.lines @@ -438,7 +450,7 @@ analyze hmap ns = case Map.lookup ns hmap of Namespace.Py -> Meta.detectAll "#" contentLines |> \Meta.Parsed {..} -> Target - { builder = Local user host, + { builder = "python", wrapper = Nothing, compiler = CPython, compilerFlags = @@ -447,17 +459,19 @@ analyze hmap ns = case Map.lookup ns hmap of [ "-c", "\"import py_compile;import os;" <> "py_compile.compile(file='" - <> str path + <> str quapath <> "', cfile=os.getenv('BIZ_ROOT')+'/_/int/" - <> str path + <> str quapath <> "', doraise=True)\"" ], sysdeps = psys, langdeps = pdep, outPath = outToPath pout, out = pout, - srcs = Set.singleton path, - packageSet = "pythonPackages", + -- implement detectPythonImports, then I can fill this out + srcs = Set.empty, + packageSet = "python.packages", + mainModule = Namespace.toModule namespace, .. } |> Just @@ -470,15 +484,17 @@ analyze hmap ns = case Map.lookup ns hmap of sysdeps = psys, wrapper = Nothing, compiler = Gcc, - builder = Local user host, + builder = "c", out = pout, - packageSet = "cPackages", + packageSet = "c.packages", + mainModule = Namespace.toModule namespace, compilerFlags = case pout of Meta.Bin o -> ["-o", o, path] <> Set.toList parg |> map Text.pack _ -> panic "can only bild C exes, not libs", outPath = outToPath pout, - srcs = Set.singleton absPath, + -- implement detectCImports, then I can fill this out + srcs = Set.empty, .. } |> Just @@ -489,10 +505,11 @@ analyze hmap ns = case Map.lookup ns hmap of |> \out -> detectHaskellImports hmap contentLines +> \(langdeps, srcs) -> Target - { builder = Local user host, + { builder = "haskell", wrapper = Nothing, compiler = Ghc, - packageSet = "ghcPackages", + packageSet = "haskell.packages", + mainModule = Namespace.toModule namespace, compilerFlags = [ "-Werror", "-threaded", @@ -502,7 +519,7 @@ analyze hmap ns = case Map.lookup ns hmap of "-hidir", ".", "--make", - "$BIZ_ROOT" path + "$BIZ_ROOT" quapath ] ++ case out of Meta.Bin o -> @@ -528,20 +545,22 @@ analyze hmap ns = case Map.lookup ns hmap of { sysdeps = Set.empty, wrapper = Nothing, compiler = Sbcl, - packageSet = "sbclWith", + packageSet = "lisp.sbclWith", + mainModule = Namespace.toModule namespace, compilerFlags = map Text.pack [ "--eval", "(require :asdf)", "--load", - absPath, + quapath, "--eval", "(sb-ext:save-lisp-and-die #p\"" <> (root outToPath out) <> "\" :toplevel #'main :executable t)" ], - builder = Local user host, + builder = "base", outPath = outToPath out, - srcs = Set.singleton absPath, + -- add local src imports to detectLispImports, then i can fill this out + srcs = Set.empty, .. } Namespace.Nix -> @@ -552,7 +571,7 @@ analyze hmap ns = case Map.lookup ns hmap of sysdeps = Set.empty, compiler = NixBuild, compilerFlags = - [ absPath, + [ quapath, "--out-link", root nixdir Namespace.toPath namespace, "--builders", @@ -561,8 +580,10 @@ analyze hmap ns = case Map.lookup ns hmap of |> map Text.pack, out = Meta.None, outPath = outToPath Meta.None, - srcs = Set.singleton absPath, + srcs = Set.empty, packageSet = "", + mainModule = Namespace.toModule namespace, + builder = "base", .. } |> Just @@ -573,19 +594,20 @@ analyze hmap ns = case Map.lookup ns hmap of { langdeps = pdep, sysdeps = psys, compiler = Guile, - packageSet = "guilePackages", + packageSet = "scheme.guilePackages", + mainModule = Namespace.toModule namespace, compilerFlags = [ "compile", "--r7rs", "--load-path=" ++ root, - "--output=" ++ root intdir replaceExtension path ".scm.go", - absPath + "--output=" ++ root intdir replaceExtension quapath ".scm.go", + quapath ] |> map Text.pack, - builder = Local user host, + builder = "base", outPath = outToPath pout, out = pout, - srcs = Set.singleton absPath, + srcs = Set.empty, -- implement detectSchemeImports -- TODO: wrapper should just be removed, instead rely on -- upstream nixpkgs builders to make wrappers wrapper = @@ -615,7 +637,8 @@ analyze hmap ns = case Map.lookup ns hmap of -- this packageSet doesn't actually exist because everyone in -- nix just generates nix expressions for rust dependencies with -- Cargo.lock, so I have to make it in order to use rust deps - packageSet = "rustPackages", + packageSet = "rust.packages", + mainModule = Namespace.toModule namespace, wrapper = Nothing, sysdeps = psys <> Set.singleton "rustc", out = pout, @@ -629,9 +652,10 @@ analyze hmap ns = case Map.lookup ns hmap of o ] _ -> panic "can't build rust libs", - builder = Local user host, + builder = "base", outPath = outToPath pout, - srcs = Set.singleton absPath, + -- implement detectRustImports + srcs = Set.empty, .. } |> Just @@ -651,9 +675,11 @@ detectHaskellImports hmap contentLines = +> \files -> findDeps root files +> \deps -> - (pkgs <> deps, Set.fromList files) + (pkgs <> deps, map (stripRoot root) files |> Set.fromList) |> pure where + stripRoot :: FilePath -> FilePath -> QualifiedPath + stripRoot root f = fromMaybe f (List.stripPrefix (root <> "/") f) filepaths :: [String] -> IO [FilePath] filepaths imports = imports @@ -742,7 +768,7 @@ build andTest loud analysis = Ghc -> case out of Meta.None -> pure (Exit.ExitSuccess, mempty) Meta.Bin _ -> do - Log.info ["bild", "nix", toLog builder, nschunk namespace] + Log.info ["bild", "nix", user <> "@" <> host, nschunk namespace] result <- nixBuild loud target if andTest && (isSuccess <| fst result) then test loud target @@ -761,7 +787,7 @@ build andTest loud analysis = Dir.setPermissions (root outToPath out) (Dir.setOwnerExecutable True p) pure (Exit.ExitSuccess, mempty) NixBuild -> do - Log.info ["bild", "nix", toLog builder, nschunk namespace] + Log.info ["bild", "nix", user <> "@" <> host, nschunk namespace] proc loud namespace (toNixFlag compiler) compilerFlags Copy -> do Log.warn ["bild", "copy", "not implemented yet", nschunk namespace] @@ -773,11 +799,6 @@ build andTest loud analysis = Log.info ["bild", "dev", "lisp", nschunk namespace] proc loud namespace (toNixFlag compiler) compilerFlags --- | Format for logging -toLog :: Builder -> Text -toLog (Local u h) = "local://" <> u <> "@" <> h -toLog (Remote u h) = "remote://" <> u <> "@" <> h - data Proc = Proc { loud :: Bool, cmd :: String, @@ -884,7 +905,7 @@ lispRequires = isQuote c = c `elem` ['\'', ':'] nixBuild :: Bool -> Target -> IO (Exit.ExitCode, ByteString) -nixBuild loud Target {..} = +nixBuild loud target@(Target {..}) = Env.getEnv "BIZ_ROOT" +> \root -> instantiate root |> run +> \case (_, "") -> panic "instantiate did not produce a drv" @@ -909,15 +930,7 @@ nixBuild loud Target {..} = -- is tightly coupled with the code in the nix builder and there's no -- way around that, methinks. args = - [ argstr "srcs" <| unwords <| map str <| Set.toList srcs <> [root path], - argstr "main" <| str <| Namespace.toModule namespace, - argstr "root" <| str root, - argstr "packageSet" packageSet, - (langdeps == mempty) ?: (mempty, argstr "langdeps" <| unwords <| map str <| Set.toList langdeps), - (sysdeps == mempty) ?: (mempty, argstr "sysdeps" <| unwords <| map str <| Set.toList sysdeps), - argstr "name" <| str <| outname out, - argstr "compileLine" <| unwords <| (Text.pack <| toNixFlag compiler) : compilerFlags, - ["--attr", selectBuilder namespace], + [ argstr "analysisJSON" <| str <| Aeson.encode <| (Map.singleton namespace target :: Analysis), [str <| root "Biz/Bild/Builder.nix"] ] |> mconcat @@ -949,11 +962,3 @@ nixBuild loud Target {..} = onFailure = Log.fail ["bild", "symlink", nschunk namespace] >> Log.br, onSuccess = pure () } - -selectBuilder :: Namespace -> Text -selectBuilder = \case - Namespace _ Namespace.Hs -> "haskell" - Namespace _ Namespace.Py -> "python" - Namespace _ Namespace.Rs -> "base" - Namespace _ Namespace.C -> "c" - Namespace _ ext -> panic <| "no builder for " <> show ext diff --git a/Biz/Bild.nix b/Biz/Bild.nix index ee0b1fe..1955d2c 100644 --- a/Biz/Bild.nix +++ b/Biz/Bild.nix @@ -1,79 +1,79 @@ { nixpkgs ? import ./Bild/Nixpkgs.nix }: -rec { +let constants = import ./Bild/Constants.nix; + lib = nixpkgs.lib; +# put all of our stuff in the 'bild' namespace +in nixpkgs // { bild = rec { + # provided by .envrc + root = builtins.getEnv "BIZ_ROOT"; - # internal usage - private = { - inherit nixpkgs; + inherit (nixpkgs) sources; - # provided by .envrc - root = builtins.getEnv "BIZ_ROOT"; + haskell = rec { + inherit (constants) ghcCompiler; - selectAttrs = deps: packageSet: - nixpkgs.lib.attrsets.attrVals deps packageSet; + # all available packages + deps = import ./Bild/Deps/Haskell.nix; + packages = lib.attrsets.getAttrs deps nixpkgs.haskellPackages; - # returns true if a is a subset of b, where a and b are attrsets - subset = a: b: builtins.all - (x: builtins.elem x b) a; + # make a ghc with dependencies + ghcWith = nixpkgs.haskell.packages.${ghcCompiler}.ghcWithHoogle; - # 44 = lib.strings.stringLength "/nix/store/gia2r9mxhc900y1m97dlmr1g3rm3ich3-" - dropNixStore = s: nixpkgs.lib.strings.substring 44 (nixpkgs.lib.strings.stringLength s) s; + # ghc with all packages, used for generating bild's package database + ghcPackageSetFull = ghcWith (p: lib.attrsets.attrVals deps p); - haskellDeps = import ./Bild/Deps/Haskell.nix; - - ghcPackages = nixpkgs.haskellPackages; - ghcWith = nixpkgs.haskell.packages.${constants.ghcCompiler}.ghcWithHoogle; - - sbclWith = nixpkgs.lispPackages_new.sbclWithPackages; - - pythonPackages = nixpkgs.python3Packages; - pythonWith = nixpkgs.python3.withPackages; - - cPackages = nixpkgs.pkgs; - - ghcPackageSetFull = private.ghcWith private.haskellDeps; - ghcPackageSetBild = private.ghcWith (hpkgs: with hpkgs; [ + # bild's dependencies, needs to be hand-written + ghcPackageSetBild = ghcWith (hpkgs: with hpkgs; [ aeson async base bytestring conduit conduit-extra containers directory docopt filepath process protolude rainbow regex-applicative split tasty tasty-hunit tasty-quickcheck text hostname wai # can remove when removed from Biz.Log ]); + }; + lisp = { + sbclWith = nixpkgs.lispPackages_new.sbclWithPackages; }; - # generally-useful things from nixpkgs - inherit (nixpkgs) lib stdenv sources; + python = { + packages = nixpkgs.python3Packages; + pythonWith = nixpkgs.python3.withPackages; + buildPythonApplication = nixpkgs.python3.pkgs.buildPythonApplication; + }; + + # c packages are just nixpkgs, filtered to just the list of deps i want + c.packages = lib.attrsets.getAttrs (import ./Bild/Deps/C.nix) nixpkgs.pkgs; # expose some packages for inclusion in os/image builds pkgs = with nixpkgs.pkgs; { inherit git; }; - # remove this when I switch to all-nix builds + # this is needed to do builds without calling out to nix, remove this when I + # switch to all-nix builds bildRuntimeDeps = with nixpkgs; [ pkg-config # this is just to get access to ghc-pkg in bild - (private.ghcWith (hpkgs: with hpkgs; [])) + (haskell.ghcWith (hpkgs: with hpkgs; [])) - /* disable until nixified builds are complete */ # lisp deps guile - (private.sbclWith (p: with p; [asdf alexandria])) # just enough to build Example.lisp + (lisp.sbclWith (p: with p; [asdf alexandria])) # just enough to build Example.lisp ]; # a standard nix build for `bild` - this should be the only hand-written # builder we need - bild = stdenv.mkDerivation { + bild = nixpkgs.stdenv.mkDerivation { name = "bild"; src = ../.; - nativeBuildInputs = [ private.ghcPackageSetBild ]; + nativeBuildInputs = [ haskell.ghcPackageSetBild ]; buildInputs = [ nixpkgs.makeWrapper ]; propagatedBuildInputs = bildRuntimeDeps; strictDeps = true; buildPhase = '' - mkdir -p $out/bin $out/lib/ghc-${private.ghcPackageSetFull.version} + mkdir -p $out/bin $out/lib/ghc-${haskell.ghcPackageSetFull.version} cp -r \ - ${private.ghcPackageSetFull}/lib/ghc-${private.ghcPackageSetFull.version}/package.conf.d \ - $out/lib/ghc-${private.ghcPackageSetFull.version} + ${haskell.ghcPackageSetFull}/lib/ghc-${haskell.ghcPackageSetFull.version}/package.conf.d \ + $out/lib/ghc-${haskell.ghcPackageSetFull.version} ghc \ -threaded \ -Werror \ @@ -84,14 +84,14 @@ rec { ''; installPhase = '' wrapProgram $out/bin/bild \ - --prefix PATH : ${lib.makeBinPath [ private.ghcPackageSetBild ]} \ + --prefix PATH : ${lib.makeBinPath [ haskell.ghcPackageSetBild ]} \ --set GHC_PACKAGE_PATH \ - $out/lib/ghc-${private.ghcPackageSetFull.version}/package.conf.d + $out/lib/ghc-${haskell.ghcPackageSetFull.version}/package.conf.d ''; }; # wrapper around bild - runBildAnalyze = main: stdenv.mkDerivation rec { + runBildAnalyze = main: nixpkgs.stdenv.mkDerivation rec { name = "bild-analysis"; src = ../.; USER = "nixbld"; @@ -109,43 +109,19 @@ rec { installPhase = "exit 0"; }; - # gather data needed for compiling by analyzing the main module - analyze = main: - let - path = lib.strings.removePrefix (builtins.getEnv "BIZ_ROOT" + "/") (toString main); - in - lib.attrsets.getAttrFromPath [path] - (lib.trivial.importJSON - (runBildAnalyze main + "/analysis.json")); - - buildPythonApplication = nixpkgs.python310.pkgs.buildPythonApplication; - - # build a ghc executable - ghc = main: - let - data = analyze main; - ghc = private.ghcWith (hp: private.selectAttrs data.langdeps hp); - module = lib.strings.concatStringsSep "." data.namespace.path; - in stdenv.mkDerivation { - name = module; - src = ../.; - nativeBuildInputs = [ ghc ] ++ private.selectAttrs data.sysdeps nixpkgs.pkgs; - strictDeps = true; - buildPhase = '' - set -eux - mkdir -p $out/bin - : compiling with ghc - ${ghc}/bin/ghc \ - -Werror \ - -i. \ - --make ${main} \ - -main-is ${module} \ - -o $out/bin/${data.out} - ''; - # the install process was handled above - installPhase = "exit 0"; - } // { env = ghc; }; + # gather data needed for compiling by analyzing the main module. returns the + # json object of the build + analyze = main: builtins.readFile (runBildAnalyze main + "/analysis.json"); + + # i think this isn't going to work because we have a nix-in-nix problem. + # instead: + # - get the store path some other way. if i can pass that to bild.os, then nix + # should automatically get all the deps required + # - just do runBildAnalyze, pass the args to Bild/Builder.nix, should continue + # no problem + run = main: import ./Bild/Builder.nix { analysisJSON = analyze main; }; + # the main development environment env = let linters = with nixpkgs.pkgs; [ ormolu hlint deadnix indent black]; in nixpkgs.pkgs.mkShell { @@ -156,7 +132,7 @@ rec { ctags figlet git - haskell.packages.${constants.ghcCompiler}.fast-tags + nixpkgs.haskell.packages.${constants.ghcCompiler}.fast-tags hlint lolcat #nixops # fails to build @@ -165,34 +141,14 @@ rec { wemux ]; shellHook = '' - export GHC_PACKAGE_PATH=${bild}/lib/ghc-${private.ghcPackageSetFull.version}/package.conf.d + export GHC_PACKAGE_PATH=${bild}/lib/ghc-${haskell.ghcPackageSetFull.version}/package.conf.d ''; }; # build an operating system. 'cfg' is the NixOS config os = cfg: (nixpkgs.nixos (_args: cfg)).toplevel; - # build a rust executable - rust = main: - let - data = analyze main; - rustc = nixpkgs.pkgs.rustc; - in stdenv.mkDerivation { - name = lib.string.concatStringsSep "::" data.namespace.path; - src = ../.; - nativeBuildInputs = [ rustc ]; - strictDeps = true; - buildPhase = '' - set -eux - mkdir -p $out/bin - : compiling with rustc - ${rustc}/bin/rustc \ - ${main} \ - -o $out/bin/${data.out} - ''; - installPhase = "exit 0"; - }; - # build a docker image image = nixpkgs.pkgs.dockerTools.buildImage; +}; } diff --git a/Biz/Bild/Builder.nix b/Biz/Bild/Builder.nix index 2b62b89..a5a31c7 100644 --- a/Biz/Bild/Builder.nix +++ b/Biz/Bild/Builder.nix @@ -3,115 +3,125 @@ This is the library of nix builders. Some rules to follow: - Keep this code as minimal as possible. I'd rather write Haskell than Nix, wouldn't you? - Try to reuse as much upstream Nix as possible. -- Path-like args such as 'srcs' should always be absolute paths. */ -{ srcs ? "" # list of all source files, as a space-separated string -, main # the entrypoint or main module (not a path) -, root # path to git root -, packageSet # name mapped to private.${packageSet}, e.g. 'ghcWith' -, langdeps ? null # list of deps (as a string), split and passed to packageSet -, sysdeps ? null -, name # exe name -, compileLine ? "" # Target.compiler <> Target.compilerFlags -}: -with import (/. + root + "/Biz/Bild.nix") {}; -with builtins; +{ analysisJSON, nixpkgs ? import ../Bild.nix {} }: +with nixpkgs; let - srcs_ = (lib.strings.splitString " " srcs) ++ [main]; + analysis = builtins.fromJSON analysisJSON; + build = _: target: let + name = target.out; + root = builtins.getEnv "BIZ_ROOT"; + mainModule = target.mainModule; + compileLine = + lib.strings.concatStringsSep " " ([target.compiler] ++ target.compilerFlags); - isEmpty = x: x == null || x == []; + allSources = target.srcs ++ [target.quapath]; - skip = ["_" ".direnv"]; - filter = file: type: - if elem (baseNameOf file) skip then false - # TODO: this means any new directory will cause a rebuild. this bad. i - # should recurse into the directory and match against the srcs. for now I - # just use postUnpack to delete empty dirs - else if type == "directory" then true - else if type == "regular" then (builtins.elem file srcs_) - else false; + isEmpty = x: x == null || x == []; - # clean up empty dirs - postUnpack = "find . -type d -empty -delete"; + skip = ["_" ".direnv"]; + filter = file: type: + if lib.lists.elem (builtins.baseNameOf file) skip then false + # TODO: this means any new directory will cause a rebuild. this bad. i + # should recurse into the directory and match against the srcs. for now I + # just use postUnpack to delete empty dirs + else if type == "directory" then true + else if type == "regular" then lib.trivial.pipe file + [ (f: lib.strings.removePrefix "${root}/" f) + (f: lib.lists.elem f allSources) + ] + else false; - src = lib.sources.cleanSourceWith {inherit filter; src = lib.sources.cleanSource root;}; + # clean up empty dirs + #postUnpack = "find $src -type d -empty -delete"; - langdeps_ = - if isEmpty langdeps then - [] - else - private.selectAttrs (lib.strings.splitString " " langdeps) private.${packageSet}; - sysdeps_ = - if isEmpty sysdeps then - [] - else - private.selectAttrs (lib.strings.splitString " " sysdeps) private.nixpkgs.pkgs; - BIZ_ROOT = "."; -in { - base = stdenv.mkDerivation rec { - inherit name src BIZ_ROOT postUnpack; - buildInputs = langdeps_ ++ sysdeps_; - installPhase = "install -D ${name} $out/bin/${name}"; - buildPhase = compileLine; - }; + src = lib.sources.cleanSourceWith {inherit filter; src = lib.sources.cleanSource root;}; - haskell = stdenv.mkDerivation rec { - inherit name src BIZ_ROOT postUnpack; - buildInputs = sysdeps_ ++ [ - (private.ghcWith (p: - (private.selectAttrs (lib.strings.splitString " " langdeps) p) - )) - ]; - installPhase = "install -D ${name} $out/bin/${name}"; - buildPhase = compileLine; - }; + langdeps_ = + if isEmpty target.langdeps then + [] + else + lib.attrsets.attrVals + target.langdeps + (lib.attrsets.getAttrFromPath (lib.strings.splitString "." target.packageSet) bild); + sysdeps_ = + if isEmpty target.sysdeps then + [] + else + lib.attrsets.attrVals target.sysdeps pkgs; + BIZ_ROOT = "."; - c = stdenv.mkDerivation rec { - inherit name src BIZ_ROOT postUnpack; - buildInputs = langdeps_ ++ sysdeps_; - installPhase = "install -D ${name} $out/bin/${name}"; - buildPhase = lib.strings.concatStringsSep " " [ - compileLine - (if isEmpty langdeps then "" else - "$(pkg-config --cflags ${langdeps})") - (if isEmpty sysdeps then "" else - "$(pkg-config --libs ${sysdeps})") - ]; - }; + builders = { + base = stdenv.mkDerivation rec { + inherit name src BIZ_ROOT; + buildInputs = langdeps_ ++ sysdeps_; + installPhase = "install -D ${name} $out/bin/${name}"; + buildPhase = compileLine; + }; - python = buildPythonApplication rec { - inherit name src BIZ_ROOT postUnpack; - propagatedBuildInputs = [ (private.pythonWith (_: langdeps_)) ] ++ sysdeps_; - buildInputs = sysdeps_; - checkInputs = [(private.pythonWith (p: with p; [black mypy pylint]))]; - checkPhase = '' - check() { - $@ || { echo "fail: $name: $3"; exit 1; } - } - check python -m black --quiet --exclude 'setup\.py$' --check . - check python -m pylint --errors-only . - check python -m mypy --strict --no-error-summary --exclude 'setup\.py$' . - check python -m ${main} test - ''; - preBuild = '' - # initialize possibly-empty subdirectories as python modules - find . -type d -exec touch {}/__init__.py \; - # generate a minimal setup.py - cat > setup.py << EOF - from setuptools import setup, find_packages - setup( - name='${name}', - entry_points={'console_scripts':['${name} = ${main}:main']}, - version='0.0.0', - url='git://simatime.com/biz.git', - author='dev', - author_email='dev@simatime.com', - description='nil', - packages=find_packages(), - install_requires=[], - ) - EOF - ''; - pythonImportsCheck = [main]; # sanity check - }; -} + haskell = stdenv.mkDerivation rec { + inherit name src BIZ_ROOT; + buildInputs = sysdeps_ ++ [ + (bild.haskell.ghcWith (p: + (lib.attrsets.attrVals target.langdeps p) + )) + ]; + installPhase = "install -D ${name} $out/bin/${name}"; + buildPhase = compileLine; + }; + + c = stdenv.mkDerivation rec { + inherit name src BIZ_ROOT; + buildInputs = langdeps_ ++ sysdeps_; + installPhase = "install -D ${name} $out/bin/${name}"; + buildPhase = lib.strings.concatStringsSep " " [ + compileLine + (if isEmpty langdeps_ then "" else + "$(pkg-config --cflags ${lib.strings.concatStringsSep " " target.langdeps})") + (if isEmpty sysdeps_ then "" else + "$(pkg-config --libs ${lib.strings.concatStringsSep " " target.sysdeps})") + ]; + }; + + python = bild.python.buildPythonApplication rec { + inherit name src BIZ_ROOT; + propagatedBuildInputs = [ (bild.python.pythonWith (_: langdeps_)) ] ++ sysdeps_; + buildInputs = sysdeps_; + checkInputs = [(bild.python.pythonWith (p: with p; [black mypy pylint]))]; + checkPhase = '' + check() { + $@ || { echo "fail: $name: $3"; exit 1; } + } + check python -m black --quiet --exclude 'setup\.py$' --check . + check python -m pylint --errors-only . + check python -m mypy --strict --no-error-summary --exclude 'setup\.py$' . + check python -m ${mainModule} test + ''; + preBuild = '' + # initialize possibly-empty subdirectories as python modules + find . -type d -exec touch {}/__init__.py \; + # generate a minimal setup.py + cat > setup.py << EOF + from setuptools import setup, find_packages + setup( + name='${name}', + entry_points={'console_scripts':['${name} = ${mainModule}:main']}, + version='0.0.0', + url='git://simatime.com/biz.git', + author='dev', + author_email='dev@simatime.com', + description='nil', + packages=find_packages(), + install_requires=[], + ) + EOF + ''; + pythonImportsCheck = [mainModule]; # sanity check + }; + }; + in builders.${target.builder}; +# the caller gives us the Analysis type, which is a hashmap, but i need to +# return a single drv, so just take the first one for now. ideally i would only +# pass Target, one at a time, (perhaps parallelized in haskell land) and then i +# wouldn't need all of this let nesting +in builtins.head (lib.attrsets.mapAttrsToList build analysis) diff --git a/Biz/Bild/Deps/C.nix b/Biz/Bild/Deps/C.nix new file mode 100644 index 0000000..45cae1b --- /dev/null +++ b/Biz/Bild/Deps/C.nix @@ -0,0 +1,3 @@ +[ + "libsodium" +] diff --git a/Biz/Bild/Deps/Haskell.nix b/Biz/Bild/Deps/Haskell.nix index 066fd86..f34bfab 100644 --- a/Biz/Bild/Deps/Haskell.nix +++ b/Biz/Bild/Deps/Haskell.nix @@ -1,76 +1,74 @@ -hpkgs: +# This is the global set of Haskell packages which gets deployed to Hoogle, and +# is available for selecting. -# This is the global set of Haskell packages which gets deployed to Hoogle. - -with hpkgs; [ - MonadRandom - QuickCheck - SafeSemaphore - acid-state - aeson - async - bytestring - clay - cmark - cmark-lucid - conduit - conduit-extra - config-ini - containers - directory - docopt - envy - fast-logger - filepath - github - hashids - haskeline - hmacaroons - hostname - http-types - ixset - katip - lucid - monad-logger - monad-metrics - mtl - neat-interpolation - network-uri - niv - optparse-simple - parsec - process - protolude - quickcheck-instances - rainbow - random - regex-applicative - req - safecopy - servant - servant-auth - servant-auth-server - servant-lucid - servant-server - split - stm - stripe-haskell - tasty - tasty-hunit - tasty-quickcheck - text - time - transformers - unagi-chan - unix - unordered-containers - uuid - vector - wai - wai-app-static - wai-extra - wai-middleware-metrics - warp - x509 + "MonadRandom" + "QuickCheck" + "SafeSemaphore" + "acid-state" + "aeson" + "async" + "base" + "bytestring" + "clay" + "cmark" + "cmark-lucid" + "conduit" + "conduit-extra" + "config-ini" + "containers" + "directory" + "docopt" + "envy" + "fast-logger" + "filepath" + "github" + "hashids" + "haskeline" + "hostname" + "http-types" + "ixset" + "katip" + "lucid" + "monad-logger" + "monad-metrics" + "mtl" + "neat-interpolation" + "network-uri" + "niv" + "optparse-simple" + "parsec" + "process" + "protolude" + "quickcheck-instances" + "rainbow" + "random" + "regex-applicative" + "req" + "safecopy" + "servant" + "servant-auth" + "servant-auth-server" + "servant-lucid" + "servant-server" + "split" + "stm" + "stripe-haskell" + "tasty" + "tasty-hunit" + "tasty-quickcheck" + "text" + "time" + "transformers" + "unagi-chan" + "unix" + "unordered-containers" + "uuid" + "vector" + "wai" + "wai-app-static" + "wai-extra" + "wai-middleware-metrics" + "warp" + "x509" ] diff --git a/Biz/Bild/Sources.json b/Biz/Bild/Sources.json index 5d05ea0..e4fcfd4 100644 --- a/Biz/Bild/Sources.json +++ b/Biz/Bild/Sources.json @@ -56,18 +56,6 @@ "type": "git", "version": "2021-06-14-unstable" }, - "hmacaroons": { - "branch": "master", - "description": "Pure haskell implementation of macaroons", - "homepage": "https://jtanguy.github.io/hmacaroons", - "owner": "jtanguy", - "repo": "hmacaroons", - "rev": "6fbca87836a4baef171c5ffc774387766c709fbf", - "sha256": "0qd1ifx1rzcv8rc74vb5xxgi544qxclx8ky3wjg0nbj22hpvvg6j", - "type": "tarball", - "url": "https://github.com/jtanguy/hmacaroons/archive/6fbca87836a4baef171c5ffc774387766c709fbf.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, "inspekt3d": { "branch": "master", "sha256": "0lan6930g5a9z4ack9jj0zdd0mb2s6q2xzpiwcjdc3pvl9b1nbw4", diff --git a/Biz/Cloud.nix b/Biz/Cloud.nix index 05abfa7..1ae94ac 100644 --- a/Biz/Cloud.nix +++ b/Biz/Cloud.nix @@ -1,5 +1,5 @@ -{ bild ? import ./Bild.nix {} }: - +{ nixpkgs ? import ./Bild.nix {} }: +with nixpkgs; # Cloud infrastructure, always online. Mostly for messaging-related stuff. bild.os { diff --git a/Biz/Dev.nix b/Biz/Dev.nix index 6be4856..245e58c 100644 --- a/Biz/Dev.nix +++ b/Biz/Dev.nix @@ -1,5 +1,5 @@ -{ bild ? import ./Bild.nix {} }: - +{ nixpkgs ? import ./Bild.nix {} }: +with nixpkgs; # Dev machine for work and building stuff. bild.os { @@ -21,7 +21,7 @@ bild.os { services.dragons = { enable = true; port = 8095; - package = bild.ghc ./Dragons.hs; + package = bild.run ./Dragons.hs; keep = "/var/dragons/keep"; depo = "/var/dragons/depo"; }; diff --git a/Biz/Dev/Configuration.nix b/Biz/Dev/Configuration.nix index bcf184d..7fdefa4 100644 --- a/Biz/Dev/Configuration.nix +++ b/Biz/Dev/Configuration.nix @@ -63,7 +63,7 @@ in { services.my-hoogle.enable = true; services.my-hoogle.port = ports.hoogle; services.my-hoogle.home = "//hoogle.simatime.com"; - services.my-hoogle.packages = import ../Bild/Deps/Haskell.nix; + services.my-hoogle.packages = pkgset: lib.attrsets.attrVals (import ../Bild/Deps/Haskell.nix) pkgset; services.my-hoogle.haskellPackages = pkgs.haskell.packages.${ghcCompiler}; services.my-hoogle.host = "0.0.0.0"; diff --git a/Biz/Dragons/Analysis.nix b/Biz/Dragons/Analysis.nix index 0896dbe..de431a7 100644 --- a/Biz/Dragons/Analysis.nix +++ b/Biz/Dragons/Analysis.nix @@ -1,5 +1,5 @@ -{ bild ? import ../Bild.nix {} }: - +{ nixpkgs ? import ../Bild.nix {} }: +with nixpkgs; # Run this like so: # # bild Biz/Dragons/Analysis.nix @@ -14,7 +14,7 @@ bild.image { fromImageTag = "latest"; contents = [ bild.pkgs.git - (bild.ghc ./Analysis.hs) + (bild.run ./Analysis.hs) ]; config.Cmd = [ "/bin/dragons-analyze" ]; } diff --git a/Biz/Que.nix b/Biz/Que.nix index b94f4d2..103aef0 100644 --- a/Biz/Que.nix +++ b/Biz/Que.nix @@ -1,6 +1,5 @@ -{ bild ? import ./Bild.nix {} -, nixpkgs ? import ./Bild/Nixpkgs.nix -}: +{ nixpkgs ? import ./Bild.nix {} }: +with nixpkgs; # The production server for que.run @@ -17,7 +16,7 @@ bild.os { services.que-server = { enable = true; port = 80; - package = bild.ghc ./Que/Host.hs; + package = bild.run ./Que/Host.hs; }; boot.loader.grub.device = "/dev/vda"; fileSystems."/" = { device = "/dev/vda1"; fsType = "ext4"; }; @@ -33,7 +32,7 @@ bild.os { defaultGateway = "157.245.224.1"; defaultGateway6 = "2604:a880:2:d1::1"; dhcpcd.enable = false; - usePredictableInterfaceNames = nixpkgs.lib.mkForce true; + usePredictableInterfaceNames = lib.mkForce true; interfaces = { eth0 = { ipv4.addresses = [ @@ -53,7 +52,7 @@ bild.os { que-website = { enable = true; namespace = "_"; - package = bild.ghc ./Que/Site.hs; + package = bild.run ./Que/Site.hs; }; udev.extraRules = '' diff --git a/shell.nix b/shell.nix index dabf712..1db7442 100644 --- a/shell.nix +++ b/shell.nix @@ -1,3 +1,3 @@ (import ./Biz/Bild.nix { nixpkgs = import ./Biz/Bild/Nixpkgs.nix; -}).env +}).bild.env -- cgit v1.2.3