From aa5f12a5a2365a35f17becc66cf7a67fa86a4440 Mon Sep 17 00:00:00 2001 From: Ben Sima Date: Mon, 8 Apr 2024 16:02:03 -0400 Subject: Reorganize and update nixpkgs upstreams This patch does a few things: 1. Switches from nixpkgs-unstable to nixos-unstable{,-small}, simply because nixpkgs-unstable is not in cache.nixos.org, but nixos-unstable is, and -small is the same but requires all tests to pass. So we should prefer nixos-unstable-small, whenever possible. 2. Reorganizes the nixpkgs import code such that Nixpkgs.nix returns an attrset of all the nixpkgs that I want to use, rather than putting other nixpkgs branches into the main one as an overlay. This is much simpler and explicit, but it meant I had to change a lot of usages throughtout the nix codebase. 3. As a consequence of 2, moves the overlays into separate files so they can be re-used across nixpkgs branches. --- Biz/Bild/Builder.nix | 16 +++++----- Biz/Bild/Deps.nix | 55 +--------------------------------- Biz/Bild/Functions.nix | 33 +++++++++++++++++++++ Biz/Bild/Haskell.nix | 44 +++++++++++++++++++++++++++ Biz/Bild/Nixpkgs.nix | 80 ++++++++++++++++++++++++-------------------------- Biz/Bild/Python.nix | 23 +++++++++++++++ Biz/Bild/Sources.json | 27 ++++++++++++----- 7 files changed, 166 insertions(+), 112 deletions(-) create mode 100644 Biz/Bild/Functions.nix create mode 100644 Biz/Bild/Haskell.nix create mode 100644 Biz/Bild/Python.nix (limited to 'Biz/Bild') diff --git a/Biz/Bild/Builder.nix b/Biz/Bild/Builder.nix index 4bef830..d1fc48b 100644 --- a/Biz/Bild/Builder.nix +++ b/Biz/Bild/Builder.nix @@ -3,9 +3,9 @@ wouldn't you? - Try to reuse as much upstream Nix as possible. */ -{ analysisJSON, nixpkgs ? import ../Bild.nix { } }: -with nixpkgs; +{ analysisJSON, bild ? import ../Bild.nix { } }: let + lib = bild.lib; analysis = builtins.fromJSON analysisJSON; build = _: target: let @@ -53,7 +53,7 @@ let sysdeps_ = if isEmpty target.sysdeps then [ ] else - lib.attrsets.attrVals target.sysdeps pkgs; + lib.attrsets.attrVals target.sysdeps bild.pkgs; rundeps_ = if isEmpty target.rundeps then [ ] @@ -63,16 +63,16 @@ let CODEROOT = "."; builders = { - base = stdenv.mkDerivation rec { + base = bild.stdenv.mkDerivation rec { inherit name src CODEROOT preBuild; buildInputs = langdeps_ ++ sysdeps_; installPhase = "install -D ${name} $out/bin/${name}"; buildPhase = compileLine; }; - haskell = stdenv.mkDerivation rec { + haskell = bild.stdenv.mkDerivation rec { inherit name src CODEROOT preBuild; - nativeBuildInputs = [ makeWrapper ]; + nativeBuildInputs = [ bild.makeWrapper ]; buildInputs = sysdeps_ ++ [ (bild.haskell.ghcWith (p: (lib.attrsets.attrVals target.langdeps p))) @@ -85,7 +85,7 @@ let ''; }; - c = stdenv.mkDerivation rec { + c = bild.stdenv.mkDerivation rec { inherit name src CODEROOT preBuild; buildInputs = langdeps_ ++ sysdeps_; installPhase = "install -D ${name} $out/bin/${name}"; @@ -155,4 +155,4 @@ let # return a single drv, so just take the first one for now. ideally i would only # pass Target, one at a time, (perhaps parallelized in haskell land) and then i # wouldn't need all of this let nesting -in builtins.head (lib.attrsets.mapAttrsToList build analysis) +in builtins.head (bild.lib.attrsets.mapAttrsToList build analysis) diff --git a/Biz/Bild/Deps.nix b/Biz/Bild/Deps.nix index 9a18c90..8f44dde 100644 --- a/Biz/Bild/Deps.nix +++ b/Biz/Bild/Deps.nix @@ -1,59 +1,6 @@ _self: super: -with import ./Constants.nix; -let - buildCabal = sel: name: sel.callCabal2nix name super.sources.${name} { }; - buildCabalSubdir = sel: - { name, src ? super.sources.${name}, subdir ? name }: - sel.callCabal2nix name (src + "/${subdir}") { }; -in rec { - haskell = super.haskell // { - packages = super.haskell.packages // { - "${ghcCompiler}" = super.haskell.packages."${ghcCompiler}".override - (_old: { - overrides = with super.pkgs.haskell.lib; - sel: sup: - super.overridePinnedDeps (buildCabal sel) // { - acid-state = - dontCheck sup.acid-state; # mac: "too many open files" - clay = doJailbreak sup.clay; - envy = doJailbreak sup.envy; - fast-tags = - sup.fast-tags.overrideAttrs (old: old // { patches = [ ]; }); - generic-data = dontCheck - sup.generic-data; # https://github.com/Lysxia/generic-data/issues/56 - readable = - doJailbreak sup.readable; # why is this even being built? - servant-auth = doJailbreak (buildCabalSubdir sel { - name = "servant-auth"; - subdir = "servant-auth/servant-auth"; - }); - servant-server = doJailbreak sup.servant-server; - stripe-core = doJailbreak sup.stripe-core; - stripe-haskell = dontCheck sup.stripe-haskell; - stripe-http-client = - doJailbreak (dontCheck sup.stripe-http-client); - temporary = dontCheck sup.temporary; - wai-middleware-metrics = dontCheck sup.wai-middleware-metrics; - }; - }); - }; - }; - - python3 = super.python3.override { - packageOverrides = _: pysuper: - with pysuper.pkgs.python3Packages; { - exllama = callPackage ./Deps/exllama.nix { - cudaPackages = super.pkgs.cudaPackages_11_7; - }; - interegular = callPackage ./Deps/interegular.nix { }; - outlines = callPackage ./Deps/outlines.nix { }; - perscache = callPackage ./Deps/perscache.nix { }; - }; - }; - - python3Packages = python3.pkgs; - +{ autogen = super.callPackage ./Deps/autogen.nix { }; guile = super.guile_3_0; diff --git a/Biz/Bild/Functions.nix b/Biz/Bild/Functions.nix new file mode 100644 index 0000000..b725a00 --- /dev/null +++ b/Biz/Bild/Functions.nix @@ -0,0 +1,33 @@ +_: super: { + # Given a generic `builder`, will generate an attrset for all the packages + # pinned by `deps` with `builder` applied to the package. This attrset can + # then be merged with the rest of the packages in the set as part of an + # overlay or overrides. + overridePinnedDeps = builder: + super.lib.genAttrs (builtins.attrNames super.sources) builder; + + # Modifies a derivation with our source and version, keeping super build + # rules. This will fail if build steps have changed, or if no build + # rules are available upstream. + overrideSource = depName: + if super ? "${depName}" && super.${depName} ? overrideAttrs then + super.${depName}.overrideAttrs (attrs: + attrs // rec { + version = + super.sources.${depName}.version or super.sources.${depName}.rev; + src = super.sources.${depName}; + }) + else + null; + + # Simply override the 'src' attr on a drv. This is meant to be a simpler + # alternative to 'overrideSource' above. In an overaly, use it like: + # mypkg = super.overrideSource super.mypkg super.sources.mypkg; + overrideSrc = dep: src: + dep.overrideAttrs (attrs: + attrs // { + version = src.version or src.rev; + src = src; + }); +} + diff --git a/Biz/Bild/Haskell.nix b/Biz/Bild/Haskell.nix new file mode 100644 index 0000000..2c0529a --- /dev/null +++ b/Biz/Bild/Haskell.nix @@ -0,0 +1,44 @@ +_self: super: + +let + ghcCompiler = "ghc924"; + buildCabal = sel: name: sel.callCabal2nix name super.sources.${name} { }; + buildCabalSubdir = sel: + { name, src ? super.sources.${name}, subdir ? name }: + sel.callCabal2nix name (src + "/${subdir}") { }; +in rec { + + haskell = super.haskell // { + packages = super.haskell.packages // { + "${ghcCompiler}" = super.haskell.packages."${ghcCompiler}".override + (_old: { + overrides = with super.pkgs.haskell.lib; + sel: sup: + super.overridePinnedDeps (buildCabal sel) // { + acid-state = + dontCheck sup.acid-state; # mac: "too many open files" + clay = doJailbreak sup.clay; + envy = doJailbreak sup.envy; + fast-tags = + sup.fast-tags.overrideAttrs (old: old // { patches = [ ]; }); + generic-data = dontCheck + sup.generic-data; # https://github.com/Lysxia/generic-data/issues/56 + readable = + doJailbreak sup.readable; # why is this even being built? + servant-auth = doJailbreak (buildCabalSubdir sel { + name = "servant-auth"; + subdir = "servant-auth/servant-auth"; + }); + servant-server = doJailbreak sup.servant-server; + stripe-core = doJailbreak sup.stripe-core; + stripe-haskell = dontCheck sup.stripe-haskell; + stripe-http-client = + doJailbreak (dontCheck sup.stripe-http-client); + temporary = dontCheck sup.temporary; + wai-middleware-metrics = dontCheck sup.wai-middleware-metrics; + }; + }); + }; + }; + +} diff --git a/Biz/Bild/Nixpkgs.nix b/Biz/Bild/Nixpkgs.nix index 8cb3448..f0e8698 100644 --- a/Biz/Bild/Nixpkgs.nix +++ b/Biz/Bild/Nixpkgs.nix @@ -2,54 +2,50 @@ let sources = import ./Sources.nix { sourcesFile = ./Sources.json; }; config = { + allowAliases = true; allowBroken = true; allowUnfree = true; + checkMeta = true; cudaSupport = true; }; system = __currentSystem; -in import sources.nixos-23_05 { - inherit system config; - overlays = [ - (_: _: { inherit sources; }) - - # add some functions - (_: old: { - # Given a generic `builder`, will generate an attrset for all the packages - # pinned by `deps` with `builder` applied to the package. This attrset can - # then be merged with the rest of the packages in the set as part of an - # overlay or overrides. - overridePinnedDeps = builder: - old.lib.genAttrs (builtins.attrNames sources) builder; - - # Modifies a derivation with our source and version, keeping old build - # rules. This will fail if build steps have changed, or if no build - # rules are available upstream. - overrideSource = depName: - if old ? "${depName}" && old.${depName} ? overrideAttrs then - old.${depName}.overrideAttrs (attrs: - attrs // rec { - version = sources.${depName}.version or sources.${depName}.rev; - src = sources.${depName}; - }) - else - null; - }) - - # override pinned deps with our sources, this must come before other - # package overlays, because of the 'null' above - (_: pkgs: pkgs.overridePinnedDeps pkgs.overrideSource) - - # add other nixpkgs distributions - (_: _: { - unstable = import sources.nixpkgs-unstable { inherit system config; }; - nixos-23_11 = import sources.nixos-23_11 { inherit system config; }; - }) - - # add our hand-written derivations - (import ./Deps.nix) - (_: pkgs: { niv = import pkgs.sources.niv { }; }) - ]; +in { + nixos-23_05 = import sources.nixos-23_05 { + inherit system config; + overlays = [ + (_: _: { inherit sources; }) + + (import ./Functions.nix) + + # override pinned deps with our sources, this must come before other + # package overlays, because of the 'null' from 'overrideSource' + (_: pkgs: pkgs.overridePinnedDeps pkgs.overrideSource) + + (import ./Deps.nix) + (import ./Python.nix) + (import ./Haskell.nix) + ]; + }; + + nixos-23_11 = import sources.nixos-23_11 { + inherit system config; + overlays = [ + (_: _: { inherit sources; }) + (import ./Functions.nix) + (import ./Deps.nix) + ]; + }; + + nixos-unstable-small = import sources.nixos-unstable-small { + inherit system config; + overlays = [ + (_: _: { inherit sources; }) + (import ./Functions.nix) + (import ./Deps.nix) + ]; + }; + } diff --git a/Biz/Bild/Python.nix b/Biz/Bild/Python.nix new file mode 100644 index 0000000..51f35f1 --- /dev/null +++ b/Biz/Bild/Python.nix @@ -0,0 +1,23 @@ +_self: super: { + python3 = super.python3.override { + packageOverrides = _pyself: pysuper: + with pysuper.pkgs.python3Packages; + let dontCheck = p: p.overridePythonAttrs (_: { doCheck = false; }); + in { + exllama = callPackage ./Deps/exllama.nix { + cudaPackages = super.pkgs.cudaPackages_11_7; + }; + exllamav2 = callPackage ./Deps/exllamav2.nix { + cudaPackages = super.pkgs.cudaPackages_11_7; + }; + interegular = callPackage ./Deps/interegular.nix { }; + mypy = dontCheck pysuper.mypy; + outlines = callPackage ./Deps/outlines.nix { }; + perscache = callPackage ./Deps/perscache.nix { }; + safetensors = super.nixos-23_11.python310Packages.safetensors.override { + buildPythonPackage = pysuper.pkgs.python3Packages.buildPythonPackage; + }; + tokenizers = dontCheck pysuper.tokenizers; + }; + }; +} diff --git a/Biz/Bild/Sources.json b/Biz/Bild/Sources.json index 52112b5..ed6b096 100644 --- a/Biz/Bild/Sources.json +++ b/Biz/Bild/Sources.json @@ -121,17 +121,28 @@ "url_template": "https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/archive//nixos-mailserver-.tar.gz", "version": "master" }, - "nixpkgs-unstable": { - "branch": "master", - "description": "Nix Packages collection", - "homepage": "https://github.com/nixos/nixpkgs", - "name": "nixpkgs", + "nixos-unstable": { + "branch": "nixos-unstable", + "description": "Nix Packages collection & NixOS", + "homepage": "", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "fd281bd6b7d3e32ddfa399853946f782553163b5", + "sha256": "1hy81yj2dcg6kfsm63xcqf8kvigxglim1rcg1xpmy2rb6a8vqvsj", + "type": "tarball", + "url": "https://github.com/nixos/nixpkgs/archive/fd281bd6b7d3e32ddfa399853946f782553163b5.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixos-unstable-small": { + "branch": "nixos-unstable-small", + "description": "Nix Packages collection & NixOS", + "homepage": "", "owner": "nixos", "repo": "nixpkgs", - "rev": "0b6c600141ae4c516593d05b35294c44d55344da", - "sha256": "0x76w3rnkmwgscjljd33pv50r2057lbhcwhf1qvmp36vxrxa98w5", + "rev": "6cc8dbb00974248cdd1b7ebd05cbc7c0799ce974", + "sha256": "09lyljxmzvwc71j5s2k0ya3y04ya3lxy8bgcm36gi8vmrn42dgv0", "type": "tarball", - "url": "https://github.com/nixos/nixpkgs/archive/0b6c600141ae4c516593d05b35294c44d55344da.tar.gz", + "url": "https://github.com/nixos/nixpkgs/archive/6cc8dbb00974248cdd1b7ebd05cbc7c0799ce974.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "outlines": { -- cgit v1.2.3