From 6513755670892983db88a6633b8c1ea6019c03d1 Mon Sep 17 00:00:00 2001 From: Ben Sima Date: Fri, 15 Nov 2024 14:55:37 -0500 Subject: Re-namespace some stuff to Omni I was getting confused about what is a product and what is internal infrastructure; I think it is good to keep those things separate. So I moved a bunch of stuff to an Omni namespace, actually most stuff went there. Only things that are explicitly external products are still in the Biz namespace. --- Omni/Bild/Builder.nix | 168 +++++++ Omni/Bild/CcacheWrapper.nix | 57 +++ Omni/Bild/Constants.nix | 1 + Omni/Bild/Deps.hs | 694 +++++++++++++++++++++++++++ Omni/Bild/Deps.nix | 45 ++ Omni/Bild/Deps/C.nix | 1 + Omni/Bild/Deps/Haskell.nix | 72 +++ Omni/Bild/Deps/Python.nix | 1 + Omni/Bild/Deps/accelerate.nix | 16 + Omni/Bild/Deps/bitsandbytes.nix | 86 ++++ Omni/Bild/Deps/guile-opengl.nix | 32 ++ Omni/Bild/Deps/inspekt3d.nix | 30 ++ Omni/Bild/Deps/interegular.nix | 21 + Omni/Bild/Deps/lion-pytorch.nix | 27 ++ Omni/Bild/Deps/llm-ollama.nix | 40 ++ Omni/Bild/Deps/llm-sentence-transformers.nix | 42 ++ Omni/Bild/Deps/nostr-rs-relay.nix | 19 + Omni/Bild/Deps/outlines.nix | 34 ++ Omni/Bild/Deps/perscache.nix | 25 + Omni/Bild/Example.c | 15 + Omni/Bild/Example.hs | 45 ++ Omni/Bild/Example.lisp | 4 + Omni/Bild/Example.py | 45 ++ Omni/Bild/Example.rs | 4 + Omni/Bild/Functions.nix | 33 ++ Omni/Bild/Haskell.nix | 36 ++ Omni/Bild/Meta.hs | 108 +++++ Omni/Bild/Nixpkgs.nix | 43 ++ Omni/Bild/Python.nix | 17 + Omni/Bild/Sources.json | 198 ++++++++ Omni/Bild/Sources.nix | 207 ++++++++ 31 files changed, 2166 insertions(+) create mode 100644 Omni/Bild/Builder.nix create mode 100644 Omni/Bild/CcacheWrapper.nix create mode 100644 Omni/Bild/Constants.nix create mode 100644 Omni/Bild/Deps.hs create mode 100644 Omni/Bild/Deps.nix create mode 100644 Omni/Bild/Deps/C.nix create mode 100644 Omni/Bild/Deps/Haskell.nix create mode 100644 Omni/Bild/Deps/Python.nix create mode 100644 Omni/Bild/Deps/accelerate.nix create mode 100644 Omni/Bild/Deps/bitsandbytes.nix create mode 100644 Omni/Bild/Deps/guile-opengl.nix create mode 100644 Omni/Bild/Deps/inspekt3d.nix create mode 100644 Omni/Bild/Deps/interegular.nix create mode 100644 Omni/Bild/Deps/lion-pytorch.nix create mode 100644 Omni/Bild/Deps/llm-ollama.nix create mode 100644 Omni/Bild/Deps/llm-sentence-transformers.nix create mode 100644 Omni/Bild/Deps/nostr-rs-relay.nix create mode 100644 Omni/Bild/Deps/outlines.nix create mode 100644 Omni/Bild/Deps/perscache.nix create mode 100644 Omni/Bild/Example.c create mode 100644 Omni/Bild/Example.hs create mode 100644 Omni/Bild/Example.lisp create mode 100644 Omni/Bild/Example.py create mode 100644 Omni/Bild/Example.rs create mode 100644 Omni/Bild/Functions.nix create mode 100644 Omni/Bild/Haskell.nix create mode 100644 Omni/Bild/Meta.hs create mode 100644 Omni/Bild/Nixpkgs.nix create mode 100644 Omni/Bild/Python.nix create mode 100644 Omni/Bild/Sources.json create mode 100644 Omni/Bild/Sources.nix (limited to 'Omni/Bild') diff --git a/Omni/Bild/Builder.nix b/Omni/Bild/Builder.nix new file mode 100644 index 0000000..a78f311 --- /dev/null +++ b/Omni/Bild/Builder.nix @@ -0,0 +1,168 @@ +/* This is the library of nix builders. Some rules to follow: + - Keep this code as minimal as possible. I'd rather write Haskell than Nix, + wouldn't you? + - Try to reuse as much upstream Nix as possible. +*/ +{ analysisJSON, bild }: +with bild; +let + analysis = builtins.fromJSON analysisJSON; + + # common bash functions for the builder + commonBash = builtins.toFile "common.bash" '' + # Check that a command succeeds, fail and log if not. + function check { + $@ || { echo "fail: $name: $3"; exit 1; } + } + ''; + + build = _: target: + let + name = target.out; + root = builtins.getEnv "CODEROOT"; + mainModule = target.mainModule; + compileLine = lib.strings.concatStringsSep " " + ([ target.compiler ] ++ target.compilerFlags); + + allSources = target.srcs ++ [ target.quapath ]; + + isEmpty = x: x == null || x == [ ]; + + skip = [ "_" ".direnv" ]; + filter = file: type: + if lib.lists.elem (builtins.baseNameOf file) skip then + false + # TODO: this means any new directory will cause a rebuild. this bad. i + # should recurse into the directory and match against the srcs. for now I + # just use preBuild to delete empty dirs + else if type == "directory" then + true + else if type == "regular" then + lib.trivial.pipe file [ + (f: lib.strings.removePrefix "${root}/" f) + (f: lib.lists.elem f allSources) + ] + else + false; + + # remove empty directories, leftover from the src filter + preBuild = "find . -type d -empty -delete"; + + src = lib.sources.cleanSourceWith { + inherit filter; + src = lib.sources.cleanSource root; + }; + + langdeps_ = if isEmpty target.langdeps then + [ ] + else + lib.attrsets.attrVals target.langdeps (lib.attrsets.getAttrFromPath + (lib.strings.splitString "." target.packageSet) bild); + + sysdeps_ = if isEmpty target.sysdeps then + [ ] + else + lib.attrsets.attrVals target.sysdeps pkgs; + + rundeps_ = if isEmpty target.rundeps then + [ ] + else + lib.attrsets.attrVals target.rundeps pkgs; + + CODEROOT = "."; + + builders = { + base = stdenv.mkDerivation rec { + inherit name src CODEROOT preBuild; + buildInputs = langdeps_ ++ sysdeps_; + installPhase = "install -D ${name} $out/bin/${name}"; + buildPhase = compileLine; + }; + + haskell = stdenv.mkDerivation rec { + inherit name src CODEROOT preBuild; + nativeBuildInputs = [ makeWrapper ]; + buildInputs = sysdeps_ ++ [ + (haskell.ghcWith (p: (lib.attrsets.attrVals target.langdeps p))) + ]; + buildPhase = compileLine; + installPhase = '' + install -D ${name} $out/bin/${name} + wrapProgram $out/bin/${name} \ + --prefix PATH : ${lib.makeBinPath rundeps_} + ''; + }; + + c = stdenv.mkDerivation rec { + inherit name src CODEROOT preBuild; + buildInputs = langdeps_ ++ sysdeps_; + installPhase = "install -D ${name} $out/bin/${name}"; + buildPhase = lib.strings.concatStringsSep " " [ + compileLine + (if isEmpty langdeps_ then + "" + else + "$(pkg-config --cflags ${ + lib.strings.concatStringsSep " " target.langdeps + })") + (if isEmpty sysdeps_ then + "" + else + "$(pkg-config --libs ${ + lib.strings.concatStringsSep " " target.sysdeps + })") + ]; + }; + + python = python.buildPythonApplication rec { + inherit name src CODEROOT; + nativeBuildInputs = [ makeWrapper ]; + propagatedBuildInputs = langdeps_ ++ sysdeps_ ++ rundeps_; + buildInputs = sysdeps_; + nativeCheckInputs = [ pkgs.ruff python.packages.mypy ]; + checkPhase = '' + . ${commonBash} + cp ${../../pyproject.toml} ./pyproject.toml + check ruff format --exclude 'setup.py' --check . + check ruff check --exclude 'setup.py' --exclude '__init__.py' . + touch ./py.typed + check python -m mypy \ + --explicit-package-bases \ + --no-error-summary \ + --exclude 'setup\.py$' \ + . + ''; + installCheck = '' + . ${commonBash} + check python -m ${mainModule} test + ''; + preBuild = '' + # remove empty directories, leftover from the src filter + find . -type d -empty -delete + # initialize remaining dirs as python modules + find . -type d -exec touch {}/__init__.py \; + # generate a minimal setup.py + cat > setup.py << EOF + from setuptools import find_packages, setup + setup( + name="${name}", + entry_points={"console_scripts":["${name} = ${mainModule}:main"]}, + version="0.0.0", + url="git://simatime.com/omni.git", + author="dev", + author_email="dev@simatime.com", + description="nil", + packages=find_packages(), + install_requires=[], + ) + EOF + ''; + pythonImportsCheck = [ mainModule ]; # sanity check + }; + }; + in builders.${target.builder}; + # the bild caller gives us the Analysis type, which is a hashmap, but i need to + # return a single drv, so just take the first one for now. ideally i would only + # pass Target, one at a time, (perhaps parallelized in haskell land) and then i + # wouldn't need all of this let nesting +in builtins.head (lib.attrsets.mapAttrsToList build analysis) diff --git a/Omni/Bild/CcacheWrapper.nix b/Omni/Bild/CcacheWrapper.nix new file mode 100644 index 0000000..78e5a08 --- /dev/null +++ b/Omni/Bild/CcacheWrapper.nix @@ -0,0 +1,57 @@ +self: super: + +let + # this should come from config.programs.ccache.cacheDir but I can't figure out + # how to access that from a nixpkgs overlay, so just hardcode the default + ccacheDir = "/var/cache/ccache"; + + # https://github.com/NixOS/nixpkgs/pull/216363#issuecomment-1430356886 + fixwebkit = pkg: + self.useCcacheStdenv (pkg.overrideAttrs (attrs: rec { + preConfigure = attrs.preConfigure + '' + # not sure which of these works so just do them both + export NUMBER_OF_PROCESSORS=$NIX_BUILD_CORES + ninjaFlagsArray+=("-l$NIX_BUILD_CORES") + ''; + })); +in { + ccacheWrapper = super.ccacheWrapper.override { + extraConfig = '' + export CCACHE_COMPRESS=1 + export CCACHE_DIR="${ccacheDir}" + export CCACHE_UMASK=007 + if [ ! -d "$CCACHE_DIR" ] + then + echo "=====" + echo "Directory '$CCACHE_DIR' does not exist" + echo "Please create it with:" + echo " sudo mkdir -m0770 '$CCACHE_DIR'" + echo " sudo chown root:nixbld '$CCACHE_DIR'" + echo "=====" + exit 1 + fi + if [ ! -w "$CCACHE_DIR" ] + then + echo "=====" + echo "Directory '$CCACHE_DIR' is not accessible for user $(whoami)" + echo "Please verify its access permissions" + echo "=====" + exit 1 + fi + ''; + }; + + useCcacheStdenv = pkg: pkg.override { stdenv = super.ccacheStdenv; }; + + cudann = self.useCcacheStdenv super.cudann; + llvm = self.useCcacheStdenv super.llvm; + magma = self.useCcacheStdenv super.magma; + nvcc = self.useCcacheStdenv super.nvcc; + onnx = self.useCcacheStdenv super.onnx; + onnxruntime = self.useCcacheStdenv super.onnxruntime; + webkit = fixwebkit super.webkit; + webkitgtk = fixwebkit super.webkitgtk; + webkitgtk_4_1 = fixwebkit super.webkitgtk_4_1; + webkitgtk_5_0 = fixwebkit super.webkitgtk_5_0; + webkitgtk_6_0 = fixwebkit super.webkitgtk_6_0; +} diff --git a/Omni/Bild/Constants.nix b/Omni/Bild/Constants.nix new file mode 100644 index 0000000..20c992e --- /dev/null +++ b/Omni/Bild/Constants.nix @@ -0,0 +1 @@ +{ ghcCompiler = "ghc948"; } diff --git a/Omni/Bild/Deps.hs b/Omni/Bild/Deps.hs new file mode 100644 index 0000000..c2fe53f --- /dev/null +++ b/Omni/Bild/Deps.hs @@ -0,0 +1,694 @@ +{-# LANGUAGE DerivingStrategies #-} +{-# LANGUAGE GeneralizedNewtypeDeriving #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TupleSections #-} +{-# LANGUAGE ViewPatterns #-} + +-- | A specific-purpose dependency manager. +-- +-- : out deps +module Omni.Bild.Deps where + +import Alpha hiding (map, packageName, str, tshow) +import Data.Aeson ((.=)) +import qualified Data.Aeson as Aeson +import qualified Data.Aeson.Key as K +import qualified Data.Aeson.KeyMap as KM +import qualified Data.ByteString as B +import qualified Data.ByteString.Char8 as B8 +import qualified Data.HashMap.Strict as HMS +import Data.HashMap.Strict.Extended +import qualified Data.Text as T +import Data.Text.Extended +import GHC.Show +import qualified Network.HTTP.Simple as HTTP +import Niv.Cmd (Cmd, description, extraLogs, parseCmdShortcut, parsePackageSpec, updateCmd) +import Niv.Git.Cmd +import Niv.GitHub.Cmd +import Niv.Local.Cmd +import Niv.Logger +import Niv.Sources +import Niv.Update +import qualified Options.Applicative as Opts +import qualified Options.Applicative.Help.Pretty as Opts +import qualified System.Directory as Dir +import System.Environment (getEnv) +import System.FilePath (takeDirectory, ()) +import UnliftIO +import Prelude + +newtype NIO a = NIO {runNIO :: ReaderT FindSourcesJson IO a} + deriving (Functor, Applicative, Monad, MonadIO, MonadReader FindSourcesJson) + +instance MonadUnliftIO NIO where + withRunInIO = wrappedWithRunInIO NIO runNIO + +getFindSourcesJson :: NIO FindSourcesJson +-- getFindSourcesJson = ask +getFindSourcesJson = do + root <- li <| getEnv "CODEROOT" + pure <| AtPath <| root "Omni/Bild/Sources.json" + +li :: (MonadIO io) => IO a -> io a +li = liftIO + +main :: IO () +main = + getArgs +> \case + ["test"] -> pure () + args -> cli args + +cli :: [String] -> IO () +cli args = do + ((fsj, colors), nio) <- + pure args +> Opts.handleParseResult <. execParserPure' Opts.defaultPrefs opts + setColors colors + runReaderT (runNIO nio) fsj + where + execParserPure' pprefs pinfo [] = + Opts.Failure + <| Opts.parserFailure pprefs pinfo (Opts.ShowHelpText Nothing) mempty + execParserPure' pprefs pinfo args_ = Opts.execParserPure pprefs pinfo args_ + opts = Opts.info ((,) parseColors) <*> (parseCommand <**> Opts.helper)) <| mconcat desc + desc = + [ Opts.fullDesc, + Opts.headerDoc + <| Just + <| "deps - specific-purpose dependency manager" + ] + parseFindSourcesJson = + AtPath + Opts.short 's' + <> Opts.metavar "FILE" + <> Opts.help "Use FILE instead of Omni/Bild/Sources.json" + ) + <|> pure Auto + parseColors = + (\case True -> Never; False -> Always) + Opts.help "Don't use colors in output" + ) + +parseCommand :: Opts.Parser (NIO ()) +parseCommand = + Opts.subparser + ( Opts.command "init" parseCmdInit + <> Opts.command "add" parseCmdAdd + <> Opts.command "show" parseCmdShow + <> Opts.command "update" parseCmdUpdate + <> Opts.command "modify" parseCmdModify + <> Opts.command "drop" parseCmdDrop + ) + +parsePackageName :: Opts.Parser PackageName +parsePackageName = + PackageName + parsePackageSpec githubCmd + +------------------------------------------------------------------------------- +-- INIT +------------------------------------------------------------------------------- + +-- | Whether or not to fetch nixpkgs +data FetchNixpkgs + = NoNixpkgs + | NixpkgsFast -- Pull latest known nixpkgs + | NixpkgsCustom T.Text Nixpkgs -- branch, nixpkgs + deriving (Show) + +data Nixpkgs = Nixpkgs T.Text T.Text -- owner, repo + +instance Show Nixpkgs where + show (Nixpkgs o r) = T.unpack o <> "/" <> T.unpack r + +parseCmdInit :: Opts.ParserInfo (NIO ()) +parseCmdInit = Opts.info (cmdInit Opts.helper) <| mconcat desc + where + desc = + [ Opts.fullDesc, + Opts.progDesc + "Initialize a Nix project. Existing files won't be modified." + ] + +parseNixpkgs :: Opts.Parser FetchNixpkgs +parseNixpkgs = parseNixpkgsFast <|> parseNixpkgsLatest <|> parseNixpkgsCustom <|> parseNoNixpkgs <|> pure NixpkgsFast + where + parseNixpkgsFast = + Opts.flag' + NixpkgsFast + ( Opts.long "fast" + <> Opts.help "Use the latest nixpkgs cached at 'https://github.com/nmattia/niv/blob/master/data/nixpkgs.json'. This is the default." + ) + parseNixpkgsLatest = + Opts.flag' + (NixpkgsCustom "master" (Nixpkgs "NixOS" "nixpkgs")) + ( Opts.long "latest" + <> Opts.help "Pull the latest unstable nixpkgs from NixOS/nixpkgs." + ) + parseNixpkgsCustom = + flip NixpkgsCustom + Opts.showDefault + <> Opts.help "Use a custom nixpkgs repository from GitHub." + <> Opts.metavar "OWNER/REPO" + ) + <*> Opts.strOption + ( Opts.long "nixpkgs-branch" + <> Opts.short 'b' + <> Opts.help "The nixpkgs branch when using --nixpkgs ...." + <> Opts.showDefault + ) + parseNoNixpkgs = + Opts.flag' + NoNixpkgs + ( Opts.long "no-nixpkgs" + <> Opts.help "Don't add a nixpkgs entry to Sources.json." + ) + customNixpkgsReader = + Opts.maybeReader <| \(T.pack -> repo) -> case T.splitOn "/" repo of + [owner, reponame] -> Just (Nixpkgs owner reponame) + _ -> Nothing + +cmdInit :: FetchNixpkgs -> NIO () +cmdInit nixpkgs = do + job "Initializing" <| do + fsj <- getFindSourcesJson + -- Writes all the default files + -- a path, a "create" function and an update function for each file. + forM_ + [ ( pathNixSourcesNix, + (`createFile` initNixSourcesNixContent), + \path content -> do + if shouldUpdateNixSourcesNix content + then do + say "Updating sources.nix" + li <| B.writeFile path initNixSourcesNixContent + else say "Not updating sources.nix" + ), + ( pathNixSourcesJson fsj, + \path -> do + createFile path initNixSourcesJsonContent + + -- Import nixpkgs, if necessary + initNixpkgs nixpkgs, + \path _content -> dontCreateFile path + ) + ] + <| \(path, onCreate, onUpdate) -> do + exists <- li <| Dir.doesFileExist path + if exists then li (B.readFile path) +> onUpdate path else onCreate path + case fsj of + Auto -> pure () + AtPath fp -> + tsay + <| T.unlines + [ T.unwords + [ tbold <| tblue "INFO:", + "You are using a custom path for sources.json." + ], + " You need to configure the sources.nix to use " <> tbold (T.pack fp) <> ":", + tbold " import sources.nix { sourcesFile = PATH ; }; ", + T.unwords + [ " where", + tbold "PATH", + "is the relative path from sources.nix to", + tbold (T.pack fp) <> "." + ] + ] + where + createFile :: FilePath -> B.ByteString -> NIO () + createFile path content = + li <| do + let dir = takeDirectory path + Dir.createDirectoryIfMissing True dir + say <| "Creating " <> path + B.writeFile path content + dontCreateFile :: FilePath -> NIO () + dontCreateFile path = say <| "Not creating " <> path + +initNixpkgs :: FetchNixpkgs -> NIO () +initNixpkgs nixpkgs = + case nixpkgs of + NoNixpkgs -> say "Not importing 'nixpkgs'." + NixpkgsFast -> do + say "Using known 'nixpkgs' ..." + packageSpec <- HTTP.getResponseBody do + say "Importing 'nixpkgs' ..." + let (owner, repo) = case nixpkgs' of + Nixpkgs o r -> (o, r) + cmdAdd + githubCmd + (PackageName "nixpkgs") + ( specToFreeAttrs + <| PackageSpec + <| KM.fromList + [ "owner" .= owner, + "repo" .= repo, + "branch" .= branch + ] + ) + +------------------------------------------------------------------------------- +-- ADD +------------------------------------------------------------------------------- + +parseCmdAdd :: Opts.ParserInfo (NIO ()) +parseCmdAdd = + Opts.info + ((parseCommands <|> parseShortcuts) <**> Opts.helper) + <| description githubCmd + where + -- XXX: this should parse many shortcuts (github, git). Right now we only + -- parse GitHub because the git interface is still experimental. note to + -- implementer: it'll be tricky to have the correct arguments show up + -- without repeating "PACKAGE PACKAGE PACKAGE" for every package type. + parseShortcuts = parseShortcut githubCmd + parseShortcut cmd = uncurry (cmdAdd cmd) Opts.helper) (description gitCmd) + parseCmdAddLocal = + Opts.info (parseCmd localCmd <**> Opts.helper) (description localCmd) + parseCmdAddGitHub = + Opts.info (parseCmd githubCmd <**> Opts.helper) (description githubCmd) + parseCommands = + Opts.subparser + ( Opts.hidden + <> Opts.commandGroup "Experimental commands:" + <> Opts.command "git" parseCmdAddGit + <> Opts.command "github" parseCmdAddGitHub + <> Opts.command "local" parseCmdAddLocal + ) + +-- | only used in shortcuts (niv add foo/bar ...) because PACKAGE is NOT +-- optional +parseShortcutArgs :: Cmd -> Opts.Parser (PackageName, Attrs) +parseShortcutArgs cmd = collapse parsePackageSpec cmd + where + collapse specAndName pspec = (pname, specToLockedAttrs <| pspec <> baseSpec) + where + (pname, baseSpec) = case specAndName of + ((_, spec), Just pname') -> (pname', PackageSpec spec) + ((pname', spec), Nothing) -> (pname', PackageSpec spec) + parseNameAndShortcut = + (,) + optName + optName = + Opts.optional + <| PackageName + Opts.short 'n' + <> Opts.metavar "NAME" + <> Opts.help "Set the package name to " + ) + +-- | only used in command (niv add ...) because PACKAGE is optional +parseCmdArgs :: Cmd -> Opts.Parser (PackageName, Attrs) +parseCmdArgs cmd = collapse parsePackageSpec cmd + where + collapse specAndName pspec = (pname, specToLockedAttrs <| pspec <> baseSpec) + where + (pname, baseSpec) = case specAndName of + (Just (_, spec), Just pname') -> (pname', PackageSpec spec) + (Just (pname', spec), Nothing) -> (pname', PackageSpec spec) + (Nothing, Just pname') -> (pname', PackageSpec KM.empty) + (Nothing, Nothing) -> (PackageName "unnamed", PackageSpec KM.empty) + parseNameAndShortcut = + (,) + optName + optName = + Opts.optional + <| PackageName + Opts.short 'n' + <> Opts.metavar "NAME" + <> Opts.help "Set the package name to " + ) + +cmdAdd :: Cmd -> PackageName -> Attrs -> NIO () +cmdAdd cmd packageName attrs = do + job ("Adding package " <> T.unpack (unPackageName packageName)) <| do + fsj <- getFindSourcesJson + sources <- unSources li (abortUpdateFailed [(packageName, e)]) + Right finalSpec -> do + say <| "Writing new sources file" + li + <| setSources fsj + <| Sources + <| HMS.insert packageName finalSpec sources + +------------------------------------------------------------------------------- +-- SHOW +------------------------------------------------------------------------------- + +parseCmdShow :: Opts.ParserInfo (NIO ()) +parseCmdShow = + Opts.info + ((cmdShow Opts.helper) + <| Opts.progDesc "Show information about a dependency in human-readable format" + +cmdShow :: Maybe PackageName -> NIO () +cmdShow = \case + Just packageName -> do + fsj <- getFindSourcesJson + sources <- unSources showPackage packageName pspec + Nothing -> li <| abortCannotShowNoSuchPackage packageName + Nothing -> do + fsj <- getFindSourcesJson + sources <- unSources PackageName -> PackageSpec -> io () +showPackage (PackageName pname) (PackageSpec spec) = do + tsay <| tbold pname + forM_ (KM.toList spec) <| \(attrName, attrValValue) -> do + let attrValue = case attrValValue of + Aeson.String str -> str + _ -> tfaint "" + tsay <| " " <> K.toText attrName <> ": " <> attrValue + +------------------------------------------------------------------------------- +-- UPDATE +------------------------------------------------------------------------------- + +parseCmdUpdate :: Opts.ParserInfo (NIO ()) +parseCmdUpdate = + Opts.info + ((cmdUpdate Opts.helper) + <| mconcat desc + where + desc = + [ Opts.fullDesc, + Opts.progDesc "Update dependencies", + Opts.headerDoc + <| Just + <| Opts.nest 2 + <| Opts.vcat + [ "Examples:", + Opts.fill 30 "deps update" Opts.<+> "# update all packages", + Opts.fill 30 "deps update nixpkgs" Opts.<+> "# update nixpkgs", + Opts.fill 30 "deps update my-package -v beta-0.2" Opts.<+> "# update my-package to version \"beta-0.2\"" + ] + ] + +specToFreeAttrs :: PackageSpec -> Attrs +specToFreeAttrs = KM.toHashMapText <. fmap (Free,) <. unPackageSpec + +specToLockedAttrs :: PackageSpec -> Attrs +specToLockedAttrs = KM.toHashMapText <. fmap (Locked,) <. unPackageSpec + +cmdUpdate :: Maybe (PackageName, PackageSpec) -> NIO () +cmdUpdate = \case + Just (packageName, cliSpec) -> + job ("Update " <> T.unpack (unPackageName packageName)) <| do + fsj <- getFindSourcesJson + sources <- unSources do + -- lookup the "type" to find a Cmd to run, defaulting to legacy + -- github + let cmd = case KM.lookup "type" (unPackageSpec defaultSpec) of + Just "git" -> gitCmd + Just "local" -> localCmd + _ -> githubCmd + spec = specToLockedAttrs cliSpec <> specToFreeAttrs defaultSpec + fmap attrsToSpec li <| abortCannotUpdateNoSuchPackage packageName + case eFinalSpec of + Left e -> li <| abortUpdateFailed [(packageName, e)] + Right finalSpec -> + li + <| setSources fsj + <| Sources + <| HMS.insert packageName finalSpec sources + Nothing -> + job "Updating all packages" <| do + fsj <- getFindSourcesJson + sources <- unSources do + tsay <| "Package: " <> unPackageName packageName + let initialSpec = specToFreeAttrs defaultSpec + -- lookup the "type" to find a Cmd to run, defaulting to legacy + -- github + let cmd = case KM.lookup "type" (unPackageSpec defaultSpec) of + Just "git" -> gitCmd + Just "local" -> localCmd + _ -> githubCmd + fmap attrsToSpec Cmd -> IO (Either SomeException Attrs) +doUpdate attrs cmd = do + forM_ (extraLogs cmd attrs) <| tsay + tryEvalUpdate attrs (updateCmd cmd) + +partitionEithersHMS :: + (Eq k, Hashable k) => + HMS.HashMap k (Either a b) -> + (HMS.HashMap k a, HMS.HashMap k b) +partitionEithersHMS = + flip HMS.foldlWithKey' (HMS.empty, HMS.empty) <| \(ls, rs) k -> \case + Left l -> (HMS.insert k l ls, rs) + Right r -> (ls, HMS.insert k r rs) + +------------------------------------------------------------------------------- +-- MODIFY +------------------------------------------------------------------------------- + +parseCmdModify :: Opts.ParserInfo (NIO ()) +parseCmdModify = + Opts.info + ((cmdModify optName <*> parsePackageSpec githubCmd) <**> Opts.helper) + <| mconcat desc + where + desc = + [ Opts.fullDesc, + Opts.progDesc "Modify dependency attributes without performing an update", + Opts.headerDoc + <| Just + <| Opts.vcat + [ "Examples:", + "", + " niv modify nixpkgs -v beta-0.2", + " niv modify nixpkgs -a branch=nixpkgs-unstable" + ] + ] + optName = + Opts.optional + <| PackageName + Opts.short 'n' + <> Opts.metavar "NAME" + <> Opts.help "Set the package name to " + ) + +cmdModify :: PackageName -> Maybe PackageName -> PackageSpec -> NIO () +cmdModify packageName mNewName cliSpec = do + tsay <| "Modifying package: " <> unPackageName packageName + fsj <- getFindSourcesJson + sources <- unSources pure <| attrsToSpec (specToLockedAttrs cliSpec <> specToFreeAttrs defaultSpec) + Nothing -> li <| abortCannotModifyNoSuchPackage packageName + case mNewName of + Just newName -> do + when (HMS.member newName sources) + <| li + <| abortCannotAddPackageExists newName + li <| setSources fsj <| Sources <| HMS.insert newName finalSpec <| HMS.delete packageName sources + Nothing -> + li <| setSources fsj <| Sources <| HMS.insert packageName finalSpec sources + +------------------------------------------------------------------------------- +-- DROP +------------------------------------------------------------------------------- + +parseCmdDrop :: Opts.ParserInfo (NIO ()) +parseCmdDrop = + Opts.info + ( (cmdDrop parseDropAttributes) + <**> Opts.helper + ) + <| mconcat desc + where + desc = + [ Opts.fullDesc, + Opts.progDesc "Drop dependency", + Opts.headerDoc + <| Just + <| Opts.vcat + [ "Examples:", + "", + " niv drop jq", + " niv drop my-package version" + ] + ] + parseDropAttributes :: Opts.Parser [T.Text] + parseDropAttributes = + many + <| Opts.argument Opts.str (Opts.metavar "ATTRIBUTE") + +cmdDrop :: PackageName -> [T.Text] -> NIO () +cmdDrop packageName = \case + [] -> do + tsay <| "Dropping package: " <> unPackageName packageName + fsj <- getFindSourcesJson + sources <- unSources do + tsay <| "Dropping attributes: " <> T.intercalate " " attrs + tsay <| "In package: " <> unPackageName packageName + fsj <- getFindSourcesJson + sources <- unSources + li <| abortCannotAttributesDropNoSuchPackage packageName + Just (PackageSpec packageSpec) -> + pure + <| PackageSpec + <| KM.mapMaybeWithKey + (\k v -> if K.toText k `elem` attrs then Nothing else Just v) + packageSpec + li + <| setSources fsj + <| Sources + <| HMS.insert packageName packageSpec sources + +------------------------------------------------------------------------------- +-- Files and their content +------------------------------------------------------------------------------- + +-- | Checks if content is different than default and if it does /not/ contain +-- a comment line with @niv: no_update@ +shouldUpdateNixSourcesNix :: B.ByteString -> Bool +shouldUpdateNixSourcesNix content = + content /= initNixSourcesNixContent + && not (any lineForbids (B8.lines content)) + where + lineForbids :: B8.ByteString -> Bool + lineForbids str = + case B8.uncons (B8.dropWhile isSpace str) of + Just ('#', rest) -> case B8.stripPrefix "niv:" (B8.dropWhile isSpace rest) of + Just rest' -> case B8.stripPrefix "no_update" (B8.dropWhile isSpace rest') of + Just {} -> True + _ -> False + _ -> False + _ -> False + +------------------------------------------------------------------------------- +-- Abort +------------------------------------------------------------------------------- + +abortCannotAddPackageExists :: PackageName -> IO a +abortCannotAddPackageExists (PackageName n) = + abort + <| T.unlines + [ "Cannot add package " <> n <> ".", + "The package already exists. Use", + " niv drop " <> n, + "and then re-add the package. Alternatively use", + " niv update " <> n <> " --attribute foo=bar", + "to update the package's attributes." + ] + +abortCannotUpdateNoSuchPackage :: PackageName -> IO a +abortCannotUpdateNoSuchPackage (PackageName n) = + abort + <| T.unlines + [ "Cannot update package " <> n <> ".", + "The package doesn't exist. Use", + " niv add " <> n, + "to add the package." + ] + +abortCannotModifyNoSuchPackage :: PackageName -> IO a +abortCannotModifyNoSuchPackage (PackageName n) = + abort + <| T.unlines + [ "Cannot modify package " <> n <> ".", + "The package doesn't exist. Use", + " niv add " <> n, + "to add the package." + ] + +abortCannotDropNoSuchPackage :: PackageName -> IO a +abortCannotDropNoSuchPackage (PackageName n) = + abort + <| T.unlines + [ "Cannot drop package " <> n <> ".", + "The package doesn't exist." + ] + +abortCannotShowNoSuchPackage :: PackageName -> IO a +abortCannotShowNoSuchPackage (PackageName n) = + abort + <| T.unlines + [ "Cannot show package " <> n <> ".", + "The package doesn't exist." + ] + +abortCannotAttributesDropNoSuchPackage :: PackageName -> IO a +abortCannotAttributesDropNoSuchPackage (PackageName n) = + abort + <| T.unlines + [ "Cannot drop attributes of package " <> n <> ".", + "The package doesn't exist." + ] + +abortUpdateFailed :: [(PackageName, SomeException)] -> IO a +abortUpdateFailed errs = + abort + <| T.unlines + <| ["One or more packages failed to update:"] + <> map + ( \(PackageName pname, e) -> + pname <> ": " <> tshow e + ) + errs diff --git a/Omni/Bild/Deps.nix b/Omni/Bild/Deps.nix new file mode 100644 index 0000000..9ba0b31 --- /dev/null +++ b/Omni/Bild/Deps.nix @@ -0,0 +1,45 @@ +_self: super: + +{ + # Needs upgrading for guile 3 + # inspekt3d = super.callPackage ./Deps/inspekt3d.nix {}; + + guix = super.pkgs.stdenv.mkDerivation rec { + pname = "guix"; + name = "${pname}-${version}"; + version = super.sources.guix.version; + src = super.sources.guix; + buildInputs = with super.pkgs; [ + guile + # guile-gcrypt + # guile-sql + # guile-zlib + # guile-lzlib + # guile-avahi + # guile-git + # guile-json + gnutls + gnumake + sqlite + libgcrypt + gcc + ]; + }; + + llm = super.overrideSrc super.llm super.sources.llm; + + nostr-rs-relay = super.callPackage ./Deps/nostr-rs-relay.nix { }; + + ollama = super.ollama.override { acceleration = "cuda"; }; + + # https://github.com/NixOS/nixpkgs/issues/317147#issuecomment-2147343125 + radicale = super.radicale.overrideAttrs (_old: rec { + version = "3.2.0"; + src = super.fetchFromGitHub { + owner = "Kozea"; + repo = "Radicale"; + rev = "v${version}"; + hash = "sha256-RxC8VOfdTXJZiAroDHTKjJqGWu65Z5uyb4WK1LOqubQ="; + }; + }); +} diff --git a/Omni/Bild/Deps/C.nix b/Omni/Bild/Deps/C.nix new file mode 100644 index 0000000..3f670cd --- /dev/null +++ b/Omni/Bild/Deps/C.nix @@ -0,0 +1 @@ +[ "libsodium" ] diff --git a/Omni/Bild/Deps/Haskell.nix b/Omni/Bild/Deps/Haskell.nix new file mode 100644 index 0000000..04f3a74 --- /dev/null +++ b/Omni/Bild/Deps/Haskell.nix @@ -0,0 +1,72 @@ +# This is the global set of Haskell packages which gets deployed to Hoogle, and +# is available for selecting. + +[ + "MonadRandom" + "QuickCheck" + "SafeSemaphore" + "acid-state" + "aeson" + "async" + "base" + "bytestring" + "clay" + "cmark" + "cmark-lucid" + "conduit" + "conduit-extra" + "config-ini" + "containers" + "directory" + "docopt" + "envy" + "fast-logger" + "filepath" + "github" + "haskeline" + "hostname" + "http-types" + "ixset" + "katip" + "lucid" + "monad-logger" + "mtl" + "neat-interpolation" + "network-uri" + "niv" + "optparse-simple" + "parsec" + "process" + "protolude" + "quickcheck-instances" + "rainbow" + "random" + "regex-applicative" + "req" + "safecopy" + "saltine" + "servant" + "servant-auth" + "servant-auth-server" + "servant-lucid" + "servant-server" + "split" + "stm" + "tasty" + "tasty-hunit" + "tasty-quickcheck" + "text" + "time" + "transformers" + "unagi-chan" + "unix" + "unordered-containers" + "uuid" + "vector" + "wai" + "wai-app-static" + "wai-extra" + "wai-middleware-metrics" + "warp" + "x509" +] diff --git a/Omni/Bild/Deps/Python.nix b/Omni/Bild/Deps/Python.nix new file mode 100644 index 0000000..b0b2465 --- /dev/null +++ b/Omni/Bild/Deps/Python.nix @@ -0,0 +1 @@ +[ "cryptography" "llm" "mypy" "nltk" "slixmpp" ] diff --git a/Omni/Bild/Deps/accelerate.nix b/Omni/Bild/Deps/accelerate.nix new file mode 100644 index 0000000..be1d2fd --- /dev/null +++ b/Omni/Bild/Deps/accelerate.nix @@ -0,0 +1,16 @@ +{ fetchFromGitHub, buildPythonPackage, numpy, packaging, psutil, pyyaml, torch +}: + +buildPythonPackage rec { + name = "accelerate"; + version = "0.15.0"; + propagatedBuildInputs = [ numpy packaging psutil pyyaml torch ]; + doCheck = false; + src = fetchFromGitHub { + owner = "huggingface"; + repo = "accelerate"; + rev = "v${version}"; + sha256 = "sha256-agfbOaa+Nm10HZkd2Y7zR3R37n+vLNsxCyxZax6O3Lo="; + }; +} + diff --git a/Omni/Bild/Deps/bitsandbytes.nix b/Omni/Bild/Deps/bitsandbytes.nix new file mode 100644 index 0000000..eb32aac --- /dev/null +++ b/Omni/Bild/Deps/bitsandbytes.nix @@ -0,0 +1,86 @@ +{ lib, buildPythonPackage, fetchFromGitHub, python, pythonOlder, pytestCheckHook +, setuptools, torch, einops, lion-pytorch, scipy, symlinkJoin }: + +let + pname = "bitsandbytes"; + version = "0.38.0"; + + inherit (torch) cudaPackages cudaSupport; + inherit (cudaPackages) cudaVersion; + + # NOTE: torchvision doesn't use cudnn; torch does! + # For this reason it is not included. + cuda-common-redist = with cudaPackages; [ + cuda_cccl # + libcublas # cublas_v2.h + libcurand + libcusolver # cusolverDn.h + libcusparse # cusparse.h + ]; + + cuda-native-redist = symlinkJoin { + name = "cuda-native-redist-${cudaVersion}"; + paths = with cudaPackages; + [ + cuda_cudart # cuda_runtime.h cuda_runtime_api.h + cuda_nvcc + ] ++ cuda-common-redist; + }; + + cuda-redist = symlinkJoin { + name = "cuda-redist-${cudaVersion}"; + paths = cuda-common-redist; + }; + +in buildPythonPackage { + inherit pname version; + format = "pyproject"; + + disabled = pythonOlder "3.7"; + + src = fetchFromGitHub { + owner = "TimDettmers"; + repo = pname; + rev = "refs/tags/${version}"; + hash = "sha256-gGlbzTDvZNo4MhcYzLvWuB2ec7q+Qt5/LtTbJ0Rc+Kk="; + }; + + postPatch = '' + substituteInPlace Makefile --replace "/usr/bin/g++" "g++" --replace "lib64" "lib" + substituteInPlace bitsandbytes/cuda_setup/main.py \ + --replace "binary_path = package_dir / binary_name" \ + "binary_path = Path('$out/${python.sitePackages}/${pname}')/binary_name" + '' + lib.optionalString torch.cudaSupport '' + substituteInPlace bitsandbytes/cuda_setup/main.py \ + --replace "/usr/local/cuda/lib64" "${cuda-native-redist}/lib" + ''; + + CUDA_HOME = "${cuda-native-redist}"; + + preBuild = if torch.cudaSupport then + with torch.cudaPackages; + let + cudaVersion = lib.concatStrings + (lib.splitVersion torch.cudaPackages.cudaMajorMinorVersion); + in "make CUDA_VERSION=${cudaVersion} cuda${cudaMajorVersion}x" + else + "make CUDA_VERSION=CPU cpuonly"; + + nativeBuildInputs = [ setuptools ] + ++ lib.optionals torch.cudaSupport [ cuda-native-redist ]; + buildInputs = lib.optionals torch.cudaSupport [ cuda-redist ]; + + propagatedBuildInputs = [ torch ]; + + doCheck = false; # tests require CUDA and also GPU access + nativeCheckInputs = [ pytestCheckHook einops lion-pytorch scipy ]; + + pythonImportsCheck = [ "bitsandbytes" ]; + + meta = with lib; { + homepage = "https://github.com/TimDettmers/bitsandbytes"; + description = "8-bit CUDA functions for PyTorch"; + license = licenses.mit; + maintainers = with maintainers; [ bcdarwin ]; + }; +} diff --git a/Omni/Bild/Deps/guile-opengl.nix b/Omni/Bild/Deps/guile-opengl.nix new file mode 100644 index 0000000..af01082 --- /dev/null +++ b/Omni/Bild/Deps/guile-opengl.nix @@ -0,0 +1,32 @@ +{ stdenv, lib, fetchurl, pkg-config, guile, libGL, libGLU, freeglut }: + +let + name = "guile-opengl-${version}"; + version = "0.1.0"; +in stdenv.mkDerivation { + inherit name; + + src = fetchurl { + url = "mirror://gnu/guile-opengl/${name}.tar.gz"; + sha256 = "13qfx4xh8baryxqrv986l848ygd0piqwm6s2s90pxk9c0m9vklim"; + }; + + patchPhase = '' + substituteInPlace glx/runtime.scm \ + --replace '(dynamic-link "libGL")' '(dynamic-link "${libGL}/lib/libGL.so")' + substituteInPlace glu/runtime.scm \ + --replace '(dynamic-link "libGLU")' '(dynamic-link "${libGLU}/lib/libGLU.so")' + substituteInPlace glut/runtime.scm \ + --replace '(dynamic-link "libglut")' '(dynamic-link "${freeglut}/lib/libglut.so")' + ''; + + nativeBuildInputs = [ pkg-config guile libGL libGLU freeglut ]; + + meta = with lib; { + description = "Guile bindings for the OpenGL graphics API"; + homepage = "https://www.gnu.org/software/guile-opengl/"; + license = licenses.gpl3Plus; + maintainers = with maintainers; [ vyp ]; + platforms = platforms.all; + }; +} diff --git a/Omni/Bild/Deps/inspekt3d.nix b/Omni/Bild/Deps/inspekt3d.nix new file mode 100644 index 0000000..3146350 --- /dev/null +++ b/Omni/Bild/Deps/inspekt3d.nix @@ -0,0 +1,30 @@ +{ stdenv, lib, autoreconfHook, pkg-config, guile, guile-opengl, mesa +, glibcLocales, libfive, sources }: + +stdenv.mkDerivation { + name = "inspekt3d-unstable"; + + src = sources.inspekt3d; + version = "unstable-2018-10-17"; + + nativeBuildInputs = [ pkg-config autoreconfHook ]; + buildInputs = [ guile glibcLocales mesa ]; + propagatedBuildInputs = [ guile-opengl libfive ]; + + preBuild = '' + substituteInPlace inspekt3d/library.scm \ + --replace '"libfive-guile"' '"${libfive}/lib/libfive-guile.so"' \ + --replace '"libfive"' '"${libfive}/lib/libfive.so"' + ''; + + GUILE_AUTO_COMPILE = 0; + preConfigure = "./bootstrap"; + + meta = with lib; { + description = "Lightweight 3D viewer for Libfive written in Guile Scheme"; + homepage = "https://sr.ht/~morgansmith/inspekt3d"; + license = licenses.gpl3; + maintainers = with maintainers; [ bsima ]; + platforms = platforms.all; + }; +} diff --git a/Omni/Bild/Deps/interegular.nix b/Omni/Bild/Deps/interegular.nix new file mode 100644 index 0000000..24065d8 --- /dev/null +++ b/Omni/Bild/Deps/interegular.nix @@ -0,0 +1,21 @@ +{ lib, sources, buildPythonPackage }: + +buildPythonPackage rec { + pname = "interegular"; + version = sources.interegular.rev; + format = "setuptools"; + + src = sources.interegular; + + propagatedBuildInputs = [ ]; + + doCheck = false; # no tests currently + pythonImportsCheck = [ "interegular" ]; + + meta = with lib; { + description = "Allows to check regexes for overlaps."; + homepage = "https://github.com/MegaIng/interegular"; + license = licenses.mit; + maintainers = with maintainers; [ bsima ]; + }; +} diff --git a/Omni/Bild/Deps/lion-pytorch.nix b/Omni/Bild/Deps/lion-pytorch.nix new file mode 100644 index 0000000..7b06e78 --- /dev/null +++ b/Omni/Bild/Deps/lion-pytorch.nix @@ -0,0 +1,27 @@ +{ lib, buildPythonPackage, pythonOlder, fetchFromGitHub, torch }: + +buildPythonPackage rec { + pname = "lion-pytorch"; + version = "0.1.2"; + format = "setuptools"; + disabled = pythonOlder "3.6"; + + src = fetchFromGitHub { + owner = "lucidrains"; + repo = "lion-pytorch"; + rev = "refs/tags/${version}"; + hash = "sha256-9hdpRJvCpv3PeC7f0IXpHt6i+e6LiT0QUl5jeDGelQE="; + }; + + propagatedBuildInputs = [ torch ]; + + pythonImportsCheck = [ "lion_pytorch" ]; + doCheck = false; # no tests currently + + meta = with lib; { + description = "Optimizer tuned by Google Brain using genetic algorithms"; + homepage = "https://github.com/lucidrains/lion-pytorch"; + license = licenses.mit; + maintainers = with maintainers; [ bcdarwin ]; + }; +} diff --git a/Omni/Bild/Deps/llm-ollama.nix b/Omni/Bild/Deps/llm-ollama.nix new file mode 100644 index 0000000..e2b6a66 --- /dev/null +++ b/Omni/Bild/Deps/llm-ollama.nix @@ -0,0 +1,40 @@ +{ buildPythonPackage, fetchFromGitHub, lib, llm, ollama, pytestCheckHook +, setuptools, pythonOlder, }: +buildPythonPackage rec { + pname = "llm-ollama"; + version = "0.3.0"; + pyproject = true; + + disabled = pythonOlder "3.8"; + + src = fetchFromGitHub { + owner = "taketwo"; + repo = pname; + rev = "refs/tags/${version}"; + hash = "sha256-Ar0Ux8BNGY0i764CEk7+48J6jnndlRIIMPZ9tFpXiy4="; + }; + + nativeBuildInputs = [ setuptools ]; + + buildInputs = [ llm ollama ]; + + propagatedBuildInputs = [ ollama ]; + + disabledTests = [ + # wants to mkdir in the /homeless-shelter + "test_registered_models" + ]; + + nativeCheckInputs = [ pytestCheckHook ]; + + pythonImportsCheck = [ "llm_ollama" ]; + + meta = with lib; { + homepage = "https://github.com/taketwo/llm-ollama"; + description = + "LLM plugin providing access to local Ollama models usting HTTP API"; + changelog = "https://github.com/taketwo/llm-ollama/releases/tag/${version}"; + license = licenses.asl20; + maintainers = with maintainers; [ bsima ]; + }; +} diff --git a/Omni/Bild/Deps/llm-sentence-transformers.nix b/Omni/Bild/Deps/llm-sentence-transformers.nix new file mode 100644 index 0000000..4d63c83 --- /dev/null +++ b/Omni/Bild/Deps/llm-sentence-transformers.nix @@ -0,0 +1,42 @@ +{ buildPythonPackage, fetchFromGitHub, lib, llm, sentence-transformers +, pytestCheckHook, setuptools, pythonOlder, }: +buildPythonPackage rec { + pname = "llm-sentence-transformers"; + version = "0.2"; + pyproject = true; + + disabled = pythonOlder "3.8"; + + src = fetchFromGitHub { + owner = "simonw"; + repo = pname; + rev = "refs/tags/${version}"; + hash = "sha256-1NlKPWekdVLrNkIMWXLCRWn54RlAEuEDWMCDnQHNkBc="; + }; + + nativeBuildInputs = [ setuptools ]; + + buildInputs = [ llm sentence-transformers ]; + + propagatedBuildInputs = [ sentence-transformers ]; + + # fails because of some pydantic warnings + doCheck = false; + disabledTests = [ + # wants to mkdir in the /homeless-shelter + "test_sentence_transformers" + ]; + + nativeCheckInputs = [ pytestCheckHook ]; + + pythonImportsCheck = [ "llm_sentence_transformers" ]; + + meta = with lib; { + homepage = "https://github.com/taketwo/llm-sentence-transformers"; + description = "LLM plugin for embeddings using sentence-transformers"; + changelog = + "https://github.com/taketwo/llm-sentence-transformers/releases/tag/${version}"; + license = licenses.asl20; + maintainers = with maintainers; [ bsima ]; + }; +} diff --git a/Omni/Bild/Deps/nostr-rs-relay.nix b/Omni/Bild/Deps/nostr-rs-relay.nix new file mode 100644 index 0000000..0eef13f --- /dev/null +++ b/Omni/Bild/Deps/nostr-rs-relay.nix @@ -0,0 +1,19 @@ +{ fetchFromSourcehut, rustPlatform, pkg-config, openssl }: + +rustPlatform.buildRustPackage rec { + pname = "nostr-rs-relay"; + version = "0.7.15"; + + src = fetchFromSourcehut { + owner = "~gheartsfield"; + repo = pname; + rev = version; + sha256 = "sha256-aa1uFJcpQPMVzIWpkQ2MW6LIzTnhXNQc220scbzwJ5k="; + }; + + cargoSha256 = "sha256-3593pjc4A4NsEnE/ZYsR1vSMCvw2ZJue4EIY6cFa2WA="; + + nativeBuildInputs = [ pkg-config openssl.dev ]; + + buildInputs = [ openssl.dev ]; +} diff --git a/Omni/Bild/Deps/outlines.nix b/Omni/Bild/Deps/outlines.nix new file mode 100644 index 0000000..29ef41b --- /dev/null +++ b/Omni/Bild/Deps/outlines.nix @@ -0,0 +1,34 @@ +{ lib, sources, buildPythonPackage, interegular, jinja2, lark, numpy, perscache +, pillow, pydantic, regex, scipy, tenacity, torch }: + +buildPythonPackage rec { + pname = "outlines"; + version = sources.outlines.rev; + format = "pyproject"; + + src = sources.outlines; + + propagatedBuildInputs = [ + interegular + jinja2 + lark + numpy + perscache + pillow + pydantic + regex + scipy + tenacity + torch + ]; + + doCheck = false; # no tests currently + pythonImportsCheck = [ "outlines" ]; + + meta = with lib; { + description = "Probabilistic Generative Model Programming"; + homepage = "https://github.com/normal-computing/outlines"; + license = licenses.asl20; + maintainers = with maintainers; [ bsima ]; + }; +} diff --git a/Omni/Bild/Deps/perscache.nix b/Omni/Bild/Deps/perscache.nix new file mode 100644 index 0000000..508a261 --- /dev/null +++ b/Omni/Bild/Deps/perscache.nix @@ -0,0 +1,25 @@ +{ lib, sources, buildPythonPackage, beartype, cloudpickle, icontract, pbr }: + +buildPythonPackage rec { + pname = "perscache"; + version = sources.perscache.rev; + + src = sources.perscache; + + propagatedBuildInputs = [ beartype cloudpickle icontract pbr ]; + PBR_VERSION = version; + + doCheck = false; # no tests currently + pythonImportsCheck = [ "perscache" ]; + + meta = with lib; { + description = '' + An easy to use decorator for persistent memoization: like + `functools.lrucache`, but results can be saved in any format to any + storage. + ''; + homepage = "https://github.com/leshchenko1979/perscache"; + license = licenses.mit; + maintainers = with maintainers; [ bsima ]; + }; +} diff --git a/Omni/Bild/Example.c b/Omni/Bild/Example.c new file mode 100644 index 0000000..2f4bfd4 --- /dev/null +++ b/Omni/Bild/Example.c @@ -0,0 +1,15 @@ +// : out examplesodium.exe +// : dep libsodium +// : arg -lsodium +#include + +int +main (void) +{ + if (sodium_init () < 0) + { + /* panic! the library couldn't be initialized; it is not safe to use */ + } + printf ("Omni/Bild/Example.c: Hello world!\n"); + return 0; +} diff --git a/Omni/Bild/Example.hs b/Omni/Bild/Example.hs new file mode 100644 index 0000000..025391c --- /dev/null +++ b/Omni/Bild/Example.hs @@ -0,0 +1,45 @@ +{-# LANGUAGE QuasiQuotes #-} +{-# LANGUAGE NoImplicitPrelude #-} + +-- : out example +module Omni.Bild.Example where + +-- Both internal and external language dependencies are detected automatically +-- by bild, for example here we import 'Crypto.Saltine' and list 'saltine' in +-- 'Deps/Haskell.nix' to indicate that this is the package we want made +-- available to bild, which will index the external package and record its +-- modules for lookup. +import Alpha +import qualified Crypto.Saltine as Saltine +import qualified Crypto.Saltine.Core.SecretBox as Saltine +import qualified Omni.Cli as Cli +import qualified Omni.Test as Test + +main :: IO () +main = Cli.main <| Cli.Plan help move test pure + +move :: Cli.Arguments -> IO () +move _ = putStrLn "Hello world" + +test :: Test.Tree +test = + Test.group + "Omni.Bild.Example" + [ Test.unit "can use saltine package" <| do + Saltine.sodiumInit + k <- Saltine.newKey + n <- Saltine.newNonce + let msg = "foobar" + let encryptedMsg = Saltine.secretbox k n <| str "foobar" + Just msg Test.@=? str str: + """ + Encrypt and decrypt `name`. + + Example taken from `cryptography` docs. + + Raises: + ValueError: if decryption fails + """ + key = cryptography.fernet.Fernet.generate_key() + f = cryptography.fernet.Fernet(key) + token = f.encrypt(hello(name).encode("utf-8")) + ret = f.decrypt(token).decode("utf-8") + if ret != hello(name): + msg = "en/decryption failed!" + raise ValueError(msg) + return ret + + +def hello(name: str) -> str: + """Say hello.""" + return f"Hello {name}" + + +def main() -> None: + """Entrypoint.""" + if "test" in sys.argv: + sys.stdout.write("testing success") + sys.stdout.write(cryptic_hello("world")) + + +if __name__ == "__main__": + main() diff --git a/Omni/Bild/Example.rs b/Omni/Bild/Example.rs new file mode 100644 index 0000000..ba98dda --- /dev/null +++ b/Omni/Bild/Example.rs @@ -0,0 +1,4 @@ +// : out helloworld +fn main() { + println!("Hello world!"); +} diff --git a/Omni/Bild/Functions.nix b/Omni/Bild/Functions.nix new file mode 100644 index 0000000..8b87f86 --- /dev/null +++ b/Omni/Bild/Functions.nix @@ -0,0 +1,33 @@ +_: super: { + # Given a generic `builder`, will generate an attrset for all the packages + # pinned by `deps` with `builder` applied to the package. This attrset can + # then be merged with the rest of the packages in the set as part of an + # overlay or overrides. + overridePinnedDeps = builder: + super.lib.genAttrs (builtins.attrNames super.sources) builder; + + # Modifies a derivation with our source and version, keeping super build + # rules. This will fail if build steps have changed, or if no build + # rules are available upstream. + overrideSource = depName: + if super ? "${depName}" && super.${depName} ? overrideAttrs then + super.${depName}.overrideAttrs (attrs: + attrs // rec { + version = + super.sources.${depName}.version or super.sources.${depName}.rev; + src = super.sources.${depName}; + }) + else + null; + + # Simply override the 'src' attr on a drv. This is meant to be a simpler + # alternative to 'overrideSource' above. In an overlay, use it like: + # mypkg = super.overrideSrc super.mypkg super.sources.mypkg; + overrideSrc = dep: src: + dep.overrideAttrs (attrs: + attrs // { + version = src.version or src.rev; + src = src; + }); +} + diff --git a/Omni/Bild/Haskell.nix b/Omni/Bild/Haskell.nix new file mode 100644 index 0000000..c744848 --- /dev/null +++ b/Omni/Bild/Haskell.nix @@ -0,0 +1,36 @@ +_self: super: + +let + inherit (import ./Constants.nix) ghcCompiler; + + buildCabal = sel: name: sel.callCabal2nix name super.sources.${name} { }; + +in rec { + + haskell = super.haskell // { + packages = super.haskell.packages // { + "${ghcCompiler}" = super.haskell.packages."${ghcCompiler}".override + (_old: { + overrides = with super.pkgs.haskell.lib; + sel: sup: + super.overridePinnedDeps (buildCabal sel) // { + ap-normalize = dontCheck sup.ap-normalize; + clay = doJailbreak sup.clay; + cmark = doJailbreak sup.cmark; + docopt = buildCabal sel "docopt"; + linear-generics = doJailbreak sup.linear-generics; + req = doJailbreak sup.req; + servant-auth = doJailbreak sup.servant-auth; + servant-auth-server = dontCheck sup.servant-auth-server; + shellcheck = doJailbreak sup.shellcheck; + string-qq = doJailbreak sup.string-qq; + syb-with-class = doJailbreak sup.syb-with-class; + th-abstraction = doJailbreak sup.th-abstraction; + }; + }); + }; + }; + + ormolu = super.haskellPackages.ormolu; + +} diff --git a/Omni/Bild/Meta.hs b/Omni/Bild/Meta.hs new file mode 100644 index 0000000..44bcff0 --- /dev/null +++ b/Omni/Bild/Meta.hs @@ -0,0 +1,108 @@ +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE NoImplicitPrelude #-} + +-- | Small module for extracting metadata from the comments of modules. +module Omni.Bild.Meta where + +import Alpha +import qualified Data.Aeson as Aeson +import qualified Data.Char as Char +import qualified Data.Set as Set +import qualified Data.Text as Text +import qualified Text.Regex.Applicative as Regex + +-- | A third-party dependency. This gets mapped to some name in nixpkgs, +-- prefixed by package set like @haskellPackages@ or +-- @python3Packages@. +type Dep = String + +-- | This is a system-level requirement, the string gets mapped to a name in +-- nixpkgs at the top level, like @pkgs.thing@. +type Sys = String + +-- | A run-time dependency. This is some executable that will be placed on +-- @PATH@. This gets selected from @bild.pkgs@, so it must be exported there. +type Run = String + +-- | An arbitrary compiler argument that may get added to the compilation +-- command. Should be used sparingly, and not all builds will support this. +type Arg = String + +data Out = Lib String | Bin String | None + deriving (Show, Eq) + +instance Aeson.ToJSON Out where + toJSON = + Aeson.String <. Text.pack <. \case + Bin a -> a + Lib a -> a + None -> "" + +data Parsed = Parsed + { pdep :: Set Dep, + parg :: Set Arg, + pout :: Out, + psys :: Set Sys, + prun :: Set Run + } + +detect :: (Ord a) => Regex.RE Char a -> [Text] -> Set a +detect m cl = + cl + /> Text.unpack + /> Regex.match m + |> catMaybes + |> Set.fromList + +-- | 'Out' is always singular, so it gets a special function +detectOut :: Regex.RE Char Out -> [Text] -> Out +detectOut m cl = + cl + /> Text.unpack + /> Regex.match m + |> catMaybes + |> head + |> fromMaybe None + +detectAll :: [Char] -> [Text] -> Parsed +detectAll m cl = Parsed {..} + where + pout = detectOut (out m <|> lib m) cl + detect_ re = detect (re m) cl + pdep = detect_ dep + psys = detect_ sys + parg = detect_ arg + prun = detect_ run + +dep :: [Char] -> Regex.RE Char Dep +dep comment = + Regex.string (comment ++ " : dep ") + *> Regex.many (Regex.psym (not <. Char.isSpace)) + +sys :: [Char] -> Regex.RE Char Dep +sys comment = + Regex.string (comment ++ " : sys ") + *> Regex.many (Regex.psym (not <. Char.isSpace)) + +out :: [Char] -> Regex.RE Char Out +out comment = + Regex.string (comment ++ " : out ") + *> Regex.many (Regex.psym (/= ' ')) + /> Bin + +lib :: [Char] -> Regex.RE Char Out +lib comment = + Regex.string (comment ++ " : lib ") + *> Regex.many (Regex.psym (/= ' ')) + /> Lib + +arg :: [Char] -> Regex.RE Char Arg +arg comment = + Regex.string (comment ++ " : arg ") + *> Regex.many Regex.anySym + +run :: [Char] -> Regex.RE Char Run +run comment = + Regex.string (comment ++ " : run ") + *> Regex.many Regex.anySym diff --git a/Omni/Bild/Nixpkgs.nix b/Omni/Bild/Nixpkgs.nix new file mode 100644 index 0000000..212e3f1 --- /dev/null +++ b/Omni/Bild/Nixpkgs.nix @@ -0,0 +1,43 @@ +let + sources = import ./Sources.nix { sourcesFile = ./Sources.json; }; + + config = { + allowAliases = true; + allowBroken = true; + allowUnfree = true; + checkMeta = true; + cudaSupport = true; + }; + + system = __currentSystem; + + # override pinned deps with our sources, this must come before other + # package overlays, because of the 'null' from 'overrideSource' + depsOverlay = _: pkgs: pkgs.overridePinnedDeps pkgs.overrideSource; + + overlays = [ + (_: _: { inherit sources; }) + (import ./CcacheWrapper.nix) + (import ./Functions.nix) + depsOverlay + (import ./Deps.nix) + (import ./Python.nix) + (import ./Haskell.nix) + ]; + + nixos-unstable-small = + import sources.nixos-unstable-small { inherit system config overlays; }; + +in { + nixos-24_05 = import sources.nixos-24_05 { + inherit system config; + overlays = overlays ++ [ + (_: _: { + # backport newer packages from unstable + unstable = nixos-unstable-small.pkgs; + }) + ]; + }; + + inherit nixos-unstable-small; +} diff --git a/Omni/Bild/Python.nix b/Omni/Bild/Python.nix new file mode 100644 index 0000000..6e4f390 --- /dev/null +++ b/Omni/Bild/Python.nix @@ -0,0 +1,17 @@ +_self: super: { + python3 = super.python3.override { + packageOverrides = _pyself: pysuper: + with pysuper.pkgs.python3Packages; + let dontCheck = p: p.overridePythonAttrs (_: { doCheck = false; }); + in { + interegular = callPackage ./Deps/interegular.nix { }; + llm-ollama = callPackage ./Deps/llm-ollama.nix { }; + llm-sentence-transformers = + callPackage ./Deps/llm-sentence-transformers.nix { }; + mypy = dontCheck pysuper.mypy; + outlines = callPackage ./Deps/outlines.nix { }; + perscache = callPackage ./Deps/perscache.nix { }; + tokenizers = dontCheck pysuper.tokenizers; + }; + }; +} diff --git a/Omni/Bild/Sources.json b/Omni/Bild/Sources.json new file mode 100644 index 0000000..2939283 --- /dev/null +++ b/Omni/Bild/Sources.json @@ -0,0 +1,198 @@ +{ + "clay": { + "branch": "master", + "description": "A CSS preprocessor as embedded Haskell.", + "homepage": "", + "owner": "sebastiaanvisser", + "repo": "clay", + "rev": "dcc4fc6d8b55af4814bd3f9bbb6d32e2fa2751a8", + "sha256": "1dm71z1q7yaq0kl2yb0vr0lsbd8byq5qkdb2kvr26jq48nfq2xdc", + "type": "tarball", + "url": "https://github.com/sebastiaanvisser/clay/archive/dcc4fc6d8b55af4814bd3f9bbb6d32e2fa2751a8.tar.gz", + "url_template": "https://github.com///archive/.tar.gz", + "version": "cc7729b1b42a79e261091ff7835f7fc2a7ae3cee" + }, + "docopt": { + "branch": "main", + "description": "A command-line interface description language and parser that will make you smile", + "homepage": "http://docopt.org/", + "owner": "docopt", + "repo": "docopt.hs", + "rev": "47516acafeae3e1fdc447716e6ea05c2b918ff3a", + "sha256": "07skrfhzx51yn4qvig3ps34qra9s5g6m4k2z42h9ys0ypyk2wf8w", + "type": "tarball", + "url": "https://github.com/docopt/docopt.hs/archive/47516acafeae3e1fdc447716e6ea05c2b918ff3a.tar.gz", + "url_template": "https://github.com///archive/.tar.gz", + "version": "0.7.0.8" + }, + "ghc-exactprint": { + "branch": "master", + "description": "GHC version of haskell-src-exts exactPrint", + "homepage": null, + "owner": "alanz", + "repo": "ghc-exactprint", + "rev": "3e70715a756c46761a3a6a086a6be5dee4e60d22", + "sha256": "1mhmk1555n7qr25iwbm8kbjs24c9j0q01j4m2kmz6zh7r1gjayxs", + "type": "tarball", + "url": "https://github.com/alanz/ghc-exactprint/archive/3e70715a756c46761a3a6a086a6be5dee4e60d22.tar.gz", + "url_template": "https://github.com///archive/.tar.gz", + "version": "0.6.3.4" + }, + "guix": { + "branch": "master", + "repo": "https://git.savannah.gnu.org/git/guix.git", + "rev": "a25e0518954b48753ff44ad116d0a6fb47dfb6cb", + "type": "git", + "version": "2021-06-14-unstable" + }, + "inspekt3d": { + "branch": "master", + "type": "git", + "repo": "https://git.sr.ht/~morgansmith/inspekt3d", + "rev": "703f52ccbfedad2bf5240bf8183d1b573c9d54ef" + }, + "interegular": { + "branch": "master", + "description": "Allows to check regexes for overlaps. Based on greenery by @qntm.", + "homepage": null, + "owner": "MegaIng", + "repo": "interegular", + "rev": "v0.2.1", + "sha256": "14f3jvnczq6qay2qp4rxchbdhkj00qs8kpacl0nrxgr0785km36k", + "type": "tarball", + "url": "https://github.com/MegaIng/interegular/archive/v0.2.1.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "llm": { + "branch": "main", + "description": "Access large language models from the command-line", + "homepage": "https://llm.datasette.io", + "owner": "simonw", + "repo": "llm", + "rev": "0.13.1", + "sha256": "0305xpmigk219i2n1slgpz3jwvpx5pdp5s8dkjz85w75xivakbin", + "type": "tarball", + "url": "https://github.com/simonw/llm/archive/0.13.1.tar.gz", + "url_template": "https://github.com///archive/.tar.gz", + "version": "0.13.1" + }, + "niv": { + "branch": "master", + "description": "Easy dependency management for Nix projects", + "homepage": "https://github.com/nmattia/niv", + "owner": "nmattia", + "repo": "niv", + "rev": "e80fc8fae87cc91f449533fca6b9cadf8be69e6c", + "sha256": "024hnxvqk8z5n2n54rj05l91q38g9y8nwvrj46xml13kjmg4shb3", + "type": "tarball", + "url": "https://github.com/nmattia/niv/archive/e80fc8fae87cc91f449533fca6b9cadf8be69e6c.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixos-23_05": { + "branch": "nixos-23.05", + "description": "Nix Packages collection & NixOS", + "homepage": "", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "70bdadeb94ffc8806c0570eb5c2695ad29f0e421", + "sha256": "05cbl1k193c9la9xhlz4y6y8ijpb2mkaqrab30zij6z4kqgclsrd", + "type": "tarball", + "url": "https://github.com/nixos/nixpkgs/archive/70bdadeb94ffc8806c0570eb5c2695ad29f0e421.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixos-23_11": { + "branch": "nixos-23.11", + "description": "Nix Packages collection & NixOS", + "homepage": "", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "219951b495fc2eac67b1456824cc1ec1fd2ee659", + "sha256": "065jy7qivlbdqmbvd7r9h97b23f21axmc4r7sqmq2h0j82rmymxv", + "type": "tarball", + "url": "https://github.com/nixos/nixpkgs/archive/219951b495fc2eac67b1456824cc1ec1fd2ee659.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixos-24_05": { + "branch": "nixos-24.05", + "description": "Nix Packages collection & NixOS", + "homepage": "", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "a9b86fc2290b69375c5542b622088eb6eca2a7c3", + "sha256": "1mssfzy1nsansjmp5ckyl8vbk32va3abchpg19ljyak0xblxnjs1", + "type": "tarball", + "url": "https://github.com/nixos/nixpkgs/archive/a9b86fc2290b69375c5542b622088eb6eca2a7c3.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixos-mailserver": { + "repo": "https://gitlab.com/simple-nixos-mailserver/nixos-mailserver", + "rev": "f535d8123c4761b2ed8138f3d202ea710a334a1d", + "sha256": "0csx2i8p7gbis0n5aqpm57z5f9cd8n9yabq04bg1h4mkfcf7mpl6", + "type": "tarball", + "url": "https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/archive/f535d8123c4761b2ed8138f3d202ea710a334a1d/nixos-mailserver-f535d8123c4761b2ed8138f3d202ea710a334a1d.tar.gz", + "url_template": "https://gitlab.com/simple-nixos-mailserver/nixos-mailserver/-/archive//nixos-mailserver-.tar.gz", + "version": "master" + }, + "nixos-unstable-small": { + "branch": "nixos-unstable-small", + "description": "Nix Packages collection & NixOS", + "homepage": "", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "a5e6a9e979367ee14f65d9c38119c30272f8455f", + "sha256": "08yfk81kpsizdzlbi8whpaarb0w0rw9aynlrvhn5gr5dfpv9hbsf", + "type": "tarball", + "url": "https://github.com/nixos/nixpkgs/archive/a5e6a9e979367ee14f65d9c38119c30272f8455f.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "outlines": { + "branch": "main", + "description": "Generative Model Programming", + "homepage": "https://normal-computing.github.io/outlines/", + "owner": "normal-computing", + "repo": "outlines", + "rev": "0.0.8", + "sha256": "1yvx5c5kplmr56nffqcb6ssjnmlikkaw32hxl6i4b607v3s0s6jv", + "type": "tarball", + "url": "https://github.com/normal-computing/outlines/archive/0.0.8.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "perscache": { + "branch": "master", + "description": "An easy to use decorator for persistent memoization: like `functools.lrucache`, but results can be saved in any format to any storage.", + "homepage": null, + "owner": "leshchenko1979", + "repo": "perscache", + "rev": "0.6.1", + "sha256": "0j2775pjll4vw1wmxkjhnb5z6z83x5lhg89abj2d8ivd17n4rhjf", + "type": "tarball", + "url": "https://github.com/leshchenko1979/perscache/archive/0.6.1.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "radicale": { + "branch": "master", + "description": "A simple CalDAV (calendar) and CardDAV (contact) server.", + "homepage": "https://radicale.org", + "owner": "kozea", + "repo": "radicale", + "rev": "d7ce2f0b98589400b0e1718cfd7bb29b7ebeaebe", + "sha256": "08himwwwikhnn4amqzgbbqq323xhfy7yf5vaqczkm2fw6h1s3skg", + "type": "tarball", + "url": "https://github.com/kozea/radicale/archive/d7ce2f0b98589400b0e1718cfd7bb29b7ebeaebe.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "regex-applicative": { + "branch": "master", + "description": "Regex-based parsing with applicative interface", + "homepage": "", + "owner": "feuerbach", + "repo": "regex-applicative", + "rev": "449519c38e65753345e9a008362c011cb7a0a4d9", + "revision": "449519c38e65753345e9a008362c011cb7a0a4d9", + "sha256": "1vdrhsjzij5dm7rn10sic5dv9574yb0lyhzfv9psh7b08dsj8g1k", + "type": "tarball", + "url": "https://github.com/feuerbach/regex-applicative/archive/449519c38e65753345e9a008362c011cb7a0a4d9.tar.gz", + "url_template": "https://github.com///archive/.tar.gz", + "version": "0.3.4" + } +} diff --git a/Omni/Bild/Sources.nix b/Omni/Bild/Sources.nix new file mode 100644 index 0000000..f7af81e --- /dev/null +++ b/Omni/Bild/Sources.nix @@ -0,0 +1,207 @@ +# This file has been generated by Niv. + +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: name: spec: + let name' = sanitizeName name + "-src"; + in if spec.builtin or true then + builtins_fetchurl { + inherit (spec) url sha256; + name = name'; + } + else + pkgs.fetchurl { + inherit (spec) url sha256; + name = name'; + }; + + fetch_tarball = pkgs: name: spec: + let name' = sanitizeName name + "-src"; + in if spec.builtin or true then + builtins_fetchTarball { + name = name'; + inherit (spec) url sha256; + } + else + pkgs.fetchzip { + name = name'; + inherit (spec) url sha256; + }; + + fetch_git = name: spec: + let + ref = if spec ? ref then + spec.ref + else if spec ? branch then + "refs/heads/${spec.branch}" + else if spec ? tag then + "refs/tags/${spec.tag}" + else + abort + "In git source '${name}': Please specify `ref`, `tag` or `branch`!"; + in builtins.fetchGit { + url = spec.repo; + inherit (spec) rev; + inherit ref; + }; + + fetch_local = spec: spec.path; + + fetch_builtin-tarball = name: + throw '' + [${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=tarball -a builtin=true''; + + fetch_builtin-url = name: + throw '' + [${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=file -a builtin=true''; + + # + # Various helpers + # + + # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 + sanitizeName = name: + (concatMapStrings (s: if builtins.isList s then "-" else s) + (builtins.split "[^[:alnum:]+._?=-]+" + ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name))); + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: system: + let + sourcesNixpkgs = import + (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { + inherit system; + }; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in if builtins.hasAttr "nixpkgs" sources then + sourcesNixpkgs + else if hasNixpkgsPath && !hasThisAsNixpkgsPath then + import { } + else + abort '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if !builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then + fetch_file pkgs name spec + else if spec.type == "tarball" then + fetch_tarball pkgs name spec + else if spec.type == "git" then + fetch_git name spec + else if spec.type == "local" then + fetch_local spec + else if spec.type == "builtin-tarball" then + fetch_builtin-tarball name + else if spec.type == "builtin-url" then + fetch_builtin-url name + else + abort + "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # If the environment variable NIV_OVERRIDE_${name} is set, then use + # the path directly as opposed to the fetched source. + replace = name: drv: + let + saneName = stringAsChars + (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name; + ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; + in if ersatz == "" then + drv + else + # this turns the string into an actual Nix path (for both absolute and + # relative paths) + if builtins.substring 0 1 ersatz == "/" then + /. + ersatz + else + /. + builtins.getEnv "PWD" + "/${ersatz}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or (f: set: + with builtins; + listToAttrs (map (attr: { + name = attr; + value = f attr set.${attr}; + }) (attrNames set))); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 + range = first: last: + if first > last then + [ ] + else + builtins.genList (n: first + n) (last - first + 1); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 + stringToCharacters = s: + map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 + stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); + concatMapStrings = f: list: concatStrings (map f list); + concatStrings = builtins.concatStringsSep ""; + + # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 + optionalAttrs = cond: as: if cond then as else { }; + + # fetchTarball version that is compatible between all the versions of Nix + # deadnix: skip + builtins_fetchTarball = { url, name ? null, sha256 }@attrs: + let inherit (builtins) lessThan nixVersion fetchTarball; + in if lessThan nixVersion "1.12" then + fetchTarball + ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + # deadnix: skip + builtins_fetchurl = { url, name ? null, sha256 }@attrs: + let inherit (builtins) lessThan nixVersion fetchurl; + in if lessThan nixVersion "1.12" then + fetchurl + ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs (name: spec: + if builtins.hasAttr "outPath" spec then + abort + "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = replace name (fetch config.pkgs name spec); }) + config.sources; + + # The "config" used by the fetchers + mkConfig = { sourcesFile ? + if builtins.pathExists ./sources.json then ./sources.json else null + , sources ? if isNull sourcesFile then + { } + else + builtins.fromJSON (builtins.readFile sourcesFile) + , system ? builtins.currentSystem, pkgs ? mkPkgs sources system }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; + }; + +in mkSources (mkConfig { }) // { + __functor = _: settings: mkSources (mkConfig settings); +} -- cgit v1.2.3