Add simonw/llm as cli/library client for running LLMs

This is basically exactly the client library that I would write myself. Some
parts of it are still beta quality, but it's the sort of thing that I would
contribute to anyway.

Unfortunately I couldn't get the llm-llama-cpp plugin to work because it depends
on llama-cpp-python which is not packaged for nix and is hard to package because
the upstream project vendors a patched version of llama.cpp. So I'm stuck with
ollama for now, but that's fine because it actually works.

6 files changed, 61 insertions, 2 deletions

diff --git a/Biz/Bild.nix b/Biz/Bild.nix
index ec049d8..894ebe6 100644
--- a/Biz/Bild.nix
+++ b/Biz/Bild.nix
@@ -82,6 +82,8 @@ let
       inherit bat bc cmark ctags deadnix fd figlet fzf git git-branchless
         gitlint guile hlint indent jq lolcat mypy nixfmt ormolu pkg-config
         ripgrep rustc tree wemux;
+      llm = nixpkgs.nixos-unstable-small.llm.withPlugins
+        [ nixpkgs.nixos-unstable-small.python3.pkgs.llm-ollama ];
       ollama = nixpkgs.nixos-unstable-small.ollama;
       ruff = nixpkgs.nixos-unstable-small.ruff;
       shellcheck = nixpkgs.nixos-unstable-small.shellcheck;
@@ -186,6 +188,7 @@ let
         gitlint
         jq
         lolcat
+        llm
         ormolu
         ripgrep
         stable.haskell.packages.${constants.ghcCompiler}.fast-tags
diff --git a/Biz/Bild/Deps.nix b/Biz/Bild/Deps.nix
index 8bf2272..46fa00f 100644
--- a/Biz/Bild/Deps.nix
+++ b/Biz/Bild/Deps.nix
@@ -30,6 +30,8 @@ _self: super:
     ];
   };
 
+  llm = super.overrideSrc super.llm super.sources.llm;
+
   nostr-rs-relay = super.callPackage ./Deps/nostr-rs-relay.nix { };
 
   ollama = super.callPackage ./Deps/ollama.nix { acceleration = "cuda"; };
diff --git a/Biz/Bild/Deps/llm-ollama.nix b/Biz/Bild/Deps/llm-ollama.nix
new file mode 100644
index 0000000..e2b6a66
--- /dev/null
+++ b/Biz/Bild/Deps/llm-ollama.nix
@@ -0,0 +1,40 @@
+{ buildPythonPackage, fetchFromGitHub, lib, llm, ollama, pytestCheckHook
+, setuptools, pythonOlder, }:
+buildPythonPackage rec {
+  pname = "llm-ollama";
+  version = "0.3.0";
+  pyproject = true;
+
+  disabled = pythonOlder "3.8";
+
+  src = fetchFromGitHub {
+    owner = "taketwo";
+    repo = pname;
+    rev = "refs/tags/${version}";
+    hash = "sha256-Ar0Ux8BNGY0i764CEk7+48J6jnndlRIIMPZ9tFpXiy4=";
+  };
+
+  nativeBuildInputs = [ setuptools ];
+
+  buildInputs = [ llm ollama ];
+
+  propagatedBuildInputs = [ ollama ];
+
+  disabledTests = [
+    # wants to mkdir in the /homeless-shelter
+    "test_registered_models"
+  ];
+
+  nativeCheckInputs = [ pytestCheckHook ];
+
+  pythonImportsCheck = [ "llm_ollama" ];
+
+  meta = with lib; {
+    homepage = "https://github.com/taketwo/llm-ollama";
+    description =
+      "LLM plugin providing access to local Ollama models usting HTTP API";
+    changelog = "https://github.com/taketwo/llm-ollama/releases/tag/${version}";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ bsima ];
+  };
+}
diff --git a/Biz/Bild/Functions.nix b/Biz/Bild/Functions.nix
index b725a00..8b87f86 100644
--- a/Biz/Bild/Functions.nix
+++ b/Biz/Bild/Functions.nix
@@ -21,8 +21,8 @@ _: super: {
       null;
 
   # Simply override the 'src' attr on a drv. This is meant to be a simpler
-  # alternative to 'overrideSource' above. In an overaly, use it like:
-  #    mypkg = super.overrideSource super.mypkg super.sources.mypkg;
+  # alternative to 'overrideSource' above. In an overlay, use it like:
+  #    mypkg = super.overrideSrc super.mypkg super.sources.mypkg;
   overrideSrc = dep: src:
     dep.overrideAttrs (attrs:
       attrs // {
diff --git a/Biz/Bild/Python.nix b/Biz/Bild/Python.nix
index c559e42..2385987 100644
--- a/Biz/Bild/Python.nix
+++ b/Biz/Bild/Python.nix
@@ -7,6 +7,7 @@ _self: super: {
         exllama = callPackage ./Deps/exllama.nix { };
         exllamav2 = callPackage ./Deps/exllamav2.nix { };
         interegular = callPackage ./Deps/interegular.nix { };
+        llm-ollama = callPackage ./Deps/llm-ollama.nix { };
         mypy = dontCheck pysuper.mypy;
         outlines = callPackage ./Deps/outlines.nix { };
         perscache = callPackage ./Deps/perscache.nix { };
diff --git a/Biz/Bild/Sources.json b/Biz/Bild/Sources.json
index 40bcdfb..3b1e4fd 100644
--- a/Biz/Bild/Sources.json
+++ b/Biz/Bild/Sources.json
@@ -77,6 +77,19 @@
         "url": "https://github.com/MegaIng/interegular/archive/v0.2.1.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
+    "llm": {
+        "branch": "main",
+        "description": "Access large language models from the command-line",
+        "homepage": "https://llm.datasette.io",
+        "owner": "simonw",
+        "repo": "llm",
+        "rev": "0.13.1",
+        "sha256": "0305xpmigk219i2n1slgpz3jwvpx5pdp5s8dkjz85w75xivakbin",
+        "type": "tarball",
+        "url": "https://github.com/simonw/llm/archive/0.13.1.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz",
+        "version": "0.13.1"
+    },
     "niv": {
         "branch": "master",
         "description": "Easy dependency management for Nix projects",