From 2d33aa547ff6a516c90ca2b47b13e2add200583a Mon Sep 17 00:00:00 2001 From: Ben Sima Date: Tue, 14 May 2024 09:35:45 -0400 Subject: Add simonw/llm as cli/library client for running LLMs This is basically exactly the client library that I would write myself. Some parts of it are still beta quality, but it's the sort of thing that I would contribute to anyway. Unfortunately I couldn't get the llm-llama-cpp plugin to work because it depends on llama-cpp-python which is not packaged for nix and is hard to package because the upstream project vendors a patched version of llama.cpp. So I'm stuck with ollama for now, but that's fine because it actually works. --- Biz/Bild.nix | 3 +++ Biz/Bild/Deps.nix | 2 ++ Biz/Bild/Deps/llm-ollama.nix | 40 ++++++++++++++++++++++++++++++++++++++++ Biz/Bild/Functions.nix | 4 ++-- Biz/Bild/Python.nix | 1 + Biz/Bild/Sources.json | 13 +++++++++++++ 6 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 Biz/Bild/Deps/llm-ollama.nix diff --git a/Biz/Bild.nix b/Biz/Bild.nix index ec049d8..894ebe6 100644 --- a/Biz/Bild.nix +++ b/Biz/Bild.nix @@ -82,6 +82,8 @@ let inherit bat bc cmark ctags deadnix fd figlet fzf git git-branchless gitlint guile hlint indent jq lolcat mypy nixfmt ormolu pkg-config ripgrep rustc tree wemux; + llm = nixpkgs.nixos-unstable-small.llm.withPlugins + [ nixpkgs.nixos-unstable-small.python3.pkgs.llm-ollama ]; ollama = nixpkgs.nixos-unstable-small.ollama; ruff = nixpkgs.nixos-unstable-small.ruff; shellcheck = nixpkgs.nixos-unstable-small.shellcheck; @@ -186,6 +188,7 @@ let gitlint jq lolcat + llm ormolu ripgrep stable.haskell.packages.${constants.ghcCompiler}.fast-tags diff --git a/Biz/Bild/Deps.nix b/Biz/Bild/Deps.nix index 8bf2272..46fa00f 100644 --- a/Biz/Bild/Deps.nix +++ b/Biz/Bild/Deps.nix @@ -30,6 +30,8 @@ _self: super: ]; }; + llm = super.overrideSrc super.llm super.sources.llm; + nostr-rs-relay = super.callPackage ./Deps/nostr-rs-relay.nix { }; ollama = super.callPackage ./Deps/ollama.nix { acceleration = "cuda"; }; diff --git a/Biz/Bild/Deps/llm-ollama.nix b/Biz/Bild/Deps/llm-ollama.nix new file mode 100644 index 0000000..e2b6a66 --- /dev/null +++ b/Biz/Bild/Deps/llm-ollama.nix @@ -0,0 +1,40 @@ +{ buildPythonPackage, fetchFromGitHub, lib, llm, ollama, pytestCheckHook +, setuptools, pythonOlder, }: +buildPythonPackage rec { + pname = "llm-ollama"; + version = "0.3.0"; + pyproject = true; + + disabled = pythonOlder "3.8"; + + src = fetchFromGitHub { + owner = "taketwo"; + repo = pname; + rev = "refs/tags/${version}"; + hash = "sha256-Ar0Ux8BNGY0i764CEk7+48J6jnndlRIIMPZ9tFpXiy4="; + }; + + nativeBuildInputs = [ setuptools ]; + + buildInputs = [ llm ollama ]; + + propagatedBuildInputs = [ ollama ]; + + disabledTests = [ + # wants to mkdir in the /homeless-shelter + "test_registered_models" + ]; + + nativeCheckInputs = [ pytestCheckHook ]; + + pythonImportsCheck = [ "llm_ollama" ]; + + meta = with lib; { + homepage = "https://github.com/taketwo/llm-ollama"; + description = + "LLM plugin providing access to local Ollama models usting HTTP API"; + changelog = "https://github.com/taketwo/llm-ollama/releases/tag/${version}"; + license = licenses.asl20; + maintainers = with maintainers; [ bsima ]; + }; +} diff --git a/Biz/Bild/Functions.nix b/Biz/Bild/Functions.nix index b725a00..8b87f86 100644 --- a/Biz/Bild/Functions.nix +++ b/Biz/Bild/Functions.nix @@ -21,8 +21,8 @@ _: super: { null; # Simply override the 'src' attr on a drv. This is meant to be a simpler - # alternative to 'overrideSource' above. In an overaly, use it like: - # mypkg = super.overrideSource super.mypkg super.sources.mypkg; + # alternative to 'overrideSource' above. In an overlay, use it like: + # mypkg = super.overrideSrc super.mypkg super.sources.mypkg; overrideSrc = dep: src: dep.overrideAttrs (attrs: attrs // { diff --git a/Biz/Bild/Python.nix b/Biz/Bild/Python.nix index c559e42..2385987 100644 --- a/Biz/Bild/Python.nix +++ b/Biz/Bild/Python.nix @@ -7,6 +7,7 @@ _self: super: { exllama = callPackage ./Deps/exllama.nix { }; exllamav2 = callPackage ./Deps/exllamav2.nix { }; interegular = callPackage ./Deps/interegular.nix { }; + llm-ollama = callPackage ./Deps/llm-ollama.nix { }; mypy = dontCheck pysuper.mypy; outlines = callPackage ./Deps/outlines.nix { }; perscache = callPackage ./Deps/perscache.nix { }; diff --git a/Biz/Bild/Sources.json b/Biz/Bild/Sources.json index 40bcdfb..3b1e4fd 100644 --- a/Biz/Bild/Sources.json +++ b/Biz/Bild/Sources.json @@ -77,6 +77,19 @@ "url": "https://github.com/MegaIng/interegular/archive/v0.2.1.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, + "llm": { + "branch": "main", + "description": "Access large language models from the command-line", + "homepage": "https://llm.datasette.io", + "owner": "simonw", + "repo": "llm", + "rev": "0.13.1", + "sha256": "0305xpmigk219i2n1slgpz3jwvpx5pdp5s8dkjz85w75xivakbin", + "type": "tarball", + "url": "https://github.com/simonw/llm/archive/0.13.1.tar.gz", + "url_template": "https://github.com///archive/.tar.gz", + "version": "0.13.1" + }, "niv": { "branch": "master", "description": "Easy dependency management for Nix projects", -- cgit v1.2.3