Rewrite mktags.sh to MakeTags.py

This makes tags for all third-party dependencies, and uses universal-ctags
instead of fast-tags. It's so nice having tags for /everything/. It's also
pretty fast, and caches the result for third-party deps anyway.

Oh yeah this also added an explicit list of third-party Python packages. I need
to filter the entire pkgset down to just the ones I'm using in order to make
tags for them.

5 files changed, 134 insertions, 48 deletions

diff --git a/Biz/Bild.nix b/Biz/Bild.nix
index 8e33d93..c24cecd 100644
--- a/Biz/Bild.nix
+++ b/Biz/Bild.nix
@@ -14,6 +14,19 @@ let
 
   unstable = nixpkgs.nixos-unstable-small;
 
+  # get the .src attributes of all drvs in each pkgset in the `sources` list,
+  # and concat them with `:` into a Unix-style search path.
+  # makeSourcesPath :: [pkgset] -> str
+  makeSourcesPath = with stable;
+    sources:
+    lib.trivial.pipe sources [
+      (builtins.map lib.attrsets.attrValues)
+      lib.lists.flatten
+      (builtins.filter (pkg: pkg != null))
+      (builtins.map (pkg: pkg.src))
+      (lib.strings.concatStringsSep ":")
+    ];
+
   # this is the main library definitions, recursive references can be made with
   # `self.thing`, like in Python objects
   self = {
@@ -68,7 +81,8 @@ let
     lisp = { sbclWith = stable.sbcl.withPackages; };
 
     python = {
-      packages = stable.python3.pkgs;
+      packages = self.lib.attrsets.getAttrs (import ./Bild/Deps/Python.nix)
+        stable.python3.pkgs;
       pythonWith = stable.python3.withPackages;
       buildPythonApplication = stable.python3.pkgs.buildPythonApplication;
     };
@@ -83,8 +97,7 @@ let
         git-branchless gitlint groff guile hlint indent jq lolcat mypy nixfmt
         ormolu pkg-config ripgrep rustc tree wemux;
       llama-cpp = unstable.llama-cpp;
-      llm = python3.withPackages
-        (p: with p; [ p.llm-ollama ]);
+      llm = python3.withPackages (p: with p; [ p.llm-ollama ]);
       ollama = unstable.ollama;
       ruff = unstable.ruff;
       shellcheck = unstable.shellcheck;
@@ -192,12 +205,18 @@ let
         llm
         ormolu
         ripgrep
-        stable.haskell.packages.${constants.ghcCompiler}.fast-tags
         tree
         wemux
       ];
       shellHook = ''
         export GHC_PACKAGE_PATH=${self.bild}/lib/ghc-${self.haskell.ghcVersion}/package.conf.d
+        export ALL_SOURCES=${
+          makeSourcesPath [
+            self.python.packages
+            self.haskell.packages
+            self.c.packages
+          ]
+        }
       '';
     };
 
diff --git a/Biz/Bild/Deps/Python.nix b/Biz/Bild/Deps/Python.nix
new file mode 100644
index 0000000..b0b2465
--- /dev/null
+++ b/Biz/Bild/Deps/Python.nix
@@ -0,0 +1 @@
+[ "cryptography" "llm" "mypy" "nltk" "slixmpp" ]
diff --git a/Biz/Ide/MakeTags.py b/Biz/Ide/MakeTags.py
new file mode 100755
index 0000000..add07c0
--- /dev/null
+++ b/Biz/Ide/MakeTags.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+"""
+Make tags for internal or external code.
+
+This should run fast, and be executable with just Python, meaning it does not
+require a build step.
+"""
+
+# : out maketags
+# : run universal-ctags
+import argparse
+import os
+import pathlib
+import subprocess
+import tarfile
+import zipfile
+
+
+def main() -> None:
+    """Run ctags on internal or external source code.
+
+    Raises:
+        ValueError: if CODEROOT is not set
+        ArgumentError: when explicit paths aren't provided
+    """
+    coderoot = os.environ.get("CODEROOT")
+    if coderoot is None:
+        msg = "CODEROOT not set"
+        raise ValueError(msg)
+    cabsrc = pathlib.Path(coderoot) / "_" / "src"
+    cli = argparse.ArgumentParser()
+    cli.add_argument(
+        "paths",
+        nargs="*",
+        default=".",
+        help="List of paths to run ctags on. Defaults to '.'",
+    )
+    cli.add_argument(
+        "-x",
+        "--external",
+        action="store_true",
+        help=" ".join([
+            "Use this when `paths` is a list of external packages,",
+            f"they will be extracted or linked into {cabsrc}",
+        ]),
+    )
+    args = cli.parse_args()
+    if args.external and args.paths == ".":
+        msg = "requires explicit paths"
+        raise argparse.ArgumentError(argument=args.external, message=msg)
+    if args.external:
+        extract_and_copy(cabsrc, args.paths)
+        ctags(["--recurse=yes"], cwd=cabsrc)
+    else:
+        ctags(["--exclude=*_/*", "--recurse=yes"], cwd=pathlib.Path(coderoot))
+
+
+def strip_nix_hash(path: str) -> str:
+    """Remove the /nix/store/ and hash prefix from a path."""
+    hash_len = 33
+    return path.removeprefix("/nix/store/")[hash_len:]
+
+
+def extract_and_copy(cabsrc: pathlib.Path, paths: list[str]) -> None:
+    """
+    Extract and copy or link sources.
+
+    Loop over `paths`, if the path is an archive, extract it into `cabsrc`. If
+    its a directory, just symlink the directory into `cabsrc`. Either way, we
+    end up with a directory full of source trees for running ctags on.
+    """
+    for path in paths:
+        outpath: pathlib.Path = cabsrc / strip_nix_hash(path)
+        if outpath.exists():
+            continue
+        if path.endswith(".zip"):
+            out = outpath.with_suffix("")
+            if out.exists():
+                continue
+            zipfile.ZipFile(path).extractall(out)  # noqa: S202
+        elif path.endswith(".tar.gz"):
+            out = outpath.with_suffix("").with_suffix("")
+            if out.exists():
+                continue
+            with tarfile.open(path) as tarball:
+                tarball.extractall(out)  # noqa: S202
+        elif pathlib.Path(path).is_dir():
+            outpath.symlink_to(path)
+
+
+def ctags(args: list[str], cwd: pathlib.Path = pathlib.Path()) -> None:
+    """Call `ctags` with `args` for both emacs and vim."""
+    os.chdir(cwd)
+    excludes = [
+        "--exclude=.mypy_cache",
+        "--exclude=.git",
+        "--exclude=.direnv",
+        "--exclude=.ruff_cache",
+    ]
+    subprocess.check_call(["ctags", *excludes, *args])
+    subprocess.check_call(["ctags", "-e", *excludes, *args])
+
+
+if __name__ == "__main__":
+    main()
diff --git a/Biz/Ide/hooks/post-checkout b/Biz/Ide/hooks/post-checkout
index 72ec5a7..4f58c74 100755
--- a/Biz/Ide/hooks/post-checkout
+++ b/Biz/Ide/hooks/post-checkout
@@ -1,16 +1,18 @@
 #!/usr/bin/env bash
 set -e
-mktags=${CODEROOT:?}/Biz/Ide/mktags.sh
+function MakeTags {
+  ${CODEROOT:?}/Biz/Ide/MakeTags.py
+}
 old=$1
 new=$2
 # filter out only the changed haskell files
 mapfile -t changed < <(git diff --diff-filter=d --name-only "$old" "$new" -- '*.hs')
 if [[ ! -r tags ]] || [[ ! -r TAGS ]]
 then
-    $mktags "$CODEROOT"/**/*
+    MakeTags "$CODEROOT"/**/*
 elif [[ ${#changed[@]} -gt 0 ]]
 then
-    $mktags "${changed[@]}"
+    MakeTags "${changed[@]}"
 fi
 ## START BRANCHLESS CONFIG
 
diff --git a/Biz/Ide/mktags.sh b/Biz/Ide/mktags.sh
deleted file mode 100755
index aae1505..0000000
--- a/Biz/Ide/mktags.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-#
-# script to generate tags automatically if there are none.
-#
-  set -euo pipefail
-  files="$*"
-  vimtags=${CODEROOT:?}/tags
-  emacstags=${CODEROOT:?}/TAGS
-#
-  if [[ ! -r $emacstags ]]; then
-      echo Generating emacs TAGS from scratch...
-      fast-tags -e -R "${CODEROOT:?}"
-      ctags -e \
-           --append=yes \
-           --recurse=yes \
-           --exclude="$CODEROOT/_/*" \
-           "${CODEROOT:?}"
-  else
-      fast-tags -e "$files"
-      ctags -e \
-           --append=yes \
-           --exclude="_/*" \
-           "$files"
-  fi
-#
-  if [[ ! -r $vimtags ]]; then
-      echo Generating vim tags from scratch...
-      fast-tags -R "${CODEROOT:?}"
-      ctags \
-           --append=yes \
-           --recurse=yes \
-           --exclude="${CODEROOT:?}/_/*" \
-           "${CODEROOT:?}"
-  else
-      fast-tags "$files"
-      ctags \
-           --append=yes \
-           --exclude="_/*" \
-           "$files"
-  fi
-##