From 8bcc88ed658fa859cc37f86fc404d415f7aa2275 Mon Sep 17 00:00:00 2001 From: eikek Date: Sun, 10 Mar 2024 00:59:56 +0100 Subject: [PATCH] Document flake dev setup --- .gitignore | 1 + flake.lock | 6 +- flake.nix | 26 ++- nix/test-vm.nix | 7 + website/site/content/docs/dev/development.md | 186 ++++++++++++++---- website/site/content/docs/install/nix.md | 42 ++++ .../site/content/docs/install/quickstart.md | 8 +- 7 files changed, 231 insertions(+), 45 deletions(-) diff --git a/.gitignore b/.gitignore index 18234e40..a921d704 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ #artwork/*.png +.envrc target/ local/ node_modules/ diff --git a/flake.lock b/flake.lock index d2dfeb62..e2f18d87 100644 --- a/flake.lock +++ b/flake.lock @@ -6,11 +6,11 @@ "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1709936258, - "narHash": "sha256-ziYmDU/5v++oYSSwyMqEOr2V75rO+dMQA5aEdwH8amw=", + "lastModified": 1710025500, + "narHash": "sha256-niJf4WZ4GHQ+RaP+Qj4+6P/2SWN1Z4co36/ETvIg0Qg=", "owner": "eikek", "repo": "devshell-tools", - "rev": "59900a3731a88508257525754a704ad2f8a3278e", + "rev": "57d7d292571e291fe8213a1655529f739cfa174d", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index af1469a6..c2ae0178 100644 --- a/flake.nix +++ b/flake.nix @@ -15,10 +15,11 @@ }: flake-utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; + sbt17 = pkgs.sbt.override { jre = pkgs.jdk17; }; devshellPkgs = with pkgs; [ jq scala-cli - sbt + sbt17 netcat jdk17 @@ -85,6 +86,7 @@ (builtins.attrValues devshell-tools.legacyPackages.${system}.cnt-scripts) ++ devshellPkgs; + DOCSPELL_ENV = "dev"; DEV_CONTAINER = "docsp-dev"; SBT_OPTS = "-Xmx2G -Xss4m"; }; @@ -93,6 +95,7 @@ (builtins.attrValues devshell-tools.legacyPackages.${system}.vm-scripts) ++ devshellPkgs; + DOCSPELL_ENV = "dev"; SBT_OPTS = "-Xmx2G -Xss4m"; DEV_VM = "dev-vm"; VM_SSH_PORT = "10022"; @@ -132,10 +135,16 @@ system = "x86_64-linux"; modules = [ { - services.dev-postgres.enable = true; + services.dev-postgres = { + enable = true; + databases = ["docspell"]; + }; services.dev-email.enable = true; services.dev-minio.enable = true; - services.dev-solr.enable = true; + services.dev-solr = { + enable = true; + cores = ["docspell"]; + }; } ]; }; @@ -143,10 +152,17 @@ system = "x86_64-linux"; modules = [ { - services.dev-postgres.enable = true; + services.dev-postgres = { + enable = true; + databases = ["docspell"]; + }; services.dev-email.enable = true; services.dev-minio.enable = true; - services.dev-solr.enable = true; + services.dev-solr = { + enable = true; + cores = ["docspell"]; + }; + port-forward.ssh = 10022; port-forward.dev-postgres = 6534; port-forward.dev-smtp = 10025; diff --git a/nix/test-vm.nix b/nix/test-vm.nix index 673b26de..91ae32f3 100644 --- a/nix/test-vm.nix +++ b/nix/test-vm.nix @@ -27,6 +27,13 @@ in { port-forward.dev-webmail = 8080; port-forward.dev-solr = 8983; + environment.systemPackages = with pkgs; [ + jq + htop + iotop + coreutils + ]; + networking = { hostName = "docspell-test-vm"; firewall.allowedTCPPorts = [7880]; diff --git a/website/site/content/docs/dev/development.md b/website/site/content/docs/dev/development.md index 3cc95543..4fb379ba 100644 --- a/website/site/content/docs/dev/development.md +++ b/website/site/content/docs/dev/development.md @@ -65,6 +65,159 @@ docspell.joex { } ``` +## Installing Nix + +It is recommended to install [nix](https://nixos.org/nix). You can use +the official installer or [this +one](https://github.com/DeterminateSystems/nix-installer), which will +enable Flakes by default. + +If not enabled, enable flakes by creating a config file: + +``` +mkdir -p ~/.config/nix #on Linux +echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf +``` + +With nix installed you can use the provided development environments +to get started quickly. + +# Nix Expressions + +The soure root contains a `flake.nix` file to install docspell via the +nix package manager and to integrate it into NixOS. + +The flake provides docspell packages of the latest release and NixOS +modules. It doesn't provide package builds from the current source +tree. + +## Dev Environments + +Additionally it provides devshells that can be used to create a +development environment for docspell. + +These two `devShell` definitions address two different setups: one +uses a NixOS container and the other a VM. Both provide the same set +of services that can be used with the local docspell instance: + +- postgresql database, with a database `docspell` and user `dev` +- solr with a core `docspell` +- email setup with smtp/imap and webmail +- minio with root user `minioadmin` + +If you are on NixOS the container is probably more convenient to use. +For other systems, the vm should be good. Drop into either shell by +running: + +``` bash +# drop into the environment setup for using a vm +nix develop .#dev-vm + +# drop into the environment setup for using a container +nix deveop .#docsp-dev +``` + +Once in such an environment, you can create the container or vm like +this: + +```bash +# dev-vm env +# - build the vm +vm-build + +# -run the vm +vm-run + +# - ssh into the vm +vm-ssh + +# docsp-dev container env +# - create the container +cnt-recreate + +# - login +cnt-login +``` + +You can use tab completion on `vm-` or `cnt-` and see other useful +commands. For instance it allows to recreate solr cores or check logs +of services on the container or vm. + +Then you can adjust your dev config file in `local/dev.conf` to +connect to the services on the vm or container. The container exposes +the default ports while the vm uses port-forwarding from the host to +the guest machine. The ports are define in `flake.nix`. For example, a +jdbc connection to postgres on the container can look like this: + +``` +jdbc.url = "jdbc:postgresql://docsp-dev:5432/docspelldev" +``` + +on the vm, it would be +``` +jdbc.url = "jdbc:postgresql://localhost:6543/docspelldev" +``` + +You can reach the webmail on both versions at port `8080`. In order to +enable sending mails between users, you need to login as some +arbitrary user so the underlying services can create the data +directories. In your dev docspell you can then connect to smtp on the +vm or container. Mails send from docspell can be checked in the +webmail. Conversely, you can send mails using webmail to any user and +have their mailbox scanned by docspell. + +### Direnv + +Using [direnv](https://direnv.net) entering the dev environment is +very convenient. Install this tool (it also has integration in several +IDEs and editors) and create a file `.envrc` in the source root: + +``` +use flake .# +``` + +The file `.envrc` is git-ignored, because are different ones possible. +Here `` refers to either `dev-cnt` or `dev-vm` - one of the +devshells defined in `flake.nix`. + +After allowing direnv to execute this file via `direnv allow` you will +be dropped into this environment whenever you enter the directory. It +will also preserve your shell, don't need to use bash. + +## Checks + +The command `nix flake check` would run all checks defined in the +flake. It will build both packages and runs a vm with docspell +installed (via NixOS modules) and check whether the services come up. + +## Test VM + +There is another VM defined in the flake that provides a full setup +for docspell. It contains docspell server and joex, a postgresql, a +solr and a email setup. The intention is to use it as an easy 'getting +started' approach with nix. + +Once it has started, you can connect to `localhost:7881` to reach +docspell. The webmail will be available at `localhost:8080`. + +You can run this vm with a single command: + +``` +nix run github:eikek/docspell#nixosConfigurations.test-vm.config.system.build.vm +``` + +It uses the same setup as the dev-vm, so you can drop into the +`.#dev-vm` development shell and use `vm-ssh` to connect to the +running test vm. + +Once connected to the machine, you can see the docspell config file via + +```bash +systemctl show docspell-joex.service | grep "ExecStart=" | sed 's/^ExecStart=.*path=\([^;]*\).*/\1/' | xargs tac | grep -m 1 . | awk '{print $NF}' | tr -d '"' | xargs jq '.' +# or replace "joex" with "restserver" +systemctl show docspell-restserver.service | grep "ExecStart=" | sed 's/^ExecStart=.*path=\([^;]*\).*/\1/' | xargs tac | grep -m 1 . | awk '{print $NF}' | tr -d '"' | xargs jq '.' +``` + # Developing Frontend The frontend is a SPA written in [Elm](https://elm-lang.org). The UI @@ -124,39 +277,6 @@ These tasks must not show any errors (it is checked by the CI). The warnings should also be fixed. -# Nix Expressions - -The directory `/nix` contains Nix Flake to install docspell via -the nix package manager and to integrate it into NixOS. - -Flake implements `checks` output which can be run with `nix flake check` -and it defines a development VM which can be used to interactively work -with docspell. - -To run the VM, issue: - -```bash -cd $PROJECT_ROOT/nix -nix run '.#nixosConfigurations.dev-vm.config.system.build.vm -``` - -To open docspell, wait for docspell-restserver service to report that -http listener is up and connect to `localhost:64080`. - -To ssh into the machine, run: - -```bash -ssh -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -p 64022 root@localhost -``` - -Once connected to the machine, you can see the docspell config file via - -```bash -systemd-show docspell-joex.service | grep ExecStart | cut -d'=' -f2 | xargs cat | tail -n1 | awk '{print $NF}'| sed 's/.$//' | xargs cat | jq -``` - # Release The CI and making a release is done via github actions. The workflow diff --git a/website/site/content/docs/install/nix.md b/website/site/content/docs/install/nix.md index 296d6911..dce291d5 100644 --- a/website/site/content/docs/install/nix.md +++ b/website/site/content/docs/install/nix.md @@ -5,6 +5,34 @@ weight = 24 # Nix +Docspell is a flake, you need to enable flakes in order to make use of +it. You can also use the provided expressions without Flakes, which is +described below. + +## Try it out {try-it-out} + +You can try out the server and joex packages by running the following: + +``` +nix run github:eikek/docspell#docspell-server +nix run github:eikek/docspell#docspell-joex +``` + +While this works, it will be only a very basic setup. The database +defaults to a directory in `/tmp` and no fulltext search enabled. Then +for processing documents, some external tools are required which would +need to be present on yout system to make it work. + +A more elaborate setup with PostgreSQL and SOLR can be started using +the `test-vm`: + +``` +nix run github:eikek/docspell#nixosConfigurations.test-vm.config.system.build.vm +``` + +The vm contains all the required tools. After starting up, you can +find docspell at `http://localhost:7881`. + ## Install via Nix Docspell can be installed via the [nix](https://nixos.org/nix) package @@ -12,6 +40,16 @@ manager. Docspell is currently not part of the [nixpkgs collection](https://nixos.org/nixpkgs/), but you can use the flake from this repository. +You could install the server and joex by running the following: +``` +nix profile install github:eikek/docspell#docspell-server +nix profile install github:eikek/docspell#docspell-joex +``` + +This would install the packages on your system. If you use NixOS you +probably want to use the provided [NixOS modules](#nixos) instead. + + ## Upgrading Since [downgrading](@/docs/install/downgrading.md) is not supported, @@ -164,3 +202,7 @@ in The same way import NixOS modules from `nix/modules/{joex|server}.nix`. + +An alternative can be to use `builtins.getFlake` to fetch the flake +and get access to its output. But this requires to use a flake enabled +nix, which then defeats the idea of "not using flakes". diff --git a/website/site/content/docs/install/quickstart.md b/website/site/content/docs/install/quickstart.md index fa02b4e1..4aaefdb7 100644 --- a/website/site/content/docs/install/quickstart.md +++ b/website/site/content/docs/install/quickstart.md @@ -20,10 +20,10 @@ To get started, here are some quick links: applies to the `zip` and `deb` files. The files can be downloaded from the [release page](https://github.com/eikek/docspell/releases/latest). -- via the [nix package manager](@/docs/install/nix.md) and/or as a - [NixOS module](@/docs/install/nix.md#nixos) through a flake. - If you use nix/nixos, you know what to do. The linked page contains - some examples. +- via the [nix package manager](@/docs/install/nix.md#try-it-out) + and/or as a [NixOS module](@/docs/install/nix.md#nixos) through a + flake. If you use nix/nixos, you know what to do. The linked page + contains some examples. - [Unraid](https://www.unraid.net/): There are user provided [notes and unraid templates](https://github.com/vakilando/unraid-docker-templates)