From 2e182748032cbd36d2360a1fb19a9e58e0b16bf3 Mon Sep 17 00:00:00 2001 From: eikek Date: Sat, 9 Mar 2024 01:28:44 +0100 Subject: [PATCH 1/6] Extend nix flake setup --- flake.lock | 130 +++++++++ flake.nix | 164 +++++++++++ nix/checks/configuration-test.nix | 31 +-- nix/checks/default.nix | 3 +- nix/dev-vm/default.nix | 28 -- nix/docker.nix | 79 ++++++ nix/flake.lock | 26 -- nix/flake.nix | 134 --------- nix/modules/joex.nix | 329 +++++++++++------------ nix/modules/server.nix | 155 ++++++----- nix/pkg.nix | 57 ++++ nix/test-vm.nix | 77 ++++++ project/build.nix | 44 +-- website/shell.nix | 25 +- website/site/content/docs/install/nix.md | 33 ++- 15 files changed, 824 insertions(+), 491 deletions(-) create mode 100644 flake.lock create mode 100644 flake.nix delete mode 100644 nix/dev-vm/default.nix create mode 100644 nix/docker.nix delete mode 100644 nix/flake.lock delete mode 100644 nix/flake.nix create mode 100644 nix/pkg.nix create mode 100644 nix/test-vm.nix diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..d2dfeb62 --- /dev/null +++ b/flake.lock @@ -0,0 +1,130 @@ +{ + "nodes": { + "devshell-tools": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + }, + "locked": { + "lastModified": 1709936258, + "narHash": "sha256-ziYmDU/5v++oYSSwyMqEOr2V75rO+dMQA5aEdwH8amw=", + "owner": "eikek", + "repo": "devshell-tools", + "rev": "59900a3731a88508257525754a704ad2f8a3278e", + "type": "github" + }, + "original": { + "owner": "eikek", + "repo": "devshell-tools", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1709126324, + "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "d465f4819400de7c8d874d50b982301f28a84605", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_2": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1709126324, + "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "d465f4819400de7c8d874d50b982301f28a84605", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1709309926, + "narHash": "sha256-VZFBtXGVD9LWTecGi6eXrE0hJ/mVB3zGUlHImUs2Qak=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "79baff8812a0d68e24a836df0a364c678089e2c7", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1709677081, + "narHash": "sha256-tix36Y7u0rkn6mTm0lA45b45oab2cFLqAzDbJxeXS+c=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "880992dcc006a5e00dd0591446fdf723e6a51a64", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "devshell-tools": "devshell-tools", + "flake-utils": "flake-utils_2", + "nixpkgs": "nixpkgs_2" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..af1469a6 --- /dev/null +++ b/flake.nix @@ -0,0 +1,164 @@ +{ + description = "Docspell"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11"; + devshell-tools.url = "github:eikek/devshell-tools"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { + self, + nixpkgs, + flake-utils, + devshell-tools, + }: + flake-utils.lib.eachDefaultSystem (system: let + pkgs = nixpkgs.legacyPackages.${system}; + devshellPkgs = with pkgs; [ + jq + scala-cli + sbt + + netcat + jdk17 + wget + which + dpkg + elmPackages.elm + fakeroot + zola + yarn + ]; + docspellPkgs = pkgs.callPackage (import ./nix/pkg.nix) {}; + dockerAmd64 = pkgs.pkgsCross.gnu64.callPackage (import ./nix/docker.nix) { + inherit (docspellPkgs) docspell-server docspell-joex; + }; + dockerArm64 = pkgs.pkgsCross.aarch64-multiplatform.callPackage (import ./nix/docker.nix) { + inherit (docspellPkgs) docspell-server docspell-joex; + }; + in { + formatter = pkgs.alejandra; + + packages = { + inherit (docspellPkgs) docspell-server docspell-joex; + }; + + legacyPackages = { + docker = { + amd64 = { + inherit (dockerAmd64) docspell-server docspell-joex; + }; + arm64 = { + inherit (dockerArm64) docspell-server docspell-joex; + }; + }; + }; + + checks = { + build-server = self.packages.${system}.docspell-server; + build-joex = self.packages.${system}.docspell-joex; + + test = with import (nixpkgs + "/nixos/lib/testing-python.nix") + { + inherit system; + }; + makeTest { + name = "docspell"; + nodes = { + machine = {...}: { + nixpkgs.overlays = [self.overlays.default]; + imports = [ + self.nixosModules.default + ./nix/checks + ]; + }; + }; + + testScript = builtins.readFile ./nix/checks/testScript.py; + }; + }; + + devShells = { + dev-cnt = pkgs.mkShellNoCC { + buildInputs = + (builtins.attrValues devshell-tools.legacyPackages.${system}.cnt-scripts) + ++ devshellPkgs; + + DEV_CONTAINER = "docsp-dev"; + SBT_OPTS = "-Xmx2G -Xss4m"; + }; + dev-vm = pkgs.mkShellNoCC { + buildInputs = + (builtins.attrValues devshell-tools.legacyPackages.${system}.vm-scripts) + ++ devshellPkgs; + + SBT_OPTS = "-Xmx2G -Xss4m"; + DEV_VM = "dev-vm"; + VM_SSH_PORT = "10022"; + }; + }; + }) + // { + nixosModules = { + default = {...}: { + imports = [ + ./nix/modules/server.nix + ./nix/modules/joex.nix + ]; + }; + server = import ./nix/modules/server.nix; + joex = import ./nix/modules/joex.nix; + }; + + overlays.default = final: prev: let + docspellPkgs = final.callPackage (import ./nix/pkg.nix) {}; + in { + inherit (docspellPkgs) docspell-server docspell-joex; + }; + + nixosConfigurations = { + test-vm = devshell-tools.lib.mkVm { + system = "x86_64-linux"; + modules = [ + self.nixosModules.default + { + nixpkgs.overlays = [self.overlays.default]; + } + ./nix/test-vm.nix + ]; + }; + docsp-dev = devshell-tools.lib.mkContainer { + system = "x86_64-linux"; + modules = [ + { + services.dev-postgres.enable = true; + services.dev-email.enable = true; + services.dev-minio.enable = true; + services.dev-solr.enable = true; + } + ]; + }; + dev-vm = devshell-tools.lib.mkVm { + system = "x86_64-linux"; + modules = [ + { + services.dev-postgres.enable = true; + services.dev-email.enable = true; + services.dev-minio.enable = true; + services.dev-solr.enable = true; + port-forward.ssh = 10022; + port-forward.dev-postgres = 6534; + port-forward.dev-smtp = 10025; + port-forward.dev-imap = 10143; + port-forward.dev-webmail = 8080; + port-forward.dev-minio-api = 9000; + port-forward.dev-minio-console = 9001; + port-forward.dev-solr = 8983; + networking.hostName = "dev-vm"; + } + ]; + }; + }; + }; +} diff --git a/nix/checks/configuration-test.nix b/nix/checks/configuration-test.nix index 84ccb577..ac1ec897 100644 --- a/nix/checks/configuration-test.nix +++ b/nix/checks/configuration-test.nix @@ -1,5 +1,8 @@ -{ config, pkgs, ... }: -let +{ + config, + pkgs, + ... +}: let full-text-search = { enabled = true; backend = "postgresql"; @@ -9,9 +12,7 @@ let }; }; }; -in -{ - +in { i18n = { defaultLocale = "en_US.UTF-8"; }; @@ -21,12 +22,11 @@ in password = "root"; }; - services.docspell-joex = { enable = true; bind.address = "0.0.0.0"; base-url = "http://localhost:7878"; - jvmArgs = [ "-J-Xmx1536M" ]; + jvmArgs = ["-J-Xmx1536M"]; inherit full-text-search; }; services.docspell-restserver = { @@ -69,14 +69,12 @@ in }; }; - environment.systemPackages = - [ - pkgs.jq - pkgs.inetutils - pkgs.htop - pkgs.jdk17 - ]; - + environment.systemPackages = [ + pkgs.jq + pkgs.inetutils + pkgs.htop + pkgs.jdk17 + ]; services.xserver = { enable = false; @@ -84,12 +82,11 @@ in networking = { hostName = "docspelltest"; - firewall.allowedTCPPorts = [ 7880 ]; + firewall.allowedTCPPorts = [7880]; }; system.stateVersion = "22.11"; # This slows down the build of a vm documentation.enable = false; - } diff --git a/nix/checks/default.nix b/nix/checks/default.nix index d3eb6f37..97ae31c1 100644 --- a/nix/checks/default.nix +++ b/nix/checks/default.nix @@ -1,5 +1,4 @@ -{ ... }: -{ +{...}: { imports = [ ./configuration-test.nix ]; diff --git a/nix/dev-vm/default.nix b/nix/dev-vm/default.nix deleted file mode 100644 index 01e172a1..00000000 --- a/nix/dev-vm/default.nix +++ /dev/null @@ -1,28 +0,0 @@ -# NOTE: modulesPath and imports are taken from nixpkgs#59219 -{ modulesPath, pkgs, lib, ... }: { - imports = [ (modulesPath + "/virtualisation/qemu-vm.nix") ]; - services.openssh = { - enable = true; - permitRootLogin = "yes"; - }; - - services.docspell-restserver = { - openid = lib.mkForce [ ]; - backend = lib.mkForce { - signup = { - mode = "open"; - }; - }; - }; - - # Otherwise oomkiller kills docspell - virtualisation.memorySize = 4096; - - virtualisation.forwardPorts = [ - # SSH - { from = "host"; host.port = 64022; guest.port = 22; } - # Docspell - { from = "host"; host.port = 64080; guest.port = 7880; } - ]; - -} diff --git a/nix/docker.nix b/nix/docker.nix new file mode 100644 index 00000000..d0e69596 --- /dev/null +++ b/nix/docker.nix @@ -0,0 +1,79 @@ +{ + dockerTools, + busybox, + cacert, + wget, + unpaper, + ghostscript, + ocrmypdf, + tesseract4, + python3Packages, + unoconv, + docspell-server, + docspell-joex, +}: let + mkImage = { + name, + port, + pkg, + tools, + }: + dockerTools.buildLayeredImage { + inherit name; + created = "now"; + contents = + [ + busybox + cacert + wget + pkg + ] + ++ tools; + + extraCommands = "mkdir -m 0777 tmp"; + + #https://github.com/moby/docker-image-spec/blob/main/spec.md#image-json-description + config = { + Entrypoint = ["bin/${name}" "-Dconfig.file="]; + #Cmd = ["bin/${name}" "-Dconfig.file="]; + ExposedPorts = { + "${builtins.toString port}/tcp" = {}; + }; + Env = [ + "PATH=/bin" + "SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt" + ]; + Healthcheck = { + Test = [ + "CMD" + "wget" + "--spider" + "http://localhost:${builtins.toString port}/api/info/version" + ]; + Interval = 60000000000; #1min + Timeout = 10000000000; #10s + Retries = 2; + StartInterval = 10000000000; + }; + Labels = { + #https://github.com/microscaling/microscaling/blob/55a2d7b91ce7513e07f8b1fd91bbed8df59aed5a/Dockerfile#L22-L33 + "org.label-schema.vcs-ref" = "v${pkg.version}"; + "org.label-schema.vcs-url" = "https://github.com/eikek/docspell"; + }; + }; + tag = "v${pkg.version}"; + }; +in { + docspell-server = mkImage { + name = "docspell-restserver"; + port = 7880; + pkg = docspell-server; + tools = []; + }; + docspell-joex = mkImage { + name = "docspell-joex"; + port = 7878; + pkg = docspell-joex; + tools = [unpaper ghostscript ocrmypdf tesseract4 python3Packages.weasyprint unoconv]; + }; +} diff --git a/nix/flake.lock b/nix/flake.lock deleted file mode 100644 index 61eca2df..00000000 --- a/nix/flake.lock +++ /dev/null @@ -1,26 +0,0 @@ -{ - "nodes": { - "nixpkgs": { - "locked": { - "lastModified": 1706373441, - "narHash": "sha256-S1hbgNbVYhuY2L05OANWqmRzj4cElcbLuIkXTb69xkk=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "56911ef3403a9318b7621ce745f5452fb9ef6867", - "type": "github" - }, - "original": { - "id": "nixpkgs", - "ref": "nixos-23.11", - "type": "indirect" - } - }, - "root": { - "inputs": { - "nixpkgs": "nixpkgs" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/nix/flake.nix b/nix/flake.nix deleted file mode 100644 index a8cb0785..00000000 --- a/nix/flake.nix +++ /dev/null @@ -1,134 +0,0 @@ -{ - description = "Docspell flake"; - inputs = { - nixpkgs.url = "nixpkgs/nixos-23.11"; - }; - - outputs = { self, nixpkgs }: - let - supportedSystems = [ "x86_64-linux" "aarch64-linux" ]; - forAllSystems = nixpkgs.lib.genAttrs supportedSystems; - nixpkgsFor = forAllSystems (system: import nixpkgs { inherit system; }); - # Version config - cfg = { - v0_41_0 = rec { - version = "0.41.0"; - server = { - url = "https://github.com/eikek/docspell/releases/download/v${version}/docspell-restserver-${version}.zip"; - sha256 = "sha256-JFftIzI94UNLLh96I++qFsBZhOkquPIPhNhtS2Ov8wI="; - }; - joex = { - url = "https://github.com/eikek/docspell/releases/download/v${version}/docspell-joex-${version}.zip"; - sha256 = "sha256-flKWjEsMd2/XT3Bu6EjFgf3lCojvLbKFDEXemP1K+/8="; - }; - }; - }; - current_version = cfg.v0_41_0; - inherit (current_version) version; - in - rec - { - overlays.default = final: prev: { - docspell-server = with final; stdenv.mkDerivation { - inherit version; - pname = "docspell-server"; - - src = fetchzip current_version.server; - buildInputs = [ jdk17 ]; - buildPhase = "true"; - - installPhase = '' - mkdir -p $out/{bin,docspell-restserver-${version}} - cp -R * $out/docspell-restserver-${version}/ - cat > $out/bin/docspell-restserver <<-EOF - #!${bash}/bin/bash - $out/docspell-restserver-${version}/bin/docspell-restserver -java-home ${jdk17} "\$@" - EOF - chmod 755 $out/bin/docspell-restserver - ''; - }; - docspell-joex = with final; stdenv.mkDerivation rec { - inherit version; - - pname = "docspell-joex"; - - src = fetchzip current_version.joex; - - buildInputs = [ jdk17 ]; - - buildPhase = "true"; - - installPhase = '' - mkdir -p $out/{bin,docspell-joex-${version}} - cp -R * $out/docspell-joex-${version}/ - cat > $out/bin/docspell-joex <<-EOF - #!${bash}/bin/bash - $out/docspell-joex-${version}/bin/docspell-joex -java-home ${jdk17} "\$@" - EOF - chmod 755 $out/bin/docspell-joex - ''; - }; - - }; - - packages = forAllSystems (system: - { - default = (import nixpkgs { - inherit system; - overlays = [ self.overlays.default ]; - }).docspell-joex; - }); - - checks = forAllSystems - (system: { - build = self.packages.${system}.default; - - test = - with import (nixpkgs + "/nixos/lib/testing-python.nix") - { - inherit system; - }; - - makeTest { - name = "docspell"; - nodes = { - machine = { ... }: { - imports = [ - self.nixosModules.default - ./checks - ]; - }; - }; - - testScript = builtins.readFile ./checks/testScript.py; - }; - }); - - nixosModules = { - default = { ... }: { - imports = [ - ((import ./modules/server.nix) self.overlays.default) - ((import ./modules/joex.nix) self.overlays.default) - ]; - }; - server = ((import ./modules/server.nix) self.overlays.default); - joex = ((import ./modules/joex.nix) self.overlays.default); - }; - - nixosConfigurations = - let - lib = nixpkgs.lib; - in - { - dev-vm = lib.makeOverridable nixpkgs.lib.nixosSystem { - system = "x86_64-linux"; - modules = [ - self.nixosModules.default - ./checks - # nixos-shell specific module. Should be kept outside nix flake checks - ./dev-vm - ]; - }; - }; - }; -} diff --git a/nix/modules/joex.nix b/nix/modules/joex.nix index 7890d690..5e893be6 100644 --- a/nix/modules/joex.nix +++ b/nix/modules/joex.nix @@ -1,11 +1,17 @@ -overlay: { config, lib, pkgs, ... }: - -with lib; -let +{ + config, + lib, + pkgs, + ... +}: +with lib; let cfg = config.services.docspell-joex; # Extract the config without the extraConfig attribute. It will be merged later declared_config = attrsets.filterAttrs (n: v: n != "extraConfig") cfg; - user = if cfg.runAs == null then "docspell" else cfg.runAs; + user = + if cfg.runAs == null + then "docspell" + else cfg.runAs; configFile = pkgs.writeText "docspell-joex.conf" '' {"docspell": { "joex": ${builtins.toJSON (lib.recursiveUpdate declared_config cfg.extraConfig)} @@ -85,7 +91,7 @@ let schedule = "Sun *-*-* 00:00:00 UTC"; sender-account = ""; smtp-id = ""; - recipients = [ ]; + recipients = []; subject = "Docspell {{ latestVersion }} is available"; body = '' Hello, @@ -116,21 +122,21 @@ let working-dir = "/tmp/docspell-extraction"; command = { program = "${pkgs.ghostscript}/bin/gs"; - args = [ "-dNOPAUSE" "-dBATCH" "-dSAFER" "-sDEVICE=tiffscaled8" "-sOutputFile={{outfile}}" "{{infile}}" ]; + args = ["-dNOPAUSE" "-dBATCH" "-dSAFER" "-sDEVICE=tiffscaled8" "-sOutputFile={{outfile}}" "{{infile}}"]; timeout = "5 minutes"; }; }; unpaper = { command = { program = "${pkgs.unpaper}/bin/unpaper"; - args = [ "{{infile}}" "{{outfile}}" ]; + args = ["{{infile}}" "{{outfile}}"]; timeout = "5 minutes"; }; }; tesseract = { command = { program = "${pkgs.tesseract4}/bin/tesseract"; - args = [ "{{file}}" "stdout" "-l" "{{lang}}" ]; + args = ["{{file}}" "stdout" "-l" "{{lang}}"]; timeout = "5 minutes"; }; }; @@ -179,7 +185,7 @@ let wkhtmlpdf = { command = { program = ""; - args = [ "--encoding" "UTF-8" "-" "{{outfile}}" ]; + args = ["--encoding" "UTF-8" "-" "{{outfile}}"]; timeout = "2 minutes"; }; working-dir = "/tmp/docspell-convert"; @@ -204,7 +210,7 @@ let tesseract = { command = { program = "${pkgs.tesseract4}/bin/tesseract"; - args = [ "{{infile}}" "out" "-l" "{{lang}}" "pdf" "txt" ]; + args = ["{{infile}}" "out" "-l" "{{lang}}" "pdf" "txt"]; timeout = "5 minutes"; }; working-dir = "/tmp/docspell-convert"; @@ -213,7 +219,7 @@ let unoconv = { command = { program = "${pkgs.unoconv}/bin/unoconv"; - args = [ "-f" "pdf" "-o" "{{outfile}}" "{{infile}}" ]; + args = ["-f" "pdf" "-o" "{{outfile}}" "{{infile}}"]; timeout = "2 minutes"; }; working-dir = "/tmp/docspell-convert"; @@ -240,7 +246,7 @@ let }; files = { chunk-size = 524288; - valid-mime-types = [ ]; + valid-mime-types = []; }; full-text-search = { enabled = false; @@ -259,9 +265,9 @@ let user = "pguser"; password = ""; }; - pg-config = { }; + pg-config = {}; pg-query-parser = "websearch_to_tsquery"; - pg-rank-normalization = [ 4 ]; + pg-rank-normalization = [4]; }; migration = { index-all-chunk = 10; @@ -291,9 +297,7 @@ let }; }; }; -in -{ - +in { ## interface options = { services.docspell-joex = { @@ -321,12 +325,11 @@ in }; jvmArgs = mkOption { type = types.listOf types.str; - default = [ ]; - example = [ "-J-Xmx1G" ]; + default = []; + example = ["-J-Xmx1G"]; description = "The options passed to the executable for setting jvm arguments."; }; - app-id = mkOption { type = types.str; default = defaults.app-id; @@ -340,7 +343,7 @@ in }; bind = mkOption { - type = types.submodule ({ + type = types.submodule { options = { address = mkOption { type = types.str; @@ -353,13 +356,13 @@ in description = "The port to bind the REST server"; }; }; - }); + }; default = defaults.bind; description = "Address and port bind the rest server."; }; logging = mkOption { - type = types.submodule ({ + type = types.submodule { options = { minimum-level = mkOption { type = types.str; @@ -377,7 +380,7 @@ in description = "Set of logger and their levels"; }; }; - }); + }; default = defaults.logging; description = "Settings for logging"; }; @@ -394,7 +397,7 @@ in }; jdbc = mkOption { - type = types.submodule ({ + type = types.submodule { options = { url = mkOption { type = types.str; @@ -421,13 +424,13 @@ in description = "The password to connect to the database."; }; }; - }); + }; default = defaults.jdbc; description = "Database connection settings"; }; send-mail = mkOption { - type = types.submodule ({ + type = types.submodule { options = { list-id = mkOption { type = types.str; @@ -443,15 +446,14 @@ in https://tools.ietf.org/html/rfc2919 for a formal specification ''; }; - }; - }); + }; default = defaults.send-mail; description = "Settings for sending mails."; }; scheduler = mkOption { - type = types.submodule ({ + type = types.submodule { options = { pool-size = mkOption { type = types.int; @@ -501,13 +503,13 @@ in ''; }; }; - }); + }; default = defaults.scheduler; description = "Settings for the scheduler"; }; periodic-scheduler = mkOption { - type = types.submodule ({ + type = types.submodule { options = { wakeup-period = mkOption { type = types.str; @@ -520,7 +522,7 @@ in ''; }; }; - }); + }; default = defaults.periodic-scheduler; description = '' Settings for the periodic scheduler. @@ -528,10 +530,10 @@ in }; user-tasks = mkOption { - type = types.submodule ({ + type = types.submodule { options = { scan-mailbox = mkOption { - type = types.submodule ({ + type = types.submodule { options = { max-folders = mkOption { type = types.int; @@ -565,18 +567,18 @@ in ''; }; }; - }); + }; default = defaults.user-tasks.scan-mailbox; description = "Allows to import e-mails by scanning a mailbox."; }; }; - }); + }; default = defaults.user-tasks; description = "Configuration for the user tasks."; }; house-keeping = mkOption { - type = types.submodule ({ + type = types.submodule { options = { schedule = mkOption { type = types.str; @@ -587,7 +589,7 @@ in ''; }; cleanup-invites = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -600,7 +602,7 @@ in description = "The minimum age of invites to be deleted."; }; }; - }); + }; default = defaults.house-keeping.cleanup-invites; description = '' This task removes invitation keys that have been created but not @@ -609,7 +611,7 @@ in ''; }; cleanup-jobs = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -633,9 +635,8 @@ in whether more or less memory should be used. ''; }; - }; - }); + }; default = defaults.house-keeping.cleanup-jobs; description = '' Jobs store their log output in the database. Normally this data @@ -644,7 +645,7 @@ in ''; }; cleanup-remember-me = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -657,13 +658,13 @@ in description = "The miminum age of remember me tokens to delete."; }; }; - }); + }; default = defaults.house-keeping.cleanup-remember-me; description = "Settings for cleaning up remember me tokens."; }; cleanup-downloads = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -676,13 +677,13 @@ in description = "The miminum age of a download file to delete."; }; }; - }); + }; default = defaults.house-keeping.cleanup-downloads; description = ""; }; check-nodes = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -695,12 +696,12 @@ in description = "How often the node must be unreachable, before it is removed."; }; }; - }); + }; default = defaults.house-keeping.cleanup-nodes; description = "Removes node entries that are not reachable anymore."; }; }; - }); + }; default = defaults.house-keeping; description = '' Docspell uses periodic house keeping tasks, like cleaning expired @@ -709,7 +710,7 @@ in }; update-check = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -753,7 +754,7 @@ in recipients = mkOption { type = types.listOf types.str; default = defaults.update-check.recipients; - example = [ "josh.doe@gmail.com" ]; + example = ["josh.doe@gmail.com"]; description = '' A list of recipient e-mail addresses. ''; @@ -781,7 +782,7 @@ in ''; }; }; - }); + }; default = defaults.update-check; description = '' A periodic task to check for new releases of docspell. It can @@ -791,10 +792,10 @@ in }; extraction = mkOption { - type = types.submodule ({ + type = types.submodule { options = { pdf = mkOption { - type = types.submodule ({ + type = types.submodule { options = { min-text-len = mkOption { type = types.int; @@ -808,12 +809,12 @@ in ''; }; }; - }); + }; default = defaults.extraction.pdf; description = "Settings for PDF extraction"; }; preview = mkOption { - type = types.submodule ({ + type = types.submodule { options = { dpi = mkOption { type = types.int; @@ -830,12 +831,12 @@ in ''; }; }; - }); + }; default = defaults.extraction.preview; description = ""; }; ocr = mkOption { - type = types.submodule ({ + type = types.submodule { options = { max-image-size = mkOption { type = types.int; @@ -846,7 +847,7 @@ in ''; }; page-range = mkOption { - type = types.submodule ({ + type = types.submodule { options = { begin = mkOption { type = types.int; @@ -854,7 +855,7 @@ in description = "Specifies the first N pages of a file to process."; }; }; - }); + }; default = defaults.extraction.page-range; description = '' Defines what pages to process. If a PDF with 600 pages is @@ -871,7 +872,7 @@ in ''; }; ghostscript = mkOption { - type = types.submodule ({ + type = types.submodule { options = { working-dir = mkOption { type = types.str; @@ -879,7 +880,7 @@ in description = "Directory where the extraction processes can put their temp files"; }; command = mkOption { - type = types.submodule ({ + type = types.submodule { options = { program = mkOption { type = types.str; @@ -897,20 +898,20 @@ in description = "The timeout when executing the command"; }; }; - }); + }; default = defaults.extraction.ghostscript.command; description = "The system command"; }; }; - }); + }; default = defaults.extraction.ghostscript; description = "The ghostscript command."; }; unpaper = mkOption { - type = types.submodule ({ + type = types.submodule { options = { command = mkOption { - type = types.submodule ({ + type = types.submodule { options = { program = mkOption { type = types.str; @@ -928,20 +929,20 @@ in description = "The timeout when executing the command"; }; }; - }); + }; default = defaults.extraction.unpaper.command; description = "The system command"; }; }; - }); + }; default = defaults.extraction.unpaper; description = "The unpaper command."; }; tesseract = mkOption { - type = types.submodule ({ + type = types.submodule { options = { command = mkOption { - type = types.submodule ({ + type = types.submodule { options = { program = mkOption { type = types.str; @@ -959,23 +960,22 @@ in description = "The timeout when executing the command"; }; }; - }); + }; default = defaults.extraction.tesseract.command; description = "The system command"; }; }; - }); + }; default = defaults.extraction.tesseract; description = "The tesseract command."; }; - }; - }); + }; default = defaults.extraction.ocr; description = ""; }; }; - }); + }; default = defaults.extraction; description = '' Configuration of text extraction @@ -990,7 +990,7 @@ in }; text-analysis = mkOption { - type = types.submodule ({ + type = types.submodule { options = { max-length = mkOption { type = types.int; @@ -1015,7 +1015,7 @@ in }; nlp = mkOption { - type = types.submodule ({ + type = types.submodule { options = { mode = mkOption { type = types.str; @@ -1073,7 +1073,7 @@ in }; regex-ner = mkOption { - type = types.submodule ({ + type = types.submodule { options = { max-entries = mkOption { type = types.int; @@ -1104,18 +1104,18 @@ in ''; }; }; - }); + }; default = defaults.text-analysis.nlp.regex-ner; description = ""; }; }; - }); + }; default = defaults.text-analysis.nlp; description = "Configure NLP"; }; classification = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -1147,9 +1147,8 @@ in good results with *my* dataset. ''; }; - }; - }); + }; default = defaults.text-analysis.classification; description = '' Settings for doing document classification. @@ -1167,13 +1166,13 @@ in ''; }; }; - }); + }; default = defaults.text-analysis; description = "Settings for text analysis"; }; convert = mkOption { - type = types.submodule ({ + type = types.submodule { options = { chunk-size = mkOption { type = types.int; @@ -1202,7 +1201,7 @@ in ''; }; markdown = mkOption { - type = types.submodule ({ + type = types.submodule { options = { internal-css = mkOption { type = types.str; @@ -1212,7 +1211,7 @@ in ''; }; }; - }); + }; default = defaults.convert.markdown; description = '' Settings when processing markdown files (and other text files) @@ -1224,12 +1223,12 @@ in ''; }; html-converter = mkOption { - type = types.enum [ "wkhtmlpdf" "weasyprint" ]; + type = types.enum ["wkhtmlpdf" "weasyprint"]; default = "weasyprint"; description = "Which tool to use for converting html to pdfs"; }; wkhtmlpdf = mkOption { - type = types.submodule ({ + type = types.submodule { options = { working-dir = mkOption { type = types.str; @@ -1237,7 +1236,7 @@ in description = "Directory where the conversion processes can put their temp files"; }; command = mkOption { - type = types.submodule ({ + type = types.submodule { options = { program = mkOption { type = types.str; @@ -1255,12 +1254,12 @@ in description = "The timeout when executing the command"; }; }; - }); + }; default = defaults.convert.wkhtmlpdf.command; description = "The system command"; }; }; - }); + }; default = defaults.convert.wkhtmlpdf; description = '' To convert HTML files into PDF files, the external tool @@ -1268,7 +1267,7 @@ in ''; }; weasyprint = mkOption { - type = types.submodule ({ + type = types.submodule { options = { working-dir = mkOption { type = types.str; @@ -1276,7 +1275,7 @@ in description = "Directory where the conversion processes can put their temp files"; }; command = mkOption { - type = types.submodule ({ + type = types.submodule { options = { program = mkOption { type = types.str; @@ -1294,12 +1293,12 @@ in description = "The timeout when executing the command"; }; }; - }); + }; default = defaults.convert.weasyprint.command; description = "The system command"; }; }; - }); + }; default = defaults.convert.weasyprint; description = '' To convert HTML files into PDF files, the external tool @@ -1307,7 +1306,7 @@ in ''; }; tesseract = mkOption { - type = types.submodule ({ + type = types.submodule { options = { working-dir = mkOption { type = types.str; @@ -1315,7 +1314,7 @@ in description = "Directory where the conversion processes can put their temp files"; }; command = mkOption { - type = types.submodule ({ + type = types.submodule { options = { program = mkOption { type = types.str; @@ -1333,12 +1332,12 @@ in description = "The timeout when executing the command"; }; }; - }); + }; default = defaults.convert.tesseract.command; description = "The system command"; }; }; - }); + }; default = defaults.convert.tesseract; description = '' To convert image files to PDF files, tesseract is used. This @@ -1346,7 +1345,7 @@ in ''; }; unoconv = mkOption { - type = types.submodule ({ + type = types.submodule { options = { working-dir = mkOption { type = types.str; @@ -1354,7 +1353,7 @@ in description = "Directory where the conversion processes can put their temp files"; }; command = mkOption { - type = types.submodule ({ + type = types.submodule { options = { program = mkOption { type = types.str; @@ -1372,12 +1371,12 @@ in description = "The timeout when executing the command"; }; }; - }); + }; default = defaults.convert.unoconv.command; description = "The system command"; }; }; - }); + }; default = defaults.convert.unoconv; description = '' To convert "office" files to PDF files, the external tool @@ -1392,7 +1391,7 @@ in }; ocrmypdf = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -1405,7 +1404,7 @@ in description = "Directory where the conversion processes can put their temp files"; }; command = mkOption { - type = types.submodule ({ + type = types.submodule { options = { program = mkOption { type = types.str; @@ -1423,12 +1422,12 @@ in description = "The timeout when executing the command"; }; }; - }); + }; default = defaults.convert.ocrmypdf.command; description = "The system command"; }; }; - }); + }; default = defaults.convert.ocrmypdf; description = '' The tool ocrmypdf can be used to convert pdf files to pdf files @@ -1449,9 +1448,8 @@ in converted to PDF/A. ''; }; - }; - }); + }; default = defaults.convert; description = '' Configuration for converting files into PDFs. @@ -1462,7 +1460,7 @@ in ''; }; files = mkOption { - type = types.submodule ({ + type = types.submodule { options = { chunk-size = mkOption { type = types.int; @@ -1489,12 +1487,12 @@ in ''; }; }; - }); + }; default = defaults.files; description = "Settings for how files are stored."; }; full-text-search = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -1514,7 +1512,7 @@ in }; solr = mkOption { - type = types.submodule ({ + type = types.submodule { options = { url = mkOption { type = types.str; @@ -1546,13 +1544,13 @@ in description = "The default combiner for tokens. One of {AND, OR}."; }; }; - }); + }; default = defaults.full-text-search.solr; description = "Configuration for the SOLR backend."; }; postgresql = mkOption { - type = types.submodule ({ + type = types.submodule { options = { use-default-connection = mkOption { type = types.bool; @@ -1560,7 +1558,7 @@ in description = "Whether to use the primary db connection."; }; jdbc = mkOption { - type = types.submodule ({ + type = types.submodule { options = { url = mkOption { type = types.str; @@ -1580,7 +1578,7 @@ in description = "The password to connect to the database."; }; }; - }); + }; default = defaults.full-text-search.postgresql.jdbc; description = "Database connection settings"; }; @@ -1600,13 +1598,13 @@ in description = ""; }; }; - }); + }; default = defaults.full-text-search.postgresql; description = "PostgreSQL for fulltext search"; }; migration = mkOption { - type = types.submodule ({ + type = types.submodule { options = { index-all-chunk = mkOption { type = types.int; @@ -1618,17 +1616,17 @@ in ''; }; }; - }); + }; default = defaults.full-text-search.migration; description = "Settings for running the index migration tasks"; }; }; - }); + }; default = defaults.full-text-search; description = "Configuration for full-text search."; }; addons = mkOption { - type = types.submodule ({ + type = types.submodule { options = { working-dir = mkOption { type = types.str; @@ -1641,7 +1639,7 @@ in description = "Cache directory"; }; executor-config = mkOption { - type = types.submodule ({ + type = types.submodule { options = { runner = mkOption { type = types.str; @@ -1659,7 +1657,7 @@ in description = ""; }; nspawn = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -1682,12 +1680,12 @@ in description = ""; }; }; - }); + }; default = defaults.addons.executor-config.nspawn; description = ""; }; nix-runner = mkOption { - type = types.submodule ({ + type = types.submodule { options = { nix-binary = mkOption { type = types.str; @@ -1700,12 +1698,12 @@ in description = ""; }; }; - }); + }; default = defaults.addons.executor-config.nix-runner; description = ""; }; docker-runner = mkOption { - type = types.submodule ({ + type = types.submodule { options = { docker-binary = mkOption { type = types.str; @@ -1718,24 +1716,24 @@ in description = ""; }; }; - }); + }; default = defaults.addons.executor-config.docker-runner; description = ""; }; }; - }); + }; default = defaults.addons.executor-config; description = ""; }; }; - }); + }; default = defaults.addons; description = "Addon executor config"; }; extraConfig = mkOption { type = types.attrs; description = "Extra configuration for docspell server. Overwrites values in case of a conflict."; - default = { }; + default = {}; example = '' { files = { @@ -1754,9 +1752,6 @@ in ## implementation config = mkIf config.services.docspell-joex.enable { - - nixpkgs.overlays = [ overlay ]; - users.users."${user}" = mkIf (cfg.runAs == null) { name = user; isSystemUser = true; @@ -1765,43 +1760,35 @@ in description = "Docspell user"; group = user; }; - users.groups."${user}" = mkIf (cfg.runAs == null) { }; + users.groups."${user}" = mkIf (cfg.runAs == null) {}; # Setting up a unoconv listener to improve conversion performance - systemd.services.unoconv = - let - cmd = "${pkgs.unoconv}/bin/unoconv --listener -v"; - in - { - description = "Unoconv Listener"; - after = [ "networking.target" ]; - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - Restart = "always"; - }; - script = - "${pkgs.su}/bin/su -s ${pkgs.bash}/bin/sh ${user} -c \"${cmd}\""; + systemd.services.unoconv = let + cmd = "${pkgs.unoconv}/bin/unoconv --listener -v"; + in { + description = "Unoconv Listener"; + after = ["networking.target"]; + wantedBy = ["multi-user.target"]; + serviceConfig = { + Restart = "always"; }; + script = "${pkgs.su}/bin/su -s ${pkgs.bash}/bin/sh ${user} -c \"${cmd}\""; + }; - systemd.services.docspell-joex = - let - args = builtins.concatStringsSep " " cfg.jvmArgs; - cmd = "${pkgs.docspell-joex}/bin/docspell-joex ${args} -- ${configFile}"; - waitTarget = - if cfg.waitForTarget != null - then - [ cfg.waitForTarget ] - else - [ ]; - in - { - description = "Docspell Joex"; - after = ([ "networking.target" ] ++ waitTarget); - wantedBy = [ "multi-user.target" ]; - path = [ pkgs.gawk ]; + systemd.services.docspell-joex = let + args = builtins.concatStringsSep " " cfg.jvmArgs; + cmd = "${pkgs.docspell-joex}/bin/docspell-joex ${args} -- ${configFile}"; + waitTarget = + if cfg.waitForTarget != null + then [cfg.waitForTarget] + else []; + in { + description = "Docspell Joex"; + after = ["networking.target"] ++ waitTarget; + wantedBy = ["multi-user.target"]; + path = [pkgs.gawk]; - script = - "${pkgs.su}/bin/su -s ${pkgs.bash}/bin/sh ${user} -c \"${cmd}\""; - }; + script = "${pkgs.su}/bin/su -s ${pkgs.bash}/bin/sh ${user} -c \"${cmd}\""; + }; }; } diff --git a/nix/modules/server.nix b/nix/modules/server.nix index 4a98c8dc..925c2352 100644 --- a/nix/modules/server.nix +++ b/nix/modules/server.nix @@ -1,11 +1,17 @@ -overlay: { config, lib, pkgs, ... }: - -with lib; -let +{ + config, + lib, + pkgs, + ... +}: +with lib; let cfg = config.services.docspell-restserver; # Extract the config without the extraConfig attribute. It will be merged later declared_config = attrsets.filterAttrs (n: v: n != "extraConfig") cfg; - user = if cfg.runAs == null then "docspell" else cfg.runAs; + user = + if cfg.runAs == null + then "docspell" + else cfg.runAs; configFile = pkgs.writeText "docspell-server.conf" '' {"docspell": {"server": ${builtins.toJSON (lib.recursiveUpdate declared_config cfg.extraConfig)} @@ -44,7 +50,7 @@ let source-name = "integration"; allowed-ips = { enabled = false; - ips = [ "127.0.0.1" ]; + ips = ["127.0.0.1"]; }; http-basic = { enabled = false; @@ -78,9 +84,9 @@ let user = "pguser"; password = ""; }; - pg-config = { }; + pg-config = {}; pg-query-parser = "websearch_to_tsquery"; - pg-rank-normalization = [ 4 ]; + pg-rank-normalization = [4]; }; }; auth = { @@ -126,19 +132,17 @@ let }; files = { chunk-size = 524288; - valid-mime-types = [ ]; + valid-mime-types = []; }; addons = { enabled = false; allow-impure = true; - allowed-urls = [ "*" ]; - denied-urls = [ ]; + allowed-urls = ["*"]; + denied-urls = []; }; }; }; -in -{ - +in { ## interface options = { services.docspell-restserver = { @@ -156,12 +160,11 @@ in }; jvmArgs = mkOption { type = types.listOf types.str; - default = [ ]; - example = [ "-J-Xmx1G" ]; + default = []; + example = ["-J-Xmx1G"]; description = "The options passed to the executable for setting jvm arguments."; }; - app-name = mkOption { type = types.str; default = defaults.app-name; @@ -232,7 +235,7 @@ in }; bind = mkOption { - type = types.submodule ({ + type = types.submodule { options = { address = mkOption { type = types.str; @@ -245,13 +248,13 @@ in description = "The port to bind the REST server"; }; }; - }); + }; default = defaults.bind; description = "Address and port bind the rest server."; }; server-options = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enable-http-2 = mkOption { type = types.bool; @@ -269,13 +272,13 @@ in description = "Timeout when waiting for the response."; }; }; - }); + }; default = defaults.server-options; description = "Tuning the http server"; }; logging = mkOption { - type = types.submodule ({ + type = types.submodule { options = { minimum-level = mkOption { type = types.str; @@ -293,13 +296,13 @@ in description = "Set of logger and their levels"; }; }; - }); + }; default = defaults.logging; description = "Settings for logging"; }; auth = mkOption { - type = types.submodule ({ + type = types.submodule { options = { server-secret = mkOption { type = types.str; @@ -320,7 +323,7 @@ in ''; }; remember-me = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -333,18 +336,18 @@ in description = "The time a remember-me token is valid."; }; }; - }); + }; default = defaults.auth.remember-me; description = "Settings for Remember-Me"; }; }; - }); + }; default = defaults.auth; description = "Authentication"; }; download-all = mkOption { - type = types.submodule ({ + type = types.submodule { options = { max-files = mkOption { type = types.int; @@ -357,7 +360,7 @@ in description = "The maximum (uncompressed) size of the zip file contents."; }; }; - }); + }; default = defaults.download-all; description = ""; }; @@ -387,7 +390,7 @@ in description = "How to retrieve the collective name."; }; provider = mkOption { - type = (types.submodule { + type = types.submodule { options = { provider-id = mkOption { type = types.str; @@ -436,18 +439,18 @@ in description = "The expected algorithm used to sign the token."; }; }; - }); + }; default = defaults.openid.provider; description = "The config for an OpenID Connect provider."; }; }; }); - default = [ ]; + default = []; description = "A list of OIDC provider configurations."; }; integration-endpoint = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -467,7 +470,7 @@ in ''; }; allowed-ips = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -480,7 +483,7 @@ in description = "The ips/ip patterns to allow"; }; }; - }); + }; default = defaults.integration-endpoint.allowed-ips; description = '' IPv4 addresses to allow access. An empty list, if enabled, @@ -491,7 +494,7 @@ in ''; }; http-basic = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -514,14 +517,14 @@ in description = "The password to check."; }; }; - }); + }; default = defaults.integration-endpoint.http-basic; description = '' Requests are expected to use http basic auth when uploading files. ''; }; http-header = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -539,7 +542,7 @@ in description = "The value of the header to check."; }; }; - }); + }; default = defaults.integration-endpoint.http-header; description = '' Requests are expected to supply some specific header when @@ -547,7 +550,7 @@ in ''; }; }; - }); + }; default = defaults.integration-endpoint; description = '' This endpoint allows to upload files to any collective. The @@ -566,7 +569,7 @@ in }; admin-endpoint = mkOption { - type = types.submodule ({ + type = types.submodule { options = { secret = mkOption { type = types.str; @@ -574,13 +577,13 @@ in description = "The secret used to call admin endpoints."; }; }; - }); + }; default = defaults.admin-endpoint; description = "An endpoint for administration tasks."; }; full-text-search = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -597,7 +600,7 @@ in description = "The backend to use, either solr or postgresql"; }; solr = mkOption { - type = types.submodule ({ + type = types.submodule { options = { url = mkOption { type = types.str; @@ -629,13 +632,13 @@ in description = "The default combiner for tokens. One of {AND, OR}."; }; }; - }); + }; default = defaults.full-text-search.solr; description = "Configuration for the SOLR backend."; }; postgresql = mkOption { - type = types.submodule ({ + type = types.submodule { options = { use-default-connection = mkOption { type = types.bool; @@ -643,7 +646,7 @@ in description = "Whether to use the primary db connection."; }; jdbc = mkOption { - type = types.submodule ({ + type = types.submodule { options = { url = mkOption { type = types.str; @@ -663,7 +666,7 @@ in description = "The password to connect to the database."; }; }; - }); + }; default = defaults.full-text-search.postgresql.jdbc; description = "Database connection settings"; }; @@ -683,18 +686,18 @@ in description = ""; }; }; - }); + }; default = defaults.full-text-search.postgresql; description = "PostgreSQL for fulltext search"; }; }; - }); + }; default = defaults.full-text-search; description = "Configuration for full-text search."; }; backend = mkOption { - type = types.submodule ({ + type = types.submodule { options = { mail-debug = mkOption { type = types.bool; @@ -707,7 +710,7 @@ in ''; }; jdbc = mkOption { - type = types.submodule ({ + type = types.submodule { options = { url = mkOption { type = types.str; @@ -734,12 +737,12 @@ in description = "The password to connect to the database."; }; }; - }); + }; default = defaults.backend.jdbc; description = "Database connection settings"; }; signup = mkOption { - type = types.submodule ({ + type = types.submodule { options = { mode = mkOption { type = types.str; @@ -772,12 +775,12 @@ in ''; }; }; - }); + }; default = defaults.backend.signup; description = "Registration settings"; }; files = mkOption { - type = types.submodule ({ + type = types.submodule { options = { chunk-size = mkOption { type = types.int; @@ -804,12 +807,12 @@ in ''; }; }; - }); + }; default = defaults.backend.files; description = "Settings for how files are stored."; }; addons = mkOption { - type = types.submodule ({ + type = types.submodule { options = { enabled = mkOption { type = types.bool; @@ -832,19 +835,19 @@ in description = "Url patterns to deny to install"; }; }; - }); + }; default = defaults.backend.addons; description = "Addon config"; }; }; - }); + }; default = defaults.backend; description = "Configuration for the backend"; }; extraConfig = mkOption { type = types.attrs; description = "Extra configuration for docspell server. Overwrites values in case of a conflict."; - default = { }; + default = {}; example = '' { files = { @@ -863,8 +866,6 @@ in ## implementation config = mkIf config.services.docspell-restserver.enable { - - nixpkgs.overlays = [ overlay ]; users.users."${user}" = mkIf (cfg.runAs == null) { name = user; isSystemUser = true; @@ -873,24 +874,20 @@ in description = "Docspell user"; group = user; }; - users.groups."${user}" = mkIf (cfg.runAs == null) { }; + users.groups."${user}" = mkIf (cfg.runAs == null) {}; - - systemd.services.docspell-restserver = - let - args = builtins.concatStringsSep " " cfg.jvmArgs; - cmd = "${pkgs.docspell-server}/bin/docspell-restserver ${args} -- ${configFile}"; - in - { - description = "Docspell Rest Server"; - after = [ "networking.target" ]; - wantedBy = [ "multi-user.target" ]; - path = [ pkgs.gawk ]; - preStart = '' + systemd.services.docspell-restserver = let + args = builtins.concatStringsSep " " cfg.jvmArgs; + cmd = "${pkgs.docspell-server}/bin/docspell-restserver ${args} -- ${configFile}"; + in { + description = "Docspell Rest Server"; + after = ["networking.target"]; + wantedBy = ["multi-user.target"]; + path = [pkgs.gawk]; + preStart = '' ''; - script = - "${pkgs.su}/bin/su -s ${pkgs.bash}/bin/sh ${user} -c \"${cmd}\""; - }; + script = "${pkgs.su}/bin/su -s ${pkgs.bash}/bin/sh ${user} -c \"${cmd}\""; + }; }; } diff --git a/nix/pkg.nix b/nix/pkg.nix new file mode 100644 index 00000000..1b5733ba --- /dev/null +++ b/nix/pkg.nix @@ -0,0 +1,57 @@ +{ + stdenv, + bash, + fetchzip, + jdk17, +}: let + version = "0.41.0"; + server = { + url = "https://github.com/eikek/docspell/releases/download/v${version}/docspell-restserver-${version}.zip"; + sha256 = "sha256-JFftIzI94UNLLh96I++qFsBZhOkquPIPhNhtS2Ov8wI="; + }; + joex = { + url = "https://github.com/eikek/docspell/releases/download/v${version}/docspell-joex-${version}.zip"; + sha256 = "sha256-flKWjEsMd2/XT3Bu6EjFgf3lCojvLbKFDEXemP1K+/8="; + }; +in { + docspell-server = stdenv.mkDerivation { + inherit version; + pname = "docspell-server"; + + src = fetchzip server; + buildInputs = [jdk17]; + buildPhase = "true"; + + installPhase = '' + mkdir -p $out/{bin,docspell-restserver-${version}} + cp -R * $out/docspell-restserver-${version}/ + cat > $out/bin/docspell-restserver <<-EOF + #!${bash}/bin/bash + $out/docspell-restserver-${version}/bin/docspell-restserver -java-home ${jdk17} "\$@" + EOF + chmod 755 $out/bin/docspell-restserver + ''; + }; + + docspell-joex = stdenv.mkDerivation rec { + inherit version; + + pname = "docspell-joex"; + + src = fetchzip joex; + + buildInputs = [jdk17]; + + buildPhase = "true"; + + installPhase = '' + mkdir -p $out/{bin,docspell-joex-${version}} + cp -R * $out/docspell-joex-${version}/ + cat > $out/bin/docspell-joex <<-EOF + #!${bash}/bin/bash + $out/docspell-joex-${version}/bin/docspell-joex -java-home ${jdk17} "\$@" + EOF + chmod 755 $out/bin/docspell-joex + ''; + }; +} diff --git a/nix/test-vm.nix b/nix/test-vm.nix new file mode 100644 index 00000000..673b26de --- /dev/null +++ b/nix/test-vm.nix @@ -0,0 +1,77 @@ +{ + config, + pkgs, + ... +}: let + full-text-search = { + enabled = true; + backend = "solr"; + solr.url = "http://localhost:8983/solr/docspell"; + }; + jdbc = { + url = "jdbc:postgresql://localhost:5432/docspell"; + user = "dev"; + password = "dev"; + }; +in { + services.dev-postgres = { + enable = true; + databases = ["docspell"]; + }; + services.dev-email.enable = true; + services.dev-solr = { + enable = true; + cores = ["docspell"]; + }; + + port-forward.dev-webmail = 8080; + port-forward.dev-solr = 8983; + + networking = { + hostName = "docspell-test-vm"; + firewall.allowedTCPPorts = [7880]; + }; + + virtualisation.memorySize = 6144; + + virtualisation.forwardPorts = [ + { + from = "host"; + host.port = 7881; + guest.port = 7880; + } + ]; + + services.docspell-restserver = { + enable = true; + bind.address = "0.0.0.0"; + backend = { + addons.enabled = true; + signup.mode = "open"; + inherit jdbc; + }; + integration-endpoint = { + enabled = true; + http-header = { + enabled = true; + header-value = "test123"; + }; + }; + admin-endpoint = { + secret = "admin123"; + }; + inherit full-text-search; + }; + + services.docspell-joex = { + enable = true; + bind.address = "0.0.0.0"; + inherit jdbc full-text-search; + addons = { + executor-config = { + runner = "nix-flake,trivial"; + nspawn.enabled = true; + }; + }; + }; +} diff --git a/project/build.nix b/project/build.nix index 0693a950..73f21840 100644 --- a/project/build.nix +++ b/project/build.nix @@ -1,23 +1,35 @@ let nixpkgs = builtins.fetchTarball { #url = "https://github.com/NixOS/nixpkgs/archive/92e990a8d6bc35f1089c76dd8ba68b78da90da59.tar.gz"; - url = "channel:nixos-23.05"; + url = "channel:nixos-23.05"; }; - pkgs = import nixpkgs { }; + pkgs = import nixpkgs {}; initScript = pkgs.writeScript "docspell-build-init" '' - export LD_LIBRARY_PATH= - ${pkgs.bash}/bin/bash -c "sbt -mem 4096 -java-home ${pkgs.openjdk17}/lib/openjdk" + export LD_LIBRARY_PATH= + ${pkgs.bash}/bin/bash -c "sbt -mem 4096 -java-home ${pkgs.openjdk17}/lib/openjdk" ''; -in with pkgs; +in + with pkgs; + buildFHSUserEnv { + name = "docspell-sbt"; + targetPkgs = pkgs: + with pkgs; [ + netcat + jdk17 + wget + which + dpkg + sbt + git + elmPackages.elm + ncurses + fakeroot + mc + zola + yarn -buildFHSUserEnv { - name = "docspell-sbt"; - targetPkgs = pkgs: with pkgs; [ - netcat jdk17 wget which dpkg sbt git elmPackages.elm ncurses fakeroot mc - zola yarn - - # haskells http client needs this (to download elm packages) - iana-etc - ]; - runScript = initScript; -} + # haskells http client needs this (to download elm packages) + iana-etc + ]; + runScript = initScript; + } diff --git a/website/shell.nix b/website/shell.nix index 8c531cfc..3811e5bc 100644 --- a/website/shell.nix +++ b/website/shell.nix @@ -4,17 +4,16 @@ let #url = "https://github.com/NixOS/nixpkgs/archive/e6badb26fc0d238fda2432c45b7dd4e782eb8200.tar.gz"; #url = "https://github.com/NixOs/nixpkgs/archive/0f316e4d72daed659233817ffe52bf08e081b5de.tar.gz"; #21.11 }; - pkgs = import nixpkgs { }; + pkgs = import nixpkgs {}; in -with pkgs; - - mkShell { - buildInputs = [ - zola - yarn - sbt - elmPackages.elm - nodejs - inotifyTools - ]; - } + with pkgs; + mkShell { + buildInputs = [ + zola + yarn + sbt + elmPackages.elm + nodejs + inotifyTools + ]; + } diff --git a/website/site/content/docs/install/nix.md b/website/site/content/docs/install/nix.md index 343b4e0c..296d6911 100644 --- a/website/site/content/docs/install/nix.md +++ b/website/site/content/docs/install/nix.md @@ -8,9 +8,9 @@ weight = 24 ## Install via Nix Docspell can be installed via the [nix](https://nixos.org/nix) package -manager, which is available for Linux and OSX. Docspell is currently not -part of the [nixpkgs collection](https://nixos.org/nixpkgs/), but you -can use the flake from this repository. +manager. Docspell is currently not part of the [nixpkgs +collection](https://nixos.org/nixpkgs/), but you can use the flake +from this repository. ## Upgrading @@ -31,7 +31,7 @@ There are the following modules provided: ```nix # flake.nix -inputs.docspell.url = "github:eikek/docspell?dir=nix/"; +inputs.docspell.url = "github:eikek/docspell"; # in modules imports = [ @@ -134,10 +134,33 @@ inputs.docspell.url = "github:eikek/docspell?dir=nix/"; ''; }; - networking = { hostName = "docspellexample"; firewall.allowedTCPPorts = [7880]; }; } ``` + +You can also look at `nix/test-vm.nix` for another example. + +## Without Flakes + +Of course, you can also use it without flakes. There is `nix/pkg.nix` +which contains the derivation of both packages, `docspell-server` and +`docspell-joex`. Just call it with your nixpkgs instance as usual: + +``` nix +let + repo = fetchurl { + url = "https://github.com/eikek/docspell"; + sha256 = "sha256-X2mM+Z5s8Xm1E6zrZ0wcRaivLEvqbk5Dn+GSXkZHdLM="; + }; + docspellPkgs = pkgs.callPackage (import "${repo}/nix/pkg.nix") {}; +in + # + # use docspellPkgs.docspell-server or docspellPkgs.docspell-joex + # +``` + +The same way import NixOS modules from +`nix/modules/{joex|server}.nix`. From 8bcc88ed658fa859cc37f86fc404d415f7aa2275 Mon Sep 17 00:00:00 2001 From: eikek Date: Sun, 10 Mar 2024 00:59:56 +0100 Subject: [PATCH 2/6] Document flake dev setup --- .gitignore | 1 + flake.lock | 6 +- flake.nix | 26 ++- nix/test-vm.nix | 7 + website/site/content/docs/dev/development.md | 186 ++++++++++++++---- website/site/content/docs/install/nix.md | 42 ++++ .../site/content/docs/install/quickstart.md | 8 +- 7 files changed, 231 insertions(+), 45 deletions(-) diff --git a/.gitignore b/.gitignore index 18234e40..a921d704 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ #artwork/*.png +.envrc target/ local/ node_modules/ diff --git a/flake.lock b/flake.lock index d2dfeb62..e2f18d87 100644 --- a/flake.lock +++ b/flake.lock @@ -6,11 +6,11 @@ "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1709936258, - "narHash": "sha256-ziYmDU/5v++oYSSwyMqEOr2V75rO+dMQA5aEdwH8amw=", + "lastModified": 1710025500, + "narHash": "sha256-niJf4WZ4GHQ+RaP+Qj4+6P/2SWN1Z4co36/ETvIg0Qg=", "owner": "eikek", "repo": "devshell-tools", - "rev": "59900a3731a88508257525754a704ad2f8a3278e", + "rev": "57d7d292571e291fe8213a1655529f739cfa174d", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index af1469a6..c2ae0178 100644 --- a/flake.nix +++ b/flake.nix @@ -15,10 +15,11 @@ }: flake-utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; + sbt17 = pkgs.sbt.override { jre = pkgs.jdk17; }; devshellPkgs = with pkgs; [ jq scala-cli - sbt + sbt17 netcat jdk17 @@ -85,6 +86,7 @@ (builtins.attrValues devshell-tools.legacyPackages.${system}.cnt-scripts) ++ devshellPkgs; + DOCSPELL_ENV = "dev"; DEV_CONTAINER = "docsp-dev"; SBT_OPTS = "-Xmx2G -Xss4m"; }; @@ -93,6 +95,7 @@ (builtins.attrValues devshell-tools.legacyPackages.${system}.vm-scripts) ++ devshellPkgs; + DOCSPELL_ENV = "dev"; SBT_OPTS = "-Xmx2G -Xss4m"; DEV_VM = "dev-vm"; VM_SSH_PORT = "10022"; @@ -132,10 +135,16 @@ system = "x86_64-linux"; modules = [ { - services.dev-postgres.enable = true; + services.dev-postgres = { + enable = true; + databases = ["docspell"]; + }; services.dev-email.enable = true; services.dev-minio.enable = true; - services.dev-solr.enable = true; + services.dev-solr = { + enable = true; + cores = ["docspell"]; + }; } ]; }; @@ -143,10 +152,17 @@ system = "x86_64-linux"; modules = [ { - services.dev-postgres.enable = true; + services.dev-postgres = { + enable = true; + databases = ["docspell"]; + }; services.dev-email.enable = true; services.dev-minio.enable = true; - services.dev-solr.enable = true; + services.dev-solr = { + enable = true; + cores = ["docspell"]; + }; + port-forward.ssh = 10022; port-forward.dev-postgres = 6534; port-forward.dev-smtp = 10025; diff --git a/nix/test-vm.nix b/nix/test-vm.nix index 673b26de..91ae32f3 100644 --- a/nix/test-vm.nix +++ b/nix/test-vm.nix @@ -27,6 +27,13 @@ in { port-forward.dev-webmail = 8080; port-forward.dev-solr = 8983; + environment.systemPackages = with pkgs; [ + jq + htop + iotop + coreutils + ]; + networking = { hostName = "docspell-test-vm"; firewall.allowedTCPPorts = [7880]; diff --git a/website/site/content/docs/dev/development.md b/website/site/content/docs/dev/development.md index 3cc95543..4fb379ba 100644 --- a/website/site/content/docs/dev/development.md +++ b/website/site/content/docs/dev/development.md @@ -65,6 +65,159 @@ docspell.joex { } ``` +## Installing Nix + +It is recommended to install [nix](https://nixos.org/nix). You can use +the official installer or [this +one](https://github.com/DeterminateSystems/nix-installer), which will +enable Flakes by default. + +If not enabled, enable flakes by creating a config file: + +``` +mkdir -p ~/.config/nix #on Linux +echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf +``` + +With nix installed you can use the provided development environments +to get started quickly. + +# Nix Expressions + +The soure root contains a `flake.nix` file to install docspell via the +nix package manager and to integrate it into NixOS. + +The flake provides docspell packages of the latest release and NixOS +modules. It doesn't provide package builds from the current source +tree. + +## Dev Environments + +Additionally it provides devshells that can be used to create a +development environment for docspell. + +These two `devShell` definitions address two different setups: one +uses a NixOS container and the other a VM. Both provide the same set +of services that can be used with the local docspell instance: + +- postgresql database, with a database `docspell` and user `dev` +- solr with a core `docspell` +- email setup with smtp/imap and webmail +- minio with root user `minioadmin` + +If you are on NixOS the container is probably more convenient to use. +For other systems, the vm should be good. Drop into either shell by +running: + +``` bash +# drop into the environment setup for using a vm +nix develop .#dev-vm + +# drop into the environment setup for using a container +nix deveop .#docsp-dev +``` + +Once in such an environment, you can create the container or vm like +this: + +```bash +# dev-vm env +# - build the vm +vm-build + +# -run the vm +vm-run + +# - ssh into the vm +vm-ssh + +# docsp-dev container env +# - create the container +cnt-recreate + +# - login +cnt-login +``` + +You can use tab completion on `vm-` or `cnt-` and see other useful +commands. For instance it allows to recreate solr cores or check logs +of services on the container or vm. + +Then you can adjust your dev config file in `local/dev.conf` to +connect to the services on the vm or container. The container exposes +the default ports while the vm uses port-forwarding from the host to +the guest machine. The ports are define in `flake.nix`. For example, a +jdbc connection to postgres on the container can look like this: + +``` +jdbc.url = "jdbc:postgresql://docsp-dev:5432/docspelldev" +``` + +on the vm, it would be +``` +jdbc.url = "jdbc:postgresql://localhost:6543/docspelldev" +``` + +You can reach the webmail on both versions at port `8080`. In order to +enable sending mails between users, you need to login as some +arbitrary user so the underlying services can create the data +directories. In your dev docspell you can then connect to smtp on the +vm or container. Mails send from docspell can be checked in the +webmail. Conversely, you can send mails using webmail to any user and +have their mailbox scanned by docspell. + +### Direnv + +Using [direnv](https://direnv.net) entering the dev environment is +very convenient. Install this tool (it also has integration in several +IDEs and editors) and create a file `.envrc` in the source root: + +``` +use flake .# +``` + +The file `.envrc` is git-ignored, because are different ones possible. +Here `` refers to either `dev-cnt` or `dev-vm` - one of the +devshells defined in `flake.nix`. + +After allowing direnv to execute this file via `direnv allow` you will +be dropped into this environment whenever you enter the directory. It +will also preserve your shell, don't need to use bash. + +## Checks + +The command `nix flake check` would run all checks defined in the +flake. It will build both packages and runs a vm with docspell +installed (via NixOS modules) and check whether the services come up. + +## Test VM + +There is another VM defined in the flake that provides a full setup +for docspell. It contains docspell server and joex, a postgresql, a +solr and a email setup. The intention is to use it as an easy 'getting +started' approach with nix. + +Once it has started, you can connect to `localhost:7881` to reach +docspell. The webmail will be available at `localhost:8080`. + +You can run this vm with a single command: + +``` +nix run github:eikek/docspell#nixosConfigurations.test-vm.config.system.build.vm +``` + +It uses the same setup as the dev-vm, so you can drop into the +`.#dev-vm` development shell and use `vm-ssh` to connect to the +running test vm. + +Once connected to the machine, you can see the docspell config file via + +```bash +systemctl show docspell-joex.service | grep "ExecStart=" | sed 's/^ExecStart=.*path=\([^;]*\).*/\1/' | xargs tac | grep -m 1 . | awk '{print $NF}' | tr -d '"' | xargs jq '.' +# or replace "joex" with "restserver" +systemctl show docspell-restserver.service | grep "ExecStart=" | sed 's/^ExecStart=.*path=\([^;]*\).*/\1/' | xargs tac | grep -m 1 . | awk '{print $NF}' | tr -d '"' | xargs jq '.' +``` + # Developing Frontend The frontend is a SPA written in [Elm](https://elm-lang.org). The UI @@ -124,39 +277,6 @@ These tasks must not show any errors (it is checked by the CI). The warnings should also be fixed. -# Nix Expressions - -The directory `/nix` contains Nix Flake to install docspell via -the nix package manager and to integrate it into NixOS. - -Flake implements `checks` output which can be run with `nix flake check` -and it defines a development VM which can be used to interactively work -with docspell. - -To run the VM, issue: - -```bash -cd $PROJECT_ROOT/nix -nix run '.#nixosConfigurations.dev-vm.config.system.build.vm -``` - -To open docspell, wait for docspell-restserver service to report that -http listener is up and connect to `localhost:64080`. - -To ssh into the machine, run: - -```bash -ssh -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -p 64022 root@localhost -``` - -Once connected to the machine, you can see the docspell config file via - -```bash -systemd-show docspell-joex.service | grep ExecStart | cut -d'=' -f2 | xargs cat | tail -n1 | awk '{print $NF}'| sed 's/.$//' | xargs cat | jq -``` - # Release The CI and making a release is done via github actions. The workflow diff --git a/website/site/content/docs/install/nix.md b/website/site/content/docs/install/nix.md index 296d6911..dce291d5 100644 --- a/website/site/content/docs/install/nix.md +++ b/website/site/content/docs/install/nix.md @@ -5,6 +5,34 @@ weight = 24 # Nix +Docspell is a flake, you need to enable flakes in order to make use of +it. You can also use the provided expressions without Flakes, which is +described below. + +## Try it out {try-it-out} + +You can try out the server and joex packages by running the following: + +``` +nix run github:eikek/docspell#docspell-server +nix run github:eikek/docspell#docspell-joex +``` + +While this works, it will be only a very basic setup. The database +defaults to a directory in `/tmp` and no fulltext search enabled. Then +for processing documents, some external tools are required which would +need to be present on yout system to make it work. + +A more elaborate setup with PostgreSQL and SOLR can be started using +the `test-vm`: + +``` +nix run github:eikek/docspell#nixosConfigurations.test-vm.config.system.build.vm +``` + +The vm contains all the required tools. After starting up, you can +find docspell at `http://localhost:7881`. + ## Install via Nix Docspell can be installed via the [nix](https://nixos.org/nix) package @@ -12,6 +40,16 @@ manager. Docspell is currently not part of the [nixpkgs collection](https://nixos.org/nixpkgs/), but you can use the flake from this repository. +You could install the server and joex by running the following: +``` +nix profile install github:eikek/docspell#docspell-server +nix profile install github:eikek/docspell#docspell-joex +``` + +This would install the packages on your system. If you use NixOS you +probably want to use the provided [NixOS modules](#nixos) instead. + + ## Upgrading Since [downgrading](@/docs/install/downgrading.md) is not supported, @@ -164,3 +202,7 @@ in The same way import NixOS modules from `nix/modules/{joex|server}.nix`. + +An alternative can be to use `builtins.getFlake` to fetch the flake +and get access to its output. But this requires to use a flake enabled +nix, which then defeats the idea of "not using flakes". diff --git a/website/site/content/docs/install/quickstart.md b/website/site/content/docs/install/quickstart.md index fa02b4e1..4aaefdb7 100644 --- a/website/site/content/docs/install/quickstart.md +++ b/website/site/content/docs/install/quickstart.md @@ -20,10 +20,10 @@ To get started, here are some quick links: applies to the `zip` and `deb` files. The files can be downloaded from the [release page](https://github.com/eikek/docspell/releases/latest). -- via the [nix package manager](@/docs/install/nix.md) and/or as a - [NixOS module](@/docs/install/nix.md#nixos) through a flake. - If you use nix/nixos, you know what to do. The linked page contains - some examples. +- via the [nix package manager](@/docs/install/nix.md#try-it-out) + and/or as a [NixOS module](@/docs/install/nix.md#nixos) through a + flake. If you use nix/nixos, you know what to do. The linked page + contains some examples. - [Unraid](https://www.unraid.net/): There are user provided [notes and unraid templates](https://github.com/vakilando/unraid-docker-templates) From f3f246d798f965276448ad580caa71b48790b587 Mon Sep 17 00:00:00 2001 From: eikek Date: Sun, 10 Mar 2024 10:22:36 +0100 Subject: [PATCH 3/6] Rename server -> restserver in nix setup While I'd like to rename it the other way around, it would be a much more breaking change. So for now, this way. --- flake.nix | 14 +++++++------- nix/docker.nix | 6 +++--- nix/modules/server.nix | 4 ++-- nix/pkg.nix | 5 +++-- website/site/content/docs/install/nix.md | 8 ++++---- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/flake.nix b/flake.nix index c2ae0178..b8a8cdc0 100644 --- a/flake.nix +++ b/flake.nix @@ -33,31 +33,31 @@ ]; docspellPkgs = pkgs.callPackage (import ./nix/pkg.nix) {}; dockerAmd64 = pkgs.pkgsCross.gnu64.callPackage (import ./nix/docker.nix) { - inherit (docspellPkgs) docspell-server docspell-joex; + inherit (docspellPkgs) docspell-restserver docspell-joex; }; dockerArm64 = pkgs.pkgsCross.aarch64-multiplatform.callPackage (import ./nix/docker.nix) { - inherit (docspellPkgs) docspell-server docspell-joex; + inherit (docspellPkgs) docspell-restserver docspell-joex; }; in { formatter = pkgs.alejandra; packages = { - inherit (docspellPkgs) docspell-server docspell-joex; + inherit (docspellPkgs) docspell-restserver docspell-joex; }; legacyPackages = { docker = { amd64 = { - inherit (dockerAmd64) docspell-server docspell-joex; + inherit (dockerAmd64) docspell-restserver docspell-joex; }; arm64 = { - inherit (dockerArm64) docspell-server docspell-joex; + inherit (dockerArm64) docspell-restserver docspell-joex; }; }; }; checks = { - build-server = self.packages.${system}.docspell-server; + build-server = self.packages.${system}.docspell-restserver; build-joex = self.packages.${system}.docspell-joex; test = with import (nixpkgs + "/nixos/lib/testing-python.nix") @@ -117,7 +117,7 @@ overlays.default = final: prev: let docspellPkgs = final.callPackage (import ./nix/pkg.nix) {}; in { - inherit (docspellPkgs) docspell-server docspell-joex; + inherit (docspellPkgs) docspell-restserver docspell-joex; }; nixosConfigurations = { diff --git a/nix/docker.nix b/nix/docker.nix index d0e69596..29e9f870 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -9,7 +9,7 @@ tesseract4, python3Packages, unoconv, - docspell-server, + docspell-restserver, docspell-joex, }: let mkImage = { @@ -64,10 +64,10 @@ tag = "v${pkg.version}"; }; in { - docspell-server = mkImage { + docspell-restserver = mkImage { name = "docspell-restserver"; port = 7880; - pkg = docspell-server; + pkg = docspell-restserver; tools = []; }; docspell-joex = mkImage { diff --git a/nix/modules/server.nix b/nix/modules/server.nix index 925c2352..0943e42e 100644 --- a/nix/modules/server.nix +++ b/nix/modules/server.nix @@ -12,7 +12,7 @@ with lib; let if cfg.runAs == null then "docspell" else cfg.runAs; - configFile = pkgs.writeText "docspell-server.conf" '' + configFile = pkgs.writeText "docspell-restserver.conf" '' {"docspell": {"server": ${builtins.toJSON (lib.recursiveUpdate declared_config cfg.extraConfig)} }} @@ -878,7 +878,7 @@ in { systemd.services.docspell-restserver = let args = builtins.concatStringsSep " " cfg.jvmArgs; - cmd = "${pkgs.docspell-server}/bin/docspell-restserver ${args} -- ${configFile}"; + cmd = "${pkgs.docspell-restserver}/bin/docspell-restserver ${args} -- ${configFile}"; in { description = "Docspell Rest Server"; after = ["networking.target"]; diff --git a/nix/pkg.nix b/nix/pkg.nix index 1b5733ba..a37db9c2 100644 --- a/nix/pkg.nix +++ b/nix/pkg.nix @@ -14,9 +14,9 @@ sha256 = "sha256-flKWjEsMd2/XT3Bu6EjFgf3lCojvLbKFDEXemP1K+/8="; }; in { - docspell-server = stdenv.mkDerivation { + docspell-restserver = stdenv.mkDerivation { inherit version; - pname = "docspell-server"; + pname = "docspell-restserver"; src = fetchzip server; buildInputs = [jdk17]; @@ -30,6 +30,7 @@ in { $out/docspell-restserver-${version}/bin/docspell-restserver -java-home ${jdk17} "\$@" EOF chmod 755 $out/bin/docspell-restserver + ''; }; diff --git a/website/site/content/docs/install/nix.md b/website/site/content/docs/install/nix.md index dce291d5..be705a14 100644 --- a/website/site/content/docs/install/nix.md +++ b/website/site/content/docs/install/nix.md @@ -14,7 +14,7 @@ described below. You can try out the server and joex packages by running the following: ``` -nix run github:eikek/docspell#docspell-server +nix run github:eikek/docspell#docspell-restserver nix run github:eikek/docspell#docspell-joex ``` @@ -42,7 +42,7 @@ from this repository. You could install the server and joex by running the following: ``` -nix profile install github:eikek/docspell#docspell-server +nix profile install github:eikek/docspell#docspell-restserver nix profile install github:eikek/docspell#docspell-joex ``` @@ -184,7 +184,7 @@ You can also look at `nix/test-vm.nix` for another example. ## Without Flakes Of course, you can also use it without flakes. There is `nix/pkg.nix` -which contains the derivation of both packages, `docspell-server` and +which contains the derivation of both packages, `docspell-restserver` and `docspell-joex`. Just call it with your nixpkgs instance as usual: ``` nix @@ -196,7 +196,7 @@ let docspellPkgs = pkgs.callPackage (import "${repo}/nix/pkg.nix") {}; in # - # use docspellPkgs.docspell-server or docspellPkgs.docspell-joex + # use docspellPkgs.docspell-restserver or docspellPkgs.docspell-joex # ``` From 3aad3b7be452264774e816e90efece52d85ce5e9 Mon Sep 17 00:00:00 2001 From: eikek Date: Sun, 10 Mar 2024 15:36:40 +0100 Subject: [PATCH 4/6] Remove other now obsolete nix files --- .github/workflows/ci-docs.yml | 6 +--- .github/workflows/website.yml | 8 ++--- flake.nix | 1 + project/build.nix | 35 ------------------- website/README.md | 11 +++--- website/push-docs.sh | 2 +- website/shell.nix | 19 ---------- .../content/blog/2022-01-31_create_post.md | 10 +++--- 8 files changed, 16 insertions(+), 76 deletions(-) delete mode 100644 project/build.nix delete mode 100644 website/shell.nix diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 2ae985e7..15de027f 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -17,9 +17,5 @@ jobs: with: elm-version: 0.19.1 - uses: cachix/install-nix-action@v26 - with: - nix_path: nixpkgs=channel:nixos-23.05 - - name: Print nixpkgs version - run: nix-instantiate --eval -E '(import {}).lib.version' - name: Build website (${{ env.DOCSPELL_VERSION }}) - run: nix-shell website/shell.nix --run "sbt make-website" + run: nix develop .#dev-cnt --command sbt make-website diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index 585b7ff7..375d7f67 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -16,13 +16,9 @@ jobs: with: elm-version: 0.19.1 - uses: cachix/install-nix-action@v26 - with: - nix_path: nixpkgs=channel:nixos-23.05 - - name: Print nixpkgs version - run: nix-instantiate --eval -E '(import {}).lib.version' - name: Build website (${{ env.DOCSPELL_VERSION }}) - run: nix-shell website/shell.nix --run "sbt make-website" + run: nix develop .#dev-cnt --command sbt make-website - name: Publish website (${{ env.DOCSPELL_VERSION }}) env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: sbt publish-website + run: nix develop .#dev-cnt --command sbt publish-website diff --git a/flake.nix b/flake.nix index b8a8cdc0..4acb2d17 100644 --- a/flake.nix +++ b/flake.nix @@ -30,6 +30,7 @@ fakeroot zola yarn + inotifyTools ]; docspellPkgs = pkgs.callPackage (import ./nix/pkg.nix) {}; dockerAmd64 = pkgs.pkgsCross.gnu64.callPackage (import ./nix/docker.nix) { diff --git a/project/build.nix b/project/build.nix deleted file mode 100644 index 73f21840..00000000 --- a/project/build.nix +++ /dev/null @@ -1,35 +0,0 @@ -let - nixpkgs = builtins.fetchTarball { - #url = "https://github.com/NixOS/nixpkgs/archive/92e990a8d6bc35f1089c76dd8ba68b78da90da59.tar.gz"; - url = "channel:nixos-23.05"; - }; - pkgs = import nixpkgs {}; - initScript = pkgs.writeScript "docspell-build-init" '' - export LD_LIBRARY_PATH= - ${pkgs.bash}/bin/bash -c "sbt -mem 4096 -java-home ${pkgs.openjdk17}/lib/openjdk" - ''; -in - with pkgs; - buildFHSUserEnv { - name = "docspell-sbt"; - targetPkgs = pkgs: - with pkgs; [ - netcat - jdk17 - wget - which - dpkg - sbt - git - elmPackages.elm - ncurses - fakeroot - mc - zola - yarn - - # haskells http client needs this (to download elm packages) - iana-etc - ]; - runScript = initScript; - } diff --git a/website/README.md b/website/README.md index 66c63ed6..a3d2df97 100644 --- a/website/README.md +++ b/website/README.md @@ -15,19 +15,20 @@ Sbt is used to build the site. Install things by running `yarn install`. -Open terminal for each script below: +Use a dev [environment](https://docspell.org/docs/dev/development/) +and open terminal for each script below: 1. Starting the server ``` shell - nix-shell --run "cd site && zola serve" + cd site && zola serve ``` 2. Building the stylesheet ``` shell - nix-shell --run ./scripts/run-styles.sh + ./scripts/run-styles.sh ``` 3. Building some javascript files ``` shell - nix-shell --run ./scripts/run-elm.sh + ./scripts/run-elm.sh ``` Open browser at `localhost:1111`. @@ -40,7 +41,7 @@ be finally deployed. To see this, start sbt and change into the website project. ``` shell -nix-shell website/shell.nix --run sbt +$ sbt sbt> project website ``` diff --git a/website/push-docs.sh b/website/push-docs.sh index bc017433..d2c090a6 100755 --- a/website/push-docs.sh +++ b/website/push-docs.sh @@ -37,7 +37,7 @@ git clone git@github.com:eikek/docspell.git "$temp" cd "$temp" && git checkout --track origin/gh-pages && rm -rf "$temp"/* echo "Create new website from current working directory" -cd $cdir && nix-shell website/shell.nix --run 'sbt make-website' +cd $cdir && sbt make-website echo "Copying new site to target" cp -R "$cdir"/website/target/zola-site/* "$temp/" diff --git a/website/shell.nix b/website/shell.nix deleted file mode 100644 index 3811e5bc..00000000 --- a/website/shell.nix +++ /dev/null @@ -1,19 +0,0 @@ -let - nixpkgs = builtins.fetchTarball { - url = "channel:nixos-23.05"; - #url = "https://github.com/NixOS/nixpkgs/archive/e6badb26fc0d238fda2432c45b7dd4e782eb8200.tar.gz"; - #url = "https://github.com/NixOs/nixpkgs/archive/0f316e4d72daed659233817ffe52bf08e081b5de.tar.gz"; #21.11 - }; - pkgs = import nixpkgs {}; -in - with pkgs; - mkShell { - buildInputs = [ - zola - yarn - sbt - elmPackages.elm - nodejs - inotifyTools - ]; - } diff --git a/website/site/content/blog/2022-01-31_create_post.md b/website/site/content/blog/2022-01-31_create_post.md index 69bd8ab2..e330694c 100644 --- a/website/site/content/blog/2022-01-31_create_post.md +++ b/website/site/content/blog/2022-01-31_create_post.md @@ -63,11 +63,11 @@ you can look at the results while writing. If you want to see a live view of the page while editing, some tools are required. The easiest way to get these is to install -[nix](https://nixos.org/) and run `nix-shell website/shell.nix` to get -an environment with all these tools installed. Otherwise install the -programs mentioned in `website/shell.nix`, which are: -[yarn](https://yarnpkg.com/), [zola](https://www.getzola.org/), -[elm](https://elm-lang.org) and [sbt](https://scala-sbt.org). +[nix](https://nixos.org/) and run `nix develop .#dev-vm` to get an +environment with all these tools installed. Otherwise install the +programs: [yarn](https://yarnpkg.com/), +[zola](https://www.getzola.org/), [elm](https://elm-lang.org) and +[sbt](https://scala-sbt.org). Then clone the sources to your machine and build the complete site once, so that all assets and required stuff is present: From 8a41ed3fd382334ac4fa9509317ff0b566871f7e Mon Sep 17 00:00:00 2001 From: eikek Date: Sun, 10 Mar 2024 15:50:06 +0100 Subject: [PATCH 5/6] Github actions use nix --- .github/workflows/ci-docs.yml | 9 +++------ .github/workflows/ci.yml | 20 +++----------------- .github/workflows/docker-image.yml | 2 +- .github/workflows/release-nightly.yml | 24 ++++-------------------- .github/workflows/release.yml | 20 +++----------------- .github/workflows/website.yml | 11 ++++------- flake.nix | 22 ++++++++++++++-------- 7 files changed, 32 insertions(+), 76 deletions(-) diff --git a/.github/workflows/ci-docs.yml b/.github/workflows/ci-docs.yml index 15de027f..e2fa7bca 100644 --- a/.github/workflows/ci-docs.yml +++ b/.github/workflows/ci-docs.yml @@ -6,16 +6,13 @@ on: - "master" jobs: check-website: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 + - uses: cachix/install-nix-action@v26 - name: Set current version run: echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV - - uses: jorelali/setup-elm@v5 - with: - elm-version: 0.19.1 - - uses: cachix/install-nix-action@v26 - name: Build website (${{ env.DOCSPELL_VERSION }}) - run: nix develop .#dev-cnt --command sbt make-website + run: nix develop .#ci --command sbt make-website diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 873d0ab2..272a9654 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,32 +5,18 @@ on: - master jobs: ci-matrix: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - java: [ 'openjdk@1.17' ] steps: - uses: actions/checkout@v4.1.1 with: fetch-depth: 100 - - uses: jorelali/setup-elm@v5 - with: - elm-version: 0.19.1 - - uses: bahmutov/npm-install@v1 - with: - working-directory: modules/webapp - name: Fetch tags run: git fetch --depth=100 origin +refs/tags/*:refs/tags/* - - uses: olafurpg/setup-scala@v14 - with: - java-version: ${{ matrix.java }} - # - name: Coursier cache - # uses: coursier/cache-action@v6 + - uses: cachix/install-nix-action@v26 - name: sbt ci ${{ github.ref }} - run: sbt ci - env: - SBT_OPTS: "-Xmx2G -Xss4m" + run: nix develop .#ci --command sbt ci ci: runs-on: ubuntu-22.04 needs: [ci-matrix] diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 997e0972..77e2a3a0 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -4,7 +4,7 @@ on: types: [ published ] jobs: docker-images: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4.1.1 with: diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml index fa371efa..ec18ef90 100644 --- a/.github/workflows/release-nightly.yml +++ b/.github/workflows/release-nightly.yml @@ -5,36 +5,20 @@ on: - "master" jobs: release-nightly: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest strategy: fail-fast: true - matrix: - java: [ 'openjdk@1.17' ] steps: - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 - - uses: olafurpg/setup-scala@v14 - with: - java-version: ${{ matrix.java }} - - uses: jorelali/setup-elm@v5 - with: - elm-version: 0.19.1 - - uses: bahmutov/npm-install@v1 - with: - working-directory: modules/webapp - # - name: Coursier cache - # uses: coursier/cache-action@v6 + - uses: cachix/install-nix-action@v26 - name: Set current version run: echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV - name: sbt ci ${{ github.ref }} - run: sbt ci - env: - SBT_OPTS: "-Xmx2G -Xss4m" + run: nix develop .#ci --command sbt ci - name: sbt make-pkg (${{ env.DOCSPELL_VERSION }}) - run: sbt make-pkg - env: - SBT_OPTS: "-Xmx2G -Xss4m" + run: nix develop .#ci --command sbt make-pkg - uses: "marvinpinto/action-automatic-releases@latest" with: repo_token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 49d33cb9..9a55c2f2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,32 +5,18 @@ on: - 'v*' jobs: release: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest strategy: fail-fast: true - matrix: - java: [ 'openjdk@1.17' ] steps: - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 - - uses: olafurpg/setup-scala@v14 - with: - java-version: ${{ matrix.java }} - - uses: jorelali/setup-elm@v5 - with: - elm-version: 0.19.1 - - uses: bahmutov/npm-install@v1 - with: - working-directory: modules/webapp - # - name: Coursier cache - # uses: coursier/cache-action@v6 + - uses: cachix/install-nix-action@v26 - name: Set current version run: echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV - name: sbt make-pkg (${{ env.DOCSPELL_VERSION }}) - run: sbt make-pkg - env: - SBT_OPTS: "-Xmx2G -Xss4m" + run: nix develop .#ci --command sbt make-pkg - uses: meeDamian/github-release@2.0 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index 375d7f67..fa02f578 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -5,20 +5,17 @@ on: - "current-docs" jobs: publish-website: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4.1.1 with: fetch-depth: 0 + - uses: cachix/install-nix-action@v26 - name: Set current version run: echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV - - uses: jorelali/setup-elm@v5 - with: - elm-version: 0.19.1 - - uses: cachix/install-nix-action@v26 - name: Build website (${{ env.DOCSPELL_VERSION }}) - run: nix develop .#dev-cnt --command sbt make-website + run: nix develop .#ci --command sbt make-website - name: Publish website (${{ env.DOCSPELL_VERSION }}) env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: nix develop .#dev-cnt --command sbt publish-website + run: nix develop .#ci --command sbt publish-website diff --git a/flake.nix b/flake.nix index 4acb2d17..513af86f 100644 --- a/flake.nix +++ b/flake.nix @@ -16,22 +16,24 @@ flake-utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs.legacyPackages.${system}; sbt17 = pkgs.sbt.override { jre = pkgs.jdk17; }; - devshellPkgs = with pkgs; [ - jq - scala-cli + ciPkgs = with pkgs; [ sbt17 - - netcat jdk17 - wget - which dpkg elmPackages.elm fakeroot zola yarn - inotifyTools + nodejs ]; + devshellPkgs = ciPkgs ++ (with pkgs; [ + jq + scala-cli + netcat + wget + which + inotifyTools + ]); docspellPkgs = pkgs.callPackage (import ./nix/pkg.nix) {}; dockerAmd64 = pkgs.pkgsCross.gnu64.callPackage (import ./nix/docker.nix) { inherit (docspellPkgs) docspell-restserver docspell-joex; @@ -101,6 +103,10 @@ DEV_VM = "dev-vm"; VM_SSH_PORT = "10022"; }; + ci = pkgs.mkShellNoCC { + buildInputs = ciPkgs; + SBT_OPTS = "-Xmx2G -Xss4m"; + }; }; }) // { From ba8435c7dcfdff1916f1523ce0617767a4837cb3 Mon Sep 17 00:00:00 2001 From: eikek Date: Sun, 10 Mar 2024 16:44:31 +0100 Subject: [PATCH 6/6] Disable strict external link checking This is so brittle, only works sometimes. --- website/site/config.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/website/site/config.toml b/website/site/config.toml index c8df8075..3c513b60 100644 --- a/website/site/config.toml +++ b/website/site/config.toml @@ -20,6 +20,7 @@ highlight_themes_css = [ ] [link_checker] +external_level = "warn" skip_prefixes = [ "http://localhost", "https://docs.github.com", # doesn't allow checking anymore