Prepare nixos setup for full-text-search and new consumedir settings

nix/configuration-test.nix

@@ -1,9 +1,13 @@
 { config, pkgs, ... }:
 let
   docspell = import ./release.nix;
+  full-text-search = {
+    enabled = true;
+    solr.url = "http://localhost:${toString config.services.solr.port}/solr/docspell";
+  };
 in
 {
-  imports = docspell.modules;
+  imports = docspell.modules ++ [ ./solr.nix ];
 
   i18n = {
     defaultLocale = "en_US.UTF-8";
@@ -28,25 +32,42 @@ in
 
   services.docspell-joex = {
     enable = true;
+    waitForTarget = "solr-init.target";
     bind.address = "0.0.0.0";
     base-url = "http://localhost:7878";
+    inherit full-text-search;
   };
   services.docspell-restserver = {
-    bind.address = "0.0.0.0";
     enable = true;
+    bind.address = "0.0.0.0";
+    integration-endpoint = {
+      enabled = true;
+      http-header = {
+        enabled = true;
+        header-value = "test123";
+      };
+    };
+    inherit full-text-search;
   };
   services.docspell-consumedir = {
     enable = true;
-    watchDirs = ["/tmp/test"];
-    urls = ["http://localhost:7880/api/v1/open/upload/item/blabla"];
+    integration-endpoint = {
+      enabled = true;
+      header = "Docspell-Integration:test123";
+    };
+    watchDirs = ["/tmp/docs"];
+    urls = ["http://localhost:7880/api/v1/open/integration/item"];
   };
 
   environment.systemPackages =
     [ pkgs.docspell.tools
       pkgs.docspell.server
-      pkgs.docspell.joex pkgs.jq
+      pkgs.docspell.joex
+      pkgs.jq
+      pkgs.telnet
     ];
 
+
   services.xserver = {
     enable = false;
   };
@@ -56,6 +77,11 @@ in
     firewall.allowedTCPPorts = [7880];
   };
 
+  system.activationScripts = {
+    initUploadDir = ''
+      mkdir -p ${builtins.concatStringsSep " " config.services.docspell-consumedir.watchDirs}
+    '';
+  };
   system.stateVersion = "20.03";
 
 }

nix/module-consumedir.nix

@@ -45,6 +45,40 @@ in {
         description = "Check for duplicates and update only if the file is not already present.";
       };
 
+      integration-endpoint = mkOption {
+        type = types.submodule({
+          options = {
+            enabled = mkOption {
+              type = types.bool;
+              default = false;
+              description = "Whether to upload to the integration endpoint.";
+            };
+            header = mkOption {
+              type = types.str;
+              default = "";
+              description = ''
+                The `header:value` string matching the configured header-name
+                 and value for the integration endpoint.
+              '';
+            };
+            user = mkOption {
+              type = types.str;
+              default = "";
+              description = ''
+                The `user:password` string matching the configured user and password
+                for the integration endpoint. Since both are separated by a colon, the
+                user name may not contain a colon (the password can).
+              '';
+            };
+          };
+        });
+        default = {
+          enabled = false;
+          header = "";
+          user = "";
+        };
+        description = "Settings for using the integration endpoint.";
+      };
       urls = mkOption {
         type = types.listOf types.str;
         example = [ "http://localhost:7880/api/v1/open/upload/item/abced-12345-abcde-12345" ];
@@ -68,6 +102,17 @@ in {
              (if cfg.verbose then ["-v"] else []) ++
              (if cfg.deleteFiles then ["-d"] else []) ++
              (if cfg.distinct then [ "-m" ] else []) ++
+             (if cfg.integration-endpoint.enabled then [ "-i" ] else []) ++
+             (if cfg.integration-endpoint.header != ""
+              then
+                [ "--iheader" cfg.integration-endpoint.header ]
+              else
+                []) ++
+             (if cfg.integration-endpoint.user != ""
+              then
+                [ "--iuser" cfg.integration-endpoint.user ]
+              else
+                []) ++
              (map (a: "'" + a + "'") cfg.urls);
       cmd = "${pkgs.docspell.tools}/bin/consumedir.sh " + (builtins.concatStringsSep " " args);
     in

nix/module-joex.nix

@@ -168,6 +168,16 @@ in {
           user is created.
         '';
       };
+      waitForTarget = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = ''
+          If not null, joex depends on this systemd target. This is
+          useful if full-text-search is enabled and the solr instance
+          is running on the same machine.
+        '';
+      };
+
 
       app-id = mkOption {
         type = types.str;
@@ -999,10 +1009,16 @@ in {
     systemd.services.docspell-joex =
       let
         cmd = "${pkgs.docspell.joex}/bin/docspell-joex ${configFile}";
+        waitTarget =
+          if cfg.waitForTarget != null
+          then
+            [ cfg.waitForTarget ]
+          else
+            [];
       in
         {
           description = "Docspell Joex";
-          after = [ "networking.target" ];
+          after = ([ "networking.target" ] ++ waitTarget);
           wantedBy = [ "multi-user.target" ];
           path = [ pkgs.gawk ];
 

nix/solr.nix

@@ -0,0 +1,35 @@
+{ config, pkgs, ... }:
+
+# This module sets up solr with one core. It is a bit tedious…. If you
+# know a better solution, please let me know.
+{
+
+  services.solr = {
+    enable = true;
+  };
+  # This is needed to run solr script as user solr
+  users.users.solr.useDefaultShell = true;
+
+  systemd.services.solr-init =
+    let
+      solrPort = toString config.services.solr.port;
+      initSolr = ''
+        if [ ! -f ${config.services.solr.stateDir}/docspell_core ]; then
+          while ! echo "" | ${pkgs.telnet}/bin/telnet localhost ${solrPort}
+          do
+             echo "Waiting for SOLR become ready..."
+             sleep 1.5
+          done
+          ${pkgs.su}/bin/su -s ${pkgs.bash}/bin/sh solr -c "${pkgs.solr}/bin/solr create_core -c docspell -p ${solrPort}";
+          touch ${config.services.solr.stateDir}/docspell_core
+        fi
+      '';
+    in
+      { script = initSolr;
+        after = [ "solr.target" ];
+        wantedBy = [ "multi-user.target" ];
+        requires = [ "solr.target" ];
+        description = "Create a core at solr";
+      };
+
+}