From 719327905336b016101805e6b0cd30004ae1b053 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Sun, 28 Jun 2020 13:36:46 +0200 Subject: [PATCH] Prepare nixos setup for full-text-search and new consumedir settings --- nix/configuration-test.nix | 36 +++++++++++++++++++++++++----- nix/module-consumedir.nix | 45 ++++++++++++++++++++++++++++++++++++++ nix/module-joex.nix | 18 ++++++++++++++- nix/solr.nix | 35 +++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 6 deletions(-) create mode 100644 nix/solr.nix diff --git a/nix/configuration-test.nix b/nix/configuration-test.nix index 71948b19..2dcd4e12 100644 --- a/nix/configuration-test.nix +++ b/nix/configuration-test.nix @@ -1,9 +1,13 @@ { config, pkgs, ... }: let docspell = import ./release.nix; + full-text-search = { + enabled = true; + solr.url = "http://localhost:${toString config.services.solr.port}/solr/docspell"; + }; in { - imports = docspell.modules; + imports = docspell.modules ++ [ ./solr.nix ]; i18n = { defaultLocale = "en_US.UTF-8"; @@ -28,25 +32,42 @@ in services.docspell-joex = { enable = true; + waitForTarget = "solr-init.target"; bind.address = "0.0.0.0"; base-url = "http://localhost:7878"; + inherit full-text-search; }; services.docspell-restserver = { - bind.address = "0.0.0.0"; enable = true; + bind.address = "0.0.0.0"; + integration-endpoint = { + enabled = true; + http-header = { + enabled = true; + header-value = "test123"; + }; + }; + inherit full-text-search; }; services.docspell-consumedir = { enable = true; - watchDirs = ["/tmp/test"]; - urls = ["http://localhost:7880/api/v1/open/upload/item/blabla"]; + integration-endpoint = { + enabled = true; + header = "Docspell-Integration:test123"; + }; + watchDirs = ["/tmp/docs"]; + urls = ["http://localhost:7880/api/v1/open/integration/item"]; }; environment.systemPackages = [ pkgs.docspell.tools pkgs.docspell.server - pkgs.docspell.joex pkgs.jq + pkgs.docspell.joex + pkgs.jq + pkgs.telnet ]; + services.xserver = { enable = false; }; @@ -56,6 +77,11 @@ in firewall.allowedTCPPorts = [7880]; }; + system.activationScripts = { + initUploadDir = '' + mkdir -p ${builtins.concatStringsSep " " config.services.docspell-consumedir.watchDirs} + ''; + }; system.stateVersion = "20.03"; } diff --git a/nix/module-consumedir.nix b/nix/module-consumedir.nix index 6f363e7a..66c485e9 100644 --- a/nix/module-consumedir.nix +++ b/nix/module-consumedir.nix @@ -45,6 +45,40 @@ in { description = "Check for duplicates and update only if the file is not already present."; }; + integration-endpoint = mkOption { + type = types.submodule({ + options = { + enabled = mkOption { + type = types.bool; + default = false; + description = "Whether to upload to the integration endpoint."; + }; + header = mkOption { + type = types.str; + default = ""; + description = '' + The `header:value` string matching the configured header-name + and value for the integration endpoint. + ''; + }; + user = mkOption { + type = types.str; + default = ""; + description = '' + The `user:password` string matching the configured user and password + for the integration endpoint. Since both are separated by a colon, the + user name may not contain a colon (the password can). + ''; + }; + }; + }); + default = { + enabled = false; + header = ""; + user = ""; + }; + description = "Settings for using the integration endpoint."; + }; urls = mkOption { type = types.listOf types.str; example = [ "http://localhost:7880/api/v1/open/upload/item/abced-12345-abcde-12345" ]; @@ -68,6 +102,17 @@ in { (if cfg.verbose then ["-v"] else []) ++ (if cfg.deleteFiles then ["-d"] else []) ++ (if cfg.distinct then [ "-m" ] else []) ++ + (if cfg.integration-endpoint.enabled then [ "-i" ] else []) ++ + (if cfg.integration-endpoint.header != "" + then + [ "--iheader" cfg.integration-endpoint.header ] + else + []) ++ + (if cfg.integration-endpoint.user != "" + then + [ "--iuser" cfg.integration-endpoint.user ] + else + []) ++ (map (a: "'" + a + "'") cfg.urls); cmd = "${pkgs.docspell.tools}/bin/consumedir.sh " + (builtins.concatStringsSep " " args); in diff --git a/nix/module-joex.nix b/nix/module-joex.nix index c4ebe8fa..d92afc02 100644 --- a/nix/module-joex.nix +++ b/nix/module-joex.nix @@ -168,6 +168,16 @@ in { user is created. ''; }; + waitForTarget = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + If not null, joex depends on this systemd target. This is + useful if full-text-search is enabled and the solr instance + is running on the same machine. + ''; + }; + app-id = mkOption { type = types.str; @@ -999,10 +1009,16 @@ in { systemd.services.docspell-joex = let cmd = "${pkgs.docspell.joex}/bin/docspell-joex ${configFile}"; + waitTarget = + if cfg.waitForTarget != null + then + [ cfg.waitForTarget ] + else + []; in { description = "Docspell Joex"; - after = [ "networking.target" ]; + after = ([ "networking.target" ] ++ waitTarget); wantedBy = [ "multi-user.target" ]; path = [ pkgs.gawk ]; diff --git a/nix/solr.nix b/nix/solr.nix new file mode 100644 index 00000000..b7d73cfb --- /dev/null +++ b/nix/solr.nix @@ -0,0 +1,35 @@ +{ config, pkgs, ... }: + +# This module sets up solr with one core. It is a bit tedious…. If you +# know a better solution, please let me know. +{ + + services.solr = { + enable = true; + }; + # This is needed to run solr script as user solr + users.users.solr.useDefaultShell = true; + + systemd.services.solr-init = + let + solrPort = toString config.services.solr.port; + initSolr = '' + if [ ! -f ${config.services.solr.stateDir}/docspell_core ]; then + while ! echo "" | ${pkgs.telnet}/bin/telnet localhost ${solrPort} + do + echo "Waiting for SOLR become ready..." + sleep 1.5 + done + ${pkgs.su}/bin/su -s ${pkgs.bash}/bin/sh solr -c "${pkgs.solr}/bin/solr create_core -c docspell -p ${solrPort}"; + touch ${config.services.solr.stateDir}/docspell_core + fi + ''; + in + { script = initSolr; + after = [ "solr.target" ]; + wantedBy = [ "multi-user.target" ]; + requires = [ "solr.target" ]; + description = "Create a core at solr"; + }; + +}