Update dependency tailwindcss to v3.4.7

Update mariadb-java-client to 3.4.1

.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+# Scala Steward: Reformat with scalafmt 3.8.2
+1c566cd5182d41f4cc06040fc347ddb4be617779

.github/release-drafter.yml

@@ -0,0 +1,42 @@
+name-template: "$RESOLVED_VERSION"
+tag-template: "$RESOLVED_VERSION"
+template: |
+  ## What’s Changed
+
+  $CHANGES
+
+categories:
+  - title: "🚀 Features"
+    labels:
+      - 'feature'
+      - 'enhancement'
+  - title: "🐛 Bug Fixes"
+    labels:
+      - 'fix'
+      - 'bug'
+  - title: "💚 Maintenance"
+    labels:
+      - 'chore'
+      - 'documentation'
+  - title: "🧱 Dependencies"
+    labels:
+      - 'dependencies'
+      - 'type: dependencies'
+
+change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
+version-resolver:
+  major:
+    labels:
+      - 'breaking'
+  minor:
+    labels:
+      - 'feature'
+      - 'enhancement'
+  patch:
+    labels:
+      - 'chore'
+      - 'documentation'
+      - 'dependencies'
+  default: patch
+exclude-labels:
+  - 'skip-changelog'

.github/renovate.json

@@ -1,6 +1,6 @@
 {
   "automerge": true,
-  "labels": ["type: dependencies"],
+  "labels": ["dependencies"],
   "packageRules": [
     {
       "matchManagers": [

.github/stale.yml

@@ -1,16 +0,0 @@
-# Number of days of inactivity before an issue becomes stale
-daysUntilStale: 30
-# Number of days of inactivity before a stale issue is closed
-daysUntilClose: 7
-onlyLabels:
-  - question
-# Label to use when marking an issue as stale
-staleLabel: stale
-# Comment to post when marking an issue as stale. Set to `false` to disable
-markComment: >
-  This issue has been automatically marked as stale because it has not
-  had recent activity. It will be closed if no further activity
-  occurs. This only applies to 'question' issues. Always feel free to
-  reopen or create new issues. Thank you!
-# Comment to post when closing a stale issue. Set to `false` to disable
-closeComment: false

.github/workflows/ci-docs.yml

@@ -6,20 +6,13 @@ on:
       - "master"
 jobs:
   check-website:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.1.7
         with:
           fetch-depth: 0
+      - uses: cachix/install-nix-action@v27
       - name: Set current version
         run:  echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
-      - uses: jorelali/setup-elm@v5
-        with:
-          elm-version: 0.19.1
-      - uses: cachix/install-nix-action@v25
-        with:
-          nix_path: nixpkgs=channel:nixos-23.05
-      - name: Print nixpkgs version
-        run: nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
       - name: Build website (${{ env.DOCSPELL_VERSION }})
-        run: nix-shell website/shell.nix --run "sbt make-website"
+        run: nix develop .#ci --command sbt make-website

.github/workflows/ci.yml

@@ -5,30 +5,18 @@ on:
       - master
 jobs:
   ci-matrix:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      matrix:
-        java: [ 'openjdk@1.17' ]
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.1.7
         with:
           fetch-depth: 100
-      - uses: jorelali/setup-elm@v5
-        with:
-          elm-version: 0.19.1
-      - uses: bahmutov/npm-install@v1
-        with:
-          working-directory: modules/webapp
       - name: Fetch tags
         run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
-      - uses: olafurpg/setup-scala@v14
+      - uses: cachix/install-nix-action@v27
-        with:
-          java-version: ${{ matrix.java }}
-      # - name: Coursier cache
-      #   uses: coursier/cache-action@v6
       - name: sbt ci ${{ github.ref }}
-        run: sbt ci
+        run: nix develop .#ci --command sbt ci
   ci:
     runs-on: ubuntu-22.04
     needs: [ci-matrix]

.github/workflows/docker-image.yml

@@ -4,9 +4,9 @@ on:
     types: [ published ]
 jobs:
   docker-images:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.1.7
         with:
           fetch-depth: 0
       - name: Set current version

.github/workflows/release-drafter.yml

@@ -0,0 +1,14 @@
+name: Release Drafter
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  update_release_draft:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: release-drafter/release-drafter@v6
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release-nightly.yml

@@ -5,32 +5,20 @@ on:
       - "master"
 jobs:
   release-nightly:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: true
-      matrix:
-        java: [ 'openjdk@1.17' ]
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.1.7
         with:
           fetch-depth: 0
-      - uses: olafurpg/setup-scala@v14
+      - uses: cachix/install-nix-action@v27
-        with:
-          java-version: ${{ matrix.java }}
-      - uses: jorelali/setup-elm@v5
-        with:
-          elm-version: 0.19.1
-      - uses: bahmutov/npm-install@v1
-        with:
-          working-directory: modules/webapp
-      # - name: Coursier cache
-      #   uses: coursier/cache-action@v6
       - name: Set current version
         run:  echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
       - name: sbt ci ${{ github.ref }}
-        run: sbt ci
+        run: nix develop .#ci --command sbt ci
       - name: sbt make-pkg (${{ env.DOCSPELL_VERSION }})
-        run: sbt make-pkg
+        run: nix develop .#ci --command sbt make-pkg
       - uses: "marvinpinto/action-automatic-releases@latest"
         with:
           repo_token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/release.yml

@@ -5,30 +5,18 @@ on:
       - 'v*'
 jobs:
   release:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: true
-      matrix:
-        java: [ 'openjdk@1.17' ]
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.1.7
         with:
           fetch-depth: 0
-      - uses: olafurpg/setup-scala@v14
+      - uses: cachix/install-nix-action@v27
-        with:
-          java-version: ${{ matrix.java }}
-      - uses: jorelali/setup-elm@v5
-        with:
-          elm-version: 0.19.1
-      - uses: bahmutov/npm-install@v1
-        with:
-          working-directory: modules/webapp
-      # - name: Coursier cache
-      #   uses: coursier/cache-action@v6
       - name: Set current version
         run:  echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
       - name: sbt make-pkg (${{ env.DOCSPELL_VERSION }})
-        run: sbt make-pkg
+        run: nix develop .#ci --command sbt make-pkg
       - uses: meeDamian/github-release@2.0
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/stale.yml

@@ -0,0 +1,21 @@
+name: 'Handle stale issues'
+on:
+  schedule:
+    - cron: '30 1 * * *'
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      # https://github.com/actions/stale
+      - uses: actions/stale@v9
+        with:
+          days-before-stale: 30
+          days-before-close: 7
+          only-labels: question
+          stale-issue-label: stale
+          stale-issue-message: >
+            This issue has been automatically marked as stale because it has not
+            had recent activity. It will be closed if no further activity
+            occurs. This only applies to 'question' issues. Always feel free to
+            reopen or create new issues. Thank you!

.github/workflows/website.yml

@@ -5,24 +5,17 @@ on:
       - "current-docs"
 jobs:
   publish-website:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.1.7
         with:
           fetch-depth: 0
+      - uses: cachix/install-nix-action@v27
       - name: Set current version
         run:  echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
-      - uses: jorelali/setup-elm@v5
-        with:
-          elm-version: 0.19.1
-      - uses: cachix/install-nix-action@v25
-        with:
-          nix_path: nixpkgs=channel:nixos-23.05
-      - name: Print nixpkgs version
-        run: nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
       - name: Build website (${{ env.DOCSPELL_VERSION }})
-        run: nix-shell website/shell.nix --run "sbt make-website"
+        run: nix develop .#ci --command sbt make-website
       - name: Publish website (${{ env.DOCSPELL_VERSION }})
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: sbt publish-website
+        run: nix develop .#ci --command sbt publish-website

.gitignore

@@ -1,4 +1,5 @@
 #artwork/*.png
+.envrc
 target/
 local/
 node_modules/

.mergify.yml

@@ -6,7 +6,7 @@ pull_request_rules:
       assign:
         users: [eikek]
       label:
-        add: ["type: dependencies"]
+        add: ["dependencies"]
   - name: automatically merge Scala Steward PRs on CI success
     conditions:
       - author=eikek-scala-steward[bot]

.scalafmt.conf

@@ -1,4 +1,4 @@
-version = "3.7.17"
+version = "3.8.2"
 
 preset = default
 align.preset = some

Changelog.md

@@ -1020,7 +1020,7 @@ Additionally there are some other minor features and bug fixes.
     to be able to add a request header. Check [this for
     firefox](https://addons.mozilla.org/en-US/firefox/addon/modheader-firefox/)
     or [this for
-    chromium](https://chrome.google.com/webstore/detail/modheader/idgpnmonknjnojddfkpgkljpfnnfcklj)
+    chromium](https://chromewebstore.google.com/detail/modheader-modify-http-hea/idgpnmonknjnojddfkpgkljpfnnfcklj)
   - then add the request header `Docspell-Ui` with value `1`.
     Reloading the page gets you back the old ui.
 - With new Web-UI, certain features and fixes were realized, but not

build.sbt

@@ -15,11 +15,14 @@ val scalafixSettings = Seq(
 
 val sharedSettings = Seq(
   organization := "com.github.eikek",
-  scalaVersion := "2.13.12",
+  scalaVersion := "2.13.14",
   organizationName := "Eike K. & Contributors",
-  licenses += ("AGPL-3.0-or-later", url(
+  licenses += (
-    "https://spdx.org/licenses/AGPL-3.0-or-later.html"
+    "AGPL-3.0-or-later",
-  )),
+    url(
+      "https://spdx.org/licenses/AGPL-3.0-or-later.html"
+    )
+  ),
   startYear := Some(2020),
   headerLicenseStyle := HeaderLicenseStyle.SpdxSyntax,
   headerSources / excludeFilter := HiddenFileFilter || "*.java" || "StringUtil.scala",
@@ -677,7 +680,11 @@ val restapi = project
     openapiTargetLanguage := Language.Scala,
     openapiPackage := Pkg("docspell.restapi.model"),
     openapiSpec := (Compile / resourceDirectory).value / "docspell-openapi.yml",
-    openapiStaticGen := OpenApiDocGenerator.Redoc
+    openapiStaticGen := OpenApiDocGenerator.Redoc,
+    openapiRedoclyCmd := Seq("redocly-cli"),
+    openapiRedoclyConfig := Some(
+      (LocalRootProject / baseDirectory).value / "project" / "redocly.yml"
+    )
   )
   .dependsOn(common, query.jvm, notificationApi, jsonminiq, addonlib)
 
@@ -697,7 +704,11 @@ val joexapi = project
     openapiTargetLanguage := Language.Scala,
     openapiPackage := Pkg("docspell.joexapi.model"),
     openapiSpec := (Compile / resourceDirectory).value / "joex-openapi.yml",
-    openapiStaticGen := OpenApiDocGenerator.Redoc
+    openapiStaticGen := OpenApiDocGenerator.Redoc,
+    openapiRedoclyCmd := Seq("redocly-cli"),
+    openapiRedoclyConfig := Some(
+      (LocalRootProject / baseDirectory).value / "project" / "redocly.yml"
+    )
   )
   .dependsOn(common, loggingScribe, addonlib)
 

docker/docker-compose/docker-compose.yml

@@ -109,7 +109,7 @@ services:
       - restserver
 
   db:
-    image: postgres:16.1
+    image: postgres:16.3
     container_name: postgres_db
     restart: unless-stopped
     volumes:

docker/dockerfiles/joex.dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:20231219
+FROM alpine:3.20.2
 
 ARG version=
 ARG joex_url=
@@ -77,7 +77,7 @@ RUN \
   wget https://github.com/tesseract-ocr/tessdata/raw/main/khm.traineddata && \
   mv khm.traineddata /usr/share/tessdata
 
-# Using these data files for japanese, because they work better. See #973
+# Using these data files for japanese, because they work better. Includes vertical data. See #973 and #2445.
 RUN \
   wget https://raw.githubusercontent.com/tesseract-ocr/tessdata_fast/master/jpn_vert.traineddata && \
   wget https://raw.githubusercontent.com/tesseract-ocr/tessdata_fast/master/jpn.traineddata && \

docker/dockerfiles/restserver.dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:20231219
+FROM alpine:3.20.2
 
 ARG version=
 ARG restserver_url=

flake.lock

@@ -0,0 +1,130 @@
+{
+  "nodes": {
+    "devshell-tools": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      },
+      "locked": {
+        "lastModified": 1710099997,
+        "narHash": "sha256-WmBKTLdth6I/D+0//9enbIXohGsBjepbjIAm9pCYj0U=",
+        "owner": "eikek",
+        "repo": "devshell-tools",
+        "rev": "e82faf976d318b3829f6f7f6785db6f3c7b65267",
+        "type": "github"
+      },
+      "original": {
+        "owner": "eikek",
+        "repo": "devshell-tools",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1709126324,
+        "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "d465f4819400de7c8d874d50b982301f28a84605",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
+      "inputs": {
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1709126324,
+        "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "d465f4819400de7c8d874d50b982301f28a84605",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1709309926,
+        "narHash": "sha256-VZFBtXGVD9LWTecGi6eXrE0hJ/mVB3zGUlHImUs2Qak=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "79baff8812a0d68e24a836df0a364c678089e2c7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-23.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1709677081,
+        "narHash": "sha256-tix36Y7u0rkn6mTm0lA45b45oab2cFLqAzDbJxeXS+c=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "880992dcc006a5e00dd0591446fdf723e6a51a64",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-23.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "devshell-tools": "devshell-tools",
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": "nixpkgs_2"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,193 @@
+{
+  description = "Docspell";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11";
+    devshell-tools.url = "github:eikek/devshell-tools";
+    flake-utils.url = "github:numtide/flake-utils";
+  };
+
+  outputs = {
+    self,
+    nixpkgs,
+    flake-utils,
+    devshell-tools,
+  }:
+    flake-utils.lib.eachDefaultSystem (system: let
+      pkgs = nixpkgs.legacyPackages.${system};
+      sbt17 = pkgs.sbt.override {jre = pkgs.jdk17;};
+      ciPkgs = with pkgs; [
+        sbt17
+        jdk17
+        dpkg
+        elmPackages.elm
+        fakeroot
+        zola
+        yarn
+        nodejs
+        redocly-cli
+        tailwindcss
+      ];
+      devshellPkgs =
+        ciPkgs
+        ++ (with pkgs; [
+          jq
+          scala-cli
+          netcat
+          wget
+          which
+          inotifyTools
+        ]);
+      docspellPkgs = pkgs.callPackage (import ./nix/pkg.nix) {};
+      dockerAmd64 = pkgs.pkgsCross.gnu64.callPackage (import ./nix/docker.nix) {
+        inherit (docspellPkgs) docspell-restserver docspell-joex;
+      };
+      dockerArm64 = pkgs.pkgsCross.aarch64-multiplatform.callPackage (import ./nix/docker.nix) {
+        inherit (docspellPkgs) docspell-restserver docspell-joex;
+      };
+    in {
+      formatter = pkgs.alejandra;
+
+      packages = {
+        inherit (docspellPkgs) docspell-restserver docspell-joex;
+      };
+
+      legacyPackages = {
+        docker = {
+          amd64 = {
+            inherit (dockerAmd64) docspell-restserver docspell-joex;
+          };
+          arm64 = {
+            inherit (dockerArm64) docspell-restserver docspell-joex;
+          };
+        };
+      };
+
+      checks = {
+        build-server = self.packages.${system}.docspell-restserver;
+        build-joex = self.packages.${system}.docspell-joex;
+
+        test = with import (nixpkgs + "/nixos/lib/testing-python.nix")
+        {
+          inherit system;
+        };
+          makeTest {
+            name = "docspell";
+            nodes = {
+              machine = {...}: {
+                nixpkgs.overlays = [self.overlays.default];
+                imports = [
+                  self.nixosModules.default
+                  ./nix/checks
+                ];
+              };
+            };
+
+            testScript = builtins.readFile ./nix/checks/testScript.py;
+          };
+      };
+
+      devShells = {
+        dev-cnt = pkgs.mkShellNoCC {
+          buildInputs =
+            (builtins.attrValues devshell-tools.legacyPackages.${system}.cnt-scripts)
+            ++ devshellPkgs;
+
+          DOCSPELL_ENV = "dev";
+          DEV_CONTAINER = "docsp-dev";
+          SBT_OPTS = "-Xmx2G -Xss4m";
+        };
+        dev-vm = pkgs.mkShellNoCC {
+          buildInputs =
+            (builtins.attrValues devshell-tools.legacyPackages.${system}.vm-scripts)
+            ++ devshellPkgs;
+
+          DOCSPELL_ENV = "dev";
+          SBT_OPTS = "-Xmx2G -Xss4m";
+          DEV_VM = "dev-vm";
+          VM_SSH_PORT = "10022";
+        };
+        ci = pkgs.mkShellNoCC {
+          buildInputs = ciPkgs;
+          SBT_OPTS = "-Xmx2G -Xss4m";
+        };
+      };
+    })
+    // {
+      nixosModules = {
+        default = {...}: {
+          imports = [
+            ./nix/modules/server.nix
+            ./nix/modules/joex.nix
+          ];
+        };
+        server = import ./nix/modules/server.nix;
+        joex = import ./nix/modules/joex.nix;
+      };
+
+      overlays.default = final: prev: let
+        docspellPkgs = final.callPackage (import ./nix/pkg.nix) {};
+      in {
+        inherit (docspellPkgs) docspell-restserver docspell-joex;
+      };
+
+      nixosConfigurations = {
+        test-vm = devshell-tools.lib.mkVm {
+          system = "x86_64-linux";
+          modules = [
+            self.nixosModules.default
+            {
+              nixpkgs.overlays = [self.overlays.default];
+            }
+            ./nix/test-vm.nix
+          ];
+        };
+        docsp-dev = devshell-tools.lib.mkContainer {
+          system = "x86_64-linux";
+          modules = [
+            {
+              services.dev-postgres = {
+                enable = true;
+                databases = ["docspell"];
+              };
+              services.dev-email.enable = true;
+              services.dev-minio.enable = true;
+              services.dev-solr = {
+                enable = true;
+                cores = ["docspell"];
+              };
+            }
+          ];
+        };
+        dev-vm = devshell-tools.lib.mkVm {
+          system = "x86_64-linux";
+          modules = [
+            {
+              networking.hostName = "dev-vm";
+              virtualisation.memorySize = 2048;
+
+              services.dev-postgres = {
+                enable = true;
+                databases = ["docspell"];
+              };
+              services.dev-email.enable = true;
+              services.dev-minio.enable = true;
+              services.dev-solr = {
+                enable = true;
+                cores = ["docspell"];
+                heap = 512;
+              };
+              port-forward.ssh = 10022;
+              port-forward.dev-postgres = 6534;
+              port-forward.dev-smtp = 10025;
+              port-forward.dev-imap = 10143;
+              port-forward.dev-webmail = 8080;
+              port-forward.dev-minio-api = 9000;
+              port-forward.dev-minio-console = 9001;
+              port-forward.dev-solr = 8983;
+            }
+          ];
+        };
+      };
+    };
+}

modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala

@@ -38,9 +38,9 @@ final case class AddonArchive(url: LenientUri, name: String, version: String) {
           Files[F].createDirectories(target) *>
             reader(url)
               .through(Zip[F](logger.some).unzip(glob = glob, targetDir = target.some))
-              .evalTap(_ => Directory.unwrapSingle[F](logger, target))
               .compile
               .drain
+              .flatTap(_ => Directory.unwrapSingle[F](logger, target))
               .as(target)
       }
   }

modules/addonlib/src/main/scala/docspell/addons/runner/RunnerUtil.scala

@@ -110,7 +110,7 @@ private[addons] object RunnerUtil {
   ): F[AddonResult] =
     for {
       stdout <-
-        if (ctx.meta.options.exists(_.collectOutput)) CollectOut.buffer[F]
+        if (ctx.meta.parseResult) CollectOut.buffer[F]
         else CollectOut.none[F].pure[F]
       cmdResult <- SysExec(cmd, logger, ctx.baseDir.some)
         .flatMap(
@@ -135,7 +135,7 @@ private[addons] object RunnerUtil {
             out <- stdout.get
             _ <- logger.debug(s"Addon stdout: $out")
             result = Option
-              .when(ctx.meta.options.exists(_.collectOutput) && out.nonEmpty)(
+              .when(ctx.meta.parseResult && out.nonEmpty)(
                 JsonParser
                   .decode[AddonOutput](out)
                   .fold(AddonResult.decodingError, AddonResult.success)

modules/addonlib/src/test/scala/docspell/addons/AddonArchiveTest.scala

@@ -9,7 +9,7 @@ package docspell.addons
 import cats.effect._
 import cats.syntax.option._
 
-import docspell.common.UrlReader
+import docspell.common._
 import docspell.logging.TestLoggingConfig
 
 import munit._
@@ -42,10 +42,20 @@ class AddonArchiveTest extends CatsEffectSuite with TestLoggingConfig with Fixtu
     } yield ()
   }
 
+  tempDir.test("read archive from zip with yaml only") { dir =>
+    for {
+      aa <- AddonArchive.read[IO](singleFileAddonUrl, UrlReader.defaultReader[IO], None)
+      _ = assertEquals(aa.version, "0.7.0")
+      path <- aa.extractTo(UrlReader.defaultReader[IO], dir)
+      read <- AddonArchive.read[IO](aa.url, UrlReader.defaultReader[IO], path.some)
+      _ = assertEquals(aa, read)
+    } yield ()
+  }
+
   tempDir.test("Read generated addon from path") { dir =>
     AddonGenerator.successAddon("mini-addon").use { addon =>
       for {
-        archive <- IO(AddonArchive(addon.url, "", ""))
+        archive <- IO(AddonArchive(addon.url, "test-addon", "0.1.0"))
         path <- archive.extractTo[IO](UrlReader.defaultReader[IO], dir)
 
         read <- AddonArchive.read[IO](addon.url, UrlReader.defaultReader[IO], path.some)

modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala

@@ -142,7 +142,7 @@ class AddonExecutorTest extends CatsEffectSuite with Fixtures with TestLoggingCo
       AddonExecutionResult.executionResultMonoid
         .combine(
           AddonExecutionResult.empty,
-          AddonExecutionResult(Nil, true)
+          AddonExecutionResult(Nil, pure = true)
         )
         .pure
     )

modules/addonlib/src/test/scala/docspell/addons/AddonGenerator.scala

@@ -27,9 +27,9 @@ object AddonGenerator {
   ): Resource[IO, AddonArchive] =
     output match {
       case None =>
-        generate(name, version, false)("exit 0")
+        generate(name, version, collectOutput = false)("exit 0")
       case Some(out) =>
-        generate(name, version, true)(
+        generate(name, version, collectOutput = true)(
           s"""
              |cat <<-EOF
              |${out.asJson.noSpaces}
@@ -77,8 +77,9 @@ object AddonGenerator {
       meta = AddonMeta.Meta(name, version, None),
       triggers = Set(AddonTriggerType.ExistingItem: AddonTriggerType).some,
       args = None,
-      runner =
+      runner = AddonMeta
-        AddonMeta.Runner(None, None, AddonMeta.TrivialRunner(true, "addon.sh").some).some,
+        .Runner(None, None, AddonMeta.TrivialRunner(enable = true, "addon.sh").some)
+        .some,
       options =
         AddonMeta.Options(networking = !collectOutput, collectOutput = collectOutput).some
     )

modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala

@@ -35,4 +35,13 @@ class AddonMetaTest extends CatsEffectSuite with TestLoggingConfig with Fixtures
       _ = assertEquals(meta, dummyAddonMeta)
     } yield ()
   }
+
+  test("parse yaml with defaults") {
+    val yamlStr = """meta:
+                    |  name: "test"
+                    |  version: "0.1.0"
+                    |""".stripMargin
+    val meta = AddonMeta.fromYamlString(yamlStr).fold(throw _, identity)
+    assert(meta.parseResult)
+  }
 }

modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala

@@ -31,6 +31,9 @@ trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite =>
   val miniAddonUrl =
     LenientUri.fromJava(getClass.getResource("/minimal-addon.zip"))
 
+  val singleFileAddonUrl =
+    LenientUri.fromJava(getClass.getResource("/docspell-addon-single-file.zip"))
+
   val dummyAddonMeta =
     AddonMeta(
       meta =
@@ -40,13 +43,13 @@ trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite =>
       ),
       None,
       runner = Runner(
-        nix = NixRunner(true).some,
+        nix = NixRunner(enable = true).some,
         docker = DockerRunner(
           enable = true,
           image = None,
           build = "Dockerfile".some
         ).some,
-        trivial = TrivialRunner(true, "src/addon.sh").some
+        trivial = TrivialRunner(enable = true, "src/addon.sh").some
       ).some,
       options = Options(networking = true, collectOutput = true).some
     )
@@ -55,7 +58,7 @@ trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite =>
     Path(s"/tmp/target/test-temp")
 
   val tempDir =
-    ResourceFixture[Path](
+    ResourceFunFixture[Path](
       Resource.eval(Files[IO].createDirectories(baseTempDir)) *>
         Files[IO]
           .tempDirectory(baseTempDir.some, "run-", PosixPermissions.fromOctal("777"))
@@ -65,7 +68,7 @@ trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite =>
       runner: RunnerType,
       runners: RunnerType*
   ): AddonExecutorConfig = {
-    val nspawn = NSpawn(false, "sudo", "systemd-nspawn", Duration.millis(100))
+    val nspawn = NSpawn(enabled = false, "sudo", "systemd-nspawn", Duration.millis(100))
     AddonExecutorConfig(
       runner = runner :: runners.toList,
       runTimeout = Duration.minutes(2),

modules/analysis/src/main/scala/docspell/analysis/date/DateFind.scala

@@ -125,6 +125,7 @@ object DateFind {
         case Language.Dutch      => dmy.or(ymd).or(mdy)
         case Language.Latvian    => dmy.or(lavLong).or(ymd)
         case Language.Japanese   => ymd
+        case Language.JpnVert    => ymd
         case Language.Hebrew     => dmy
         case Language.Lithuanian => ymd
         case Language.Polish     => dmy

modules/analysis/src/main/scala/docspell/analysis/date/MonthName.scala

@@ -54,6 +54,8 @@ object MonthName {
         latvian
       case Language.Japanese =>
         japanese
+      case Language.JpnVert =>
+        japanese
       case Language.Hebrew =>
         hebrew
       case Language.Lithuanian =>

modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala

@@ -22,7 +22,7 @@ import munit._
 
 class StanfordNerAnnotatorSuite extends FunSuite with TestLoggingConfig {
   lazy val germanClassifier =
-    new StanfordCoreNLP(Properties.nerGerman(None, false))
+    new StanfordCoreNLP(Properties.nerGerman(None, highRecall = false))
   lazy val englishClassifier =
     new StanfordCoreNLP(Properties.nerEnglish(None))
 

modules/backend/src/main/scala/docspell/backend/Config.scala

@@ -90,6 +90,6 @@ object Config {
   }
   object Addons {
     val disabled: Addons =
-      Addons(false, false, UrlMatcher.False, UrlMatcher.True)
+      Addons(enabled = false, allowImpure = false, UrlMatcher.False, UrlMatcher.True)
   }
 }

modules/backend/src/main/scala/docspell/backend/auth/Login.scala

@@ -127,7 +127,7 @@ object Login {
           _ <- logF.trace(s"Account lookup: $data")
           res <- data match {
             case Some(d) if checkNoPassword(d, Set(AccountSource.OpenId)) =>
-              doLogin(config, d.account, false)
+              doLogin(config, d.account, rememberMe = false)
             case Some(d) if checkNoPassword(d, Set(AccountSource.Local)) =>
               config.onAccountSourceConflict match {
                 case OnAccountSourceConflict.Fail =>
@@ -145,7 +145,7 @@ object Login {
                           AccountSource.OpenId
                         )
                       )
-                    res <- doLogin(config, d.account, false)
+                    res <- doLogin(config, d.account, rememberMe = false)
                   } yield res
               }
             case _ =>
@@ -212,7 +212,12 @@ object Login {
         val okResult: F[Result] =
           for {
             _ <- store.transact(RUser.updateLogin(sf.token.account))
-            newToken <- AuthToken.user(sf.token.account, false, config.serverSecret, None)
+            newToken <- AuthToken.user(
+              sf.token.account,
+              requireSecondFactor = false,
+              config.serverSecret,
+              None
+            )
             rem <- OptionT
               .whenF(sf.rememberMe && config.rememberMe.enabled)(
                 insertRememberToken(store, sf.token.account, config)
@@ -239,7 +244,9 @@ object Login {
         (for {
           _ <- validateToken
           key <- EitherT.fromOptionF(
-            store.transact(RTotp.findEnabledByUserId(sf.token.account.userId, true)),
+            store.transact(
+              RTotp.findEnabledByUserId(sf.token.account.userId, enabled = true)
+            ),
             Result.invalidAuth
           )
           now <- EitherT.right[Result](Timestamp.current[F])
@@ -255,7 +262,12 @@ object Login {
         def okResult(acc: AccountInfo) =
           for {
             _ <- store.transact(RUser.updateLogin(acc))
-            token <- AuthToken.user(acc, false, config.serverSecret, None)
+            token <- AuthToken.user(
+              acc,
+              requireSecondFactor = false,
+              config.serverSecret,
+              None
+            )
           } yield Result.ok(token, None)
 
         def rememberedLogin(rid: Ident) =

modules/backend/src/main/scala/docspell/backend/joex/AddonOps.scala

@@ -93,7 +93,7 @@ object AddonOps {
           AddonResult.executionFailed(
             new Exception(s"Addon run config ${id.id} not found.")
           ) :: Nil,
-          false
+          pure = false
         ) :: Nil,
         Nil
       )

modules/backend/src/main/scala/docspell/backend/joex/AddonPrepare.scala

@@ -72,7 +72,7 @@ private[joex] class AddonPrepare[F[_]: Sync](store: Store[F]) extends LoggerExte
 
       token <- AuthToken.user(
         account,
-        false,
+        requireSecondFactor = false,
         secret.getOrElse(ByteVector.empty),
         tokenValidity.some
       )

modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala

@@ -194,7 +194,14 @@ object OCollective {
           id <- Ident.randomId[F]
           settings = sett.emptyTrash.getOrElse(EmptyTrash.default)
           args = EmptyTrashArgs(cid, settings.minAge)
-          ut = UserTask(id, EmptyTrashArgs.taskName, true, settings.schedule, None, args)
+          ut = UserTask(
+            id,
+            EmptyTrashArgs.taskName,
+            enabled = true,
+            settings.schedule,
+            None,
+            args
+          )
           _ <- uts.updateOneTask(UserTaskScope.collective(cid), args.makeSubject.some, ut)
           _ <- joex.notifyAllNodes
         } yield ()
@@ -220,7 +227,7 @@ object OCollective {
           ut = UserTask(
             id,
             LearnClassifierArgs.taskName,
-            true,
+            enabled = true,
             CalEvent(WeekdayComponent.All, DateEvent.All, TimeEvent.All),
             None,
             args
@@ -239,7 +246,7 @@ object OCollective {
           ut = UserTask(
             id,
             EmptyTrashArgs.taskName,
-            true,
+            enabled = true,
             CalEvent(WeekdayComponent.All, DateEvent.All, TimeEvent.All),
             None,
             args

modules/backend/src/main/scala/docspell/backend/ops/ONotification.scala

@@ -114,14 +114,14 @@ object ONotification {
           )
           _ <- notMod.send(logbuf._2.andThen(log), ev, ch)
           logs <- logbuf._1.get
-          res = SendTestResult(true, logs)
+          res = SendTestResult(success = true, logs)
         } yield res).attempt
           .map {
             case Right(res) => res
             case Left(ex) =>
               val ev =
                 LogEvent.of(Level.Error, "Failed sending sample event").addError(ex)
-              SendTestResult(false, Vector(ev))
+              SendTestResult(success = false, Vector(ev))
           }
 
       def listChannels(userId: Ident): F[Vector[Channel]] =

modules/backend/src/main/scala/docspell/backend/ops/OTotp.scala

@@ -120,7 +120,9 @@ object OTotp {
       def confirmInit(accountId: AccountInfo, otp: OnetimePassword): F[ConfirmResult] =
         for {
           _ <- log.info(s"Confirm TOTP setup for account ${accountId.asString}")
-          key <- store.transact(RTotp.findEnabledByUserId(accountId.userId, false))
+          key <- store.transact(
+            RTotp.findEnabledByUserId(accountId.userId, enabled = false)
+          )
           now <- Timestamp.current[F]
           res <- key match {
             case None =>
@@ -129,7 +131,7 @@ object OTotp {
               val check = totp.checkPassword(r.secret, otp, now.value)
               if (check)
                 store
-                  .transact(RTotp.setEnabled(accountId.userId, true))
+                  .transact(RTotp.setEnabled(accountId.userId, enabled = true))
                   .map(_ => ConfirmResult.Success)
               else ConfirmResult.Failed.pure[F]
           }
@@ -140,7 +142,7 @@ object OTotp {
           case Some(pw) =>
             for {
               _ <- log.info(s"Validating TOTP, because it is requested to disable it.")
-              key <- store.transact(RTotp.findEnabledByLogin(accountId, true))
+              key <- store.transact(RTotp.findEnabledByLogin(accountId, enabled = true))
               now <- Timestamp.current[F]
               res <- key match {
                 case None =>
@@ -149,7 +151,7 @@ object OTotp {
                   val check = totp.checkPassword(r.secret, pw, now.value)
                   if (check)
                     UpdateResult.fromUpdate(
-                      store.transact(RTotp.setEnabled(r.userId, false))
+                      store.transact(RTotp.setEnabled(r.userId, enabled = false))
                     )
                   else
                     log.info(s"TOTP code was invalid. Not disabling it.") *> UpdateResult
@@ -160,15 +162,15 @@ object OTotp {
           case None =>
             UpdateResult.fromUpdate {
               (for {
-                key <- OptionT(RTotp.findEnabledByLogin(accountId, true))
+                key <- OptionT(RTotp.findEnabledByLogin(accountId, enabled = true))
-                n <- OptionT.liftF(RTotp.setEnabled(key.userId, false))
+                n <- OptionT.liftF(RTotp.setEnabled(key.userId, enabled = false))
               } yield n).mapK(store.transform).getOrElse(0)
             }
         }
 
       def state(acc: AccountInfo): F[OtpState] =
         for {
-          record <- store.transact(RTotp.findEnabledByUserId(acc.userId, true))
+          record <- store.transact(RTotp.findEnabledByUserId(acc.userId, enabled = true))
           result = record match {
             case Some(r) =>
               OtpState.Enabled(r.created)

modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala

@@ -159,7 +159,7 @@ object OUpload {
             data.meta.skipDuplicates,
             data.meta.fileFilter.some,
             data.meta.tags.some,
-            false,
+            reprocess = false,
             data.meta.attachmentsOnly,
             data.meta.customData
           )

modules/backend/src/test/scala/docspell/backend/auth/AuthTokenTest.scala

@@ -32,9 +32,12 @@ class AuthTokenTest extends CatsEffectSuite {
   val otherSecret = ByteVector.fromValidHex("16bad")
 
   test("validate") {
-    val token1 = AuthToken.user[IO](user, false, secret, None).unsafeRunSync()
+    val token1 =
+      AuthToken.user[IO](user, requireSecondFactor = false, secret, None).unsafeRunSync()
     val token2 =
-      AuthToken.user[IO](user, false, secret, Duration.seconds(10).some).unsafeRunSync()
+      AuthToken
+        .user[IO](user, requireSecondFactor = false, secret, Duration.seconds(10).some)
+        .unsafeRunSync()
     assert(token1.validate(secret, Duration.seconds(5)))
     assert(!token1.validate(otherSecret, Duration.seconds(5)))
     assert(!token1.copy(account = john).validate(secret, Duration.seconds(5)))
@@ -46,9 +49,12 @@ class AuthTokenTest extends CatsEffectSuite {
   }
 
   test("signature") {
-    val token1 = AuthToken.user[IO](user, false, secret, None).unsafeRunSync()
+    val token1 =
+      AuthToken.user[IO](user, requireSecondFactor = false, secret, None).unsafeRunSync()
     val token2 =
-      AuthToken.user[IO](user, false, secret, Duration.seconds(10).some).unsafeRunSync()
+      AuthToken
+        .user[IO](user, requireSecondFactor = false, secret, Duration.seconds(10).some)
+        .unsafeRunSync()
 
     assert(token1.sigValid(secret))
     assert(token1.sigInvalid(otherSecret))

modules/common/src/main/scala/docspell/common/Language.scala

@@ -123,6 +123,11 @@ object Language {
     val iso3 = "jpn"
   }
 
+  /*It's not an ISO value, but this needs to be unique and tesseract will need jpn_vert for it's scan from the config of /etc/docspell-joex/docspell-joex.conf.*/
+  case object JpnVert extends Language {
+    val iso2 = "ja_vert"
+    val iso3 = "jpn_vert"
+  }
   case object Hebrew extends Language {
     val iso2 = "he"
     val iso3 = "heb"
@@ -172,6 +177,7 @@ object Language {
       Romanian,
       Latvian,
       Japanese,
+      JpnVert,
       Hebrew,
       Lithuanian,
       Polish,

modules/common/src/main/scala/docspell/common/LenientUri.scala

@@ -78,7 +78,11 @@ case class LenientUri(
       .covary[F]
       .rethrow
       .flatMap(url =>
-        fs2.io.readInputStream(Sync[F].delay(url.openStream()), chunkSize, true)
+        fs2.io.readInputStream(
+          Sync[F].delay(url.openStream()),
+          chunkSize,
+          closeAfterUse = true
+        )
       )
 
   def readText[F[_]: Sync](chunkSize: Int): F[String] =
@@ -121,7 +125,7 @@ object LenientUri {
     val isRoot = true
     val isEmpty = false
     def /(seg: String): Path =
-      NonEmptyPath(NonEmptyList.of(seg), false)
+      NonEmptyPath(NonEmptyList.of(seg), trailingSlash = false)
     def asString = "/"
   }
   case object EmptyPath extends Path {
@@ -129,7 +133,7 @@ object LenientUri {
     val isRoot = false
     val isEmpty = true
     def /(seg: String): Path =
-      NonEmptyPath(NonEmptyList.of(seg), false)
+      NonEmptyPath(NonEmptyList.of(seg), trailingSlash = false)
     def asString = ""
   }
   case class NonEmptyPath(segs: NonEmptyList[String], trailingSlash: Boolean)

modules/common/src/main/scala/docspell/common/MimeType.scala

@@ -194,7 +194,7 @@ object MimeType {
           val csValueStart = in.substring(n + "charset=".length).trim
           val csName = csValueStart.indexOf(';') match {
             case -1 => unquote(csValueStart).trim
-            case n  => unquote(csValueStart.substring(0, n)).trim
+            case n2 => unquote(csValueStart.substring(0, n2)).trim
           }
           if (Charset.isSupported(csName)) Right((Some(Charset.forName(csName)), ""))
           else Right((None, ""))

modules/common/src/main/scala/docspell/common/SystemCommand.scala

@@ -1,212 +0,0 @@
-/*
- * Copyright 2020 Eike K. & Contributors
- *
- * SPDX-License-Identifier: AGPL-3.0-or-later
- */
-
-package docspell.common
-
-import java.io.InputStream
-import java.lang.ProcessBuilder.Redirect
-import java.util.concurrent.TimeUnit
-
-import scala.jdk.CollectionConverters._
-
-import cats.effect._
-import cats.implicits._
-import fs2.io.file.Path
-import fs2.{Stream, io, text}
-
-import docspell.common.{exec => newExec}
-import docspell.logging.Logger
-
-// better use `SysCmd` and `SysExec`
-object SystemCommand {
-
-  final case class Config(
-      program: String,
-      args: Seq[String],
-      timeout: Duration,
-      env: Map[String, String] = Map.empty
-  ) {
-
-    def toSysCmd = newExec
-      .SysCmd(program, newExec.Args(args))
-      .withTimeout(timeout)
-      .addEnv(newExec.Env(env))
-
-    def mapArgs(f: String => String): Config =
-      Config(program, args.map(f), timeout)
-
-    def replace(repl: Map[String, String]): Config =
-      mapArgs(s =>
-        repl.foldLeft(s) { case (res, (k, v)) =>
-          res.replace(k, v)
-        }
-      )
-
-    def withEnv(key: String, value: String): Config =
-      copy(env = env.updated(key, value))
-
-    def addEnv(moreEnv: Map[String, String]): Config =
-      copy(env = env ++ moreEnv)
-
-    def appendArgs(extraArgs: Args): Config =
-      copy(args = args ++ extraArgs.args)
-
-    def appendArgs(extraArgs: Seq[String]): Config =
-      copy(args = args ++ extraArgs)
-
-    def toCmd: List[String] =
-      program :: args.toList
-
-    lazy val cmdString: String =
-      toCmd.mkString(" ")
-  }
-
-  final case class Args(args: Vector[String]) extends Iterable[String] {
-    override def iterator = args.iterator
-
-    def prepend(a: String): Args = Args(a +: args)
-
-    def prependWhen(flag: Boolean)(a: String): Args =
-      prependOption(Option.when(flag)(a))
-
-    def prependOption(value: Option[String]): Args =
-      value.map(prepend).getOrElse(this)
-
-    def append(a: String, as: String*): Args =
-      Args(args ++ (a +: as.toVector))
-
-    def appendOption(value: Option[String]): Args =
-      value.map(append(_)).getOrElse(this)
-
-    def appendOptionVal(first: String, second: Option[String]): Args =
-      second.map(b => append(first, b)).getOrElse(this)
-
-    def appendWhen(flag: Boolean)(a: String, as: String*): Args =
-      if (flag) append(a, as: _*) else this
-
-    def appendWhenNot(flag: Boolean)(a: String, as: String*): Args =
-      if (!flag) append(a, as: _*) else this
-
-    def append(p: Path): Args =
-      append(p.toString)
-
-    def append(as: Iterable[String]): Args =
-      Args(args ++ as.toVector)
-  }
-  object Args {
-    val empty: Args = Args()
-
-    def apply(as: String*): Args =
-      Args(as.toVector)
-  }
-
-  final case class Result(rc: Int, stdout: String, stderr: String)
-
-  def exec[F[_]: Sync](
-      cmd: Config,
-      logger: Logger[F],
-      wd: Option[Path] = None,
-      stdin: Stream[F, Byte] = Stream.empty
-  ): Stream[F, Result] =
-    startProcess(cmd, wd, logger, stdin) { proc =>
-      Stream.eval {
-        for {
-          _ <- writeToProcess(stdin, proc)
-          term <- Sync[F].blocking(proc.waitFor(cmd.timeout.seconds, TimeUnit.SECONDS))
-          _ <-
-            if (term)
-              logger.debug(s"Command `${cmd.cmdString}` finished: ${proc.exitValue}")
-            else
-              logger.warn(
-                s"Command `${cmd.cmdString}` did not finish in ${cmd.timeout.formatExact}!"
-              )
-          _ <- if (!term) timeoutError(proc, cmd) else Sync[F].pure(())
-          out <-
-            if (term) inputStreamToString(proc.getInputStream)
-            else Sync[F].pure("")
-          err <-
-            if (term) inputStreamToString(proc.getErrorStream)
-            else Sync[F].pure("")
-        } yield Result(proc.exitValue, out, err)
-      }
-    }
-
-  def execSuccess[F[_]: Sync](
-      cmd: Config,
-      logger: Logger[F],
-      wd: Option[Path] = None,
-      stdin: Stream[F, Byte] = Stream.empty
-  ): Stream[F, Result] =
-    exec(cmd, logger, wd, stdin).flatMap { r =>
-      if (r.rc != 0)
-        Stream.raiseError[F](
-          new Exception(
-            s"Command `${cmd.cmdString}` returned non-zero exit code ${r.rc}. Stderr: ${r.stderr}"
-          )
-        )
-      else Stream.emit(r)
-    }
-
-  private def startProcess[F[_]: Sync, A](
-      cmd: Config,
-      wd: Option[Path],
-      logger: Logger[F],
-      stdin: Stream[F, Byte]
-  )(
-      f: Process => Stream[F, A]
-  ): Stream[F, A] = {
-    val log = logger.debug(s"Running external command: ${cmd.cmdString}")
-    val hasStdin = stdin.take(1).compile.last.map(_.isDefined)
-    val proc = log *> hasStdin.flatMap(flag =>
-      Sync[F].blocking {
-        val pb = new ProcessBuilder(cmd.toCmd.asJava)
-          .redirectInput(if (flag) Redirect.PIPE else Redirect.INHERIT)
-          .redirectError(Redirect.PIPE)
-          .redirectOutput(Redirect.PIPE)
-
-        val pbEnv = pb.environment()
-        cmd.env.foreach { case (key, value) =>
-          pbEnv.put(key, value)
-        }
-        wd.map(_.toNioPath.toFile).foreach(pb.directory)
-        pb.start()
-      }
-    )
-    Stream
-      .bracket(proc)(p =>
-        logger.debug(s"Closing process: `${cmd.cmdString}`").map(_ => p.destroy())
-      )
-      .flatMap(f)
-  }
-
-  private def inputStreamToString[F[_]: Sync](in: InputStream): F[String] =
-    io.readInputStream(Sync[F].pure(in), 16 * 1024, closeAfterUse = false)
-      .through(text.utf8.decode)
-      .chunks
-      .map(_.toVector.mkString)
-      .fold1(_ + _)
-      .compile
-      .last
-      .map(_.getOrElse(""))
-
-  private def writeToProcess[F[_]: Sync](
-      data: Stream[F, Byte],
-      proc: Process
-  ): F[Unit] =
-    data
-      .through(io.writeOutputStream(Sync[F].blocking(proc.getOutputStream)))
-      .compile
-      .drain
-
-  private def timeoutError[F[_]: Sync](proc: Process, cmd: Config): F[Unit] =
-    Sync[F].blocking(proc.destroyForcibly()).attempt *> {
-      Sync[F].raiseError(
-        new Exception(
-          s"Command `${cmd.cmdString}` timed out (${cmd.timeout.formatExact})"
-        )
-      )
-    }
-}

modules/common/src/main/scala/docspell/common/UrlMatcher.scala

@@ -62,7 +62,7 @@ object UrlMatcher {
       // strip path to only match prefixes
       val mPath: LenientUri.Path =
         NonEmptyList.fromList(url.path.segments.take(pathSegmentCount)) match {
-          case Some(nel) => LenientUri.NonEmptyPath(nel, false)
+          case Some(nel) => LenientUri.NonEmptyPath(nel, trailingSlash = false)
           case None      => LenientUri.RootPath
         }
 

modules/common/src/main/scala/docspell/common/exec/Env.scala

@@ -17,6 +17,9 @@ case class Env(values: Map[String, String]) {
   def addAll(e: Env): Env =
     Env(values ++ e.values)
 
+  def modifyValue(f: String => String): Env =
+    Env(values.view.mapValues(f).toMap)
+
   def ++(e: Env) = addAll(e)
 
   def foreach(f: (String, String) => Unit): Unit =

modules/common/src/main/scala/docspell/common/exec/ExternalCommand.scala

@@ -0,0 +1,89 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common.exec
+
+import docspell.common.Duration
+import docspell.common.Ident
+import docspell.common.exec.Env
+import docspell.common.exec.ExternalCommand.ArgMapping
+import docspell.common.exec.SysCmd
+
+final case class ExternalCommand(
+    program: String,
+    args: Seq[String],
+    timeout: Duration,
+    env: Map[String, String] = Map.empty,
+    argMappings: Map[Ident, ArgMapping] = Map.empty
+) {
+  def withVars(vars: Map[String, String]): ExternalCommand.WithVars =
+    ExternalCommand.WithVars(this, vars)
+
+  import ExternalCommand.pattern
+
+  def resolve(vars: Map[String, String]): SysCmd = {
+    val replace = ExternalCommand.replaceString(vars) _
+    val resolvedArgMappings =
+      argMappings.view.mapValues(_.resolve(replace).firstMatch).toMap
+    val resolvedArgs = args.map(replace).flatMap { arg =>
+      resolvedArgMappings
+        .find(e => pattern(e._1.id) == arg)
+        .map(_._2)
+        .getOrElse(List(arg))
+    }
+
+    SysCmd(replace(program), resolvedArgs: _*)
+      .withTimeout(timeout)
+      .withEnv(_ => Env(env).modifyValue(replace))
+  }
+}
+
+object ExternalCommand {
+  private val openPattern = "{{"
+  private val closePattern = "}}"
+
+  private def pattern(s: String): String = s"${openPattern}${s}${closePattern}"
+
+  def apply(program: String, args: Seq[String], timeout: Duration): ExternalCommand =
+    ExternalCommand(program, args, timeout, Map.empty, Map.empty)
+
+  final case class ArgMapping(
+      value: String,
+      mappings: List[ArgMatch]
+  ) {
+    private[exec] def resolve(replace: String => String): ArgMapping =
+      ArgMapping(replace(value), mappings.map(_.resolve(replace)))
+
+    def firstMatch: List[String] =
+      mappings.find(am => value.matches(am.matches)).map(_.args).getOrElse(Nil)
+  }
+
+  final case class ArgMatch(
+      matches: String,
+      args: List[String]
+  ) {
+    private[exec] def resolve(replace: String => String): ArgMatch =
+      ArgMatch(replace(matches), args.map(replace))
+  }
+
+  private def replaceString(vars: Map[String, String])(in: String): String =
+    vars.foldLeft(in) { case (result, (name, value)) =>
+      val key = s"{{$name}}"
+      result.replace(key, value)
+    }
+
+  final case class WithVars(cmd: ExternalCommand, vars: Map[String, String]) {
+    def resolved: SysCmd = cmd.resolve(vars)
+    def append(more: (String, String)*): WithVars =
+      WithVars(cmd, vars ++ more.toMap)
+
+    def withVar(key: String, value: String): WithVars =
+      WithVars(cmd, vars.updated(key, value))
+
+    def withVarOption(key: String, value: Option[String]): WithVars =
+      value.map(withVar(key, _)).getOrElse(this)
+  }
+}

modules/common/src/main/scala/docspell/common/exec/SysExec.scala

@@ -38,6 +38,20 @@ trait SysExec[F[_]] {
 
   def waitFor(timeout: Option[Duration] = None): F[Int]
 
+  /** Uses `waitFor` and throws when return code is non-zero. Logs stderr and stdout while
+    * waiting.
+    */
+  def runToSuccess(logger: Logger[F], timeout: Option[Duration] = None)(implicit
+      F: Async[F]
+  ): F[Int]
+
+  /** Uses `waitFor` and throws when return code is non-zero. Logs stderr while waiting
+    * and collects stdout once finished successfully.
+    */
+  def runToSuccessStdout(logger: Logger[F], timeout: Option[Duration] = None)(implicit
+      F: Async[F]
+  ): F[String]
+
   /** Sends a signal to the process to terminate it immediately */
   def cancel: F[Unit]
 
@@ -75,6 +89,12 @@ object SysExec {
       proc <- startProcess(logger, cmd, workdir, stdin)
       fibers <- Resource.eval(Ref.of[F, List[F[Unit]]](Nil))
     } yield new SysExec[F] {
+      private lazy val basicName: String =
+        cmd.program.lastIndexOf(java.io.File.separatorChar.toInt) match {
+          case n if n > 0 => cmd.program.drop(n + 1)
+          case _          => cmd.program.takeRight(16)
+        }
+
       def stdout: Stream[F, Byte] =
         fs2.io.readInputStream(
           Sync[F].blocking(proc.getInputStream),
@@ -107,6 +127,39 @@ object SysExec {
             )
       }
 
+      def runToSuccess(logger: Logger[F], timeout: Option[Duration])(implicit
+          F: Async[F]
+      ): F[Int] =
+        logOutputs(logger, basicName).use(_.waitFor(timeout).flatMap {
+          case rc if rc == 0 => Sync[F].pure(0)
+          case rc =>
+            Sync[F].raiseError(
+              new Exception(s"Command `${cmd.program}` returned non-zero exit code ${rc}")
+            )
+        })
+
+      def runToSuccessStdout(logger: Logger[F], timeout: Option[Duration])(implicit
+          F: Async[F]
+      ): F[String] =
+        F.background(
+          stderrLines
+            .through(line => Stream.eval(logger.debug(s"[$basicName (err)]: $line")))
+            .compile
+            .drain
+        ).use { f1 =>
+          waitFor(timeout)
+            .flatMap {
+              case rc if rc == 0 => stdout.through(fs2.text.utf8.decode).compile.string
+              case rc =>
+                Sync[F].raiseError[String](
+                  new Exception(
+                    s"Command `${cmd.program}` returned non-zero exit code ${rc}"
+                  )
+                )
+            }
+            .flatTap(_ => f1)
+        }
+
       def consumeOutputs(out: Pipe[F, String, Unit], err: Pipe[F, String, Unit])(implicit
           F: Async[F]
       ): Resource[F, SysExec[F]] =

modules/common/src/main/scala/docspell/common/util/Directory.scala

@@ -6,6 +6,7 @@
 
 package docspell.common.util
 
+import cats.data.OptionT
 import cats.effect._
 import cats.syntax.all._
 import cats.{Applicative, Monad}
@@ -26,10 +27,10 @@ object Directory {
     (dir :: dirs.toList).traverse_(Files[F].createDirectories(_))
 
   def nonEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] =
-    List(
+    OptionT
-      Files[F].isDirectory(dir),
+      .whenM(Files[F].isDirectory(dir))(Files[F].list(dir).take(1).compile.toList)
-      Files[F].list(dir).take(1).compile.last.map(_.isDefined)
+      .map(_.nonEmpty)
-    ).sequence.map(_.forall(identity))
+      .isDefined
 
   def isEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] =
     nonEmpty(dir).map(b => !b)

modules/common/src/test/scala/docspell/common/exec/ExternalCommandTest.scala

@@ -0,0 +1,74 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common.exec
+
+import docspell.common.Duration
+import docspell.common.Ident
+import docspell.common.exec.Args
+import docspell.common.exec.Env
+import docspell.common.exec.ExternalCommand._
+import docspell.common.exec.SysCmd
+
+import munit.FunSuite
+
+class ExternalCommandTest extends FunSuite {
+
+  test("resolve") {
+    val cmd = ExternalCommand(
+      program = "tesseract",
+      args = "{{infile}}" :: "{{lang-spec}}" :: "out" :: "pdf" :: "txt" :: Nil,
+      timeout = Duration.minutes(5),
+      env = Map.empty,
+      argMappings = Map(
+        Ident.unsafe("lang-spec") -> ArgMapping(
+          value = "{{lang}}",
+          mappings = List(
+            ArgMatch(
+              matches = "jpn_vert",
+              args = List("-l", "jpn_vert", "-c", "preserve_interword_spaces=1")
+            ),
+            ArgMatch(
+              matches = ".*",
+              args = List("-l", "{{lang}}")
+            )
+          )
+        )
+      )
+    )
+
+    val varsDe = Map("lang" -> "de", "encoding" -> "UTF_8", "infile" -> "text.jpg")
+    assertEquals(
+      cmd.resolve(varsDe),
+      SysCmd(
+        "tesseract",
+        Args.of("text.jpg", "-l", "de", "out", "pdf", "txt"),
+        Env.empty,
+        Duration.minutes(5)
+      )
+    )
+
+    val varsJpnVert = varsDe.updated("lang", "jpn_vert")
+    assertEquals(
+      cmd.resolve(varsJpnVert),
+      SysCmd(
+        "tesseract",
+        Args.of(
+          "text.jpg",
+          "-l",
+          "jpn_vert",
+          "-c",
+          "preserve_interword_spaces=1",
+          "out",
+          "pdf",
+          "txt"
+        ),
+        Env.empty,
+        Duration.minutes(5)
+      )
+    )
+  }
+}

modules/common/src/test/scala/docspell/common/util/DirectoryTest.scala

@@ -16,7 +16,7 @@ import munit.CatsEffectSuite
 
 class DirectoryTest extends CatsEffectSuite with TestLoggingConfig {
   val logger = docspell.logging.getLogger[IO]
-  val tempDir = ResourceFixture(
+  val tempDir = ResourceFunFixture(
     Files[IO].tempDirectory(Path("target").some, "directory-test-", None)
   )
 

modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala

@@ -11,7 +11,8 @@ import cats.implicits._
 import fs2.io.file.{Files, Path}
 import fs2.{Pipe, Stream}
 
-import docspell.common._
+import docspell.common.exec.ExternalCommand
+import docspell.common.exec.SysExec
 import docspell.common.util.File
 import docspell.convert.ConversionResult
 import docspell.convert.ConversionResult.{Handler, successPdf, successPdfTxt}
@@ -21,11 +22,11 @@ private[extern] object ExternConv {
 
   def toPDF[F[_]: Async: Files, A](
       name: String,
-      cmdCfg: SystemCommand.Config,
+      cmdCfg: ExternalCommand.WithVars,
       wd: Path,
       useStdin: Boolean,
       logger: Logger[F],
-      reader: (Path, SystemCommand.Result) => F[ConversionResult[F]]
+      reader: (Path, Int) => F[ConversionResult[F]]
   )(in: Stream[F, Byte], handler: Handler[F, A]): F[A] =
     Stream
       .resource(File.withTempDir[F](wd, s"docspell-$name"))
@@ -33,32 +34,21 @@ private[extern] object ExternConv {
         val inFile = dir.resolve("infile").absolute.normalize
         val out = dir.resolve("out.pdf").absolute.normalize
         val sysCfg =
-          cmdCfg.replace(
+          cmdCfg
-            Map(
+            .withVar("outfile", out.toString)
-              "{{outfile}}" -> out.toString
+            .withVarOption("infile", Option.when(!useStdin)(inFile.toString))
-            ) ++
+            .resolved
-              (if (!useStdin) Map("{{infile}}" -> inFile.toString)
-               else Map.empty)
-          )
 
         val createInput: Pipe[F, Byte, Unit] =
           if (useStdin) _ => Stream.emit(())
           else storeDataToFile(name, logger, inFile)
 
-        in.through(createInput).flatMap { _ =>
+        in.through(createInput).evalMap { _ =>
-          SystemCommand
+          SysExec(sysCfg, logger, Some(dir), Option.when(useStdin)(in))
-            .exec[F](
+            .flatMap(_.logOutputs(logger, name))
-              sysCfg,
+            .use { proc =>
-              logger,
+              proc.waitFor().flatMap(rc => reader(out, rc).flatMap(handler.run))
-              Some(dir),
+            }
-              if (useStdin) in
-              else Stream.empty
-            )
-            .evalMap(result =>
-              logResult(name, result, logger)
-                .flatMap(_ => reader(out, result))
-                .flatMap(handler.run)
-            )
         }
       }
       .compile
@@ -74,9 +64,9 @@ private[extern] object ExternConv {
   def readResult[F[_]: Async: Files](
       chunkSize: Int,
       logger: Logger[F]
-  )(out: Path, result: SystemCommand.Result): F[ConversionResult[F]] =
+  )(out: Path, result: Int): F[ConversionResult[F]] =
     File.existsNonEmpty[F](out).flatMap {
-      case true if result.rc == 0 =>
+      case true if result == 0 =>
         val outTxt = out.resolveSibling(out.fileName.toString + ".txt")
         File.existsNonEmpty[F](outTxt).flatMap {
           case true =>
@@ -88,13 +78,13 @@ private[extern] object ExternConv {
             successPdf(File.readAll(out, chunkSize)).pure[F]
         }
       case true =>
-        logger.warn(s"Command not successful (rc=${result.rc}), but file exists.") *>
+        logger.warn(s"Command not successful (rc=${result}), but file exists.") *>
           successPdf(File.readAll(out, chunkSize)).pure[F]
 
       case false =>
         ConversionResult
           .failure[F](
-            new Exception(s"Command result=${result.rc}. No output file found.")
+            new Exception(s"Command result=${result}. No output file found.")
           )
           .pure[F]
     }
@@ -103,25 +93,25 @@ private[extern] object ExternConv {
       outPrefix: String,
       chunkSize: Int,
       logger: Logger[F]
-  )(out: Path, result: SystemCommand.Result): F[ConversionResult[F]] = {
+  )(out: Path, result: Int): F[ConversionResult[F]] = {
     val outPdf = out.resolveSibling(s"$outPrefix.pdf")
     File.existsNonEmpty[F](outPdf).flatMap {
       case true =>
         val outTxt = out.resolveSibling(s"$outPrefix.txt")
         File.exists(outTxt).flatMap { txtExists =>
           val pdfData = File.readAll(out, chunkSize)
-          if (result.rc == 0)
+          if (result == 0)
             if (txtExists) successPdfTxt(pdfData, File.readText(outTxt)).pure[F]
             else successPdf(pdfData).pure[F]
           else
-            logger.warn(s"Command not successful (rc=${result.rc}), but file exists.") *>
+            logger.warn(s"Command not successful (rc=${result}), but file exists.") *>
               successPdf(pdfData).pure[F]
         }
 
       case false =>
         ConversionResult
           .failure[F](
-            new Exception(s"Command result=${result.rc}. No output file found.")
+            new Exception(s"Command result=${result}. No output file found.")
           )
           .pure[F]
     }
@@ -138,14 +128,6 @@ private[extern] object ExternConv {
         .drain ++
         Stream.eval(storeFile(in, inFile))
 
-  private def logResult[F[_]: Sync](
-      name: String,
-      result: SystemCommand.Result,
-      logger: Logger[F]
-  ): F[Unit] =
-    logger.debug(s"$name stdout: ${result.stdout}") *>
-      logger.debug(s"$name stderr: ${result.stderr}")
-
   private def storeFile[F[_]: Async: Files](
       in: Stream[F, Byte],
       target: Path

modules/convert/src/main/scala/docspell/convert/extern/OcrMyPdf.scala

@@ -24,14 +24,16 @@ object OcrMyPdf {
       logger: Logger[F]
   )(in: Stream[F, Byte], handler: Handler[F, A]): F[A] =
     if (cfg.enabled) {
-      val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
+      val reader: (Path, Int) => F[ConversionResult[F]] =
         ExternConv.readResult[F](chunkSize, logger)
 
+      val cmd = cfg.command.withVars(Map("lang" -> lang.iso3))
+
       ExternConv.toPDF[F, A](
         "ocrmypdf",
-        cfg.command.replace(Map("{{lang}}" -> lang.iso3)),
+        cmd,
         cfg.workingDir,
-        false,
+        useStdin = false,
         logger,
         reader
       )(in, handler)

modules/convert/src/main/scala/docspell/convert/extern/OcrMyPdfConfig.scala

@@ -8,10 +8,10 @@ package docspell.convert.extern
 
 import fs2.io.file.Path
 
-import docspell.common.SystemCommand
+import docspell.common.exec.ExternalCommand
 
 case class OcrMyPdfConfig(
     enabled: Boolean,
-    command: SystemCommand.Config,
+    command: ExternalCommand,
     workingDir: Path
 )

modules/convert/src/main/scala/docspell/convert/extern/Tesseract.scala

@@ -24,17 +24,18 @@ object Tesseract {
       logger: Logger[F]
   )(in: Stream[F, Byte], handler: Handler[F, A]): F[A] = {
     val outBase = cfg.command.args.tail.headOption.getOrElse("out")
-    val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
+    val reader: (Path, Int) => F[ConversionResult[F]] =
       ExternConv.readResultTesseract[F](outBase, chunkSize, logger)
 
+    val cmd = cfg.command.withVars(Map("lang" -> lang.iso3))
+
     ExternConv.toPDF[F, A](
       "tesseract",
-      cfg.command.replace(Map("{{lang}}" -> lang.iso3)),
+      cmd,
       cfg.workingDir,
-      false,
+      useStdin = false,
       logger,
       reader
     )(in, handler)
   }
-
 }

modules/convert/src/main/scala/docspell/convert/extern/TesseractConfig.scala

@@ -8,6 +8,6 @@ package docspell.convert.extern
 
 import fs2.io.file.Path
 
-import docspell.common.SystemCommand
+import docspell.common.exec.ExternalCommand
 
-case class TesseractConfig(command: SystemCommand.Config, workingDir: Path)
+case class TesseractConfig(command: ExternalCommand, workingDir: Path)

modules/convert/src/main/scala/docspell/convert/extern/Unoconv.scala

@@ -10,7 +10,6 @@ import cats.effect._
 import fs2.Stream
 import fs2.io.file.{Files, Path}
 
-import docspell.common._
 import docspell.convert.ConversionResult
 import docspell.convert.ConversionResult.Handler
 import docspell.logging.Logger
@@ -22,14 +21,15 @@ object Unoconv {
       chunkSize: Int,
       logger: Logger[F]
   )(in: Stream[F, Byte], handler: Handler[F, A]): F[A] = {
-    val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
+    val reader: (Path, Int) => F[ConversionResult[F]] =
       ExternConv.readResult[F](chunkSize, logger)
+    val cmd = cfg.command.withVars(Map.empty)
 
     ExternConv.toPDF[F, A](
       "unoconv",
-      cfg.command,
+      cmd,
       cfg.workingDir,
-      false,
+      useStdin = false,
       logger,
       reader
     )(
@@ -37,5 +37,4 @@ object Unoconv {
       handler
     )
   }
-
 }

modules/convert/src/main/scala/docspell/convert/extern/UnoconvConfig.scala

@@ -8,6 +8,6 @@ package docspell.convert.extern
 
 import fs2.io.file.Path
 
-import docspell.common.SystemCommand
+import docspell.common.exec.ExternalCommand
 
-case class UnoconvConfig(command: SystemCommand.Config, workingDir: Path)
+case class UnoconvConfig(command: ExternalCommand, workingDir: Path)

modules/convert/src/main/scala/docspell/convert/extern/Weasyprint.scala

@@ -27,10 +27,10 @@ object Weasyprint {
       sanitizeHtml: SanitizeHtml,
       logger: Logger[F]
   )(in: Stream[F, Byte], handler: Handler[F, A]): F[A] = {
-    val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
+    val reader: (Path, Int) => F[ConversionResult[F]] =
       ExternConv.readResult[F](chunkSize, logger)
 
-    val cmdCfg = cfg.command.replace(Map("{{encoding}}" -> charset.name()))
+    val cmdCfg = cfg.command.withVars(Map("encoding" -> charset.name()))
 
     // html sanitize should (among other) remove links to invalid
     // protocols like cid: which is not supported by further
@@ -51,5 +51,4 @@ object Weasyprint {
         handler
       )
   }
-
 }

modules/convert/src/main/scala/docspell/convert/extern/WeasyprintConfig.scala

@@ -8,6 +8,6 @@ package docspell.convert.extern
 
 import fs2.io.file.Path
 
-import docspell.common.SystemCommand
+import docspell.common.exec.ExternalCommand
 
-case class WeasyprintConfig(command: SystemCommand.Config, workingDir: Path)
+case class WeasyprintConfig(command: ExternalCommand, workingDir: Path)

modules/convert/src/main/scala/docspell/convert/extern/WkHtmlPdf.scala

@@ -27,10 +27,10 @@ object WkHtmlPdf {
       sanitizeHtml: SanitizeHtml,
       logger: Logger[F]
   )(in: Stream[F, Byte], handler: Handler[F, A]): F[A] = {
-    val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
+    val reader: (Path, Int) => F[ConversionResult[F]] =
       ExternConv.readResult[F](chunkSize, logger)
 
-    val cmdCfg = cfg.command.replace(Map("{{encoding}}" -> charset.name()))
+    val cmdCfg = cfg.command.withVars(Map("encoding" -> charset.name()))
 
     // html sanitize should (among other) remove links to invalid
     // protocols like cid: which is not supported by further
@@ -58,5 +58,4 @@ object WkHtmlPdf {
         handler
       )
   }
-
 }

modules/convert/src/main/scala/docspell/convert/extern/WkHtmlPdfConfig.scala

@@ -8,6 +8,6 @@ package docspell.convert.extern
 
 import fs2.io.file.Path
 
-import docspell.common.SystemCommand
+import docspell.common.exec.ExternalCommand
 
-case class WkHtmlPdfConfig(command: SystemCommand.Config, workingDir: Path)
+case class WkHtmlPdfConfig(command: ExternalCommand, workingDir: Path)

modules/convert/src/test/scala/docspell/convert/ConversionTest.scala

@@ -15,6 +15,7 @@ import cats.implicits._
 import fs2.Stream
 
 import docspell.common._
+import docspell.common.exec._
 import docspell.common.util.File
 import docspell.convert.ConversionResult.Handler
 import docspell.convert.ConvertConfig.HtmlConverter
@@ -36,7 +37,7 @@ class ConversionTest extends FunSuite with FileChecks with TestLoggingConfig {
     3000 * 3000,
     MarkdownConfig("body { padding: 2em 5em; }"),
     WkHtmlPdfConfig(
-      SystemCommand.Config(
+      ExternalCommand(
         "wkhtmltopdf",
         Seq("-s", "A4", "--encoding", "UTF-8", "-", "{{outfile}}"),
         Duration.seconds(20)
@@ -44,7 +45,7 @@ class ConversionTest extends FunSuite with FileChecks with TestLoggingConfig {
       target
     ),
     WeasyprintConfig(
-      SystemCommand.Config(
+      ExternalCommand(
         "weasyprint",
         Seq("--encoding", "UTF-8", "-", "{{outfile}}"),
         Duration.seconds(20)
@@ -53,7 +54,7 @@ class ConversionTest extends FunSuite with FileChecks with TestLoggingConfig {
     ),
     HtmlConverter.Wkhtmltopdf,
     TesseractConfig(
-      SystemCommand.Config(
+      ExternalCommand(
         "tesseract",
         Seq("{{infile}}", "out", "-l", "deu", "pdf", "txt"),
         Duration.seconds(20)
@@ -61,7 +62,7 @@ class ConversionTest extends FunSuite with FileChecks with TestLoggingConfig {
       target
     ),
     UnoconvConfig(
-      SystemCommand.Config(
+      ExternalCommand(
         "unoconv",
         Seq("-f", "pdf", "-o", "{{outfile}}", "{{infile}}"),
         Duration.seconds(20)
@@ -69,8 +70,8 @@ class ConversionTest extends FunSuite with FileChecks with TestLoggingConfig {
       target
     ),
     OcrMyPdfConfig(
-      true,
+      enabled = true,
-      SystemCommand.Config(
+      ExternalCommand(
         "ocrmypdf",
         Seq(
           "-l",
@@ -86,7 +87,7 @@ class ConversionTest extends FunSuite with FileChecks with TestLoggingConfig {
       ),
       target
     ),
-    ConvertConfig.DecryptPdf(true, Nil)
+    ConvertConfig.DecryptPdf(enabled = true, Nil)
   )
 
   val conversion =

modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala

@@ -14,6 +14,7 @@ import cats.effect.unsafe.implicits.global
 import fs2.io.file.Path
 
 import docspell.common._
+import docspell.common.exec._
 import docspell.common.util.File
 import docspell.convert._
 import docspell.files.ExampleFiles
@@ -27,7 +28,7 @@ class ExternConvTest extends FunSuite with FileChecks with TestLoggingConfig {
   val target = File.path(Paths.get("target"))
 
   test("convert html to pdf") {
-    val cfg = SystemCommand.Config(
+    val cfg = ExternalCommand(
       "wkhtmltopdf",
       Seq("-s", "A4", "--encoding", "UTF-8", "-", "{{outfile}}"),
       Duration.seconds(20)
@@ -53,7 +54,7 @@ class ExternConvTest extends FunSuite with FileChecks with TestLoggingConfig {
   }
 
   test("convert office to pdf") {
-    val cfg = SystemCommand.Config(
+    val cfg = ExternalCommand(
       "unoconv",
       Seq("-f", "pdf", "-o", "{{outfile}}", "{{infile}}"),
       Duration.seconds(20)
@@ -80,7 +81,7 @@ class ExternConvTest extends FunSuite with FileChecks with TestLoggingConfig {
   }
 
   test("convert image to pdf") {
-    val cfg = SystemCommand.Config(
+    val cfg = ExternalCommand(
       "tesseract",
       Seq("{{infile}}", "out", "-l", "deu", "pdf", "txt"),
       Duration.seconds(20)
@@ -105,5 +106,4 @@ class ExternConvTest extends FunSuite with FileChecks with TestLoggingConfig {
       )
       .unsafeRunSync()
   }
-
 }

modules/extract/src/main/scala/docspell/extract/ocr/Ocr.scala

@@ -10,7 +10,8 @@ import cats.effect._
 import fs2.Stream
 import fs2.io.file.{Files, Path}
 
-import docspell.common._
+import docspell.common.exec.ExternalCommand
+import docspell.common.exec.SysExec
 import docspell.common.util.File
 import docspell.logging.Logger
 
@@ -77,14 +78,17 @@ object Ocr {
       else cfg.ghostscript.command.args
     val cmd = cfg.ghostscript.command
       .copy(args = xargs)
-      .replace(
+      .withVars(
         Map(
-          "{{infile}}" -> "-",
+          "infile" -> "-",
-          "{{outfile}}" -> "%d.tif"
+          "outfile" -> "%d.tif"
         )
       )
-    SystemCommand
+      .resolved
-      .execSuccess(cmd, logger, wd = Some(wd), stdin = pdf)
+
+    Stream
+      .resource(SysExec(cmd, logger, Some(wd), Some(pdf)))
+      .evalMap(_.runToSuccess(logger))
       .flatMap(_ => File.listFiles(pathEndsWith(".tif"), wd))
   }
 
@@ -93,18 +97,22 @@ object Ocr {
     */
   private[extract] def runGhostscriptFile[F[_]: Async: Files](
       pdf: Path,
-      ghostscript: SystemCommand.Config,
+      ghostscript: ExternalCommand,
       wd: Path,
       logger: Logger[F]
   ): Stream[F, Path] = {
-    val cmd = ghostscript.replace(
+    val cmd = ghostscript
-      Map(
+      .withVars(
-        "{{infile}}" -> pdf.absolute.toString,
+        Map(
-        "{{outfile}}" -> "%d.tif"
+          "infile" -> pdf.absolute.toString,
+          "outfile" -> "%d.tif"
+        )
       )
-    )
+      .resolved
-    SystemCommand
+
-      .execSuccess[F](cmd, logger, wd = Some(wd))
+    Stream
+      .resource(SysExec(cmd, logger, Some(wd)))
+      .evalMap(_.runToSuccess(logger))
       .flatMap(_ => File.listFiles(pathEndsWith(".tif"), wd))
   }
 
@@ -116,19 +124,23 @@ object Ocr {
     */
   private[extract] def runUnpaperFile[F[_]: Async](
       img: Path,
-      unpaper: SystemCommand.Config,
+      unpaper: ExternalCommand,
       wd: Option[Path],
       logger: Logger[F]
   ): Stream[F, Path] = {
     val targetFile = img.resolveSibling("u-" + img.fileName.toString).absolute
-    val cmd = unpaper.replace(
+    val cmd = unpaper
-      Map(
+      .withVars(
-        "{{infile}}" -> img.absolute.toString,
+        Map(
-        "{{outfile}}" -> targetFile.toString
+          "infile" -> img.absolute.toString,
+          "outfile" -> targetFile.toString
+        )
       )
-    )
+      .resolved
-    SystemCommand
+
-      .execSuccess[F](cmd, logger, wd = wd)
+    Stream
+      .resource(SysExec(cmd, logger, wd))
+      .evalMap(_.runToSuccess(logger))
       .map(_ => targetFile)
       .handleErrorWith { th =>
         logger
@@ -150,12 +162,14 @@ object Ocr {
     // so use the parent as working dir
     runUnpaperFile(img, config.unpaper.command, img.parent, logger).flatMap { uimg =>
       val cmd = config.tesseract.command
-        .replace(
+        .withVars(
-          Map("{{file}}" -> uimg.fileName.toString, "{{lang}}" -> fixLanguage(lang))
+          Map("file" -> uimg.fileName.toString, "lang" -> fixLanguage(lang))
         )
-      SystemCommand
+        .resolved
-        .execSuccess[F](cmd, logger, wd = uimg.parent)
+
-        .map(_.stdout)
+      Stream
+        .resource(SysExec(cmd, logger, uimg.parent))
+        .evalMap(_.runToSuccessStdout(logger))
     }
 
   /** Run tesseract on the given image file and return the extracted text. */
@@ -166,8 +180,12 @@ object Ocr {
       config: OcrConfig
   ): Stream[F, String] = {
     val cmd = config.tesseract.command
-      .replace(Map("{{file}}" -> "stdin", "{{lang}}" -> fixLanguage(lang)))
+      .withVars(Map("file" -> "stdin", "lang" -> fixLanguage(lang)))
-    SystemCommand.execSuccess(cmd, logger, stdin = img).map(_.stdout)
+      .resolved
+
+    Stream
+      .resource(SysExec(cmd, logger, None, Some(img)))
+      .evalMap(_.runToSuccessStdout(logger))
   }
 
   private def fixLanguage(lang: String): String =

modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala

@@ -6,12 +6,9 @@
 
 package docspell.extract.ocr
 
-import java.nio.file.Paths
-
 import fs2.io.file.Path
 
-import docspell.common._
+import docspell.common.exec.ExternalCommand
-import docspell.common.util.File
 
 case class OcrConfig(
     maxImageSize: Int,
@@ -25,43 +22,10 @@ object OcrConfig {
 
   case class PageRange(begin: Int)
 
-  case class Ghostscript(command: SystemCommand.Config, workingDir: Path)
+  case class Ghostscript(command: ExternalCommand, workingDir: Path)
 
-  case class Tesseract(command: SystemCommand.Config)
+  case class Tesseract(command: ExternalCommand)
 
-  case class Unpaper(command: SystemCommand.Config)
+  case class Unpaper(command: ExternalCommand)
 
-  val default = OcrConfig(
-    maxImageSize = 3000 * 3000,
-    pageRange = PageRange(10),
-    ghostscript = Ghostscript(
-      SystemCommand.Config(
-        "gs",
-        Seq(
-          "-dNOPAUSE",
-          "-dBATCH",
-          "-dSAFER",
-          "-sDEVICE=tiffscaled8",
-          "-sOutputFile={{outfile}}",
-          "{{infile}}"
-        ),
-        Duration.seconds(30)
-      ),
-      File.path(
-        Paths.get(System.getProperty("java.io.tmpdir")).resolve("docspell-extraction")
-      )
-    ),
-    unpaper = Unpaper(
-      SystemCommand
-        .Config("unpaper", Seq("{{infile}}", "{{outfile}}"), Duration.seconds(30))
-    ),
-    tesseract = Tesseract(
-      SystemCommand
-        .Config(
-          "tesseract",
-          Seq("{{file}}", "stdout", "-l", "{{lang}}"),
-          Duration.minutes(1)
-        )
-    )
-  )
 }

modules/extract/src/test/scala/docspell/extract/ocr/TextExtractionSuite.scala

@@ -6,9 +6,14 @@
 
 package docspell.extract.ocr
 
+import java.nio.file.Paths
+
 import cats.effect.IO
 import cats.effect.unsafe.implicits.global
 
+import docspell.common.Duration
+import docspell.common.exec.ExternalCommand
+import docspell.common.util.File
 import docspell.files.TestFiles
 import docspell.logging.TestLoggingConfig
 
@@ -21,7 +26,7 @@ class TextExtractionSuite extends FunSuite with TestLoggingConfig {
 
   test("extract english pdf".ignore) {
     val text = TextExtract
-      .extract[IO](letterSourceEN, logger, "eng", OcrConfig.default)
+      .extract[IO](letterSourceEN, logger, "eng", TextExtractionSuite.defaultConfig)
       .compile
       .lastOrError
       .unsafeRunSync()
@@ -31,7 +36,7 @@ class TextExtractionSuite extends FunSuite with TestLoggingConfig {
   test("extract german pdf".ignore) {
     val expect = TestFiles.letterDEText
     val extract = TextExtract
-      .extract[IO](letterSourceDE, logger, "deu", OcrConfig.default)
+      .extract[IO](letterSourceDE, logger, "deu", TextExtractionSuite.defaultConfig)
       .compile
       .lastOrError
       .unsafeRunSync()
@@ -39,3 +44,37 @@ class TextExtractionSuite extends FunSuite with TestLoggingConfig {
     assertEquals(extract.value, expect)
   }
 }
+
+object TextExtractionSuite {
+  val defaultConfig = OcrConfig(
+    maxImageSize = 3000 * 3000,
+    pageRange = OcrConfig.PageRange(10),
+    ghostscript = OcrConfig.Ghostscript(
+      ExternalCommand(
+        "gs",
+        Seq(
+          "-dNOPAUSE",
+          "-dBATCH",
+          "-dSAFER",
+          "-sDEVICE=tiffscaled8",
+          "-sOutputFile={{outfile}}",
+          "{{infile}}"
+        ),
+        Duration.seconds(30)
+      ),
+      File.path(
+        Paths.get(System.getProperty("java.io.tmpdir")).resolve("docspell-extraction")
+      )
+    ),
+    unpaper = OcrConfig.Unpaper(
+      ExternalCommand("unpaper", Seq("{{infile}}", "{{outfile}}"), Duration.seconds(30))
+    ),
+    tesseract = OcrConfig.Tesseract(
+      ExternalCommand(
+        "tesseract",
+        Seq("{{file}}", "stdout", "-l", "{{lang}}"),
+        Duration.minutes(1)
+      )
+    )
+  )
+}

modules/files/src/test/scala/docspell/files/ZipTest.scala

@@ -19,7 +19,7 @@ import munit._
 
 class ZipTest extends CatsEffectSuite with TestLoggingConfig {
   val logger = docspell.logging.getLogger[IO]
-  val tempDir = ResourceFixture(
+  val tempDir = ResourceFunFixture(
     Files[IO].tempDirectory(Path("target").some, "zip-test-", None)
   )
 

modules/fts-psql/src/main/scala/docspell/ftspsql/FtsRepository.scala

@@ -201,6 +201,7 @@ object FtsRepository extends DoobieMeta {
       case Language.Czech      => "simple"
       case Language.Latvian    => "simple"
       case Language.Japanese   => "simple"
+      case Language.JpnVert    => "simple"
       case Language.Hebrew     => "simple"
       case Language.Lithuanian => "simple"
       case Language.Polish     => "simple"

modules/fts-solr/src/main/scala/docspell/ftssolr/SolrMigration.scala

@@ -45,7 +45,7 @@ object SolrMigration {
         description,
         FtsMigration.Result.reIndexAll.pure[F]
       ),
-      true
+      dataChangeOnly = true
     )
 
   def indexAll[F[_]: Applicative](
@@ -59,7 +59,7 @@ object SolrMigration {
         description,
         FtsMigration.Result.indexAll.pure[F]
       ),
-      true
+      dataChangeOnly = true
     )
 
   def apply[F[_]: Functor](
@@ -74,6 +74,6 @@ object SolrMigration {
         description,
         task.map(_ => FtsMigration.Result.workDone)
       ),
-      false
+      dataChangeOnly = false
     )
 }

modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala

@@ -299,14 +299,22 @@ object SolrSetup {
       Map("add-field" -> body.asJson).asJson
 
     def string(field: Field): AddField =
-      AddField(field, "string", true, true, false)
+      AddField(field, "string", stored = true, indexed = true, multiValued = false)
 
     def textGeneral(field: Field): AddField =
-      AddField(field, "text_general", true, true, false)
+      AddField(field, "text_general", stored = true, indexed = true, multiValued = false)
 
     def textLang(field: Field, lang: Language): AddField =
-      if (lang == Language.Czech) AddField(field, s"text_cz", true, true, false)
+      if (lang == Language.Czech)
-      else AddField(field, s"text_${lang.iso2}", true, true, false)
+        AddField(field, s"text_cz", stored = true, indexed = true, multiValued = false)
+      else
+        AddField(
+          field,
+          s"text_${lang.iso2}",
+          stored = true,
+          indexed = true,
+          multiValued = false
+        )
   }
 
   case class DeleteField(name: Field)

modules/joex/src/main/resources/reference.conf

@@ -595,11 +595,30 @@ Docpell Update Check
     tesseract = {
       command = {
         program = "tesseract"
+        # Custom Language Mappings Below
+        # Japanese Vertical Mapping
+        arg-mappings = {
+          "tesseract_lang" = {
+            value = "{{lang}}"
+            mappings = [
+              {
+                matches = "jpn_vert"
+                args = [ "-l", "jpn_vert", "-c", "preserve_interword_spaces=1" ]
+              },
+            # Start Other Custom Language Mappings Here
+            # Default Mapping Below
+              {
+                matches = ".*"
+                args = [ "-l", "{{lang}}" ]
+              }
+            ]
+          }
+        }
+        # Default arguments for all processing go below.
         args = [
           "{{infile}}",
           "out",
-          "-l",
+          "{{tesseract_lang}}",
-          "{{lang}}",
           "pdf",
           "txt"
         ]
@@ -651,8 +670,34 @@ Docpell Update Check
       enabled = true
       command = {
         program = "ocrmypdf"
+        # Custom argument mappings for this program.
+        arg-mappings = {
+          "ocr_lang" = {
+            value = "{{lang}}"
+            # Custom Language Mappings Below
+            # Japanese Vertical Mapping
+            mappings = [
+              {
+                matches = "jpn_vert"
+                args = [ "-l", "jpn_vert", "--pdf-renderer", "sandwich", "--tesseract-pagesegmode", "5", "--output-type", "pdf" ]
+              },
+            # Japanese Mapping for OCR Optimization
+              {
+                matches = "jpn"
+                args = [ "-l", "jpn", "--output-type", "pdf" ]
+              },
+            # Start Other Custom Language Mappings Here
+            # Default Mapping Below
+              {
+                matches = ".*"
+                args = [ "-l", "{{lang}}" ]
+              }
+            ]
+          }
+        }
+        # Default arguments for all processing go below.
         args = [
-          "-l", "{{lang}}",
+          "{{ocr_lang}}",
           "--skip-text",
           "--deskew",
           "-j", "1",
@@ -893,4 +938,4 @@ Docpell Update Check
       }
     }
   }
-}
+}

modules/joex/src/main/scala/docspell/joex/emptytrash/EmptyTrashTask.scala

@@ -30,7 +30,7 @@ object EmptyTrashTask {
     UserTask(
       args.periodicTaskId,
       EmptyTrashArgs.taskName,
-      true,
+      enabled = true,
       ce,
       None,
       args

modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala

@@ -29,23 +29,23 @@ object FileCopyTask {
   case class CopyResult(success: Boolean, message: String, counter: List[Counter])
   object CopyResult {
     def noSourceImpl: CopyResult =
-      CopyResult(false, "No source BinaryStore implementation found!", Nil)
+      CopyResult(success = false, "No source BinaryStore implementation found!", Nil)
 
     def noTargetImpl: CopyResult =
-      CopyResult(false, "No target BinaryStore implementation found!", Nil)
+      CopyResult(success = false, "No target BinaryStore implementation found!", Nil)
 
     def noSourceStore(id: Ident): CopyResult =
       CopyResult(
-        false,
+        success = false,
         s"No source file repo found with id: ${id.id}. Make sure it is present in the config.",
         Nil
       )
 
     def noTargetStore: CopyResult =
-      CopyResult(false, "No target file repositories defined", Nil)
+      CopyResult(success = false, "No target file repositories defined", Nil)
 
     def success(counter: NonEmptyList[Counter]): CopyResult =
-      CopyResult(true, "Done", counter.toList)
+      CopyResult(success = true, "Done", counter.toList)
 
     implicit val binaryIdCodec: Codec[BinaryId] =
       Codec.from(
@@ -96,8 +96,10 @@ object FileCopyTask {
             .fromList(targets.filter(_ != srcConfig))
             .toRight(CopyResult.noTargetStore)
 
-          srcRepo = store.createFileRepository(srcConfig, true)
+          srcRepo = store.createFileRepository(srcConfig, withAttributeStore = true)
-          targetRepos = trgConfig.map(store.createFileRepository(_, false))
+          targetRepos = trgConfig.map(
+            store.createFileRepository(_, withAttributeStore = false)
+          )
         } yield (srcRepo, targetRepos)
 
       data match {

modules/joex/src/main/scala/docspell/joex/hk/CleanupResult.scala

@@ -13,8 +13,8 @@ case class CleanupResult(removed: Int, disabled: Boolean) {
   def asString = if (disabled) "disabled" else s"$removed"
 }
 object CleanupResult {
-  def of(n: Int): CleanupResult = CleanupResult(n, false)
+  def of(n: Int): CleanupResult = CleanupResult(n, disabled = false)
-  def disabled: CleanupResult = CleanupResult(0, true)
+  def disabled: CleanupResult = CleanupResult(0, disabled = true)
 
   implicit val jsonEncoder: Encoder[CleanupResult] =
     deriveEncoder

modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala

@@ -55,7 +55,7 @@ object HouseKeepingTask {
     UserTask(
       periodicId,
       taskName,
-      true,
+      enabled = true,
       ce,
       "Docspell house-keeping".some,
       ()

modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala

@@ -222,13 +222,13 @@ object FindProposal {
     def searchExact[F[_]: Sync](ctx: Context[F, Args], store: Store[F]): Finder[F] =
       labels =>
         labels.toList
-          .traverse(nl => search(nl, true, ctx, store))
+          .traverse(nl => search(nl, exact = true, ctx, store))
           .map(MetaProposalList.flatten)
 
     def searchFuzzy[F[_]: Sync](ctx: Context[F, Args], store: Store[F]): Finder[F] =
       labels =>
         labels.toList
-          .traverse(nl => search(nl, false, ctx, store))
+          .traverse(nl => search(nl, exact = false, ctx, store))
           .map(MetaProposalList.flatten)
   }
 

modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala

@@ -131,10 +131,10 @@ object ReProcessItem {
               data.item.source, // source-id
               None, // folder
               Seq.empty,
-              false,
+              skipDuplicate = false,
               None,
               None,
-              true,
+              reprocess = true,
               None, // attachOnly (not used when reprocessing attachments)
               None // cannot retain customData from an already existing item
             ),

modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala

@@ -75,7 +75,7 @@ object TextAnalysis {
       analyser: TextAnalyser[F],
       nerFile: RegexNerFile[F]
   )(rm: RAttachmentMeta): F[(RAttachmentMeta, AttachmentDates)] = {
-    val settings = NlpSettings(ctx.args.meta.language, false, None)
+    val settings = NlpSettings(ctx.args.meta.language, highRecall = false, None)
     for {
       customNer <- nerFile.makeFile(ctx.args.meta.collective)
       sett = settings.copy(regexNer = customNer)

modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala

@@ -28,7 +28,7 @@ object JoexRoutes {
         for {
           _ <- app.scheduler.notifyChange
           _ <- app.periodicScheduler.notifyChange
-          resp <- Ok(BasicResult(true, "Schedulers notified."))
+          resp <- Ok(BasicResult(success = true, "Schedulers notified."))
         } yield resp
 
       case GET -> Root / "running" =>
@@ -43,7 +43,7 @@ object JoexRoutes {
           _ <- Async[F].start(
             Temporal[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown
           )
-          resp <- Ok(BasicResult(true, "Shutdown initiated."))
+          resp <- Ok(BasicResult(success = true, "Shutdown initiated."))
         } yield resp
 
       case GET -> Root / "job" / Ident(id) =>
@@ -54,7 +54,9 @@ object JoexRoutes {
             job <- optJob
             log <- optLog
           } yield mkJobLog(job, log)
-          resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found")))
+          resp <- jAndL
+            .map(Ok(_))
+            .getOrElse(NotFound(BasicResult(success = false, "Not found")))
         } yield resp
 
       case POST -> Root / "job" / Ident(id) / "cancel" =>

modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala

@@ -323,7 +323,7 @@ object ScanMailboxTask {
           s"mailbox-${ctx.args.account.login.id}",
           args.itemFolder,
           Seq.empty,
-          true,
+          skipDuplicates = true,
           args.fileFilter.getOrElse(Glob.all),
           args.tags.getOrElse(Nil),
           args.language,

modules/joexapi/src/main/resources/joex-openapi.yml

@@ -18,6 +18,8 @@ servers:
   - url: /api/v1
     description: Current host
 
+security: []
+
 paths:
   /api/info/version:
     get:

modules/notification/api/src/main/scala/docspell/notification/api/Event.scala

@@ -164,7 +164,7 @@ object Event {
       for {
         id1 <- Ident.randomId[F]
         id2 <- Ident.randomId[F]
-      } yield ItemSelection(account, Nel.of(id1, id2), true, baseUrl, None)
+      } yield ItemSelection(account, Nel.of(id1, id2), more = true, baseUrl, None)
   }
 
   /** Event when a new job is added to the queue */

modules/pubsub/naive/src/test/scala/docspell/pubsub/naive/Fixtures.scala

@@ -20,9 +20,9 @@ import org.typelevel.ci._
 
 trait Fixtures extends HttpClientOps { self: CatsEffectSuite =>
 
-  val pubsubEnv = ResourceFixture(Fixtures.envResource("node-1"))
+  val pubsubEnv = ResourceFunFixture(Fixtures.envResource("node-1"))
 
-  val pubsubT = ResourceFixture {
+  val pubsubT = ResourceFunFixture {
     Fixtures
       .envResource("node-1")
       .flatMap(_.pubSub)

modules/query/shared/src/main/scala/docspell/query/ParseFailure.scala

@@ -87,10 +87,10 @@ object ParseFailure {
         SimpleMessage(offset, message)
 
       case InRange(offset, lower, upper) =>
-        if (lower == upper) ExpectMessage(offset, List(lower.toString), true)
+        if (lower == upper) ExpectMessage(offset, List(lower.toString), exhaustive = true)
         else {
           val expect = s"$lower-$upper"
-          ExpectMessage(offset, List(expect), true)
+          ExpectMessage(offset, List(expect), exhaustive = true)
         }
 
       case Length(offset, expected, actual) =>
@@ -110,6 +110,10 @@ object ParseFailure {
         ExpectMessage(offset, options.take(7), options.size < 8)
 
       case WithContext(ctx, expect) =>
-        ExpectMessage(expect.offset, s"Failed to parse near: $ctx" :: Nil, true)
+        ExpectMessage(
+          expect.offset,
+          s"Failed to parse near: $ctx" :: Nil,
+          exhaustive = true
+        )
     }
 }

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -27,6 +27,8 @@ servers:
   - url: /api/v1
     description: Current host
 
+security: []
+
 paths:
   /api/info/version:
     get:

modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala

@@ -329,7 +329,7 @@ trait Conversions {
             sourceName,
             None,
             validFileTypes,
-            false,
+            skipDuplicates = false,
             Glob.all,
             Nil,
             None,
@@ -641,82 +641,86 @@ trait Conversions {
   def basicResult(r: SetValueResult): BasicResult =
     r match {
       case SetValueResult.FieldNotFound =>
-        BasicResult(false, "The given field is unknown")
+        BasicResult(success = false, "The given field is unknown")
       case SetValueResult.ItemNotFound =>
-        BasicResult(false, "The given item is unknown")
+        BasicResult(success = false, "The given item is unknown")
       case SetValueResult.ValueInvalid(msg) =>
-        BasicResult(false, s"The value is invalid: $msg")
+        BasicResult(success = false, s"The value is invalid: $msg")
       case SetValueResult.Success =>
-        BasicResult(true, "Custom field value set successfully.")
+        BasicResult(success = true, "Custom field value set successfully.")
     }
 
   def basicResult(cr: JobCancelResult): BasicResult =
     cr match {
-      case JobCancelResult.JobNotFound => BasicResult(false, "Job not found")
+      case JobCancelResult.JobNotFound => BasicResult(success = false, "Job not found")
       case JobCancelResult.CancelRequested =>
-        BasicResult(true, "Cancel was requested at the job executor")
+        BasicResult(success = true, "Cancel was requested at the job executor")
       case JobCancelResult.Removed =>
-        BasicResult(true, "The job has been removed from the queue.")
+        BasicResult(success = true, "The job has been removed from the queue.")
     }
 
   def idResult(ar: AddResult, id: Ident, successMsg: String): IdResult =
     ar match {
-      case AddResult.Success           => IdResult(true, successMsg, id)
+      case AddResult.Success           => IdResult(success = true, successMsg, id)
-      case AddResult.EntityExists(msg) => IdResult(false, msg, Ident.unsafe(""))
+      case AddResult.EntityExists(msg) => IdResult(success = false, msg, Ident.unsafe(""))
       case AddResult.Failure(ex) =>
-        IdResult(false, s"Internal error: ${ex.getMessage}", Ident.unsafe(""))
+        IdResult(success = false, s"Internal error: ${ex.getMessage}", Ident.unsafe(""))
     }
 
   def basicResult(ar: AddResult, successMsg: String): BasicResult =
     ar match {
-      case AddResult.Success           => BasicResult(true, successMsg)
+      case AddResult.Success           => BasicResult(success = true, successMsg)
-      case AddResult.EntityExists(msg) => BasicResult(false, msg)
+      case AddResult.EntityExists(msg) => BasicResult(success = false, msg)
       case AddResult.Failure(ex) =>
-        BasicResult(false, s"Internal error: ${ex.getMessage}")
+        BasicResult(success = false, s"Internal error: ${ex.getMessage}")
     }
 
   def basicResult(ar: UpdateResult, successMsg: String): BasicResult =
     ar match {
-      case UpdateResult.Success  => BasicResult(true, successMsg)
+      case UpdateResult.Success  => BasicResult(success = true, successMsg)
-      case UpdateResult.NotFound => BasicResult(false, "Not found")
+      case UpdateResult.NotFound => BasicResult(success = false, "Not found")
       case UpdateResult.Failure(ex) =>
-        BasicResult(false, s"Error: ${ex.getMessage}")
+        BasicResult(success = false, s"Error: ${ex.getMessage}")
     }
 
   def basicResult(ur: OUpload.UploadResult): BasicResult =
     ur match {
-      case UploadResult.Success  => BasicResult(true, "Files submitted.")
+      case UploadResult.Success => BasicResult(success = true, "Files submitted.")
-      case UploadResult.NoFiles  => BasicResult(false, "There were no files to submit.")
+      case UploadResult.NoFiles =>
-      case UploadResult.NoSource => BasicResult(false, "The source id is not valid.")
+        BasicResult(success = false, "There were no files to submit.")
-      case UploadResult.NoItem   => BasicResult(false, "The item could not be found.")
+      case UploadResult.NoSource =>
+        BasicResult(success = false, "The source id is not valid.")
+      case UploadResult.NoItem =>
+        BasicResult(success = false, "The item could not be found.")
       case UploadResult.NoCollective =>
-        BasicResult(false, "The collective could not be found.")
+        BasicResult(success = false, "The collective could not be found.")
       case UploadResult.StoreFailure(_) =>
         BasicResult(
-          false,
+          success = false,
           "There were errors storing a file! See the server logs for details."
         )
     }
 
   def basicResult(cr: PassChangeResult): BasicResult =
     cr match {
-      case PassChangeResult.Success => BasicResult(true, "Password changed.")
+      case PassChangeResult.Success => BasicResult(success = true, "Password changed.")
       case PassChangeResult.UpdateFailed =>
-        BasicResult(false, "The database update failed.")
+        BasicResult(success = false, "The database update failed.")
       case PassChangeResult.PasswordMismatch =>
-        BasicResult(false, "The current password is incorrect.")
+        BasicResult(success = false, "The current password is incorrect.")
-      case PassChangeResult.UserNotFound => BasicResult(false, "User not found.")
+      case PassChangeResult.UserNotFound =>
+        BasicResult(success = false, "User not found.")
       case PassChangeResult.InvalidSource(source) =>
         BasicResult(
-          false,
+          success = false,
           s"User has invalid soure: $source. Passwords are managed elsewhere."
         )
     }
 
   def basicResult(e: Either[Throwable, _], successMsg: String): BasicResult =
     e match {
-      case Right(_) => BasicResult(true, successMsg)
+      case Right(_) => BasicResult(success = true, successMsg)
-      case Left(ex) => BasicResult(false, ex.getMessage)
+      case Left(ex) => BasicResult(success = false, ex.getMessage)
     }
 
   // MIME Type

modules/restserver/src/main/scala/docspell/restserver/http4s/BinaryUtil.scala

@@ -38,7 +38,7 @@ object BinaryUtil {
         if (matches) withResponseHeaders(dsl, NotModified())(data)
         else makeByteResp(dsl)(data)
       }
-      .getOrElse(NotFound(BasicResult(false, "Not found")))
+      .getOrElse(NotFound(BasicResult(success = false, "Not found")))
   }
 
   def respondHead[F[_]: Async](dsl: Http4sDsl[F])(
@@ -48,7 +48,7 @@ object BinaryUtil {
 
     fileData
       .map(data => withResponseHeaders(dsl, Ok())(data))
-      .getOrElse(NotFound(BasicResult(false, "Not found")))
+      .getOrElse(NotFound(BasicResult(success = false, "Not found")))
   }
 
   def respondPreview[F[_]: Async](dsl: Http4sDsl[F], req: Request[F])(
@@ -56,7 +56,7 @@ object BinaryUtil {
   ): F[Response[F]] = {
     import dsl._
     def notFound =
-      NotFound(BasicResult(false, "Not found"))
+      NotFound(BasicResult(success = false, "Not found"))
 
     QP.WithFallback.unapply(req.multiParams) match {
       case Some(bool) =>
@@ -75,7 +75,7 @@ object BinaryUtil {
           )
 
       case None =>
-        BadRequest(BasicResult(false, "Invalid query parameter 'withFallback'"))
+        BadRequest(BasicResult(success = false, "Invalid query parameter 'withFallback'"))
     }
   }
 
@@ -85,7 +85,7 @@ object BinaryUtil {
     import dsl._
     fileData
       .map(data => withResponseHeaders(dsl, Ok())(data))
-      .getOrElse(NotFound(BasicResult(false, "Not found")))
+      .getOrElse(NotFound(BasicResult(success = false, "Not found")))
   }
 
   def withResponseHeaders[F[_]: Sync](dsl: Http4sDsl[F], resp: F[Response[F]])(

modules/restserver/src/main/scala/docspell/restserver/http4s/ThrowableResponseMapper.scala

@@ -33,10 +33,10 @@ object ThrowableResponseMapper {
     def toResponse(ex: Throwable): F[Response[F]] =
       ex match {
         case _: IllegalArgumentException =>
-          BadRequest(BasicResult(false, ex.getMessage))
+          BadRequest(BasicResult(success = false, ex.getMessage))
 
         case _ =>
-          InternalServerError(BasicResult(false, ex.getMessage))
+          InternalServerError(BasicResult(success = false, ex.getMessage))
       }
   }
 }

modules/restserver/src/main/scala/docspell/restserver/routes/AddonArchiveRoutes.scala

@@ -52,7 +52,7 @@ object AddonArchiveRoutes extends AddonValidationSupport {
       case req @ POST -> Root :? Sync(sync) =>
         def create(r: Option[RAddonArchive]) =
           IdResult(
-            true,
+            success = true,
             r.fold("Addon submitted for installation")(r =>
               s"Addon installed: ${r.id.id}"
             ),
@@ -77,7 +77,7 @@ object AddonArchiveRoutes extends AddonValidationSupport {
       case PUT -> Root / Ident(id) :? Sync(sync) =>
         def create(r: Option[AddonMeta]) =
           BasicResult(
-            true,
+            success = true,
             r.fold("Addon updated in background")(m =>
               s"Addon updated: ${m.nameAndVersion}"
             )
@@ -99,8 +99,8 @@ object AddonArchiveRoutes extends AddonValidationSupport {
         for {
           flag <- backend.addons.deleteAddon(token.account.collectiveId, id)
           resp <-
-            if (flag) Ok(BasicResult(true, "Addon deleted"))
+            if (flag) Ok(BasicResult(success = true, "Addon deleted"))
-            else NotFound(BasicResult(false, "Addon not found"))
+            else NotFound(BasicResult(success = false, "Addon not found"))
         } yield resp
     }
   }
@@ -112,11 +112,11 @@ object AddonArchiveRoutes extends AddonValidationSupport {
     import dsl._
 
     def failWith(msg: String): F[Response[F]] =
-      Ok(IdResult(false, msg, Ident.unsafe("")))
+      Ok(IdResult(success = false, msg, Ident.unsafe("")))
 
     e match {
       case AddonValidationError.AddonNotFound =>
-        NotFound(BasicResult(false, "Addon not found."))
+        NotFound(BasicResult(success = false, "Addon not found."))
 
       case _ =>
         failWith(validationErrorToMessage(e))

modules/restserver/src/main/scala/docspell/restserver/routes/AddonRoutes.scala

@@ -35,5 +35,5 @@ object AddonRoutes {
         "run" -> AddonRunRoutes(backend, token)
       )
     else
-      Responses.notFoundRoute(BasicResult(false, "Addons disabled"))
+      Responses.notFoundRoute(BasicResult(success = false, "Addons disabled"))
 }

modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunConfigRoutes.scala

@@ -43,8 +43,8 @@ object AddonRunConfigRoutes {
               .map(_.leftMap(_.message))
           )
           resp <- res.fold(
-            msg => Ok(BasicResult(false, msg)),
+            msg => Ok(BasicResult(success = false, msg)),
-            id => Ok(IdResult(true, s"Addon run config added", id))
+            id => Ok(IdResult(success = true, s"Addon run config added", id))
           )
         } yield resp
 
@@ -58,8 +58,8 @@ object AddonRunConfigRoutes {
               .map(_.leftMap(_.message))
           )
           resp <- res.fold(
-            msg => Ok(BasicResult(false, msg)),
+            msg => Ok(BasicResult(success = false, msg)),
-            id => Ok(IdResult(true, s"Addon run config updated", id))
+            id => Ok(IdResult(success = true, s"Addon run config updated", id))
           )
         } yield resp
 
@@ -67,8 +67,8 @@ object AddonRunConfigRoutes {
         for {
           flag <- backend.addons.deleteAddonRunConfig(token.account.collectiveId, id)
           resp <-
-            if (flag) Ok(BasicResult(true, "Addon task deleted"))
+            if (flag) Ok(BasicResult(success = true, "Addon task deleted"))
-            else NotFound(BasicResult(false, "Addon task not found"))
+            else NotFound(BasicResult(success = false, "Addon task not found"))
         } yield resp
     }
   }

modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunRoutes.scala

@@ -35,7 +35,7 @@ object AddonRunRoutes {
           input.addonRunConfigIds.toSet,
           UserTaskScope(token.account)
         )
-        resp <- Ok(BasicResult(true, "Job for running addons submitted."))
+        resp <- Ok(BasicResult(success = true, "Job for running addons submitted."))
       } yield resp
     }
   }

modules/restserver/src/main/scala/docspell/restserver/routes/AttachmentRoutes.scala

@@ -66,7 +66,7 @@ object AttachmentRoutes {
           resp <-
             fileData
               .map(data => withResponseHeaders(Ok())(data))
-              .getOrElse(NotFound(BasicResult(false, "Not found")))
+              .getOrElse(NotFound(BasicResult(success = false, "Not found")))
         } yield resp
 
       case req @ GET -> Root / Ident(id) / "original" =>
@@ -83,7 +83,7 @@ object AttachmentRoutes {
                 if (matches) withResponseHeaders(NotModified())(data)
                 else makeByteResp(data)
               }
-              .getOrElse(NotFound(BasicResult(false, "Not found")))
+              .getOrElse(NotFound(BasicResult(success = false, "Not found")))
         } yield resp
 
       case HEAD -> Root / Ident(id) / "archive" =>
@@ -93,7 +93,7 @@ object AttachmentRoutes {
           resp <-
             fileData
               .map(data => withResponseHeaders(Ok())(data))
-              .getOrElse(NotFound(BasicResult(false, "Not found")))
+              .getOrElse(NotFound(BasicResult(success = false, "Not found")))
         } yield resp
 
       case req @ GET -> Root / Ident(id) / "archive" =>
@@ -108,7 +108,7 @@ object AttachmentRoutes {
                 if (matches) withResponseHeaders(NotModified())(data)
                 else makeByteResp(data)
               }
-              .getOrElse(NotFound(BasicResult(false, "Not found")))
+              .getOrElse(NotFound(BasicResult(success = false, "Not found")))
         } yield resp
 
       case req @ GET -> Root / Ident(id) / "preview" =>
@@ -148,7 +148,9 @@ object AttachmentRoutes {
         for {
           rm <- backend.itemSearch.findAttachmentMeta(id, user.account.collectiveId)
           md = rm.map(Conversions.mkAttachmentMeta)
-          resp <- md.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found.")))
+          resp <- md
+            .map(Ok(_))
+            .getOrElse(NotFound(BasicResult(success = false, "Not found.")))
         } yield resp
 
       case req @ POST -> Root / Ident(id) / "name" =>
@@ -169,8 +171,11 @@ object AttachmentRoutes {
             backend.attachment
               .setExtractedText(user.account.collectiveId, itemId, id, newText)
           )
-          resp <- OptionT.liftF(Ok(BasicResult(true, "Extracted text updated.")))
+          resp <- OptionT.liftF(
-        } yield resp).getOrElseF(NotFound(BasicResult(false, "Attachment not found")))
+            Ok(BasicResult(success = true, "Extracted text updated."))
+          )
+        } yield resp)
+          .getOrElseF(NotFound(BasicResult(success = false, "Attachment not found")))
 
       case DELETE -> Root / Ident(id) / "extracted-text" =>
         (for {
@@ -181,7 +186,9 @@ object AttachmentRoutes {
             backend.attachment
               .setExtractedText(user.account.collectiveId, itemId, id, "".pure[F])
           )
-          resp <- OptionT.liftF(Ok(BasicResult(true, "Extracted text cleared.")))
+          resp <- OptionT.liftF(
+            Ok(BasicResult(success = true, "Extracted text cleared."))
+          )
         } yield resp).getOrElseF(NotFound())
 
       case GET -> Root / Ident(id) / "extracted-text" =>
@@ -190,14 +197,15 @@ object AttachmentRoutes {
             backend.itemSearch.findAttachmentMeta(id, user.account.collectiveId)
           )
           resp <- OptionT.liftF(Ok(OptionalText(meta.content)))
-        } yield resp).getOrElseF(NotFound(BasicResult(false, "Attachment not found")))
+        } yield resp)
+          .getOrElseF(NotFound(BasicResult(success = false, "Attachment not found")))
 
       case DELETE -> Root / Ident(id) =>
         for {
           n <- backend.item.deleteAttachment(id, user.account.collectiveId)
           res =
-            if (n == 0) BasicResult(false, "Attachment not found")
+            if (n == 0) BasicResult(success = false, "Attachment not found")
-            else BasicResult(true, "Attachment deleted.")
+            else BasicResult(success = true, "Attachment deleted.")
           resp <- Ok(res)
         } yield resp
     }

modules/restserver/src/main/scala/docspell/restserver/routes/CalEventCheckRoutes.scala

@@ -40,9 +40,9 @@ object CalEventCheckRoutes {
           val next = ev
             .nextElapses(now.toUtcDateTime, 2)
             .map(Timestamp.atUtc)
-          CalEventCheckResult(true, "Valid.", ev.some, next)
+          CalEventCheckResult(success = true, "Valid.", ev.some, next)
         case Left(err) =>
-          CalEventCheckResult(false, err, None, Nil)
+          CalEventCheckResult(success = false, err, None, Nil)
       }
     }
 }

modules/restserver/src/main/scala/docspell/restserver/routes/ClientSettingsRoutes.scala

@@ -66,7 +66,7 @@ object ClientSettingsRoutes {
         for {
           data <- req.as[Json]
           _ <- backend.clientSettings.saveUser(clientId, user.account.userId, data)
-          res <- Ok(BasicResult(true, "Settings stored"))
+          res <- Ok(BasicResult(success = true, "Settings stored"))
         } yield res
 
       case GET -> Root / "user" / Ident(clientId) =>
@@ -97,7 +97,7 @@ object ClientSettingsRoutes {
             user.account.collectiveId,
             data
           )
-          res <- Ok(BasicResult(true, "Settings stored"))
+          res <- Ok(BasicResult(success = true, "Settings stored"))
         } yield res
 
       case GET -> Root / "collective" / Ident(clientId) =>

modules/restserver/src/main/scala/docspell/restserver/routes/CollectiveRoutes.scala

@@ -118,7 +118,7 @@ object CollectiveRoutes {
       case POST -> Root / "classifier" / "startonce" =>
         for {
           _ <- backend.collective.startLearnClassifier(user.account.collectiveId)
-          resp <- Ok(BasicResult(true, "Task submitted"))
+          resp <- Ok(BasicResult(success = true, "Task submitted"))
         } yield resp
 
       case req @ POST -> Root / "emptytrash" / "startonce" =>
@@ -127,7 +127,7 @@ object CollectiveRoutes {
           _ <- backend.collective.startEmptyTrash(
             EmptyTrashArgs(user.account.collectiveId, data.minAge)
           )
-          resp <- Ok(BasicResult(true, "Task submitted"))
+          resp <- Ok(BasicResult(success = true, "Task submitted"))
         } yield resp
     }
   }

modules/restserver/src/main/scala/docspell/restserver/routes/CustomFieldRoutes.scala

@@ -56,7 +56,7 @@ object CustomFieldRoutes {
         (for {
           field <- OptionT(backend.customFields.findById(user.account.collectiveId, id))
           res <- OptionT.liftF(Ok(convertField(field)))
-        } yield res).getOrElseF(NotFound(BasicResult(false, "Not found")))
+        } yield res).getOrElseF(NotFound(BasicResult(success = false, "Not found")))
 
       case req @ PUT -> Root / Ident(id) =>
         for {