.git-blame-ignore-revs

@@ -1,2 +0,0 @@
-# Scala Steward: Reformat with scalafmt 3.8.2
-1c566cd5182d41f4cc06040fc347ddb4be617779

.github/release-drafter.yml

@@ -1,42 +0,0 @@
-name-template: "$RESOLVED_VERSION"
-tag-template: "$RESOLVED_VERSION"
-template: |
-  ## What’s Changed
-
-  $CHANGES
-
-categories:
-  - title: "🚀 Features"
-    labels:
-      - 'feature'
-      - 'enhancement'
-  - title: "🐛 Bug Fixes"
-    labels:
-      - 'fix'
-      - 'bug'
-  - title: "💚 Maintenance"
-    labels:
-      - 'chore'
-      - 'documentation'
-  - title: "🧱 Dependencies"
-    labels:
-      - 'dependencies'
-      - 'type: dependencies'
-
-change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
-version-resolver:
-  major:
-    labels:
-      - 'breaking'
-  minor:
-    labels:
-      - 'feature'
-      - 'enhancement'
-  patch:
-    labels:
-      - 'chore'
-      - 'documentation'
-      - 'dependencies'
-  default: patch
-exclude-labels:
-  - 'skip-changelog'

.github/renovate.json

@@ -1,6 +1,6 @@
 {
   "automerge": true,
-  "labels": ["dependencies"],
+  "labels": ["type: dependencies"],
   "packageRules": [
     {
       "matchManagers": [

.github/stale.yml

@@ -0,0 +1,16 @@
+# Number of days of inactivity before an issue becomes stale
+daysUntilStale: 30
+# Number of days of inactivity before a stale issue is closed
+daysUntilClose: 7
+onlyLabels:
+  - question
+# Label to use when marking an issue as stale
+staleLabel: stale
+# Comment to post when marking an issue as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not
+  had recent activity. It will be closed if no further activity
+  occurs. This only applies to 'question' issues. Always feel free to
+  reopen or create new issues. Thank you!
+# Comment to post when closing a stale issue. Set to `false` to disable
+closeComment: false

.github/workflows/ci-docs.yml

@@ -6,13 +6,20 @@ on:
       - "master"
 jobs:
   check-website:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4.1.7
+      - uses: actions/checkout@v4.1.1
         with:
           fetch-depth: 0
-      - uses: cachix/install-nix-action@v27
       - name: Set current version
         run:  echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
+      - uses: jorelali/setup-elm@v5
+        with:
+          elm-version: 0.19.1
+      - uses: cachix/install-nix-action@v25
+        with:
+          nix_path: nixpkgs=channel:nixos-23.05
+      - name: Print nixpkgs version
+        run: nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
       - name: Build website (${{ env.DOCSPELL_VERSION }})
-        run: nix develop .#ci --command sbt make-website
+        run: nix-shell website/shell.nix --run "sbt make-website"

.github/workflows/ci.yml

@@ -5,18 +5,30 @@ on:
       - master
 jobs:
   ci-matrix:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
+      matrix:
+        java: [ 'openjdk@1.17' ]
     steps:
-      - uses: actions/checkout@v4.1.7
+      - uses: actions/checkout@v4.1.1
         with:
           fetch-depth: 100
+      - uses: jorelali/setup-elm@v5
+        with:
+          elm-version: 0.19.1
+      - uses: bahmutov/npm-install@v1
+        with:
+          working-directory: modules/webapp
       - name: Fetch tags
         run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
-      - uses: cachix/install-nix-action@v27
+      - uses: olafurpg/setup-scala@v14
+        with:
+          java-version: ${{ matrix.java }}
+      # - name: Coursier cache
+      #   uses: coursier/cache-action@v6
       - name: sbt ci ${{ github.ref }}
-        run: nix develop .#ci --command sbt ci
+        run: sbt ci
   ci:
     runs-on: ubuntu-22.04
     needs: [ci-matrix]

.github/workflows/docker-image.yml

@@ -4,9 +4,9 @@ on:
     types: [ published ]
 jobs:
   docker-images:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4.1.7
+      - uses: actions/checkout@v4.1.1
         with:
           fetch-depth: 0
       - name: Set current version

.github/workflows/release-drafter.yml

@@ -1,14 +0,0 @@
-name: Release Drafter
-
-on:
-  push:
-    branches:
-      - master
-
-jobs:
-  update_release_draft:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: release-drafter/release-drafter@v6
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release-nightly.yml

@@ -5,20 +5,32 @@ on:
       - "master"
 jobs:
   release-nightly:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: true
+      matrix:
+        java: [ 'openjdk@1.17' ]
     steps:
-      - uses: actions/checkout@v4.1.7
+      - uses: actions/checkout@v4.1.1
         with:
           fetch-depth: 0
-      - uses: cachix/install-nix-action@v27
+      - uses: olafurpg/setup-scala@v14
+        with:
+          java-version: ${{ matrix.java }}
+      - uses: jorelali/setup-elm@v5
+        with:
+          elm-version: 0.19.1
+      - uses: bahmutov/npm-install@v1
+        with:
+          working-directory: modules/webapp
+      # - name: Coursier cache
+      #   uses: coursier/cache-action@v6
       - name: Set current version
         run:  echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
       - name: sbt ci ${{ github.ref }}
-        run: nix develop .#ci --command sbt ci
+        run: sbt ci
       - name: sbt make-pkg (${{ env.DOCSPELL_VERSION }})
-        run: nix develop .#ci --command sbt make-pkg
+        run: sbt make-pkg
       - uses: "marvinpinto/action-automatic-releases@latest"
         with:
           repo_token: "${{ secrets.GITHUB_TOKEN }}"

.github/workflows/release.yml

@@ -5,18 +5,30 @@ on:
       - 'v*'
 jobs:
   release:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: true
+      matrix:
+        java: [ 'openjdk@1.17' ]
     steps:
-      - uses: actions/checkout@v4.1.7
+      - uses: actions/checkout@v4.1.1
         with:
           fetch-depth: 0
-      - uses: cachix/install-nix-action@v27
+      - uses: olafurpg/setup-scala@v14
+        with:
+          java-version: ${{ matrix.java }}
+      - uses: jorelali/setup-elm@v5
+        with:
+          elm-version: 0.19.1
+      - uses: bahmutov/npm-install@v1
+        with:
+          working-directory: modules/webapp
+      # - name: Coursier cache
+      #   uses: coursier/cache-action@v6
       - name: Set current version
         run:  echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
       - name: sbt make-pkg (${{ env.DOCSPELL_VERSION }})
-        run: nix develop .#ci --command sbt make-pkg
+        run: sbt make-pkg
       - uses: meeDamian/github-release@2.0
         with:
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/stale.yml

@@ -1,21 +0,0 @@
-name: 'Handle stale issues'
-on:
-  schedule:
-    - cron: '30 1 * * *'
-
-jobs:
-  stale:
-    runs-on: ubuntu-latest
-    steps:
-      # https://github.com/actions/stale
-      - uses: actions/stale@v9
-        with:
-          days-before-stale: 30
-          days-before-close: 7
-          only-labels: question
-          stale-issue-label: stale
-          stale-issue-message: >
-            This issue has been automatically marked as stale because it has not
-            had recent activity. It will be closed if no further activity
-            occurs. This only applies to 'question' issues. Always feel free to
-            reopen or create new issues. Thank you!

.github/workflows/website.yml

@@ -5,17 +5,24 @@ on:
       - "current-docs"
 jobs:
   publish-website:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4.1.7
+      - uses: actions/checkout@v4.1.1
         with:
           fetch-depth: 0
-      - uses: cachix/install-nix-action@v27
       - name: Set current version
         run:  echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
+      - uses: jorelali/setup-elm@v5
+        with:
+          elm-version: 0.19.1
+      - uses: cachix/install-nix-action@v25
+        with:
+          nix_path: nixpkgs=channel:nixos-23.05
+      - name: Print nixpkgs version
+        run: nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
       - name: Build website (${{ env.DOCSPELL_VERSION }})
-        run: nix develop .#ci --command sbt make-website
+        run: nix-shell website/shell.nix --run "sbt make-website"
       - name: Publish website (${{ env.DOCSPELL_VERSION }})
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: nix develop .#ci --command sbt publish-website
+        run: sbt publish-website

.gitignore

@@ -1,5 +1,4 @@
 #artwork/*.png
-.envrc
 target/
 local/
 node_modules/

.mergify.yml

@@ -6,7 +6,7 @@ pull_request_rules:
       assign:
         users: [eikek]
       label:
-        add: ["dependencies"]
+        add: ["type: dependencies"]
   - name: automatically merge Scala Steward PRs on CI success
     conditions:
       - author=eikek-scala-steward[bot]

.scalafmt.conf

@@ -1,4 +1,4 @@
-version = "3.8.2"
+version = "3.7.17"
 
 preset = default
 align.preset = some

Changelog.md

@@ -1020,7 +1020,7 @@ Additionally there are some other minor features and bug fixes.
     to be able to add a request header. Check [this for
     firefox](https://addons.mozilla.org/en-US/firefox/addon/modheader-firefox/)
     or [this for
-    chromium](https://chromewebstore.google.com/detail/modheader-modify-http-hea/idgpnmonknjnojddfkpgkljpfnnfcklj)
+    chromium](https://chrome.google.com/webstore/detail/modheader/idgpnmonknjnojddfkpgkljpfnnfcklj)
   - then add the request header `Docspell-Ui` with value `1`.
     Reloading the page gets you back the old ui.
 - With new Web-UI, certain features and fixes were realized, but not

build.sbt

@@ -15,14 +15,11 @@ val scalafixSettings = Seq(
 
 val sharedSettings = Seq(
   organization := "com.github.eikek",
-  scalaVersion := "2.13.14",
+  scalaVersion := "2.13.12",
   organizationName := "Eike K. & Contributors",
-  licenses += (
+  licenses += ("AGPL-3.0-or-later", url(
-    "AGPL-3.0-or-later",
+    "https://spdx.org/licenses/AGPL-3.0-or-later.html"
-    url(
+  )),
-      "https://spdx.org/licenses/AGPL-3.0-or-later.html"
-    )
-  ),
   startYear := Some(2020),
   headerLicenseStyle := HeaderLicenseStyle.SpdxSyntax,
   headerSources / excludeFilter := HiddenFileFilter || "*.java" || "StringUtil.scala",
@@ -680,11 +677,7 @@ val restapi = project
     openapiTargetLanguage := Language.Scala,
     openapiPackage := Pkg("docspell.restapi.model"),
     openapiSpec := (Compile / resourceDirectory).value / "docspell-openapi.yml",
-    openapiStaticGen := OpenApiDocGenerator.Redoc,
+    openapiStaticGen := OpenApiDocGenerator.Redoc
-    openapiRedoclyCmd := Seq("redocly-cli"),
-    openapiRedoclyConfig := Some(
-      (LocalRootProject / baseDirectory).value / "project" / "redocly.yml"
-    )
   )
   .dependsOn(common, query.jvm, notificationApi, jsonminiq, addonlib)
 
@@ -704,11 +697,7 @@ val joexapi = project
     openapiTargetLanguage := Language.Scala,
     openapiPackage := Pkg("docspell.joexapi.model"),
     openapiSpec := (Compile / resourceDirectory).value / "joex-openapi.yml",
-    openapiStaticGen := OpenApiDocGenerator.Redoc,
+    openapiStaticGen := OpenApiDocGenerator.Redoc
-    openapiRedoclyCmd := Seq("redocly-cli"),
-    openapiRedoclyConfig := Some(
-      (LocalRootProject / baseDirectory).value / "project" / "redocly.yml"
-    )
   )
   .dependsOn(common, loggingScribe, addonlib)
 

docker/docker-compose/docker-compose.yml

@@ -109,7 +109,7 @@ services:
       - restserver
 
   db:
-    image: postgres:16.3
+    image: postgres:16.1
     container_name: postgres_db
     restart: unless-stopped
     volumes:

docker/dockerfiles/joex.dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.20.2
+FROM alpine:20231219
 
 ARG version=
 ARG joex_url=
@@ -77,7 +77,7 @@ RUN \
   wget https://github.com/tesseract-ocr/tessdata/raw/main/khm.traineddata && \
   mv khm.traineddata /usr/share/tessdata
 
-# Using these data files for japanese, because they work better. Includes vertical data. See #973 and #2445.
+# Using these data files for japanese, because they work better. See #973
 RUN \
   wget https://raw.githubusercontent.com/tesseract-ocr/tessdata_fast/master/jpn_vert.traineddata && \
   wget https://raw.githubusercontent.com/tesseract-ocr/tessdata_fast/master/jpn.traineddata && \

docker/dockerfiles/restserver.dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.20.2
+FROM alpine:20231219
 
 ARG version=
 ARG restserver_url=

flake.lock

@@ -1,130 +0,0 @@
-{
-  "nodes": {
-    "devshell-tools": {
-      "inputs": {
-        "flake-utils": "flake-utils",
-        "nixpkgs": "nixpkgs"
-      },
-      "locked": {
-        "lastModified": 1710099997,
-        "narHash": "sha256-WmBKTLdth6I/D+0//9enbIXohGsBjepbjIAm9pCYj0U=",
-        "owner": "eikek",
-        "repo": "devshell-tools",
-        "rev": "e82faf976d318b3829f6f7f6785db6f3c7b65267",
-        "type": "github"
-      },
-      "original": {
-        "owner": "eikek",
-        "repo": "devshell-tools",
-        "type": "github"
-      }
-    },
-    "flake-utils": {
-      "inputs": {
-        "systems": "systems"
-      },
-      "locked": {
-        "lastModified": 1709126324,
-        "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "d465f4819400de7c8d874d50b982301f28a84605",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "flake-utils_2": {
-      "inputs": {
-        "systems": "systems_2"
-      },
-      "locked": {
-        "lastModified": 1709126324,
-        "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "d465f4819400de7c8d874d50b982301f28a84605",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "nixpkgs": {
-      "locked": {
-        "lastModified": 1709309926,
-        "narHash": "sha256-VZFBtXGVD9LWTecGi6eXrE0hJ/mVB3zGUlHImUs2Qak=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "79baff8812a0d68e24a836df0a364c678089e2c7",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-23.11",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs_2": {
-      "locked": {
-        "lastModified": 1709677081,
-        "narHash": "sha256-tix36Y7u0rkn6mTm0lA45b45oab2cFLqAzDbJxeXS+c=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "880992dcc006a5e00dd0591446fdf723e6a51a64",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixos-23.11",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "root": {
-      "inputs": {
-        "devshell-tools": "devshell-tools",
-        "flake-utils": "flake-utils_2",
-        "nixpkgs": "nixpkgs_2"
-      }
-    },
-    "systems": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    },
-    "systems_2": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    }
-  },
-  "root": "root",
-  "version": 7
-}

flake.nix

@@ -1,193 +0,0 @@
-{
-  description = "Docspell";
-
-  inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11";
-    devshell-tools.url = "github:eikek/devshell-tools";
-    flake-utils.url = "github:numtide/flake-utils";
-  };
-
-  outputs = {
-    self,
-    nixpkgs,
-    flake-utils,
-    devshell-tools,
-  }:
-    flake-utils.lib.eachDefaultSystem (system: let
-      pkgs = nixpkgs.legacyPackages.${system};
-      sbt17 = pkgs.sbt.override {jre = pkgs.jdk17;};
-      ciPkgs = with pkgs; [
-        sbt17
-        jdk17
-        dpkg
-        elmPackages.elm
-        fakeroot
-        zola
-        yarn
-        nodejs
-        redocly-cli
-        tailwindcss
-      ];
-      devshellPkgs =
-        ciPkgs
-        ++ (with pkgs; [
-          jq
-          scala-cli
-          netcat
-          wget
-          which
-          inotifyTools
-        ]);
-      docspellPkgs = pkgs.callPackage (import ./nix/pkg.nix) {};
-      dockerAmd64 = pkgs.pkgsCross.gnu64.callPackage (import ./nix/docker.nix) {
-        inherit (docspellPkgs) docspell-restserver docspell-joex;
-      };
-      dockerArm64 = pkgs.pkgsCross.aarch64-multiplatform.callPackage (import ./nix/docker.nix) {
-        inherit (docspellPkgs) docspell-restserver docspell-joex;
-      };
-    in {
-      formatter = pkgs.alejandra;
-
-      packages = {
-        inherit (docspellPkgs) docspell-restserver docspell-joex;
-      };
-
-      legacyPackages = {
-        docker = {
-          amd64 = {
-            inherit (dockerAmd64) docspell-restserver docspell-joex;
-          };
-          arm64 = {
-            inherit (dockerArm64) docspell-restserver docspell-joex;
-          };
-        };
-      };
-
-      checks = {
-        build-server = self.packages.${system}.docspell-restserver;
-        build-joex = self.packages.${system}.docspell-joex;
-
-        test = with import (nixpkgs + "/nixos/lib/testing-python.nix")
-        {
-          inherit system;
-        };
-          makeTest {
-            name = "docspell";
-            nodes = {
-              machine = {...}: {
-                nixpkgs.overlays = [self.overlays.default];
-                imports = [
-                  self.nixosModules.default
-                  ./nix/checks
-                ];
-              };
-            };
-
-            testScript = builtins.readFile ./nix/checks/testScript.py;
-          };
-      };
-
-      devShells = {
-        dev-cnt = pkgs.mkShellNoCC {
-          buildInputs =
-            (builtins.attrValues devshell-tools.legacyPackages.${system}.cnt-scripts)
-            ++ devshellPkgs;
-
-          DOCSPELL_ENV = "dev";
-          DEV_CONTAINER = "docsp-dev";
-          SBT_OPTS = "-Xmx2G -Xss4m";
-        };
-        dev-vm = pkgs.mkShellNoCC {
-          buildInputs =
-            (builtins.attrValues devshell-tools.legacyPackages.${system}.vm-scripts)
-            ++ devshellPkgs;
-
-          DOCSPELL_ENV = "dev";
-          SBT_OPTS = "-Xmx2G -Xss4m";
-          DEV_VM = "dev-vm";
-          VM_SSH_PORT = "10022";
-        };
-        ci = pkgs.mkShellNoCC {
-          buildInputs = ciPkgs;
-          SBT_OPTS = "-Xmx2G -Xss4m";
-        };
-      };
-    })
-    // {
-      nixosModules = {
-        default = {...}: {
-          imports = [
-            ./nix/modules/server.nix
-            ./nix/modules/joex.nix
-          ];
-        };
-        server = import ./nix/modules/server.nix;
-        joex = import ./nix/modules/joex.nix;
-      };
-
-      overlays.default = final: prev: let
-        docspellPkgs = final.callPackage (import ./nix/pkg.nix) {};
-      in {
-        inherit (docspellPkgs) docspell-restserver docspell-joex;
-      };
-
-      nixosConfigurations = {
-        test-vm = devshell-tools.lib.mkVm {
-          system = "x86_64-linux";
-          modules = [
-            self.nixosModules.default
-            {
-              nixpkgs.overlays = [self.overlays.default];
-            }
-            ./nix/test-vm.nix
-          ];
-        };
-        docsp-dev = devshell-tools.lib.mkContainer {
-          system = "x86_64-linux";
-          modules = [
-            {
-              services.dev-postgres = {
-                enable = true;
-                databases = ["docspell"];
-              };
-              services.dev-email.enable = true;
-              services.dev-minio.enable = true;
-              services.dev-solr = {
-                enable = true;
-                cores = ["docspell"];
-              };
-            }
-          ];
-        };
-        dev-vm = devshell-tools.lib.mkVm {
-          system = "x86_64-linux";
-          modules = [
-            {
-              networking.hostName = "dev-vm";
-              virtualisation.memorySize = 2048;
-
-              services.dev-postgres = {
-                enable = true;
-                databases = ["docspell"];
-              };
-              services.dev-email.enable = true;
-              services.dev-minio.enable = true;
-              services.dev-solr = {
-                enable = true;
-                cores = ["docspell"];
-                heap = 512;
-              };
-              port-forward.ssh = 10022;
-              port-forward.dev-postgres = 6534;
-              port-forward.dev-smtp = 10025;
-              port-forward.dev-imap = 10143;
-              port-forward.dev-webmail = 8080;
-              port-forward.dev-minio-api = 9000;
-              port-forward.dev-minio-console = 9001;
-              port-forward.dev-solr = 8983;
-            }
-          ];
-        };
-      };
-    };
-}

modules/addonlib/src/main/scala/docspell/addons/AddonArchive.scala

@@ -38,9 +38,9 @@ final case class AddonArchive(url: LenientUri, name: String, version: String) {
           Files[F].createDirectories(target) *>
             reader(url)
               .through(Zip[F](logger.some).unzip(glob = glob, targetDir = target.some))
+              .evalTap(_ => Directory.unwrapSingle[F](logger, target))
               .compile
               .drain
-              .flatTap(_ => Directory.unwrapSingle[F](logger, target))
               .as(target)
       }
   }

modules/addonlib/src/main/scala/docspell/addons/runner/RunnerUtil.scala

@@ -110,7 +110,7 @@ private[addons] object RunnerUtil {
   ): F[AddonResult] =
     for {
       stdout <-
-        if (ctx.meta.parseResult) CollectOut.buffer[F]
+        if (ctx.meta.options.exists(_.collectOutput)) CollectOut.buffer[F]
         else CollectOut.none[F].pure[F]
       cmdResult <- SysExec(cmd, logger, ctx.baseDir.some)
         .flatMap(
@@ -135,7 +135,7 @@ private[addons] object RunnerUtil {
             out <- stdout.get
             _ <- logger.debug(s"Addon stdout: $out")
             result = Option
-              .when(ctx.meta.parseResult && out.nonEmpty)(
+              .when(ctx.meta.options.exists(_.collectOutput) && out.nonEmpty)(
                 JsonParser
                   .decode[AddonOutput](out)
                   .fold(AddonResult.decodingError, AddonResult.success)

modules/addonlib/src/test/scala/docspell/addons/AddonArchiveTest.scala

@@ -9,7 +9,7 @@ package docspell.addons
 import cats.effect._
 import cats.syntax.option._
 
-import docspell.common._
+import docspell.common.UrlReader
 import docspell.logging.TestLoggingConfig
 
 import munit._
@@ -42,20 +42,10 @@ class AddonArchiveTest extends CatsEffectSuite with TestLoggingConfig with Fixtu
     } yield ()
   }
 
-  tempDir.test("read archive from zip with yaml only") { dir =>
-    for {
-      aa <- AddonArchive.read[IO](singleFileAddonUrl, UrlReader.defaultReader[IO], None)
-      _ = assertEquals(aa.version, "0.7.0")
-      path <- aa.extractTo(UrlReader.defaultReader[IO], dir)
-      read <- AddonArchive.read[IO](aa.url, UrlReader.defaultReader[IO], path.some)
-      _ = assertEquals(aa, read)
-    } yield ()
-  }
-
   tempDir.test("Read generated addon from path") { dir =>
     AddonGenerator.successAddon("mini-addon").use { addon =>
       for {
-        archive <- IO(AddonArchive(addon.url, "test-addon", "0.1.0"))
+        archive <- IO(AddonArchive(addon.url, "", ""))
         path <- archive.extractTo[IO](UrlReader.defaultReader[IO], dir)
 
         read <- AddonArchive.read[IO](addon.url, UrlReader.defaultReader[IO], path.some)

modules/addonlib/src/test/scala/docspell/addons/AddonExecutorTest.scala

@@ -142,7 +142,7 @@ class AddonExecutorTest extends CatsEffectSuite with Fixtures with TestLoggingCo
       AddonExecutionResult.executionResultMonoid
         .combine(
           AddonExecutionResult.empty,
-          AddonExecutionResult(Nil, pure = true)
+          AddonExecutionResult(Nil, true)
         )
         .pure
     )

modules/addonlib/src/test/scala/docspell/addons/AddonGenerator.scala

@@ -27,9 +27,9 @@ object AddonGenerator {
   ): Resource[IO, AddonArchive] =
     output match {
       case None =>
-        generate(name, version, collectOutput = false)("exit 0")
+        generate(name, version, false)("exit 0")
       case Some(out) =>
-        generate(name, version, collectOutput = true)(
+        generate(name, version, true)(
           s"""
              |cat <<-EOF
              |${out.asJson.noSpaces}
@@ -77,9 +77,8 @@ object AddonGenerator {
       meta = AddonMeta.Meta(name, version, None),
       triggers = Set(AddonTriggerType.ExistingItem: AddonTriggerType).some,
       args = None,
-      runner = AddonMeta
+      runner =
-        .Runner(None, None, AddonMeta.TrivialRunner(enable = true, "addon.sh").some)
+        AddonMeta.Runner(None, None, AddonMeta.TrivialRunner(true, "addon.sh").some).some,
-        .some,
       options =
         AddonMeta.Options(networking = !collectOutput, collectOutput = collectOutput).some
     )

modules/addonlib/src/test/scala/docspell/addons/AddonMetaTest.scala

@@ -35,13 +35,4 @@ class AddonMetaTest extends CatsEffectSuite with TestLoggingConfig with Fixtures
       _ = assertEquals(meta, dummyAddonMeta)
     } yield ()
   }
-
-  test("parse yaml with defaults") {
-    val yamlStr = """meta:
-                    |  name: "test"
-                    |  version: "0.1.0"
-                    |""".stripMargin
-    val meta = AddonMeta.fromYamlString(yamlStr).fold(throw _, identity)
-    assert(meta.parseResult)
-  }
 }

modules/addonlib/src/test/scala/docspell/addons/Fixtures.scala

@@ -31,9 +31,6 @@ trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite =>
   val miniAddonUrl =
     LenientUri.fromJava(getClass.getResource("/minimal-addon.zip"))
 
-  val singleFileAddonUrl =
-    LenientUri.fromJava(getClass.getResource("/docspell-addon-single-file.zip"))
-
   val dummyAddonMeta =
     AddonMeta(
       meta =
@@ -43,13 +40,13 @@ trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite =>
       ),
       None,
       runner = Runner(
-        nix = NixRunner(enable = true).some,
+        nix = NixRunner(true).some,
         docker = DockerRunner(
           enable = true,
           image = None,
           build = "Dockerfile".some
         ).some,
-        trivial = TrivialRunner(enable = true, "src/addon.sh").some
+        trivial = TrivialRunner(true, "src/addon.sh").some
       ).some,
       options = Options(networking = true, collectOutput = true).some
     )
@@ -58,7 +55,7 @@ trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite =>
     Path(s"/tmp/target/test-temp")
 
   val tempDir =
-    ResourceFunFixture[Path](
+    ResourceFixture[Path](
       Resource.eval(Files[IO].createDirectories(baseTempDir)) *>
         Files[IO]
           .tempDirectory(baseTempDir.some, "run-", PosixPermissions.fromOctal("777"))
@@ -68,7 +65,7 @@ trait Fixtures extends TestLoggingConfig { self: CatsEffectSuite =>
       runner: RunnerType,
       runners: RunnerType*
   ): AddonExecutorConfig = {
-    val nspawn = NSpawn(enabled = false, "sudo", "systemd-nspawn", Duration.millis(100))
+    val nspawn = NSpawn(false, "sudo", "systemd-nspawn", Duration.millis(100))
     AddonExecutorConfig(
       runner = runner :: runners.toList,
       runTimeout = Duration.minutes(2),

modules/analysis/src/main/scala/docspell/analysis/date/DateFind.scala

@@ -125,7 +125,6 @@ object DateFind {
         case Language.Dutch      => dmy.or(ymd).or(mdy)
         case Language.Latvian    => dmy.or(lavLong).or(ymd)
         case Language.Japanese   => ymd
-        case Language.JpnVert    => ymd
         case Language.Hebrew     => dmy
         case Language.Lithuanian => ymd
         case Language.Polish     => dmy

modules/analysis/src/main/scala/docspell/analysis/date/MonthName.scala

@@ -54,8 +54,6 @@ object MonthName {
         latvian
       case Language.Japanese =>
         japanese
-      case Language.JpnVert =>
-        japanese
       case Language.Hebrew =>
         hebrew
       case Language.Lithuanian =>

modules/analysis/src/test/scala/docspell/analysis/nlp/StanfordNerAnnotatorSuite.scala

@@ -22,7 +22,7 @@ import munit._
 
 class StanfordNerAnnotatorSuite extends FunSuite with TestLoggingConfig {
   lazy val germanClassifier =
-    new StanfordCoreNLP(Properties.nerGerman(None, highRecall = false))
+    new StanfordCoreNLP(Properties.nerGerman(None, false))
   lazy val englishClassifier =
     new StanfordCoreNLP(Properties.nerEnglish(None))
 

modules/backend/src/main/scala/docspell/backend/Config.scala

@@ -90,6 +90,6 @@ object Config {
   }
   object Addons {
     val disabled: Addons =
-      Addons(enabled = false, allowImpure = false, UrlMatcher.False, UrlMatcher.True)
+      Addons(false, false, UrlMatcher.False, UrlMatcher.True)
   }
 }

modules/backend/src/main/scala/docspell/backend/auth/Login.scala

@@ -127,7 +127,7 @@ object Login {
           _ <- logF.trace(s"Account lookup: $data")
           res <- data match {
             case Some(d) if checkNoPassword(d, Set(AccountSource.OpenId)) =>
-              doLogin(config, d.account, rememberMe = false)
+              doLogin(config, d.account, false)
             case Some(d) if checkNoPassword(d, Set(AccountSource.Local)) =>
               config.onAccountSourceConflict match {
                 case OnAccountSourceConflict.Fail =>
@@ -145,7 +145,7 @@ object Login {
                           AccountSource.OpenId
                         )
                       )
-                    res <- doLogin(config, d.account, rememberMe = false)
+                    res <- doLogin(config, d.account, false)
                   } yield res
               }
             case _ =>
@@ -212,12 +212,7 @@ object Login {
         val okResult: F[Result] =
           for {
             _ <- store.transact(RUser.updateLogin(sf.token.account))
-            newToken <- AuthToken.user(
+            newToken <- AuthToken.user(sf.token.account, false, config.serverSecret, None)
-              sf.token.account,
-              requireSecondFactor = false,
-              config.serverSecret,
-              None
-            )
             rem <- OptionT
               .whenF(sf.rememberMe && config.rememberMe.enabled)(
                 insertRememberToken(store, sf.token.account, config)
@@ -244,9 +239,7 @@ object Login {
         (for {
           _ <- validateToken
           key <- EitherT.fromOptionF(
-            store.transact(
+            store.transact(RTotp.findEnabledByUserId(sf.token.account.userId, true)),
-              RTotp.findEnabledByUserId(sf.token.account.userId, enabled = true)
-            ),
             Result.invalidAuth
           )
           now <- EitherT.right[Result](Timestamp.current[F])
@@ -262,12 +255,7 @@ object Login {
         def okResult(acc: AccountInfo) =
           for {
             _ <- store.transact(RUser.updateLogin(acc))
-            token <- AuthToken.user(
+            token <- AuthToken.user(acc, false, config.serverSecret, None)
-              acc,
-              requireSecondFactor = false,
-              config.serverSecret,
-              None
-            )
           } yield Result.ok(token, None)
 
         def rememberedLogin(rid: Ident) =

modules/backend/src/main/scala/docspell/backend/joex/AddonOps.scala

@@ -93,7 +93,7 @@ object AddonOps {
           AddonResult.executionFailed(
             new Exception(s"Addon run config ${id.id} not found.")
           ) :: Nil,
-          pure = false
+          false
         ) :: Nil,
         Nil
       )

modules/backend/src/main/scala/docspell/backend/joex/AddonPrepare.scala

@@ -72,7 +72,7 @@ private[joex] class AddonPrepare[F[_]: Sync](store: Store[F]) extends LoggerExte
 
       token <- AuthToken.user(
         account,
-        requireSecondFactor = false,
+        false,
         secret.getOrElse(ByteVector.empty),
         tokenValidity.some
       )

modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala

@@ -194,14 +194,7 @@ object OCollective {
           id <- Ident.randomId[F]
           settings = sett.emptyTrash.getOrElse(EmptyTrash.default)
           args = EmptyTrashArgs(cid, settings.minAge)
-          ut = UserTask(
+          ut = UserTask(id, EmptyTrashArgs.taskName, true, settings.schedule, None, args)
-            id,
-            EmptyTrashArgs.taskName,
-            enabled = true,
-            settings.schedule,
-            None,
-            args
-          )
           _ <- uts.updateOneTask(UserTaskScope.collective(cid), args.makeSubject.some, ut)
           _ <- joex.notifyAllNodes
         } yield ()
@@ -227,7 +220,7 @@ object OCollective {
           ut = UserTask(
             id,
             LearnClassifierArgs.taskName,
-            enabled = true,
+            true,
             CalEvent(WeekdayComponent.All, DateEvent.All, TimeEvent.All),
             None,
             args
@@ -246,7 +239,7 @@ object OCollective {
           ut = UserTask(
             id,
             EmptyTrashArgs.taskName,
-            enabled = true,
+            true,
             CalEvent(WeekdayComponent.All, DateEvent.All, TimeEvent.All),
             None,
             args

modules/backend/src/main/scala/docspell/backend/ops/ONotification.scala

@@ -114,14 +114,14 @@ object ONotification {
           )
           _ <- notMod.send(logbuf._2.andThen(log), ev, ch)
           logs <- logbuf._1.get
-          res = SendTestResult(success = true, logs)
+          res = SendTestResult(true, logs)
         } yield res).attempt
           .map {
             case Right(res) => res
             case Left(ex) =>
               val ev =
                 LogEvent.of(Level.Error, "Failed sending sample event").addError(ex)
-              SendTestResult(success = false, Vector(ev))
+              SendTestResult(false, Vector(ev))
           }
 
       def listChannels(userId: Ident): F[Vector[Channel]] =

modules/backend/src/main/scala/docspell/backend/ops/OTotp.scala

@@ -120,9 +120,7 @@ object OTotp {
       def confirmInit(accountId: AccountInfo, otp: OnetimePassword): F[ConfirmResult] =
         for {
           _ <- log.info(s"Confirm TOTP setup for account ${accountId.asString}")
-          key <- store.transact(
+          key <- store.transact(RTotp.findEnabledByUserId(accountId.userId, false))
-            RTotp.findEnabledByUserId(accountId.userId, enabled = false)
-          )
           now <- Timestamp.current[F]
           res <- key match {
             case None =>
@@ -131,7 +129,7 @@ object OTotp {
               val check = totp.checkPassword(r.secret, otp, now.value)
               if (check)
                 store
-                  .transact(RTotp.setEnabled(accountId.userId, enabled = true))
+                  .transact(RTotp.setEnabled(accountId.userId, true))
                   .map(_ => ConfirmResult.Success)
               else ConfirmResult.Failed.pure[F]
           }
@@ -142,7 +140,7 @@ object OTotp {
           case Some(pw) =>
             for {
               _ <- log.info(s"Validating TOTP, because it is requested to disable it.")
-              key <- store.transact(RTotp.findEnabledByLogin(accountId, enabled = true))
+              key <- store.transact(RTotp.findEnabledByLogin(accountId, true))
               now <- Timestamp.current[F]
               res <- key match {
                 case None =>
@@ -151,7 +149,7 @@ object OTotp {
                   val check = totp.checkPassword(r.secret, pw, now.value)
                   if (check)
                     UpdateResult.fromUpdate(
-                      store.transact(RTotp.setEnabled(r.userId, enabled = false))
+                      store.transact(RTotp.setEnabled(r.userId, false))
                     )
                   else
                     log.info(s"TOTP code was invalid. Not disabling it.") *> UpdateResult
@@ -162,15 +160,15 @@ object OTotp {
           case None =>
             UpdateResult.fromUpdate {
               (for {
-                key <- OptionT(RTotp.findEnabledByLogin(accountId, enabled = true))
+                key <- OptionT(RTotp.findEnabledByLogin(accountId, true))
-                n <- OptionT.liftF(RTotp.setEnabled(key.userId, enabled = false))
+                n <- OptionT.liftF(RTotp.setEnabled(key.userId, false))
               } yield n).mapK(store.transform).getOrElse(0)
             }
         }
 
       def state(acc: AccountInfo): F[OtpState] =
         for {
-          record <- store.transact(RTotp.findEnabledByUserId(acc.userId, enabled = true))
+          record <- store.transact(RTotp.findEnabledByUserId(acc.userId, true))
           result = record match {
             case Some(r) =>
               OtpState.Enabled(r.created)

modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala

@@ -159,7 +159,7 @@ object OUpload {
             data.meta.skipDuplicates,
             data.meta.fileFilter.some,
             data.meta.tags.some,
-            reprocess = false,
+            false,
             data.meta.attachmentsOnly,
             data.meta.customData
           )

modules/backend/src/test/scala/docspell/backend/auth/AuthTokenTest.scala

@@ -32,12 +32,9 @@ class AuthTokenTest extends CatsEffectSuite {
   val otherSecret = ByteVector.fromValidHex("16bad")
 
   test("validate") {
-    val token1 =
+    val token1 = AuthToken.user[IO](user, false, secret, None).unsafeRunSync()
-      AuthToken.user[IO](user, requireSecondFactor = false, secret, None).unsafeRunSync()
     val token2 =
-      AuthToken
+      AuthToken.user[IO](user, false, secret, Duration.seconds(10).some).unsafeRunSync()
-        .user[IO](user, requireSecondFactor = false, secret, Duration.seconds(10).some)
-        .unsafeRunSync()
     assert(token1.validate(secret, Duration.seconds(5)))
     assert(!token1.validate(otherSecret, Duration.seconds(5)))
     assert(!token1.copy(account = john).validate(secret, Duration.seconds(5)))
@@ -49,12 +46,9 @@ class AuthTokenTest extends CatsEffectSuite {
   }
 
   test("signature") {
-    val token1 =
+    val token1 = AuthToken.user[IO](user, false, secret, None).unsafeRunSync()
-      AuthToken.user[IO](user, requireSecondFactor = false, secret, None).unsafeRunSync()
     val token2 =
-      AuthToken
+      AuthToken.user[IO](user, false, secret, Duration.seconds(10).some).unsafeRunSync()
-        .user[IO](user, requireSecondFactor = false, secret, Duration.seconds(10).some)
-        .unsafeRunSync()
 
     assert(token1.sigValid(secret))
     assert(token1.sigInvalid(otherSecret))

modules/common/src/main/scala/docspell/common/Language.scala

@@ -123,11 +123,6 @@ object Language {
     val iso3 = "jpn"
   }
 
-  /*It's not an ISO value, but this needs to be unique and tesseract will need jpn_vert for it's scan from the config of /etc/docspell-joex/docspell-joex.conf.*/
-  case object JpnVert extends Language {
-    val iso2 = "ja_vert"
-    val iso3 = "jpn_vert"
-  }
   case object Hebrew extends Language {
     val iso2 = "he"
     val iso3 = "heb"
@@ -177,7 +172,6 @@ object Language {
       Romanian,
       Latvian,
       Japanese,
-      JpnVert,
       Hebrew,
       Lithuanian,
       Polish,

modules/common/src/main/scala/docspell/common/LenientUri.scala

@@ -78,11 +78,7 @@ case class LenientUri(
       .covary[F]
       .rethrow
       .flatMap(url =>
-        fs2.io.readInputStream(
+        fs2.io.readInputStream(Sync[F].delay(url.openStream()), chunkSize, true)
-          Sync[F].delay(url.openStream()),
-          chunkSize,
-          closeAfterUse = true
-        )
       )
 
   def readText[F[_]: Sync](chunkSize: Int): F[String] =
@@ -125,7 +121,7 @@ object LenientUri {
     val isRoot = true
     val isEmpty = false
     def /(seg: String): Path =
-      NonEmptyPath(NonEmptyList.of(seg), trailingSlash = false)
+      NonEmptyPath(NonEmptyList.of(seg), false)
     def asString = "/"
   }
   case object EmptyPath extends Path {
@@ -133,7 +129,7 @@ object LenientUri {
     val isRoot = false
     val isEmpty = true
     def /(seg: String): Path =
-      NonEmptyPath(NonEmptyList.of(seg), trailingSlash = false)
+      NonEmptyPath(NonEmptyList.of(seg), false)
     def asString = ""
   }
   case class NonEmptyPath(segs: NonEmptyList[String], trailingSlash: Boolean)

modules/common/src/main/scala/docspell/common/MimeType.scala

@@ -194,7 +194,7 @@ object MimeType {
           val csValueStart = in.substring(n + "charset=".length).trim
           val csName = csValueStart.indexOf(';') match {
             case -1 => unquote(csValueStart).trim
-            case n2 => unquote(csValueStart.substring(0, n2)).trim
+            case n  => unquote(csValueStart.substring(0, n)).trim
           }
           if (Charset.isSupported(csName)) Right((Some(Charset.forName(csName)), ""))
           else Right((None, ""))

modules/common/src/main/scala/docspell/common/SystemCommand.scala

@@ -0,0 +1,212 @@
+/*
+ * Copyright 2020 Eike K. & Contributors
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+package docspell.common
+
+import java.io.InputStream
+import java.lang.ProcessBuilder.Redirect
+import java.util.concurrent.TimeUnit
+
+import scala.jdk.CollectionConverters._
+
+import cats.effect._
+import cats.implicits._
+import fs2.io.file.Path
+import fs2.{Stream, io, text}
+
+import docspell.common.{exec => newExec}
+import docspell.logging.Logger
+
+// better use `SysCmd` and `SysExec`
+object SystemCommand {
+
+  final case class Config(
+      program: String,
+      args: Seq[String],
+      timeout: Duration,
+      env: Map[String, String] = Map.empty
+  ) {
+
+    def toSysCmd = newExec
+      .SysCmd(program, newExec.Args(args))
+      .withTimeout(timeout)
+      .addEnv(newExec.Env(env))
+
+    def mapArgs(f: String => String): Config =
+      Config(program, args.map(f), timeout)
+
+    def replace(repl: Map[String, String]): Config =
+      mapArgs(s =>
+        repl.foldLeft(s) { case (res, (k, v)) =>
+          res.replace(k, v)
+        }
+      )
+
+    def withEnv(key: String, value: String): Config =
+      copy(env = env.updated(key, value))
+
+    def addEnv(moreEnv: Map[String, String]): Config =
+      copy(env = env ++ moreEnv)
+
+    def appendArgs(extraArgs: Args): Config =
+      copy(args = args ++ extraArgs.args)
+
+    def appendArgs(extraArgs: Seq[String]): Config =
+      copy(args = args ++ extraArgs)
+
+    def toCmd: List[String] =
+      program :: args.toList
+
+    lazy val cmdString: String =
+      toCmd.mkString(" ")
+  }
+
+  final case class Args(args: Vector[String]) extends Iterable[String] {
+    override def iterator = args.iterator
+
+    def prepend(a: String): Args = Args(a +: args)
+
+    def prependWhen(flag: Boolean)(a: String): Args =
+      prependOption(Option.when(flag)(a))
+
+    def prependOption(value: Option[String]): Args =
+      value.map(prepend).getOrElse(this)
+
+    def append(a: String, as: String*): Args =
+      Args(args ++ (a +: as.toVector))
+
+    def appendOption(value: Option[String]): Args =
+      value.map(append(_)).getOrElse(this)
+
+    def appendOptionVal(first: String, second: Option[String]): Args =
+      second.map(b => append(first, b)).getOrElse(this)
+
+    def appendWhen(flag: Boolean)(a: String, as: String*): Args =
+      if (flag) append(a, as: _*) else this
+
+    def appendWhenNot(flag: Boolean)(a: String, as: String*): Args =
+      if (!flag) append(a, as: _*) else this
+
+    def append(p: Path): Args =
+      append(p.toString)
+
+    def append(as: Iterable[String]): Args =
+      Args(args ++ as.toVector)
+  }
+  object Args {
+    val empty: Args = Args()
+
+    def apply(as: String*): Args =
+      Args(as.toVector)
+  }
+
+  final case class Result(rc: Int, stdout: String, stderr: String)
+
+  def exec[F[_]: Sync](
+      cmd: Config,
+      logger: Logger[F],
+      wd: Option[Path] = None,
+      stdin: Stream[F, Byte] = Stream.empty
+  ): Stream[F, Result] =
+    startProcess(cmd, wd, logger, stdin) { proc =>
+      Stream.eval {
+        for {
+          _ <- writeToProcess(stdin, proc)
+          term <- Sync[F].blocking(proc.waitFor(cmd.timeout.seconds, TimeUnit.SECONDS))
+          _ <-
+            if (term)
+              logger.debug(s"Command `${cmd.cmdString}` finished: ${proc.exitValue}")
+            else
+              logger.warn(
+                s"Command `${cmd.cmdString}` did not finish in ${cmd.timeout.formatExact}!"
+              )
+          _ <- if (!term) timeoutError(proc, cmd) else Sync[F].pure(())
+          out <-
+            if (term) inputStreamToString(proc.getInputStream)
+            else Sync[F].pure("")
+          err <-
+            if (term) inputStreamToString(proc.getErrorStream)
+            else Sync[F].pure("")
+        } yield Result(proc.exitValue, out, err)
+      }
+    }
+
+  def execSuccess[F[_]: Sync](
+      cmd: Config,
+      logger: Logger[F],
+      wd: Option[Path] = None,
+      stdin: Stream[F, Byte] = Stream.empty
+  ): Stream[F, Result] =
+    exec(cmd, logger, wd, stdin).flatMap { r =>
+      if (r.rc != 0)
+        Stream.raiseError[F](
+          new Exception(
+            s"Command `${cmd.cmdString}` returned non-zero exit code ${r.rc}. Stderr: ${r.stderr}"
+          )
+        )
+      else Stream.emit(r)
+    }
+
+  private def startProcess[F[_]: Sync, A](
+      cmd: Config,
+      wd: Option[Path],
+      logger: Logger[F],
+      stdin: Stream[F, Byte]
+  )(
+      f: Process => Stream[F, A]
+  ): Stream[F, A] = {
+    val log = logger.debug(s"Running external command: ${cmd.cmdString}")
+    val hasStdin = stdin.take(1).compile.last.map(_.isDefined)
+    val proc = log *> hasStdin.flatMap(flag =>
+      Sync[F].blocking {
+        val pb = new ProcessBuilder(cmd.toCmd.asJava)
+          .redirectInput(if (flag) Redirect.PIPE else Redirect.INHERIT)
+          .redirectError(Redirect.PIPE)
+          .redirectOutput(Redirect.PIPE)
+
+        val pbEnv = pb.environment()
+        cmd.env.foreach { case (key, value) =>
+          pbEnv.put(key, value)
+        }
+        wd.map(_.toNioPath.toFile).foreach(pb.directory)
+        pb.start()
+      }
+    )
+    Stream
+      .bracket(proc)(p =>
+        logger.debug(s"Closing process: `${cmd.cmdString}`").map(_ => p.destroy())
+      )
+      .flatMap(f)
+  }
+
+  private def inputStreamToString[F[_]: Sync](in: InputStream): F[String] =
+    io.readInputStream(Sync[F].pure(in), 16 * 1024, closeAfterUse = false)
+      .through(text.utf8.decode)
+      .chunks
+      .map(_.toVector.mkString)
+      .fold1(_ + _)
+      .compile
+      .last
+      .map(_.getOrElse(""))
+
+  private def writeToProcess[F[_]: Sync](
+      data: Stream[F, Byte],
+      proc: Process
+  ): F[Unit] =
+    data
+      .through(io.writeOutputStream(Sync[F].blocking(proc.getOutputStream)))
+      .compile
+      .drain
+
+  private def timeoutError[F[_]: Sync](proc: Process, cmd: Config): F[Unit] =
+    Sync[F].blocking(proc.destroyForcibly()).attempt *> {
+      Sync[F].raiseError(
+        new Exception(
+          s"Command `${cmd.cmdString}` timed out (${cmd.timeout.formatExact})"
+        )
+      )
+    }
+}

modules/common/src/main/scala/docspell/common/UrlMatcher.scala

@@ -62,7 +62,7 @@ object UrlMatcher {
       // strip path to only match prefixes
       val mPath: LenientUri.Path =
         NonEmptyList.fromList(url.path.segments.take(pathSegmentCount)) match {
-          case Some(nel) => LenientUri.NonEmptyPath(nel, trailingSlash = false)
+          case Some(nel) => LenientUri.NonEmptyPath(nel, false)
           case None      => LenientUri.RootPath
         }
 

modules/common/src/main/scala/docspell/common/exec/Env.scala

@@ -17,9 +17,6 @@ case class Env(values: Map[String, String]) {
   def addAll(e: Env): Env =
     Env(values ++ e.values)
 
-  def modifyValue(f: String => String): Env =
-    Env(values.view.mapValues(f).toMap)
-
   def ++(e: Env) = addAll(e)
 
   def foreach(f: (String, String) => Unit): Unit =

modules/common/src/main/scala/docspell/common/exec/ExternalCommand.scala

@@ -1,89 +0,0 @@
-/*
- * Copyright 2020 Eike K. & Contributors
- *
- * SPDX-License-Identifier: AGPL-3.0-or-later
- */
-
-package docspell.common.exec
-
-import docspell.common.Duration
-import docspell.common.Ident
-import docspell.common.exec.Env
-import docspell.common.exec.ExternalCommand.ArgMapping
-import docspell.common.exec.SysCmd
-
-final case class ExternalCommand(
-    program: String,
-    args: Seq[String],
-    timeout: Duration,
-    env: Map[String, String] = Map.empty,
-    argMappings: Map[Ident, ArgMapping] = Map.empty
-) {
-  def withVars(vars: Map[String, String]): ExternalCommand.WithVars =
-    ExternalCommand.WithVars(this, vars)
-
-  import ExternalCommand.pattern
-
-  def resolve(vars: Map[String, String]): SysCmd = {
-    val replace = ExternalCommand.replaceString(vars) _
-    val resolvedArgMappings =
-      argMappings.view.mapValues(_.resolve(replace).firstMatch).toMap
-    val resolvedArgs = args.map(replace).flatMap { arg =>
-      resolvedArgMappings
-        .find(e => pattern(e._1.id) == arg)
-        .map(_._2)
-        .getOrElse(List(arg))
-    }
-
-    SysCmd(replace(program), resolvedArgs: _*)
-      .withTimeout(timeout)
-      .withEnv(_ => Env(env).modifyValue(replace))
-  }
-}
-
-object ExternalCommand {
-  private val openPattern = "{{"
-  private val closePattern = "}}"
-
-  private def pattern(s: String): String = s"${openPattern}${s}${closePattern}"
-
-  def apply(program: String, args: Seq[String], timeout: Duration): ExternalCommand =
-    ExternalCommand(program, args, timeout, Map.empty, Map.empty)
-
-  final case class ArgMapping(
-      value: String,
-      mappings: List[ArgMatch]
-  ) {
-    private[exec] def resolve(replace: String => String): ArgMapping =
-      ArgMapping(replace(value), mappings.map(_.resolve(replace)))
-
-    def firstMatch: List[String] =
-      mappings.find(am => value.matches(am.matches)).map(_.args).getOrElse(Nil)
-  }
-
-  final case class ArgMatch(
-      matches: String,
-      args: List[String]
-  ) {
-    private[exec] def resolve(replace: String => String): ArgMatch =
-      ArgMatch(replace(matches), args.map(replace))
-  }
-
-  private def replaceString(vars: Map[String, String])(in: String): String =
-    vars.foldLeft(in) { case (result, (name, value)) =>
-      val key = s"{{$name}}"
-      result.replace(key, value)
-    }
-
-  final case class WithVars(cmd: ExternalCommand, vars: Map[String, String]) {
-    def resolved: SysCmd = cmd.resolve(vars)
-    def append(more: (String, String)*): WithVars =
-      WithVars(cmd, vars ++ more.toMap)
-
-    def withVar(key: String, value: String): WithVars =
-      WithVars(cmd, vars.updated(key, value))
-
-    def withVarOption(key: String, value: Option[String]): WithVars =
-      value.map(withVar(key, _)).getOrElse(this)
-  }
-}

modules/common/src/main/scala/docspell/common/exec/SysExec.scala

@@ -38,20 +38,6 @@ trait SysExec[F[_]] {
 
   def waitFor(timeout: Option[Duration] = None): F[Int]
 
-  /** Uses `waitFor` and throws when return code is non-zero. Logs stderr and stdout while
-    * waiting.
-    */
-  def runToSuccess(logger: Logger[F], timeout: Option[Duration] = None)(implicit
-      F: Async[F]
-  ): F[Int]
-
-  /** Uses `waitFor` and throws when return code is non-zero. Logs stderr while waiting
-    * and collects stdout once finished successfully.
-    */
-  def runToSuccessStdout(logger: Logger[F], timeout: Option[Duration] = None)(implicit
-      F: Async[F]
-  ): F[String]
-
   /** Sends a signal to the process to terminate it immediately */
   def cancel: F[Unit]
 
@@ -89,12 +75,6 @@ object SysExec {
       proc <- startProcess(logger, cmd, workdir, stdin)
       fibers <- Resource.eval(Ref.of[F, List[F[Unit]]](Nil))
     } yield new SysExec[F] {
-      private lazy val basicName: String =
-        cmd.program.lastIndexOf(java.io.File.separatorChar.toInt) match {
-          case n if n > 0 => cmd.program.drop(n + 1)
-          case _          => cmd.program.takeRight(16)
-        }
-
       def stdout: Stream[F, Byte] =
         fs2.io.readInputStream(
           Sync[F].blocking(proc.getInputStream),
@@ -127,39 +107,6 @@ object SysExec {
             )
       }
 
-      def runToSuccess(logger: Logger[F], timeout: Option[Duration])(implicit
-          F: Async[F]
-      ): F[Int] =
-        logOutputs(logger, basicName).use(_.waitFor(timeout).flatMap {
-          case rc if rc == 0 => Sync[F].pure(0)
-          case rc =>
-            Sync[F].raiseError(
-              new Exception(s"Command `${cmd.program}` returned non-zero exit code ${rc}")
-            )
-        })
-
-      def runToSuccessStdout(logger: Logger[F], timeout: Option[Duration])(implicit
-          F: Async[F]
-      ): F[String] =
-        F.background(
-          stderrLines
-            .through(line => Stream.eval(logger.debug(s"[$basicName (err)]: $line")))
-            .compile
-            .drain
-        ).use { f1 =>
-          waitFor(timeout)
-            .flatMap {
-              case rc if rc == 0 => stdout.through(fs2.text.utf8.decode).compile.string
-              case rc =>
-                Sync[F].raiseError[String](
-                  new Exception(
-                    s"Command `${cmd.program}` returned non-zero exit code ${rc}"
-                  )
-                )
-            }
-            .flatTap(_ => f1)
-        }
-
       def consumeOutputs(out: Pipe[F, String, Unit], err: Pipe[F, String, Unit])(implicit
           F: Async[F]
       ): Resource[F, SysExec[F]] =

modules/common/src/main/scala/docspell/common/util/Directory.scala

@@ -6,7 +6,6 @@
 
 package docspell.common.util
 
-import cats.data.OptionT
 import cats.effect._
 import cats.syntax.all._
 import cats.{Applicative, Monad}
@@ -27,10 +26,10 @@ object Directory {
     (dir :: dirs.toList).traverse_(Files[F].createDirectories(_))
 
   def nonEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] =
-    OptionT
+    List(
-      .whenM(Files[F].isDirectory(dir))(Files[F].list(dir).take(1).compile.toList)
+      Files[F].isDirectory(dir),
-      .map(_.nonEmpty)
+      Files[F].list(dir).take(1).compile.last.map(_.isDefined)
-      .isDefined
+    ).sequence.map(_.forall(identity))
 
   def isEmpty[F[_]: Files: Sync](dir: Path): F[Boolean] =
     nonEmpty(dir).map(b => !b)

modules/common/src/test/scala/docspell/common/exec/ExternalCommandTest.scala

@@ -1,74 +0,0 @@
-/*
- * Copyright 2020 Eike K. & Contributors
- *
- * SPDX-License-Identifier: AGPL-3.0-or-later
- */
-
-package docspell.common.exec
-
-import docspell.common.Duration
-import docspell.common.Ident
-import docspell.common.exec.Args
-import docspell.common.exec.Env
-import docspell.common.exec.ExternalCommand._
-import docspell.common.exec.SysCmd
-
-import munit.FunSuite
-
-class ExternalCommandTest extends FunSuite {
-
-  test("resolve") {
-    val cmd = ExternalCommand(
-      program = "tesseract",
-      args = "{{infile}}" :: "{{lang-spec}}" :: "out" :: "pdf" :: "txt" :: Nil,
-      timeout = Duration.minutes(5),
-      env = Map.empty,
-      argMappings = Map(
-        Ident.unsafe("lang-spec") -> ArgMapping(
-          value = "{{lang}}",
-          mappings = List(
-            ArgMatch(
-              matches = "jpn_vert",
-              args = List("-l", "jpn_vert", "-c", "preserve_interword_spaces=1")
-            ),
-            ArgMatch(
-              matches = ".*",
-              args = List("-l", "{{lang}}")
-            )
-          )
-        )
-      )
-    )
-
-    val varsDe = Map("lang" -> "de", "encoding" -> "UTF_8", "infile" -> "text.jpg")
-    assertEquals(
-      cmd.resolve(varsDe),
-      SysCmd(
-        "tesseract",
-        Args.of("text.jpg", "-l", "de", "out", "pdf", "txt"),
-        Env.empty,
-        Duration.minutes(5)
-      )
-    )
-
-    val varsJpnVert = varsDe.updated("lang", "jpn_vert")
-    assertEquals(
-      cmd.resolve(varsJpnVert),
-      SysCmd(
-        "tesseract",
-        Args.of(
-          "text.jpg",
-          "-l",
-          "jpn_vert",
-          "-c",
-          "preserve_interword_spaces=1",
-          "out",
-          "pdf",
-          "txt"
-        ),
-        Env.empty,
-        Duration.minutes(5)
-      )
-    )
-  }
-}

modules/common/src/test/scala/docspell/common/util/DirectoryTest.scala

@@ -16,7 +16,7 @@ import munit.CatsEffectSuite
 
 class DirectoryTest extends CatsEffectSuite with TestLoggingConfig {
   val logger = docspell.logging.getLogger[IO]
-  val tempDir = ResourceFunFixture(
+  val tempDir = ResourceFixture(
     Files[IO].tempDirectory(Path("target").some, "directory-test-", None)
   )
 

modules/convert/src/main/scala/docspell/convert/extern/ExternConv.scala

@@ -11,8 +11,7 @@ import cats.implicits._
 import fs2.io.file.{Files, Path}
 import fs2.{Pipe, Stream}
 
-import docspell.common.exec.ExternalCommand
+import docspell.common._
-import docspell.common.exec.SysExec
 import docspell.common.util.File
 import docspell.convert.ConversionResult
 import docspell.convert.ConversionResult.{Handler, successPdf, successPdfTxt}
@@ -22,11 +21,11 @@ private[extern] object ExternConv {
 
   def toPDF[F[_]: Async: Files, A](
       name: String,
-      cmdCfg: ExternalCommand.WithVars,
+      cmdCfg: SystemCommand.Config,
       wd: Path,
       useStdin: Boolean,
       logger: Logger[F],
-      reader: (Path, Int) => F[ConversionResult[F]]
+      reader: (Path, SystemCommand.Result) => F[ConversionResult[F]]
   )(in: Stream[F, Byte], handler: Handler[F, A]): F[A] =
     Stream
       .resource(File.withTempDir[F](wd, s"docspell-$name"))
@@ -34,21 +33,32 @@ private[extern] object ExternConv {
         val inFile = dir.resolve("infile").absolute.normalize
         val out = dir.resolve("out.pdf").absolute.normalize
         val sysCfg =
-          cmdCfg
+          cmdCfg.replace(
-            .withVar("outfile", out.toString)
+            Map(
-            .withVarOption("infile", Option.when(!useStdin)(inFile.toString))
+              "{{outfile}}" -> out.toString
-            .resolved
+            ) ++
+              (if (!useStdin) Map("{{infile}}" -> inFile.toString)
+               else Map.empty)
+          )
 
         val createInput: Pipe[F, Byte, Unit] =
           if (useStdin) _ => Stream.emit(())
           else storeDataToFile(name, logger, inFile)
 
-        in.through(createInput).evalMap { _ =>
+        in.through(createInput).flatMap { _ =>
-          SysExec(sysCfg, logger, Some(dir), Option.when(useStdin)(in))
+          SystemCommand
-            .flatMap(_.logOutputs(logger, name))
+            .exec[F](
-            .use { proc =>
+              sysCfg,
-              proc.waitFor().flatMap(rc => reader(out, rc).flatMap(handler.run))
+              logger,
-            }
+              Some(dir),
+              if (useStdin) in
+              else Stream.empty
+            )
+            .evalMap(result =>
+              logResult(name, result, logger)
+                .flatMap(_ => reader(out, result))
+                .flatMap(handler.run)
+            )
         }
       }
       .compile
@@ -64,9 +74,9 @@ private[extern] object ExternConv {
   def readResult[F[_]: Async: Files](
       chunkSize: Int,
       logger: Logger[F]
-  )(out: Path, result: Int): F[ConversionResult[F]] =
+  )(out: Path, result: SystemCommand.Result): F[ConversionResult[F]] =
     File.existsNonEmpty[F](out).flatMap {
-      case true if result == 0 =>
+      case true if result.rc == 0 =>
         val outTxt = out.resolveSibling(out.fileName.toString + ".txt")
         File.existsNonEmpty[F](outTxt).flatMap {
           case true =>
@@ -78,13 +88,13 @@ private[extern] object ExternConv {
             successPdf(File.readAll(out, chunkSize)).pure[F]
         }
       case true =>
-        logger.warn(s"Command not successful (rc=${result}), but file exists.") *>
+        logger.warn(s"Command not successful (rc=${result.rc}), but file exists.") *>
           successPdf(File.readAll(out, chunkSize)).pure[F]
 
       case false =>
         ConversionResult
           .failure[F](
-            new Exception(s"Command result=${result}. No output file found.")
+            new Exception(s"Command result=${result.rc}. No output file found.")
           )
           .pure[F]
     }
@@ -93,25 +103,25 @@ private[extern] object ExternConv {
       outPrefix: String,
       chunkSize: Int,
       logger: Logger[F]
-  )(out: Path, result: Int): F[ConversionResult[F]] = {
+  )(out: Path, result: SystemCommand.Result): F[ConversionResult[F]] = {
     val outPdf = out.resolveSibling(s"$outPrefix.pdf")
     File.existsNonEmpty[F](outPdf).flatMap {
       case true =>
         val outTxt = out.resolveSibling(s"$outPrefix.txt")
         File.exists(outTxt).flatMap { txtExists =>
           val pdfData = File.readAll(out, chunkSize)
-          if (result == 0)
+          if (result.rc == 0)
             if (txtExists) successPdfTxt(pdfData, File.readText(outTxt)).pure[F]
             else successPdf(pdfData).pure[F]
           else
-            logger.warn(s"Command not successful (rc=${result}), but file exists.") *>
+            logger.warn(s"Command not successful (rc=${result.rc}), but file exists.") *>
               successPdf(pdfData).pure[F]
         }
 
       case false =>
         ConversionResult
           .failure[F](
-            new Exception(s"Command result=${result}. No output file found.")
+            new Exception(s"Command result=${result.rc}. No output file found.")
           )
           .pure[F]
     }
@@ -128,6 +138,14 @@ private[extern] object ExternConv {
         .drain ++
         Stream.eval(storeFile(in, inFile))
 
+  private def logResult[F[_]: Sync](
+      name: String,
+      result: SystemCommand.Result,
+      logger: Logger[F]
+  ): F[Unit] =
+    logger.debug(s"$name stdout: ${result.stdout}") *>
+      logger.debug(s"$name stderr: ${result.stderr}")
+
   private def storeFile[F[_]: Async: Files](
       in: Stream[F, Byte],
       target: Path

modules/convert/src/main/scala/docspell/convert/extern/OcrMyPdf.scala

@@ -24,16 +24,14 @@ object OcrMyPdf {
       logger: Logger[F]
   )(in: Stream[F, Byte], handler: Handler[F, A]): F[A] =
     if (cfg.enabled) {
-      val reader: (Path, Int) => F[ConversionResult[F]] =
+      val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
         ExternConv.readResult[F](chunkSize, logger)
 
-      val cmd = cfg.command.withVars(Map("lang" -> lang.iso3))
-
       ExternConv.toPDF[F, A](
         "ocrmypdf",
-        cmd,
+        cfg.command.replace(Map("{{lang}}" -> lang.iso3)),
         cfg.workingDir,
-        useStdin = false,
+        false,
         logger,
         reader
       )(in, handler)

modules/convert/src/main/scala/docspell/convert/extern/OcrMyPdfConfig.scala

@@ -8,10 +8,10 @@ package docspell.convert.extern
 
 import fs2.io.file.Path
 
-import docspell.common.exec.ExternalCommand
+import docspell.common.SystemCommand
 
 case class OcrMyPdfConfig(
     enabled: Boolean,
-    command: ExternalCommand,
+    command: SystemCommand.Config,
     workingDir: Path
 )

modules/convert/src/main/scala/docspell/convert/extern/Tesseract.scala

@@ -24,18 +24,17 @@ object Tesseract {
       logger: Logger[F]
   )(in: Stream[F, Byte], handler: Handler[F, A]): F[A] = {
     val outBase = cfg.command.args.tail.headOption.getOrElse("out")
-    val reader: (Path, Int) => F[ConversionResult[F]] =
+    val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
       ExternConv.readResultTesseract[F](outBase, chunkSize, logger)
 
-    val cmd = cfg.command.withVars(Map("lang" -> lang.iso3))
-
     ExternConv.toPDF[F, A](
       "tesseract",
-      cmd,
+      cfg.command.replace(Map("{{lang}}" -> lang.iso3)),
       cfg.workingDir,
-      useStdin = false,
+      false,
       logger,
       reader
     )(in, handler)
   }
+
 }

modules/convert/src/main/scala/docspell/convert/extern/TesseractConfig.scala

@@ -8,6 +8,6 @@ package docspell.convert.extern
 
 import fs2.io.file.Path
 
-import docspell.common.exec.ExternalCommand
+import docspell.common.SystemCommand
 
-case class TesseractConfig(command: ExternalCommand, workingDir: Path)
+case class TesseractConfig(command: SystemCommand.Config, workingDir: Path)

modules/convert/src/main/scala/docspell/convert/extern/Unoconv.scala

@@ -10,6 +10,7 @@ import cats.effect._
 import fs2.Stream
 import fs2.io.file.{Files, Path}
 
+import docspell.common._
 import docspell.convert.ConversionResult
 import docspell.convert.ConversionResult.Handler
 import docspell.logging.Logger
@@ -21,15 +22,14 @@ object Unoconv {
       chunkSize: Int,
       logger: Logger[F]
   )(in: Stream[F, Byte], handler: Handler[F, A]): F[A] = {
-    val reader: (Path, Int) => F[ConversionResult[F]] =
+    val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
       ExternConv.readResult[F](chunkSize, logger)
-    val cmd = cfg.command.withVars(Map.empty)
 
     ExternConv.toPDF[F, A](
       "unoconv",
-      cmd,
+      cfg.command,
       cfg.workingDir,
-      useStdin = false,
+      false,
       logger,
       reader
     )(
@@ -37,4 +37,5 @@ object Unoconv {
       handler
     )
   }
+
 }

modules/convert/src/main/scala/docspell/convert/extern/UnoconvConfig.scala

@@ -8,6 +8,6 @@ package docspell.convert.extern
 
 import fs2.io.file.Path
 
-import docspell.common.exec.ExternalCommand
+import docspell.common.SystemCommand
 
-case class UnoconvConfig(command: ExternalCommand, workingDir: Path)
+case class UnoconvConfig(command: SystemCommand.Config, workingDir: Path)

modules/convert/src/main/scala/docspell/convert/extern/Weasyprint.scala

@@ -27,10 +27,10 @@ object Weasyprint {
       sanitizeHtml: SanitizeHtml,
       logger: Logger[F]
   )(in: Stream[F, Byte], handler: Handler[F, A]): F[A] = {
-    val reader: (Path, Int) => F[ConversionResult[F]] =
+    val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
       ExternConv.readResult[F](chunkSize, logger)
 
-    val cmdCfg = cfg.command.withVars(Map("encoding" -> charset.name()))
+    val cmdCfg = cfg.command.replace(Map("{{encoding}}" -> charset.name()))
 
     // html sanitize should (among other) remove links to invalid
     // protocols like cid: which is not supported by further
@@ -51,4 +51,5 @@ object Weasyprint {
         handler
       )
   }
+
 }

modules/convert/src/main/scala/docspell/convert/extern/WeasyprintConfig.scala

@@ -8,6 +8,6 @@ package docspell.convert.extern
 
 import fs2.io.file.Path
 
-import docspell.common.exec.ExternalCommand
+import docspell.common.SystemCommand
 
-case class WeasyprintConfig(command: ExternalCommand, workingDir: Path)
+case class WeasyprintConfig(command: SystemCommand.Config, workingDir: Path)

modules/convert/src/main/scala/docspell/convert/extern/WkHtmlPdf.scala

@@ -27,10 +27,10 @@ object WkHtmlPdf {
       sanitizeHtml: SanitizeHtml,
       logger: Logger[F]
   )(in: Stream[F, Byte], handler: Handler[F, A]): F[A] = {
-    val reader: (Path, Int) => F[ConversionResult[F]] =
+    val reader: (Path, SystemCommand.Result) => F[ConversionResult[F]] =
       ExternConv.readResult[F](chunkSize, logger)
 
-    val cmdCfg = cfg.command.withVars(Map("encoding" -> charset.name()))
+    val cmdCfg = cfg.command.replace(Map("{{encoding}}" -> charset.name()))
 
     // html sanitize should (among other) remove links to invalid
     // protocols like cid: which is not supported by further
@@ -58,4 +58,5 @@ object WkHtmlPdf {
         handler
       )
   }
+
 }

modules/convert/src/main/scala/docspell/convert/extern/WkHtmlPdfConfig.scala

@@ -8,6 +8,6 @@ package docspell.convert.extern
 
 import fs2.io.file.Path
 
-import docspell.common.exec.ExternalCommand
+import docspell.common.SystemCommand
 
-case class WkHtmlPdfConfig(command: ExternalCommand, workingDir: Path)
+case class WkHtmlPdfConfig(command: SystemCommand.Config, workingDir: Path)

modules/convert/src/test/scala/docspell/convert/ConversionTest.scala

@@ -15,7 +15,6 @@ import cats.implicits._
 import fs2.Stream
 
 import docspell.common._
-import docspell.common.exec._
 import docspell.common.util.File
 import docspell.convert.ConversionResult.Handler
 import docspell.convert.ConvertConfig.HtmlConverter
@@ -37,7 +36,7 @@ class ConversionTest extends FunSuite with FileChecks with TestLoggingConfig {
     3000 * 3000,
     MarkdownConfig("body { padding: 2em 5em; }"),
     WkHtmlPdfConfig(
-      ExternalCommand(
+      SystemCommand.Config(
         "wkhtmltopdf",
         Seq("-s", "A4", "--encoding", "UTF-8", "-", "{{outfile}}"),
         Duration.seconds(20)
@@ -45,7 +44,7 @@ class ConversionTest extends FunSuite with FileChecks with TestLoggingConfig {
       target
     ),
     WeasyprintConfig(
-      ExternalCommand(
+      SystemCommand.Config(
         "weasyprint",
         Seq("--encoding", "UTF-8", "-", "{{outfile}}"),
         Duration.seconds(20)
@@ -54,7 +53,7 @@ class ConversionTest extends FunSuite with FileChecks with TestLoggingConfig {
     ),
     HtmlConverter.Wkhtmltopdf,
     TesseractConfig(
-      ExternalCommand(
+      SystemCommand.Config(
         "tesseract",
         Seq("{{infile}}", "out", "-l", "deu", "pdf", "txt"),
         Duration.seconds(20)
@@ -62,7 +61,7 @@ class ConversionTest extends FunSuite with FileChecks with TestLoggingConfig {
       target
     ),
     UnoconvConfig(
-      ExternalCommand(
+      SystemCommand.Config(
         "unoconv",
         Seq("-f", "pdf", "-o", "{{outfile}}", "{{infile}}"),
         Duration.seconds(20)
@@ -70,8 +69,8 @@ class ConversionTest extends FunSuite with FileChecks with TestLoggingConfig {
       target
     ),
     OcrMyPdfConfig(
-      enabled = true,
+      true,
-      ExternalCommand(
+      SystemCommand.Config(
         "ocrmypdf",
         Seq(
           "-l",
@@ -87,7 +86,7 @@ class ConversionTest extends FunSuite with FileChecks with TestLoggingConfig {
       ),
       target
     ),
-    ConvertConfig.DecryptPdf(enabled = true, Nil)
+    ConvertConfig.DecryptPdf(true, Nil)
   )
 
   val conversion =

modules/convert/src/test/scala/docspell/convert/extern/ExternConvTest.scala

@@ -14,7 +14,6 @@ import cats.effect.unsafe.implicits.global
 import fs2.io.file.Path
 
 import docspell.common._
-import docspell.common.exec._
 import docspell.common.util.File
 import docspell.convert._
 import docspell.files.ExampleFiles
@@ -28,7 +27,7 @@ class ExternConvTest extends FunSuite with FileChecks with TestLoggingConfig {
   val target = File.path(Paths.get("target"))
 
   test("convert html to pdf") {
-    val cfg = ExternalCommand(
+    val cfg = SystemCommand.Config(
       "wkhtmltopdf",
       Seq("-s", "A4", "--encoding", "UTF-8", "-", "{{outfile}}"),
       Duration.seconds(20)
@@ -54,7 +53,7 @@ class ExternConvTest extends FunSuite with FileChecks with TestLoggingConfig {
   }
 
   test("convert office to pdf") {
-    val cfg = ExternalCommand(
+    val cfg = SystemCommand.Config(
       "unoconv",
       Seq("-f", "pdf", "-o", "{{outfile}}", "{{infile}}"),
       Duration.seconds(20)
@@ -81,7 +80,7 @@ class ExternConvTest extends FunSuite with FileChecks with TestLoggingConfig {
   }
 
   test("convert image to pdf") {
-    val cfg = ExternalCommand(
+    val cfg = SystemCommand.Config(
       "tesseract",
       Seq("{{infile}}", "out", "-l", "deu", "pdf", "txt"),
       Duration.seconds(20)
@@ -106,4 +105,5 @@ class ExternConvTest extends FunSuite with FileChecks with TestLoggingConfig {
       )
       .unsafeRunSync()
   }
+
 }

modules/extract/src/main/scala/docspell/extract/ocr/Ocr.scala

@@ -10,8 +10,7 @@ import cats.effect._
 import fs2.Stream
 import fs2.io.file.{Files, Path}
 
-import docspell.common.exec.ExternalCommand
+import docspell.common._
-import docspell.common.exec.SysExec
 import docspell.common.util.File
 import docspell.logging.Logger
 
@@ -78,17 +77,14 @@ object Ocr {
       else cfg.ghostscript.command.args
     val cmd = cfg.ghostscript.command
       .copy(args = xargs)
-      .withVars(
+      .replace(
         Map(
-          "infile" -> "-",
+          "{{infile}}" -> "-",
-          "outfile" -> "%d.tif"
+          "{{outfile}}" -> "%d.tif"
         )
       )
-      .resolved
+    SystemCommand
-
+      .execSuccess(cmd, logger, wd = Some(wd), stdin = pdf)
-    Stream
-      .resource(SysExec(cmd, logger, Some(wd), Some(pdf)))
-      .evalMap(_.runToSuccess(logger))
       .flatMap(_ => File.listFiles(pathEndsWith(".tif"), wd))
   }
 
@@ -97,22 +93,18 @@ object Ocr {
     */
   private[extract] def runGhostscriptFile[F[_]: Async: Files](
       pdf: Path,
-      ghostscript: ExternalCommand,
+      ghostscript: SystemCommand.Config,
       wd: Path,
       logger: Logger[F]
   ): Stream[F, Path] = {
-    val cmd = ghostscript
+    val cmd = ghostscript.replace(
-      .withVars(
+      Map(
-        Map(
+        "{{infile}}" -> pdf.absolute.toString,
-          "infile" -> pdf.absolute.toString,
+        "{{outfile}}" -> "%d.tif"
-          "outfile" -> "%d.tif"
-        )
       )
-      .resolved
+    )
-
+    SystemCommand
-    Stream
+      .execSuccess[F](cmd, logger, wd = Some(wd))
-      .resource(SysExec(cmd, logger, Some(wd)))
-      .evalMap(_.runToSuccess(logger))
       .flatMap(_ => File.listFiles(pathEndsWith(".tif"), wd))
   }
 
@@ -124,23 +116,19 @@ object Ocr {
     */
   private[extract] def runUnpaperFile[F[_]: Async](
       img: Path,
-      unpaper: ExternalCommand,
+      unpaper: SystemCommand.Config,
       wd: Option[Path],
       logger: Logger[F]
   ): Stream[F, Path] = {
     val targetFile = img.resolveSibling("u-" + img.fileName.toString).absolute
-    val cmd = unpaper
+    val cmd = unpaper.replace(
-      .withVars(
+      Map(
-        Map(
+        "{{infile}}" -> img.absolute.toString,
-          "infile" -> img.absolute.toString,
+        "{{outfile}}" -> targetFile.toString
-          "outfile" -> targetFile.toString
-        )
       )
-      .resolved
+    )
-
+    SystemCommand
-    Stream
+      .execSuccess[F](cmd, logger, wd = wd)
-      .resource(SysExec(cmd, logger, wd))
-      .evalMap(_.runToSuccess(logger))
       .map(_ => targetFile)
       .handleErrorWith { th =>
         logger
@@ -162,14 +150,12 @@ object Ocr {
     // so use the parent as working dir
     runUnpaperFile(img, config.unpaper.command, img.parent, logger).flatMap { uimg =>
       val cmd = config.tesseract.command
-        .withVars(
+        .replace(
-          Map("file" -> uimg.fileName.toString, "lang" -> fixLanguage(lang))
+          Map("{{file}}" -> uimg.fileName.toString, "{{lang}}" -> fixLanguage(lang))
         )
-        .resolved
+      SystemCommand
-
+        .execSuccess[F](cmd, logger, wd = uimg.parent)
-      Stream
+        .map(_.stdout)
-        .resource(SysExec(cmd, logger, uimg.parent))
-        .evalMap(_.runToSuccessStdout(logger))
     }
 
   /** Run tesseract on the given image file and return the extracted text. */
@@ -180,12 +166,8 @@ object Ocr {
       config: OcrConfig
   ): Stream[F, String] = {
     val cmd = config.tesseract.command
-      .withVars(Map("file" -> "stdin", "lang" -> fixLanguage(lang)))
+      .replace(Map("{{file}}" -> "stdin", "{{lang}}" -> fixLanguage(lang)))
-      .resolved
+    SystemCommand.execSuccess(cmd, logger, stdin = img).map(_.stdout)
-
-    Stream
-      .resource(SysExec(cmd, logger, None, Some(img)))
-      .evalMap(_.runToSuccessStdout(logger))
   }
 
   private def fixLanguage(lang: String): String =

modules/extract/src/main/scala/docspell/extract/ocr/OcrConfig.scala

@@ -6,9 +6,12 @@
 
 package docspell.extract.ocr
 
+import java.nio.file.Paths
+
 import fs2.io.file.Path
 
-import docspell.common.exec.ExternalCommand
+import docspell.common._
+import docspell.common.util.File
 
 case class OcrConfig(
     maxImageSize: Int,
@@ -22,10 +25,43 @@ object OcrConfig {
 
   case class PageRange(begin: Int)
 
-  case class Ghostscript(command: ExternalCommand, workingDir: Path)
+  case class Ghostscript(command: SystemCommand.Config, workingDir: Path)
 
-  case class Tesseract(command: ExternalCommand)
+  case class Tesseract(command: SystemCommand.Config)
 
-  case class Unpaper(command: ExternalCommand)
+  case class Unpaper(command: SystemCommand.Config)
 
+  val default = OcrConfig(
+    maxImageSize = 3000 * 3000,
+    pageRange = PageRange(10),
+    ghostscript = Ghostscript(
+      SystemCommand.Config(
+        "gs",
+        Seq(
+          "-dNOPAUSE",
+          "-dBATCH",
+          "-dSAFER",
+          "-sDEVICE=tiffscaled8",
+          "-sOutputFile={{outfile}}",
+          "{{infile}}"
+        ),
+        Duration.seconds(30)
+      ),
+      File.path(
+        Paths.get(System.getProperty("java.io.tmpdir")).resolve("docspell-extraction")
+      )
+    ),
+    unpaper = Unpaper(
+      SystemCommand
+        .Config("unpaper", Seq("{{infile}}", "{{outfile}}"), Duration.seconds(30))
+    ),
+    tesseract = Tesseract(
+      SystemCommand
+        .Config(
+          "tesseract",
+          Seq("{{file}}", "stdout", "-l", "{{lang}}"),
+          Duration.minutes(1)
+        )
+    )
+  )
 }

modules/extract/src/test/scala/docspell/extract/ocr/TextExtractionSuite.scala

@@ -6,14 +6,9 @@
 
 package docspell.extract.ocr
 
-import java.nio.file.Paths
-
 import cats.effect.IO
 import cats.effect.unsafe.implicits.global
 
-import docspell.common.Duration
-import docspell.common.exec.ExternalCommand
-import docspell.common.util.File
 import docspell.files.TestFiles
 import docspell.logging.TestLoggingConfig
 
@@ -26,7 +21,7 @@ class TextExtractionSuite extends FunSuite with TestLoggingConfig {
 
   test("extract english pdf".ignore) {
     val text = TextExtract
-      .extract[IO](letterSourceEN, logger, "eng", TextExtractionSuite.defaultConfig)
+      .extract[IO](letterSourceEN, logger, "eng", OcrConfig.default)
       .compile
       .lastOrError
       .unsafeRunSync()
@@ -36,7 +31,7 @@ class TextExtractionSuite extends FunSuite with TestLoggingConfig {
   test("extract german pdf".ignore) {
     val expect = TestFiles.letterDEText
     val extract = TextExtract
-      .extract[IO](letterSourceDE, logger, "deu", TextExtractionSuite.defaultConfig)
+      .extract[IO](letterSourceDE, logger, "deu", OcrConfig.default)
       .compile
       .lastOrError
       .unsafeRunSync()
@@ -44,37 +39,3 @@ class TextExtractionSuite extends FunSuite with TestLoggingConfig {
     assertEquals(extract.value, expect)
   }
 }
-
-object TextExtractionSuite {
-  val defaultConfig = OcrConfig(
-    maxImageSize = 3000 * 3000,
-    pageRange = OcrConfig.PageRange(10),
-    ghostscript = OcrConfig.Ghostscript(
-      ExternalCommand(
-        "gs",
-        Seq(
-          "-dNOPAUSE",
-          "-dBATCH",
-          "-dSAFER",
-          "-sDEVICE=tiffscaled8",
-          "-sOutputFile={{outfile}}",
-          "{{infile}}"
-        ),
-        Duration.seconds(30)
-      ),
-      File.path(
-        Paths.get(System.getProperty("java.io.tmpdir")).resolve("docspell-extraction")
-      )
-    ),
-    unpaper = OcrConfig.Unpaper(
-      ExternalCommand("unpaper", Seq("{{infile}}", "{{outfile}}"), Duration.seconds(30))
-    ),
-    tesseract = OcrConfig.Tesseract(
-      ExternalCommand(
-        "tesseract",
-        Seq("{{file}}", "stdout", "-l", "{{lang}}"),
-        Duration.minutes(1)
-      )
-    )
-  )
-}

modules/files/src/test/scala/docspell/files/ZipTest.scala

@@ -19,7 +19,7 @@ import munit._
 
 class ZipTest extends CatsEffectSuite with TestLoggingConfig {
   val logger = docspell.logging.getLogger[IO]
-  val tempDir = ResourceFunFixture(
+  val tempDir = ResourceFixture(
     Files[IO].tempDirectory(Path("target").some, "zip-test-", None)
   )
 

modules/fts-psql/src/main/scala/docspell/ftspsql/FtsRepository.scala

@@ -201,7 +201,6 @@ object FtsRepository extends DoobieMeta {
       case Language.Czech      => "simple"
       case Language.Latvian    => "simple"
       case Language.Japanese   => "simple"
-      case Language.JpnVert    => "simple"
       case Language.Hebrew     => "simple"
       case Language.Lithuanian => "simple"
       case Language.Polish     => "simple"

modules/fts-solr/src/main/scala/docspell/ftssolr/SolrMigration.scala

@@ -45,7 +45,7 @@ object SolrMigration {
         description,
         FtsMigration.Result.reIndexAll.pure[F]
       ),
-      dataChangeOnly = true
+      true
     )
 
   def indexAll[F[_]: Applicative](
@@ -59,7 +59,7 @@ object SolrMigration {
         description,
         FtsMigration.Result.indexAll.pure[F]
       ),
-      dataChangeOnly = true
+      true
     )
 
   def apply[F[_]: Functor](
@@ -74,6 +74,6 @@ object SolrMigration {
         description,
         task.map(_ => FtsMigration.Result.workDone)
       ),
-      dataChangeOnly = false
+      false
     )
 }

modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala

@@ -299,22 +299,14 @@ object SolrSetup {
       Map("add-field" -> body.asJson).asJson
 
     def string(field: Field): AddField =
-      AddField(field, "string", stored = true, indexed = true, multiValued = false)
+      AddField(field, "string", true, true, false)
 
     def textGeneral(field: Field): AddField =
-      AddField(field, "text_general", stored = true, indexed = true, multiValued = false)
+      AddField(field, "text_general", true, true, false)
 
     def textLang(field: Field, lang: Language): AddField =
-      if (lang == Language.Czech)
+      if (lang == Language.Czech) AddField(field, s"text_cz", true, true, false)
-        AddField(field, s"text_cz", stored = true, indexed = true, multiValued = false)
+      else AddField(field, s"text_${lang.iso2}", true, true, false)
-      else
-        AddField(
-          field,
-          s"text_${lang.iso2}",
-          stored = true,
-          indexed = true,
-          multiValued = false
-        )
   }
 
   case class DeleteField(name: Field)

modules/joex/src/main/resources/reference.conf

@@ -595,30 +595,11 @@ Docpell Update Check
     tesseract = {
       command = {
         program = "tesseract"
-        # Custom Language Mappings Below
-        # Japanese Vertical Mapping
-        arg-mappings = {
-          "tesseract_lang" = {
-            value = "{{lang}}"
-            mappings = [
-              {
-                matches = "jpn_vert"
-                args = [ "-l", "jpn_vert", "-c", "preserve_interword_spaces=1" ]
-              },
-            # Start Other Custom Language Mappings Here
-            # Default Mapping Below
-              {
-                matches = ".*"
-                args = [ "-l", "{{lang}}" ]
-              }
-            ]
-          }
-        }
-        # Default arguments for all processing go below.
         args = [
           "{{infile}}",
           "out",
-          "{{tesseract_lang}}",
+          "-l",
+          "{{lang}}",
           "pdf",
           "txt"
         ]
@@ -670,34 +651,8 @@ Docpell Update Check
       enabled = true
       command = {
         program = "ocrmypdf"
-        # Custom argument mappings for this program.
-        arg-mappings = {
-          "ocr_lang" = {
-            value = "{{lang}}"
-            # Custom Language Mappings Below
-            # Japanese Vertical Mapping
-            mappings = [
-              {
-                matches = "jpn_vert"
-                args = [ "-l", "jpn_vert", "--pdf-renderer", "sandwich", "--tesseract-pagesegmode", "5", "--output-type", "pdf" ]
-              },
-            # Japanese Mapping for OCR Optimization
-              {
-                matches = "jpn"
-                args = [ "-l", "jpn", "--output-type", "pdf" ]
-              },
-            # Start Other Custom Language Mappings Here
-            # Default Mapping Below
-              {
-                matches = ".*"
-                args = [ "-l", "{{lang}}" ]
-              }
-            ]
-          }
-        }
-        # Default arguments for all processing go below.
         args = [
-          "{{ocr_lang}}",
+          "-l", "{{lang}}",
           "--skip-text",
           "--deskew",
           "-j", "1",

modules/joex/src/main/scala/docspell/joex/emptytrash/EmptyTrashTask.scala

@@ -30,7 +30,7 @@ object EmptyTrashTask {
     UserTask(
       args.periodicTaskId,
       EmptyTrashArgs.taskName,
-      enabled = true,
+      true,
       ce,
       None,
       args

modules/joex/src/main/scala/docspell/joex/filecopy/FileCopyTask.scala

@@ -29,23 +29,23 @@ object FileCopyTask {
   case class CopyResult(success: Boolean, message: String, counter: List[Counter])
   object CopyResult {
     def noSourceImpl: CopyResult =
-      CopyResult(success = false, "No source BinaryStore implementation found!", Nil)
+      CopyResult(false, "No source BinaryStore implementation found!", Nil)
 
     def noTargetImpl: CopyResult =
-      CopyResult(success = false, "No target BinaryStore implementation found!", Nil)
+      CopyResult(false, "No target BinaryStore implementation found!", Nil)
 
     def noSourceStore(id: Ident): CopyResult =
       CopyResult(
-        success = false,
+        false,
         s"No source file repo found with id: ${id.id}. Make sure it is present in the config.",
         Nil
       )
 
     def noTargetStore: CopyResult =
-      CopyResult(success = false, "No target file repositories defined", Nil)
+      CopyResult(false, "No target file repositories defined", Nil)
 
     def success(counter: NonEmptyList[Counter]): CopyResult =
-      CopyResult(success = true, "Done", counter.toList)
+      CopyResult(true, "Done", counter.toList)
 
     implicit val binaryIdCodec: Codec[BinaryId] =
       Codec.from(
@@ -96,10 +96,8 @@ object FileCopyTask {
             .fromList(targets.filter(_ != srcConfig))
             .toRight(CopyResult.noTargetStore)
 
-          srcRepo = store.createFileRepository(srcConfig, withAttributeStore = true)
+          srcRepo = store.createFileRepository(srcConfig, true)
-          targetRepos = trgConfig.map(
+          targetRepos = trgConfig.map(store.createFileRepository(_, false))
-            store.createFileRepository(_, withAttributeStore = false)
-          )
         } yield (srcRepo, targetRepos)
 
       data match {

modules/joex/src/main/scala/docspell/joex/hk/CleanupResult.scala

@@ -13,8 +13,8 @@ case class CleanupResult(removed: Int, disabled: Boolean) {
   def asString = if (disabled) "disabled" else s"$removed"
 }
 object CleanupResult {
-  def of(n: Int): CleanupResult = CleanupResult(n, disabled = false)
+  def of(n: Int): CleanupResult = CleanupResult(n, false)
-  def disabled: CleanupResult = CleanupResult(0, disabled = true)
+  def disabled: CleanupResult = CleanupResult(0, true)
 
   implicit val jsonEncoder: Encoder[CleanupResult] =
     deriveEncoder

modules/joex/src/main/scala/docspell/joex/hk/HouseKeepingTask.scala

@@ -55,7 +55,7 @@ object HouseKeepingTask {
     UserTask(
       periodicId,
       taskName,
-      enabled = true,
+      true,
       ce,
       "Docspell house-keeping".some,
       ()

modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala

@@ -222,13 +222,13 @@ object FindProposal {
     def searchExact[F[_]: Sync](ctx: Context[F, Args], store: Store[F]): Finder[F] =
       labels =>
         labels.toList
-          .traverse(nl => search(nl, exact = true, ctx, store))
+          .traverse(nl => search(nl, true, ctx, store))
           .map(MetaProposalList.flatten)
 
     def searchFuzzy[F[_]: Sync](ctx: Context[F, Args], store: Store[F]): Finder[F] =
       labels =>
         labels.toList
-          .traverse(nl => search(nl, exact = false, ctx, store))
+          .traverse(nl => search(nl, false, ctx, store))
           .map(MetaProposalList.flatten)
   }
 

modules/joex/src/main/scala/docspell/joex/process/ReProcessItem.scala

@@ -131,10 +131,10 @@ object ReProcessItem {
               data.item.source, // source-id
               None, // folder
               Seq.empty,
-              skipDuplicate = false,
+              false,
               None,
               None,
-              reprocess = true,
+              true,
               None, // attachOnly (not used when reprocessing attachments)
               None // cannot retain customData from an already existing item
             ),

modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala

@@ -75,7 +75,7 @@ object TextAnalysis {
       analyser: TextAnalyser[F],
       nerFile: RegexNerFile[F]
   )(rm: RAttachmentMeta): F[(RAttachmentMeta, AttachmentDates)] = {
-    val settings = NlpSettings(ctx.args.meta.language, highRecall = false, None)
+    val settings = NlpSettings(ctx.args.meta.language, false, None)
     for {
       customNer <- nerFile.makeFile(ctx.args.meta.collective)
       sett = settings.copy(regexNer = customNer)

modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala

@@ -28,7 +28,7 @@ object JoexRoutes {
         for {
           _ <- app.scheduler.notifyChange
           _ <- app.periodicScheduler.notifyChange
-          resp <- Ok(BasicResult(success = true, "Schedulers notified."))
+          resp <- Ok(BasicResult(true, "Schedulers notified."))
         } yield resp
 
       case GET -> Root / "running" =>
@@ -43,7 +43,7 @@ object JoexRoutes {
           _ <- Async[F].start(
             Temporal[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown
           )
-          resp <- Ok(BasicResult(success = true, "Shutdown initiated."))
+          resp <- Ok(BasicResult(true, "Shutdown initiated."))
         } yield resp
 
       case GET -> Root / "job" / Ident(id) =>
@@ -54,9 +54,7 @@ object JoexRoutes {
             job <- optJob
             log <- optLog
           } yield mkJobLog(job, log)
-          resp <- jAndL
+          resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found")))
-            .map(Ok(_))
-            .getOrElse(NotFound(BasicResult(success = false, "Not found")))
         } yield resp
 
       case POST -> Root / "job" / Ident(id) / "cancel" =>

modules/joex/src/main/scala/docspell/joex/scanmailbox/ScanMailboxTask.scala

@@ -323,7 +323,7 @@ object ScanMailboxTask {
           s"mailbox-${ctx.args.account.login.id}",
           args.itemFolder,
           Seq.empty,
-          skipDuplicates = true,
+          true,
           args.fileFilter.getOrElse(Glob.all),
           args.tags.getOrElse(Nil),
           args.language,

modules/joexapi/src/main/resources/joex-openapi.yml

@@ -18,8 +18,6 @@ servers:
   - url: /api/v1
     description: Current host
 
-security: []
-
 paths:
   /api/info/version:
     get:

modules/notification/api/src/main/scala/docspell/notification/api/Event.scala

@@ -164,7 +164,7 @@ object Event {
       for {
         id1 <- Ident.randomId[F]
         id2 <- Ident.randomId[F]
-      } yield ItemSelection(account, Nel.of(id1, id2), more = true, baseUrl, None)
+      } yield ItemSelection(account, Nel.of(id1, id2), true, baseUrl, None)
   }
 
   /** Event when a new job is added to the queue */

modules/pubsub/naive/src/test/scala/docspell/pubsub/naive/Fixtures.scala

@@ -20,9 +20,9 @@ import org.typelevel.ci._
 
 trait Fixtures extends HttpClientOps { self: CatsEffectSuite =>
 
-  val pubsubEnv = ResourceFunFixture(Fixtures.envResource("node-1"))
+  val pubsubEnv = ResourceFixture(Fixtures.envResource("node-1"))
 
-  val pubsubT = ResourceFunFixture {
+  val pubsubT = ResourceFixture {
     Fixtures
       .envResource("node-1")
       .flatMap(_.pubSub)

modules/query/shared/src/main/scala/docspell/query/ParseFailure.scala

@@ -87,10 +87,10 @@ object ParseFailure {
         SimpleMessage(offset, message)
 
       case InRange(offset, lower, upper) =>
-        if (lower == upper) ExpectMessage(offset, List(lower.toString), exhaustive = true)
+        if (lower == upper) ExpectMessage(offset, List(lower.toString), true)
         else {
           val expect = s"$lower-$upper"
-          ExpectMessage(offset, List(expect), exhaustive = true)
+          ExpectMessage(offset, List(expect), true)
         }
 
       case Length(offset, expected, actual) =>
@@ -110,10 +110,6 @@ object ParseFailure {
         ExpectMessage(offset, options.take(7), options.size < 8)
 
       case WithContext(ctx, expect) =>
-        ExpectMessage(
+        ExpectMessage(expect.offset, s"Failed to parse near: $ctx" :: Nil, true)
-          expect.offset,
-          s"Failed to parse near: $ctx" :: Nil,
-          exhaustive = true
-        )
     }
 }

modules/restapi/src/main/resources/docspell-openapi.yml

@@ -27,8 +27,6 @@ servers:
   - url: /api/v1
     description: Current host
 
-security: []
-
 paths:
   /api/info/version:
     get:

modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala

@@ -329,7 +329,7 @@ trait Conversions {
             sourceName,
             None,
             validFileTypes,
-            skipDuplicates = false,
+            false,
             Glob.all,
             Nil,
             None,
@@ -641,86 +641,82 @@ trait Conversions {
   def basicResult(r: SetValueResult): BasicResult =
     r match {
       case SetValueResult.FieldNotFound =>
-        BasicResult(success = false, "The given field is unknown")
+        BasicResult(false, "The given field is unknown")
       case SetValueResult.ItemNotFound =>
-        BasicResult(success = false, "The given item is unknown")
+        BasicResult(false, "The given item is unknown")
       case SetValueResult.ValueInvalid(msg) =>
-        BasicResult(success = false, s"The value is invalid: $msg")
+        BasicResult(false, s"The value is invalid: $msg")
       case SetValueResult.Success =>
-        BasicResult(success = true, "Custom field value set successfully.")
+        BasicResult(true, "Custom field value set successfully.")
     }
 
   def basicResult(cr: JobCancelResult): BasicResult =
     cr match {
-      case JobCancelResult.JobNotFound => BasicResult(success = false, "Job not found")
+      case JobCancelResult.JobNotFound => BasicResult(false, "Job not found")
       case JobCancelResult.CancelRequested =>
-        BasicResult(success = true, "Cancel was requested at the job executor")
+        BasicResult(true, "Cancel was requested at the job executor")
       case JobCancelResult.Removed =>
-        BasicResult(success = true, "The job has been removed from the queue.")
+        BasicResult(true, "The job has been removed from the queue.")
     }
 
   def idResult(ar: AddResult, id: Ident, successMsg: String): IdResult =
     ar match {
-      case AddResult.Success           => IdResult(success = true, successMsg, id)
+      case AddResult.Success           => IdResult(true, successMsg, id)
-      case AddResult.EntityExists(msg) => IdResult(success = false, msg, Ident.unsafe(""))
+      case AddResult.EntityExists(msg) => IdResult(false, msg, Ident.unsafe(""))
       case AddResult.Failure(ex) =>
-        IdResult(success = false, s"Internal error: ${ex.getMessage}", Ident.unsafe(""))
+        IdResult(false, s"Internal error: ${ex.getMessage}", Ident.unsafe(""))
     }
 
   def basicResult(ar: AddResult, successMsg: String): BasicResult =
     ar match {
-      case AddResult.Success           => BasicResult(success = true, successMsg)
+      case AddResult.Success           => BasicResult(true, successMsg)
-      case AddResult.EntityExists(msg) => BasicResult(success = false, msg)
+      case AddResult.EntityExists(msg) => BasicResult(false, msg)
       case AddResult.Failure(ex) =>
-        BasicResult(success = false, s"Internal error: ${ex.getMessage}")
+        BasicResult(false, s"Internal error: ${ex.getMessage}")
     }
 
   def basicResult(ar: UpdateResult, successMsg: String): BasicResult =
     ar match {
-      case UpdateResult.Success  => BasicResult(success = true, successMsg)
+      case UpdateResult.Success  => BasicResult(true, successMsg)
-      case UpdateResult.NotFound => BasicResult(success = false, "Not found")
+      case UpdateResult.NotFound => BasicResult(false, "Not found")
       case UpdateResult.Failure(ex) =>
-        BasicResult(success = false, s"Error: ${ex.getMessage}")
+        BasicResult(false, s"Error: ${ex.getMessage}")
     }
 
   def basicResult(ur: OUpload.UploadResult): BasicResult =
     ur match {
-      case UploadResult.Success => BasicResult(success = true, "Files submitted.")
+      case UploadResult.Success  => BasicResult(true, "Files submitted.")
-      case UploadResult.NoFiles =>
+      case UploadResult.NoFiles  => BasicResult(false, "There were no files to submit.")
-        BasicResult(success = false, "There were no files to submit.")
+      case UploadResult.NoSource => BasicResult(false, "The source id is not valid.")
-      case UploadResult.NoSource =>
+      case UploadResult.NoItem   => BasicResult(false, "The item could not be found.")
-        BasicResult(success = false, "The source id is not valid.")
-      case UploadResult.NoItem =>
-        BasicResult(success = false, "The item could not be found.")
       case UploadResult.NoCollective =>
-        BasicResult(success = false, "The collective could not be found.")
+        BasicResult(false, "The collective could not be found.")
       case UploadResult.StoreFailure(_) =>
         BasicResult(
-          success = false,
+          false,
           "There were errors storing a file! See the server logs for details."
         )
     }
 
   def basicResult(cr: PassChangeResult): BasicResult =
     cr match {
-      case PassChangeResult.Success => BasicResult(success = true, "Password changed.")
+      case PassChangeResult.Success => BasicResult(true, "Password changed.")
       case PassChangeResult.UpdateFailed =>
-        BasicResult(success = false, "The database update failed.")
+        BasicResult(false, "The database update failed.")
       case PassChangeResult.PasswordMismatch =>
-        BasicResult(success = false, "The current password is incorrect.")
+        BasicResult(false, "The current password is incorrect.")
-      case PassChangeResult.UserNotFound =>
+      case PassChangeResult.UserNotFound => BasicResult(false, "User not found.")
-        BasicResult(success = false, "User not found.")
       case PassChangeResult.InvalidSource(source) =>
         BasicResult(
-          success = false,
+          false,
           s"User has invalid soure: $source. Passwords are managed elsewhere."
         )
     }
 
   def basicResult(e: Either[Throwable, _], successMsg: String): BasicResult =
     e match {
-      case Right(_) => BasicResult(success = true, successMsg)
+      case Right(_) => BasicResult(true, successMsg)
-      case Left(ex) => BasicResult(success = false, ex.getMessage)
+      case Left(ex) => BasicResult(false, ex.getMessage)
     }
 
   // MIME Type

modules/restserver/src/main/scala/docspell/restserver/http4s/BinaryUtil.scala

@@ -38,7 +38,7 @@ object BinaryUtil {
         if (matches) withResponseHeaders(dsl, NotModified())(data)
         else makeByteResp(dsl)(data)
       }
-      .getOrElse(NotFound(BasicResult(success = false, "Not found")))
+      .getOrElse(NotFound(BasicResult(false, "Not found")))
   }
 
   def respondHead[F[_]: Async](dsl: Http4sDsl[F])(
@@ -48,7 +48,7 @@ object BinaryUtil {
 
     fileData
       .map(data => withResponseHeaders(dsl, Ok())(data))
-      .getOrElse(NotFound(BasicResult(success = false, "Not found")))
+      .getOrElse(NotFound(BasicResult(false, "Not found")))
   }
 
   def respondPreview[F[_]: Async](dsl: Http4sDsl[F], req: Request[F])(
@@ -56,7 +56,7 @@ object BinaryUtil {
   ): F[Response[F]] = {
     import dsl._
     def notFound =
-      NotFound(BasicResult(success = false, "Not found"))
+      NotFound(BasicResult(false, "Not found"))
 
     QP.WithFallback.unapply(req.multiParams) match {
       case Some(bool) =>
@@ -75,7 +75,7 @@ object BinaryUtil {
           )
 
       case None =>
-        BadRequest(BasicResult(success = false, "Invalid query parameter 'withFallback'"))
+        BadRequest(BasicResult(false, "Invalid query parameter 'withFallback'"))
     }
   }
 
@@ -85,7 +85,7 @@ object BinaryUtil {
     import dsl._
     fileData
       .map(data => withResponseHeaders(dsl, Ok())(data))
-      .getOrElse(NotFound(BasicResult(success = false, "Not found")))
+      .getOrElse(NotFound(BasicResult(false, "Not found")))
   }
 
   def withResponseHeaders[F[_]: Sync](dsl: Http4sDsl[F], resp: F[Response[F]])(

modules/restserver/src/main/scala/docspell/restserver/http4s/ThrowableResponseMapper.scala

@@ -33,10 +33,10 @@ object ThrowableResponseMapper {
     def toResponse(ex: Throwable): F[Response[F]] =
       ex match {
         case _: IllegalArgumentException =>
-          BadRequest(BasicResult(success = false, ex.getMessage))
+          BadRequest(BasicResult(false, ex.getMessage))
 
         case _ =>
-          InternalServerError(BasicResult(success = false, ex.getMessage))
+          InternalServerError(BasicResult(false, ex.getMessage))
       }
   }
 }

modules/restserver/src/main/scala/docspell/restserver/routes/AddonArchiveRoutes.scala

@@ -52,7 +52,7 @@ object AddonArchiveRoutes extends AddonValidationSupport {
       case req @ POST -> Root :? Sync(sync) =>
         def create(r: Option[RAddonArchive]) =
           IdResult(
-            success = true,
+            true,
             r.fold("Addon submitted for installation")(r =>
               s"Addon installed: ${r.id.id}"
             ),
@@ -77,7 +77,7 @@ object AddonArchiveRoutes extends AddonValidationSupport {
       case PUT -> Root / Ident(id) :? Sync(sync) =>
         def create(r: Option[AddonMeta]) =
           BasicResult(
-            success = true,
+            true,
             r.fold("Addon updated in background")(m =>
               s"Addon updated: ${m.nameAndVersion}"
             )
@@ -99,8 +99,8 @@ object AddonArchiveRoutes extends AddonValidationSupport {
         for {
           flag <- backend.addons.deleteAddon(token.account.collectiveId, id)
           resp <-
-            if (flag) Ok(BasicResult(success = true, "Addon deleted"))
+            if (flag) Ok(BasicResult(true, "Addon deleted"))
-            else NotFound(BasicResult(success = false, "Addon not found"))
+            else NotFound(BasicResult(false, "Addon not found"))
         } yield resp
     }
   }
@@ -112,11 +112,11 @@ object AddonArchiveRoutes extends AddonValidationSupport {
     import dsl._
 
     def failWith(msg: String): F[Response[F]] =
-      Ok(IdResult(success = false, msg, Ident.unsafe("")))
+      Ok(IdResult(false, msg, Ident.unsafe("")))
 
     e match {
       case AddonValidationError.AddonNotFound =>
-        NotFound(BasicResult(success = false, "Addon not found."))
+        NotFound(BasicResult(false, "Addon not found."))
 
       case _ =>
         failWith(validationErrorToMessage(e))

modules/restserver/src/main/scala/docspell/restserver/routes/AddonRoutes.scala

@@ -35,5 +35,5 @@ object AddonRoutes {
         "run" -> AddonRunRoutes(backend, token)
       )
     else
-      Responses.notFoundRoute(BasicResult(success = false, "Addons disabled"))
+      Responses.notFoundRoute(BasicResult(false, "Addons disabled"))
 }

modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunConfigRoutes.scala

@@ -43,8 +43,8 @@ object AddonRunConfigRoutes {
               .map(_.leftMap(_.message))
           )
           resp <- res.fold(
-            msg => Ok(BasicResult(success = false, msg)),
+            msg => Ok(BasicResult(false, msg)),
-            id => Ok(IdResult(success = true, s"Addon run config added", id))
+            id => Ok(IdResult(true, s"Addon run config added", id))
           )
         } yield resp
 
@@ -58,8 +58,8 @@ object AddonRunConfigRoutes {
               .map(_.leftMap(_.message))
           )
           resp <- res.fold(
-            msg => Ok(BasicResult(success = false, msg)),
+            msg => Ok(BasicResult(false, msg)),
-            id => Ok(IdResult(success = true, s"Addon run config updated", id))
+            id => Ok(IdResult(true, s"Addon run config updated", id))
           )
         } yield resp
 
@@ -67,8 +67,8 @@ object AddonRunConfigRoutes {
         for {
           flag <- backend.addons.deleteAddonRunConfig(token.account.collectiveId, id)
           resp <-
-            if (flag) Ok(BasicResult(success = true, "Addon task deleted"))
+            if (flag) Ok(BasicResult(true, "Addon task deleted"))
-            else NotFound(BasicResult(success = false, "Addon task not found"))
+            else NotFound(BasicResult(false, "Addon task not found"))
         } yield resp
     }
   }

modules/restserver/src/main/scala/docspell/restserver/routes/AddonRunRoutes.scala

@@ -35,7 +35,7 @@ object AddonRunRoutes {
           input.addonRunConfigIds.toSet,
           UserTaskScope(token.account)
         )
-        resp <- Ok(BasicResult(success = true, "Job for running addons submitted."))
+        resp <- Ok(BasicResult(true, "Job for running addons submitted."))
       } yield resp
     }
   }

modules/restserver/src/main/scala/docspell/restserver/routes/AttachmentRoutes.scala

@@ -66,7 +66,7 @@ object AttachmentRoutes {
           resp <-
             fileData
               .map(data => withResponseHeaders(Ok())(data))
-              .getOrElse(NotFound(BasicResult(success = false, "Not found")))
+              .getOrElse(NotFound(BasicResult(false, "Not found")))
         } yield resp
 
       case req @ GET -> Root / Ident(id) / "original" =>
@@ -83,7 +83,7 @@ object AttachmentRoutes {
                 if (matches) withResponseHeaders(NotModified())(data)
                 else makeByteResp(data)
               }
-              .getOrElse(NotFound(BasicResult(success = false, "Not found")))
+              .getOrElse(NotFound(BasicResult(false, "Not found")))
         } yield resp
 
       case HEAD -> Root / Ident(id) / "archive" =>
@@ -93,7 +93,7 @@ object AttachmentRoutes {
           resp <-
             fileData
               .map(data => withResponseHeaders(Ok())(data))
-              .getOrElse(NotFound(BasicResult(success = false, "Not found")))
+              .getOrElse(NotFound(BasicResult(false, "Not found")))
         } yield resp
 
       case req @ GET -> Root / Ident(id) / "archive" =>
@@ -108,7 +108,7 @@ object AttachmentRoutes {
                 if (matches) withResponseHeaders(NotModified())(data)
                 else makeByteResp(data)
               }
-              .getOrElse(NotFound(BasicResult(success = false, "Not found")))
+              .getOrElse(NotFound(BasicResult(false, "Not found")))
         } yield resp
 
       case req @ GET -> Root / Ident(id) / "preview" =>
@@ -148,9 +148,7 @@ object AttachmentRoutes {
         for {
           rm <- backend.itemSearch.findAttachmentMeta(id, user.account.collectiveId)
           md = rm.map(Conversions.mkAttachmentMeta)
-          resp <- md
+          resp <- md.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found.")))
-            .map(Ok(_))
-            .getOrElse(NotFound(BasicResult(success = false, "Not found.")))
         } yield resp
 
       case req @ POST -> Root / Ident(id) / "name" =>
@@ -171,11 +169,8 @@ object AttachmentRoutes {
             backend.attachment
               .setExtractedText(user.account.collectiveId, itemId, id, newText)
           )
-          resp <- OptionT.liftF(
+          resp <- OptionT.liftF(Ok(BasicResult(true, "Extracted text updated.")))
-            Ok(BasicResult(success = true, "Extracted text updated."))
+        } yield resp).getOrElseF(NotFound(BasicResult(false, "Attachment not found")))
-          )
-        } yield resp)
-          .getOrElseF(NotFound(BasicResult(success = false, "Attachment not found")))
 
       case DELETE -> Root / Ident(id) / "extracted-text" =>
         (for {
@@ -186,9 +181,7 @@ object AttachmentRoutes {
             backend.attachment
               .setExtractedText(user.account.collectiveId, itemId, id, "".pure[F])
           )
-          resp <- OptionT.liftF(
+          resp <- OptionT.liftF(Ok(BasicResult(true, "Extracted text cleared.")))
-            Ok(BasicResult(success = true, "Extracted text cleared."))
-          )
         } yield resp).getOrElseF(NotFound())
 
       case GET -> Root / Ident(id) / "extracted-text" =>
@@ -197,15 +190,14 @@ object AttachmentRoutes {
             backend.itemSearch.findAttachmentMeta(id, user.account.collectiveId)
           )
           resp <- OptionT.liftF(Ok(OptionalText(meta.content)))
-        } yield resp)
+        } yield resp).getOrElseF(NotFound(BasicResult(false, "Attachment not found")))
-          .getOrElseF(NotFound(BasicResult(success = false, "Attachment not found")))
 
       case DELETE -> Root / Ident(id) =>
         for {
           n <- backend.item.deleteAttachment(id, user.account.collectiveId)
           res =
-            if (n == 0) BasicResult(success = false, "Attachment not found")
+            if (n == 0) BasicResult(false, "Attachment not found")
-            else BasicResult(success = true, "Attachment deleted.")
+            else BasicResult(true, "Attachment deleted.")
           resp <- Ok(res)
         } yield resp
     }

modules/restserver/src/main/scala/docspell/restserver/routes/CalEventCheckRoutes.scala

@@ -40,9 +40,9 @@ object CalEventCheckRoutes {
           val next = ev
             .nextElapses(now.toUtcDateTime, 2)
             .map(Timestamp.atUtc)
-          CalEventCheckResult(success = true, "Valid.", ev.some, next)
+          CalEventCheckResult(true, "Valid.", ev.some, next)
         case Left(err) =>
-          CalEventCheckResult(success = false, err, None, Nil)
+          CalEventCheckResult(false, err, None, Nil)
       }
     }
 }

modules/restserver/src/main/scala/docspell/restserver/routes/ClientSettingsRoutes.scala

@@ -66,7 +66,7 @@ object ClientSettingsRoutes {
         for {
           data <- req.as[Json]
           _ <- backend.clientSettings.saveUser(clientId, user.account.userId, data)
-          res <- Ok(BasicResult(success = true, "Settings stored"))
+          res <- Ok(BasicResult(true, "Settings stored"))
         } yield res
 
       case GET -> Root / "user" / Ident(clientId) =>
@@ -97,7 +97,7 @@ object ClientSettingsRoutes {
             user.account.collectiveId,
             data
           )
-          res <- Ok(BasicResult(success = true, "Settings stored"))
+          res <- Ok(BasicResult(true, "Settings stored"))
         } yield res
 
       case GET -> Root / "collective" / Ident(clientId) =>

modules/restserver/src/main/scala/docspell/restserver/routes/CollectiveRoutes.scala

@@ -118,7 +118,7 @@ object CollectiveRoutes {
       case POST -> Root / "classifier" / "startonce" =>
         for {
           _ <- backend.collective.startLearnClassifier(user.account.collectiveId)
-          resp <- Ok(BasicResult(success = true, "Task submitted"))
+          resp <- Ok(BasicResult(true, "Task submitted"))
         } yield resp
 
       case req @ POST -> Root / "emptytrash" / "startonce" =>
@@ -127,7 +127,7 @@ object CollectiveRoutes {
           _ <- backend.collective.startEmptyTrash(
             EmptyTrashArgs(user.account.collectiveId, data.minAge)
           )
-          resp <- Ok(BasicResult(success = true, "Task submitted"))
+          resp <- Ok(BasicResult(true, "Task submitted"))
         } yield resp
     }
   }

modules/restserver/src/main/scala/docspell/restserver/routes/CustomFieldRoutes.scala

@@ -56,7 +56,7 @@ object CustomFieldRoutes {
         (for {
           field <- OptionT(backend.customFields.findById(user.account.collectiveId, id))
           res <- OptionT.liftF(Ok(convertField(field)))
-        } yield res).getOrElseF(NotFound(BasicResult(success = false, "Not found")))
+        } yield res).getOrElseF(NotFound(BasicResult(false, "Not found")))
 
       case req @ PUT -> Root / Ident(id) =>
         for {