Merge pull request #2539 from eikek/nix-refactor

Extend nix setup, including dev environments
This commit is contained in:
mergify[bot] 2024-03-10 17:06:08 +00:00 committed by GitHub
commit 5715f60e96
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
28 changed files with 1047 additions and 630 deletions

View File

@ -6,20 +6,13 @@ on:
- "master"
jobs:
check-website:
runs-on: ubuntu-22.04
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.1.1
with:
fetch-depth: 0
- uses: cachix/install-nix-action@v26
- name: Set current version
run: echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
- uses: jorelali/setup-elm@v5
with:
elm-version: 0.19.1
- uses: cachix/install-nix-action@v26
with:
nix_path: nixpkgs=channel:nixos-23.05
- name: Print nixpkgs version
run: nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
- name: Build website (${{ env.DOCSPELL_VERSION }})
run: nix-shell website/shell.nix --run "sbt make-website"
run: nix develop .#ci --command sbt make-website

View File

@ -5,32 +5,18 @@ on:
- master
jobs:
ci-matrix:
runs-on: ubuntu-22.04
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
java: [ 'openjdk@1.17' ]
steps:
- uses: actions/checkout@v4.1.1
with:
fetch-depth: 100
- uses: jorelali/setup-elm@v5
with:
elm-version: 0.19.1
- uses: bahmutov/npm-install@v1
with:
working-directory: modules/webapp
- name: Fetch tags
run: git fetch --depth=100 origin +refs/tags/*:refs/tags/*
- uses: olafurpg/setup-scala@v14
with:
java-version: ${{ matrix.java }}
# - name: Coursier cache
# uses: coursier/cache-action@v6
- uses: cachix/install-nix-action@v26
- name: sbt ci ${{ github.ref }}
run: sbt ci
env:
SBT_OPTS: "-Xmx2G -Xss4m"
run: nix develop .#ci --command sbt ci
ci:
runs-on: ubuntu-22.04
needs: [ci-matrix]

View File

@ -4,7 +4,7 @@ on:
types: [ published ]
jobs:
docker-images:
runs-on: ubuntu-22.04
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.1.1
with:

View File

@ -5,36 +5,20 @@ on:
- "master"
jobs:
release-nightly:
runs-on: ubuntu-22.04
runs-on: ubuntu-latest
strategy:
fail-fast: true
matrix:
java: [ 'openjdk@1.17' ]
steps:
- uses: actions/checkout@v4.1.1
with:
fetch-depth: 0
- uses: olafurpg/setup-scala@v14
with:
java-version: ${{ matrix.java }}
- uses: jorelali/setup-elm@v5
with:
elm-version: 0.19.1
- uses: bahmutov/npm-install@v1
with:
working-directory: modules/webapp
# - name: Coursier cache
# uses: coursier/cache-action@v6
- uses: cachix/install-nix-action@v26
- name: Set current version
run: echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
- name: sbt ci ${{ github.ref }}
run: sbt ci
env:
SBT_OPTS: "-Xmx2G -Xss4m"
run: nix develop .#ci --command sbt ci
- name: sbt make-pkg (${{ env.DOCSPELL_VERSION }})
run: sbt make-pkg
env:
SBT_OPTS: "-Xmx2G -Xss4m"
run: nix develop .#ci --command sbt make-pkg
- uses: "marvinpinto/action-automatic-releases@latest"
with:
repo_token: "${{ secrets.GITHUB_TOKEN }}"

View File

@ -5,32 +5,18 @@ on:
- 'v*'
jobs:
release:
runs-on: ubuntu-22.04
runs-on: ubuntu-latest
strategy:
fail-fast: true
matrix:
java: [ 'openjdk@1.17' ]
steps:
- uses: actions/checkout@v4.1.1
with:
fetch-depth: 0
- uses: olafurpg/setup-scala@v14
with:
java-version: ${{ matrix.java }}
- uses: jorelali/setup-elm@v5
with:
elm-version: 0.19.1
- uses: bahmutov/npm-install@v1
with:
working-directory: modules/webapp
# - name: Coursier cache
# uses: coursier/cache-action@v6
- uses: cachix/install-nix-action@v26
- name: Set current version
run: echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
- name: sbt make-pkg (${{ env.DOCSPELL_VERSION }})
run: sbt make-pkg
env:
SBT_OPTS: "-Xmx2G -Xss4m"
run: nix develop .#ci --command sbt make-pkg
- uses: meeDamian/github-release@2.0
with:
token: ${{ secrets.GITHUB_TOKEN }}

View File

@ -5,24 +5,17 @@ on:
- "current-docs"
jobs:
publish-website:
runs-on: ubuntu-22.04
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.1.1
with:
fetch-depth: 0
- uses: cachix/install-nix-action@v26
- name: Set current version
run: echo "DOCSPELL_VERSION=$(cat version.sbt | grep version | cut -d= -f2 | xargs)" >> $GITHUB_ENV
- uses: jorelali/setup-elm@v5
with:
elm-version: 0.19.1
- uses: cachix/install-nix-action@v26
with:
nix_path: nixpkgs=channel:nixos-23.05
- name: Print nixpkgs version
run: nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
- name: Build website (${{ env.DOCSPELL_VERSION }})
run: nix-shell website/shell.nix --run "sbt make-website"
run: nix develop .#ci --command sbt make-website
- name: Publish website (${{ env.DOCSPELL_VERSION }})
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: sbt publish-website
run: nix develop .#ci --command sbt publish-website

1
.gitignore vendored
View File

@ -1,4 +1,5 @@
#artwork/*.png
.envrc
target/
local/
node_modules/

130
flake.lock generated Normal file
View File

@ -0,0 +1,130 @@
{
"nodes": {
"devshell-tools": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs"
},
"locked": {
"lastModified": 1710025500,
"narHash": "sha256-niJf4WZ4GHQ+RaP+Qj4+6P/2SWN1Z4co36/ETvIg0Qg=",
"owner": "eikek",
"repo": "devshell-tools",
"rev": "57d7d292571e291fe8213a1655529f739cfa174d",
"type": "github"
},
"original": {
"owner": "eikek",
"repo": "devshell-tools",
"type": "github"
}
},
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1709126324,
"narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "d465f4819400de7c8d874d50b982301f28a84605",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"flake-utils_2": {
"inputs": {
"systems": "systems_2"
},
"locked": {
"lastModified": 1709126324,
"narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "d465f4819400de7c8d874d50b982301f28a84605",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1709309926,
"narHash": "sha256-VZFBtXGVD9LWTecGi6eXrE0hJ/mVB3zGUlHImUs2Qak=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "79baff8812a0d68e24a836df0a364c678089e2c7",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-23.11",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1709677081,
"narHash": "sha256-tix36Y7u0rkn6mTm0lA45b45oab2cFLqAzDbJxeXS+c=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "880992dcc006a5e00dd0591446fdf723e6a51a64",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-23.11",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"devshell-tools": "devshell-tools",
"flake-utils": "flake-utils_2",
"nixpkgs": "nixpkgs_2"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"systems_2": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

187
flake.nix Normal file
View File

@ -0,0 +1,187 @@
{
description = "Docspell";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11";
devshell-tools.url = "github:eikek/devshell-tools";
flake-utils.url = "github:numtide/flake-utils";
};
outputs = {
self,
nixpkgs,
flake-utils,
devshell-tools,
}:
flake-utils.lib.eachDefaultSystem (system: let
pkgs = nixpkgs.legacyPackages.${system};
sbt17 = pkgs.sbt.override { jre = pkgs.jdk17; };
ciPkgs = with pkgs; [
sbt17
jdk17
dpkg
elmPackages.elm
fakeroot
zola
yarn
nodejs
];
devshellPkgs = ciPkgs ++ (with pkgs; [
jq
scala-cli
netcat
wget
which
inotifyTools
]);
docspellPkgs = pkgs.callPackage (import ./nix/pkg.nix) {};
dockerAmd64 = pkgs.pkgsCross.gnu64.callPackage (import ./nix/docker.nix) {
inherit (docspellPkgs) docspell-restserver docspell-joex;
};
dockerArm64 = pkgs.pkgsCross.aarch64-multiplatform.callPackage (import ./nix/docker.nix) {
inherit (docspellPkgs) docspell-restserver docspell-joex;
};
in {
formatter = pkgs.alejandra;
packages = {
inherit (docspellPkgs) docspell-restserver docspell-joex;
};
legacyPackages = {
docker = {
amd64 = {
inherit (dockerAmd64) docspell-restserver docspell-joex;
};
arm64 = {
inherit (dockerArm64) docspell-restserver docspell-joex;
};
};
};
checks = {
build-server = self.packages.${system}.docspell-restserver;
build-joex = self.packages.${system}.docspell-joex;
test = with import (nixpkgs + "/nixos/lib/testing-python.nix")
{
inherit system;
};
makeTest {
name = "docspell";
nodes = {
machine = {...}: {
nixpkgs.overlays = [self.overlays.default];
imports = [
self.nixosModules.default
./nix/checks
];
};
};
testScript = builtins.readFile ./nix/checks/testScript.py;
};
};
devShells = {
dev-cnt = pkgs.mkShellNoCC {
buildInputs =
(builtins.attrValues devshell-tools.legacyPackages.${system}.cnt-scripts)
++ devshellPkgs;
DOCSPELL_ENV = "dev";
DEV_CONTAINER = "docsp-dev";
SBT_OPTS = "-Xmx2G -Xss4m";
};
dev-vm = pkgs.mkShellNoCC {
buildInputs =
(builtins.attrValues devshell-tools.legacyPackages.${system}.vm-scripts)
++ devshellPkgs;
DOCSPELL_ENV = "dev";
SBT_OPTS = "-Xmx2G -Xss4m";
DEV_VM = "dev-vm";
VM_SSH_PORT = "10022";
};
ci = pkgs.mkShellNoCC {
buildInputs = ciPkgs;
SBT_OPTS = "-Xmx2G -Xss4m";
};
};
})
// {
nixosModules = {
default = {...}: {
imports = [
./nix/modules/server.nix
./nix/modules/joex.nix
];
};
server = import ./nix/modules/server.nix;
joex = import ./nix/modules/joex.nix;
};
overlays.default = final: prev: let
docspellPkgs = final.callPackage (import ./nix/pkg.nix) {};
in {
inherit (docspellPkgs) docspell-restserver docspell-joex;
};
nixosConfigurations = {
test-vm = devshell-tools.lib.mkVm {
system = "x86_64-linux";
modules = [
self.nixosModules.default
{
nixpkgs.overlays = [self.overlays.default];
}
./nix/test-vm.nix
];
};
docsp-dev = devshell-tools.lib.mkContainer {
system = "x86_64-linux";
modules = [
{
services.dev-postgres = {
enable = true;
databases = ["docspell"];
};
services.dev-email.enable = true;
services.dev-minio.enable = true;
services.dev-solr = {
enable = true;
cores = ["docspell"];
};
}
];
};
dev-vm = devshell-tools.lib.mkVm {
system = "x86_64-linux";
modules = [
{
services.dev-postgres = {
enable = true;
databases = ["docspell"];
};
services.dev-email.enable = true;
services.dev-minio.enable = true;
services.dev-solr = {
enable = true;
cores = ["docspell"];
};
port-forward.ssh = 10022;
port-forward.dev-postgres = 6534;
port-forward.dev-smtp = 10025;
port-forward.dev-imap = 10143;
port-forward.dev-webmail = 8080;
port-forward.dev-minio-api = 9000;
port-forward.dev-minio-console = 9001;
port-forward.dev-solr = 8983;
networking.hostName = "dev-vm";
}
];
};
};
};
}

View File

@ -1,5 +1,8 @@
{ config, pkgs, ... }:
let
{
config,
pkgs,
...
}: let
full-text-search = {
enabled = true;
backend = "postgresql";
@ -9,9 +12,7 @@ let
};
};
};
in
{
in {
i18n = {
defaultLocale = "en_US.UTF-8";
};
@ -21,12 +22,11 @@ in
password = "root";
};
services.docspell-joex = {
enable = true;
bind.address = "0.0.0.0";
base-url = "http://localhost:7878";
jvmArgs = [ "-J-Xmx1536M" ];
jvmArgs = ["-J-Xmx1536M"];
inherit full-text-search;
};
services.docspell-restserver = {
@ -69,14 +69,12 @@ in
};
};
environment.systemPackages =
[
pkgs.jq
pkgs.inetutils
pkgs.htop
pkgs.jdk17
];
environment.systemPackages = [
pkgs.jq
pkgs.inetutils
pkgs.htop
pkgs.jdk17
];
services.xserver = {
enable = false;
@ -84,12 +82,11 @@ in
networking = {
hostName = "docspelltest";
firewall.allowedTCPPorts = [ 7880 ];
firewall.allowedTCPPorts = [7880];
};
system.stateVersion = "22.11";
# This slows down the build of a vm
documentation.enable = false;
}

View File

@ -1,5 +1,4 @@
{ ... }:
{
{...}: {
imports = [
./configuration-test.nix
];

View File

@ -1,28 +0,0 @@
# NOTE: modulesPath and imports are taken from nixpkgs#59219
{ modulesPath, pkgs, lib, ... }: {
imports = [ (modulesPath + "/virtualisation/qemu-vm.nix") ];
services.openssh = {
enable = true;
permitRootLogin = "yes";
};
services.docspell-restserver = {
openid = lib.mkForce [ ];
backend = lib.mkForce {
signup = {
mode = "open";
};
};
};
# Otherwise oomkiller kills docspell
virtualisation.memorySize = 4096;
virtualisation.forwardPorts = [
# SSH
{ from = "host"; host.port = 64022; guest.port = 22; }
# Docspell
{ from = "host"; host.port = 64080; guest.port = 7880; }
];
}

79
nix/docker.nix Normal file
View File

@ -0,0 +1,79 @@
{
dockerTools,
busybox,
cacert,
wget,
unpaper,
ghostscript,
ocrmypdf,
tesseract4,
python3Packages,
unoconv,
docspell-restserver,
docspell-joex,
}: let
mkImage = {
name,
port,
pkg,
tools,
}:
dockerTools.buildLayeredImage {
inherit name;
created = "now";
contents =
[
busybox
cacert
wget
pkg
]
++ tools;
extraCommands = "mkdir -m 0777 tmp";
#https://github.com/moby/docker-image-spec/blob/main/spec.md#image-json-description
config = {
Entrypoint = ["bin/${name}" "-Dconfig.file="];
#Cmd = ["bin/${name}" "-Dconfig.file="];
ExposedPorts = {
"${builtins.toString port}/tcp" = {};
};
Env = [
"PATH=/bin"
"SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt"
];
Healthcheck = {
Test = [
"CMD"
"wget"
"--spider"
"http://localhost:${builtins.toString port}/api/info/version"
];
Interval = 60000000000; #1min
Timeout = 10000000000; #10s
Retries = 2;
StartInterval = 10000000000;
};
Labels = {
#https://github.com/microscaling/microscaling/blob/55a2d7b91ce7513e07f8b1fd91bbed8df59aed5a/Dockerfile#L22-L33
"org.label-schema.vcs-ref" = "v${pkg.version}";
"org.label-schema.vcs-url" = "https://github.com/eikek/docspell";
};
};
tag = "v${pkg.version}";
};
in {
docspell-restserver = mkImage {
name = "docspell-restserver";
port = 7880;
pkg = docspell-restserver;
tools = [];
};
docspell-joex = mkImage {
name = "docspell-joex";
port = 7878;
pkg = docspell-joex;
tools = [unpaper ghostscript ocrmypdf tesseract4 python3Packages.weasyprint unoconv];
};
}

26
nix/flake.lock generated
View File

@ -1,26 +0,0 @@
{
"nodes": {
"nixpkgs": {
"locked": {
"lastModified": 1706373441,
"narHash": "sha256-S1hbgNbVYhuY2L05OANWqmRzj4cElcbLuIkXTb69xkk=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "56911ef3403a9318b7621ce745f5452fb9ef6867",
"type": "github"
},
"original": {
"id": "nixpkgs",
"ref": "nixos-23.11",
"type": "indirect"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs"
}
}
},
"root": "root",
"version": 7
}

View File

@ -1,134 +0,0 @@
{
description = "Docspell flake";
inputs = {
nixpkgs.url = "nixpkgs/nixos-23.11";
};
outputs = { self, nixpkgs }:
let
supportedSystems = [ "x86_64-linux" "aarch64-linux" ];
forAllSystems = nixpkgs.lib.genAttrs supportedSystems;
nixpkgsFor = forAllSystems (system: import nixpkgs { inherit system; });
# Version config
cfg = {
v0_41_0 = rec {
version = "0.41.0";
server = {
url = "https://github.com/eikek/docspell/releases/download/v${version}/docspell-restserver-${version}.zip";
sha256 = "sha256-JFftIzI94UNLLh96I++qFsBZhOkquPIPhNhtS2Ov8wI=";
};
joex = {
url = "https://github.com/eikek/docspell/releases/download/v${version}/docspell-joex-${version}.zip";
sha256 = "sha256-flKWjEsMd2/XT3Bu6EjFgf3lCojvLbKFDEXemP1K+/8=";
};
};
};
current_version = cfg.v0_41_0;
inherit (current_version) version;
in
rec
{
overlays.default = final: prev: {
docspell-server = with final; stdenv.mkDerivation {
inherit version;
pname = "docspell-server";
src = fetchzip current_version.server;
buildInputs = [ jdk17 ];
buildPhase = "true";
installPhase = ''
mkdir -p $out/{bin,docspell-restserver-${version}}
cp -R * $out/docspell-restserver-${version}/
cat > $out/bin/docspell-restserver <<-EOF
#!${bash}/bin/bash
$out/docspell-restserver-${version}/bin/docspell-restserver -java-home ${jdk17} "\$@"
EOF
chmod 755 $out/bin/docspell-restserver
'';
};
docspell-joex = with final; stdenv.mkDerivation rec {
inherit version;
pname = "docspell-joex";
src = fetchzip current_version.joex;
buildInputs = [ jdk17 ];
buildPhase = "true";
installPhase = ''
mkdir -p $out/{bin,docspell-joex-${version}}
cp -R * $out/docspell-joex-${version}/
cat > $out/bin/docspell-joex <<-EOF
#!${bash}/bin/bash
$out/docspell-joex-${version}/bin/docspell-joex -java-home ${jdk17} "\$@"
EOF
chmod 755 $out/bin/docspell-joex
'';
};
};
packages = forAllSystems (system:
{
default = (import nixpkgs {
inherit system;
overlays = [ self.overlays.default ];
}).docspell-joex;
});
checks = forAllSystems
(system: {
build = self.packages.${system}.default;
test =
with import (nixpkgs + "/nixos/lib/testing-python.nix")
{
inherit system;
};
makeTest {
name = "docspell";
nodes = {
machine = { ... }: {
imports = [
self.nixosModules.default
./checks
];
};
};
testScript = builtins.readFile ./checks/testScript.py;
};
});
nixosModules = {
default = { ... }: {
imports = [
((import ./modules/server.nix) self.overlays.default)
((import ./modules/joex.nix) self.overlays.default)
];
};
server = ((import ./modules/server.nix) self.overlays.default);
joex = ((import ./modules/joex.nix) self.overlays.default);
};
nixosConfigurations =
let
lib = nixpkgs.lib;
in
{
dev-vm = lib.makeOverridable nixpkgs.lib.nixosSystem {
system = "x86_64-linux";
modules = [
self.nixosModules.default
./checks
# nixos-shell specific module. Should be kept outside nix flake checks
./dev-vm
];
};
};
};
}

File diff suppressed because it is too large Load Diff

View File

@ -1,12 +1,18 @@
overlay: { config, lib, pkgs, ... }:
with lib;
let
{
config,
lib,
pkgs,
...
}:
with lib; let
cfg = config.services.docspell-restserver;
# Extract the config without the extraConfig attribute. It will be merged later
declared_config = attrsets.filterAttrs (n: v: n != "extraConfig") cfg;
user = if cfg.runAs == null then "docspell" else cfg.runAs;
configFile = pkgs.writeText "docspell-server.conf" ''
user =
if cfg.runAs == null
then "docspell"
else cfg.runAs;
configFile = pkgs.writeText "docspell-restserver.conf" ''
{"docspell": {"server":
${builtins.toJSON (lib.recursiveUpdate declared_config cfg.extraConfig)}
}}
@ -44,7 +50,7 @@ let
source-name = "integration";
allowed-ips = {
enabled = false;
ips = [ "127.0.0.1" ];
ips = ["127.0.0.1"];
};
http-basic = {
enabled = false;
@ -78,9 +84,9 @@ let
user = "pguser";
password = "";
};
pg-config = { };
pg-config = {};
pg-query-parser = "websearch_to_tsquery";
pg-rank-normalization = [ 4 ];
pg-rank-normalization = [4];
};
};
auth = {
@ -126,19 +132,17 @@ let
};
files = {
chunk-size = 524288;
valid-mime-types = [ ];
valid-mime-types = [];
};
addons = {
enabled = false;
allow-impure = true;
allowed-urls = [ "*" ];
denied-urls = [ ];
allowed-urls = ["*"];
denied-urls = [];
};
};
};
in
{
in {
## interface
options = {
services.docspell-restserver = {
@ -156,12 +160,11 @@ in
};
jvmArgs = mkOption {
type = types.listOf types.str;
default = [ ];
example = [ "-J-Xmx1G" ];
default = [];
example = ["-J-Xmx1G"];
description = "The options passed to the executable for setting jvm arguments.";
};
app-name = mkOption {
type = types.str;
default = defaults.app-name;
@ -232,7 +235,7 @@ in
};
bind = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
address = mkOption {
type = types.str;
@ -245,13 +248,13 @@ in
description = "The port to bind the REST server";
};
};
});
};
default = defaults.bind;
description = "Address and port bind the rest server.";
};
server-options = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
enable-http-2 = mkOption {
type = types.bool;
@ -269,13 +272,13 @@ in
description = "Timeout when waiting for the response.";
};
};
});
};
default = defaults.server-options;
description = "Tuning the http server";
};
logging = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
minimum-level = mkOption {
type = types.str;
@ -293,13 +296,13 @@ in
description = "Set of logger and their levels";
};
};
});
};
default = defaults.logging;
description = "Settings for logging";
};
auth = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
server-secret = mkOption {
type = types.str;
@ -320,7 +323,7 @@ in
'';
};
remember-me = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
enabled = mkOption {
type = types.bool;
@ -333,18 +336,18 @@ in
description = "The time a remember-me token is valid.";
};
};
});
};
default = defaults.auth.remember-me;
description = "Settings for Remember-Me";
};
};
});
};
default = defaults.auth;
description = "Authentication";
};
download-all = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
max-files = mkOption {
type = types.int;
@ -357,7 +360,7 @@ in
description = "The maximum (uncompressed) size of the zip file contents.";
};
};
});
};
default = defaults.download-all;
description = "";
};
@ -387,7 +390,7 @@ in
description = "How to retrieve the collective name.";
};
provider = mkOption {
type = (types.submodule {
type = types.submodule {
options = {
provider-id = mkOption {
type = types.str;
@ -436,18 +439,18 @@ in
description = "The expected algorithm used to sign the token.";
};
};
});
};
default = defaults.openid.provider;
description = "The config for an OpenID Connect provider.";
};
};
});
default = [ ];
default = [];
description = "A list of OIDC provider configurations.";
};
integration-endpoint = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
enabled = mkOption {
type = types.bool;
@ -467,7 +470,7 @@ in
'';
};
allowed-ips = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
enabled = mkOption {
type = types.bool;
@ -480,7 +483,7 @@ in
description = "The ips/ip patterns to allow";
};
};
});
};
default = defaults.integration-endpoint.allowed-ips;
description = ''
IPv4 addresses to allow access. An empty list, if enabled,
@ -491,7 +494,7 @@ in
'';
};
http-basic = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
enabled = mkOption {
type = types.bool;
@ -514,14 +517,14 @@ in
description = "The password to check.";
};
};
});
};
default = defaults.integration-endpoint.http-basic;
description = ''
Requests are expected to use http basic auth when uploading files.
'';
};
http-header = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
enabled = mkOption {
type = types.bool;
@ -539,7 +542,7 @@ in
description = "The value of the header to check.";
};
};
});
};
default = defaults.integration-endpoint.http-header;
description = ''
Requests are expected to supply some specific header when
@ -547,7 +550,7 @@ in
'';
};
};
});
};
default = defaults.integration-endpoint;
description = ''
This endpoint allows to upload files to any collective. The
@ -566,7 +569,7 @@ in
};
admin-endpoint = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
secret = mkOption {
type = types.str;
@ -574,13 +577,13 @@ in
description = "The secret used to call admin endpoints.";
};
};
});
};
default = defaults.admin-endpoint;
description = "An endpoint for administration tasks.";
};
full-text-search = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
enabled = mkOption {
type = types.bool;
@ -597,7 +600,7 @@ in
description = "The backend to use, either solr or postgresql";
};
solr = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
url = mkOption {
type = types.str;
@ -629,13 +632,13 @@ in
description = "The default combiner for tokens. One of {AND, OR}.";
};
};
});
};
default = defaults.full-text-search.solr;
description = "Configuration for the SOLR backend.";
};
postgresql = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
use-default-connection = mkOption {
type = types.bool;
@ -643,7 +646,7 @@ in
description = "Whether to use the primary db connection.";
};
jdbc = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
url = mkOption {
type = types.str;
@ -663,7 +666,7 @@ in
description = "The password to connect to the database.";
};
};
});
};
default = defaults.full-text-search.postgresql.jdbc;
description = "Database connection settings";
};
@ -683,18 +686,18 @@ in
description = "";
};
};
});
};
default = defaults.full-text-search.postgresql;
description = "PostgreSQL for fulltext search";
};
};
});
};
default = defaults.full-text-search;
description = "Configuration for full-text search.";
};
backend = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
mail-debug = mkOption {
type = types.bool;
@ -707,7 +710,7 @@ in
'';
};
jdbc = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
url = mkOption {
type = types.str;
@ -734,12 +737,12 @@ in
description = "The password to connect to the database.";
};
};
});
};
default = defaults.backend.jdbc;
description = "Database connection settings";
};
signup = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
mode = mkOption {
type = types.str;
@ -772,12 +775,12 @@ in
'';
};
};
});
};
default = defaults.backend.signup;
description = "Registration settings";
};
files = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
chunk-size = mkOption {
type = types.int;
@ -804,12 +807,12 @@ in
'';
};
};
});
};
default = defaults.backend.files;
description = "Settings for how files are stored.";
};
addons = mkOption {
type = types.submodule ({
type = types.submodule {
options = {
enabled = mkOption {
type = types.bool;
@ -832,19 +835,19 @@ in
description = "Url patterns to deny to install";
};
};
});
};
default = defaults.backend.addons;
description = "Addon config";
};
};
});
};
default = defaults.backend;
description = "Configuration for the backend";
};
extraConfig = mkOption {
type = types.attrs;
description = "Extra configuration for docspell server. Overwrites values in case of a conflict.";
default = { };
default = {};
example = ''
{
files = {
@ -863,8 +866,6 @@ in
## implementation
config = mkIf config.services.docspell-restserver.enable {
nixpkgs.overlays = [ overlay ];
users.users."${user}" = mkIf (cfg.runAs == null) {
name = user;
isSystemUser = true;
@ -873,24 +874,20 @@ in
description = "Docspell user";
group = user;
};
users.groups."${user}" = mkIf (cfg.runAs == null) { };
users.groups."${user}" = mkIf (cfg.runAs == null) {};
systemd.services.docspell-restserver =
let
args = builtins.concatStringsSep " " cfg.jvmArgs;
cmd = "${pkgs.docspell-server}/bin/docspell-restserver ${args} -- ${configFile}";
in
{
description = "Docspell Rest Server";
after = [ "networking.target" ];
wantedBy = [ "multi-user.target" ];
path = [ pkgs.gawk ];
preStart = ''
systemd.services.docspell-restserver = let
args = builtins.concatStringsSep " " cfg.jvmArgs;
cmd = "${pkgs.docspell-restserver}/bin/docspell-restserver ${args} -- ${configFile}";
in {
description = "Docspell Rest Server";
after = ["networking.target"];
wantedBy = ["multi-user.target"];
path = [pkgs.gawk];
preStart = ''
'';
script =
"${pkgs.su}/bin/su -s ${pkgs.bash}/bin/sh ${user} -c \"${cmd}\"";
};
script = "${pkgs.su}/bin/su -s ${pkgs.bash}/bin/sh ${user} -c \"${cmd}\"";
};
};
}

58
nix/pkg.nix Normal file
View File

@ -0,0 +1,58 @@
{
stdenv,
bash,
fetchzip,
jdk17,
}: let
version = "0.41.0";
server = {
url = "https://github.com/eikek/docspell/releases/download/v${version}/docspell-restserver-${version}.zip";
sha256 = "sha256-JFftIzI94UNLLh96I++qFsBZhOkquPIPhNhtS2Ov8wI=";
};
joex = {
url = "https://github.com/eikek/docspell/releases/download/v${version}/docspell-joex-${version}.zip";
sha256 = "sha256-flKWjEsMd2/XT3Bu6EjFgf3lCojvLbKFDEXemP1K+/8=";
};
in {
docspell-restserver = stdenv.mkDerivation {
inherit version;
pname = "docspell-restserver";
src = fetchzip server;
buildInputs = [jdk17];
buildPhase = "true";
installPhase = ''
mkdir -p $out/{bin,docspell-restserver-${version}}
cp -R * $out/docspell-restserver-${version}/
cat > $out/bin/docspell-restserver <<-EOF
#!${bash}/bin/bash
$out/docspell-restserver-${version}/bin/docspell-restserver -java-home ${jdk17} "\$@"
EOF
chmod 755 $out/bin/docspell-restserver
'';
};
docspell-joex = stdenv.mkDerivation rec {
inherit version;
pname = "docspell-joex";
src = fetchzip joex;
buildInputs = [jdk17];
buildPhase = "true";
installPhase = ''
mkdir -p $out/{bin,docspell-joex-${version}}
cp -R * $out/docspell-joex-${version}/
cat > $out/bin/docspell-joex <<-EOF
#!${bash}/bin/bash
$out/docspell-joex-${version}/bin/docspell-joex -java-home ${jdk17} "\$@"
EOF
chmod 755 $out/bin/docspell-joex
'';
};
}

84
nix/test-vm.nix Normal file
View File

@ -0,0 +1,84 @@
{
config,
pkgs,
...
}: let
full-text-search = {
enabled = true;
backend = "solr";
solr.url = "http://localhost:8983/solr/docspell";
};
jdbc = {
url = "jdbc:postgresql://localhost:5432/docspell";
user = "dev";
password = "dev";
};
in {
services.dev-postgres = {
enable = true;
databases = ["docspell"];
};
services.dev-email.enable = true;
services.dev-solr = {
enable = true;
cores = ["docspell"];
};
port-forward.dev-webmail = 8080;
port-forward.dev-solr = 8983;
environment.systemPackages = with pkgs; [
jq
htop
iotop
coreutils
];
networking = {
hostName = "docspell-test-vm";
firewall.allowedTCPPorts = [7880];
};
virtualisation.memorySize = 6144;
virtualisation.forwardPorts = [
{
from = "host";
host.port = 7881;
guest.port = 7880;
}
];
services.docspell-restserver = {
enable = true;
bind.address = "0.0.0.0";
backend = {
addons.enabled = true;
signup.mode = "open";
inherit jdbc;
};
integration-endpoint = {
enabled = true;
http-header = {
enabled = true;
header-value = "test123";
};
};
admin-endpoint = {
secret = "admin123";
};
inherit full-text-search;
};
services.docspell-joex = {
enable = true;
bind.address = "0.0.0.0";
inherit jdbc full-text-search;
addons = {
executor-config = {
runner = "nix-flake,trivial";
nspawn.enabled = true;
};
};
};
}

View File

@ -1,23 +0,0 @@
let
nixpkgs = builtins.fetchTarball {
#url = "https://github.com/NixOS/nixpkgs/archive/92e990a8d6bc35f1089c76dd8ba68b78da90da59.tar.gz";
url = "channel:nixos-23.05";
};
pkgs = import nixpkgs { };
initScript = pkgs.writeScript "docspell-build-init" ''
export LD_LIBRARY_PATH=
${pkgs.bash}/bin/bash -c "sbt -mem 4096 -java-home ${pkgs.openjdk17}/lib/openjdk"
'';
in with pkgs;
buildFHSUserEnv {
name = "docspell-sbt";
targetPkgs = pkgs: with pkgs; [
netcat jdk17 wget which dpkg sbt git elmPackages.elm ncurses fakeroot mc
zola yarn
# haskells http client needs this (to download elm packages)
iana-etc
];
runScript = initScript;
}

View File

@ -15,19 +15,20 @@ Sbt is used to build the site.
Install things by running `yarn install`.
Open terminal for each script below:
Use a dev [environment](https://docspell.org/docs/dev/development/)
and open terminal for each script below:
1. Starting the server
``` shell
nix-shell --run "cd site && zola serve"
cd site && zola serve
```
2. Building the stylesheet
``` shell
nix-shell --run ./scripts/run-styles.sh
./scripts/run-styles.sh
```
3. Building some javascript files
``` shell
nix-shell --run ./scripts/run-elm.sh
./scripts/run-elm.sh
```
Open browser at `localhost:1111`.
@ -40,7 +41,7 @@ be finally deployed. To see this, start sbt and change into the
website project.
``` shell
nix-shell website/shell.nix --run sbt
$ sbt
sbt> project website
```

View File

@ -37,7 +37,7 @@ git clone git@github.com:eikek/docspell.git "$temp"
cd "$temp" && git checkout --track origin/gh-pages && rm -rf "$temp"/*
echo "Create new website from current working directory"
cd $cdir && nix-shell website/shell.nix --run 'sbt make-website'
cd $cdir && sbt make-website
echo "Copying new site to target"
cp -R "$cdir"/website/target/zola-site/* "$temp/"

View File

@ -1,20 +0,0 @@
let
nixpkgs = builtins.fetchTarball {
url = "channel:nixos-23.05";
#url = "https://github.com/NixOS/nixpkgs/archive/e6badb26fc0d238fda2432c45b7dd4e782eb8200.tar.gz";
#url = "https://github.com/NixOs/nixpkgs/archive/0f316e4d72daed659233817ffe52bf08e081b5de.tar.gz"; #21.11
};
pkgs = import nixpkgs { };
in
with pkgs;
mkShell {
buildInputs = [
zola
yarn
sbt
elmPackages.elm
nodejs
inotifyTools
];
}

View File

@ -20,6 +20,7 @@ highlight_themes_css = [
]
[link_checker]
external_level = "warn"
skip_prefixes = [
"http://localhost",
"https://docs.github.com", # doesn't allow checking anymore

View File

@ -63,11 +63,11 @@ you can look at the results while writing.
If you want to see a live view of the page while editing, some tools
are required. The easiest way to get these is to install
[nix](https://nixos.org/) and run `nix-shell website/shell.nix` to get
an environment with all these tools installed. Otherwise install the
programs mentioned in `website/shell.nix`, which are:
[yarn](https://yarnpkg.com/), [zola](https://www.getzola.org/),
[elm](https://elm-lang.org) and [sbt](https://scala-sbt.org).
[nix](https://nixos.org/) and run `nix develop .#dev-vm` to get an
environment with all these tools installed. Otherwise install the
programs: [yarn](https://yarnpkg.com/),
[zola](https://www.getzola.org/), [elm](https://elm-lang.org) and
[sbt](https://scala-sbt.org).
Then clone the sources to your machine and build the complete site
once, so that all assets and required stuff is present:

View File

@ -65,6 +65,159 @@ docspell.joex {
}
```
## Installing Nix
It is recommended to install [nix](https://nixos.org/nix). You can use
the official installer or [this
one](https://github.com/DeterminateSystems/nix-installer), which will
enable Flakes by default.
If not enabled, enable flakes by creating a config file:
```
mkdir -p ~/.config/nix #on Linux
echo "experimental-features = nix-command flakes" >> ~/.config/nix/nix.conf
```
With nix installed you can use the provided development environments
to get started quickly.
# Nix Expressions
The soure root contains a `flake.nix` file to install docspell via the
nix package manager and to integrate it into NixOS.
The flake provides docspell packages of the latest release and NixOS
modules. It doesn't provide package builds from the current source
tree.
## Dev Environments
Additionally it provides devshells that can be used to create a
development environment for docspell.
These two `devShell` definitions address two different setups: one
uses a NixOS container and the other a VM. Both provide the same set
of services that can be used with the local docspell instance:
- postgresql database, with a database `docspell` and user `dev`
- solr with a core `docspell`
- email setup with smtp/imap and webmail
- minio with root user `minioadmin`
If you are on NixOS the container is probably more convenient to use.
For other systems, the vm should be good. Drop into either shell by
running:
``` bash
# drop into the environment setup for using a vm
nix develop .#dev-vm
# drop into the environment setup for using a container
nix deveop .#docsp-dev
```
Once in such an environment, you can create the container or vm like
this:
```bash
# dev-vm env
# - build the vm
vm-build
# -run the vm
vm-run
# - ssh into the vm
vm-ssh
# docsp-dev container env
# - create the container
cnt-recreate
# - login
cnt-login
```
You can use tab completion on `vm-` or `cnt-` and see other useful
commands. For instance it allows to recreate solr cores or check logs
of services on the container or vm.
Then you can adjust your dev config file in `local/dev.conf` to
connect to the services on the vm or container. The container exposes
the default ports while the vm uses port-forwarding from the host to
the guest machine. The ports are define in `flake.nix`. For example, a
jdbc connection to postgres on the container can look like this:
```
jdbc.url = "jdbc:postgresql://docsp-dev:5432/docspelldev"
```
on the vm, it would be
```
jdbc.url = "jdbc:postgresql://localhost:6543/docspelldev"
```
You can reach the webmail on both versions at port `8080`. In order to
enable sending mails between users, you need to login as some
arbitrary user so the underlying services can create the data
directories. In your dev docspell you can then connect to smtp on the
vm or container. Mails send from docspell can be checked in the
webmail. Conversely, you can send mails using webmail to any user and
have their mailbox scanned by docspell.
### Direnv
Using [direnv](https://direnv.net) entering the dev environment is
very convenient. Install this tool (it also has integration in several
IDEs and editors) and create a file `.envrc` in the source root:
```
use flake .#<env-name>
```
The file `.envrc` is git-ignored, because are different ones possible.
Here `<env-name>` refers to either `dev-cnt` or `dev-vm` - one of the
devshells defined in `flake.nix`.
After allowing direnv to execute this file via `direnv allow` you will
be dropped into this environment whenever you enter the directory. It
will also preserve your shell, don't need to use bash.
## Checks
The command `nix flake check` would run all checks defined in the
flake. It will build both packages and runs a vm with docspell
installed (via NixOS modules) and check whether the services come up.
## Test VM
There is another VM defined in the flake that provides a full setup
for docspell. It contains docspell server and joex, a postgresql, a
solr and a email setup. The intention is to use it as an easy 'getting
started' approach with nix.
Once it has started, you can connect to `localhost:7881` to reach
docspell. The webmail will be available at `localhost:8080`.
You can run this vm with a single command:
```
nix run github:eikek/docspell#nixosConfigurations.test-vm.config.system.build.vm
```
It uses the same setup as the dev-vm, so you can drop into the
`.#dev-vm` development shell and use `vm-ssh` to connect to the
running test vm.
Once connected to the machine, you can see the docspell config file via
```bash
systemctl show docspell-joex.service | grep "ExecStart=" | sed 's/^ExecStart=.*path=\([^;]*\).*/\1/' | xargs tac | grep -m 1 . | awk '{print $NF}' | tr -d '"' | xargs jq '.'
# or replace "joex" with "restserver"
systemctl show docspell-restserver.service | grep "ExecStart=" | sed 's/^ExecStart=.*path=\([^;]*\).*/\1/' | xargs tac | grep -m 1 . | awk '{print $NF}' | tr -d '"' | xargs jq '.'
```
# Developing Frontend
The frontend is a SPA written in [Elm](https://elm-lang.org). The UI
@ -124,39 +277,6 @@ These tasks must not show any errors (it is checked by the CI). The
warnings should also be fixed.
# Nix Expressions
The directory `/nix` contains Nix Flake to install docspell via
the nix package manager and to integrate it into NixOS.
Flake implements `checks` output which can be run with `nix flake check`
and it defines a development VM which can be used to interactively work
with docspell.
To run the VM, issue:
```bash
cd $PROJECT_ROOT/nix
nix run '.#nixosConfigurations.dev-vm.config.system.build.vm
```
To open docspell, wait for docspell-restserver service to report that
http listener is up and connect to `localhost:64080`.
To ssh into the machine, run:
```bash
ssh -o StrictHostKeyChecking=no \
-o UserKnownHostsFile=/dev/null \
-p 64022 root@localhost
```
Once connected to the machine, you can see the docspell config file via
```bash
systemd-show docspell-joex.service | grep ExecStart | cut -d'=' -f2 | xargs cat | tail -n1 | awk '{print $NF}'| sed 's/.$//' | xargs cat | jq
```
# Release
The CI and making a release is done via github actions. The workflow

View File

@ -5,12 +5,50 @@ weight = 24
# Nix
Docspell is a flake, you need to enable flakes in order to make use of
it. You can also use the provided expressions without Flakes, which is
described below.
## Try it out {try-it-out}
You can try out the server and joex packages by running the following:
```
nix run github:eikek/docspell#docspell-restserver
nix run github:eikek/docspell#docspell-joex
```
While this works, it will be only a very basic setup. The database
defaults to a directory in `/tmp` and no fulltext search enabled. Then
for processing documents, some external tools are required which would
need to be present on yout system to make it work.
A more elaborate setup with PostgreSQL and SOLR can be started using
the `test-vm`:
```
nix run github:eikek/docspell#nixosConfigurations.test-vm.config.system.build.vm
```
The vm contains all the required tools. After starting up, you can
find docspell at `http://localhost:7881`.
## Install via Nix
Docspell can be installed via the [nix](https://nixos.org/nix) package
manager, which is available for Linux and OSX. Docspell is currently not
part of the [nixpkgs collection](https://nixos.org/nixpkgs/), but you
can use the flake from this repository.
manager. Docspell is currently not part of the [nixpkgs
collection](https://nixos.org/nixpkgs/), but you can use the flake
from this repository.
You could install the server and joex by running the following:
```
nix profile install github:eikek/docspell#docspell-restserver
nix profile install github:eikek/docspell#docspell-joex
```
This would install the packages on your system. If you use NixOS you
probably want to use the provided [NixOS modules](#nixos) instead.
## Upgrading
@ -31,7 +69,7 @@ There are the following modules provided:
```nix
# flake.nix
inputs.docspell.url = "github:eikek/docspell?dir=nix/";
inputs.docspell.url = "github:eikek/docspell";
# in modules
imports = [
@ -134,10 +172,37 @@ inputs.docspell.url = "github:eikek/docspell?dir=nix/";
'';
};
networking = {
hostName = "docspellexample";
firewall.allowedTCPPorts = [7880];
};
}
```
You can also look at `nix/test-vm.nix` for another example.
## Without Flakes
Of course, you can also use it without flakes. There is `nix/pkg.nix`
which contains the derivation of both packages, `docspell-restserver` and
`docspell-joex`. Just call it with your nixpkgs instance as usual:
``` nix
let
repo = fetchurl {
url = "https://github.com/eikek/docspell";
sha256 = "sha256-X2mM+Z5s8Xm1E6zrZ0wcRaivLEvqbk5Dn+GSXkZHdLM=";
};
docspellPkgs = pkgs.callPackage (import "${repo}/nix/pkg.nix") {};
in
#
# use docspellPkgs.docspell-restserver or docspellPkgs.docspell-joex
#
```
The same way import NixOS modules from
`nix/modules/{joex|server}.nix`.
An alternative can be to use `builtins.getFlake` to fetch the flake
and get access to its output. But this requires to use a flake enabled
nix, which then defeats the idea of "not using flakes".

View File

@ -20,10 +20,10 @@ To get started, here are some quick links:
applies to the `zip` and `deb` files. The files can be downloaded
from the [release
page](https://github.com/eikek/docspell/releases/latest).
- via the [nix package manager](@/docs/install/nix.md) and/or as a
[NixOS module](@/docs/install/nix.md#nixos) through a flake.
If you use nix/nixos, you know what to do. The linked page contains
some examples.
- via the [nix package manager](@/docs/install/nix.md#try-it-out)
and/or as a [NixOS module](@/docs/install/nix.md#nixos) through a
flake. If you use nix/nixos, you know what to do. The linked page
contains some examples.
- [Unraid](https://www.unraid.net/): There are user provided [notes
and unraid
templates](https://github.com/vakilando/unraid-docker-templates)