diff --git a/.gitignore b/.gitignore index 233e2fa2..05afc7d2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,15 @@ #artwork/*.png target/ +node_modules/ dev.conf elm-stuff result _site/ -*.qcow2 \ No newline at end of file +*.qcow2 +/website/site/content/docs/changelog/ +/website/site/public/ +/website/site/static/openapi/ +/website/site/static/js/bundle.js +/website/site/templates/shortcodes/server.conf +/website/site/templates/shortcodes/sample-exim.conf +/website/site/templates/shortcodes/joex.conf diff --git a/artwork/poster.svg b/artwork/poster.svg new file mode 100644 index 00000000..871be67d --- /dev/null +++ b/artwork/poster.svg @@ -0,0 +1,200 @@ + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + Docspell + Simple Document Organizer + + + diff --git a/artwork/search.svg b/artwork/search.svg deleted file mode 100644 index 9a620ee3..00000000 --- a/artwork/search.svg +++ /dev/null @@ -1,159 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - diff --git a/artwork/stow.svg b/artwork/stow.svg deleted file mode 100644 index d53b8cb9..00000000 --- a/artwork/stow.svg +++ /dev/null @@ -1,161 +0,0 @@ - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/artwork/tag.svg b/artwork/tag.svg deleted file mode 100644 index 3809c0dc..00000000 --- a/artwork/tag.svg +++ /dev/null @@ -1,90 +0,0 @@ - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - diff --git a/build.sbt b/build.sbt index 62a8fe5e..7b9d8abf 100644 --- a/build.sbt +++ b/build.sbt @@ -1,18 +1,16 @@ import com.github.eikek.sbt.openapi._ import scala.sys.process._ +import com.typesafe.sbt.site.SitePlugin import com.typesafe.sbt.SbtGit.GitKeys._ import docspell.build._ -import microsites.ExtraMdFileConfig -val toolsPackage = taskKey[Seq[File]]("Package the scripts/extension tools") +val toolsPackage = taskKey[Seq[File]]("Package the scripts/extension tools") val elmCompileMode = settingKey[ElmCompileMode]("How to compile elm sources") - - // --- Settings val scalafixSettings = Seq( - semanticdbEnabled := true, // enable SemanticDB + semanticdbEnabled := true, // enable SemanticDB semanticdbVersion := scalafixSemanticdb.revision, //"4.3.10", // use Scalafix compatible version ThisBuild / scalafixDependencies ++= Dependencies.organizeImports ) @@ -22,7 +20,8 @@ val sharedSettings = Seq( scalaVersion := "2.13.2", scalacOptions ++= Seq( "-deprecation", - "-encoding", "UTF-8", + "-encoding", + "UTF-8", "-language:higherKinds", "-feature", "-Werror", // fail when there are warnings @@ -33,10 +32,10 @@ val sharedSettings = Seq( "-Wvalue-discard", "-Wnumeric-widen" ), - LocalRootProject/toolsPackage := { - val v = version.value + LocalRootProject / toolsPackage := { + val v = version.value val logger = streams.value.log - val dir = (LocalRootProject/baseDirectory).value / "tools" + val dir = (LocalRootProject / baseDirectory).value / "tools" packageTools(logger, dir, v) }, scalacOptions in (Compile, console) := @@ -58,135 +57,173 @@ lazy val noPublish = Seq( val elmSettings = Seq( elmCompileMode := ElmCompileMode.Debug, - Compile/resourceGenerators += Def.task { - compileElm(streams.value.log - , (Compile/baseDirectory).value - , (Compile/resourceManaged).value - , name.value - , version.value - , elmCompileMode.value) + Compile / resourceGenerators += Def.task { + compileElm( + streams.value.log, + (Compile / baseDirectory).value, + (Compile / resourceManaged).value, + name.value, + version.value, + elmCompileMode.value + ) }.taskValue, watchSources += Watched.WatchSource( - (Compile/sourceDirectory).value/"elm" - , FileFilter.globFilter("*.elm") - , HiddenFileFilter + (Compile / sourceDirectory).value / "elm", + FileFilter.globFilter("*.elm"), + HiddenFileFilter ) ) val webjarSettings = Seq( - Compile/resourceGenerators += Def.task { - copyWebjarResources(Seq((sourceDirectory in Compile).value/"webjar") - , (Compile/resourceManaged).value - , name.value - , version.value - , streams.value.log + Compile / resourceGenerators += Def.task { + copyWebjarResources( + Seq((sourceDirectory in Compile).value / "webjar"), + (Compile / resourceManaged).value, + name.value, + version.value, + streams.value.log ) }.taskValue, watchSources += Watched.WatchSource( - (Compile / sourceDirectory).value/"webjar" - , FileFilter.globFilter("*.js") || FileFilter.globFilter("*.css") - , HiddenFileFilter + (Compile / sourceDirectory).value / "webjar", + FileFilter.globFilter("*.js") || FileFilter.globFilter("*.css"), + HiddenFileFilter ) ) -def debianSettings(cfgFile: String) = Seq( - maintainer := "Eike Kettner ", - packageSummary := description.value, - packageDescription := description.value, - mappings in Universal += { - val conf = (Compile / resourceDirectory).value / "reference.conf" - if (!conf.exists) { - sys.error(s"File $conf not found") - } - conf -> s"conf/$cfgFile.conf" - }, - bashScriptExtraDefines += s"""addJava "-Dconfig.file=$${app_home}/../conf/$cfgFile.conf"""" -) +def debianSettings(cfgFile: String) = + Seq( + maintainer := "Eike Kettner ", + packageSummary := description.value, + packageDescription := description.value, + mappings in Universal += { + val conf = (Compile / resourceDirectory).value / "reference.conf" + if (!conf.exists) + sys.error(s"File $conf not found") + conf -> s"conf/$cfgFile.conf" + }, + bashScriptExtraDefines += s"""addJava "-Dconfig.file=$${app_home}/../conf/$cfgFile.conf"""" + ) val buildInfoSettings = Seq( - buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion, gitHeadCommit, gitHeadCommitDate, gitUncommittedChanges, gitDescribedVersion), + buildInfoKeys := Seq[BuildInfoKey]( + name, + version, + scalaVersion, + sbtVersion, + gitHeadCommit, + gitHeadCommitDate, + gitUncommittedChanges, + gitDescribedVersion + ), buildInfoOptions += BuildInfoOption.ToJson, buildInfoOptions += BuildInfoOption.BuildTime ) val openapiScalaSettings = Seq( - openapiScalaConfig := ScalaConfig().withJson(ScalaJson.circeSemiauto). - addMapping(CustomMapping.forType({ - case TypeDef("LocalDateTime", _) => + openapiScalaConfig := ScalaConfig() + .withJson(ScalaJson.circeSemiauto) + .addMapping(CustomMapping.forType({ + case TypeDef("LocalDateTime", _) => TypeDef("Timestamp", Imports("docspell.common.Timestamp")) - })). - addMapping(CustomMapping.forFormatType({ - case "ident" => field => - field.copy(typeDef = TypeDef("Ident", Imports("docspell.common.Ident"))) - case "collectivestate" => field => - field.copy(typeDef = TypeDef("CollectiveState", Imports("docspell.common.CollectiveState"))) - case "userstate" => field => - field.copy(typeDef = TypeDef("UserState", Imports("docspell.common.UserState"))) - case "password" => field => - field.copy(typeDef = TypeDef("Password", Imports("docspell.common.Password"))) - case "contactkind" => field => - field.copy(typeDef = TypeDef("ContactKind", Imports("docspell.common.ContactKind"))) - case "direction" => field => - field.copy(typeDef = TypeDef("Direction", Imports("docspell.common.Direction"))) - case "priority" => field => - field.copy(typeDef = TypeDef("Priority", Imports("docspell.common.Priority"))) - case "jobstate" => field => - field.copy(typeDef = TypeDef("JobState", Imports("docspell.common.JobState"))) - case "loglevel" => field => - field.copy(typeDef = TypeDef("LogLevel", Imports("docspell.common.LogLevel"))) - case "mimetype" => field => - field.copy(typeDef = TypeDef("MimeType", Imports("docspell.common.MimeType"))) - case "itemstate" => field => - field.copy(typeDef = TypeDef("ItemState", Imports("docspell.common.ItemState"))) - case "nertag" => field => - field.copy(typeDef = TypeDef("NerTag", Imports("docspell.common.NerTag"))) - case "language" => field => - field.copy(typeDef = TypeDef("Language", Imports("docspell.common.Language"))) - case "calevent" => field => - field.copy(typeDef = TypeDef("CalEvent", Imports("com.github.eikek.calev.CalEvent", - "com.github.eikek.calev.circe.CalevCirceCodec._"))) + })) + .addMapping(CustomMapping.forFormatType({ + case "ident" => + field => field.copy(typeDef = TypeDef("Ident", Imports("docspell.common.Ident"))) + case "collectivestate" => + field => + field.copy(typeDef = + TypeDef("CollectiveState", Imports("docspell.common.CollectiveState")) + ) + case "userstate" => + field => + field.copy(typeDef = TypeDef("UserState", Imports("docspell.common.UserState"))) + case "password" => + field => + field.copy(typeDef = TypeDef("Password", Imports("docspell.common.Password"))) + case "contactkind" => + field => + field.copy(typeDef = + TypeDef("ContactKind", Imports("docspell.common.ContactKind")) + ) + case "direction" => + field => + field.copy(typeDef = TypeDef("Direction", Imports("docspell.common.Direction"))) + case "priority" => + field => + field.copy(typeDef = TypeDef("Priority", Imports("docspell.common.Priority"))) + case "jobstate" => + field => + field.copy(typeDef = TypeDef("JobState", Imports("docspell.common.JobState"))) + case "loglevel" => + field => + field.copy(typeDef = TypeDef("LogLevel", Imports("docspell.common.LogLevel"))) + case "mimetype" => + field => + field.copy(typeDef = TypeDef("MimeType", Imports("docspell.common.MimeType"))) + case "itemstate" => + field => + field.copy(typeDef = TypeDef("ItemState", Imports("docspell.common.ItemState"))) + case "nertag" => + field => + field.copy(typeDef = TypeDef("NerTag", Imports("docspell.common.NerTag"))) + case "language" => + field => + field.copy(typeDef = TypeDef("Language", Imports("docspell.common.Language"))) + case "calevent" => + field => + field.copy(typeDef = + TypeDef( + "CalEvent", + Imports( + "com.github.eikek.calev.CalEvent", + "com.github.eikek.calev.circe.CalevCirceCodec._" + ) + ) + ) })) ) - // --- Modules // Base module, everything depends on this – including restapi and // joexapi modules. This should aim to have least possible // dependencies -val common = project.in(file("modules/common")). - disablePlugins(RevolverPlugin). - settings(sharedSettings). - settings(testSettings). - settings( +val common = project + .in(file("modules/common")) + .disablePlugins(RevolverPlugin) + .settings(sharedSettings) + .settings(testSettings) + .settings( name := "docspell-common", libraryDependencies ++= Dependencies.fs2 ++ - Dependencies.circe ++ - Dependencies.loggingApi ++ - Dependencies.calevCore ++ - Dependencies.calevCirce ++ - Dependencies.pureconfig.map(_ % "optional") + Dependencies.circe ++ + Dependencies.loggingApi ++ + Dependencies.calevCore ++ + Dependencies.calevCirce ++ + Dependencies.pureconfig.map(_ % "optional") ) // Some example files for testing // https://file-examples.com/index.php/sample-documents-download/sample-doc-download/ -val files = project.in(file("modules/files")). - disablePlugins(RevolverPlugin). - settings(sharedSettings). - settings(testSettings). - settings( +val files = project + .in(file("modules/files")) + .disablePlugins(RevolverPlugin) + .settings(sharedSettings) + .settings(testSettings) + .settings( name := "docspell-files", libraryDependencies ++= Dependencies.tika ++ - Dependencies.icu4j, + Dependencies.icu4j, Test / sourceGenerators += Def.task { - val base = (Test/resourceDirectory).value - val files = (base ** (_.isFile)) pair sbt.io.Path.relativeTo(base) - val lines = files.toList.map(_._2).map(s => { + val base = (Test / resourceDirectory).value + val files = (base ** (_.isFile)).pair(sbt.io.Path.relativeTo(base)) + val lines = files.toList.map(_._2).map { s => val ident = s.replaceAll("[^a-zA-Z0-9_]+", "_") ident -> s"""val $ident = createUrl("${s}")""" - }) + } val content = s"""package docspell.files object ExampleFiles extends ExampleFilesSupport { @@ -199,340 +236,355 @@ ${lines.map(_._1).mkString(",\n")} } """ - val target = (Test/sourceManaged).value/"scala"/"ExampleFiles.scala" + val target = (Test / sourceManaged).value / "scala" / "ExampleFiles.scala" IO.createDirectory(target.getParentFile) IO.write(target, content) Seq(target) }.taskValue - ).dependsOn(common) + ) + .dependsOn(common) -val store = project.in(file("modules/store")). - disablePlugins(RevolverPlugin). - settings(sharedSettings). - settings(testSettings). - settings( +val store = project + .in(file("modules/store")) + .disablePlugins(RevolverPlugin) + .settings(sharedSettings) + .settings(testSettings) + .settings( name := "docspell-store", libraryDependencies ++= Dependencies.doobie ++ - Dependencies.bitpeace ++ - Dependencies.tika ++ - Dependencies.fs2 ++ - Dependencies.databases ++ - Dependencies.flyway ++ - Dependencies.loggingApi ++ - Dependencies.emil ++ - Dependencies.emilDoobie ++ - Dependencies.calevCore ++ - Dependencies.calevFs2 - ).dependsOn(common) + Dependencies.bitpeace ++ + Dependencies.tika ++ + Dependencies.fs2 ++ + Dependencies.databases ++ + Dependencies.flyway ++ + Dependencies.loggingApi ++ + Dependencies.emil ++ + Dependencies.emilDoobie ++ + Dependencies.calevCore ++ + Dependencies.calevFs2 + ) + .dependsOn(common) -val extract = project.in(file("modules/extract")). - disablePlugins(RevolverPlugin). - settings(sharedSettings). - settings(testSettings). - settings( +val extract = project + .in(file("modules/extract")) + .disablePlugins(RevolverPlugin) + .settings(sharedSettings) + .settings(testSettings) + .settings( name := "docspell-extract", libraryDependencies ++= Dependencies.fs2 ++ - Dependencies.twelvemonkeys ++ - Dependencies.pdfbox ++ - Dependencies.poi ++ - Dependencies.commonsIO ++ - Dependencies.julOverSlf4j - ).dependsOn(common, files % "compile->compile;test->test") + Dependencies.twelvemonkeys ++ + Dependencies.pdfbox ++ + Dependencies.poi ++ + Dependencies.commonsIO ++ + Dependencies.julOverSlf4j + ) + .dependsOn(common, files % "compile->compile;test->test") -val convert = project.in(file("modules/convert")). - disablePlugins(RevolverPlugin). - settings(sharedSettings). - settings(testSettings). - settings( +val convert = project + .in(file("modules/convert")) + .disablePlugins(RevolverPlugin) + .settings(sharedSettings) + .settings(testSettings) + .settings( name := "docspell-convert", libraryDependencies ++= Dependencies.flexmark ++ - Dependencies.twelvemonkeys - ).dependsOn(common, files % "compile->compile;test->test") + Dependencies.twelvemonkeys + ) + .dependsOn(common, files % "compile->compile;test->test") -val analysis = project.in(file("modules/analysis")). - disablePlugins(RevolverPlugin). - enablePlugins(NerModelsPlugin). - settings(sharedSettings). - settings(testSettings). - settings(NerModelsPlugin.nerClassifierSettings). - settings( +val analysis = project + .in(file("modules/analysis")) + .disablePlugins(RevolverPlugin) + .enablePlugins(NerModelsPlugin) + .settings(sharedSettings) + .settings(testSettings) + .settings(NerModelsPlugin.nerClassifierSettings) + .settings( name := "docspell-analysis", libraryDependencies ++= Dependencies.fs2 ++ - Dependencies.stanfordNlpCore - ).dependsOn(common, files % "test->test") + Dependencies.stanfordNlpCore + ) + .dependsOn(common, files % "test->test") -val ftsclient = project.in(file("modules/fts-client")). - disablePlugins(RevolverPlugin). - settings(sharedSettings). - settings(testSettings). - settings( +val ftsclient = project + .in(file("modules/fts-client")) + .disablePlugins(RevolverPlugin) + .settings(sharedSettings) + .settings(testSettings) + .settings( name := "docspell-fts-client", libraryDependencies ++= Seq.empty - ).dependsOn(common) + ) + .dependsOn(common) -val ftssolr = project.in(file("modules/fts-solr")). - disablePlugins(RevolverPlugin). - settings(sharedSettings). - settings(testSettings). - settings( +val ftssolr = project + .in(file("modules/fts-solr")) + .disablePlugins(RevolverPlugin) + .settings(sharedSettings) + .settings(testSettings) + .settings( name := "docspell-fts-solr", libraryDependencies ++= Dependencies.http4sClient ++ - Dependencies.http4sCirce ++ - Dependencies.http4sDsl ++ - Dependencies.circe - ).dependsOn(common, ftsclient) - -val restapi = project.in(file("modules/restapi")). - disablePlugins(RevolverPlugin). - enablePlugins(OpenApiSchema). - settings(sharedSettings). - settings(testSettings). - settings(openapiScalaSettings). - settings( + Dependencies.http4sCirce ++ + Dependencies.http4sDsl ++ + Dependencies.circe + ) + .dependsOn(common, ftsclient) + +val restapi = project + .in(file("modules/restapi")) + .disablePlugins(RevolverPlugin) + .enablePlugins(OpenApiSchema) + .settings(sharedSettings) + .settings(testSettings) + .settings(openapiScalaSettings) + .settings( name := "docspell-restapi", libraryDependencies ++= Dependencies.circe, openapiTargetLanguage := Language.Scala, openapiPackage := Pkg("docspell.restapi.model"), - openapiSpec := (Compile/resourceDirectory).value/"docspell-openapi.yml", - ).dependsOn(common) + openapiSpec := (Compile / resourceDirectory).value / "docspell-openapi.yml", + openapiStaticArgs := Seq("-l", "html2") + ) + .dependsOn(common) -val joexapi = project.in(file("modules/joexapi")). - disablePlugins(RevolverPlugin). - enablePlugins(OpenApiSchema). - settings(sharedSettings). - settings(testSettings). - settings(openapiScalaSettings). - settings( +val joexapi = project + .in(file("modules/joexapi")) + .disablePlugins(RevolverPlugin) + .enablePlugins(OpenApiSchema) + .settings(sharedSettings) + .settings(testSettings) + .settings(openapiScalaSettings) + .settings( name := "docspell-joexapi", libraryDependencies ++= Dependencies.circe ++ - Dependencies.http4sCirce ++ - Dependencies.http4sClient, + Dependencies.http4sCirce ++ + Dependencies.http4sClient, openapiTargetLanguage := Language.Scala, openapiPackage := Pkg("docspell.joexapi.model"), - openapiSpec := (Compile/resourceDirectory).value/"joex-openapi.yml" - ).dependsOn(common) + openapiSpec := (Compile / resourceDirectory).value / "joex-openapi.yml" + ) + .dependsOn(common) -val backend = project.in(file("modules/backend")). - disablePlugins(RevolverPlugin). - settings(sharedSettings). - settings(testSettings). - settings( +val backend = project + .in(file("modules/backend")) + .disablePlugins(RevolverPlugin) + .settings(sharedSettings) + .settings(testSettings) + .settings( name := "docspell-backend", libraryDependencies ++= Dependencies.loggingApi ++ - Dependencies.fs2 ++ - Dependencies.bcrypt ++ - Dependencies.http4sClient ++ - Dependencies.emil - ).dependsOn(store, joexapi, ftsclient) + Dependencies.fs2 ++ + Dependencies.bcrypt ++ + Dependencies.http4sClient ++ + Dependencies.emil + ) + .dependsOn(store, joexapi, ftsclient) -val webapp = project.in(file("modules/webapp")). - disablePlugins(RevolverPlugin). - enablePlugins(OpenApiSchema). - settings(sharedSettings). - settings(elmSettings). - settings(webjarSettings). - settings( +val webapp = project + .in(file("modules/webapp")) + .disablePlugins(RevolverPlugin) + .enablePlugins(OpenApiSchema) + .settings(sharedSettings) + .settings(elmSettings) + .settings(webjarSettings) + .settings( name := "docspell-webapp", openapiTargetLanguage := Language.Elm, openapiPackage := Pkg("Api.Model"), - openapiSpec := (restapi/Compile/resourceDirectory).value/"docspell-openapi.yml", + openapiSpec := (restapi / Compile / resourceDirectory).value / "docspell-openapi.yml", openapiElmConfig := ElmConfig().withJson(ElmJson.decodePipeline) ) - - // --- Application(s) -val joex = project.in(file("modules/joex")). - enablePlugins(BuildInfoPlugin - , JavaServerAppPackaging - , DebianPlugin - , SystemdPlugin). - settings(sharedSettings). - settings(testSettings). - settings(debianSettings("docspell-joex")). - settings(buildInfoSettings). - settings( +val joex = project + .in(file("modules/joex")) + .enablePlugins(BuildInfoPlugin, JavaServerAppPackaging, DebianPlugin, SystemdPlugin) + .settings(sharedSettings) + .settings(testSettings) + .settings(debianSettings("docspell-joex")) + .settings(buildInfoSettings) + .settings( name := "docspell-joex", libraryDependencies ++= Dependencies.fs2 ++ - Dependencies.http4sServer ++ - Dependencies.http4sCirce ++ - Dependencies.http4sDsl ++ - Dependencies.circe ++ - Dependencies.pureconfig ++ - Dependencies.emilTnef ++ - Dependencies.emilMarkdown ++ - Dependencies.emilJsoup ++ - Dependencies.jsoup ++ - Dependencies.yamusca ++ - Dependencies.loggingApi ++ - Dependencies.logging.map(_ % Runtime), + Dependencies.http4sServer ++ + Dependencies.http4sCirce ++ + Dependencies.http4sDsl ++ + Dependencies.circe ++ + Dependencies.pureconfig ++ + Dependencies.emilTnef ++ + Dependencies.emilMarkdown ++ + Dependencies.emilJsoup ++ + Dependencies.jsoup ++ + Dependencies.yamusca ++ + Dependencies.loggingApi ++ + Dependencies.logging.map(_ % Runtime), addCompilerPlugin(Dependencies.kindProjectorPlugin), addCompilerPlugin(Dependencies.betterMonadicFor), buildInfoPackage := "docspell.joex", - reStart/javaOptions ++= Seq(s"-Dconfig.file=${(LocalRootProject/baseDirectory).value/"local"/"dev.conf"}") - ).dependsOn(store, backend, extract, convert, analysis, joexapi, restapi, ftssolr) + reStart / javaOptions ++= Seq( + s"-Dconfig.file=${(LocalRootProject / baseDirectory).value / "local" / "dev.conf"}" + ) + ) + .dependsOn(store, backend, extract, convert, analysis, joexapi, restapi, ftssolr) -val restserver = project.in(file("modules/restserver")). - enablePlugins(BuildInfoPlugin - , JavaServerAppPackaging - , DebianPlugin - , SystemdPlugin). - settings(sharedSettings). - settings(testSettings). - settings(debianSettings("docspell-server")). - settings(buildInfoSettings). - settings( +val restserver = project + .in(file("modules/restserver")) + .enablePlugins(BuildInfoPlugin, JavaServerAppPackaging, DebianPlugin, SystemdPlugin) + .settings(sharedSettings) + .settings(testSettings) + .settings(debianSettings("docspell-server")) + .settings(buildInfoSettings) + .settings( name := "docspell-restserver", libraryDependencies ++= Dependencies.http4sServer ++ - Dependencies.http4sCirce ++ - Dependencies.http4sDsl ++ - Dependencies.circe ++ - Dependencies.pureconfig ++ - Dependencies.yamusca ++ - Dependencies.webjars ++ - Dependencies.loggingApi ++ - Dependencies.logging.map(_ % Runtime), + Dependencies.http4sCirce ++ + Dependencies.http4sDsl ++ + Dependencies.circe ++ + Dependencies.pureconfig ++ + Dependencies.yamusca ++ + Dependencies.webjars ++ + Dependencies.loggingApi ++ + Dependencies.logging.map(_ % Runtime), addCompilerPlugin(Dependencies.kindProjectorPlugin), addCompilerPlugin(Dependencies.betterMonadicFor), buildInfoPackage := "docspell.restserver", - Compile/sourceGenerators += Def.task { - createWebjarSource(Dependencies.webjars, (Compile/sourceManaged).value) + Compile / sourceGenerators += Def.task { + createWebjarSource(Dependencies.webjars, (Compile / sourceManaged).value) }.taskValue, - Compile/resourceGenerators += Def.task { - copyWebjarResources(Seq((restapi/Compile/resourceDirectory).value/"docspell-openapi.yml") - , (Compile/resourceManaged).value - , name.value - , version.value - , streams.value.log) + Compile / resourceGenerators += Def.task { + copyWebjarResources( + Seq((restapi / Compile / resourceDirectory).value / "docspell-openapi.yml"), + (Compile / resourceManaged).value, + name.value, + version.value, + streams.value.log + ) }.taskValue, - Compile/unmanagedResourceDirectories ++= Seq((Compile/resourceDirectory).value.getParentFile/"templates"), - reStart/javaOptions ++= Seq(s"-Dconfig.file=${(LocalRootProject/baseDirectory).value/"local"/"dev.conf"}") - ).dependsOn(restapi, joexapi, backend, webapp, ftssolr) + Compile / unmanagedResourceDirectories ++= Seq( + (Compile / resourceDirectory).value.getParentFile / "templates" + ), + reStart / javaOptions ++= Seq( + s"-Dconfig.file=${(LocalRootProject / baseDirectory).value / "local" / "dev.conf"}" + ) + ) + .dependsOn(restapi, joexapi, backend, webapp, ftssolr) +// --- Website Documentation - -// --- Microsite Documentation - -val microsite = project.in(file("modules/microsite")). - disablePlugins(RevolverPlugin). - enablePlugins(MicrositesPlugin). - disablePlugins(ReleasePlugin). - settings(sharedSettings). - settings( - name := "docspell-microsite", +val website = project + .in(file("website")) + .disablePlugins(RevolverPlugin, ReleasePlugin) + .enablePlugins(ZolaPlugin, GhpagesPlugin) + .settings(sharedSettings) + .settings( + name := "docspell-website", publishArtifact := false, skip in publish := true, - micrositeFooterText := Some( - """ - |

© 2020 Docspell, v{{site.version}}

- |""".stripMargin - ), - micrositeName := "Docspell", - micrositeDescription := "Auto-tagging Document Organizer", - micrositeDocumentationUrl := "doc", - micrositeDocumentationLabelDescription := "Documentation", - micrositeFavicons := Seq(microsites.MicrositeFavicon("favicon.png", "96x96")), - micrositeAuthor := "eikek", - micrositeGithubOwner := "eikek", - micrositeGithubRepo := "docspell", - micrositeGitterChannel := false, - micrositeShareOnSocial := false, - micrositeHighlightLanguages ++= Seq("json", "javascript"), - micrositeEditButton := Some(microsites.MicrositeEditButton("Improve this page", "/edit/master/modules/microsite/docs/{{ page.path }}")), - fork in run := true, - micrositeCompilingDocsTool := WithMdoc, - mdocVariables := Map( - "VERSION" -> version.value, - "PVERSION" -> version.value.replace('.', '_') - ), - micrositeExtraMdFiles := Map( - file("Changelog.md") -> ExtraMdFileConfig( - "changelog.md", - "docs", - Map("title" -> "Changelog", "permalink" -> "changelog") - ) - ), - Compile/resourceGenerators += Def.task { - val jekyllOut = resourceManaged.value/"main"/"jekyll" + ghpagesNoJekyll := true, + // the ghpages plugins works together with the site plugin (its a dependency) + // to make it publish the zola generated site, override their mappings with the zola output + mappings in SitePlugin.autoImport.makeSite := + Path.selectSubpaths(zolaOutputDir.value, _ => true).toSeq, + git.remoteRepo := "git@github.com:eikek/docspell", + Compile / resourceGenerators += Def.task { + val templateOut = baseDirectory.value / "site" / "templates" / "shortcodes" + val staticOut = baseDirectory.value / "site" / "static" / "openapi" + IO.createDirectories(Seq(templateOut, staticOut)) val logger = streams.value.log - val templates = Seq( - (resourceDirectory in (restserver, Compile)).value / "reference.conf" -> jekyllOut /"_includes"/"server.conf", - (resourceDirectory in (joex, Compile)).value / "reference.conf" -> jekyllOut/"_includes"/"joex.conf", - (LocalRootProject / baseDirectory).value / "tools" / "exim" / "exim.conf" -> jekyllOut/ "_includes"/"sample-exim.conf" - ) - val res1 = templates.map { case (s, t) => - logger.info(s"Copying $s -> $t") - IO.write(t, "{% raw %}\n") - IO.append(t, IO.readBytes(s)) - IO.write(t, "\n{% endraw %}", append = true) - t - } - val files = Seq( - (resourceDirectory in (restapi, Compile)).value/"docspell-openapi.yml" -> jekyllOut/"openapi"/"docspell-openapi.yml" + (resourceDirectory in (restserver, Compile)).value / "reference.conf" -> templateOut / "server.conf", + (resourceDirectory in (joex, Compile)).value / "reference.conf" -> templateOut / "joex.conf", + (LocalRootProject / baseDirectory).value / "tools" / "exim" / "exim.conf" -> templateOut / "sample-exim.conf", + (resourceDirectory in (restapi, Compile)).value / "docspell-openapi.yml" -> staticOut / "docspell-openapi.yml", + (restapi / Compile / openapiStaticDoc).value -> staticOut / "docspell-openapi.html" ) IO.copy(files) - res1 ++ files.map(_._2) + files.map(_._2) }.taskValue, - Compile/resourceGenerators += Def.task { - val staticDoc = (restapi/Compile/openapiStaticDoc).value - val target = resourceManaged.value/"main"/"jekyll"/"openapi"/"docspell-openapi.html" - IO.copy(Seq(staticDoc -> target)) + Compile / resourceGenerators += Def.task { + val changelog = (LocalRootProject / baseDirectory).value / "Changelog.md" + val targetDir = baseDirectory.value / "site" / "content" / "docs" / "changelog" + IO.createDirectory(targetDir) + val target = targetDir / "_index.md" + + IO.write( + target, + """|+++ + |title = "Changelog" + |description = "See what changed between releases." + |weight = 10 + |insert_anchor_links = "right" + |[extra] + |maketoc = false + |+++ + |""".stripMargin + ) + IO.append(target, IO.readBytes(changelog)) Seq(target) }.taskValue ) -val root = project.in(file(".")). - settings(sharedSettings). - settings(noPublish). - settings( +val root = project + .in(file(".")) + .settings(sharedSettings) + .settings(noPublish) + .settings( name := "docspell-root" - ). - aggregate(common - , extract - , convert - , analysis - , ftsclient - , ftssolr - , files - , store - , joexapi - , joex - , backend - , webapp - , restapi - , restserver) - - + ) + .aggregate( + common, + extract, + convert, + analysis, + ftsclient, + ftssolr, + files, + store, + joexapi, + joex, + backend, + webapp, + restapi, + restserver + ) // --- Helpers -def copyWebjarResources(src: Seq[File], base: File, artifact: String, version: String, logger: Logger): Seq[File] = { - val targetDir = base/"META-INF"/"resources"/"webjars"/artifact/version +def copyWebjarResources( + src: Seq[File], + base: File, + artifact: String, + version: String, + logger: Logger +): Seq[File] = { + val targetDir = base / "META-INF" / "resources" / "webjars" / artifact / version logger.info(s"Copy webjar resources from ${src.size} files/directories.") src.flatMap { dir => if (dir.isDirectory) { - val files = (dir ** "*").filter(_.isFile).get pair Path.relativeTo(dir) - files.map { case (f, name) => - val target = targetDir/name - IO.createDirectories(Seq(target.getParentFile)) - IO.copy(Seq(f -> target)) - target + val files = (dir ** "*").filter(_.isFile).get.pair(Path.relativeTo(dir)) + files.map { + case (f, name) => + val target = targetDir / name + IO.createDirectories(Seq(target.getParentFile)) + IO.copy(Seq(f -> target)) + target } } else { - val target = targetDir/dir.name + val target = targetDir / dir.name IO.createDirectories(Seq(target.getParentFile)) IO.copy(Seq(dir -> target)) Seq(target) @@ -540,19 +592,34 @@ def copyWebjarResources(src: Seq[File], base: File, artifact: String, version: S } } -def compileElm(logger: Logger, wd: File, outBase: File, artifact: String, version: String, mode: ElmCompileMode): Seq[File] = { +def compileElm( + logger: Logger, + wd: File, + outBase: File, + artifact: String, + version: String, + mode: ElmCompileMode +): Seq[File] = { logger.info("Compile elm files ...") - val target = outBase/"META-INF"/"resources"/"webjars"/artifact/version/"docspell-app.js" + val target = + outBase / "META-INF" / "resources" / "webjars" / artifact / version / "docspell-app.js" val cmd = Seq("elm", "make") ++ mode.flags ++ Seq("--output", target.toString) - val proc = Process(cmd ++ Seq(wd/"src"/"main"/"elm"/"Main.elm").map(_.toString), Some(wd)) + val proc = Process( + cmd ++ Seq(wd / "src" / "main" / "elm" / "Main.elm").map(_.toString), + Some(wd) + ) val out = proc.!! logger.info(out) Seq(target) } def createWebjarSource(wj: Seq[ModuleID], out: File): Seq[File] = { - val target = out/"Webjars.scala" - val fields = wj.map(m => s"""val ${m.name.toLowerCase.filter(_ != '-')} = "/${m.name}/${m.revision}" """).mkString("\n\n") + val target = out / "Webjars.scala" + val fields = wj + .map(m => + s"""val ${m.name.toLowerCase.filter(_ != '-')} = "/${m.name}/${m.revision}" """ + ) + .mkString("\n\n") val content = s"""package docspell.restserver.webapp |object Webjars { |$fields @@ -564,36 +631,45 @@ def createWebjarSource(wj: Seq[ModuleID], out: File): Seq[File] = { } def packageTools(logger: Logger, dir: File, version: String): Seq[File] = { - val target = dir/"target" + val target = dir / "target" IO.delete(target) IO.createDirectory(target) - val archive = target/s"docspell-tools-${version}.zip" + val archive = target / s"docspell-tools-${version}.zip" logger.info(s"Packaging tools to $archive ...") - val webext = target/"docspell-firefox-extension.xpi" - val wx = dir/"webextension" - IO.zip(Seq( - wx/"_locales/de/messages.json" -> "_locales/de/messages.json", - wx/"_locales/en/messages.json" -> "_locales/en/messages.json", - wx/"docspell.js" -> "docspell.js", - wx/"icons"/"logo-48.png" -> "icons/logo-48.png", - wx/"icons"/"logo-96.png" -> "icons/logo-96.png", - wx/"manifest.json" -> "manifest.json" - ), webext) + val webext = target / "docspell-firefox-extension.xpi" + val wx = dir / "webextension" + IO.zip( + Seq( + wx / "_locales/de/messages.json" -> "_locales/de/messages.json", + wx / "_locales/en/messages.json" -> "_locales/en/messages.json", + wx / "docspell.js" -> "docspell.js", + wx / "icons" / "logo-48.png" -> "icons/logo-48.png", + wx / "icons" / "logo-96.png" -> "icons/logo-96.png", + wx / "manifest.json" -> "manifest.json" + ), + webext + ) - IO.zip(Seq( - webext -> s"docspell-tools-${version}/firefox/docspell-extension.xpi", - wx/"native/app_manifest.json" ->s"docspell-tools-${version}/firefox/native/app_manifest.json", - wx/"native/native.py" ->s"docspell-tools-${version}/firefox/native/native.py", - dir/"ds.sh" -> s"docspell-tools-${version}/ds.sh", - dir/"consumedir.sh" -> s"docspell-tools-${version}/consumedir.sh" - ), archive) + IO.zip( + Seq( + webext -> s"docspell-tools-${version}/firefox/docspell-extension.xpi", + wx / "native/app_manifest.json" -> s"docspell-tools-${version}/firefox/native/app_manifest.json", + wx / "native/native.py" -> s"docspell-tools-${version}/firefox/native/native.py", + dir / "ds.sh" -> s"docspell-tools-${version}/ds.sh", + dir / "consumedir.sh" -> s"docspell-tools-${version}/consumedir.sh" + ), + archive + ) Seq(archive) } // --- aliases -addCommandAlias("make", ";set webapp/elmCompileMode := ElmCompileMode.Production ;root/openapiCodegen ;root/test:compile") +addCommandAlias( + "make", + ";set webapp/elmCompileMode := ElmCompileMode.Production ;root/openapiCodegen ;root/test:compile" +) addCommandAlias("make-zip", ";restserver/universal:packageBin ;joex/universal:packageBin") addCommandAlias("make-deb", ";restserver/debian:packageBin ;joex/debian:packageBin") addCommandAlias("make-tools", ";root/toolsPackage") diff --git a/modules/microsite/docs/demo.md b/modules/microsite/docs/demo.md deleted file mode 100644 index 234b60f3..00000000 --- a/modules/microsite/docs/demo.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -layout: docs -title: Demo -permalink: demo ---- - -# {{ page.title }} - -## Finding Items - - - - -## Basic Idea (First Version) - - diff --git a/modules/microsite/docs/dev/adr.md b/modules/microsite/docs/dev/adr.md deleted file mode 100644 index 8410f065..00000000 --- a/modules/microsite/docs/dev/adr.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -layout: docs -title: ADRs -permalink: dev/adr ---- - -# ADR - -Some early information about certain details can be found in a few -[ADR](https://adr.github.io/) that exist: - -- [0001 Components](adr/0001_components) -- [0002 Component Interaction](adr/0002_component_interaction) -- [0003 Encryption](adr/0003_encryption) -- [0004 ISO8601 vs Unix](adr/0004_iso8601vsEpoch) -- [0005 Job Executor](adr/0005_job-executor) -- [0006 More File Types](adr/0006_more-file-types) - - [0007 Convert HTML](adr/0007_convert_html_files) - - [0008 Convert Text](adr/0008_convert_plain_text) - - [0009 Convert Office Files](adr/0009_convert_office_docs) - - [0010 Convert Image Files](adr/0010_convert_image_files) - - [0011 Extract Text](adr/0011_extract_text) -- [0012 Periodic Tasks](adr/0012_periodic_tasks) -- [0013 Archive Files](adr/0013_archive_files) -- [0014 Full-Text Search](adr/0014_fulltext_search_engine) -- [0015 Convert PDF files](adr/0015_convert_pdf_files) diff --git a/modules/microsite/docs/doc.md b/modules/microsite/docs/doc.md deleted file mode 100644 index 81aed022..00000000 --- a/modules/microsite/docs/doc.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -layout: docs -title: Documentation -permalink: doc ---- - -# {{page.title}} - -This is the documentation for Docspell @VERSION@. - -Docspell assists in organizing large amounts of files that are -typically scanned paper documents. You can associate tags, set -correspondends, what a document is concerned with, a name, a date and -some more. If your documents are associated with this meta data, you -should be able to quickly find them later using the search feature. -But adding this manually to each document is a tedious task. What if -most of it could be attached automatically? - -## How it works - -Documents have two main properties: a correspondent (sender or -receiver that is not you) and something the document is about. Usually -it is about a person or some thing – maybe your car, or contracts -concerning some familiy member, etc. - -1. You maintain a kind of address book. It should list all possible - correspondents and the concerning people/things. This grows - incrementally with each new unknown document. -2. When docspell analyzes a document, it tries to find matches within - your address book. It can detect the correspondent and a concerning - person or thing. It will then associate this data to your - documents. -3. You can inspect what docspell has done and correct it. If docspell - has found multiple suggestions, they will be shown for you to - select one. If it is not correctly associated, very often the - correct one is just one click away. - -The set of meta data that docspell uses to draw suggestions from, must -be maintained manually. But usually, this data doesn't grow as fast as -the documents. After a while there is a quite complete address book -and only once in a while it has to be revisited. - -Besides extracting the text from documents to analyze, docspell also -converts all files into PDF files. This unifies the different formats -your documents may be in originally and makes them more accessible -from other systems and the future. - -## Terms - -In order to better understand these pages, some terms should be -explained first. - -### Item - -An **Item** is roughly your (pdf) document, only that an item may span -multiple files, which are called **attachments**. And an item has -**meta data** associated: - -- a **correspondent**: the other side of the communication. It can be - an organization or a person. -- a **concerning person** or **equipment**: a person or thing that - this item is about. Maybe it is an insurance contract about your - car. -- **tag**: an item can be tagged with custom tags. A tag can have a - *category*. This is intended for grouping tags, for example a - category `doctype` could be used to group tags like `bill`, - `contract`, `receipt` etc. Usually an item is not tagged with more - than one tag of a category. -- a **item date**: this is the date of the document – if this is not - set, the created date of the item is used. -- a **due date**: an optional date indicating that something has to be - done (e.g. paying a bill, submitting it) about this item until this - date -- a **direction**: one of "incoming" or "outgoing" -- a **name**: some item name, defaults to the file name of the - attachments -- some **notes**: arbitrary descriptive text. You can use markdown - here, which is appropriately formatted in the web application. - -### Collective - -The users of the application are part of a **collective**. A -**collective** is a group of users that share access to the same -items. The account name is therefore comprised of a *collective name* -and a *user name*. - -All users of a collective are equal; they have same permissions to -access all items. The items don't belong to a user, but to the -collective. - -That means, to identify yourself when signing in, you have to give the -collective name and your user name. By default it is separated by a -slash `/`, for example `smith/john`. If your user name is the same as -the collective name, you can omit one; so `smith/smith` can be -abbreviated to just `smith`. diff --git a/modules/microsite/docs/doc/install.md b/modules/microsite/docs/doc/install.md deleted file mode 100644 index 90236233..00000000 --- a/modules/microsite/docs/doc/install.md +++ /dev/null @@ -1,235 +0,0 @@ ---- -layout: docs -title: Installation -permalink: doc/install ---- - -# {{ page.title }} - -This page contains detailed installation instructions. For a quick -start, refer to [this page](../getit). - -Docspell has been developed and tested on a GNU/Linux system. It may -run on Windows and MacOS machines, too (ghostscript and tesseract are -available on these systems). But I've never tried. - -Docspell consists of two components that are started in separate -processes: - -1. *REST Server* This is the main application, providing the REST Api - and the web application. -2. *Joex* (job executor) This is the component that does the document - processing. - -They can run on multiple machines. All REST server and Joex instances -should be on the same network. It is not strictly required that they -can reach each other, but the components can then notify themselves -about new or done work. - -While this is possible, the simple setup is to start both components -once on the same machine. - -The [download page](https://github.com/eikek/docspell/releases) -provides pre-compiled packages and the [development page](../dev) -contains build instructions. - - -## Prerequisites - -The two components have one prerequisite in common: they both require -Java to run. While this is the only requirement for the *REST server*, -the *Joex* components requires some more external programs. - -### Java - -Very often, Java is already installed. You can check this by opening a -terminal and typing `java -version`. Otherwise install Java using your -package manager or see [this site](https://adoptopenjdk.net/) for -other options. - -It is enough to install the JRE. The JDK is required, if you want to -build docspell from source. - -Docspell has been tested with Java version 1.8 (or sometimes referred -to as JRE 8 and JDK 8, respectively). The pre-build packages are also -build using JDK 8. But a later version of Java should work as well. - -The next tools are only required on machines running the *Joex* -component. - -### External Programs for Joex - -- [Ghostscript](http://pages.cs.wisc.edu/~ghost/) (the `gs` command) - is used to extract/convert PDF files into images that are then fed - to ocr. It is available on most GNU/Linux distributions. -- [Unpaper](https://github.com/Flameeyes/unpaper) is a program that - pre-processes images to yield better results when doing ocr. If this - is not installed, docspell tries without it. However, it is - recommended to install, because it [improves text - extraction](https://github.com/tesseract-ocr/tesseract/wiki/ImproveQuality) - (at the expense of a longer runtime). -- [Tesseract](https://github.com/tesseract-ocr/tesseract) is the tool - doing the OCR (converts images into text). It can also convert - images into pdf files. It is a widely used open source OCR engine. - Tesseract 3 and 4 should work with docspell; you can adopt the - command line in the configuration file, if necessary. -- [Unoconv](https://github.com/unoconv/unoconv) is used to convert - office documents into PDF files. It uses libreoffice/openoffice. -- [wkhtmltopdf](https://wkhtmltopdf.org/) is used to convert HTML into - PDF files. -- [OCRmyPDF](https://github.com/jbarlow83/OCRmyPDF) can be optionally - used to convert PDF to PDF files. It adds an OCR layer to scanned - PDF files to make them searchable. It also creates PDF/A files from - the input pdf. - -The performance of `unoconv` can be improved by starting `unoconv -l` -in a separate process. This runs a libreoffice/openoffice listener -therefore avoids starting one each time `unoconv` is called. - -### Example Debian - -On Debian this should install all joex requirements: - -``` bash -sudo apt-get install ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng unpaper unoconv wkhtmltopdf ocrmypdf -``` - - -## Database - -Both components must have access to a SQL database. Docspell has -support these databases: - -- PostreSQL -- MariaDB -- H2 - -The H2 database is an interesting option for personal and mid-size -setups, as it requires no additional work. It is integrated into -docspell and works really well. It is also configured as the default -database. - -For large installations, PostgreSQL or MariaDB is recommended. Create -a database and a user with enough privileges (read, write, create -table) to that database. - -When using H2, make sure that all components access the same database -– the jdbc url must point to the same file. Then, it is important to -add the options -`;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_SERVER=TRUE` at the end -of the url. See the [config page](configure#jdbc) for an example. - - -## Installing from ZIP files - -After extracting the zip files, you'll find a start script in the -`bin/` folder. - - -## Installing from DEB packages - -The DEB packages can be installed on Debian, or Debian based Distros: - -``` bash -$ sudo dpkg -i docspell*.deb -``` - -Then the start scripts are in your `$PATH`. Run `docspell-restserver` -or `docspell-joex` from a terminal window. - -The packages come with a systemd unit file that will be installed to -autostart the services. - - -## Running - -Run the start script (in the corresponding `bin/` directory when using -the zip files): - -``` -$ ./docspell-restserver*/bin/docspell-restserver -$ ./docspell-joex*/bin/docspell-joex -``` - -This will startup both components using the default configuration. The -configuration should be adopted to your needs. For example, the -database connection is configured to use a H2 database in the `/tmp` -directory. Please refer to the [configuration page](configure) for how -to create a custom config file. Once you have your config file, simply -pass it as argument to the command: - -``` -$ ./docspell-restserver*/bin/docspell-restserver /path/to/server-config.conf -$ ./docspell-joex*/bin/docspell-joex /path/to/joex-config.conf -``` - -After starting the rest server, you can reach the web application at -path `/app`, so using default values it would be -`http://localhost:7880/app`. - -You should be able to create a new account and sign in. Check the -[configuration page](configure) to further customize docspell. - - -### Options - -The start scripts support some options to configure the JVM. One often -used setting is the maximum heap size of the JVM. By default, java -determines it based on properties of the current machine. You can -specify it by given java startup options to the command: - -``` -$ ./docspell-restserver*/bin/docspell-restserver -J-Xmx1G -- /path/to/server-config.conf -``` - -This would limit the maximum heap to 1GB. The double slash separates -internal options and the arguments to the program. Another frequently -used option is to change the default temp directory. Usually it is -`/tmp`, but it may be desired to have a dedicated temp directory, -which can be configured: - -``` -$ ./docspell-restserver*/bin/docspell-restserver -J-Xmx1G -Djava.io.tmpdir=/path/to/othertemp -- /path/to/server-config.conf -``` - -The command: - -``` -$ ./docspell-restserver*/bin/docspell-restserver -h -``` - -gives an overview of supported options. - - -## Raspberry Pi, and similiar - -Both component can run next to each other on a raspberry pi or -similiar device. - - -### REST Server - -The REST server component runs very well on the Raspberry Pi and -similiar devices. It doesn't require much resources, because the heavy -work is done by the joex components. - - -### Joex - -Running the joex component on the Raspberry Pi is possible, but will -result in long processing times for OCR. Files that don't require OCR -are no problem. - -Tested on a RPi model 3 (4 cores, 1G RAM) processing a PDF (scanned -with 300dpi) with two pages took 9:52. You can speed it up -considerably by uninstalling the `unpaper` command, because this step -takes quite long. This, of course, reduces the quality of OCR. But -without `unpaper` the same sample pdf was then processed in 1:24, a -speedup of 8 minutes. - -You should limit the joex pool size to 1 and, depending on your model -and the amount of RAM, set a heap size of at least 500M -(`-J-Xmx500M`). - -For personal setups, when you don't need the processing results asap, -this can work well enough. diff --git a/modules/microsite/docs/doc/tools.md b/modules/microsite/docs/doc/tools.md deleted file mode 100644 index 8595ab3c..00000000 --- a/modules/microsite/docs/doc/tools.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -layout: docs -title: Tools -permalink: doc/tools ---- - -# {{ page.title }} - -The `tools/` folder contains some scripts and other resources intented -for integrating docspell. - -- [ds.sh](ds) A script to quickly upload files from the command - line. -- [Consume Directory](consumedir) A script to watch a directory - for new files and upload them to docspell. -- [Browser Extension](browserext) An extension for firefox to - upload files from your browser via *right-click -> upload to - docspell*. -- [SMTP Gateway](smtpgateway) Start a SMTP server that forwards all - mails to docspell. diff --git a/modules/microsite/docs/getit.md b/modules/microsite/docs/getit.md deleted file mode 100644 index 61893ab0..00000000 --- a/modules/microsite/docs/getit.md +++ /dev/null @@ -1,107 +0,0 @@ ---- -layout: docs -title: Quickstart -permalink: getit ---- - -# {{ page.title }} - -There are the following quick ways to get docspell to run on your -machine: - -- [Download, Unpack, Run](#without-docker) You can download - pre-compiled binaries from the [Release - Page](https://github.com/eikek/docspell/releases). There are `deb` - packages and generic zip files. -- [With Docker](#with-docker) -- [NixOs Module](doc/nix#docspell-as-a-service-on-nixos) - -Check the [demo videos](demo) to see the basic idea. Refer to the -[documentation](doc) for more information on how to use docspell. - - -## Download, Unpack, Run - -### Prerequisite - -Install Java (use your package manager or look -[here](https://adoptopenjdk.net/)). - -OCR functionality requires the following tools: - -- [tesseract](https://github.com/tesseract-ocr/tesseract), -- [ghostscript](http://pages.cs.wisc.edu/~ghost/) and possibly -- [unpaper](https://github.com/Flameeyes/unpaper). - -The last is not really required, but improves OCR. - -PDF conversion requires the following tools: - -- [unoconv](https://github.com/unoconv/unoconv) -- [wkhtmltopdf](https://wkhtmltopdf.org/) - - -### Using zip files - -You need to download the two files: - -- [docspell-restserver-{{site.version}}.zip](https://github.com/eikek/docspell/releases/download/v{{site.version}}/docspell-restserver-{{site.version}}.zip) -- [docspell-joex-{{site.version}}.zip](https://github.com/eikek/docspell/releases/download/v{{site.version}}/docspell-joex-{{site.version}}.zip) - - -1. Unzip both files: - ``` bash - $ unzip docspell-*.zip - ``` -2. Open two terminal windows and navigate to the the directory - containing the zip files. -3. Start both components executing: - ``` bash - $ ./docspell-restserver*/bin/docspell-restserver - ``` - in one terminal and - ``` bash - $ ./docspell-joex*/bin/docspell-joex - ``` - in the other. -4. Point your browser to: -5. Register a new account, sign in and try it. - -Note, that this setup doesn't include watching a directory. You can -use the [`consumedir.sh`](doc/tools/consumedir) tool for this or use -the docker variant below. - -## With Docker - -There is a [docker-compose](https://docs.docker.com/compose/) setup -available in the `/docker` folder. - -1. Clone the github repository - ```bash - $ git clone https://github.com/eikek/docspell - ``` -2. Change into the `docker` directory: - ```bash - $ cd docspell/docker - ``` -3. Run `docker-compose up`: - - ```bash - $ export DOCSPELL_HEADER_VALUE="my-secret-123" - $ docker-compose up - ``` - - The environment variable defines a secret that is shared between - some containers. You can define whatever you like. Please see the - [`consumedir.sh`](doc/tools/consumedir#docker) docs for additional - info. -4. Goto , signup and login. When signing up, - you can choose the same name for collective and user. Then login - with this name and the password. - -5. (Optional) Create a folder `./docs/` (the name you - chose for the collective at registration) and place files in there - for importing them. - -The directory contains a file `docspell.conf` that you can -[modify](doc/configure) as needed. diff --git a/modules/microsite/docs/index.md b/modules/microsite/docs/index.md deleted file mode 100644 index 39d4ac69..00000000 --- a/modules/microsite/docs/index.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -layout: homeFeatures -features: - - first: ["Stow documents away", "Most of the time documents (emails, postal mail) are received or created. It should be fast to stow them away, knowing that they can be found if necessary.", "uploading"] - - second: ["Automatic Tagging", "All documents are analyzed and tagged automatically. It may not always be correct; results can be reviewed and corrected.", "metadata"] - - third: ["Work with them", "Search for documents by their meta data or via full-text search. Send them via e-mail. Add your own tags, names etc to better match your workflow.", "finding"] ---- - - -## License - -This project is distributed under the -[GPLv3](http://www.gnu.org/licenses/gpl-3.0.html) diff --git a/modules/microsite/src/main/resources/microsite/css/docspell.css b/modules/microsite/src/main/resources/microsite/css/docspell.css deleted file mode 100644 index b96be06f..00000000 --- a/modules/microsite/src/main/resources/microsite/css/docspell.css +++ /dev/null @@ -1,46 +0,0 @@ -#site-main { - background: aliceblue; -} - -#masthead { - background: url(../img/back-master-small.jpg); - background-repeat: no-repeat; - background-size: 100% 100%; - animation: none; - height: 26vh; -} - -.thumbnail { - border: 1px solid #aaa; -} - -.features-image { - height: 200px; - position: relative; -} - -.docs #sidebar-wrapper .sidebar-nav .sidebar-nav-item.open>a, .docs #sidebar-wrapper .sidebar-nav .sidebar-nav-item.open button { - background: #495680; -} -.docs #sidebar-wrapper .sidebar-nav .sidebar-nav-item .sub-section { - background: #172651; -} -.docs #sidebar-wrapper .sidebar-nav .sidebar-nav-item .sub-section a.active { - font-weight: bold; -} -.docs #sidebar-wrapper { - background: #172651; -} - -.docs #sidebar-wrapper #sidebar-brand { - background: #495680; - -} - -.docs h4 { - text-decoration: underline; -} - -.docs .thumbnail img { - width: 100%; -} diff --git a/modules/microsite/src/main/resources/microsite/data/menu.yml b/modules/microsite/src/main/resources/microsite/data/menu.yml deleted file mode 100644 index 0b5e0108..00000000 --- a/modules/microsite/src/main/resources/microsite/data/menu.yml +++ /dev/null @@ -1,94 +0,0 @@ -options: - - title: Home - url: index.html - - - title: Demo - url: demo - - - title: Quickstart - url: getit - - - title: Features/Limitations - url: features - - - title: Documentation - url: doc - - nested_options: - - title: Installation - url: doc/install - - - title: Configuring - url: doc/configure - - - title: Nix/NixOS - url: doc/nix - - - title: Reverse Proxy - url: doc/reverseproxy - - - title: Meta Data - url: doc/metadata - - - title: Finding Items - url: doc/finding - - - title: Curate Items - url: doc/curate - - - title: Uploads - url: doc/uploading - - - title: Processing Queue - url: doc/processing - - - title: E-Mail Settings - url: doc/emailsettings - - - title: Send via E-Mail - url: doc/mailitem - - - title: Notify on due Items - url: doc/notifydueitems - - - title: Scan Mailboxes - url: doc/scanmailbox - - - title: Joex - url: doc/joex - - - title: Tools - url: doc/tools - - nested_options: - - title: Upload CLI - url: doc/tools/ds - - - title: Consume Directory - url: doc/tools/consumedir - - - title: Browser Extension (Firefox) - url: doc/tools/browserext - - - title: SMTP Gateway - url: doc/tools/smtpgateway - - - title: Api - url: api - - nested_options: - - title: REST Api Doc - url: openapi/docspell-openapi.html - - - title: REST OpenApi Spec - url: openapi/docspell-openapi.yml - - - title: Development - url: dev - - nested_options: - - title: ADRs - url: dev/adr - - - title: Changelog - url : changelog diff --git a/modules/microsite/src/main/resources/microsite/img/docspell-demo.gif b/modules/microsite/src/main/resources/microsite/img/docspell-demo.gif deleted file mode 100644 index b7ca2be9..00000000 Binary files a/modules/microsite/src/main/resources/microsite/img/docspell-demo.gif and /dev/null differ diff --git a/modules/microsite/src/main/resources/microsite/img/docspell-try.gif b/modules/microsite/src/main/resources/microsite/img/docspell-try.gif deleted file mode 100644 index 2f4ce6ce..00000000 Binary files a/modules/microsite/src/main/resources/microsite/img/docspell-try.gif and /dev/null differ diff --git a/modules/microsite/src/main/resources/microsite/img/favicon.png b/modules/microsite/src/main/resources/microsite/img/favicon.png deleted file mode 120000 index 713c4fb8..00000000 --- a/modules/microsite/src/main/resources/microsite/img/favicon.png +++ /dev/null @@ -1 +0,0 @@ -../../../../../../../artwork/logo-96.png \ No newline at end of file diff --git a/modules/microsite/src/main/resources/microsite/img/features-header.svg b/modules/microsite/src/main/resources/microsite/img/features-header.svg deleted file mode 120000 index 75fe7ef4..00000000 --- a/modules/microsite/src/main/resources/microsite/img/features-header.svg +++ /dev/null @@ -1 +0,0 @@ -../../../../../../../artwork/logo-only.svg \ No newline at end of file diff --git a/modules/microsite/src/main/resources/microsite/img/first-feature-icon.svg b/modules/microsite/src/main/resources/microsite/img/first-feature-icon.svg deleted file mode 120000 index 7d6f35b8..00000000 --- a/modules/microsite/src/main/resources/microsite/img/first-feature-icon.svg +++ /dev/null @@ -1 +0,0 @@ -../../../../../../../artwork/stow.svg \ No newline at end of file diff --git a/modules/microsite/src/main/resources/microsite/img/light-navbar-brand.svg b/modules/microsite/src/main/resources/microsite/img/light-navbar-brand.svg deleted file mode 120000 index 76d21b70..00000000 --- a/modules/microsite/src/main/resources/microsite/img/light-navbar-brand.svg +++ /dev/null @@ -1 +0,0 @@ -../../../../../../../artwork/logo-only-36.svg \ No newline at end of file diff --git a/modules/microsite/src/main/resources/microsite/img/light-sidebar-brand.svg b/modules/microsite/src/main/resources/microsite/img/light-sidebar-brand.svg deleted file mode 120000 index 15434edc..00000000 --- a/modules/microsite/src/main/resources/microsite/img/light-sidebar-brand.svg +++ /dev/null @@ -1 +0,0 @@ -light-navbar-brand.svg \ No newline at end of file diff --git a/modules/microsite/src/main/resources/microsite/img/search-bar.png b/modules/microsite/src/main/resources/microsite/img/search-bar.png deleted file mode 100644 index 5ac9dc0b..00000000 Binary files a/modules/microsite/src/main/resources/microsite/img/search-bar.png and /dev/null differ diff --git a/modules/microsite/src/main/resources/microsite/img/search-menu.png b/modules/microsite/src/main/resources/microsite/img/search-menu.png deleted file mode 100644 index 0100c7ab..00000000 Binary files a/modules/microsite/src/main/resources/microsite/img/search-menu.png and /dev/null differ diff --git a/modules/microsite/src/main/resources/microsite/img/second-feature-icon.svg b/modules/microsite/src/main/resources/microsite/img/second-feature-icon.svg deleted file mode 120000 index 52db5991..00000000 --- a/modules/microsite/src/main/resources/microsite/img/second-feature-icon.svg +++ /dev/null @@ -1 +0,0 @@ -../../../../../../../artwork/tag.svg \ No newline at end of file diff --git a/modules/microsite/src/main/resources/microsite/img/third-feature-icon.svg b/modules/microsite/src/main/resources/microsite/img/third-feature-icon.svg deleted file mode 120000 index 89be9542..00000000 --- a/modules/microsite/src/main/resources/microsite/img/third-feature-icon.svg +++ /dev/null @@ -1 +0,0 @@ -../../../../../../../artwork/search.svg \ No newline at end of file diff --git a/modules/microsite/src/main/resources/microsite/img/wand-white.png b/modules/microsite/src/main/resources/microsite/img/wand-white.png deleted file mode 100644 index c670c27b..00000000 Binary files a/modules/microsite/src/main/resources/microsite/img/wand-white.png and /dev/null differ diff --git a/modules/microsite/src/main/resources/microsite/static/docspell-demo.webm b/modules/microsite/src/main/resources/microsite/static/docspell-demo.webm deleted file mode 100644 index 59f6230c..00000000 Binary files a/modules/microsite/src/main/resources/microsite/static/docspell-demo.webm and /dev/null differ diff --git a/modules/microsite/src/main/resources/microsite/static/docspell-search-2020-06-13.webm b/modules/microsite/src/main/resources/microsite/static/docspell-search-2020-06-13.webm deleted file mode 100644 index ea6786aa..00000000 Binary files a/modules/microsite/src/main/resources/microsite/static/docspell-search-2020-06-13.webm and /dev/null differ diff --git a/modules/microsite/src/main/resources/microsite/static/docspell-search-2020-06-24.webm b/modules/microsite/src/main/resources/microsite/static/docspell-search-2020-06-24.webm deleted file mode 100644 index 016a6076..00000000 Binary files a/modules/microsite/src/main/resources/microsite/static/docspell-search-2020-06-24.webm and /dev/null differ diff --git a/project/Cmd.scala b/project/Cmd.scala new file mode 100644 index 00000000..9fc27547 --- /dev/null +++ b/project/Cmd.scala @@ -0,0 +1,40 @@ +package docspell.build + +import sbt._ +import scala.sys.process._ +import java.util.concurrent.atomic.AtomicReference + +/** Helper for running external commands. */ +object Cmd { + + case class Result(rc: Int, out: String, err: String) { + + def throwIfNot(success: Int): Result = + if (rc != success) sys.error(s"Unsuccessful return: $rc") + else this + } + + def run(cmd: Seq[String], wd: File, logger: Logger): Unit = { + val res = Cmd.exec(cmd, Some(wd)) + logger.info(res.out) + logger.error(res.err) + res.throwIfNot(0) + } + + def exec(cmd: Seq[String], wd: Option[File]): Result = { + val capt = new Capture + val rc = Process(cmd, wd).!(capt.logger) + Result(rc, capt.out.get.mkString("\n"), capt.err.get.mkString("\n")) + } + + final private class Capture { + val err = new AtomicReference[List[String]](Nil) + val out = new AtomicReference[List[String]](Nil) + + val logger = ProcessLogger( + line => out.getAndAccumulate(List(line), _ ++ _), + line => err.getAndAccumulate(List(line), _ ++ _) + ) + + } +} diff --git a/project/ZolaPlugin.scala b/project/ZolaPlugin.scala new file mode 100644 index 00000000..1a50f9bf --- /dev/null +++ b/project/ZolaPlugin.scala @@ -0,0 +1,106 @@ +package docspell.build + +import sbt._ +import sbt.Keys._ +import scala.sys.process._ + +object ZolaPlugin extends AutoPlugin { + + object autoImport { + val zolaRootDir = settingKey[File]("The root directory of zola") + val zolaOutputDir = settingKey[File]("The directory to put the final site") + val zolaCommand = settingKey[String]("The zola executable") + val zolaTestBaseUrl = + settingKey[String]("The base-url to use when building the test site.") + + val zolaBuild = taskKey[Unit]( + "Build the site using zola, which is then available in 'zolaOutputDir'." + ) + val zolaBuildTest = taskKey[Unit]( + "Build the site using zola, which is then available in 'zolaOutputDir'. " + + "It uses a different base-url. So the final site can be tested using " + + "'python -m SimpleHTTPServer 1234' for example." + ) + val zolaCheck = taskKey[Unit]("Runs zola check to check links") + } + + import autoImport._ + + def zolaSettings: Seq[Setting[_]] = + Seq( + zolaRootDir := baseDirectory.value / "site", + zolaOutputDir := target.value / "zola-site", + zolaCommand := "zola", + zolaTestBaseUrl := "http://localhost:1234", + zolaBuild := { + val logger = streams.value.log + logger.info("Building web site using zola ...") + buildSite(zolaCommand.value, zolaRootDir.value, zolaOutputDir.value, None, logger) + logger.info("Website built") + }, + zolaBuildTest := { + val logger = streams.value.log + val baseurl = zolaTestBaseUrl.value + logger.info("Building web site (test) using zola ...") + buildSite( + zolaCommand.value, + zolaRootDir.value, + zolaOutputDir.value, + Some(baseurl), + logger + ) + logger.info(s"Website built. Check it with base-url $baseurl") + }, + zolaCheck := { + val logger = streams.value.log + logger.info("Checking web site using zola ...") + checkSite(zolaCommand.value, zolaRootDir.value, logger) + } + ) + + override def projectSettings: Seq[Setting[_]] = + zolaSettings + + def buildSite( + zolaCmd: String, + inDir: File, + outDir: File, + base: Option[String], + logger: Logger + ): Unit = { + val baseUrl = base match { + case Some(url) => + Seq("--base-url", url) + case None => + runYarnInstall("yarn", inDir.getParentFile, logger) + runElmCompile("elm", inDir.getParentFile, outDir, logger) + Seq.empty + } + Cmd.run( + Seq(zolaCmd, "build", "-o", outDir.absolutePath.toString) ++ baseUrl, + inDir, + logger + ) + } + + def checkSite(zolaCmd: String, inDir: File, logger: Logger): Unit = + Cmd.run(Seq(zolaCmd, "check"), inDir, logger) + + def runYarnInstall(yarnCmd: String, inDir: File, logger: Logger): Unit = + Cmd.run(Seq(yarnCmd, "install"), inDir, logger) + + def runElmCompile(elmCmd: String, inDir: File, zolaOut: File, logger: Logger): Unit = + Cmd.run( + Seq( + elmCmd, + "make", + "--output", + (zolaOut / "static" / "js" / "bundle.js").absolutePath.toString, + "--optimize", + (inDir / "elm" / "Main.elm").toString + ), + inDir, + logger + ) + +} diff --git a/project/build.nix b/project/build.nix index 522d61e9..21eba62c 100644 --- a/project/build.nix +++ b/project/build.nix @@ -1,14 +1,20 @@ -with import { }; let - initScript = writeScript "docspell-build-init" '' + nixpkgsUnstable = builtins.fetchTarball { + url = "https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz"; + }; + pkgsUnstable = import nixpkgsUnstable { }; + initScript = pkgsUnstable.writeScript "docspell-build-init" '' export LD_LIBRARY_PATH= - ${bash}/bin/bash -c sbt + ${pkgsUnstable.bash}/bin/bash -c sbt ''; -in +in with pkgsUnstable; + buildFHSUserEnv { name = "docspell-sbt"; targetPkgs = pkgs: with pkgs; [ - netcat jdk8 wget which zsh dpkg sbt git elmPackages.elm ncurses fakeroot mc jekyll + netcat jdk8 wget which zsh dpkg sbt git elmPackages.elm ncurses fakeroot mc + zola yarn + # haskells http client needs this (to download elm packages) iana-etc ]; diff --git a/project/plugins.sbt b/project/plugins.sbt index c367e325..13e6212d 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,9 +1,9 @@ addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.19") -addSbtPlugin("com.47deg" % "sbt-microsites" % "1.2.1") addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.9.0") addSbtPlugin("com.github.eikek" % "sbt-openapi-schema" % "0.6.1") addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.13") addSbtPlugin("com.jsuereth" % "sbt-pgp" % "2.0.1") +addSbtPlugin("com.typesafe.sbt" % "sbt-ghpages" % "0.6.3") addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0") addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.7.4") addSbtPlugin("io.spray" % "sbt-revolver" % "0.9.1") diff --git a/website/README.md b/website/README.md new file mode 100644 index 00000000..2dbebd8d --- /dev/null +++ b/website/README.md @@ -0,0 +1,30 @@ +# Website + +This is the docspell website and documentation. + +## Building + +The website is created using [zola](https://github.com/getzola/zola) +static site generator. The (very minimal) dynamic parts are written in +Elm. + +The `build.sh` script builds the site. + + +## Development + +Install things by running `yarn install`. + +Open two terminals. In first run: + +``` shell +nix-shell --run ./run-elm.sh +``` + +and in the second + +``` shell +nix-shell --run "cd site && zola serve" +``` + +Open browser at `localhost:1111`. diff --git a/website/build.sh b/website/build.sh new file mode 100755 index 00000000..567601a8 --- /dev/null +++ b/website/build.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -e + +yarn install +elm make --output site/static/js/bundle.js --optimize elm/Main.elm +cd site && zola build +cd .. + +echo "Site is in site/public." diff --git a/website/elm-analyse.json b/website/elm-analyse.json new file mode 100644 index 00000000..837c4ada --- /dev/null +++ b/website/elm-analyse.json @@ -0,0 +1,9 @@ +{ + "excludedPaths": [ + "modules/webapp/target/elm-src/" + ], + "checks" : { + "ImportAll": false, + "SingleFieldRecord": false + } +} diff --git a/website/elm.json b/website/elm.json new file mode 100644 index 00000000..daaa0710 --- /dev/null +++ b/website/elm.json @@ -0,0 +1,28 @@ +{ + "type": "application", + "source-directories": [ + "elm" + ], + "elm-version": "0.19.1", + "dependencies": { + "direct": { + "elm/browser": "1.0.2", + "elm/core": "1.0.5", + "elm/html": "1.0.0", + "elm/random": "1.0.0", + "elm-community/random-extra": "3.1.0", + "elm-explorations/markdown": "1.0.0" + }, + "indirect": { + "elm/json": "1.1.3", + "elm/time": "1.0.0", + "elm/url": "1.0.0", + "elm/virtual-dom": "1.0.2", + "owanturist/elm-union-find": "1.0.0" + } + }, + "test-dependencies": { + "direct": {}, + "indirect": {} + } +} diff --git a/website/elm/ExtraAttr.elm b/website/elm/ExtraAttr.elm new file mode 100644 index 00000000..7d7fb64c --- /dev/null +++ b/website/elm/ExtraAttr.elm @@ -0,0 +1,36 @@ +module ExtraAttr exposing (..) + +import Html exposing (..) +import Html.Attributes exposing (..) + + +ariaExpanded : Bool -> Attribute msg +ariaExpanded flag = + attribute "aria-expanded" + (if flag then + "true" + + else + "false" + ) + + +ariaHidden : Bool -> Attribute msg +ariaHidden flag = + attribute "aria-hidden" + (if flag then + "true" + + else + "false" + ) + + +ariaLabel : String -> Attribute msg +ariaLabel name = + attribute "aria-label" name + + +role : String -> Attribute msg +role name = + attribute "role" name diff --git a/website/elm/Feature.elm b/website/elm/Feature.elm new file mode 100644 index 00000000..246aa7ad --- /dev/null +++ b/website/elm/Feature.elm @@ -0,0 +1,116 @@ +module Feature exposing (..) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Markdown + + +type alias Feature = + { image : String + , header : String + , description : String + } + + +featureBox : Int -> Feature -> Html msg +featureBox index f = + case isOdd index of + False -> + div [ class "columns is-vcentered box mb-5" ] + [ div [ class "column is-three-quarter" ] + [ figure [ class "image is-2by1 feature-image" ] + [ img [ src f.image ] [] + ] + ] + , div [ class "column" ] + [ h2 [ class "title" ] + [ text f.header + ] + , Markdown.toHtml [] + f.description + ] + ] + + True -> + div [ class "columns is-vcentered box mb-5" ] + [ div [ class "column is-three-quarter" ] + [ h2 [ class "title" ] + [ text f.header + ] + , Markdown.toHtml [] + f.description + ] + , div [ class "column" ] + [ figure [ class "image is-2by1 feature-image" ] + [ img [ src f.image ] [] + ] + ] + ] + + +features : List Feature +features = + [ { image = "img/user-feature.png" + , header = "Multi-User per Account" + , description = """ +Each account (a *collective*) can have multiple users that share the +same files. For example, everyone in your family can work with your +files while using their own account with their own settings. +""" + } + , { image = "img/ocr-feature.png" + , header = "Text Extraction with OCR" + , description = """ +Text is extracted from all files. For scanned documents/images, OCR is used by utilising tesseract. The text is analysed and is available for full-text search. +""" + } + , { image = "img/analyze-feature.png" + , header = "Text Analysis" + , description = """ +The extracted text is analyzed and is used to find properties that can be annotated to your documents automatically. +""" + } + , { image = "img/filetype-feature.svg" + , header = "Support for many files" + , description = """ +Docspell can read many file types. ZIP and EML (e-mail file format) files are extracted and their contents imported. +""" + } + , { image = "img/convertpdf-feature.svg" + , header = "Conversion to PDF" + , description = """ +All files are converted to PDF. Don't worry about the originals. Original files are stored, too and can be downloaded untouched. When creating PDFs from image data (often returned from scanners), the resulting PDF contains the extracted text and is searchable. +""" + } + , { image = "img/fts-feature.png" + , header = "Full-Text Search" + , description = """ +The extracted text of all files and some properties, like names and notes, are available for full-text search. Full-text search can also be used to further constrain the results of the search-menu where you can search by tags, correspondent, etc. +""" + } + , { image = "img/sendmail-feature.png" + , header = "Send via E-Mail" + , description = """ + +Users can define SMTP settings in the app and are then able to send items out via E-Mail. This is often useful to share with other people. There is e-mail-address completion from your address book, of course. + +""" + } + , { image = "img/scanmailbox-feature.png" + , header = "Import Mailboxes" + , description = """ +Users can define IMAP settings so that docspell can import their e-mails. This can be done periodically based on a schedule. Imported mails can be moved away into another folder or deleted. +""" + } + , { image = "img/notify-feature.png" + , header = "Notifications" + , description = """ +Users can be notified by e-mail for documents whose due-date comes closer. +""" + } + ] + + +isOdd : Int -> Bool +isOdd num = + modBy 2 num == 1 diff --git a/website/elm/GetStarted.elm b/website/elm/GetStarted.elm new file mode 100644 index 00000000..9e6df77c --- /dev/null +++ b/website/elm/GetStarted.elm @@ -0,0 +1,91 @@ +module GetStarted exposing (..) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Icons +import Markdown + + +getStarted : String -> List (Html msg) +getStarted version = + [ div [ class "content container is-size-5" ] + [ Markdown.toHtml [] + """Docspell consists of several components. The easiest way to get started is probably to use docker and +[docker-compose](https://docs.docker.com/compose/).""" + , Markdown.toHtml [] + ("""1. Clone the github repository + ```bash + $ git clone https://github.com/eikek/docspell + ``` + Alternatively, [download](https://github.com/eikek/docspell/archive/v""" + ++ version + ++ """.zip) the sources and extract the zip file. +2. Change into the `docker` directory: + ```bash + $ cd docspell/docker + ``` +3. Run `docker-compose up`: + + ```bash + $ export DOCSPELL_HEADER_VALUE="my-secret-123" + $ docker-compose up + ``` + + The environment variable defines a secret that is shared between + some containers. You can define whatever you like. Please see the + [`consumedir.sh`](doc/tools/consumedir#docker) docs for additional + info. +4. Goto , signup and login. When signing up, + you can choose the same name for collective and user. Then login + with this name and the password. + +5. (Optional) Create a folder `./docs/` (the name you + chose for the collective at registration) and place files in there + for importing them. + +The directory contains a file `docspell.conf` that you can +[modify](docs/configure) as needed. + """ + ) + ] + , div [ class "content container" ] + [ div [ class "notification is-info is-light" ] + [ text "If you don't use docker, there are other ways that are " + , text "described in the relevant " + , a [ href "/docs/install" ] + [ text "documentation page" + ] + ] + ] + , div [ class "content container" ] + [ div [ class "notification is-success is-light" ] + [ div [ class "content is-medium" ] + [ h3 [ class "title" ] + [ text "Where to go from here?" + ] + , ul [] + [ li [] + [ text "The " + , a [ href "/docs/intro" ] + [ text "introduction" ] + , text " writes about the goals and basic idea." + ] + , li [] + [ text "There is a comprehensive " + , a [ href "/docs" ] + [ text "documentation" + ] + , text " available." + ] + , li [] + [ text "The source code is hosted on " + , a [ href "https://github.com/eikek/docspell" ] + [ text "github" + ] + , text "." + ] + ] + ] + ] + ] + ] diff --git a/website/elm/Icons.elm b/website/elm/Icons.elm new file mode 100644 index 00000000..1e16ec5a --- /dev/null +++ b/website/elm/Icons.elm @@ -0,0 +1,54 @@ +module Icons exposing (..) + +import Html exposing (..) +import Html.Attributes exposing (..) + + +copyright : Html msg +copyright = + img [ src "icons/copyright-40.svg" ] [] + + +infoSquared : Html msg +infoSquared = + img [ src "icons/info-square-40.svg" ] [] + + +refresh : Html msg +refresh = + img [ src "icons/refresh-40.svg" ] [] + + +logo : Html msg +logo = + img [ src "icons/logo-only-36.svg" ] [] + + +logoMC : Html msg +logoMC = + img [ src "icons/logo-only-mc.svg" ] [] + + +logoWidth : Int -> Html msg +logoWidth w = + img [ src "icons/logo-only.svg", width w ] [] + + +home : Html msg +home = + img [ src "icons/home-40.svg" ] [] + + +docs : Html msg +docs = + img [ src "icons/notes-40.svg" ] [] + + +github : Html msg +github = + img [ src "/icons/github-40.svg" ] [] + + +githubGreen : Html msg +githubGreen = + img [ src "/icons/github-40-green.svg" ] [] diff --git a/website/elm/Main.elm b/website/elm/Main.elm new file mode 100644 index 00000000..ba19c1ce --- /dev/null +++ b/website/elm/Main.elm @@ -0,0 +1,324 @@ +module Main exposing (..) + +import Browser exposing (Document) +import Browser.Navigation exposing (Key) +import ExtraAttr exposing (..) +import Feature exposing (Feature) +import GetStarted +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (..) +import Icons +import Random +import Random.List + + + +-- MAIN + + +main : Program Flags Model Msg +main = + Browser.element + { init = init + , view = view + , update = update + , subscriptions = subscriptions + } + + + +--- Model + + +type alias Flags = + { version : String + } + + +type alias Model = + { navbarOpen : Bool + , features : List Feature + , flags : Flags + } + + +type Msg + = ToggleNavbarMenu + | ShuffleFeatures + | ListShuffled (List Feature) + + + +--- Init + + +viewFeatureCount : Int +viewFeatureCount = + 10 + + +init : Flags -> ( Model, Cmd Msg ) +init flags = + ( { navbarOpen = False + , features = List.take viewFeatureCount Feature.features + , flags = flags + } + , Cmd.none + ) + + +shuffleFeatures : Cmd Msg +shuffleFeatures = + Random.List.shuffle Feature.features + |> Random.map (List.take viewFeatureCount) + |> Random.generate ListShuffled + + + +--- Update + + +update : Msg -> Model -> ( Model, Cmd Msg ) +update msg model = + case msg of + ToggleNavbarMenu -> + ( { model | navbarOpen = not model.navbarOpen } + , Cmd.none + ) + + ShuffleFeatures -> + ( model, shuffleFeatures ) + + ListShuffled lf -> + ( { model | features = lf } + , Cmd.none + ) + + +subscriptions : Model -> Sub Msg +subscriptions _ = + Sub.none + + + +--- View + + +view : Model -> Html Msg +view model = + node "body" + [] + [ mainHero model + , featureHero model + , section [ class "section" ] + [ div [ class "container" ] + (List.indexedMap Feature.featureBox model.features + ++ [ div [ class "columns box" ] + [ div [ class "column is-full" ] + [ div [ class "content has-text-centered is-medium" ] + [ text "A more complete list can be found in " + , a [ href "/docs/features" ] [ text "here" ] + , text "." + ] + ] + ] + ] + ) + ] + , getStartedHero model + , div [ class "section" ] + (GetStarted.getStarted model.flags.version) + , footHero model + ] + + +footHero : Model -> Html Msg +footHero model = + footer + [ id "footer" + , class "footer" + ] + [ div [ class "has-text-centered" ] + [ span [] + [ text ("Docspell, " ++ model.flags.version) + ] + , span [ class "pr-1 pl-1" ] + [ text " • " + ] + , a + [ href "https://spdx.org/licenses/GPL-3.0-or-later.html" + , target "_blank" + ] + [ text "GPLv3+" + ] + , span [ class "pr-1 pl-1" ] + [ text " • " + ] + , a + [ href "https://github.com/eikek/docspell" + , target "_blank" + ] + [ text "Source Code" + ] + , span [ class "pr-1 pl-1" ] + [ text " • " + ] + , span [] + [ text "© 2020 " + ] + , a + [ href "https://github.com/eikek" + , target "_blank" + ] + [ text "@eikek" + ] + ] + ] + + +getStartedHero : Model -> Html Msg +getStartedHero _ = + section + [ id "get-started" + , class "hero is-primary is-bold" + ] + [ div [ class "hero-body" ] + [ div [ class "container" ] + [ h2 [ class "title" ] + [ text "Get Started" + ] + ] + ] + ] + + +featureHero : Model -> Html Msg +featureHero model = + section + [ id "feature-selection" + , class "hero is-info is-bold" + ] + [ div + [ class "hero-body" + ] + [ div [ class "container" ] + [ h2 [ class "title" ] + [ text "Feature Selection" + ] + ] + ] + ] + + +mainHero : Model -> Html Msg +mainHero model = + section + [ id "hero-main" + , class "hero is-fullheight is-primary" + ] + [ div [ class "hero-head" ] + [ nav [ class "navbar" ] + [ div [ class "navbar-brand" ] + [ a + [ class "navbar-item" + , href "/" + ] + [ span [ class "icon is-large" ] + [ Icons.logo + ] + , text "Docspell" + ] + , a + [ role "button" + , onClick ToggleNavbarMenu + , classList + [ ( "navbar-burger", True ) + , ( "is-active", model.navbarOpen ) + ] + , ariaLabel "menu" + , ariaExpanded False + ] + [ span [ ariaHidden True ] [] + , span [ ariaHidden True ] [] + , span [ ariaHidden True ] [] + ] + ] + , div + [ classList + [ ( "navbar-menu", True ) + , ( "is-active", model.navbarOpen ) + ] + ] + [ div [ class "navbar-start" ] + [ a + [ href "docs/" + , class "navbar-item" + ] + [ span [ class "icon" ] + [ Icons.docs + ] + , span [] + [ text "Documentation" + ] + ] + , a + [ target "_blank" + , href "https://github.com/eikek/docspell" + , class "navbar-item" + ] + [ span [ class "icon" ] + [ Icons.github + ] + , span [] + [ text "Github" + ] + ] + ] + ] + ] + ] + , div [ class "hero-body" ] + [ div + [ class "container has-text-centered" + ] + [ Icons.logoWidth 112 + , h1 [ class "title main-title is-2" ] + [ text "Docspell" + ] + , h2 [ class "subtitle is-3" ] + [ text "Simple document organizer" + ] + , p [ class "content is-medium" ] + [ text "Docspell can assist in organizing your piles of " + , text "digital documents, resulting from scanners, e-mails " + , text "and other sources with miminal effort." + ] + , div [ class " buttons is-centered" ] + [ a + [ class "button is-primary is-medium" + , href "#get-started" + ] + [ text "Get Started" + ] + , a + [ class "button is-info is-medium" + , href "#feature-selection" + ] + [ text "Features" + ] + ] + ] + ] + , div [ class "hero-foot" ] + [ span [ class "unsplash-credit" ] + [ text "Photo by " + , a + [ href "https://unsplash.com/@numericcitizen" + , target "_blank" + ] + [ text "JF Martin" + ] + ] + ] + ] diff --git a/website/package.json b/website/package.json new file mode 100644 index 00000000..1679a004 --- /dev/null +++ b/website/package.json @@ -0,0 +1,6 @@ +{ + "license": "GPL-3.0-or-later", + "dependencies": { + "bulma": "^0.9.0" + } +} diff --git a/website/run-elm.sh b/website/run-elm.sh new file mode 100755 index 00000000..785bc548 --- /dev/null +++ b/website/run-elm.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +CMD="elm make --output site/static/js/bundle.js --optimize elm/Main.elm" +$CMD + +inotifywait -m -e close_write -r elm/ | + while read f; do + $CMD + done diff --git a/website/shell.nix b/website/shell.nix new file mode 100644 index 00000000..4fed5990 --- /dev/null +++ b/website/shell.nix @@ -0,0 +1,15 @@ +let + nixpkgsUnstable = builtins.fetchTarball { + url = "https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz"; + }; + pkgsUnstable = import nixpkgsUnstable { }; +in +with pkgsUnstable; + + mkShell { + buildInputs = [ + zola + yarn + inotifyTools + ]; + } diff --git a/website/site/config.toml b/website/site/config.toml new file mode 100644 index 00000000..f701f182 --- /dev/null +++ b/website/site/config.toml @@ -0,0 +1,29 @@ +# The URL the site will be built for +base_url = "https://docspell.org" + +# Whether to automatically compile all Sass files in the sass directory +compile_sass = true + +# Whether to do syntax highlighting +# Theme can be customised by setting the `highlight_theme` variable to a theme supported by Zola +highlight_code = true + +highlight_theme = "gruvbox-dark" + +# Whether to build a search index to be used later on by a JavaScript library +build_search_index = true + +[link_checker] +skip_prefixes = [ + "http://localhost", + "/openapi", + "https://www.abisource.com" # has bad ssl config +] +skip_anchor_prefixes = [ + "https://github.com", + "https://package.elm-lang.org" +] + +[extra] +# Put all your custom variables here +version = "0.9.0-SNAPSHOT" diff --git a/website/site/content/doc/_index.md b/website/site/content/doc/_index.md new file mode 100644 index 00000000..502bf030 --- /dev/null +++ b/website/site/content/doc/_index.md @@ -0,0 +1,3 @@ ++++ +redirect_to = "/docs" ++++ diff --git a/website/site/content/docs/_index.md b/website/site/content/docs/_index.md new file mode 100644 index 00000000..630db7b9 --- /dev/null +++ b/website/site/content/docs/_index.md @@ -0,0 +1,9 @@ ++++ +title = "Overview" +template = "overview.html" +insert_anchor_links = "right" ++++ + +# Note + +This content is not rendered. Everything is in the template. diff --git a/modules/microsite/docs/api.md b/website/site/content/docs/api/_index.md similarity index 90% rename from modules/microsite/docs/api.md rename to website/site/content/docs/api/_index.md index ce7e9291..fd0b8718 100644 --- a/modules/microsite/docs/api.md +++ b/website/site/content/docs/api/_index.md @@ -1,19 +1,20 @@ ---- -layout: docs -title: Api -permalink: api ---- - -# {{page.title}} ++++ +title = "Api" +description = "Contains documentation about the REST API." +weight = 70 +insert_anchor_links = "right" +[extra] +mktoc = true ++++ Docspell is designed as a REST server that uses JSON to exchange data. The REST api can be used to integrate docspell into your workflow. -[Docspell REST Api Doc](openapi/docspell-openapi.html) +[Docspell REST Api Doc](/openapi/docspell-openapi.html) The "raw" `openapi.yml` specification file can be found -[here](openapi/docspell-openapi.yml). +[here](/openapi/docspell-openapi.yml). The routes can be divided into protected and unprotected routes. The unprotected, or open routes are at `/open/*` while the protected @@ -56,7 +57,7 @@ These examples use the great command line tool ### Login -``` +``` bash $ curl -X POST -d '{"account": "smith", "password": "test"}' http://localhost:7880/api/v1/open/auth/login {"collective":"smith" ,"user":"smith" @@ -69,7 +70,7 @@ $ curl -X POST -d '{"account": "smith", "password": "test"}' http://localhost:78 ### Get new token -``` +``` bash $ curl -XPOST -H 'X-Docspell-Auth: 1568142350115-ZWlrZS9laWtl-$2a$10$rGZUFDAVNIKh4Tj6u6tlI.-O2euwCvmBT0TlyDmIHR1ZsLQPAI=' http://localhost:7880/api/v1/sec/auth/session {"collective":"smith" ,"user":"smith" @@ -82,7 +83,7 @@ $ curl -XPOST -H 'X-Docspell-Auth: 1568142350115-ZWlrZS9laWtl-$2a$10$rGZUFDAVNIK ### Get some insights -``` +``` bash $ curl -H 'X-Docspell-Auth: 1568142446077-ZWlrZS9laWtl-$2a$10$3B0teJ9rMpsBJPzHfZZPoO-WeA1bkfEONBN8fyzWE8DeaAHtUc=' http://localhost:7880/api/v1/sec/collective/insights {"incomingCount":3 ,"outgoingCount":1 diff --git a/modules/microsite/docs/doc/configure.md b/website/site/content/docs/configure/_index.md similarity index 91% rename from modules/microsite/docs/doc/configure.md rename to website/site/content/docs/configure/_index.md index 4a6ad60a..7606078c 100644 --- a/modules/microsite/docs/doc/configure.md +++ b/website/site/content/docs/configure/_index.md @@ -1,10 +1,11 @@ ---- -layout: docs -title: Configuring -permalink: doc/configure ---- - -# {{ page.title }} ++++ +title = "Configuration" +insert_anchor_links = "right" +description = "There are several tools distributed with docspell, like a program to watch a folder and import files to docspell." +weight = 40 +[extra] +mktoc = true ++++ Docspell's executable can take one argument – a configuration file. If that is not given, the defaults are used. The config file overrides @@ -13,13 +14,13 @@ necessary. This applies to the restserver and the joex as well. -## Important Config Options +# Important Config Options The configuration of both components uses separate namespaces. The configuration for the REST server is below `docspell.server`, while the one for joex is below `docspell.joex`. -### JDBC +## JDBC This configures the connection to the database. This has to be specified for the rest server and joex. By default, a H2 database in @@ -27,7 +28,7 @@ the current `/tmp` directory is configured. The config looks like this (both components): -``` +``` conf docspell.joex.jdbc { url = ... user = ... @@ -55,7 +56,7 @@ include these options: ;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_SERVER=TRUE ``` -#### Examples +### Examples PostgreSQL: ``` @@ -73,14 +74,14 @@ url = "jdbc:h2:///path/to/a/file.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_ ``` -### Full-Text Search: SOLR +## Full-Text Search: SOLR [Apache SOLR](https://lucene.apache.org/solr) is used to provide the full-text search. Both docspell components must provide the same connection setup. This is defined in the `full-text-search.solr` subsection: -``` +``` conf ... full-text-search { enabled = true @@ -108,7 +109,7 @@ While the `full-text-search.solr` options are the same for joex and the restserver, there are some settings that differ. The restserver has this additional setting, that may be of interest: -``` +``` conf full-text-search { recreate-key = "test123" } @@ -117,7 +118,7 @@ full-text-search { This key is required if you want docspell to drop and re-create the entire index. This is possible via a REST call: -``` shell +``` bash $ curl -XPOST http://localhost:7880/api/v1/open/fts/reIndexAll/test123 ``` @@ -138,13 +139,13 @@ The solr index doesn't contain any new information, it can be regenerated any time using the above REST call. Thus it doesn't need to be backed up. -### Bind +## Bind The host and port the http server binds to. This applies to both components. The joex component also exposes a small REST api to inspect its state and notify the scheduler. -``` +``` conf docspell.server.bind { address = localhost port = 7880 @@ -158,7 +159,7 @@ docspell.joex.bind { By default, it binds to `localhost` and some predefined port. This must be changed, if components are on different machines. -### baseurl +## Baseurl The base url is an important setting that defines the http URL where the corresponding component can be reached. It applies to both @@ -174,7 +175,7 @@ docspell.server.baseurl = ... docspell.joex.baseurl = ... ``` -#### Examples +### Examples ``` docspell.server.baseurl = "https://docspell.example.com" @@ -182,19 +183,19 @@ docspell.joex.baseurl = "http://192.168.101.10" ``` -### app-id +## App-id The `app-id` is the identifier of the corresponding instance. It *must be unique* for all instances. By default the REST server uses `rest1` and joex `joex1`. It is recommended to overwrite this setting to have an explicit and stable identifier. -``` +``` conf docspell.server.app-id = "rest1" docspell.joex.app-id = "joex1" ``` -### registration options +## Registration Options This defines if and how new users can create accounts. There are 3 options: @@ -205,7 +206,7 @@ options: This applies only to the REST sevrer component. -``` +``` conf docspell.server.signup { mode = "open" @@ -227,11 +228,11 @@ this. If it is not set, then invitation won't work. New invitation keys can be generated from within the web application or via REST calls (using `curl`, for example). -``` +``` bash curl -X POST -d '{"password":"blabla"}' "http://localhost:7880/api/v1/open/signup/newinvite" ``` -### Authentication +## Authentication Authentication works in two ways: @@ -245,7 +246,7 @@ token can be given as a normal http header or via a cookie header. These settings apply only to the REST server. -``` +``` conf docspell.server.auth { server-secret = "hex:caffee" # or "b64:Y2FmZmVlCg==" session-valid = "5 minutes" @@ -264,7 +265,7 @@ just some minutes, the web application obtains new ones periodically. So a short time is recommended. -## File Format +# File Format The format of the configuration files can be [HOCON](https://github.com/lightbend/config/blob/master/HOCON.md#hocon-human-optimized-config-object-notation), @@ -272,27 +273,25 @@ JSON or whatever the used [config library](https://github.com/lightbend/config) understands. The default values below are in HOCON format, which is recommended, since it allows comments and has some [advanced -features](https://github.com/lightbend/config/blob/master/README.md#features-of-hocon). Please -refer to their documentation for more on this. +features](https://github.com/lightbend/config#features-of-hocon). +Please refer to their documentation for more on this. Here are the default configurations. -## Default Config +# Default Config +## Rest Server -### Rest Server +{{ incl_conf(path="templates/shortcodes/server.conf") }} -``` -{% include server.conf %} -``` -### Joex +## Joex -``` -{% include joex.conf %} -``` -## Logging +{{ incl_conf(path="templates/shortcodes/joex.conf") }} + + +# Logging By default, docspell logs to stdout. This works well, when managed by systemd or other inits. Logging is done by @@ -302,7 +301,7 @@ for how to configure logging. If you created your logback config file, it can be added as argument to the executable using this syntax: -``` +``` bash /path/to/docspell -Dlogback.configurationFile=/path/to/your/logging-config-file ``` diff --git a/website/site/content/docs/dev/_index.md b/website/site/content/docs/dev/_index.md new file mode 100644 index 00000000..da91edbc --- /dev/null +++ b/website/site/content/docs/dev/_index.md @@ -0,0 +1,9 @@ ++++ +title = "Development" +description = "Contains build instructions and other internal notes." +weight = 300 +sort_by = "weight" +insert_anchor_links = "right" +template = "pages.html" +redirect_to = "/docs/dev/building" ++++ diff --git a/modules/microsite/docs/dev/adr/0000_use_markdown_architectural_decision_records.md b/website/site/content/docs/dev/adr/0000_use_markdown_architectural_decision_records.md similarity index 84% rename from modules/microsite/docs/dev/adr/0000_use_markdown_architectural_decision_records.md rename to website/site/content/docs/dev/adr/0000_use_markdown_architectural_decision_records.md index 8209471a..0cf80e8d 100644 --- a/modules/microsite/docs/dev/adr/0000_use_markdown_architectural_decision_records.md +++ b/website/site/content/docs/dev/adr/0000_use_markdown_architectural_decision_records.md @@ -1,18 +1,15 @@ ---- -layout: docs -title: Use Markdown Architectural Decision Records -permalink: dev/adr/0000 ---- ++++ +title = "Use Markdown Architectural Decision Records" +weight = 10 ++++ -# Use Markdown Architectural Decision Records - -## Context and Problem Statement +# Context and Problem Statement We want to [record architectural decisions](https://adr.github.io/) made in this project. Which format and structure should these records follow? -## Considered Options +# Considered Options * [MADR](https://adr.github.io/madr/) 2.1.0 - The Markdown Architectural Decision Records * [Michael Nygard's template](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions) - The first incarnation of the term "ADR" @@ -23,7 +20,7 @@ follow? * Formless - No conventions for file format and structure -## Decision Outcome +# Decision Outcome Chosen option: "MADR 2.1.0", because diff --git a/modules/microsite/docs/dev/adr/0001_components.md b/website/site/content/docs/dev/adr/0001_components.md similarity index 88% rename from modules/microsite/docs/dev/adr/0001_components.md rename to website/site/content/docs/dev/adr/0001_components.md index b1e051fd..f31ebabb 100644 --- a/modules/microsite/docs/dev/adr/0001_components.md +++ b/website/site/content/docs/dev/adr/0001_components.md @@ -1,29 +1,26 @@ ---- -layout: docs -title: Components -permalink: dev/adr/0001_components ---- ++++ +title = "Components" +weight = 20 ++++ -# Components - -## Context and Problem Statement +# Context and Problem Statement How should the application be structured into its main components? The goal is to be able to have multiple rest servers/webapps and multiple document processor components working togehter. -## Decision Outcome +# Decision Outcome The following are the "main" modules. There may be more helper modules and libraries that support implementing a feature. -### store +## store The code related to database access. It also provides the job queue. It is designed as a library. -### joex +## joex Joex stands for "job executor". @@ -37,7 +34,7 @@ It provides the document processing code. It provides a http rest server to get insight into the joex state and also to be notified for new jobs. -### backend +## backend It provides all the logic, except document processing, as a set of "operations". An operation can be directly mapped to a rest @@ -45,7 +42,7 @@ endpoint. It is designed as a library. -### rest api +## rest api This module contains the specification for the rest server as an `openapi.yml` file. It is packaged as a scala library that also @@ -54,7 +51,7 @@ provides types and conversions to/from json. The idea is that the `rest server` module can depend on it as well as rest clients. -### rest server +## rest server This is the main application. It directly depends on the `backend` module, and each rest endpoint maps to a "backend operation". It is @@ -62,6 +59,6 @@ also responsible for converting the json data inside http requests to/from types recognized by the `backend` module. -### webapp +## webapp This module provides the user interface as a web application. diff --git a/modules/microsite/docs/dev/adr/0002_component_interaction.md b/website/site/content/docs/dev/adr/0002_component_interaction.md similarity index 89% rename from modules/microsite/docs/dev/adr/0002_component_interaction.md rename to website/site/content/docs/dev/adr/0002_component_interaction.md index 1a94c30f..888b233f 100644 --- a/modules/microsite/docs/dev/adr/0002_component_interaction.md +++ b/website/site/content/docs/dev/adr/0002_component_interaction.md @@ -1,12 +1,9 @@ ---- -layout: docs -title: Component Interaction -permalink: dev/adr/0002_component_interaction ---- ++++ +title = "Component Interaction" +weight = 30 ++++ -# Component Interaction - -## Context and Problem Statement +# Context and Problem Statement There are multiple web applications with their rest servers and there are multiple document processors. These processes must communicate: @@ -18,14 +15,14 @@ are multiple document processors. These processes must communicate: user that is currently logged in. Since it's not known which rest-server the user is using right now, all must be notified. -## Considered Options +# Considered Options 1. JMS (ActiveMQ or similiar): Message Broker as another active component 2. Akka: using a cluster 3. DB: Register with "call back urls" -## Decision Outcome +# Decision Outcome Choosing option 3: DB as central synchronisation point. @@ -42,7 +39,7 @@ It works roughly like this: - each component has db access - rest servers can list all processors and vice versa -### Positive Consequences +## Positive Consequences - complexity of the whole application is not touched - since a lot of data must be transferred to the document processors, @@ -53,7 +50,7 @@ It works roughly like this: - no additional knowledge required - simple to understand and so not hard to debug -### Negative Consequences +## Negative Consequences - all components must have db access. this also is a security con, because if one of those processes is hacked, db access is diff --git a/modules/microsite/docs/dev/adr/0003_encryption.md b/website/site/content/docs/dev/adr/0003_encryption.md similarity index 93% rename from modules/microsite/docs/dev/adr/0003_encryption.md rename to website/site/content/docs/dev/adr/0003_encryption.md index 4d4ef7ff..bc7a134f 100644 --- a/modules/microsite/docs/dev/adr/0003_encryption.md +++ b/website/site/content/docs/dev/adr/0003_encryption.md @@ -1,13 +1,10 @@ ---- -layout: docs -title: Encryption -permalink: dev/adr/0003_encryption ---- - -# Encryption ++++ +title = "Encryption" +weight = 40 ++++ -## Context and Problem Statement +# Context and Problem Statement Since docspell may store important documents, it should be possible to encrypt them on the server. It should be (almost) transparent to the @@ -29,21 +26,21 @@ So in short, encryption means: such that files can be downloaded in clear form -## Decision Drivers +# Decision Drivers * major driver is to provide most possible privacy for users * even at the expense of less features; currently I think that the associated meta data is enough for finding documents (i.e. full text search is not needed) -## Considered Options +# Considered Options It is clear, that only blobs (file contents) can be encrypted, but not the associated metadata. And the extracted text must be encrypted, too, obviously. -### Public Key Encryption (PKE) +## Public Key Encryption (PKE) With PKE that the server can automatically encrypt files using publicly available key data. It wouldn't require a user to provide a @@ -68,7 +65,7 @@ attacks and also for accidental leakage (for example, if a bug in the software would access a file of another user). -### No Encryption +## No Encryption If only blobs are encrypted, against which type of attack would it provide protection? @@ -87,7 +84,7 @@ But the downside is, that it increases complexity *a lot*. And since this is a personal tool for personal use, is it worth the effort? -## Decision Outcome +# Decision Outcome No encryption, because of its complexity. diff --git a/modules/microsite/docs/dev/adr/0004_iso8601vsEpoch.md b/website/site/content/docs/dev/adr/0004_iso8601vsEpoch.md similarity index 87% rename from modules/microsite/docs/dev/adr/0004_iso8601vsEpoch.md rename to website/site/content/docs/dev/adr/0004_iso8601vsEpoch.md index 174a9808..4bda7210 100644 --- a/modules/microsite/docs/dev/adr/0004_iso8601vsEpoch.md +++ b/website/site/content/docs/dev/adr/0004_iso8601vsEpoch.md @@ -1,12 +1,9 @@ ---- -layout: docs -title: ISO8601 vs Millis -permalink: dev/adr/0004_iso8601vsEpoch ---- ++++ +title = "ISO8601 vs Millis as Date-Time transfer" +weight = 50 ++++ -# ISO8601 vs Millis as Date-Time transfer - -## Context and Problem Statement +# Context and Problem Statement The question is whether the REST Api should return an ISO8601 formatted string in UTC timezone, or the unix time (number of @@ -29,7 +26,7 @@ information in a ISO8601 string than in the epoch millis. To avoid confusion, all date/time values should use the same encoding. -## Decision Outcome +# Decision Outcome I go with the epoch time. Every timestamp/date-time values is transfered as Unix timestamp. diff --git a/modules/microsite/docs/dev/adr/0005_job-executor.md b/website/site/content/docs/dev/adr/0005_job-executor.md similarity index 96% rename from modules/microsite/docs/dev/adr/0005_job-executor.md rename to website/site/content/docs/dev/adr/0005_job-executor.md index 8d1add2d..01744e71 100644 --- a/modules/microsite/docs/dev/adr/0005_job-executor.md +++ b/website/site/content/docs/dev/adr/0005_job-executor.md @@ -1,12 +1,9 @@ ---- -layout: docs -title: Joex - Job Executor -permalink: dev/adr/0005_job-executor ---- ++++ +title = "Joex - Job Executor" +weight = 60 ++++ -# Job Executor - -## Context and Problem Statement +# Context and Problem Statement Docspell is a multi-user application. When processing user's documents, there must be some thought on how to distribute all the @@ -40,7 +37,7 @@ associated task. This is used to run the document processing jobs program was not installed or whatever). In such a case it is good to know that the task will be retried later. -## Considered Options +# Considered Options In contrast to other ADRs this is just some sketching of thoughts for the current implementation. @@ -71,7 +68,7 @@ the current implementation. jobs that are due at the specific point in time ignoring the priority hint. -### More Details +## More Details A job has these properties @@ -115,7 +112,7 @@ Once there are no jobs in the queue the executor goes into sleep and must be waked to run again. If a job is submitted, the executors are notified. -### Stuck Jobs +## Stuck Jobs A job is going into _stuck_ state, if the task has failed. In this state, the task is rerun after a while until a maximum retry count is diff --git a/modules/microsite/docs/dev/adr/0006_more-file-types.md b/website/site/content/docs/dev/adr/0006_more-file-types.md similarity index 85% rename from modules/microsite/docs/dev/adr/0006_more-file-types.md rename to website/site/content/docs/dev/adr/0006_more-file-types.md index d406ade7..3c27490e 100644 --- a/modules/microsite/docs/dev/adr/0006_more-file-types.md +++ b/website/site/content/docs/dev/adr/0006_more-file-types.md @@ -1,12 +1,9 @@ ---- -layout: docs -title: More File Types -permalink: dev/adr/0006_more-file-types ---- ++++ +title = "More File Types" +weight = 70 ++++ -# More File Types - -## Context and Problem Statement +# Context and Problem Statement Docspell currently only supports PDF files. This has simplified early development and design a lot and so helped with starting the project. @@ -38,7 +35,7 @@ since the conversion may not always create best results, the original file should be kept. -## Decision Drivers +# Decision Drivers People expect that software like Docspell support the most common document types, like all the “office documents” (`docx`, `rtf`, `odt`, @@ -47,15 +44,15 @@ those files instead of PDF. Some (older) scanners may not be able to scan into PDF files but only to image files. -## Considered Options +# Considered Options This ADR does not evaluate different options. It rather documents why this feature is realized and the thoughts that lead to how it is implemented. -## Realization +# Realization -### Data Model +## Data Model The `attachment` table holds one file. There will be another table `attachment_source` that holds the original file. It looks like this: @@ -90,7 +87,7 @@ pointing to an `attachment_source` is also not correct, because it allows the same `attachment_source` record to be associated with many `attachment` records. This would do even more harm, in my opinion. -### Migration +## Migration Creating a new table and not altering existing ones, should simplify data migration. @@ -101,7 +98,7 @@ inserted into the new table. This presents the trivial case where the attachment and source are the same. -### Processing +## Processing The first step in processing is now converting the file into a pdf. If it already is a pdf, nothing is done. This step is before text @@ -113,7 +110,7 @@ If conversion is not supported for the input file, it is skipped. If conversion fails, the error is propagated to let the retry mechanism take care. -#### What types? +### What types? Which file types should be supported? At a first step, all major office documents, common images, plain text (i.e. markdown) and html @@ -124,20 +121,18 @@ There is always the preference to use jvm internal libraries in order to be more platform independent and to reduce external dependencies. But this is not always possible (like doing OCR). -
- -
+{{ figure(file="process-files.png") }} -#### Conversion +### Conversion - Office documents (`doc`, `docx`, `xls`, `xlsx`, `odt`, `ods`): - unoconv (see [ADR 9](0009_convert_office_docs)) -- HTML (`html`): wkhtmltopdf (see [ADR 7](0007_convert_html_files)) + unoconv (see [ADR 9](@/docs/dev/adr/0009_convert_office_docs.md)) +- HTML (`html`): wkhtmltopdf (see [ADR 7](@/docs/dev/adr/0007_convert_html_files.md)) - Text/Markdown (`txt`, `md`): Java-Lib flexmark + wkhtmltopdf - Images (`jpg`, `png`, `tif`): Tesseract (see [ADR - 10](0010_convert_image_files)) + 10](@/docs/dev/adr/0010_convert_image_files.md)) -#### Text Extraction +### Text Extraction - Office documents (`doc`, `docx`, `xls`, `xlsx`): Apache Poi - Office documends (`odt`, `ods`): Apache Tika (including the sources) @@ -146,10 +141,10 @@ But this is not always possible (like doing OCR). - Text/Markdown: n.a. - PDF: Apache PDFBox or Tesseract -## Links +# Links -* [Convert HTML Files](0007_convert_html_files) -* [Convert Plain Text](0008_convert_plain_text) -* [Convert Office Documents](0009_convert_office_docs) -* [Convert Image Files](0010_convert_image_files) -* [Extract Text from Files](0011_extract_text) +* [Convert HTML Files](@/docs/dev/adr/0007_convert_html_files.md) +* [Convert Plain Text](@/docs/dev/adr/0008_convert_plain_text.md) +* [Convert Office Documents](@/docs/dev/adr/0009_convert_office_docs.md) +* [Convert Image Files](@/docs/dev/adr/0010_convert_image_files.md) +* [Extract Text from Files](@/docs/dev/adr/0011_extract_text.md) diff --git a/modules/microsite/docs/dev/adr/0007_convert_html_files.md b/website/site/content/docs/dev/adr/0007_convert_html_files.md similarity index 52% rename from modules/microsite/docs/dev/adr/0007_convert_html_files.md rename to website/site/content/docs/dev/adr/0007_convert_html_files.md index c121a879..164505d3 100644 --- a/modules/microsite/docs/dev/adr/0007_convert_html_files.md +++ b/website/site/content/docs/dev/adr/0007_convert_html_files.md @@ -1,12 +1,9 @@ ---- -layout: docs -title: Convert HTML Files -permalink: dev/adr/0007_convert_html_files ---- ++++ +title = "Convert HTML Files" +weight = 80 ++++ -# {{ page.title }} - -## Context and Problem Statement +# Context and Problem Statement How can HTML documents be converted into a PDF file that looks as much as possible like the original? @@ -17,7 +14,7 @@ has a better outcome, then an external tool is fine, too. Since Docspell is free software, the tools must also be free. -## Considered Options +# Considered Options * [pandoc](https://pandoc.org/) external command * [wkhtmltopdf](https://wkhtmltopdf.org/) external command @@ -25,9 +22,7 @@ Since Docspell is free software, the tools must also be free. Native (firefox) view: -
- -
+{{ figure(file="example-html-native.jpg") }} Note: the example html is from [here](https://www.sparksuite.com/open-source/invoice.html). @@ -36,36 +31,28 @@ I downloaded the HTML file to disk together with its resources (using *Save as...* in the browser). -### Pandoc +## Pandoc -
- -
+{{ figure(file="example-html-pandoc-latex.jpg") }} -
- -
+{{ figure(file="example-html-pandoc-html.jpg") }} Not showing the version using `context` pdf-engine, since it looked very similiar to the latex variant. -### wkhtmltopdf +## wkhtmltopdf -
- -
+{{ figure(file="example-html-wkhtmltopdf.jpg") }} -### Unoconv +## Unoconv -
- -
+{{ figure(file="example-html-unoconv.jpg") }} -## Decision Outcome +# Decision Outcome wkhtmltopdf. diff --git a/modules/microsite/docs/dev/adr/0008_convert_plain_text.md b/website/site/content/docs/dev/adr/0008_convert_plain_text.md similarity index 82% rename from modules/microsite/docs/dev/adr/0008_convert_plain_text.md rename to website/site/content/docs/dev/adr/0008_convert_plain_text.md index 2545712c..743f53cc 100644 --- a/modules/microsite/docs/dev/adr/0008_convert_plain_text.md +++ b/website/site/content/docs/dev/adr/0008_convert_plain_text.md @@ -1,12 +1,9 @@ ---- -layout: docs -title: Convert Text Files -permalink: dev/adr/0008_convert_plain_text ---- ++++ +title = "Convert Text Files" +weight = 90 ++++ -# {{ page.title }} - -## Context and Problem Statement +# Context and Problem Statement How can plain text and markdown documents be converted into a PDF files? @@ -97,15 +94,15 @@ the end. ``` -## Considered Options +# Considered Options * [flexmark](https://github.com/vsch/flexmark-java) for markdown to HTML, then use existing machinery described in [adr - 7](./0007_convert_html_files) + 7](@/docs/dev/adr/0007_convert_html_files.md) * [pandoc](https://pandoc.org/) external command -### flexmark markdown library for java +## flexmark markdown library for java Process files with [flexmark](https://github.com/vsch/flexmark-java) and then create a PDF from the resulting html. @@ -136,17 +133,13 @@ def renderMarkdown(): ExitCode = { Then run the result through `wkhtmltopdf`. Markdown file: -
- -
+{{ figure(file="example-md-java.jpg") }} TXT file: -
- -
+{{ figure(file="example-txt-java.jpg") }} -### pandoc +## pandoc Command: @@ -155,27 +148,19 @@ pandoc -f markdown -t html -o test.pdf microsite/docs/doc.md ``` Markdown/Latex: -
- -
+{{ figure(file="example-md-pandoc-latex.jpg") }} Markdown/Html: -
- -
+{{ figure(file="example-md-pandoc-html.jpg") }} Text/Latex: -
- -
+{{ figure(file="example-txt-pandoc-latex.jpg") }} Text/Html: -
- -
+{{ figure(file="example-txt-pandoc-html.jpg") }} -## Decision Outcome +# Decision Outcome Java library "flexmark". diff --git a/modules/microsite/docs/dev/adr/0009_convert_office_docs.md b/website/site/content/docs/dev/adr/0009_convert_office_docs.md similarity index 75% rename from modules/microsite/docs/dev/adr/0009_convert_office_docs.md rename to website/site/content/docs/dev/adr/0009_convert_office_docs.md index 9d5d6d0c..40f74c11 100644 --- a/modules/microsite/docs/dev/adr/0009_convert_office_docs.md +++ b/website/site/content/docs/dev/adr/0009_convert_office_docs.md @@ -1,12 +1,9 @@ ---- -layout: docs -title: Convert Office Documents -permalink: dev/adr/0009_convert_office_docs ---- ++++ +title = "Convert Office Documents" +weight = 100 ++++ -# {{ page.title }} - -## Context and Problem Statement +# Context and Problem Statement How can office documents, like `docx` or `odt` be converted into a PDF file that looks as much as possible like the original? @@ -16,13 +13,13 @@ has a better outcome, then an external tool is fine, too. Since Docspell is free software, the tools must also be free. -## Considered Options +# Considered Options * [Apache POI](https://poi.apache.org) together with [this](https://search.maven.org/artifact/fr.opensagres.xdocreport/org.apache.poi.xwpf.converter.pdf/1.0.6/jar) library * [pandoc](https://pandoc.org/) external command -* [abiword]() external command +* [abiword](https://www.abisource.com/) external command * [Unoconv](https://github.com/unoconv/unoconv) external command To choose an option, some documents are converted to pdf and compared. @@ -34,11 +31,9 @@ Here is the native view to compare with: ODT: -
- -
+{{ figure(file="example-odt-native.jpg") }} -### `XWPFConverter` +## `XWPFConverter` I couldn't get any example to work. There were exceptions: @@ -69,7 +64,7 @@ The project (not Apache Poi, the other) seems unmaintained. I could not find any website and the artifact in maven central is from 2016. -### Pandoc +## Pandoc I know pandoc as a very great tool when converting between markup documents. So this tries it with office documents. It supports `docx` @@ -93,9 +88,7 @@ pandoc -f odt -o test.pdf example.odt Results ODT: -
- -
+{{ figure(file="example-odt-pandoc-latex.jpg") }} ``` @@ -104,9 +97,7 @@ pandoc -f odt -o test.pdf example.docx Results DOCX: -
- -
+{{ figure(file="example-docx-pandoc-latex.jpg") }} ---- @@ -119,16 +110,12 @@ pandoc -f odt -t context -o test.pdf example.odt Results ODT: -
- -
+{{ figure(file="example-odt-pandoc-context.jpg") }} Results DOCX: -
- -
+{{ figure(file="example-docx-pandoc-context.jpg") }} ---- @@ -141,15 +128,11 @@ pandoc -f odt -t ms -o test.pdf example.odt Results ODT: -
- -
+{{ figure(file="example-odt-pandoc-ms.jpg") }} Results DOCX: -
- -
+{{ figure(file="example-docx-pandoc-ms.jpg") }} --- @@ -162,18 +145,14 @@ $ pandoc --extract-media . -f odt -t html -o test.pdf example.odt Results ODT: -
- -
+{{ figure(file="example-odt-pandoc-html.jpg") }} Results DOCX: -
- -
+{{ figure(file="example-docx-pandoc-html.jpg") }} -### Abiword +## Abiword Trying with: @@ -183,15 +162,13 @@ abiword --to=pdf example.odt Results: -
- -
+{{ figure(file="example-odt-abiword.jpg") }} Trying with a `docx` file failed. It worked with a `doc` file. -### Unoconv +## Unoconv Unoconv relies on libreoffice/openoffice, so installing it will result in installing parts of libreoffice, which is a very large dependency. @@ -204,17 +181,13 @@ unoconv -f pdf example.odt Results ODT: -
- -
+{{ figure(file="example-odt-unoconv.jpg") }} Results DOCX: -
- -
+{{ figure(file="example-docx-unoconv.jpg") }} -## Decision Outcome +# Decision Outcome Unoconv. diff --git a/modules/microsite/docs/dev/adr/0010_convert_image_files.md b/website/site/content/docs/dev/adr/0010_convert_image_files.md similarity index 95% rename from modules/microsite/docs/dev/adr/0010_convert_image_files.md rename to website/site/content/docs/dev/adr/0010_convert_image_files.md index b59c04d0..77542df7 100644 --- a/modules/microsite/docs/dev/adr/0010_convert_image_files.md +++ b/website/site/content/docs/dev/adr/0010_convert_image_files.md @@ -1,12 +1,9 @@ ---- -layout: docs -title: Convert Image Files -permalink: dev/adr/0010_convert_image_files ---- ++++ +title = "Convert Image Files" +weight = 110 ++++ -# {{ page.title }} - -## Context and Problem Statement +# Context and Problem Statement How to convert image files properly to pdf? @@ -21,9 +18,9 @@ though: The focus is on document images, maybe from digital cameras or scanners. -## Considered Options +# Considered Options -* [pdfbox]() library +* [pdfbox](https://pdfbox.apache.org/) library * [imagemagick](https://www.imagemagick.org/) external command * [img2pdf](https://github.com/josch/img2pdf) external command * [tesseract](https://github.com/tesseract-ocr/tesseract) external command @@ -50,7 +47,7 @@ Size: - letter-en.png 191k - letter-en.tiff 4.0M -### pdfbox +## pdfbox Using a java library is preferred, if the quality is good enough. There is an @@ -99,7 +96,7 @@ Size: - letter-en.png 142k - letter-en.tiff 142k -### img2pdf +## img2pdf This is a python tool that adds the image into the pdf without reencoding. @@ -120,7 +117,7 @@ Size: - letter-en.png 191k - letter-en.tiff 192k -### ImageMagick +## ImageMagick The well known imagemagick tool can convert images to pdfs, too. @@ -141,7 +138,7 @@ Size: - letter-en.tiff 5.1M -### Tesseract +## Tesseract Docspell already relies on tesseract for doing OCR. And in contrast to all other candidates, it can create PDFs that are searchable. Of @@ -175,7 +172,7 @@ Size: - letter-en.tiff 183k -## Decision +# Decision Tesseract. diff --git a/modules/microsite/docs/dev/adr/0011_extract_text.md b/website/site/content/docs/dev/adr/0011_extract_text.md similarity index 87% rename from modules/microsite/docs/dev/adr/0011_extract_text.md rename to website/site/content/docs/dev/adr/0011_extract_text.md index 54fd6348..d27ab83d 100644 --- a/modules/microsite/docs/dev/adr/0011_extract_text.md +++ b/website/site/content/docs/dev/adr/0011_extract_text.md @@ -1,12 +1,10 @@ ---- -layout: docs -title: Extract Text from Files -permalink: dev/adr/0011_extract_text ---- ++++ +title = "Extract Text from Files" +weight = 120 ++++ -# Extract Text from Files -## Context and Problem Statement +# Context and Problem Statement With support for more file types there must be a way to extract text from all of them. It is better to extract text from the source files, @@ -16,15 +14,15 @@ There are multiple options and multiple file types. Again, most priority is to use a java/scala library to reduce external dependencies. -## Considered Options +# Considered Options -### MS Office Documents +## MS Office Documents There is only one library I know: [Apache POI](https://poi.apache.org/). It supports `doc(x)` and `xls(x)`. However, it doesn't support open-document format (odt and ods). -### OpenDocument Format +## OpenDocument Format There are two libraries: @@ -43,12 +41,12 @@ text. I created tests that extracted text from my odt/ods files. It worked at first sight, but running the tests in a loop resulted in strange nullpointer exceptions (it only worked the first run). -### Richtext +## Richtext Richtext is supported by the jdk (using `RichtextEditorKit` from swing). -### PDF +## PDF For "image" pdf files, tesseract is used. For "text" PDF files, the library [Apache PDFBox](https://pdfbox.apache.org) can be used. @@ -56,20 +54,20 @@ library [Apache PDFBox](https://pdfbox.apache.org) can be used. There also is [iText](https://github.com/itext/itext7) with a AGPL license. -### Images +## Images For images and "image" PDF files, there is already tesseract in place. -### HTML +## HTML HTML must be converted into a PDF file before text can be extracted. -### Text/Markdown +## Text/Markdown These files can be used as-is, obviously. -## Decision Outcome +# Decision Outcome - MS Office files: POI library - Open Document files: Tika, but integrating the few source files that diff --git a/modules/microsite/docs/dev/adr/0012_periodic_tasks.md b/website/site/content/docs/dev/adr/0012_periodic_tasks.md similarity index 96% rename from modules/microsite/docs/dev/adr/0012_periodic_tasks.md rename to website/site/content/docs/dev/adr/0012_periodic_tasks.md index 44fc8ef1..e87d64e1 100644 --- a/modules/microsite/docs/dev/adr/0012_periodic_tasks.md +++ b/website/site/content/docs/dev/adr/0012_periodic_tasks.md @@ -1,12 +1,9 @@ ---- -layout: docs -title: Periodic Tasks -permalink: dev/adr/0012_periodic_tasks ---- ++++ +title = "Periodic Tasks" +weight = 130 ++++ -# Periodic Tasks - -## Context and Problem Statement +# Context and Problem Statement Currently there is a `Scheduler` that consumes tasks off a queue in the database. This allows multiple job executors running in parallel @@ -27,7 +24,7 @@ than once. If a periodic tasks takes longer than the time between runs, it must wait for the next interval. -## Considered Options +# Considered Options 1. Adding a `timer` and `nextrun` field to the current `job` table 2. Creating a separate table for periodic tasks diff --git a/modules/microsite/docs/dev/adr/0013_archive_files.md b/website/site/content/docs/dev/adr/0013_archive_files.md similarity index 87% rename from modules/microsite/docs/dev/adr/0013_archive_files.md rename to website/site/content/docs/dev/adr/0013_archive_files.md index b653c353..2f74745f 100644 --- a/modules/microsite/docs/dev/adr/0013_archive_files.md +++ b/website/site/content/docs/dev/adr/0013_archive_files.md @@ -1,13 +1,10 @@ ---- -layout: docs -title: Archive Files -permalink: dev/adr/0013_archive_files ---- - -# {{ page.title }} ++++ +title = "Archive Files" +weight = 140 ++++ -## Context and Problem Statement +# Context and Problem Statement Docspell should have support for files that contain the actual files that matter, like zip files and other such things. It should extract @@ -20,7 +17,7 @@ the file unmodified. On the other hand, files in there need to be text analysed and converted to pdf files. -## Decision Outcome +# Decision Outcome There is currently a table `attachment_source` which holds references to "original" files. These are the files as uploaded by the user, @@ -40,6 +37,6 @@ Archive may contain other archives. Then the inner archives will not be saved. The archive file is extracted recursively, until there is no known archive file found. -## Initial Support +# Initial Support Initial support is implemented for ZIP and EML (e-mail files) files. diff --git a/modules/microsite/docs/dev/adr/0014_fulltext_search_engine.md b/website/site/content/docs/dev/adr/0014_fulltext_search_engine.md similarity index 90% rename from modules/microsite/docs/dev/adr/0014_fulltext_search_engine.md rename to website/site/content/docs/dev/adr/0014_fulltext_search_engine.md index ed6d8948..e979eeae 100644 --- a/modules/microsite/docs/dev/adr/0014_fulltext_search_engine.md +++ b/website/site/content/docs/dev/adr/0014_fulltext_search_engine.md @@ -1,14 +1,11 @@ ---- -layout: docs -title: Fulltext Search Engine -permalink: dev/adr/0014_fulltext_search_engine ---- - -# {{ page.title }} ++++ +title = "Fulltext Search Engine" +weight = 150 ++++ It should be possible to search the contents of all documents. -## Context and Problem Statement +# Context and Problem Statement To allow searching the documents contents efficiently, a separate index is necessary. The "defacto standard" for fulltext search on the @@ -21,14 +18,14 @@ feature, it shouldn't have a huge impact on the application, i.e. if the fulltext search component is down or broken, docspell should still work (just the fulltext search is then not working). -## Considered Options +# Considered Options * [Apache SOLR](https://lucene.apache.org/solr) * [ElasticSearch](https://www.elastic.co/elasticsearch/) * [PostgreSQL](https://www.postgresql.org/docs/12/textsearch.html) * All of them or a subset -## Decision Outcome +# Decision Outcome If docspell is running on PostgreSQL, it would be nice to also use it for fulltext search to save the cost of running another component. But diff --git a/modules/microsite/docs/dev/adr/0015_convert_pdf_files.md b/website/site/content/docs/dev/adr/0015_convert_pdf_files.md similarity index 91% rename from modules/microsite/docs/dev/adr/0015_convert_pdf_files.md rename to website/site/content/docs/dev/adr/0015_convert_pdf_files.md index b2f3ec02..445bbebd 100644 --- a/modules/microsite/docs/dev/adr/0015_convert_pdf_files.md +++ b/website/site/content/docs/dev/adr/0015_convert_pdf_files.md @@ -1,12 +1,9 @@ ---- -layout: docs -title: Convert PDF Files -permalink: dev/adr/0015_convert_pdf_files ---- ++++ +title = "Convert PDF Files" +weight = 160 ++++ -# {{ page.title }} - -## Context and Problem Statement +# Context and Problem Statement Some PDFs contain only images (when coming from a scanner) and therefore one is not able to click into the pdf and select text for @@ -18,20 +15,20 @@ For images, this works already as tesseract is used to create the PDF files. Tesseract creates the files with an additional text layer containing the OCRed text. -## Considered Options +# Considered Options * [ocrmypdf](https://github.com/jbarlow83/OCRmyPDF) OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched -### ocrmypdf +## ocrmypdf This is a very nice python tool, that uses tesseract to do OCR on each page and add the extracted text as a pdf text layer to the page. Additionally it creates PDF/A type pdfs, which are great for archiving. This fixes exactly the things stated above. -#### Integration +### Integration Docspell already has this built in for images. When converting images to a PDF (which is done early in processing), the process creates a @@ -61,7 +58,7 @@ converted file containing the OCR-ed text as a pdf layer. If ocrmypdf is disabled, the converted file and the source file are the same for PDFs. -## Decision Outcome +# Decision Outcome Add ocrmypdf as an optional conversion from PDF to PDF. Ocrmypdf is distributed under the GPL-3 license. diff --git a/website/site/content/docs/dev/adr/_index.md b/website/site/content/docs/dev/adr/_index.md new file mode 100644 index 00000000..83de24d2 --- /dev/null +++ b/website/site/content/docs/dev/adr/_index.md @@ -0,0 +1,14 @@ ++++ +title = "ADRs" +description = "Contains some ADRs, which are internal notes on decisions made." +weight = 300 +sort_by = "weight" +insert_anchor_links = "right" +template = "pages.html" +[extra] +mktoc = true ++++ + +This contains a list of ADRs, most of them are from very early. It +often just contains notes that could go nowhere else, but still should +be captured. diff --git a/modules/microsite/docs/dev/adr/img/example-docx-pandoc-context.jpg b/website/site/content/docs/dev/adr/example-docx-pandoc-context.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-docx-pandoc-context.jpg rename to website/site/content/docs/dev/adr/example-docx-pandoc-context.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-docx-pandoc-html.jpg b/website/site/content/docs/dev/adr/example-docx-pandoc-html.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-docx-pandoc-html.jpg rename to website/site/content/docs/dev/adr/example-docx-pandoc-html.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-docx-pandoc-latex.jpg b/website/site/content/docs/dev/adr/example-docx-pandoc-latex.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-docx-pandoc-latex.jpg rename to website/site/content/docs/dev/adr/example-docx-pandoc-latex.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-docx-pandoc-ms.jpg b/website/site/content/docs/dev/adr/example-docx-pandoc-ms.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-docx-pandoc-ms.jpg rename to website/site/content/docs/dev/adr/example-docx-pandoc-ms.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-docx-unoconv.jpg b/website/site/content/docs/dev/adr/example-docx-unoconv.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-docx-unoconv.jpg rename to website/site/content/docs/dev/adr/example-docx-unoconv.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-html-native.jpg b/website/site/content/docs/dev/adr/example-html-native.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-html-native.jpg rename to website/site/content/docs/dev/adr/example-html-native.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-html-pandoc-html.jpg b/website/site/content/docs/dev/adr/example-html-pandoc-html.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-html-pandoc-html.jpg rename to website/site/content/docs/dev/adr/example-html-pandoc-html.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-html-pandoc-latex.jpg b/website/site/content/docs/dev/adr/example-html-pandoc-latex.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-html-pandoc-latex.jpg rename to website/site/content/docs/dev/adr/example-html-pandoc-latex.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-html-unoconv.jpg b/website/site/content/docs/dev/adr/example-html-unoconv.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-html-unoconv.jpg rename to website/site/content/docs/dev/adr/example-html-unoconv.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-html-wkhtmltopdf.jpg b/website/site/content/docs/dev/adr/example-html-wkhtmltopdf.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-html-wkhtmltopdf.jpg rename to website/site/content/docs/dev/adr/example-html-wkhtmltopdf.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-md-java.jpg b/website/site/content/docs/dev/adr/example-md-java.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-md-java.jpg rename to website/site/content/docs/dev/adr/example-md-java.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-md-pandoc-html.jpg b/website/site/content/docs/dev/adr/example-md-pandoc-html.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-md-pandoc-html.jpg rename to website/site/content/docs/dev/adr/example-md-pandoc-html.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-md-pandoc-latex.jpg b/website/site/content/docs/dev/adr/example-md-pandoc-latex.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-md-pandoc-latex.jpg rename to website/site/content/docs/dev/adr/example-md-pandoc-latex.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-odt-abiword.jpg b/website/site/content/docs/dev/adr/example-odt-abiword.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-odt-abiword.jpg rename to website/site/content/docs/dev/adr/example-odt-abiword.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-odt-native.jpg b/website/site/content/docs/dev/adr/example-odt-native.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-odt-native.jpg rename to website/site/content/docs/dev/adr/example-odt-native.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-odt-pandoc-context.jpg b/website/site/content/docs/dev/adr/example-odt-pandoc-context.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-odt-pandoc-context.jpg rename to website/site/content/docs/dev/adr/example-odt-pandoc-context.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-odt-pandoc-html.jpg b/website/site/content/docs/dev/adr/example-odt-pandoc-html.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-odt-pandoc-html.jpg rename to website/site/content/docs/dev/adr/example-odt-pandoc-html.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-odt-pandoc-latex.jpg b/website/site/content/docs/dev/adr/example-odt-pandoc-latex.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-odt-pandoc-latex.jpg rename to website/site/content/docs/dev/adr/example-odt-pandoc-latex.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-odt-pandoc-ms.jpg b/website/site/content/docs/dev/adr/example-odt-pandoc-ms.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-odt-pandoc-ms.jpg rename to website/site/content/docs/dev/adr/example-odt-pandoc-ms.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-odt-unoconv.jpg b/website/site/content/docs/dev/adr/example-odt-unoconv.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-odt-unoconv.jpg rename to website/site/content/docs/dev/adr/example-odt-unoconv.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-txt-java.jpg b/website/site/content/docs/dev/adr/example-txt-java.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-txt-java.jpg rename to website/site/content/docs/dev/adr/example-txt-java.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-txt-pandoc-html.jpg b/website/site/content/docs/dev/adr/example-txt-pandoc-html.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-txt-pandoc-html.jpg rename to website/site/content/docs/dev/adr/example-txt-pandoc-html.jpg diff --git a/modules/microsite/docs/dev/adr/img/example-txt-pandoc-latex.jpg b/website/site/content/docs/dev/adr/example-txt-pandoc-latex.jpg similarity index 100% rename from modules/microsite/docs/dev/adr/img/example-txt-pandoc-latex.jpg rename to website/site/content/docs/dev/adr/example-txt-pandoc-latex.jpg diff --git a/modules/microsite/docs/dev/adr/img/process-files.png b/website/site/content/docs/dev/adr/process-files.png similarity index 100% rename from modules/microsite/docs/dev/adr/img/process-files.png rename to website/site/content/docs/dev/adr/process-files.png diff --git a/modules/microsite/docs/dev/adr/process-files.puml b/website/site/content/docs/dev/adr/process-files.puml similarity index 100% rename from modules/microsite/docs/dev/adr/process-files.puml rename to website/site/content/docs/dev/adr/process-files.puml diff --git a/modules/microsite/docs/dev/adr/template.md b/website/site/content/docs/dev/adr/template.md similarity index 98% rename from modules/microsite/docs/dev/adr/template.md rename to website/site/content/docs/dev/adr/template.md index d531e39e..7fd56f41 100644 --- a/modules/microsite/docs/dev/adr/template.md +++ b/website/site/content/docs/dev/adr/template.md @@ -1,7 +1,7 @@ ---- -layout: docs -title: Short Title ---- ++++ +title = "Short Title" +draft = true ++++ # [short title of solved problem and solution] diff --git a/website/site/content/docs/dev/building.md b/website/site/content/docs/dev/building.md new file mode 100644 index 00000000..45768223 --- /dev/null +++ b/website/site/content/docs/dev/building.md @@ -0,0 +1,22 @@ ++++ +title = "Building Docspell" +weight = 0 ++++ + + +You must install [sbt](https://scala-sbt.org) and [Elm](https://elm-lang.org). + +Clone the sources and run: + +- `make` to compile all sources (Elm + Scala) +- `make-zip` to create zip packages +- `make-deb` to create debian packages +- `make-tools` to create a zip containing the script in `tools/` +- `make-pkg` for a clean compile + building all packages (zip + deb) + +The `zip` and `deb` files can be found afterwards in: + +``` +modules/restserver/target/universal +modules/joex/target/universal +``` diff --git a/modules/microsite/docs/dev.md b/website/site/content/docs/dev/development.md similarity index 78% rename from modules/microsite/docs/dev.md rename to website/site/content/docs/dev/development.md index 96a0954b..4460aa27 100644 --- a/modules/microsite/docs/dev.md +++ b/website/site/content/docs/dev/development.md @@ -1,33 +1,9 @@ ---- -layout: docs -title: Development -permalink: dev ---- ++++ +title = "Tips & Setup" +weight = 20 ++++ - -# {{page.title}} - - -## Building - -[Sbt](https://scala-sbt.org) is used to build the application. Clone -the sources and run: - -- `make` to compile all sources (Elm + Scala) -- `make-zip` to create zip packages -- `make-deb` to create debian packages -- `make-tools` to create a zip containing the script in `tools/` -- `make-pkg` for a clean compile + building all packages (zip + deb) - -The zip files can be found afterwards in: - -``` -modules/restserver/target/universal -modules/joex/target/universal -``` - - -## Starting Servers with `reStart` +# Starting Servers with `reStart` When developing, it's very convenient to use the [revolver sbt plugin](https://github.com/spray/sbt-revolver). Start the sbt console @@ -53,14 +29,14 @@ sbt:docspell-root> reStart ``` -## Custom config file +# Custom config file The sbt build is setup such that a file `dev.conf` in the directory `local` (at root of the source tree) is picked up as config file, if it exists. So you can create a custom config file for development. For example, a custom database for development may be setup this way: -``` +``` conf #jdbcurl = "jdbc:h2:///home/dev/workspace/projects/docspell/local/docspell-demo.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_SERVER=TRUE" jdbcurl = "jdbc:postgresql://localhost:5432/docspelldev" #jdbcurl = "jdbc:mariadb://localhost:3306/docspelldev" @@ -87,17 +63,17 @@ docspell.joex { } ``` -## Nix Expressions +# Nix Expressions The directory `/nix` contains nix expressions to install docspell via the nix package manager and to integrate it into NixOS. -### Testing NixOS Modules +## Testing NixOS Modules The modules can be build by building the `configuration-test.nix` file together with some nixpkgs version. For example: -``` shell +``` bash nixos-rebuild build-vm -I nixos-config=./configuration-test.nix \ -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09.tar.gz ``` @@ -108,21 +84,27 @@ the system configuration can be found behind the `./result/system` symlink. So it is possible to look at the generated systemd config for example: -``` shell +``` bash cat result/system/etc/systemd/system/docspell-joex.service ``` And with some more commands (there probably is an easier way…) the config file can be checked: -``` shell +``` bash cat result/system/etc/systemd/system/docspell-joex.service | grep ExecStart | cut -d'=' -f2 | xargs cat | tail -n1 | awk '{print $NF}'| sed 's/.$//' | xargs cat | jq ``` To see the module in action, the vm can be started (the first line sets more memory for the vm): -``` shell +``` bash export QEMU_OPTS="-m 2048" ./result/bin/run-docspelltest-vm ``` + + +# Background Info + +There is a list of [ADRs](@/docs/dev/adr/_index.md) containing +internal/background info for various topics. diff --git a/modules/microsite/docs/features.md b/website/site/content/docs/features/_index.md similarity index 58% rename from modules/microsite/docs/features.md rename to website/site/content/docs/features/_index.md index 85af705a..8999bb40 100644 --- a/modules/microsite/docs/features.md +++ b/website/site/content/docs/features/_index.md @@ -1,8 +1,9 @@ ---- -layout: docs -title: Features and Limitations -permalink: features ---- ++++ +title = "Features and Limitations" +weight = 10 +insert_anchor_links = "right" +description = "A list of features and limitations." ++++ # Features @@ -11,27 +12,28 @@ permalink: features account) - Handle multiple documents as one unit - OCR using [tesseract](https://github.com/tesseract-ocr/tesseract) -- [Full-Text Search](doc/finding#full-text-search) based on [Apache - SOLR](https://lucene.apache.org/solr) +- [Full-Text Search](@/docs/webapp/finding.md#full-text-search) based + on [Apache SOLR](https://lucene.apache.org/solr) - Conversion to PDF: all files are converted into a PDF file. PDFs with only images (as often returned from scanners) are converted into searchable PDF/A pdfs. - Non-destructive: all your uploaded files are never modified and can always be downloaded untouched - Text is analysed to find and attach meta data automatically -- [Manage document processing](doc/processing): cancel jobs, set - priorities +- [Manage document processing](@/docs/webapp/processing.md): cancel + jobs, set priorities - Everything available via a [documented](https://www.openapis.org/) - [REST Api](api); allows to [generate + [REST Api](@/docs/api/_index.md); allows to [generate clients](https://openapi-generator.tech/docs/generators) for (almost) any language - mobile-friendly Web-UI -- [Create “share-urls”](doc/uploading#anonymous-upload) to upload files - anonymously -- [Send documents via e-mail](doc/mailitem) -- [E-Mail notification](doc/notifydueitems) for documents with due dates -- [Read your mailboxes](doc/scanmailbox) via IMAP to import mails into - docspell +- [Create “share-urls”](@/docs/webapp/uploading.md#anonymous-upload) + to upload files anonymously +- [Send documents via e-mail](@/docs/webapp/mailitem.md) +- [E-Mail notification](@/docs/webapp/notifydueitems.md) for documents + with due dates +- [Read your mailboxes](@/docs/webapp/scanmailbox.md) via IMAP to + import mails into docspell - REST server and document processing are separate applications which can be scaled-out independently - Everything stored in a SQL database: PostgreSQL, MariaDB or H2 @@ -50,14 +52,14 @@ permalink: features - zip - [eml](https://en.wikipedia.org/wiki/Email#Filename_extensions) (e-mail files in plain text MIME) -- [Tooling](doc/tools): - - [Watch a folder](doc/tools/consumedir): watch folders for changes - and send files to docspell - - [Simple CLI for uploading files](doc/tools/ds) - - [Firefox plugin](doc/tools/browserext): right click on a link and - send the file to docspell - - [SMTP Gateway](doc/tools/smtpgateway): Setup a SMTP server that - delivers mails directly to docspell. +- Tooling: + - [Watch a folder](@/docs/tools/consumedir.md): watch folders for + changes and send files to docspell + - [Simple CLI for uploading files](@/docs/tools/ds.md) + - [Firefox plugin](@/docs/tools/browserext.md): right click on a + link and send the file to docspell + - [SMTP Gateway](@/docs/tools/smtpgateway.md): Setup a SMTP server + that delivers mails directly to docspell. - License: GPLv3 diff --git a/website/site/content/docs/install/_index.md b/website/site/content/docs/install/_index.md new file mode 100644 index 00000000..6d43a90b --- /dev/null +++ b/website/site/content/docs/install/_index.md @@ -0,0 +1,9 @@ ++++ +title = "Installation and Deployment" +description = "There are multiple ways to install Docspell. This section contains detailed instructions." +weight = 30 +sort_by = "weight" +insert_anchor_links = "right" +template = "pages.html" +redirect_to = "/docs/install/quickstart" ++++ diff --git a/modules/microsite/docs/doc/nix.md b/website/site/content/docs/install/installing.md similarity index 64% rename from modules/microsite/docs/doc/nix.md rename to website/site/content/docs/install/installing.md index e74c6b8e..9b6a094e 100644 --- a/modules/microsite/docs/doc/nix.md +++ b/website/site/content/docs/install/installing.md @@ -1,10 +1,96 @@ ---- -layout: docs -title: Nix/NixOS -permalink: doc/nix ---- ++++ +title = "Installing" +weight = 20 ++++ -# {{ page.title }} +# Docker + +There is a [docker-compose](https://docs.docker.com/compose/) setup +available in the `/docker` folder. This setup is also taking care of +all the necessary [prerequisites](@/docs/install/prereq.md) and +creates a container to watch a directory for incoming files. It's only +3 steps: + +1. Clone the github repository + ```bash + $ git clone https://github.com/eikek/docspell + ``` +2. Change into the `docker` directory: + ```bash + $ cd docspell/docker + ``` +3. Run `docker-compose up`: + + ```bash + $ export DOCSPELL_HEADER_VALUE="my-secret-123" + $ docker-compose up + ``` + + The environment variable defines a secret that is shared between + some containers. You can define whatever you like. Please see the + [consumedir.sh](@/docs/tools/consumedir.md#docker) docs for + additional info. +4. Goto `http://localhost:7880`, signup and login. When signing up, + you can choose the same name for collective and user. Then login + with this name and the password. + +5. (Optional) Create a folder `./docs/` (the name you + chose for the collective at registration) and place files in there + for importing them. + +The directory contains a file `docspell.conf` that you can +[modify](@/docs/configure/_index.md) as needed. + +# Download, Unpack, Run + +You can install via zip or deb archives. Please see the +[prerequisites](@/docs/install/prereq.md) first. + +## Using zip files + +You need to download the two files: + +- [docspell-restserver-{{version()}}.zip](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-restserver-{{version()}}.zip) +- [docspell-joex-{{version()}}.zip](https://github.com/eikek/docspell/releases/download/v{{version()}}/docspell-joex-{{version()}}.zip) + + +1. Unzip both files: + ``` bash + $ unzip docspell-*.zip + ``` +2. Open two terminal windows and navigate to the the directory + containing the zip files. +3. Start both components executing: + ``` bash + $ ./docspell-restserver*/bin/docspell-restserver + ``` + in one terminal and + ``` bash + $ ./docspell-joex*/bin/docspell-joex + ``` + in the other. +4. Point your browser to: +5. Register a new account, sign in and try it. + +Note, that this setup doesn't include watching a directory. You can +use the [consumedir.sh](@/docs/tools/consumedir.md) tool for this or +use the docker variant below. + +## Using deb files + +The DEB packages can be installed on Debian, or Debian based Distros: + +``` bash +$ sudo dpkg -i docspell*.deb +``` + +Then the start scripts are in your `$PATH`. Run `docspell-restserver` +or `docspell-joex` from a terminal window. + +The packages come with a systemd unit file that will be installed to +autostart the services. + +# Nix ## Install via Nix @@ -56,15 +142,16 @@ $ nix-env -iA nixpkgs.docspell.server nixpkgs.docspell.joex nixpkgs.docspell.too You may need to replace `nixpkgs` with `nixos` when you're on NixOS. -The expression `docspell.currentPkg` refers to the most current release -of Docspell. So even if you use the tarball of the current master -branch, the `release.nix` file only contains derivations for releases. -The expression `docspell.currentPkg` is a shortcut for selecting the -most current release. For example it translates to `docspell.pkg -docspell.cfg.v@PVERSION@` – if the current version is `@VERSION@`. +The expression `docspell.currentPkg` refers to the most current +release of Docspell. So even if you use the tarball of the current +master branch, the `release.nix` file only contains derivations for +releases. The expression `docspell.currentPkg` is a shortcut for +selecting the most current release. For example it translates to +`docspell.pkg docspell.cfg.v{{ pversion() }}` – if the current version +is `{{version()}}`. -## Docspell as a service on NixOS +## Docspell on NixOS {#nixos} If you are running [NixOS](https://nixos.org), there is a module definition for installing Docspell as a service using systemd. @@ -132,7 +219,7 @@ The modules files are only applicable to the newest version of Docspell. If you really need an older version, checkout the appropriate commit. -## NixOs Example +### NixOS Example This is a example system configuration that installs docspell with a postgres database. This snippet can be used to create a vm (using diff --git a/website/site/content/docs/install/prereq.md b/website/site/content/docs/install/prereq.md new file mode 100644 index 00000000..ae50d063 --- /dev/null +++ b/website/site/content/docs/install/prereq.md @@ -0,0 +1,107 @@ ++++ +title = "Prerequisites" +weight = 10 ++++ + +The two components have one prerequisite in common: they both require +Java to run. While this is the only requirement for the *REST server*, +the *Joex* components requires some more external programs. + +The rest server and joex components are not required to "see" each +other, though it is recommended. + +# Java + +Very often, Java is already installed. You can check this by opening a +terminal and typing `java -version`. Otherwise install Java using your +package manager or see [this site](https://adoptopenjdk.net/) for +other options. + +It is enough to install the JRE. The JDK is required, if you want to +build docspell from source. + +Docspell has been tested with Java version 1.8 (or sometimes referred +to as JRE 8 and JDK 8, respectively). The pre-build packages are also +build using JDK 8. But a later version of Java should work as well. + +The next tools are only required on machines running the *Joex* +component. + +# External Programs for Joex + +- [Ghostscript](http://pages.cs.wisc.edu/~ghost/) (the `gs` command) + is used to extract/convert PDF files into images that are then fed + to ocr. It is available on most GNU/Linux distributions. +- [Unpaper](https://github.com/Flameeyes/unpaper) is a program that + pre-processes images to yield better results when doing ocr. If this + is not installed, docspell tries without it. However, it is + recommended to install, because it [improves text + extraction](https://github.com/tesseract-ocr/tesseract/wiki/ImproveQuality) + (at the expense of a longer runtime). +- [Tesseract](https://github.com/tesseract-ocr/tesseract) is the tool + doing the OCR (converts images into text). It can also convert + images into pdf files. It is a widely used open source OCR engine. + Tesseract 3 and 4 should work with docspell; you can adopt the + command line in the configuration file, if necessary. +- [Unoconv](https://github.com/unoconv/unoconv) is used to convert + office documents into PDF files. It uses libreoffice/openoffice. +- [wkhtmltopdf](https://wkhtmltopdf.org/) is used to convert HTML into + PDF files. +- [OCRmyPDF](https://github.com/jbarlow83/OCRmyPDF) can be optionally + used to convert PDF to PDF files. It adds an OCR layer to scanned + PDF files to make them searchable. It also creates PDF/A files from + the input pdf. + +The performance of `unoconv` can be improved by starting `unoconv -l` +in a separate process. This runs a libreoffice/openoffice listener and +therefore avoids starting one each time `unoconv` is called. + +## Example Debian + +On Debian this should install all joex requirements: + +``` bash +sudo apt-get install ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng unpaper unoconv wkhtmltopdf ocrmypdf +``` + +# Apache SOLR + +SOLR is used to provide the fulltext search feature. This feature can +be disabled, so installing SOLR is optional. But without it, there is +no fulltext search. + +When installing manually (i.e. not via docker), just install solr and +create a core as described in the [solr +documentation](https://lucene.apache.org/solr/guide/8_4/installing-solr.html). +That will provide you with the connection url (the last part is the +core name). + +When using the provided `docker-compose.yml` setup, SOLR is already setup. + +SOLR must be reachable from all joex and all rest server components. + +# Database + +Both components must have access to a SQL database. The SQL database +contains all data (including binary files) and is the central +component of docspell. Docspell has support these databases: + +- PostreSQL +- MariaDB +- H2 + +The H2 database is an interesting option for personal and mid-size +setups, as it requires no additional work. It is integrated into +docspell and works really well out of the box. It is also configured +as the default database. + +When using H2, make sure that all components access the same database +– the jdbc url must point to the same file. Then, it is important to +add the options +`;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_SERVER=TRUE` at the end +of the url. See the [config page](@/docs/configure/_index.md#jdbc) for +an example. + +For large installations, PostgreSQL or MariaDB is recommended. Create +a database and a user with enough privileges (read, write, create +table) to that database. diff --git a/website/site/content/docs/install/quickstart.md b/website/site/content/docs/install/quickstart.md new file mode 100644 index 00000000..385a7aa4 --- /dev/null +++ b/website/site/content/docs/install/quickstart.md @@ -0,0 +1,23 @@ ++++ +title = "Quickstart" +weight = 0 ++++ + +To get started, here are some quick links: + +- Using [docker and + docker-compose](@/docs/install/installing.md#docker). This sets up + everything: all prerequisites, both docspell components and a + container running the [consumedir.sh](@/docs/tools/consumedir.md) + script to import files that are dropped in a folder. +- [Download, Unpack and + Run](@/docs/install/installing.md#download-unpack-run). This option + is also very quick, but you need to check the + [prerequisites](@/docs/install/prereq.md) yourself. Database is + already setup, but you'd need to setup SOLR (when using fulltext + search) and install some programs for the joex component. This + applies to the `zip` and `deb` files. The files can be downloaded + from the [release page](https://github.com/eikek/docspell/releases/latest). +- via the [nix package manager](@/docs/install/installing.md#nix) and/or as a [NixOS + module](@/docs/install/installing.md#nixos). If you use nix/nixos, you + know what to do. The linked page contains some examples. diff --git a/modules/microsite/docs/doc/reverseproxy.md b/website/site/content/docs/install/reverseproxy.md similarity index 95% rename from modules/microsite/docs/doc/reverseproxy.md rename to website/site/content/docs/install/reverseproxy.md index 17f42127..3932aebd 100644 --- a/modules/microsite/docs/doc/reverseproxy.md +++ b/website/site/content/docs/install/reverseproxy.md @@ -1,10 +1,7 @@ ---- -layout: docs -title: Reverse Proxy -permalink: doc/reverseproxy ---- - -# {{ page.title }} ++++ +title = "Reverse Proxy" +weight = 50 ++++ This contains examples for how to use docspell behind a reverse proxy. @@ -15,7 +12,7 @@ For the examples below, assume the following: `localhost:7880` instead. - The external domain/hostname is `docspell.example.com` -## Configuring Docspell +# Configuring Docspell These settings require a complement config part in the docspell configuration file: @@ -24,7 +21,7 @@ configuration file: to change the `bind.address` setting to be either `0.0.0.0` or the ip address of the network interface that the reverse proxy server connects to. - ``` + ``` conf docspell.server { # Where the server binds to. bind { @@ -38,7 +35,7 @@ configuration file: - Docspell needs to know the external url. The `base-url` setting must point to the external address. Using above values, it must be set to `https://docspell.example.com`. - ``` + ``` conf docspell.server { # This is the base URL this application is deployed to. This is used # to create absolute URLs and to configure the cookie. @@ -54,7 +51,7 @@ server and web application. If you have examples for more servers, please let me know or add it to this site. -## Nginx +# Nginx This defines two servers: one listens for http traffic and redirects to the https variant. Additionally it defines the let's encrypt @@ -64,7 +61,7 @@ The https server endpoint is configured with the let's encrypt certificates and acts as a proxy for the application at `192.168.1.11:7880`. -``` +``` conf server { listen 0.0.0.0:80 ; listen [::]:80 ; diff --git a/website/site/content/docs/install/rpi.md b/website/site/content/docs/install/rpi.md new file mode 100644 index 00000000..3fb19450 --- /dev/null +++ b/website/site/content/docs/install/rpi.md @@ -0,0 +1,37 @@ ++++ +title = "Raspberry-Pi and Similiar" +weight = 40 ++++ + +# Raspberry Pi, and similiar + +Both component can run next to each other on a raspberry pi or +similiar device. + + +## REST Server + +The REST server component runs very well on the Raspberry Pi and +similiar devices. It doesn't require much resources, because the heavy +work is done by the joex components. + + +## Joex + +Running the joex component on the Raspberry Pi is possible, but will +result in long processing times for OCR. Files that don't require OCR +are no problem. + +Tested on a RPi model 3 (4 cores, 1G RAM) processing a PDF (scanned +with 300dpi) with two pages took 9:52. You can speed it up +considerably by uninstalling the `unpaper` command, because this step +takes quite long. This, of course, reduces the quality of OCR. But +without `unpaper` the same sample pdf was then processed in 1:24, a +speedup of 8 minutes. + +You should limit the joex pool size to 1 and, depending on your model +and the amount of RAM, set a heap size of at least 500M +(`-J-Xmx500M`). + +For personal setups, when you don't need the processing results asap, +this can work well enough. diff --git a/website/site/content/docs/install/running.md b/website/site/content/docs/install/running.md new file mode 100644 index 00000000..295f01c9 --- /dev/null +++ b/website/site/content/docs/install/running.md @@ -0,0 +1,66 @@ ++++ +title = "Running" +weight = 30 ++++ + +# Running + +Run the start script (in the corresponding `bin/` directory when using +the zip files): + +``` +$ ./docspell-restserver*/bin/docspell-restserver +$ ./docspell-joex*/bin/docspell-joex +``` + +This will startup both components using the default configuration. The +configuration should be adopted to your needs. For example, the +database connection is configured to use a H2 database in the `/tmp` +directory. Please refer to the [configuration +page](@/docs/configure/_index.md) for how to create a custom config +file. Once you have your config file, simply pass it as argument to +the command: + +``` +$ ./docspell-restserver*/bin/docspell-restserver /path/to/server-config.conf +$ ./docspell-joex*/bin/docspell-joex /path/to/joex-config.conf +``` + +After starting the rest server, you can reach the web application at +path `/app`, so using default values it would be +`http://localhost:7880/app`. There also is a redirect from `/` to +`/app`. + +You should be able to create a new account and sign in. Check the +[configuration page](@/docs/configure/_index.md) to further customize +docspell. + + +## Options + +The start scripts support some options to configure the JVM. One often +used setting is the maximum heap size of the JVM. By default, java +determines it based on properties of the current machine. You can +specify it by given java startup options to the command: + +``` +$ ./docspell-restserver*/bin/docspell-restserver -J-Xmx1G -- /path/to/server-config.conf +``` + +This would limit the maximum heap to 1GB. The double slash separates +internal options and the arguments to the program. Another frequently +used option is to change the default temp directory. Usually it is +`/tmp`, but it may be desired to have a dedicated temp directory, +which can be configured: + +``` +$ ./docspell-restserver*/bin/docspell-restserver -J-Xmx1G -Djava.io.tmpdir=/path/to/othertemp -- /path/to/server-config.conf +``` + +The command: + +``` +$ ./docspell-restserver*/bin/docspell-restserver -h +``` + +gives an overview of supported options. diff --git a/website/site/content/docs/intro/_index.md b/website/site/content/docs/intro/_index.md new file mode 100644 index 00000000..39ee5de7 --- /dev/null +++ b/website/site/content/docs/intro/_index.md @@ -0,0 +1,134 @@ ++++ +title = "Introduction" +weight = 0 +description = "Gives a short introduction to the goals of docspell and an overview of the components involved when running docspell." +insert_anchor_links = "right" +[extra] +mktoc = true ++++ + +# Introduction + +Docspell aims to be a simple yet effective document organizer that +makes stowing documents away very quick and finding them later +reliable (and also fast). It doesn't require technical background or +studying huge manuals in order to use it. With this in mind, it is +rather opinionated and more targeted for home use and small/medium +organizations. + +Docspell analyzes the text of your files and tries to find metadata +that will be annotated automatically. This metadata is taken from an +address book that must be maintained manually. Docspell then looks for +candidates for: + +- Correspondents +- Concerned person or things +- A date + +It will propose a few candidates and sets the most likely one to your +item. + +This might be wrong, so it is recommended to curate the results. +However, very often the correct one is either set or within the +proposals where you fix it by a single click. + +Besides these properties, there are more metadata you can use to +organize your files, for example tags, folders and notes. + +Docspell is also for programmers. Everything is available via a REST +or HTTP api and can be easily used within your own scripts and tools, +for example using `curl`. There are also features for "advanced use" +and many configuration options. + + +# Components + +Docspell consists of multiple components that run in separate +processes: + +- REST server +- JOEX, short for *job executor* +- Fulltext Search Index (optional, currently Apache SOLR) + +The REST server provides the Api and the web application. The web +application is a +[SPA](https://en.wikipedia.org/wiki/Single-page_application) written +in [Elm](https://elm-lang.org) and is a client to the REST api. All +features are available via a http/rest api. + +The *joex* is the component that does the “heavy work”, executing +long-running tasks, like processing files or importing your mails +periodically. While the joex component also exposes a small REST api +for controlling it, the main user interface is all inside the rest +server api. + +The rest server and the job executor can be started multiple times in +order to scale out. It must be ensured, that all connect to the same +database. And it is also recommended (though not strictly required), +that all components can reach each other. + +The fulltext search index is another separate component, where +currently only [SOLR](https://lucene.apache.org/solr) is supported. +Fulltext search is optional, so the SOLR component is not required if +docspell is run without fulltext search support. + + +# Terms + +In order to better understand the following pages, some terms are +explained. + +## Item + +An *item* is roughly your document, only that an item may span +multiple files, which are called *attachments*. An item has *meta +data* associated: + +- a *correspondent*: the other side of the communication. It can be + an organization or a person. +- a *concerning person* or *equipment*: a person or thing that + this item is about. Maybe it is an insurance contract about your + car. +- *tag*: an item can be tagged with one or more tags (or labels). A + tag can have a *category*. This is intended for grouping tags, for + example a category `doctype` could be used to group tags like + `bill`, `contract`, `receipt` etc. Usually an item is not tagged + with more than one tag of a category. +- a *folder*: a folder is similiar to a tag, but an item can only be + in exactly one folder (or none). Furhtermore folders allow to + associate users, so that items are only visible to the users who are + members of a folder. +- an *item date*: this is the date of the document – if this is not + set, the created date of the item is used. +- a *due date*: an optional date indicating that something has to be + done (e.g. paying a bill, submitting it) about this item until this + date +- a *direction*: one of "incoming" or "outgoing" +- a *name*: some item name, defaults to the file name of the + attachments +- some *notes*: arbitrary descriptive text. You can use markdown + here, which is properly formatted in the web application. + +## Collective + +The users of the application are part of a *collective*. A +*collective* is a group of users that share access to the same +items. The account name is therefore comprised of a *collective name* +and a *user name*. + +All users of a collective are equal; they have same permissions to +access all items. The items don't belong to a user, but to the +collective. + +That means, to identify yourself when signing in, you have to give the +collective name and your user name. By default it is separated by a +slash `/`, for example `smith/john`. If your user name is the same as +the collective name, you can omit one; so `smith/smith` can be +abbreviated to just `smith`. + +By default, all users can see all items of their collective. A +*folder* can be used to implement other visibilities: Every user can +create a folder and associate members. It is possible to put items in +these folders and docspell shows only items that are either in no +specific folder or in a folder where the current user is owner or +member. diff --git a/modules/microsite/docs/doc/joex.md b/website/site/content/docs/joex/_index.md similarity index 91% rename from modules/microsite/docs/doc/joex.md rename to website/site/content/docs/joex/_index.md index 974621e5..c9ac7fd7 100644 --- a/modules/microsite/docs/doc/joex.md +++ b/website/site/content/docs/joex/_index.md @@ -1,10 +1,13 @@ ---- -layout: docs -title: Joex -permalink: doc/joex ---- ++++ +title = "Joex" +description = "More information about the job executor component." +weight = 90 +insert_anchor_links = "right" +[extra] +mktoc = true ++++ -# {{ page.title }} +# Introduction Joex is short for *Job Executor* and it is the component managing long running tasks in docspell. One of these long running tasks is the file @@ -18,7 +21,8 @@ parallel is limited with respect to the hardware it is running on. For larger installations, it is probably better to run several joex components on different machines. That works out of the box, as long as all components point to the same database and use different -`app-id`s (see [configuring docspell](./configure#app-id)). +`app-id`s (see [configuring +docspell](@/docs/configure/_index.md#app-id)). When files are submitted to docspell, they are stored in the database and all known joex components are notified about new work. Then they @@ -26,7 +30,7 @@ compete on getting the next job from the queue. After a job finishes and no job is waiting in the queue, joex will sleep until notified again. It will also periodically notify itself as a fallback. -## Task vs Job +# Task vs Job Just for the sake of this document, a task denotes the code that has to be executed or the thing that has to be done. It emerges in a job, @@ -35,7 +39,7 @@ up and executed eventually. A job maintains a state and other things, while a task is just code. -## Scheduler and Queue +# Scheduler and Queue The scheduler is the part that runs and monitors the long running jobs. It works together with the job queue, which defines what job to @@ -51,13 +55,14 @@ next. The default is `4, 1`, meaning to first select 4 high priority jobs and then 1 low priority job, then starting over. If no such job exists, its falls back to the other priority. -The priority can be set on a *Source* (see [uploads](uploading)). -Uploading through the web application will always use priority *high*. -The idea is that while logged in, jobs are more important that those -submitted when not logged in. +The priority can be set on a *Source* (see +[uploads](@/docs/webapp/uploading.md)). Uploading through the web +application will always use priority *high*. The idea is that while +logged in, jobs are more important that those submitted when not +logged in. -## Scheduler Config +# Scheduler Config The relevant part of the config file regarding the scheduler is shown below with some explanations. @@ -125,7 +130,7 @@ reach a joex component. This periodic wakup is just to ensure that jobs are eventually run. -## Periodic Tasks +# Periodic Tasks The job executor can execute tasks periodically. These tasks are stored in the database such that they can be submitted into the job @@ -134,7 +139,7 @@ something with a task. So a periodic task is never submitted twice. It is also not submitted, if a previous task has not finished yet. -## Starting on demand +# Starting on demand The job executor and rest server can be started multiple times. This is especially useful for the job executor. For example, when @@ -149,7 +154,7 @@ Once the files have been processced you can stop the additional executors. -## Shutting down +# Shutting down If a job executor is sleeping and not executing any jobs, you can just quit using SIGTERM or `Ctrl-C` when running in a terminal. But if diff --git a/website/site/content/docs/tools/_index.md b/website/site/content/docs/tools/_index.md new file mode 100644 index 00000000..8a53a66c --- /dev/null +++ b/website/site/content/docs/tools/_index.md @@ -0,0 +1,11 @@ ++++ +title = "Tools" +description = "There are several tools distributed with docspell, like a program to watch a folder and import files to docspell." +weight = 60 +insert_anchor_links = "right" +template = "pages.html" +redirect_to = "docs/tools/ds" +sort_by = "weight" +[extra] +mktoc = false ++++ diff --git a/modules/microsite/docs/doc/tools/browserext.md b/website/site/content/docs/tools/browserext.md similarity index 88% rename from modules/microsite/docs/doc/tools/browserext.md rename to website/site/content/docs/tools/browserext.md index 3fbbca7f..419469f2 100644 --- a/modules/microsite/docs/doc/tools/browserext.md +++ b/website/site/content/docs/tools/browserext.md @@ -1,10 +1,8 @@ ---- -layout: docs -title: Browser Extension (Firefox) -permalink: doc/tools/browserext ---- - -# {{ page.title }} ++++ +title = "Browser Extension (Firefox)" +description = "An extension for firefox to upload files from your browser via right-click → upload to docspell." +weight = 30 ++++ The idea is to click on a file in firefox and send it to docspell. It is downloaded in the context of your current page. Then handed to an @@ -12,18 +10,16 @@ application that pushes it to docspell. There is a browser add-on implementing this in `tools/webextension`. This add-on only works with firefox. -## Install +Installation is a bit complicated, since you need to install external +tools and the web extension. Both work together. -This is a bit complicated, since you need to install external tools -and the web extension. Both work together. - -### Install `ds.sh` +# Install `ds.sh` First copy the `ds.sh` tool somewhere in your `PATH`, maybe `/usr/local/bin` as described above. -### Install the native part +# Install the native part Then install the "native" part of the web extension: @@ -54,7 +50,7 @@ the tool. Or create a file `$HOME/.config/docspell/ds.cmd` whose content is the path to the `ds.sh` script. -### Install the extension +# Install the extension An extension file can be build using the `make-xpi.sh` script. But installing it in "standard" firefox won't work, because [Mozilla diff --git a/modules/microsite/docs/doc/tools/consumedir.md b/website/site/content/docs/tools/consumedir.md similarity index 80% rename from modules/microsite/docs/doc/tools/consumedir.md rename to website/site/content/docs/tools/consumedir.md index 04029d1c..05653b2b 100644 --- a/modules/microsite/docs/doc/tools/consumedir.md +++ b/website/site/content/docs/tools/consumedir.md @@ -1,10 +1,10 @@ ---- -layout: docs -title: Consume Directory -permalink: doc/tools/consumedir ---- ++++ +title = "Consume Directory" +description = "A script to watch a directory for new files and upload them to docspell." +weight = 20 ++++ -# {{ page.title }} +# Introduction The `consumerdir.sh` is a bash script that works in two modes: @@ -43,16 +43,16 @@ $ consumedir.sh -o --path ~/Downloads --path ~/pdfs/ -m -dv http://localhost:788 The URL can be any docspell url that accepts uploads without authentication. This is usually a [source -url](../uploading#anonymous-upload). It is also possible to use the -script with the [integration -endpoint](../uploading#integration-endpoint). +url](@/docs/webapp/uploading.md#anonymous-upload). It is also possible +to use the script with the [integration +endpoint](@/docs/webapp/uploading.md#integration-endpoint). ## Integration Endpoint When given the `-i` or `--integration` option, the script changes its behaviour slightly to work with the [integration -endpoint](../uploading#integration-endpoint). +endpoint](@/docs/webapp/uploading.md#integration-endpoint). First, if `-i` is given, it implies `-r` – so the directories are watched or traversed recursively. The script then assumes that there @@ -92,12 +92,12 @@ about duplicates. This allows to keep your files organized using the file-system and have them mirrored into docspell as well. -## Systemd +# Systemd The script can be used with systemd to run as a service. This is an example unit file: -``` +``` systemd [Unit] After=networking.target Description=Docspell Consumedir @@ -111,26 +111,27 @@ ExecStart=/bin/su -s /bin/bash someuser -c "consumedir.sh --path '/a/path/' -m ' This unit file is just an example, it needs some fiddling. It assumes an existing user `someuser` that is used to run this service. The url `http://localhost:7880/api/v1/open/upload/...` is an anonymous upload -url as described [here](../uploading#anonymous-upload). +url as described [here](@/docs/webapp/uploading.md#anonymous-upload). -## Docker +# Docker The provided docker image runs this script to watch a single directory, `./docs` in current directory, for new files. If a new file is detected, it is pushed to docspell. This utilizes the [integration -endpoint](../uploading#integration-endpoint), which is enabled in the -config file, to allow uploading documents for all collectives. A -subfolder must be created for each registered collective. The docker -containers are configured to use http-header protection for the -integration endpoint. This requires you to provide a secret, that is -shared between the rest-server and the `consumedir.sh` script. This -can be done by defining an environment variable which gets picked up -by the containers defined in `docker-compose.yml`: +endpoint](@/docs/webapp/uploading.md#integration-endpoint), which is +enabled in the config file, to allow uploading documents for all +collectives. A subfolder must be created for each registered +collective. The docker containers are configured to use http-header +protection for the integration endpoint. This requires you to provide +a secret, that is shared between the rest-server and the +`consumedir.sh` script. This can be done by defining an environment +variable which gets picked up by the containers defined in +`docker-compose.yml`: -``` +``` bash export DOCSPELL_HEADER_VALUE="my-secret" docker-compose up ``` diff --git a/modules/microsite/docs/doc/tools/ds.md b/website/site/content/docs/tools/ds.md similarity index 69% rename from modules/microsite/docs/doc/tools/ds.md rename to website/site/content/docs/tools/ds.md index c727e3db..e59bf842 100644 --- a/modules/microsite/docs/doc/tools/ds.md +++ b/website/site/content/docs/tools/ds.md @@ -1,15 +1,15 @@ ---- -layout: docs -title: Upload CLI -permalink: doc/tools/ds ---- ++++ +title = "Upload CLI" +description = "A script to quickly upload files from the command line." +weight = 10 ++++ -# {{ page.title }} +# Introduction -A bash script is provided to quickly upload files from the command -line. It reads a configuration file containing the URLs to upload to. -Then each file given to the script will be uploaded to al URLs in the -config. +The `tools/ds.sh` is a bash script to quickly upload files from the +command line. It reads a configuration file containing the URLs to +upload to. Then each file given to the script will be uploaded to al +URLs in the config. The config file is expected in `$XDG_CONFIG_HOME/docspell/ds.conf`. `$XDG_CONFIG_HOME` defaults to @@ -25,10 +25,10 @@ url.2 = http://localhost:7880/api/v1/open/upload/item/6DxhjkvWf9S-CkWqF3Kr892-Wg ``` The key must start with `url`. The urls should be [anonymous upload -urls](./uploading#anonymous-upload). +urls](@/docs/webapp/uploading.md#anonymous-upload). -## Usage +# Usage - The `-c` option allows to specifiy a different config file. - The `-h` option shows a help overview. diff --git a/modules/microsite/src/main/resources/microsite/img/exim-mail.png b/website/site/content/docs/tools/exim-mail.png similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/exim-mail.png rename to website/site/content/docs/tools/exim-mail.png diff --git a/modules/microsite/docs/doc/tools/smtpgateway.md b/website/site/content/docs/tools/smtpgateway.md similarity index 87% rename from modules/microsite/docs/doc/tools/smtpgateway.md rename to website/site/content/docs/tools/smtpgateway.md index 9a73fdcf..3646cf31 100644 --- a/modules/microsite/docs/doc/tools/smtpgateway.md +++ b/website/site/content/docs/tools/smtpgateway.md @@ -1,16 +1,14 @@ ---- -layout: docs -title: SMTP Gateway with Exim -permalink: doc/tools/smtpgateway ---- - -# {{ page.title }} ++++ +title = "SMTP Gateway with Exim" +description = "Start a SMTP server that forwards all mails to docspell." +weight = 40 ++++ One possible use case for the [integration -endpoint](../uploading#integration-endpoint) is a SMTP server that -forwards all local mail to docspell. This way there is no periodic -polling involved and documents (e-mails) get into docspell without -delay. +endpoint](@/docs/webapp/uploading.md#integration-endpoint) is a SMTP +server that forwards all local mail to docspell. This way there is no +periodic polling involved and documents (e-mails) get into docspell +without delay. The `tools/exim` folder contains a docker file and a sample `exim.conf` to help start with this setup. Note that these files @@ -18,29 +16,28 @@ provide a minimal setup, you might want to add tls and spam protection when opening it to the public. -## What you need +# What you need You need to own a domain and add the appropriate MX records to point to your server. In this document, the domain `test.org` is used. You need to enable the [integration -endpoint](../uploading#integration-endpoint) in the docspell -configuration. +endpoint](@/docs/webapp/uploading.md#integration-endpoint) in the +docspell configuration. -## Exim +# Exim [Exim](http://exim.org/) is a popular smtp server (message transfer agent). It is used here only because of previous knowledge, but same can be achieved with other MTAs. -## The Config File +# The Config File Here is the example config file for exim: -``` -{% include sample-exim.conf %} -``` +{{ incl_conf(path="templates/shortcodes/sample-exim.conf") }} + Exim has good [documentation](https://www.exim.org/docs.html), look there for more info. The following is only a quick summary of the file @@ -86,18 +83,18 @@ notes about the used options (see `man curl`): `somename`. -## Install with Docker +# Install with Docker Go into the `tools/exim` directory and build the docker image: -``` shell +``` bash docker build -t ds-exim:latest -f exim.dockerfile . ``` Then start docspell somewhere and configure the integration endpoint to use http-header protection; i.e. set this in the config file: -``` +``` conf docspell.server { integration-endpoint { enabled = true @@ -114,12 +111,12 @@ variables as needed. Finally start the container: -``` shell +``` bash docker-compose up ``` -## Test Run +# Test Run Now it is possible to send mails to this MTA which will be immediatly uploaded to docspell for the collective corresponding to the @@ -127,7 +124,7 @@ uploaded to docspell for the collective corresponding to the session (the collective is named `family`): ``` -fish ~> telnet localhost 25 +~> telnet localhost 25 Trying ::1... Connected to localhost. Escape character is '^]'. @@ -157,20 +154,18 @@ this is just a test mail. quit 221 test.org closing connection Connection closed by foreign host. -fish ~> +~> ``` The mail is processed and results in an item: -
- -
+{{ figure(file="exim-mail.png") }} However, if a mail is to an unknown collective or not to the configured local domain, the server rejects it immediately: -``` shell -fish ~> telnet localhost 25 +``` bash +~> telnet localhost 25 Trying ::1... Connected to localhost. Escape character is '^]'. @@ -191,5 +186,5 @@ rcpt to: quit 221 test.org closing connection Connection closed by foreign host. -fish ~> +~> ``` diff --git a/website/site/content/docs/webapp/_index.md b/website/site/content/docs/webapp/_index.md new file mode 100644 index 00000000..d465fd0d --- /dev/null +++ b/website/site/content/docs/webapp/_index.md @@ -0,0 +1,12 @@ ++++ +title = "Web-UI" +summary = true +description = "This section describes the features of the web application." +weight = 50 +insert_anchor_links = "right" +template = "pages.html" +sort_by = "weight" +redirect_to = "docs/webapp/uploading" ++++ + +No content here. diff --git a/modules/microsite/docs/doc/curate.md b/website/site/content/docs/webapp/curate.md similarity index 75% rename from modules/microsite/docs/doc/curate.md rename to website/site/content/docs/webapp/curate.md index 2f1236f7..1d3e14b2 100644 --- a/modules/microsite/docs/doc/curate.md +++ b/website/site/content/docs/webapp/curate.md @@ -1,16 +1,13 @@ ---- -layout: docs -title: Curate Items -permalink: doc/curate ---- - -# {{page.title}} ++++ +title = "Curate Items" +weight = 20 ++++ Curating the items meta data helps finding them later. This page describes how you can quickly go through those items and correct or amend with existing data. -## Select New items +# Select New items After files have been uploaded and the job executor created the corresponding items, they will show up on the main page. All items, @@ -18,23 +15,19 @@ the job executor has created are initially marked as *New*. The option *only New* in the left search menu can be used to select only new items: -
- -
+{{ figure(file="docspell-curate-1.jpg") }} -## Check selected items +# Check selected items Then you can go through all new items and check their metadata: Click on the first item to open the detail view. This shows the documents and the meta data in the header. -
- -
+{{ figure(file="docspell-curate-2.jpg") }} -## Modify if necessary +# Modify if necessary To change something, click the *Edit* button in the menu above the document view. This will open a form next to your documents. You can @@ -43,9 +36,7 @@ item status is *New*, you'll see the suggestions docspell found during processing. If there were multiple candidates, you can select another one by clicking its name in the suggestion list. -
- -
+{{ figure(file="docspell-curate-3.jpg") }} When you change something in the form, it is immediatly applied. Only @@ -53,19 +44,18 @@ when changing text fields, a click on the *Save* symbol next to the field is required. -## Confirm +# Confirm If everything looks good, click the *Confirm* button to confirm the current data. The *New* status goes away and also the suggestions are hidden in this state. You can always go back by clicking the *Unconfirm* button. -
- -
+ +{{ figure(file="docspell-curate-5.jpg") }} -## Proceed with next item +# Proceed with next item To look at the next item in the search results, click the *Next* button in the menu (next to the *Edit* button). Clicking next, will @@ -73,6 +63,4 @@ keep the current view, so you can continue checking the data. If you are on the last item, the view switches to the listing view when clicking *Next*. -
- -
+{{ figure(file="docspell-curate-6.jpg") }} diff --git a/modules/microsite/src/main/resources/microsite/img/docspell-curate-1.jpg b/website/site/content/docs/webapp/docspell-curate-1.jpg similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/docspell-curate-1.jpg rename to website/site/content/docs/webapp/docspell-curate-1.jpg diff --git a/modules/microsite/src/main/resources/microsite/img/docspell-curate-2.jpg b/website/site/content/docs/webapp/docspell-curate-2.jpg similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/docspell-curate-2.jpg rename to website/site/content/docs/webapp/docspell-curate-2.jpg diff --git a/modules/microsite/src/main/resources/microsite/img/docspell-curate-3.jpg b/website/site/content/docs/webapp/docspell-curate-3.jpg similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/docspell-curate-3.jpg rename to website/site/content/docs/webapp/docspell-curate-3.jpg diff --git a/modules/microsite/src/main/resources/microsite/img/docspell-curate-5.jpg b/website/site/content/docs/webapp/docspell-curate-5.jpg similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/docspell-curate-5.jpg rename to website/site/content/docs/webapp/docspell-curate-5.jpg diff --git a/modules/microsite/src/main/resources/microsite/img/docspell-curate-6.jpg b/website/site/content/docs/webapp/docspell-curate-6.jpg similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/docspell-curate-6.jpg rename to website/site/content/docs/webapp/docspell-curate-6.jpg diff --git a/modules/microsite/docs/doc/emailsettings.md b/website/site/content/docs/webapp/emailsettings.md similarity index 95% rename from modules/microsite/docs/doc/emailsettings.md rename to website/site/content/docs/webapp/emailsettings.md index 050f90ac..bf8f1412 100644 --- a/modules/microsite/docs/doc/emailsettings.md +++ b/website/site/content/docs/webapp/emailsettings.md @@ -1,10 +1,9 @@ ---- -layout: docs -title: E-Mail Settings -permalink: doc/emailsettings ---- - -# {{page.title}} ++++ +title = "E-Mail Settings" +weight = 40 +[extra] +mktoc = true ++++ Docspell has a good integration for E-Mail. You can send e-mails related to an item and you can import e-mails from your mailbox into @@ -25,7 +24,7 @@ your e-mail account to send mails on behalf of you and receive your mails.* -## SMTP Settings +# SMTP Settings For sending mail, you need to provide information to connect to a SMTP server. Every e-mail provider has this information somewhere @@ -33,9 +32,7 @@ available. Configure this in *User Settings -> E-Mail Settings (SMTP)*: -
- -
+{{ figure(file="mail-settings-1.png") }} First, you need to provide some name that is used to recognize this account. This name is also used in URLs to docspell and so it must not @@ -64,7 +61,7 @@ possible to set up these settings for multiple providers, so you can choose from which account you want to send mails. -## IMAP Settings +# IMAP Settings For receiving e-mails, you need to provide information to connect to an IMAP server. Your e-mail provider should have this information @@ -72,9 +69,7 @@ somewhere available. Configure this in *User Settings -> E-Mail Settings (IMAP)*: -
- -
+{{ figure(file="mail-settings-2.png") }} First you need to define a *Name* to recognize this connection inside docspell. This name is also used in URLs to docspell and so it must @@ -93,7 +88,7 @@ Here is an example for posteo.de: - SSL: use `StartTLS` -## SSL / TLS / StartTLS +# SSL / TLS / StartTLS *Please Note: If `SSL` is set to `None`, then mails will be sent unencrypted to your mail provider! If `Ignore certificate check` is @@ -102,7 +97,7 @@ provider is wrongly configured for SSL/TLS. This flag should only be enabled if you know why.* -## GMail +# GMail Authenticating with GMail may be not so simple. GMail implements an authentication scheme called *XOAUTH2* (at least for Imap). It will @@ -132,7 +127,7 @@ Download the `oauth2.py` script from [here](https://github.com/google/gmail-oauth2-tools) and first create an *oauth2-token*: -``` shell +``` bash ./oauth2.py --user=your.name@gmail.com \ --client_id=106701....d8c.apps.googleusercontent.com \ --client_secret=5Z1...Kir_t \ @@ -176,7 +171,8 @@ Access Token Expiration Seconds: 3599 The problem is that the access token expires. Docspell doesn't support updating the access token. It could be worked around by setting up a cron-job or similiar which uses the `oauth2.py` tool to generate new -access tokens and update your imap settings via a [REST](../api) call. +access tokens and update your imap settings via a +[REST](@/docs/api/_index.md) call. ``` bash #!/usr/bin/env bash diff --git a/modules/microsite/docs/doc/finding.md b/website/site/content/docs/webapp/finding.md similarity index 88% rename from modules/microsite/docs/doc/finding.md rename to website/site/content/docs/webapp/finding.md index 78b17c47..61d532ca 100644 --- a/modules/microsite/docs/doc/finding.md +++ b/website/site/content/docs/webapp/finding.md @@ -1,10 +1,9 @@ ---- -layout: docs -title: Finding Items -permalink: doc/finding ---- - -# {{ page.title }} ++++ +title = "Finding Items" +weight = 30 +[extra] +mktoc = true ++++ Items can be searched by their annotated meta data and their contents using full text search. The landing page shows a list of current @@ -15,15 +14,15 @@ menu with many options. Both are active at the same time, but only one is visible. You can switch between them without affecting the results. -## Search Bar +# Search Bar - +{{ imgright(file="search-bar.png") }} By default, the search bar is shown. It provides a refined view of the search menu. The dropdown contains different options to do a quick search. -### *All Names* and *Contents* +## *All Names* and *Contents* These two options correspond to the same named field in the search menu. If you switch between search menu and search bar (by clicking @@ -55,7 +54,7 @@ restricted by the search term given in the search-bar, but also by what is specified in the search menu. -### *Contents Only* +## *Contents Only* This option has no corresponding part in the search menu. Searching with this option active, there is only a full text search done in the @@ -66,33 +65,33 @@ respect to the search term. This ordering is returned from the full text search engine and is simply transfered unmodified. -## Search Menu +# Search Menu - +{{ imgright(file="search-menu.png") }} The search menu can be opened by clicking the left icon in the top bar. It shows some options to constrain the item list: -### Show new items +## Show new items Clicking the checkbox "Only new" shows items that have not been "Confirmed". All items that have been created by docspell and not looked at are marked as "new" automatically. -### Names +## Names Searches in names of certain properties. The `All Names` field is the same as the search in the search bar (see above). The `Name` field only searches in the name property of an item. -### Folder +## Folder Set a folder to only show items in that folder. If no folder is set, all accessible items are shown. These are all items that either have no folder set, or a folder where the current user is member. -### Tags +## Tags Specify a list of tags that the items must have. When adding tags to the "Include" list, an item must have all these tags in order to be @@ -101,15 +100,15 @@ included in the results. When adding tags to the "Exclude" list, then an item is removed from the results if it has at least one of these tags. -### Correspondent +## Correspondent Pick a correspondent to show only these items. -### Concerned +## Concerned Pick a concerned entity to show only these items. -### Date +## Date Specify a date range to show only items whose date property is within this range. If you want to see items of a specific day, choose the @@ -118,18 +117,18 @@ same day for both fields. For items that don't have an explicitly date property set, the created date is used. -### Due Date +## Due Date Specify a date range to show only items whose due date property is within this range. Items without a due date are not shown. -### Direction +## Direction Specify whether to show only incoming, only outgoing or all items. -## Customize Substring Search +# Customize Substring Search The substring search of the *All Names* and *Name* field can be customized in the following way: A wildcard `*` can be used at the @@ -145,10 +144,10 @@ unless one of the following is true: - The term is enclosed in quotes `"`. -## Full Text Search +# Full Text Search -### The Query +## The Query The query string for full text search is very powerful. Docspell currently supports [Apache SOLR](https://lucene.apache.org/solr/) as @@ -171,19 +170,9 @@ It will by default search all indexed fields, which are: attachment contents, attachment names, item name and item notes. -### The Results +## The Results When using full text search, each item in the result list is annotated with the highlighted occurrence of the match. -
- -
- - -## Screencast - - +{{ figure(file="search-content-results.png") }} diff --git a/modules/microsite/src/main/resources/microsite/img/mail-item-1.jpg b/website/site/content/docs/webapp/mail-item-1.jpg similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/mail-item-1.jpg rename to website/site/content/docs/webapp/mail-item-1.jpg diff --git a/modules/microsite/src/main/resources/microsite/img/mail-item-2.jpg b/website/site/content/docs/webapp/mail-item-2.jpg similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/mail-item-2.jpg rename to website/site/content/docs/webapp/mail-item-2.jpg diff --git a/modules/microsite/src/main/resources/microsite/img/mail-item-3.jpg b/website/site/content/docs/webapp/mail-item-3.jpg similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/mail-item-3.jpg rename to website/site/content/docs/webapp/mail-item-3.jpg diff --git a/modules/microsite/src/main/resources/microsite/img/mail-item-4.jpg b/website/site/content/docs/webapp/mail-item-4.jpg similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/mail-item-4.jpg rename to website/site/content/docs/webapp/mail-item-4.jpg diff --git a/modules/microsite/src/main/resources/microsite/img/mail-settings-1.png b/website/site/content/docs/webapp/mail-settings-1.png similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/mail-settings-1.png rename to website/site/content/docs/webapp/mail-settings-1.png diff --git a/modules/microsite/src/main/resources/microsite/img/mail-settings-2.png b/website/site/content/docs/webapp/mail-settings-2.png similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/mail-settings-2.png rename to website/site/content/docs/webapp/mail-settings-2.png diff --git a/modules/microsite/docs/doc/mailitem.md b/website/site/content/docs/webapp/mailitem.md similarity index 80% rename from modules/microsite/docs/doc/mailitem.md rename to website/site/content/docs/webapp/mailitem.md index 2d08fa1d..1fa31e9a 100644 --- a/modules/microsite/docs/doc/mailitem.md +++ b/website/site/content/docs/webapp/mailitem.md @@ -1,10 +1,9 @@ ---- -layout: docs -title: Send items via E-Mail -permalink: doc/mailitem ---- - -# {{page.title}} ++++ +title = "Send items via E-Mail" +weight = 50 +[extra] +mktoc = true ++++ You can send e-mails from within docspell attaching the files of an item. This is useful to collaborate or share certain documents with @@ -13,13 +12,13 @@ people outside docspell. All sent mails are stored attached to the item. -## E-Mail Settings (SMTP) +# E-Mail Settings (SMTP) To send mails, there are SMTP settings required. Please see the page -about [e-mail settings](emailsettings#smtp-settings). +about [e-mail settings](@/docs/webapp/emailsettings.md#smtp-settings). -## Sending Mails +# Sending Mails Currently, it is possible to send mails related to only one item. You can define the mail body and docspell will add the attachments of an @@ -28,9 +27,7 @@ item, or you may choose to send the mail without any attachments. In the item detail view, click on the envelope icon to open the mail form: -
- -
+{{ figure(file="mail-item-1.jpg") }} Then write the mail. Multiple recipients may be specified. The input field shows completion proposals from all contacts in your address @@ -56,25 +53,18 @@ Once you click *Send*, the docspell server will send the mail using your connection settings. If that succeeds the mail is saved to the database and you'll see a message in the form. -## Accessing Sent Mails +# Accessing Sent Mails If there is an e-mail for an item, a tab shows up at the right side, next to the attachments. - -
- -
+{{ figure(file="mail-item-2.jpg") }} This tab shows a list of all mails that have been sent related to this item. -
- -
+{{ figure(file="mail-item-3.jpg") }} Clicking on a mail opens it in detail. -
- -
+{{ figure(file="mail-item-4.jpg") }} diff --git a/modules/microsite/docs/doc/metadata.md b/website/site/content/docs/webapp/metadata.md similarity index 93% rename from modules/microsite/docs/doc/metadata.md rename to website/site/content/docs/webapp/metadata.md index 881e346f..665d0815 100644 --- a/modules/microsite/docs/doc/metadata.md +++ b/website/site/content/docs/webapp/metadata.md @@ -1,12 +1,11 @@ ---- -layout: docs -title: Meta Data -permalink: doc/metadata ---- ++++ +title = "Meta Data" +weight = 10 +[extra] +mktoc = true ++++ -# {{ page.title }} - -## Meta Data +# Metadata Docspell processes each uploaded file. Processing involves extracting archives, extracting text, anlyzing the extracted text and converting @@ -20,8 +19,7 @@ known meta data. The *Meta Data* page allows to manage this meta data: - Equipments - Folders - -### Tags +## Tags Items can be tagged with multiple custom tags (aka labels). This allows to describe many different workflows people may have with their @@ -40,7 +38,7 @@ not automatically associate tags to your items. The tags are only meant to be used manually for now. -### Organization and Person +## Organization and Person The organization entity represents an non-personal (organization or company) correspondent of an item. Docspell will choose one or more @@ -68,7 +66,7 @@ document processing. They might be useful when using this as a real address book. -### Equipment +## Equipment The equipment entity is almost like a tag. In fact, it could be replaced by a tag with a specific known category. The difference is @@ -81,7 +79,7 @@ Equipments don't have contact information, so the only property that is used to find matches during document processing is its name. -### Folders +## Folders Folders provide a way to divide all documents into disjoint subsets. Unlike with tags, an item can have at most one folder or none. A @@ -102,11 +100,11 @@ can guess the item-id. While folders are *not* taken into account when processing documents, they can be specified with the upload request or a [source -url](uploading#anonymous-upload) to have them automatically set when -they arrive. +url](@/docs/webapp/uploading.md#anonymous-upload) to have them +automatically set when files arrive. -## Document Language +# Document Language An important setting is the language of your documents. This helps OCR and text analysis. You can select between English and German diff --git a/modules/microsite/src/main/resources/microsite/img/notify-due-items.jpg b/website/site/content/docs/webapp/notify-due-items.jpg similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/notify-due-items.jpg rename to website/site/content/docs/webapp/notify-due-items.jpg diff --git a/modules/microsite/docs/doc/notifydueitems.md b/website/site/content/docs/webapp/notifydueitems.md similarity index 89% rename from modules/microsite/docs/doc/notifydueitems.md rename to website/site/content/docs/webapp/notifydueitems.md index 5a53eea7..f97e3fc4 100644 --- a/modules/microsite/docs/doc/notifydueitems.md +++ b/website/site/content/docs/webapp/notifydueitems.md @@ -1,26 +1,23 @@ ---- -layout: docs -title: Notify about due items -permalink: doc/notifydueitems ---- - -# {{page.title}} ++++ +title = "Notify about due items" +weight = 60 +[extra] +mktoc = true ++++ A user that provides valid email (smtp) settings, can be notified by docspell about due items. You will then receive an e-mail containing a list of items, sorted by their due date. You need first define smtp settings, please see [this -page](mailitem#e-mail-settings). +page](@/docs/webapp/emailsettings.md#smtp-settings). Notifying works simply by searching for due items periodically. It will be submitted to the job queue and is picked up by an available -[job executor](joex) eventually. This can be setup in the user -settings page. +[job executor](@/docs/joex/_index.md) eventually. This can be setup in +the user settings page. -
- -
+{{ figure(file="notify-due-items.jpg") }} At first, the task can be disabled/enabled any time. @@ -73,4 +70,5 @@ a free job executor). If you click *Submit* these settings are saved and the task runs periodically. -You can see the task executing at the [processing page](processing). +You can see the task executing at the [processing +page](@/docs/webapp/processing.md). diff --git a/modules/microsite/src/main/resources/microsite/img/processing-queue.jpg b/website/site/content/docs/webapp/processing-queue.jpg similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/processing-queue.jpg rename to website/site/content/docs/webapp/processing-queue.jpg diff --git a/modules/microsite/docs/doc/processing.md b/website/site/content/docs/webapp/processing.md similarity index 89% rename from modules/microsite/docs/doc/processing.md rename to website/site/content/docs/webapp/processing.md index 1554e815..dfd3decf 100644 --- a/modules/microsite/docs/doc/processing.md +++ b/website/site/content/docs/webapp/processing.md @@ -1,10 +1,9 @@ ---- -layout: docs -title: Processing Queue -permalink: doc/processing ---- - -# {{ page.title }} ++++ +title = "Processing Queue" +weight = 80 +[extra] +mktoc = true ++++ The page *Processing Queue* shows the current state of document @@ -18,9 +17,7 @@ page refreshes itself automatically to show the progress. Example screenshot: -
- -
+{{ figure(file="processing-queue.jpg") }} You can cancel running jobs or remove waiting ones from the queue. If you click on the small file symbol on finished jobs, you can inspect diff --git a/modules/microsite/src/main/resources/microsite/img/scanmailbox-detail.png b/website/site/content/docs/webapp/scanmailbox-detail.png similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/scanmailbox-detail.png rename to website/site/content/docs/webapp/scanmailbox-detail.png diff --git a/modules/microsite/src/main/resources/microsite/img/scanmailbox-list.png b/website/site/content/docs/webapp/scanmailbox-list.png similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/scanmailbox-list.png rename to website/site/content/docs/webapp/scanmailbox-list.png diff --git a/modules/microsite/docs/doc/scanmailbox.md b/website/site/content/docs/webapp/scanmailbox.md similarity index 90% rename from modules/microsite/docs/doc/scanmailbox.md rename to website/site/content/docs/webapp/scanmailbox.md index ea0d3dee..c660bf50 100644 --- a/modules/microsite/docs/doc/scanmailbox.md +++ b/website/site/content/docs/webapp/scanmailbox.md @@ -1,41 +1,36 @@ ---- -layout: docs -title: Scan Mailboxes -permalink: doc/scanmailbox ---- - -# {{page.title}} ++++ +title = "Scan Mailboxes" +weight = 70 +[extra] +mktoc = true ++++ User that provide valid email (imap) settings, can import mails from their mailbox into docspell periodically. You need first define imap settings, please see [this -page](emailsettings#imap-settings). +page](@/docs/webapp/emailsettings.md#imap-settings). Go to *User Settings -> Scan Mailbox Task*. You can define periodic tasks that connects to your mailbox and import mails into docspell. It is possible to define multiple tasks, for example, if you have multiple e-mail accounts you want to import periodically. -
- -
+{{ figure(file="scanmailbox-list.png") }} -## Details +# Details Creating a task requires the following information: -
- -
+{{ figure(file="scanmailbox-detail.png") }} You can enable or disable this task. A disabled task will not run periodically. You can still choose to run it manually if you click the `Start Once` button. Then you need to specify which [IMAP -connection](emailsettings#imap-settings) to use. +connection](@/docs/webapp/emailsettings.md#imap-settings) to use. A list of folders is required. Docspell will only look into these folders. You can specify multiple folders. The "Inbox" folder is a @@ -89,7 +84,7 @@ multiples of `7` added to it. In other words, it matches `1`, `1+7`, `1+7+7`, `1+7+7+7` and so on. -## Reading Mails twice / Duplicates +# Reading Mails twice / Duplicates Since users can move around mails in their mailboxes, it can happen that docspell unintentionally reads a mail multiple times. If docspell @@ -109,13 +104,13 @@ In later versions, docspell may use the checksum of the generated eml file to look for duplicates, too. -## How it works +# How it works Docspell will go through all folders and download mails in “batches”. This size can be set by the admin in the [configuration -file](configure#joex) and applies to all these tasks (same for all -users). This batch only contains the mail headers and not the complete -mail. +file](@/docs/configure/_index.md#joex) and applies to all these tasks +(same for all users). This batch only contains the mail headers and +not the complete mail. Then each mail is downloaded completely one by one and converted into an [eml](https://en.wikipedia.org/wiki/Email#Filename_extensions) file diff --git a/website/site/content/docs/webapp/search-bar.png b/website/site/content/docs/webapp/search-bar.png new file mode 100644 index 00000000..b7424bf3 Binary files /dev/null and b/website/site/content/docs/webapp/search-bar.png differ diff --git a/modules/microsite/src/main/resources/microsite/img/search-content-results.png b/website/site/content/docs/webapp/search-content-results.png similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/search-content-results.png rename to website/site/content/docs/webapp/search-content-results.png diff --git a/website/site/content/docs/webapp/search-menu.png b/website/site/content/docs/webapp/search-menu.png new file mode 100644 index 00000000..c35b4ad6 Binary files /dev/null and b/website/site/content/docs/webapp/search-menu.png differ diff --git a/modules/microsite/src/main/resources/microsite/img/sources-form.png b/website/site/content/docs/webapp/sources-form.png similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/sources-form.png rename to website/site/content/docs/webapp/sources-form.png diff --git a/modules/microsite/docs/doc/uploading.md b/website/site/content/docs/webapp/uploading.md similarity index 79% rename from modules/microsite/docs/doc/uploading.md rename to website/site/content/docs/webapp/uploading.md index d99222b6..25ced2c2 100644 --- a/modules/microsite/docs/doc/uploading.md +++ b/website/site/content/docs/webapp/uploading.md @@ -1,17 +1,13 @@ ---- -layout: docs -title: Uploads -permalink: doc/uploading ---- - -# {{page.title}} - ++++ +title = "Uploads" +weight = 0 ++++ This page describes, how files can get into docspell. Technically, there is just one way: via http multipart/form-data requests. -## Authenticated Upload +# Authenticated Upload From within the web application there is the "Upload Files" page. There you can select multiple files to upload. You can also @@ -29,32 +25,27 @@ This obviously requires an authenticated user. While this is handy for ad-hoc uploads, it is very inconvenient for automating it by custom scripts. For this the next variant exists. -## Anonymous Upload +# Anonymous Upload It is also possible to upload files without authentication. This should make tools that interact with docspell much easier to write. - -### Creating Anonymous Uploads - Go to "Collective Settings" and then to the "Source" tab. A *Source* -identifies an endpoint where files can be uploaded -anonymously. Creating a new source creates a long unique id which is -part on an url that can be used to upload files. You can choose any -time to deactivate or delete the source at which point uploading is -not possible anymore. The idea is to give this URL away safely. You -can delete it any time and no passwords or secrets are visible, even -your username is not visible. +identifies an endpoint where files can be uploaded anonymously. +Creating a new source creates a long unique id which is part of an url +that can be used to upload files. You can choose any time to +deactivate or delete the source at which point uploading is not +possible anymore. The idea is to give this URL away safely. You can +delete it any time and no passwords or secrets are visible, even your +username is not visible. Example screenshot: -
- -
+{{ figure(file="sources-form.png") }} This example shows a source with name "test". Besides a description and a name that is only used for displaying purposes, a priority and a -[folder](metadata#folders) can be specified. +[folder](@/docs/webapp/metadata.md#folders) can be specified. The priority is used for the processing jobs that are submitted when files are uploaded via this endpoint. @@ -72,9 +63,9 @@ your account. You could give this url to people for sending files directly into your docspell. The second url is the API url, which accepts the requests to upload -files (which is used by the first url). +files (it is used by the upload page, the first url). -For example, this url can be used to upload files with curl: +For example, the api url can be used to upload files with curl: ``` bash $ curl -XPOST -F file=@test.pdf http://localhost:7880/api/v1/open/upload/item/CqpFTb7UmGe-9nMVPZSmnwc-AHH6nWFh52t-M1JFQ9y7cdH @@ -83,11 +74,14 @@ $ curl -XPOST -F file=@test.pdf http://localhost:7880/api/v1/open/upload/item/Cq You could add more `-F file=@/path/to/your/file.pdf` to upload multiple files (note, the `@` is required by curl, so it knows that -the following is a file). +the following is a file). There is a [script +provided](@/docs/tools/ds.md) that uses this to upload files from the +command line. When files are uploaded to an source endpoint, the items resulting from this uploads are marked with the name of the source. So you know -which source an item originated. +which source an item originated. There is also a counter incremented +for each reqest. If files are uploaded using the web applications *Upload files* page, the source is implicitly set to `webapp`. If you also want to let @@ -95,7 +89,7 @@ docspell count the files uploaded through the web interface, just create a source (can be inactive) with that name (`webapp`). -## Integration Endpoint +# Integration Endpoint Another option for uploading files is the special *integration endpoint*. This endpoint allows an admin to upload files to any @@ -106,8 +100,8 @@ collective, that is known by name. ``` The endpoint is behind `/api/v1/open`, so this route is not protected -by an authentication token (see [REST Api](../api) for more -information). However, it can be protected via settings in the +by an authentication token (see [REST Api](@/docs/api/_index.md) for +more information). However, it can be protected via settings in the configuration file. The idea is that this endpoint is controlled by an administrator and not the user of the application. The admin can enable this endpoint and choose between some methods to protect it. @@ -117,7 +111,7 @@ on the same host or network). The endpoint is disabled by default, an admin must change the `docspell.server.integration-endpoint.enabled` flag to `true` in the -[configuration file](configure#rest-server). +[configuration file](@/docs/configure/_index.md#rest-server). If queried by a `GET` request, it returns whether it is enabled and the collective exists. @@ -129,10 +123,10 @@ checksum with: /api/v1/open/integration/checkfile/[collective-name]/[sha256-checksum] ``` -See the [SMTP gateway](tools/smtpgateway) or the [consumedir -script](tools/consumedir) for examples to use this endpoint. +See the [SMTP gateway](@/docs/tools/smtpgateway.md) or the [consumedir +script](@/docs/tools/consumedir.md) for examples to use this endpoint. -## The Request +# The Request This gives more details about the request for uploads. It is a http `multipart/form-data` request, with two possible fields: @@ -151,7 +145,7 @@ unstructured binary files. The `meta` content must be `application/json` containing this structure: -``` +``` elm { multiple: Bool , direction: Maybe String , folder: Maybe String @@ -176,7 +170,7 @@ This kind of request is very common and most programming languages have support for this. For example, here is another curl command uploading two files with meta data: -``` +``` bash curl -XPOST -F meta='{"multiple":false, "direction": "outgoing"}' \ -F file=@letter-en-source.pdf \ -F file=@letter-de-source.pdf \ diff --git a/website/site/sass/styles.sass b/website/site/sass/styles.sass new file mode 100644 index 00000000..608687b0 --- /dev/null +++ b/website/site/sass/styles.sass @@ -0,0 +1,102 @@ +@charset "utf-8" + +// Set your brand colors +$dblue: #1c2d61 +$infoblue: #193e74 +$dred: #9c0011 +$dwhite: #f0f8ff +$code: $dblue +$pre: $dblue + +@import url('https://fonts.googleapis.com/css?family=Libre+Baskerville:400,700') +@import url('https://fonts.googleapis.com/css?family=Montserrat:400,600') +@import url('https://fonts.googleapis.com/css?family=Source+Code+Pro:400') + +// Update Bulma's global variables +$family-serif: "Libre Baskerville",serif +$family-sans-serif: "Montserrat", sans-serif +$family-monospace: "Source Code Pro",monospace +//$family-primary: $family-serif + +/* $grey-dark: $brown; +/* $grey-light: $beige-light; +$primary: $dred +$info: $infoblue + +//josh-rose-trYl7JYATH0-unsplash +//tersius-van-rhyn-xcQWMPm9fG8-unsplash +//cassie-boca-x-tbVqkfQCU-unsplash +//jf-martin-Ofs3LjEUcrk-unsplash + // background: linear-gradient(rgba(0, 0, 0, 0.5), rgba(0, 0, 0, 1)), url(img/jf-martin-Ofs3LjEUcrk-unsplash.jpg) +//jesse-gardner-EqdpXeemf58-unsplash +#hero-main + background: linear-gradient(rgba(0, 0, 0, 0.6), rgba(0, 0, 0, 1)), url('img/jf-martin-Ofs3LjEUcrk-unsplash.jpg') + background-size: 100% 100% + background-repeat: no-repeat + .hero-body h1, h2, p + text-shadow: -2px -2px black + color: #fff + +.main-title + font-size: 4rem + +.card.full-height + height: 100% + &:hover + box-shadow: 1px 1px 3px $info; + +h1 + font-family: $family-serif + +.doc + h1 + border-bottom: 1px solid #ccc + padding-bottom: 0.15em + +a.zola-anchor + padding-left: 0.75rem + display: none + font-family: $family-monospace + +h1:hover a.zola-anchor + display: inline + +h2:hover a.zola-anchor + display: inline + +h3:hover a.zola-anchor + display: inline + +h4:hover a.zola-anchor + display: inline + +nav.breadcrumb + background: $dwhite + padding-top: 0.25rem + padding-bottom: 0.25rem + a + font-weight: bold + color: $dblue + &:hover + color: black + +span.unsplash-credit + color: #555 + margin-right: 0.5em + float: right + font-size: smaller + a + color: #555 + &:hover + color: #777 + + +//import all of bulma +@import "../../node_modules/bulma/bulma.sass" + +p.has-text + color: $text + +.card.full-height + .card-content + height: 100% diff --git a/modules/microsite/docs/CNAME b/website/site/static/CNAME similarity index 100% rename from modules/microsite/docs/CNAME rename to website/site/static/CNAME diff --git a/website/site/static/favicon-mc.ico b/website/site/static/favicon-mc.ico new file mode 100644 index 00000000..2965c866 Binary files /dev/null and b/website/site/static/favicon-mc.ico differ diff --git a/website/site/static/favicon.ico b/website/site/static/favicon.ico new file mode 100644 index 00000000..1cb3c64e Binary files /dev/null and b/website/site/static/favicon.ico differ diff --git a/website/site/static/icons/eye-40.svg b/website/site/static/icons/eye-40.svg new file mode 100644 index 00000000..e305ed34 --- /dev/null +++ b/website/site/static/icons/eye-40.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/website/site/static/icons/ghost-40.svg b/website/site/static/icons/ghost-40.svg new file mode 100644 index 00000000..42483f76 --- /dev/null +++ b/website/site/static/icons/ghost-40.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/website/site/static/icons/github-40.svg b/website/site/static/icons/github-40.svg new file mode 100644 index 00000000..df004111 --- /dev/null +++ b/website/site/static/icons/github-40.svg @@ -0,0 +1,4 @@ + + + + diff --git a/website/site/static/icons/home-40.svg b/website/site/static/icons/home-40.svg new file mode 100644 index 00000000..30849564 --- /dev/null +++ b/website/site/static/icons/home-40.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/website/site/static/icons/info-square-40.svg b/website/site/static/icons/info-square-40.svg new file mode 100644 index 00000000..538939f1 --- /dev/null +++ b/website/site/static/icons/info-square-40.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/website/site/static/icons/logo-only-36.svg b/website/site/static/icons/logo-only-36.svg new file mode 100644 index 00000000..cfc22dc0 --- /dev/null +++ b/website/site/static/icons/logo-only-36.svg @@ -0,0 +1,129 @@ + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + diff --git a/website/site/static/icons/logo-only-mc.svg b/website/site/static/icons/logo-only-mc.svg new file mode 100644 index 00000000..61dc53de --- /dev/null +++ b/website/site/static/icons/logo-only-mc.svg @@ -0,0 +1,137 @@ + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + diff --git a/website/site/static/icons/logo-only.svg b/website/site/static/icons/logo-only.svg new file mode 100644 index 00000000..3a577dc6 --- /dev/null +++ b/website/site/static/icons/logo-only.svg @@ -0,0 +1,129 @@ + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + diff --git a/website/site/static/icons/notes-40.svg b/website/site/static/icons/notes-40.svg new file mode 100644 index 00000000..bfccbc8d --- /dev/null +++ b/website/site/static/icons/notes-40.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/website/site/static/icons/refresh-40.svg b/website/site/static/icons/refresh-40.svg new file mode 100644 index 00000000..a10715ec --- /dev/null +++ b/website/site/static/icons/refresh-40.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/website/site/static/icons/square-plus-40.svg b/website/site/static/icons/square-plus-40.svg new file mode 100644 index 00000000..a201824e --- /dev/null +++ b/website/site/static/icons/square-plus-40.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/website/site/static/img/analyze-feature.png b/website/site/static/img/analyze-feature.png new file mode 100644 index 00000000..847520ed Binary files /dev/null and b/website/site/static/img/analyze-feature.png differ diff --git a/modules/microsite/src/main/resources/microsite/img/back-master-small.jpg b/website/site/static/img/back-master-small.jpg similarity index 100% rename from modules/microsite/src/main/resources/microsite/img/back-master-small.jpg rename to website/site/static/img/back-master-small.jpg diff --git a/website/site/static/img/cassie-boca-x-tbVqkfQCU-unsplash.jpg b/website/site/static/img/cassie-boca-x-tbVqkfQCU-unsplash.jpg new file mode 100644 index 00000000..ee5834a6 Binary files /dev/null and b/website/site/static/img/cassie-boca-x-tbVqkfQCU-unsplash.jpg differ diff --git a/website/site/static/img/convertpdf-feature.svg b/website/site/static/img/convertpdf-feature.svg new file mode 100644 index 00000000..2c766e11 --- /dev/null +++ b/website/site/static/img/convertpdf-feature.svg @@ -0,0 +1,618 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + pdf + + + + + + + + + + + + + + + + + + + + + + → + + diff --git a/website/site/static/img/filetype-feature.svg b/website/site/static/img/filetype-feature.svg new file mode 100644 index 00000000..c8808dd9 --- /dev/null +++ b/website/site/static/img/filetype-feature.svg @@ -0,0 +1,1654 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + zip + + + + + + + + + + + + + + html + + + + + + + + + + + + + + odt + + + + + + + + + + + jpgpngtiff + + + + + + + + + + + doc/x + + + + + + + + + + + xls/x + + + + + + + + pdf + + + + + + + + + + + + + + + rtf + + + + + + + + + + + + + txt + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ods + + + + + + + + + @ + + eml + + + + diff --git a/website/site/static/img/fts-feature.png b/website/site/static/img/fts-feature.png new file mode 100644 index 00000000..f33192f0 Binary files /dev/null and b/website/site/static/img/fts-feature.png differ diff --git a/website/site/static/img/jesse-gardner-EqdpXeemf58-unsplash.jpg b/website/site/static/img/jesse-gardner-EqdpXeemf58-unsplash.jpg new file mode 100644 index 00000000..c2b22de1 Binary files /dev/null and b/website/site/static/img/jesse-gardner-EqdpXeemf58-unsplash.jpg differ diff --git a/website/site/static/img/jf-martin-Ofs3LjEUcrk-unsplash.jpg b/website/site/static/img/jf-martin-Ofs3LjEUcrk-unsplash.jpg new file mode 100644 index 00000000..c845c14c Binary files /dev/null and b/website/site/static/img/jf-martin-Ofs3LjEUcrk-unsplash.jpg differ diff --git a/website/site/static/img/josh-rose-trYl7JYATH0-unsplash.jpg b/website/site/static/img/josh-rose-trYl7JYATH0-unsplash.jpg new file mode 100644 index 00000000..6633fb7e Binary files /dev/null and b/website/site/static/img/josh-rose-trYl7JYATH0-unsplash.jpg differ diff --git a/website/site/static/img/notify-feature.png b/website/site/static/img/notify-feature.png new file mode 100644 index 00000000..b2b085f2 Binary files /dev/null and b/website/site/static/img/notify-feature.png differ diff --git a/website/site/static/img/ocr-feature.png b/website/site/static/img/ocr-feature.png new file mode 100644 index 00000000..9053f8d9 Binary files /dev/null and b/website/site/static/img/ocr-feature.png differ diff --git a/website/site/static/img/poster.png b/website/site/static/img/poster.png new file mode 100644 index 00000000..e1688f8c Binary files /dev/null and b/website/site/static/img/poster.png differ diff --git a/website/site/static/img/scanmailbox-feature.png b/website/site/static/img/scanmailbox-feature.png new file mode 100644 index 00000000..05b603d9 Binary files /dev/null and b/website/site/static/img/scanmailbox-feature.png differ diff --git a/website/site/static/img/sendmail-feature.png b/website/site/static/img/sendmail-feature.png new file mode 100644 index 00000000..2bf428ee Binary files /dev/null and b/website/site/static/img/sendmail-feature.png differ diff --git a/website/site/static/img/tersius-van-rhyn-xcQWMPm9fG8-unsplash.jpg b/website/site/static/img/tersius-van-rhyn-xcQWMPm9fG8-unsplash.jpg new file mode 100644 index 00000000..7def9f67 Binary files /dev/null and b/website/site/static/img/tersius-van-rhyn-xcQWMPm9fG8-unsplash.jpg differ diff --git a/website/site/static/img/user-feature.png b/website/site/static/img/user-feature.png new file mode 100644 index 00000000..d557f282 Binary files /dev/null and b/website/site/static/img/user-feature.png differ diff --git a/website/site/templates/404.html b/website/site/templates/404.html new file mode 100644 index 00000000..06a97a14 --- /dev/null +++ b/website/site/templates/404.html @@ -0,0 +1,41 @@ + + + + + + + Not Found + + + +
+
+ {% include "navbar.html" %} +
+
+

+ Not Found +

+
+
+ +
+
+
+
+ Not Found Ghost +
+
+
+

+ Not Found :( +

+

+ → Documentation? +

+
+
+
+ {% include "footer.html" %} + + diff --git a/website/site/templates/footer.html b/website/site/templates/footer.html new file mode 100644 index 00000000..163d9458 --- /dev/null +++ b/website/site/templates/footer.html @@ -0,0 +1,23 @@ + diff --git a/website/site/templates/index.html b/website/site/templates/index.html new file mode 100644 index 00000000..e9eab111 --- /dev/null +++ b/website/site/templates/index.html @@ -0,0 +1,20 @@ + + + + {% include "meta.html" %} + Docspell – Simple document organizer + + + + + + + diff --git a/website/site/templates/meta.html b/website/site/templates/meta.html new file mode 100644 index 00000000..10869d43 --- /dev/null +++ b/website/site/templates/meta.html @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/website/site/templates/navbar.html b/website/site/templates/navbar.html new file mode 100644 index 00000000..92210868 --- /dev/null +++ b/website/site/templates/navbar.html @@ -0,0 +1,24 @@ + diff --git a/website/site/templates/overview.html b/website/site/templates/overview.html new file mode 100644 index 00000000..62917fb1 --- /dev/null +++ b/website/site/templates/overview.html @@ -0,0 +1,59 @@ + + + + {% include "meta.html" %} + {{ section.title }} – Docspell Documentation + + + +
+
+ {% include "navbar.html" %} +
+
+

+ Docspell Documentation +

+

+ {{ section.title }} +

+
+
+ +
+
+
+ This is the documentation for Docspell, version {{ config.extra.version }}. +
+
+ {% for section in section.subsections %} + + {% set sub = get_section(path=section) %} + +
+
+ +
+
+
+

+ {{ sub.title | title }} +

+

+ {{ sub.description }} +

+
+
+
+ +
+
+ {% endfor %} +
+
+
+ + + {% include "footer.html" %} + + diff --git a/website/site/templates/page.html b/website/site/templates/page.html new file mode 100644 index 00000000..1dc83604 --- /dev/null +++ b/website/site/templates/page.html @@ -0,0 +1,92 @@ + + + + {% include "meta.html" %} + {{ page.title }} – Docspell Documentation + + + +
+
+ {% include "navbar.html" %} +
+
+

+ {{ page.title }} +

+

+ Docspell Documentation +

+
+
+ + +
+
+
+
+
    + {% set last = page.ancestors | last %} + {% set s = get_section(path=last) %} + +

    + {{ s.title }} +

    + {% for sub in s.pages %} + {% set sub_is_current = page.permalink == sub.permalink %} +
  • + + {{ sub.title | title }} + + {% if sub_is_current %} + + {% endif %} +
    • + {% endfor %} +
+
+
+ +
+
+
+ {{ page.content | safe }} +
+
+
+
+
+ + {% include "footer.html" %} + + diff --git a/website/site/templates/pages.html b/website/site/templates/pages.html new file mode 100644 index 00000000..01c143f7 --- /dev/null +++ b/website/site/templates/pages.html @@ -0,0 +1,65 @@ + + + + {% include "meta.html" %} + {{ section.title }} – Docspell Documentation + + + +
+
+ {% include "navbar.html" %} +
+
+

+ Docspell Documentation +

+

+ {{ section.title }} +

+
+
+ + +
+
+
+ + +
+
+
+ {{ section.content | safe }} +
+
+
+
+ + {% include "footer.html" %} + + diff --git a/website/site/templates/section.html b/website/site/templates/section.html new file mode 100644 index 00000000..55d5b047 --- /dev/null +++ b/website/site/templates/section.html @@ -0,0 +1,85 @@ + + + + {% include "meta.html" %} + {{ section.title }} – Docspell Documentation + + + +
+
+ {% include "navbar.html" %} +
+
+

+ {{ section.title }} +

+

+ Docspell Documentation +

+
+
+ + {% if section.extra.mktoc %} +
+
+
+
+ +
+
+ +
+
+
+ {{ section.content | safe }} +
+
+
+
+
+ {% endif %} + + {% if not section.extra.mktoc %} +
+
+
+ {{ section.content | safe }} +
+
+
+ {% endif %} + + {% include "footer.html" %} + + + diff --git a/website/site/templates/shortcodes/figure.html b/website/site/templates/shortcodes/figure.html new file mode 100644 index 00000000..14446372 --- /dev/null +++ b/website/site/templates/shortcodes/figure.html @@ -0,0 +1,5 @@ +
+ {% set len = page.components | length %} + {% set p = page.components | slice(end=len - 1) | join(sep='/') %} + +
diff --git a/website/site/templates/shortcodes/imgright.html b/website/site/templates/shortcodes/imgright.html new file mode 100644 index 00000000..cd028127 --- /dev/null +++ b/website/site/templates/shortcodes/imgright.html @@ -0,0 +1,3 @@ +{% set len = page.components | length %} +{% set p = page.components | slice(end=len - 1) | join(sep='/') %} + diff --git a/website/site/templates/shortcodes/incl_conf.html b/website/site/templates/shortcodes/incl_conf.html new file mode 100644 index 00000000..d1dda1f8 --- /dev/null +++ b/website/site/templates/shortcodes/incl_conf.html @@ -0,0 +1,4 @@ +``` conf +{% set data = load_data(path=path) %} +{{ data | safe }} +``` diff --git a/website/site/templates/shortcodes/pversion.html b/website/site/templates/shortcodes/pversion.html new file mode 100644 index 00000000..ae4a60a3 --- /dev/null +++ b/website/site/templates/shortcodes/pversion.html @@ -0,0 +1 @@ +{{ config.extra.version | replace(from='.', to='_') }} diff --git a/website/site/templates/shortcodes/version.html b/website/site/templates/shortcodes/version.html new file mode 100644 index 00000000..a01b2689 --- /dev/null +++ b/website/site/templates/shortcodes/version.html @@ -0,0 +1 @@ +{{ config.extra.version }} diff --git a/website/yarn.lock b/website/yarn.lock new file mode 100644 index 00000000..6444860a --- /dev/null +++ b/website/yarn.lock @@ -0,0 +1,8 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +bulma@^0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/bulma/-/bulma-0.9.0.tgz#948c5445a49e9d7546f0826cb3820d17178a814f" + integrity sha512-rV75CJkubNUroAt0qCRkjznZLoaXq/ctfMXsMvKSL84UetbSyx5REl96e8GoQ04G4Tkw0XF3STECffTOQrbzOQ==