docspell.joex {

  # This is the id of this node. If you run more than one server, you
  # have to make sure to provide unique ids per node.
  app-id = "joex1"


  # This is the base URL this application is deployed to. This is used
  # to register this joex instance such that docspell rest servers can
  # reach them
  base-url = "http://localhost:7878"

  # Where the REST server binds to.
  #
  # JOEX provides a very simple REST interface to inspect its state.
  bind {
    address = "localhost"
    port = 7878
  }

  # The database connection.
  #
  # By default a H2 file-based database is configured. You can provide
  # a postgresql or mariadb connection here. When using H2 use the
  # PostgreSQL compatibility mode and AUTO_SERVER feature.
  #
  # It must be the same connection as the rest server is using.
  jdbc {
    url = "jdbc:h2://"${java.io.tmpdir}"/docspell-demo.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_SERVER=TRUE"
    user = "sa"
    password = ""
  }

  # Enable or disable debugging for e-mail related functionality. This
  # applies to both sending and receiving mails. For security reasons
  # logging is not very extensive on authentication failures. Setting
  # this to true, results in a lot of data printed to stdout.
  mail-debug = false

  send-mail {
    # This is used as the List-Id e-mail header when mails are sent
    # from docspell to its users (example: for notification mails). It
    # is not used when sending to external recipients. If it is empty,
    # no such header is added. Using this header is often useful when
    # filtering mails.
    #
    # It should be a string in angle brackets. See
    # https://tools.ietf.org/html/rfc2919 for a formal specification
    # of this header.
    list-id = ""
  }

  # Configuration for the job scheduler.
  scheduler {

    # Each scheduler needs a unique name. This defaults to the node
    # name, which must be unique, too.
    name = ${docspell.joex.app-id}

    # Number of processing allowed in parallel.
    pool-size = 2

    # A counting scheme determines the ratio of how high- and low-prio
    # jobs are run. For example: 4,1 means run 4 high prio jobs, then
    # 1 low prio and then start over.
    counting-scheme = "4,1"

    # How often a failed job should be retried until it enters failed
    # state. If a job fails, it becomes "stuck" and will be retried
    # after a delay.
    retries = 2

    # The delay until the next try is performed for a failed job. This
    # delay is increased exponentially with the number of retries.
    retry-delay = "1 minute"

    # The queue size of log statements from a job.
    log-buffer-size = 500

    # If no job is left in the queue, the scheduler will wait until a
    # notify is requested (using the REST interface). To also retry
    # stuck jobs, it will notify itself periodically.
    wakeup-period = "30 minutes"
  }

  periodic-scheduler {

    # Each scheduler needs a unique name. This defaults to the node
    # name, which must be unique, too.
    name = ${docspell.joex.app-id}

    # A fallback to start looking for due periodic tasks regularily.
    # Usually joex instances should be notified via REST calls if
    # external processes change tasks. But these requests may get
    # lost.
    wakeup-period = "10 minutes"
  }

  # Configuration for the user-tasks.
  user-tasks {
    # Allows to import e-mails by scanning a mailbox.
    scan-mailbox {
      # A limit of how many folders to scan through. If a user
      # configures more than this, only upto this limit folders are
      # scanned and a warning is logged.
      max-folders = 50

      # How many mails (headers only) to retrieve in one chunk.
      #
      # If this is greater than `max-mails' it is set automatically to
      # the value of `max-mails'.
      mail-chunk-size = 50

      # A limit on how many mails to process in one job run. This is
      # meant to avoid too heavy resource allocation to one
      # user/collective.
      #
      # If more than this number of mails is encountered, a warning is
      # logged.
      max-mails = 500
    }
  }


  # Docspell uses periodic house keeping tasks, like cleaning expired
  # invites, that can be configured here.
  house-keeping {

    # When the house keeping tasks execute. Default is to run every
    # week.
    schedule = "Sun *-*-* 00:00:00"

    # This task removes invitation keys that have been created but not
    # used. The timespan here must be greater than the `invite-time'
    # setting in the rest server config file.
    cleanup-invites = {

      # Whether this task is enabled.
      enabled = true

      # The minimum age of invites to be deleted.
      older-than = "30 days"
    }

    # This task removes expired remember-me tokens. The timespan
    # should be greater than the `valid` time in the restserver
    # config.
    cleanup-remember-me = {
      # Whether the job is enabled.
      enabled = true

      # The minimum age of tokens to be deleted.
      older-than = "30 days"
    }

    # Jobs store their log output in the database. Normally this data
    # is only interesting for some period of time. The processing logs
    # of old files can be removed eventually.
    cleanup-jobs = {

      # Whether this task is enabled.
      enabled = true

      # The minimum age of jobs to delete. It is matched against the
      # `finished' timestamp.
      older-than = "30 days"

      # This defines how many jobs are deleted in one transaction.
      # Since the data to delete may get large, it can be configured
      # whether more or less memory should be used.
      delete-batch = "100"
    }
  }

  # Configuration of text extraction
  extraction {
    # For PDF files it is first tried to read the text parts of the
    # PDF. But PDFs can be complex documents and they may contain text
    # and images. If the returned text is shorter than the value
    # below, OCR is run afterwards. Then both extracted texts are
    # compared and the longer will be used.
    pdf {
      min-text-len = 500
    }

    preview {
      # When rendering a pdf page, use this dpi. This results in
      # scaling the image. A standard A4 page rendered at 96dpi
      # results in roughly 790x1100px image. Using 32 results in
      # roughly 200x300px image.
      #
      # Note, when this is changed, you might want to re-generate
      # preview images. Check the api for this, there is an endpoint
      # to regenerate all for a collective.
      dpi = 32
    }

    # Extracting text using OCR works for image and pdf files. It will
    # first run ghostscript to create a gray image from a pdf. Then
    # unpaper is run to optimize the image for the upcoming ocr, which
    # will be done by tesseract. All these programs must be available
    # in your PATH or the absolute path can be specified below.
    ocr {

      # Images greater than this size are skipped. Note that every
      # image is loaded completely into memory for doing OCR.
      max-image-size = 14000000

      # Defines what pages to process. If a PDF with 600 pages is
      # submitted, it is probably not necessary to scan through all of
      # them. This would take a long time and occupy resources for no
      # value. The first few pages should suffice. The default is first
      # 10 pages.
      #
      # If you want all pages being processed, set this number to -1.
      #
      # Note: if you change the ghostscript command below, be aware that
      # this setting (if not -1) will add another parameter to the
      # beginning of the command.
      page-range {
        begin = 10
      }

      # The ghostscript command.
      ghostscript {
        command {
          program = "gs"
          args = [ "-dNOPAUSE"
                 , "-dBATCH"
                 , "-dSAFER"
                 , "-sDEVICE=tiffscaled8"
                 , "-sOutputFile={{outfile}}"
                 , "{{infile}}"
                 ]
          timeout = "5 minutes"
        }
        working-dir = ${java.io.tmpdir}"/docspell-extraction"
      }

      # The unpaper command.
      unpaper {
        command {
          program = "unpaper"
          args = [ "{{infile}}", "{{outfile}}" ]
          timeout = "5 minutes"
        }
      }

      # The tesseract command.
      tesseract {
        command {
          program = "tesseract"
          args = ["{{file}}"
                 , "stdout"
                 , "-l"
                 , "{{lang}}"
                 ]
          timeout = "5 minutes"
        }
      }
    }
  }

  # Settings for text analysis
  text-analysis {
    # Maximum length of text to be analysed.
    #
    # All text to analyse must fit into RAM. A large document may take
    # too much heap. Also, most important information is at the
    # beginning of a document, so in most cases the first two pages
    # should suffice. Default is 10000, which are about 2-3 pages
    # (just a rough guess, of course).
    max-length = 10000

    # A working directory for the analyser to store temporary/working
    # files.
    working-dir = ${java.io.tmpdir}"/docspell-analysis"

    regex-ner {
      # Whether to enable custom NER annotation. This uses the address
      # book of a collective as input for NER tagging (to automatically
      # find correspondent and concerned entities). If the address book
      # is large, this can be quite memory intensive and also makes text
      # analysis slower. But it greatly improves accuracy. If this is
      # false, NER tagging uses only statistical models (that also work
      # quite well).
      #
      # This setting might be moved to the collective settings in the
      # future.
      enabled = true

      # The NER annotation uses a file of patterns that is derived from
      # a collective's address book. This is is the time how long this
      # file will be kept until a check for a state change is done.
      file-cache-time = "1 minute"
    }

    # Settings for doing document classification.
    #
    # This works by learning from existing documents. A collective can
    # specify a tag category and the system will try to predict a tag
    # from this category for new incoming documents.
    #
    # This requires a satstical model that is computed from all
    # existing documents. This process is run periodically as
    # configured by the collective. It may require a lot of memory,
    # depending on the amount of data.
    #
    # It utilises this NLP library: https://nlp.stanford.edu/.
    classification {
      # Whether to enable classification globally. Each collective can
      # decide to disable it. If it is disabled here, no collective
      # can use classification.
      enabled = true

      # If concerned with memory consumption, this restricts the
      # number of items to consider. More are better for training. A
      # negative value or zero means no train on all items.
      item-count = 0

      # These settings are used to configure the classifier. If
      # multiple are given, they are all tried and the "best" is
      # chosen at the end. See
      # https://nlp.stanford.edu/nlp/javadoc/javanlp/edu/stanford/nlp/classify/ColumnDataClassifier.html
      # for more info about these settings. The settings here yielded
      # good results with *my* dataset.
      #
      # Enclose regexps in triple quotes.
      classifiers = [
        { "useSplitWords" = "true"
          "splitWordsTokenizerRegexp" = """[\p{L}][\p{L}0-9]*|(?:\$ ?)?[0-9]+(?:\.[0-9]{2})?%?|\s+|."""
          "splitWordsIgnoreRegexp" = """\s+"""
          "useSplitPrefixSuffixNGrams" = "true"
          "maxNGramLeng" = "4"
          "minNGramLeng" = "1"
          "splitWordShape" = "chris4"
          "intern" = "true" # makes it slower but saves memory
        }
      ]
    }
  }

  # Configuration for converting files into PDFs.
  #
  # Most of it is delegated to external tools, which can be configured
  # below. They must be in the PATH environment or specify the full
  # path below via the `program` key.
  convert {

    # The chunk size used when storing files. This should be the same
    # as used with the rest server.
    chunk-size = 524288

    # A string used to change the filename of the converted pdf file.
    # If empty, the original file name is used for the pdf file ( the
    # extension is always replaced with `pdf`).
    converted-filename-part = "converted"

    # When reading images, this is the maximum size. Images that are
    # larger are not processed.
    max-image-size = ${docspell.joex.extraction.ocr.max-image-size}

    # Settings when processing markdown files (and other text files)
    # to HTML.
    #
    # In order to support text formats, text files are first converted
    # to HTML using a markdown processor. The resulting HTML is then
    # converted to a PDF file.
    markdown {

      # The CSS that is used to style the resulting HTML.
      internal-css = """
        body { padding: 2em 5em; }
      """
    }

    # To convert HTML files into PDF files, the external tool
    # wkhtmltopdf is used.
    wkhtmlpdf {
      command = {
        program = "wkhtmltopdf"
        args = [
          "-s",
          "A4",
          "--encoding",
          "{{encoding}}",
          "--load-error-handling", "ignore",
          "--load-media-error-handling", "ignore",
          "-",
          "{{outfile}}"
        ]
        timeout = "2 minutes"
      }
      working-dir = ${java.io.tmpdir}"/docspell-convert"
    }

    # To convert image files to PDF files, tesseract is used. This
    # also extracts the text in one go.
    tesseract = {
      command = {
        program = "tesseract"
        args = [
          "{{infile}}",
          "out",
          "-l",
          "{{lang}}",
          "pdf",
          "txt"
        ]
        timeout = "5 minutes"
      }
      working-dir = ${java.io.tmpdir}"/docspell-convert"
    }

    # To convert "office" files to PDF files, the external tool
    # unoconv is used. Unoconv uses libreoffice/openoffice for
    # converting. So it supports all formats that are possible to read
    # with libreoffice/openoffic.
    #
    # Note: to greatly improve performance, it is recommended to start
    # a libreoffice listener by running `unoconv -l` in a separate
    # process.
    unoconv = {
      command = {
        program = "unoconv"
        args = [
          "-f",
          "pdf",
          "-o",
          "{{outfile}}",
          "{{infile}}"
        ]
        timeout = "2 minutes"
      }
      working-dir = ${java.io.tmpdir}"/docspell-convert"
    }

    # The tool ocrmypdf can be used to convert pdf files to pdf files
    # in order to add extracted text as a separate layer. This makes
    # image-only pdfs searchable and you can select and copy/paste the
    # text. It also converts pdfs into pdf/a type pdfs, which are best
    # suited for archiving. So it makes sense to use this even for
    # text-only pdfs.
    #
    # It is recommended to install ocrympdf, but it also is optional.
    # If it is enabled but fails, the error is not fatal and the
    # processing will continue using the original pdf for extracting
    # text. You can also disable it to remove the errors from the
    # processing logs.
    #
    # The `--skip-text` option is necessary to not fail on "text" pdfs
    # (where ocr is not necessary). In this case, the pdf will be
    # converted to PDF/A.
    ocrmypdf = {
      enabled = true
      command = {
        program = "ocrmypdf"
        args = [
          "-l", "{{lang}}",
          "--skip-text",
          "--deskew",
          "-j", "1",
          "{{infile}}",
          "{{outfile}}"
        ]
        timeout = "5 minutes"
      }
      working-dir = ${java.io.tmpdir}"/docspell-convert"
    }
  }

  # General config for processing documents
  processing {
    # Restricts proposals for due dates. Only dates earlier than this
    # number of years in the future are considered.
    max-due-date-years = 10
  }

  # The same section is also present in the rest-server config. It is
  # used when submitting files into the job queue for processing.
  #
  # Currently, these settings may affect memory usage of all nodes, so
  # it should be the same on all nodes.
  files {
    # Defines the chunk size (in bytes) used to store the files.
    # This will affect the memory footprint when uploading and
    # downloading files. At most this amount is loaded into RAM for
    # down- and uploading.
    #
    # It also defines the chunk size used for the blobs inside the
    # database.
    chunk-size = 524288

    # The file content types that are considered valid. Docspell
    # will only pass these files to processing. The processing code
    # itself has also checks for which files are supported and which
    # not. This affects the uploading part and can be used to
    # restrict file types that should be handed over to processing.
    # By default all files are allowed.
    valid-mime-types = [ ]
  }

  # Configuration of the full-text search engine.
  full-text-search {
    # The full-text search feature can be disabled. It requires an
    # additional index server which needs additional memory and disk
    # space. It can be enabled later any time.
    #
    # Currently the SOLR search platform is supported.
    enabled = false

    # Configuration for the SOLR backend.
    solr = {
      # The URL to solr
      url = "http://localhost:8983/solr/docspell"
      # Used to tell solr when to commit the data
      commit-within = 1000
      # If true, logs request and response bodies
      log-verbose = false
      # The defType parameter to lucene that defines the parser to
      # use. You might want to try "edismax" or look here:
      # https://lucene.apache.org/solr/guide/8_4/query-syntax-and-parsing.html#query-syntax-and-parsing
      def-type = "lucene"
      # The default combiner for tokens. One of {AND, OR}.
      q-op = "OR"
    }

    # Settings for running the index migration tasks
    migration = {
      # Chunk size to use when indexing data from the database. This
      # many attachments are loaded into memory and pushed to the
      # full-text index.
      index-all-chunk = 10
    }
  }
}