Integrate support for more files into processing and upload

The restriction that only pdf files can be uploaded is removed. All
files can now be uploaded. The processing may not process all. It is
still possible to restrict file uploads by types via a configuration.

modules/joex/src/main/resources/reference.conf

@@ -65,66 +65,143 @@ docspell.joex {
   }
 
   # Configuration of text extraction
-  #
-  # Extracting text currently only work for image and pdf files. It
-  # will first run ghostscript to create a gray image from a pdf. Then
-  # unpaper is run to optimize the image for the upcoming ocr, which
-  # will be done by tesseract. All these programs must be available in
-  # your PATH or the absolute path can be specified below.
   extraction {
-    allowed-content-types = [ "application/pdf", "image/jpeg", "image/png" ]
-
-    # Defines what pages to process. If a PDF with 600 pages is
-    # submitted, it is probably not necessary to scan through all of
-    # them. This would take a long time and occupy resources for no
-    # value. The first few pages should suffice. The default is first
-    # 10 pages.
-    #
-    # If you want all pages being processed, set this number to -1.
-    #
-    # Note: if you change the ghostscript command below, be aware that
-    # this setting (if not -1) will add another parameter to the
-    # beginning of the command.
-    page-range {
-      begin = 10
+    # For PDF files it is first tried to read the text parts of the
+    # PDF. But PDFs can be complex documents and they may contain text
+    # and images. If the returned text is shorter than the value
+    # below, OCR is run afterwards. Then both extracted texts are
+    # compared and the longer will be used.
+    pdf {
+      min-text-len = 10
     }
 
-    # The ghostscript command.
-    ghostscript {
-      command {
-        program = "gs"
-        args = [ "-dNOPAUSE"
-               , "-dBATCH"
-               , "-dSAFER"
-               , "-sDEVICE=tiffscaled8"
-               , "-sOutputFile={{outfile}}"
-               , "{{infile}}"
-               ]
-        timeout = "5 minutes"
+    # Extracting text using OCR works for image and pdf files. It will
+    # first run ghostscript to create a gray image from a pdf. Then
+    # unpaper is run to optimize the image for the upcoming ocr, which
+    # will be done by tesseract. All these programs must be available
+    # in your PATH or the absolute path can be specified below.
+    ocr {
+
+      # Images greater than this size are skipped. Note that every
+      # image is loaded completely into memory for doing OCR.
+      max-image-size = 14000000
+
+      # Defines what pages to process. If a PDF with 600 pages is
+      # submitted, it is probably not necessary to scan through all of
+      # them. This would take a long time and occupy resources for no
+      # value. The first few pages should suffice. The default is first
+      # 10 pages.
+      #
+      # If you want all pages being processed, set this number to -1.
+      #
+      # Note: if you change the ghostscript command below, be aware that
+      # this setting (if not -1) will add another parameter to the
+      # beginning of the command.
+      page-range {
+        begin = 10
       }
-      working-dir = ${java.io.tmpdir}"/docspell-extraction"
-    }
 
-    # The unpaper command.
-    unpaper {
-      command {
-        program = "unpaper"
-        args = [ "{{infile}}", "{{outfile}}" ]
-        timeout = "5 minutes"
+      # The ghostscript command.
+      ghostscript {
+        command {
+          program = "gs"
+          args = [ "-dNOPAUSE"
+                 , "-dBATCH"
+                 , "-dSAFER"
+                 , "-sDEVICE=tiffscaled8"
+                 , "-sOutputFile={{outfile}}"
+                 , "{{infile}}"
+                 ]
+          timeout = "5 minutes"
+        }
+        working-dir = ${java.io.tmpdir}"/docspell-extraction"
       }
-    }
 
-    # The tesseract command.
-    tesseract {
-      command {
-        program = "tesseract"
-        args = ["{{file}}"
-               , "stdout"
-               , "-l"
-               , "{{lang}}"
-               ]
-        timeout = "5 minutes"
+      # The unpaper command.
+      unpaper {
+        command {
+          program = "unpaper"
+          args = [ "{{infile}}", "{{outfile}}" ]
+          timeout = "5 minutes"
+        }
+      }
+
+      # The tesseract command.
+      tesseract {
+        command {
+          program = "tesseract"
+          args = ["{{file}}"
+                 , "stdout"
+                 , "-l"
+                 , "{{lang}}"
+                 ]
+          timeout = "5 minutes"
+        }
       }
     }
   }
+
+  # Configuration for converting files into PDFs.
+  #
+  # Most of it is delegated to external tools, which can be configured
+  # below. They must be in the PATH environment or specify the full
+  # path below via the `program` key.
+  convert {
+    chunk-size = 524288
+
+    max-image-size = 12000000
+
+    markdown {
+      internal-css = """
+        body { padding: 2em 5em; }
+      """
+    }
+
+    wkhtmlpdf {
+      cmd = {
+        program = "wkhtmltopdf"
+        args = [
+          "-s",
+          "A4",
+          "--encoding",
+          "UTF-8",
+          "-",
+          "{{outfile}}"
+        ]
+        timeout = "20 seconds"
+      }
+      working-dir = ${java.io.tmpdir}"/docspell-convert"
+    }
+
+    tesseract = {
+      cmd = {
+        program = "tesseract"
+        args = [
+          "{{infile}}",
+          "out",
+          "-l",
+          "{{lang}}",
+          "pdf",
+          "txt"
+        ]
+        timeout = "120 seconds"
+      }
+      working-dir = ${java.io.tmpdir}"/docspell-convert"
+    }
+
+    unoconv = {
+      cmd = {
+        program = "unoconv"
+        args = [
+          "-f",
+          "pdf",
+          "-o",
+          "{{outfile}}",
+          "{{infile}}"
+        ]
+        timeout = "20 seconds"
+      }
+      working-dir = ${java.io.tmpdir}"/docspell-convert"
+    }
+  }
 }