From 532caed84c41ceae0743e5b4246547ff733ec5ff Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Wed, 24 Jun 2020 21:25:46 +0200 Subject: [PATCH] Consistent logging of request/responses to solr Using a middleware. Also add missing changesets for mariadb. --- .../scala/docspell/ftssolr/SolrConfig.scala | 2 +- .../docspell/ftssolr/SolrFtsClient.scala | 23 +++++++++++++++---- .../scala/docspell/ftssolr/SolrQuery.scala | 4 ---- .../scala/docspell/ftssolr/SolrSetup.scala | 5 +--- .../scala/docspell/ftssolr/SolrUpdate.scala | 9 +++----- .../joex/src/main/resources/reference.conf | 8 +++++-- .../src/main/resources/reference.conf | 9 ++++++-- .../mariadb/V1.7.0__fts-migration.sql | 10 ++++++++ modules/webapp/src/main/webjar/docspell.css | 6 ++--- 9 files changed, 50 insertions(+), 26 deletions(-) create mode 100644 modules/store/src/main/resources/db/migration/mariadb/V1.7.0__fts-migration.sql diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrConfig.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrConfig.scala index 89ecb1ea..f2a04d64 100644 --- a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrConfig.scala +++ b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrConfig.scala @@ -2,6 +2,6 @@ package docspell.ftssolr import docspell.common._ -final case class SolrConfig(url: LenientUri, commitWithin: Int) +final case class SolrConfig(url: LenientUri, commitWithin: Int, logVerbose: Boolean) object SolrConfig {} diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala index cd269bef..4c43c18b 100644 --- a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala +++ b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrFtsClient.scala @@ -4,6 +4,8 @@ import fs2.Stream import cats.effect._ import cats.implicits._ import org.http4s.client.Client +import org.http4s.client.middleware.Logger +import org.log4s.getLogger import docspell.common._ import docspell.ftsclient._ @@ -50,17 +52,30 @@ final class SolrFtsClient[F[_]: Effect]( } object SolrFtsClient { + private[this] val logger = getLogger def apply[F[_]: ConcurrentEffect]( cfg: SolrConfig, httpClient: Client[F] - ): Resource[F, FtsClient[F]] = + ): Resource[F, FtsClient[F]] = { + val client = loggingMiddleware(cfg, httpClient) Resource.pure[F, FtsClient[F]]( new SolrFtsClient( - SolrUpdate(cfg, httpClient), - SolrSetup(cfg, httpClient), - SolrQuery(cfg, httpClient) + SolrUpdate(cfg, client), + SolrSetup(cfg, client), + SolrQuery(cfg, client) ) ) + } + + private def loggingMiddleware[F[_]: Concurrent]( + cfg: SolrConfig, + client: Client[F] + ): Client[F] = + Logger( + logHeaders = true, + logBody = cfg.logVerbose, + logAction = Some((msg: String) => Sync[F].delay(logger.trace(msg))) + )(client) } diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrQuery.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrQuery.scala index 21a84360..69a893d7 100644 --- a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrQuery.scala +++ b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrQuery.scala @@ -7,7 +7,6 @@ import org.http4s.circe._ import org.http4s.circe.CirceEntityDecoder._ import org.http4s.client.dsl.Http4sClientDsl import _root_.io.circe.syntax._ -import org.log4s.getLogger import docspell.ftsclient._ import JsonCodec._ @@ -42,8 +41,6 @@ trait SolrQuery[F[_]] { } object SolrQuery { - private[this] val logger = getLogger - def apply[F[_]: ConcurrentEffect](cfg: SolrConfig, client: Client[F]): SolrQuery[F] = { val dsl = new Http4sClientDsl[F] {} import dsl._ @@ -53,7 +50,6 @@ object SolrQuery { def query(q: QueryData): F[FtsResult] = { val req = Method.POST(q.asJson, url) - logger.trace(s"Running query: $req : ${q.asJson}") client.expect[FtsResult](req) } diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala index d07cacf0..c25431a8 100644 --- a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala +++ b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrSetup.scala @@ -6,7 +6,6 @@ import cats.implicits._ import org.http4s.client.Client import org.http4s.circe._ import org.http4s.client.dsl.Http4sClientDsl -import org.log4s.getLogger import _root_.io.circe.syntax._ import _root_.io.circe._ import _root_.io.circe.generic.semiauto._ @@ -19,7 +18,6 @@ trait SolrSetup[F[_]] { } object SolrSetup { - private[this] val logger = getLogger def apply[F[_]: ConcurrentEffect](cfg: SolrConfig, client: Client[F]): SolrSetup[F] = { val dsl = new Http4sClientDsl[F] {} @@ -59,8 +57,7 @@ object SolrSetup { private def run(cmd: Json): F[Unit] = { val req = Method.POST(cmd, url) - logger.debug(s"Running request $req: ${cmd.noSpaces}") - client.expect[String](req).map(r => logger.debug(s"Response: $r")) + client.expect[Unit](req) } private def addStringField(field: Field): F[Unit] = diff --git a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrUpdate.scala b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrUpdate.scala index ce4a8ed9..7740bf00 100644 --- a/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrUpdate.scala +++ b/modules/fts-solr/src/main/scala/docspell/ftssolr/SolrUpdate.scala @@ -2,13 +2,11 @@ package docspell.ftssolr import cats.effect._ import org.http4s._ -import cats.implicits._ import org.http4s.client.Client import org.http4s.circe._ import org.http4s.client.dsl.Http4sClientDsl import _root_.io.circe._ import _root_.io.circe.syntax._ -import org.log4s.getLogger import docspell.ftsclient._ import JsonCodec._ @@ -23,7 +21,6 @@ trait SolrUpdate[F[_]] { } object SolrUpdate { - private[this] val logger = getLogger def apply[F[_]: ConcurrentEffect](cfg: SolrConfig, client: Client[F]): SolrUpdate[F] = { val dsl = new Http4sClientDsl[F] {} @@ -37,17 +34,17 @@ object SolrUpdate { def add(tds: List[TextData]): F[Unit] = { val req = Method.POST(tds.asJson, url) - client.expect[String](req).map(r => logger.trace(s"Req: $req Response: $r")) + client.expect[Unit](req) } def update(tds: List[TextData]): F[Unit] = { val req = Method.POST(tds.filter(minOneChange).map(SetFields).asJson, url) - client.expect[String](req).map(r => logger.trace(s"Req: $req Response: $r")) + client.expect[Unit](req) } def delete(q: String): F[Unit] = { val req = Method.POST(Delete(q).asJson, url) - client.expect[String](req).map(r => logger.trace(s"Req: $req Response: $r")) + client.expect[Unit](req) } private val minOneChange: TextData => Boolean = diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf index 214f08f5..524f5735 100644 --- a/modules/joex/src/main/resources/reference.conf +++ b/modules/joex/src/main/resources/reference.conf @@ -368,16 +368,20 @@ docspell.joex { # Configuration of the full-text search engine. full-text-search { # The full-text search feature can be disabled. It requires an - # additional index server available which needs additional - # memory and disk space. It can be enabled later any time. + # additional index server which needs additional memory and disk + # space. It can be enabled later any time. # # Currently the SOLR search platform is supported. enabled = false # Configuration for the SOLR backend. solr = { + # The URL to solr url = "http://localhost:8983/solr/docspell" + # Used to tell solr when to commit the data commit-within = 1000 + # If true, logs request and response bodies + log-verbose = false } # Settings for running the index migration tasks diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index 6edca43c..23954f8a 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -87,8 +87,8 @@ docspell.server { # Configuration of the full-text search engine. full-text-search { # The full-text search feature can be disabled. It requires an - # additional index server available which needs additional - # memory and disk space. It can be enabled later any time. + # additional index server which needs additional memory and disk + # space. It can be enabled later any time. # # Currently the SOLR search platform is supported. enabled = false @@ -103,9 +103,14 @@ docspell.server { # Configuration for the SOLR backend. solr = { + # The URL to solr url = "http://localhost:8983/solr/docspell" + # Used to tell solr when to commit the data commit-within = 1000 + # If true, logs request and response bodies + log-verbose = false } + } # Configuration for the backend. diff --git a/modules/store/src/main/resources/db/migration/mariadb/V1.7.0__fts-migration.sql b/modules/store/src/main/resources/db/migration/mariadb/V1.7.0__fts-migration.sql new file mode 100644 index 00000000..fce07b37 --- /dev/null +++ b/modules/store/src/main/resources/db/migration/mariadb/V1.7.0__fts-migration.sql @@ -0,0 +1,10 @@ +CREATE TABLE `fts_migration` ( + `id` varchar(254) not null primary key, + `version` int not null, + `fts_engine` varchar(254) not null, + `description` varchar(254) not null, + `created` timestamp not null +); + +CREATE UNIQUE INDEX `fts_migration_version_engine_idx` +ON `fts_migration`(`version`, `fts_engine`); diff --git a/modules/webapp/src/main/webjar/docspell.css b/modules/webapp/src/main/webjar/docspell.css index 5babbe01..c09ba8cb 100644 --- a/modules/webapp/src/main/webjar/docspell.css +++ b/modules/webapp/src/main/webjar/docspell.css @@ -89,7 +89,7 @@ } .default-layout .ui.cards .ui.card .content.search-highlight { - background: rgba(246, 255, 158, 0.15); + background: rgba(246, 255, 158, 0.1); font-size: smaller; max-height: 25em; overflow: auto; @@ -97,11 +97,11 @@ .default-layout .ui.cards .ui.card .content.search-highlight .ui.list .item .content .header { } .default-layout .ui.cards .ui.card .content.search-highlight .ui.list .item .content .description { - color: rgba(0,0,0,.55); + color: rgba(0,0,0,.6); margin-left: 0.75em; } .default-layout .ui.cards .ui.card .content.search-highlight .ui.list .item .content .description strong > em { - color: rgba(0,0,0,.65); + background: rgba(220, 255, 71, 0.6); } .markdown-preview {