diff --git a/.gitignore b/.gitignore index 773ab69e..1e4b858d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ target/ dev.conf -elm-stuff \ No newline at end of file +elm-stuff +result +_site/ \ No newline at end of file diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000..94a9ed02 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md index e0fbad6f..f181c940 100644 --- a/README.md +++ b/README.md @@ -1 +1,11 @@ + + # Docspell + + +Docspell is a personal document organizer. You'll need a scanner to +convert your papers into PDF files. Docspell can then assist in +organizing the resulting mess :wink:. + +See the [microsite](https://eikek.github.io/docspell/) for more +information. diff --git a/artwork/icon.png b/artwork/icon.png new file mode 100644 index 00000000..f41a3178 Binary files /dev/null and b/artwork/icon.png differ diff --git a/build.sbt b/build.sbt index 603e38c9..94732190 100644 --- a/build.sbt +++ b/build.sbt @@ -1,39 +1,43 @@ import com.github.eikek.sbt.openapi._ import scala.sys.process._ import com.typesafe.sbt.SbtGit.GitKeys._ +import docspell.build._ val sharedSettings = Seq( organization := "com.github.eikek", - scalaVersion := "2.13.0", + scalaVersion := "2.13.1", scalacOptions ++= Seq( "-deprecation", "-encoding", "UTF-8", "-language:higherKinds", "-language:postfixOps", "-feature", - "-Xfatal-warnings", // fail when there are warnings + "-Werror", // fail when there are warnings "-unchecked", - "-Xlint", + "-Xlint:_", "-Ywarn-dead-code", "-Ywarn-numeric-widen", "-Ywarn-value-discard" ), - scalacOptions in (Compile, console) := Seq() + scalacOptions in (Compile, console) := + (scalacOptions.value.filter(o => !o.contains("Xlint")) ++ Seq("-Xlint:_,-unused")), + scalacOptions in (Test, console) := + (scalacOptions.value.filter(o => !o.contains("Xlint")) ++ Seq("-Xlint:_,-unused")) ) val testSettings = Seq( testFrameworks += new TestFramework("minitest.runner.Framework"), - libraryDependencies ++= Dependencies.miniTest + libraryDependencies ++= Dependencies.miniTest ++ Dependencies.logging.map(_ % Test) ) val elmSettings = Seq( - Compile/resourceGenerators += (Def.task { + Compile/resourceGenerators += Def.task { compileElm(streams.value.log , (Compile/baseDirectory).value , (Compile/resourceManaged).value , name.value , version.value) - }).taskValue, + }.taskValue, watchSources += Watched.WatchSource( (Compile/sourceDirectory).value/"elm" , FileFilter.globFilter("*.elm") @@ -42,14 +46,14 @@ val elmSettings = Seq( ) val webjarSettings = Seq( - Compile/resourceGenerators += (Def.task { + Compile/resourceGenerators += Def.task { copyWebjarResources(Seq((sourceDirectory in Compile).value/"webjar") , (Compile/resourceManaged).value , name.value , version.value , streams.value.log ) - }).taskValue, + }.taskValue, watchSources += Watched.WatchSource( (Compile / sourceDirectory).value/"webjar" , FileFilter.globFilter("*.js") || FileFilter.globFilter("*.css") @@ -57,7 +61,7 @@ val webjarSettings = Seq( ) ) -val debianSettings = Seq( +def debianSettings(cfgFile: String) = Seq( maintainer := "Eike Kettner ", packageSummary := description.value, packageDescription := description.value, @@ -66,9 +70,9 @@ val debianSettings = Seq( if (!conf.exists) { sys.error(s"File $conf not found") } - conf -> "conf/docspell.conf" + conf -> s"conf/$cfgFile.conf" }, - bashScriptExtraDefines += """addJava "-Dconfig.file=${app_home}/../conf/docspell.conf"""" + bashScriptExtraDefines += s"""addJava "-Dconfig.file=$${app_home}/../conf/$cfgFile.conf"""" ) val buildInfoSettings = Seq( @@ -77,7 +81,47 @@ val buildInfoSettings = Seq( buildInfoOptions += BuildInfoOption.BuildTime ) +val openapiScalaSettings = Seq( + openapiScalaConfig := ScalaConfig().withJson(ScalaJson.circeSemiauto). + addMapping(CustomMapping.forType({ + case TypeDef("LocalDateTime", _) => + TypeDef("Timestamp", Imports("docspell.common.Timestamp")) + })). + addMapping(CustomMapping.forFormatType({ + case "ident" => field => + field.copy(typeDef = TypeDef("Ident", Imports("docspell.common.Ident"))) + case "collectivestate" => field => + field.copy(typeDef = TypeDef("CollectiveState", Imports("docspell.common.CollectiveState"))) + case "userstate" => field => + field.copy(typeDef = TypeDef("UserState", Imports("docspell.common.UserState"))) + case "password" => field => + field.copy(typeDef = TypeDef("Password", Imports("docspell.common.Password"))) + case "contactkind" => field => + field.copy(typeDef = TypeDef("ContactKind", Imports("docspell.common.ContactKind"))) + case "direction" => field => + field.copy(typeDef = TypeDef("Direction", Imports("docspell.common.Direction"))) + case "priority" => field => + field.copy(typeDef = TypeDef("Priority", Imports("docspell.common.Priority"))) + case "jobstate" => field => + field.copy(typeDef = TypeDef("JobState", Imports("docspell.common.JobState"))) + case "loglevel" => field => + field.copy(typeDef = TypeDef("LogLevel", Imports("docspell.common.LogLevel"))) + case "mimetype" => field => + field.copy(typeDef = TypeDef("MimeType", Imports("docspell.common.MimeType"))) + case "itemstate" => field => + field.copy(typeDef = TypeDef("ItemState", Imports("docspell.common.ItemState"))) + case "nertag" => field => + field.copy(typeDef = TypeDef("NerTag", Imports("docspell.common.NerTag"))) + case "language" => field => + field.copy(typeDef = TypeDef("Language", Imports("docspell.common.Language"))) + })) +) +val reStartSettings = Seq( + javaOptions in reStart ++= Seq(s"-Dconfig.file=${(LocalRootProject/baseDirectory).value/"dev.conf"}") +) + +// --- Modules val common = project.in(file("modules/common")). settings(sharedSettings). @@ -85,7 +129,10 @@ val common = project.in(file("modules/common")). settings( name := "docspell-common", libraryDependencies ++= - Dependencies.fs2 + Dependencies.fs2 ++ + Dependencies.circe ++ + Dependencies.loggingApi ++ + Dependencies.pureconfig.map(_ % "optional") ) val store = project.in(file("modules/store")). @@ -96,16 +143,31 @@ val store = project.in(file("modules/store")). libraryDependencies ++= Dependencies.doobie ++ Dependencies.bitpeace ++ + Dependencies.tika ++ Dependencies.fs2 ++ Dependencies.databases ++ Dependencies.flyway ++ Dependencies.loggingApi - ) + ).dependsOn(common) +val text = project.in(file("modules/text")). + enablePlugins(NerModelsPlugin). + settings(sharedSettings). + settings(testSettings). + settings(NerModelsPlugin.nerClassifierSettings). + settings( + name := "docspell-text", + libraryDependencies ++= + Dependencies.fs2 ++ + Dependencies.tika ++ + Dependencies.stanfordNlpCore + ).dependsOn(common) + val restapi = project.in(file("modules/restapi")). enablePlugins(OpenApiSchema). settings(sharedSettings). settings(testSettings). + settings(openapiScalaSettings). settings( name := "docspell-restapi", libraryDependencies ++= @@ -113,22 +175,21 @@ val restapi = project.in(file("modules/restapi")). openapiTargetLanguage := Language.Scala, openapiPackage := Pkg("docspell.restapi.model"), openapiSpec := (Compile/resourceDirectory).value/"docspell-openapi.yml", - openapiScalaConfig := ScalaConfig().withJson(ScalaJson.circeSemiauto) - ) + ).dependsOn(common) val joexapi = project.in(file("modules/joexapi")). enablePlugins(OpenApiSchema). settings(sharedSettings). settings(testSettings). + settings(openapiScalaSettings). settings( name := "docspell-joexapi", libraryDependencies ++= Dependencies.circe, openapiTargetLanguage := Language.Scala, openapiPackage := Pkg("docspell.joexapi.model"), - openapiSpec := (Compile/resourceDirectory).value/"joex-openapi.yml", - openapiScalaConfig := ScalaConfig().withJson(ScalaJson.circeSemiauto) - ) + openapiSpec := (Compile/resourceDirectory).value/"joex-openapi.yml" + ).dependsOn(common) val joex = project.in(file("modules/joex")). enablePlugins(BuildInfoPlugin @@ -137,7 +198,7 @@ val joex = project.in(file("modules/joex")). , SystemdPlugin). settings(sharedSettings). settings(testSettings). - settings(debianSettings). + settings(debianSettings("docspell-joex")). settings(buildInfoSettings). settings( name := "docspell-joex", @@ -147,11 +208,12 @@ val joex = project.in(file("modules/joex")). Dependencies.circe ++ Dependencies.pureconfig ++ Dependencies.loggingApi ++ - Dependencies.logging, + Dependencies.logging.map(_ % Runtime), addCompilerPlugin(Dependencies.kindProjectorPlugin), addCompilerPlugin(Dependencies.betterMonadicFor), - buildInfoPackage := "docspell.joex" - ).dependsOn(store, joexapi, restapi) + buildInfoPackage := "docspell.joex", + reStart/javaOptions ++= Seq(s"-Dconfig.file=${(LocalRootProject/baseDirectory).value/"dev.conf"}") + ).dependsOn(store, text, joexapi, restapi) val backend = project.in(file("modules/backend")). settings(sharedSettings). @@ -160,7 +222,9 @@ val backend = project.in(file("modules/backend")). name := "docspell-backend", libraryDependencies ++= Dependencies.loggingApi ++ - Dependencies.fs2 + Dependencies.fs2 ++ + Dependencies.bcrypt ++ + Dependencies.http4sClient ).dependsOn(store) val webapp = project.in(file("modules/webapp")). @@ -183,7 +247,7 @@ val restserver = project.in(file("modules/restserver")). , SystemdPlugin). settings(sharedSettings). settings(testSettings). - settings(debianSettings). + settings(debianSettings("docspell-server")). settings(buildInfoSettings). settings( name := "docspell-restserver", @@ -194,39 +258,122 @@ val restserver = project.in(file("modules/restserver")). Dependencies.yamusca ++ Dependencies.webjars ++ Dependencies.loggingApi ++ - Dependencies.logging, + Dependencies.logging.map(_ % Runtime), addCompilerPlugin(Dependencies.kindProjectorPlugin), addCompilerPlugin(Dependencies.betterMonadicFor), buildInfoPackage := "docspell.restserver", - Compile/sourceGenerators += (Def.task { + Compile/sourceGenerators += Def.task { createWebjarSource(Dependencies.webjars, (Compile/sourceManaged).value) - }).taskValue, - Compile/unmanagedResourceDirectories ++= Seq((Compile/resourceDirectory).value.getParentFile/"templates") + }.taskValue, + Compile/resourceGenerators += Def.task { + copyWebjarResources(Seq((restapi/Compile/resourceDirectory).value/"docspell-openapi.yml") + , (Compile/resourceManaged).value + , name.value + , version.value + , streams.value.log) + }.taskValue, + Compile/unmanagedResourceDirectories ++= Seq((Compile/resourceDirectory).value.getParentFile/"templates"), + reStart/javaOptions ++= Seq(s"-Dconfig.file=${(LocalRootProject/baseDirectory).value/"dev.conf"}") ).dependsOn(restapi, joexapi, backend, webapp) +val microsite = project.in(file("modules/microsite")). + enablePlugins(MicrositesPlugin). + settings(sharedSettings). + settings( + name := "docspell-microsite", + publishArtifact := false, + scalacOptions -= "-Yno-imports", + scalacOptions ~= { _ filterNot (_ startsWith "-Ywarn") }, + scalacOptions ~= { _ filterNot (_ startsWith "-Xlint") }, + scalaVersion := "2.12.9", + skip in publish := true, + micrositeFooterText := Some( + """ + |

© 2019 Docspell, v{{site.version}}

+ |""".stripMargin + ), + micrositeName := "Docspell", + micrositeDescription := "Docspell – A Document Organizer", + micrositeBaseUrl := "/docspell", + micrositeAuthor := "eikek", + micrositeGithubOwner := "eikek", + micrositeGithubRepo := "docspell", + micrositeGitterChannel := false, + micrositeFavicons := Seq(microsites.MicrositeFavicon("favicon.png", "96x96")), + micrositeShareOnSocial := false, + micrositeHighlightLanguages ++= Seq("json", "javascript"), + micrositePalette := Map( + "brand-primary" -> "#5d000a", // link color + "brand-secondary" -> "#172651", //sidebar background + "brand-tertiary" -> "#495680", //main brand background + "gray-dark" -> "#050913", //header font color + "gray" -> "#131f43", //font color + "gray-light" -> "#E3E2E3", + "gray-lighter" -> "#f8fbff", //body background + "white-color" -> "#FFFFFF"), + fork in tut := true, + scalacOptions in Tut ~= (_.filterNot(Set("-Ywarn-unused-import", "-Ywarn-dead-code", "-Werror"))), + resourceGenerators in Tut += Def.task { + val conf1 = (resourceDirectory in (restserver, Compile)).value / "reference.conf" + val conf2 = (resourceDirectory in (joex, Compile)).value / "reference.conf" + val out1 = resourceManaged.value/"main"/"jekyll"/"_includes"/"server.conf" + val out2 = resourceManaged.value/"main"/"jekyll"/"_includes"/"joex.conf" + streams.value.log.info(s"Copying reference.conf: $conf1 -> $out1, $conf2 -> $out2") + IO.write(out1, "{% raw %}\n") + IO.append(out1, IO.readBytes(conf1)) + IO.write(out1, "\n{% endraw %}", append = true) + IO.write(out2, "{% raw %}\n") + IO.append(out2, IO.readBytes(conf2)) + IO.write(out2, "\n{% endraw %}", append = true) + val oa1 = (resourceDirectory in (restapi, Compile)).value/"docspell-openapi.yml" + val oaout = resourceManaged.value/"main"/"jekyll"/"openapi"/"docspell-openapi.yml" + IO.copy(Seq(oa1 -> oaout)) + Seq(out1, out2, oaout) + }.taskValue, + resourceGenerators in Tut += Def.task { + val staticDoc = (restapi/Compile/openapiStaticDoc).value + val target = resourceManaged.value/"main"/"jekyll"/"openapi"/"docspell-openapi.html" + IO.copy(Seq(staticDoc -> target)) + Seq(target) + }.taskValue, + micrositeCompilingDocsTool := WithTut //WithMdoc +// mdocIn := sourceDirectory.value / "main" / "tut" + ) + val root = project.in(file(".")). settings(sharedSettings). settings( name := "docspell-root" ). - aggregate(common, store, joexapi, joex, backend, webapp, restapi, restserver) + aggregate(common + , text + , store + , joexapi + , joex + , backend + , webapp + , restapi + , restserver + , microsite) + +// --- helpers + def copyWebjarResources(src: Seq[File], base: File, artifact: String, version: String, logger: Logger): Seq[File] = { val targetDir = base/"META-INF"/"resources"/"webjars"/artifact/version + logger.info(s"Copy webjar resources from ${src.size} files/directories.") src.flatMap { dir => if (dir.isDirectory) { val files = (dir ** "*").filter(_.isFile).get pair Path.relativeTo(dir) files.map { case (f, name) => val target = targetDir/name - logger.info(s"Copy $f -> $target") IO.createDirectories(Seq(target.getParentFile)) IO.copy(Seq(f -> target)) target } } else { val target = targetDir/dir.name - logger.info(s"Copy $dir -> $target") IO.createDirectories(Seq(target.getParentFile)) IO.copy(Seq(dir -> target)) Seq(target) @@ -255,3 +402,9 @@ def createWebjarSource(wj: Seq[ModuleID], out: File): Seq[File] = { IO.write(target, content) Seq(target) } + + +addCommandAlias("make", ";root/openapiCodegen ;root/test:compile") +addCommandAlias("make-zip", ";restserver/universal:packageBin ;joex/universal:packageBin") +addCommandAlias("make-deb", ";restserver/debian:packageBin ;joex/debian:packageBin") +addCommandAlias("make-pkg", ";make-zip ;make-deb") diff --git a/doc/dev.md b/doc/dev.md deleted file mode 100644 index 595797b8..00000000 --- a/doc/dev.md +++ /dev/null @@ -1,110 +0,0 @@ -# Development Documentation - - -## initial thoughts - -* First there is a web app, where user can login, look at their - documents etc -* User can do queries and edit document meta data -* User can manage upload endpoints - -Upload endpoints allow to receive "items". There are the following -different options: - -1. Upload a single item by uploading one file. -2. Upload a single item by uploading a zip file. -3. Upload multiple items by uploading a zip file (one entry = one - item) - -Files are received and stored in the database, always. Only if a size -constraint is not fulfilled the response is an error. Files are marked -as `RECEIVED`. Idea is that most files are valid, so are saved -anyways. - -Then a job for a new item is inserted into the processing queue and -processing begins eventually. - -External processes access the queue on the same database and take jobs -for processing. - -Processing: - -1. check mimetype and error if not supported - - want to use the servers mimetype instead of advertised one from - the client -2. extract text and other meta data -3. do some analysis -4. tag item/set meta data -5. encrypt files + text, if configured - -If an error occurs, it can be inspected in the "queue screen". The web -app shows notifications in this case. User can download the file and -remove it. Otherwise, files will be deleted after some period. Errors -are also counted per source, so one can decide whether to block a -source. - -Once processing is done, the item is put in the INBOX. - -## Modules - -### processor - -### backend - -### store - -### backend server - -### webapp - -## Flow - - -1. webapp: calls rest route -2. server: - 1. convert json -> data - 2. choose backend operation -3. backend: execute logic - 1. store: load or save from/to db -4. server: - 1. convert data -> json - - -backend: -- need better name -- contains all logic encoded as operations -- operation: A -> Either[E, B] -- middleware translates userId -> required data - - e.g. userId -> public key -- operations can fail - - common error class is used - - can be converted to json easily - - -New Items: - -1. upload endpoint -2. server: - 1. convert json->data -3. store: add job to queue -4. processor: - 1. eventually takes the job - 2. execute job - 3. notify about result - - -Processors - -- multiple processors possible -- multiple backend servers possible -- separate processes -- register on database - - unique id - - url - - servers and processors -- once a job is added to the queue notify all processors - - take all registered urls from db - - call them, skip failing ones -- processors wake up and take next job based on their config -- first free processor gets a new job -- once done, notify registered backend server diff --git a/doc/install.md b/doc/install.md deleted file mode 100644 index 739ee74a..00000000 --- a/doc/install.md +++ /dev/null @@ -1 +0,0 @@ -# Installation and Setup diff --git a/doc/user.md b/doc/user.md deleted file mode 100644 index 8e6b09eb..00000000 --- a/doc/user.md +++ /dev/null @@ -1,69 +0,0 @@ -# User Documentation - -## Concepts - - -## UI Screens - -The web application is the provided user interface. THere are the following screens - - -### Login - -### Change Password - -### Document Overview - -- search menu on the left, 25% -- listing in the middle, 25% - - choose to list all documents - - or grouped by date, corresp. etc -- right: document preview + details, 50% - -### Document Edit - -- search menu + listing is replaced by edit screen, 50% -- document preview 50% - -### Manage Additional Data - -CRUD for - -- Organisation -- Person -- Equipment -- Sources, which are the possible upload endpoints. - -### Collective Settings - -- keystore -- collective data -- manage users - -### User Settings - -- preferences (language, ...) -- smtp servers - -### Super Settings - -- admin only -- enable/disable registration -- settings for the app and collectives - - e.g. can block collectives or users -- CRUD for all entities - -### Collective Processing Queue - -- user can inspect current processing -- see errors and progress -- see jobs in queue -- cancel jobs -- see some stats about executors, so one can make an educated guess as - to when the next job is executed - -### Admin Processing Queue - -- see which external processing workers are registered -- cancel/pause jobs -- some stats diff --git a/elm.json b/elm.json index e974e727..58e79f4f 100644 --- a/elm.json +++ b/elm.json @@ -7,18 +7,26 @@ "elm-version": "0.19.0", "dependencies": { "direct": { + "CurrySoftware/elm-datepicker": "3.1.0", "NoRedInk/elm-json-decode-pipeline": "1.0.0", + "NoRedInk/elm-simple-fuzzy": "1.0.3", "elm/browser": "1.0.1", "elm/core": "1.0.2", + "elm/file": "1.0.5", "elm/html": "1.0.0", "elm/http": "2.0.0", "elm/json": "1.1.3", - "elm/url": "1.0.0" + "elm/time": "1.0.0", + "elm/url": "1.0.0", + "elm-explorations/markdown": "1.0.0", + "justinmimbs/date": "3.1.2", + "ryannhg/date-format": "2.3.0", + "truqu/elm-base64": "2.0.4" }, "indirect": { "elm/bytes": "1.0.8", - "elm/file": "1.0.5", - "elm/time": "1.0.0", + "elm/parser": "1.1.0", + "elm/regex": "1.0.0", "elm/virtual-dom": "1.0.2" } }, diff --git a/modules/backend/src/main/scala/docspell/backend/BackendApp.scala b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala new file mode 100644 index 00000000..757b24ae --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/BackendApp.scala @@ -0,0 +1,66 @@ +package docspell.backend + +import cats.effect.{Blocker, ConcurrentEffect, ContextShift, Resource} +import docspell.backend.auth.Login +import docspell.backend.ops._ +import docspell.backend.signup.OSignup +import docspell.store.Store +import docspell.store.ops.ONode +import docspell.store.queue.JobQueue + +import scala.concurrent.ExecutionContext + +trait BackendApp[F[_]] { + + def login: Login[F] + def signup: OSignup[F] + def collective: OCollective[F] + def source: OSource[F] + def tag: OTag[F] + def equipment: OEquipment[F] + def organization: OOrganization[F] + def upload: OUpload[F] + def node: ONode[F] + def job: OJob[F] + def item: OItem[F] +} + +object BackendApp { + + def create[F[_]: ConcurrentEffect](cfg: Config, store: Store[F], httpClientEc: ExecutionContext): Resource[F, BackendApp[F]] = + for { + queue <- JobQueue(store) + loginImpl <- Login[F](store) + signupImpl <- OSignup[F](store) + collImpl <- OCollective[F](store) + sourceImpl <- OSource[F](store) + tagImpl <- OTag[F](store) + equipImpl <- OEquipment[F](store) + orgImpl <- OOrganization(store) + uploadImpl <- OUpload(store, queue, cfg, httpClientEc) + nodeImpl <- ONode(store) + jobImpl <- OJob(store, httpClientEc) + itemImpl <- OItem(store) + } yield new BackendApp[F] { + val login: Login[F] = loginImpl + val signup: OSignup[F] = signupImpl + val collective: OCollective[F] = collImpl + val source = sourceImpl + val tag = tagImpl + val equipment = equipImpl + val organization = orgImpl + val upload = uploadImpl + val node = nodeImpl + val job = jobImpl + val item = itemImpl + } + + def apply[F[_]: ConcurrentEffect: ContextShift](cfg: Config + , connectEC: ExecutionContext + , httpClientEc: ExecutionContext + , blocker: Blocker): Resource[F, BackendApp[F]] = + for { + store <- Store.create(cfg.jdbc, connectEC, blocker) + backend <- create(cfg, store, httpClientEc) + } yield backend +} diff --git a/modules/backend/src/main/scala/docspell/backend/Common.scala b/modules/backend/src/main/scala/docspell/backend/Common.scala new file mode 100644 index 00000000..bd86c900 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/Common.scala @@ -0,0 +1,10 @@ +package docspell.backend + +import cats.effect._ +import org.mindrot.jbcrypt.BCrypt + +object Common { + + def genSaltString[F[_]: Sync]: F[String] = + Sync[F].delay(BCrypt.gensalt()) +} diff --git a/modules/backend/src/main/scala/docspell/backend/Config.scala b/modules/backend/src/main/scala/docspell/backend/Config.scala new file mode 100644 index 00000000..7869019c --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/Config.scala @@ -0,0 +1,16 @@ +package docspell.backend + +import docspell.backend.signup.{Config => SignupConfig} +import docspell.common.MimeType +import docspell.store.JdbcConfig + +case class Config( jdbc: JdbcConfig + , signup: SignupConfig + , files: Config.Files) { + +} + +object Config { + + case class Files(chunkSize: Int, validMimeTypes: Seq[MimeType]) +} \ No newline at end of file diff --git a/modules/backend/src/main/scala/docspell/backend/PasswordCrypt.scala b/modules/backend/src/main/scala/docspell/backend/PasswordCrypt.scala new file mode 100644 index 00000000..003ad510 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/PasswordCrypt.scala @@ -0,0 +1,13 @@ +package docspell.backend + +import docspell.common.Password +import org.mindrot.jbcrypt.BCrypt + +object PasswordCrypt { + + def crypt(pass: Password): Password = + Password(BCrypt.hashpw(pass.pass, BCrypt.gensalt())) + + def check(plain: Password, hashed: Password): Boolean = + BCrypt.checkpw(plain.pass, hashed.pass) +} diff --git a/modules/backend/src/main/scala/docspell/backend/auth/AuthToken.scala b/modules/backend/src/main/scala/docspell/backend/auth/AuthToken.scala new file mode 100644 index 00000000..57d3f5ca --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/auth/AuthToken.scala @@ -0,0 +1,81 @@ +package docspell.backend.auth + +import cats.effect._ +import cats.implicits._ +import java.time.Instant + +import javax.crypto.Mac +import javax.crypto.spec.SecretKeySpec +import scodec.bits.ByteVector + +import docspell.backend.Common +import AuthToken._ +import docspell.common._ + +case class AuthToken(millis: Long, account: AccountId, salt: String, sig: String) { + def asString = s"$millis-${b64enc(account.asString)}-$salt-$sig" + + def sigValid(key: ByteVector): Boolean = { + val newSig = AuthToken.sign(this, key) + AuthToken.constTimeEq(sig, newSig) + } + def sigInvalid(key: ByteVector): Boolean = + !sigValid(key) + + def notExpired(validity: Duration): Boolean = + !isExpired(validity) + + def isExpired(validity: Duration): Boolean = { + val ends = Instant.ofEpochMilli(millis).plusMillis(validity.millis) + Instant.now.isAfter(ends) + } + + def validate(key: ByteVector, validity: Duration): Boolean = + sigValid(key) && notExpired(validity) +} + +object AuthToken { + private val utf8 = java.nio.charset.StandardCharsets.UTF_8 + + def fromString(s: String): Either[String, AuthToken] = + s.split("\\-", 4) match { + case Array(ms, as, salt, sig) => + for { + millis <- asInt(ms).toRight("Cannot read authenticator data") + acc <- b64dec(as).toRight("Cannot read authenticator data") + accId <- AccountId.parse(acc) + } yield AuthToken(millis, accId, salt, sig) + + case _ => + Left("Invalid authenticator") + } + + def user[F[_]: Sync](accountId: AccountId, key: ByteVector): F[AuthToken] = { + for { + salt <- Common.genSaltString[F] + millis = Instant.now.toEpochMilli + cd = AuthToken(millis, accountId, salt, "") + sig = sign(cd, key) + } yield cd.copy(sig = sig) + } + + private def sign(cd: AuthToken, key: ByteVector): String = { + val raw = cd.millis.toString + cd.account.asString + cd.salt + val mac = Mac.getInstance("HmacSHA1") + mac.init(new SecretKeySpec(key.toArray, "HmacSHA1")) + ByteVector.view(mac.doFinal(raw.getBytes(utf8))).toBase64 + } + + private def b64enc(s: String): String = + ByteVector.view(s.getBytes(utf8)).toBase64 + + private def b64dec(s: String): Option[String] = + ByteVector.fromValidBase64(s).decodeUtf8.toOption + + private def asInt(s: String): Option[Long] = + Either.catchNonFatal(s.toLong).toOption + + private def constTimeEq(s1: String, s2: String): Boolean = + s1.zip(s2).foldLeft(true)({ case (r, (c1, c2)) => r & c1 == c2 }) & s1.length == s2.length + +} diff --git a/modules/backend/src/main/scala/docspell/backend/auth/Login.scala b/modules/backend/src/main/scala/docspell/backend/auth/Login.scala new file mode 100644 index 00000000..d093b43a --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/auth/Login.scala @@ -0,0 +1,89 @@ +package docspell.backend.auth + +import cats.effect._ +import cats.implicits._ +import Login._ +import docspell.common._ +import docspell.store.Store +import docspell.store.queries.QLogin +import docspell.store.records.RUser +import org.mindrot.jbcrypt.BCrypt +import scodec.bits.ByteVector +import org.log4s._ + +trait Login[F[_]] { + + def loginSession(config: Config)(sessionKey: String): F[Result] + + def loginUserPass(config: Config)(up: UserPass): F[Result] + +} + +object Login { + private[this] val logger = getLogger + + case class Config(serverSecret: ByteVector, sessionValid: Duration) + + case class UserPass(user: String, pass: String) { + def hidePass: UserPass = + if (pass.isEmpty) copy(pass = "") + else copy(pass = "***") + } + + sealed trait Result { + def toEither: Either[String, AuthToken] + } + object Result { + case class Ok(session: AuthToken) extends Result { + val toEither = Right(session) + } + case object InvalidAuth extends Result { + val toEither = Left("Authentication failed.") + } + case object InvalidTime extends Result { + val toEither = Left("Authentication failed.") + } + + def ok(session: AuthToken): Result = Ok(session) + def invalidAuth: Result = InvalidAuth + def invalidTime: Result = InvalidTime + } + + def apply[F[_]: Effect](store: Store[F]): Resource[F, Login[F]] = Resource.pure(new Login[F] { + + def loginSession(config: Config)(sessionKey: String): F[Result] = + AuthToken.fromString(sessionKey) match { + case Right(at) => + if (at.sigInvalid(config.serverSecret)) Result.invalidAuth.pure[F] + else if (at.isExpired(config.sessionValid)) Result.invalidTime.pure[F] + else Result.ok(at).pure[F] + case Left(err) => + Result.invalidAuth.pure[F] + } + + def loginUserPass(config: Config)(up: UserPass): F[Result] = { + AccountId.parse(up.user) match { + case Right(acc) => + val okResult= + store.transact(RUser.updateLogin(acc)) *> + AuthToken.user(acc, config.serverSecret).map(Result.ok) + for { + data <- store.transact(QLogin.findUser(acc)) + _ <- Sync[F].delay(logger.trace(s"Account lookup: $data")) + res <- if (data.exists(check(up.pass))) okResult + else Result.invalidAuth.pure[F] + } yield res + case Left(err) => + Result.invalidAuth.pure[F] + } + } + + private def check(given: String)(data: QLogin.Data): Boolean = { + val collOk = data.collectiveState == CollectiveState.Active || + data.collectiveState == CollectiveState.ReadOnly + val userOk = data.userState == UserState.Active + val passOk = BCrypt.checkpw(given, data.password.pass) + collOk && userOk && passOk + } + }) +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala new file mode 100644 index 00000000..67fe0736 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OCollective.scala @@ -0,0 +1,111 @@ +package docspell.backend.ops + +import cats.implicits._ +import cats.effect.{Effect, Resource} +import docspell.common._ +import docspell.store.{AddResult, Store} +import docspell.store.records.{RCollective, RUser} +import OCollective._ +import docspell.backend.PasswordCrypt +import docspell.store.queries.QCollective + +trait OCollective[F[_]] { + + def find(name: Ident): F[Option[RCollective]] + + def updateLanguage(collective: Ident, lang: Language): F[AddResult] + + def listUser(collective: Ident): F[Vector[RUser]] + + def add(s: RUser): F[AddResult] + + def update(s: RUser): F[AddResult] + + def deleteUser(login: Ident, collective: Ident): F[AddResult] + + def insights(collective: Ident): F[InsightData] + + def changePassword(accountId: AccountId, current: Password, newPass: Password): F[PassChangeResult] +} + +object OCollective { + + type InsightData = QCollective.InsightData + val insightData = QCollective.InsightData + + sealed trait PassChangeResult + object PassChangeResult { + case object UserNotFound extends PassChangeResult + case object PasswordMismatch extends PassChangeResult + case object UpdateFailed extends PassChangeResult + case object Success extends PassChangeResult + + def userNotFound: PassChangeResult = UserNotFound + def passwordMismatch: PassChangeResult = PasswordMismatch + def success: PassChangeResult = Success + def updateFailed: PassChangeResult = UpdateFailed + } + + case class RegisterData(collName: Ident, login: Ident, password: Password, invite: Option[Ident]) + + sealed trait RegisterResult { + def toEither: Either[Throwable, Unit] + } + object RegisterResult { + case object Success extends RegisterResult { + val toEither = Right(()) + } + case class CollectiveExists(id: Ident) extends RegisterResult { + val toEither = Left(new Exception()) + } + case class Error(ex: Throwable) extends RegisterResult { + val toEither = Left(ex) + } + } + + + def apply[F[_]:Effect](store: Store[F]): Resource[F, OCollective[F]] = + Resource.pure(new OCollective[F] { + def find(name: Ident): F[Option[RCollective]] = + store.transact(RCollective.findById(name)) + + def updateLanguage(collective: Ident, lang: Language): F[AddResult] = + store.transact(RCollective.updateLanguage(collective, lang)). + attempt.map(AddResult.fromUpdate) + + def listUser(collective: Ident): F[Vector[RUser]] = { + store.transact(RUser.findAll(collective, _.login)) + } + + def add(s: RUser): F[AddResult] = + store.add(RUser.insert(s.copy(password = PasswordCrypt.crypt(s.password))), RUser.exists(s.login)) + + def update(s: RUser): F[AddResult] = + store.add(RUser.update(s), RUser.exists(s.login)) + + def deleteUser(login: Ident, collective: Ident): F[AddResult] = + store.transact(RUser.delete(login, collective)). + attempt.map(AddResult.fromUpdate) + + def insights(collective: Ident): F[InsightData] = + store.transact(QCollective.getInsights(collective)) + + def changePassword(accountId: AccountId, current: Password, newPass: Password): F[PassChangeResult] = { + val q = for { + optUser <- RUser.findByAccount(accountId) + check = optUser.map(_.password).map(p => PasswordCrypt.check(current, p)) + n <- check.filter(identity).traverse(_ => RUser.updatePassword(accountId, PasswordCrypt.crypt(newPass))) + res = check match { + case Some(true) => + if (n.getOrElse(0) > 0) PassChangeResult.success else PassChangeResult.updateFailed + case Some(false) => + PassChangeResult.passwordMismatch + case None => + PassChangeResult.userNotFound + } + } yield res + + store.transact(q) + } + }) +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OEquipment.scala b/modules/backend/src/main/scala/docspell/backend/ops/OEquipment.scala new file mode 100644 index 00000000..8ac4698f --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OEquipment.scala @@ -0,0 +1,54 @@ +package docspell.backend.ops + +import cats.implicits._ +import cats.effect.{Effect, Resource} +import docspell.common.{AccountId, Ident} +import docspell.store.{AddResult, Store} +import docspell.store.records.{REquipment, RItem} + +trait OEquipment[F[_]] { + + def findAll(account: AccountId): F[Vector[REquipment]] + + def add(s: REquipment): F[AddResult] + + def update(s: REquipment): F[AddResult] + + def delete(id: Ident, collective: Ident): F[AddResult] +} + + +object OEquipment { + + def apply[F[_]: Effect](store: Store[F]): Resource[F, OEquipment[F]] = + Resource.pure(new OEquipment[F] { + def findAll(account: AccountId): F[Vector[REquipment]] = + store.transact(REquipment.findAll(account.collective, _.name)) + + def add(e: REquipment): F[AddResult] = { + def insert = REquipment.insert(e) + def exists = REquipment.existsByName(e.cid, e.name) + + val msg = s"An equipment '${e.name}' already exists" + store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity)) + } + + def update(e: REquipment): F[AddResult] = { + def insert = REquipment.update(e) + def exists = REquipment.existsByName(e.cid, e.name) + + val msg = s"An equipment '${e.name}' already exists" + store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity)) + } + + def delete(id: Ident, collective: Ident): F[AddResult] = { + val io = for { + n0 <- RItem.removeConcEquip(collective, id) + n1 <- REquipment.delete(id, collective) + } yield n0 + n1 + store.transact(io). + attempt. + map(AddResult.fromUpdate) + } + }) +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala new file mode 100644 index 00000000..4db3409f --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OItem.scala @@ -0,0 +1,159 @@ +package docspell.backend.ops + +import fs2.Stream +import cats.implicits._ +import cats.effect.{Effect, Resource} +import doobie._ +import doobie.implicits._ +import docspell.store.{AddResult, Store} +import docspell.store.queries.{QAttachment, QItem} +import OItem.{AttachmentData, ItemData, ListItem, Query} +import bitpeace.{FileMeta, RangeDef} +import docspell.common.{Direction, Ident, ItemState, MetaProposalList, Timestamp} +import docspell.store.records.{RAttachment, RAttachmentMeta, RItem, RTagItem} + +trait OItem[F[_]] { + + def findItem(id: Ident, collective: Ident): F[Option[ItemData]] + + def findItems(q: Query, maxResults: Int): F[Vector[ListItem]] + + def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] + + def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] + + def setDirection(item: Ident, direction: Direction, collective: Ident): F[AddResult] + + def setCorrOrg(item: Ident, org: Option[Ident], collective: Ident): F[AddResult] + + def setCorrPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] + + def setConcPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] + + def setConcEquip(item: Ident, equip: Option[Ident], collective: Ident): F[AddResult] + + def setNotes(item: Ident, notes: Option[String], collective: Ident): F[AddResult] + + def setName(item: Ident, notes: String, collective: Ident): F[AddResult] + + def setState(item: Ident, state: ItemState, collective: Ident): F[AddResult] + + def setItemDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] + + def setItemDueDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] + + def getProposals(item: Ident, collective: Ident): F[MetaProposalList] + + def delete(itemId: Ident, collective: Ident): F[Int] + + def findAttachmentMeta(id: Ident, collective: Ident): F[Option[RAttachmentMeta]] +} + +object OItem { + + type Query = QItem.Query + val Query = QItem.Query + + type ListItem = QItem.ListItem + val ListItem = QItem.ListItem + + type ItemData = QItem.ItemData + val ItemData = QItem.ItemData + + case class AttachmentData[F[_]](ra: RAttachment, meta: FileMeta, data: Stream[F, Byte]) + + + def apply[F[_]: Effect](store: Store[F]): Resource[F, OItem[F]] = + Resource.pure(new OItem[F] { + + def findItem(id: Ident, collective: Ident): F[Option[ItemData]] = + store.transact(QItem.findItem(id)). + map(opt => opt.flatMap(_.filterCollective(collective))) + + def findItems(q: Query, maxResults: Int): F[Vector[ListItem]] = { + store.transact(QItem.findItems(q).take(maxResults.toLong)).compile.toVector + } + + def findAttachment(id: Ident, collective: Ident): F[Option[AttachmentData[F]]] = { + store.transact(RAttachment.findByIdAndCollective(id, collective)). + flatMap({ + case Some(ra) => + store.bitpeace.get(ra.fileId.id).unNoneTerminate.compile.last. + map(_.map(m => AttachmentData[F](ra, m, store.bitpeace.fetchData2(RangeDef.all)(Stream.emit(m))))) + case None => + (None: Option[AttachmentData[F]]).pure[F] + }) + } + + def setTags(item: Ident, tagIds: List[Ident], collective: Ident): F[AddResult] = { + val db = for { + cid <- RItem.getCollective(item) + nd <- if (cid.contains(collective)) RTagItem.deleteItemTags(item) else 0.pure[ConnectionIO] + ni <- if (tagIds.nonEmpty && cid.contains(collective)) RTagItem.insertItemTags(item, tagIds) else 0.pure[ConnectionIO] + } yield nd + ni + + store.transact(db). + attempt. + map(AddResult.fromUpdate) + } + + def setDirection(item: Ident, direction: Direction, collective: Ident): F[AddResult] = + store.transact(RItem.updateDirection(item, collective, direction)). + attempt. + map(AddResult.fromUpdate) + + def setCorrOrg(item: Ident, org: Option[Ident], collective: Ident): F[AddResult] = + store.transact(RItem.updateCorrOrg(item, collective, org)). + attempt. + map(AddResult.fromUpdate) + + def setCorrPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] = + store.transact(RItem.updateCorrPerson(item, collective, person)). + attempt. + map(AddResult.fromUpdate) + + def setConcPerson(item: Ident, person: Option[Ident], collective: Ident): F[AddResult] = + store.transact(RItem.updateConcPerson(item, collective, person)). + attempt. + map(AddResult.fromUpdate) + + def setConcEquip(item: Ident, equip: Option[Ident], collective: Ident): F[AddResult] = + store.transact(RItem.updateConcEquip(item, collective, equip)). + attempt. + map(AddResult.fromUpdate) + + def setNotes(item: Ident, notes: Option[String], collective: Ident): F[AddResult] = + store.transact(RItem.updateNotes(item, collective, notes)). + attempt. + map(AddResult.fromUpdate) + + def setName(item: Ident, name: String, collective: Ident): F[AddResult] = + store.transact(RItem.updateName(item, collective, name)). + attempt. + map(AddResult.fromUpdate) + + def setState(item: Ident, state: ItemState, collective: Ident): F[AddResult] = + store.transact(RItem.updateStateForCollective(item, state, collective)). + attempt. + map(AddResult.fromUpdate) + + def setItemDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] = + store.transact(RItem.updateDate(item, collective, date)). + attempt. + map(AddResult.fromUpdate) + + def setItemDueDate(item: Ident, date: Option[Timestamp], collective: Ident): F[AddResult] = + store.transact(RItem.updateDueDate(item, collective, date)). + attempt. + map(AddResult.fromUpdate) + + def delete(itemId: Ident, collective: Ident): F[Int] = + QItem.delete(store)(itemId, collective) + + def getProposals(item: Ident, collective: Ident): F[MetaProposalList] = + store.transact(QAttachment.getMetaProposals(item, collective)) + + def findAttachmentMeta(id: Ident, collective: Ident): F[Option[RAttachmentMeta]] = + store.transact(QAttachment.getAttachmentMeta(id, collective)) + }) +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala b/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala new file mode 100644 index 00000000..99d7861a --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OJob.scala @@ -0,0 +1,84 @@ +package docspell.backend.ops + +import cats.implicits._ +import cats.effect.{ConcurrentEffect, Resource} +import docspell.backend.ops.OJob.{CollectiveQueueState, JobCancelResult} +import docspell.common.{Ident, JobState} +import docspell.store.Store +import docspell.store.queries.QJob +import docspell.store.records.{RJob, RJobLog} + +import scala.concurrent.ExecutionContext + +trait OJob[F[_]] { + + def queueState(collective: Ident, maxResults: Int): F[CollectiveQueueState] + + def cancelJob(id: Ident, collective: Ident): F[JobCancelResult] +} + +object OJob { + + sealed trait JobCancelResult + object JobCancelResult { + case object Removed extends JobCancelResult + case object CancelRequested extends JobCancelResult + case object JobNotFound extends JobCancelResult + } + + case class JobDetail(job: RJob, logs: Vector[RJobLog]) + case class CollectiveQueueState(jobs: Vector[JobDetail]) { + def queued: Vector[JobDetail] = + jobs.filter(r => JobState.queued.contains(r.job.state)) + def done: Vector[JobDetail] = + jobs.filter(r => JobState.done.contains(r.job.state)) + def running: Vector[JobDetail] = + jobs.filter(_.job.state == JobState.Running) + } + + def apply[F[_]: ConcurrentEffect](store: Store[F], clientEC: ExecutionContext): Resource[F, OJob[F]] = + Resource.pure(new OJob[F] { + + def queueState(collective: Ident, maxResults: Int): F[CollectiveQueueState] = { + store.transact(QJob.queueStateSnapshot(collective).take(maxResults.toLong)). + map(t => JobDetail(t._1, t._2)). + compile.toVector. + map(CollectiveQueueState) + } + + def cancelJob(id: Ident, collective: Ident): F[JobCancelResult] = { + def mustCancel(job: Option[RJob]): Option[(RJob, Ident)] = + for { + worker <- job.flatMap(_.worker) + job <- job.filter(j => j.state == JobState.Scheduled || j.state == JobState.Running) + } yield (job, worker) + + def canDelete(j: RJob): Boolean = + mustCancel(j.some).isEmpty + + val tryDelete = for { + job <- RJob.findByIdAndGroup(id, collective) + jobm = job.filter(canDelete) + del <- jobm.traverse(j => RJob.delete(j.id)) + } yield del match { + case Some(n) => Right(JobCancelResult.Removed: JobCancelResult) + case None => Left(mustCancel(job)) + } + + def tryCancel(job: RJob, worker: Ident): F[JobCancelResult] = + OJoex.cancelJob(job.id, worker, store, clientEC). + map(flag => if (flag) JobCancelResult.CancelRequested else JobCancelResult.JobNotFound) + + for { + tryDel <- store.transact(tryDelete) + result <- tryDel match { + case Right(r) => r.pure[F] + case Left(Some((job, worker))) => + tryCancel(job, worker) + case Left(None) => + (JobCancelResult.JobNotFound: OJob.JobCancelResult).pure[F] + } + } yield result + } + }) +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala b/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala new file mode 100644 index 00000000..6c120ee1 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OJoex.scala @@ -0,0 +1,54 @@ +package docspell.backend.ops + +import cats.implicits._ +import cats.effect.ConcurrentEffect +import docspell.common.{Ident, NodeType} +import docspell.store.Store +import docspell.store.records.RNode +import org.http4s.client.blaze.BlazeClientBuilder +import org.http4s.Method._ +import org.http4s.{Request, Uri} + +import scala.concurrent.ExecutionContext +import org.log4s._ + +object OJoex { + private [this] val logger = getLogger + + def notifyAll[F[_]: ConcurrentEffect](store: Store[F], clientExecutionContext: ExecutionContext): F[Unit] = { + for { + nodes <- store.transact(RNode.findAll(NodeType.Joex)) + _ <- nodes.toList.traverse(notifyJoex[F](clientExecutionContext)) + } yield () + } + + def cancelJob[F[_]: ConcurrentEffect](jobId: Ident, worker: Ident, store: Store[F], clientEc: ExecutionContext): F[Boolean] = + for { + node <- store.transact(RNode.findById(worker)) + cancel <- node.traverse(joexCancel(clientEc)(_, jobId)) + } yield cancel.getOrElse(false) + + + private def joexCancel[F[_]: ConcurrentEffect](ec: ExecutionContext)(node: RNode, job: Ident): F[Boolean] = { + val notifyUrl = node.url/"api"/"v1"/"job"/job.id/"cancel" + BlazeClientBuilder[F](ec).resource.use { client => + val req = Request[F](POST, Uri.unsafeFromString(notifyUrl.asString)) + client.expect[String](req).map(_ => true) + } + } + + private def notifyJoex[F[_]: ConcurrentEffect](ec: ExecutionContext)(node: RNode): F[Unit] = { + val notifyUrl = node.url/"api"/"v1"/"notify" + val execute = BlazeClientBuilder[F](ec).resource.use { client => + val req = Request[F](POST, Uri.unsafeFromString(notifyUrl.asString)) + client.expect[String](req).map(_ => ()) + } + execute.attempt.map { + case Right(_) => + () + case Left(_) => + logger.warn(s"Notifying Joex instance '${node.id.id}/${node.url.asString}' failed.") + () + } + } +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OOrganization.scala b/modules/backend/src/main/scala/docspell/backend/ops/OOrganization.scala new file mode 100644 index 00000000..356d7a56 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OOrganization.scala @@ -0,0 +1,81 @@ +package docspell.backend.ops + +import cats.implicits._ +import cats.effect.{Effect, Resource} +import docspell.common._ +import docspell.store._ +import docspell.store.records._ +import OOrganization._ +import docspell.store.queries.QOrganization + +trait OOrganization[F[_]] { + def findAllOrg(account: AccountId): F[Vector[OrgAndContacts]] + + def findAllOrgRefs(account: AccountId): F[Vector[IdRef]] + + def addOrg(s: OrgAndContacts): F[AddResult] + + def updateOrg(s: OrgAndContacts): F[AddResult] + + def findAllPerson(account: AccountId): F[Vector[PersonAndContacts]] + + def findAllPersonRefs(account: AccountId): F[Vector[IdRef]] + + def addPerson(s: PersonAndContacts): F[AddResult] + + def updatePerson(s: PersonAndContacts): F[AddResult] + + def deleteOrg(orgId: Ident, collective: Ident): F[AddResult] + + def deletePerson(personId: Ident, collective: Ident): F[AddResult] +} + +object OOrganization { + + case class OrgAndContacts(org: ROrganization, contacts: Seq[RContact]) + + case class PersonAndContacts(person: RPerson, contacts: Seq[RContact]) + + def apply[F[_] : Effect](store: Store[F]): Resource[F, OOrganization[F]] = + Resource.pure(new OOrganization[F] { + + def findAllOrg(account: AccountId): F[Vector[OrgAndContacts]] = + store.transact(QOrganization.findOrgAndContact(account.collective, _.name)). + map({ case (org, cont) => OrgAndContacts(org, cont) }). + compile.toVector + + def findAllOrgRefs(account: AccountId): F[Vector[IdRef]] = + store.transact(ROrganization.findAllRef(account.collective, _.name)) + + def addOrg(s: OrgAndContacts): F[AddResult] = + QOrganization.addOrg(s.org, s.contacts, s.org.cid)(store) + + def updateOrg(s: OrgAndContacts): F[AddResult] = + QOrganization.updateOrg(s.org, s.contacts, s.org.cid)(store) + + def findAllPerson(account: AccountId): F[Vector[PersonAndContacts]] = + store.transact(QOrganization.findPersonAndContact(account.collective, _.name)). + map({ case (person, cont) => PersonAndContacts(person, cont) }). + compile.toVector + + def findAllPersonRefs(account: AccountId): F[Vector[IdRef]] = + store.transact(RPerson.findAllRef(account.collective, _.name)) + + def addPerson(s: PersonAndContacts): F[AddResult] = + QOrganization.addPerson(s.person, s.contacts, s.person.cid)(store) + + def updatePerson(s: PersonAndContacts): F[AddResult] = + QOrganization.updatePerson(s.person, s.contacts, s.person.cid)(store) + + def deleteOrg(orgId: Ident, collective: Ident): F[AddResult] = + store.transact(QOrganization.deleteOrg(orgId, collective)). + attempt. + map(AddResult.fromUpdate) + + def deletePerson(personId: Ident, collective: Ident): F[AddResult] = + store.transact(QOrganization.deletePerson(personId, collective)). + attempt. + map(AddResult.fromUpdate) + + }) +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OSource.scala b/modules/backend/src/main/scala/docspell/backend/ops/OSource.scala new file mode 100644 index 00000000..227ae752 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OSource.scala @@ -0,0 +1,48 @@ +package docspell.backend.ops + +import cats.implicits._ +import cats.effect.{Effect, Resource} +import docspell.common.{AccountId, Ident} +import docspell.store.{AddResult, Store} +import docspell.store.records.RSource + +trait OSource[F[_]] { + + def findAll(account: AccountId): F[Vector[RSource]] + + def add(s: RSource): F[AddResult] + + def update(s: RSource): F[AddResult] + + def delete(id: Ident, collective: Ident): F[AddResult] +} + +object OSource { + + def apply[F[_]: Effect](store: Store[F]): Resource[F, OSource[F]] = + Resource.pure(new OSource[F] { + def findAll(account: AccountId): F[Vector[RSource]] = + store.transact(RSource.findAll(account.collective, _.abbrev)) + + def add(s: RSource): F[AddResult] = { + def insert = RSource.insert(s) + def exists = RSource.existsByAbbrev(s.cid, s.abbrev) + + val msg = s"A source with abbrev '${s.abbrev}' already exists" + store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity)) + } + + def update(s: RSource): F[AddResult] = { + def insert = RSource.updateNoCounter(s) + def exists = RSource.existsByAbbrev(s.cid, s.abbrev) + + val msg = s"A source with abbrev '${s.abbrev}' already exists" + store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity)) + } + + def delete(id: Ident, collective: Ident): F[AddResult] = + store.transact(RSource.delete(id, collective)). + attempt. + map(AddResult.fromUpdate) + }) +} diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OTag.scala b/modules/backend/src/main/scala/docspell/backend/ops/OTag.scala new file mode 100644 index 00000000..54ccec58 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OTag.scala @@ -0,0 +1,56 @@ +package docspell.backend.ops + +import cats.implicits._ +import cats.effect.{Effect, Resource} +import docspell.common.{AccountId, Ident} +import docspell.store.{AddResult, Store} +import docspell.store.records.{RTag, RTagItem} + +trait OTag[F[_]] { + + def findAll(account: AccountId): F[Vector[RTag]] + + def add(s: RTag): F[AddResult] + + def update(s: RTag): F[AddResult] + + def delete(id: Ident, collective: Ident): F[AddResult] +} + + +object OTag { + + def apply[F[_]: Effect](store: Store[F]): Resource[F, OTag[F]] = + Resource.pure(new OTag[F] { + def findAll(account: AccountId): F[Vector[RTag]] = + store.transact(RTag.findAll(account.collective, _.name)) + + def add(t: RTag): F[AddResult] = { + def insert = RTag.insert(t) + def exists = RTag.existsByName(t) + + val msg = s"A tag '${t.name}' already exists" + store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity)) + } + + def update(t: RTag): F[AddResult] = { + def insert = RTag.update(t) + def exists = RTag.existsByName(t) + + val msg = s"A tag '${t.name}' already exists" + store.add(insert, exists).map(_.fold(identity, _.withMsg(msg), identity)) + } + + def delete(id: Ident, collective: Ident): F[AddResult] = { + val io = for { + optTag <- RTag.findByIdAndCollective(id, collective) + n0 <- optTag.traverse(t => RTagItem.deleteTag(t.tagId)) + n1 <- optTag.traverse(t => RTag.delete(t.tagId, collective)) + } yield n0.getOrElse(0) + n1.getOrElse(0) + store.transact(io). + attempt. + map(AddResult.fromUpdate) + } + }) +} + diff --git a/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala new file mode 100644 index 00000000..0351c1d9 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/ops/OUpload.scala @@ -0,0 +1,103 @@ +package docspell.backend.ops + +import bitpeace.MimetypeHint +import cats.implicits._ +import cats.effect.{ConcurrentEffect, Effect, Resource} +import docspell.backend.Config +import fs2.Stream +import docspell.common._ +import docspell.common.syntax.all._ +import docspell.store.Store +import docspell.store.queue.JobQueue +import docspell.store.records.{RCollective, RJob, RSource} +import org.log4s._ + +import scala.concurrent.ExecutionContext + +trait OUpload[F[_]] { + + def submit(data: OUpload.UploadData[F], account: AccountId): F[OUpload.UploadResult] + + def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] +} + +object OUpload { + private [this] val logger = getLogger + + case class File[F[_]](name: Option[String], advertisedMime: Option[MimeType], data: Stream[F, Byte]) + + case class UploadMeta( direction: Option[Direction] + , sourceAbbrev: String + , validFileTypes: Seq[MimeType]) + + case class UploadData[F[_]]( multiple: Boolean + , meta: UploadMeta + , files: Vector[File[F]], priority: Priority, tracker: Option[Ident]) + + sealed trait UploadResult + object UploadResult { + case object Success extends UploadResult + case object NoFiles extends UploadResult + case object NoSource extends UploadResult + } + + def apply[F[_]: ConcurrentEffect](store: Store[F], queue: JobQueue[F], cfg: Config, httpClientEC: ExecutionContext): Resource[F, OUpload[F]] = + Resource.pure(new OUpload[F] { + + def submit(data: OUpload.UploadData[F], account: AccountId): F[OUpload.UploadResult] = { + for { + files <- data.files.traverse(saveFile).map(_.flatten) + pred <- checkFileList(files) + lang <- store.transact(RCollective.findLanguage(account.collective)) + meta = ProcessItemArgs.ProcessMeta(account.collective, lang.getOrElse(Language.German), data.meta.direction, data.meta.sourceAbbrev, data.meta.validFileTypes) + args = if (data.multiple) files.map(f => ProcessItemArgs(meta, List(f))) else Vector(ProcessItemArgs(meta, files.toList)) + job <- pred.traverse(_ => makeJobs(args, account, data.priority, data.tracker)) + _ <- logger.fdebug(s"Storing jobs: $job") + res <- job.traverse(submitJobs) + _ <- store.transact(RSource.incrementCounter(data.meta.sourceAbbrev, account.collective)) + } yield res.fold(identity, identity) + } + + def submit(data: OUpload.UploadData[F], sourceId: Ident): F[OUpload.UploadResult] = + for { + sOpt <- store.transact(RSource.find(sourceId)).map(_.toRight(UploadResult.NoSource)) + abbrev = sOpt.map(_.abbrev).toOption.getOrElse(data.meta.sourceAbbrev) + updata = data.copy(meta = data.meta.copy(sourceAbbrev = abbrev)) + accId = sOpt.map(source => AccountId(source.cid, source.sid)) + result <- accId.traverse(acc => submit(updata, acc)) + } yield result.fold(identity, identity) + + private def submitJobs(jobs: Vector[RJob]): F[OUpload.UploadResult] = { + for { + _ <- logger.fdebug(s"Storing jobs: $jobs") + _ <- queue.insertAll(jobs) + _ <- OJoex.notifyAll(store, httpClientEC) + } yield UploadResult.Success + } + + private def saveFile(file: File[F]): F[Option[ProcessItemArgs.File]] = { + logger.finfo(s"Receiving file $file") *> + store.bitpeace.saveNew(file.data, cfg.files.chunkSize, MimetypeHint(file.name, None), None). + compile.lastOrError.map(fm => Ident.unsafe(fm.id)).attempt. + map(_.fold(ex => { + logger.warn(ex)(s"Could not store file for processing!") + None + }, id => Some(ProcessItemArgs.File(file.name, id)))) + } + + private def checkFileList(files: Seq[ProcessItemArgs.File]): F[Either[UploadResult, Unit]] = + Effect[F].pure(if (files.isEmpty) Left(UploadResult.NoFiles) else Right(())) + + private def makeJobs(args: Vector[ProcessItemArgs], account: AccountId, prio: Priority, tracker: Option[Ident]): F[Vector[RJob]] = { + def create(id: Ident, now: Timestamp, arg: ProcessItemArgs): RJob = + RJob.newJob(id, ProcessItemArgs.taskName, account.collective, arg, arg.makeSubject, now, account.user, prio, tracker) + + for { + id <- Ident.randomId[F] + now <- Timestamp.current[F] + jobs = args.map(a => create(id, now, a)) + } yield jobs + + } + }) +} diff --git a/modules/backend/src/main/scala/docspell/backend/signup/Config.scala b/modules/backend/src/main/scala/docspell/backend/signup/Config.scala new file mode 100644 index 00000000..f1c053e8 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/signup/Config.scala @@ -0,0 +1,41 @@ +package docspell.backend.signup + +import docspell.common.{Duration, Password} +import io.circe._ + +case class Config(mode: Config.Mode, newInvitePassword: Password, inviteTime: Duration) + +object Config { + sealed trait Mode { self: Product => + final def name: String = + productPrefix.toLowerCase + } + object Mode { + + case object Open extends Mode + + case object Invite extends Mode + + case object Closed extends Mode + + def fromString(str: String): Either[String, Mode] = + str.toLowerCase match { + case "open" => Right(Open) + case "invite" => Right(Invite) + case "closed" => Right(Closed) + case _ => Left(s"Invalid signup mode: $str") + } + def unsafe(str: String): Mode = + fromString(str).fold(sys.error, identity) + + implicit val jsonEncoder: Encoder[Mode] = + Encoder.encodeString.contramap(_.name) + implicit val jsonDecoder: Decoder[Mode] = + Decoder.decodeString.emap(fromString) + } + + def open: Mode = Mode.Open + def invite: Mode = Mode.Invite + def closed: Mode = Mode.Closed + +} diff --git a/modules/backend/src/main/scala/docspell/backend/signup/NewInviteResult.scala b/modules/backend/src/main/scala/docspell/backend/signup/NewInviteResult.scala new file mode 100644 index 00000000..b963d4a6 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/signup/NewInviteResult.scala @@ -0,0 +1,19 @@ +package docspell.backend.signup + +import docspell.common.Ident + +sealed trait NewInviteResult { self: Product => + + final def name: String = + productPrefix.toLowerCase +} + +object NewInviteResult { + case class Success(id: Ident) extends NewInviteResult + case object InvitationDisabled extends NewInviteResult + case object PasswordMismatch extends NewInviteResult + + def passwordMismatch: NewInviteResult = PasswordMismatch + def invitationClosed: NewInviteResult = InvitationDisabled + def success(id: Ident): NewInviteResult = Success(id) +} diff --git a/modules/backend/src/main/scala/docspell/backend/signup/OSignup.scala b/modules/backend/src/main/scala/docspell/backend/signup/OSignup.scala new file mode 100644 index 00000000..1c6ad36d --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/signup/OSignup.scala @@ -0,0 +1,83 @@ +package docspell.backend.signup + +import cats.implicits._ +import cats.effect.{Effect, Resource} +import docspell.backend.PasswordCrypt +import docspell.backend.ops.OCollective.RegisterData +import docspell.common._ +import docspell.store.{AddResult, Store} +import docspell.store.records.{RCollective, RInvitation, RUser} +import doobie.free.connection.ConnectionIO + +trait OSignup[F[_]] { + + def register(cfg: Config)(data: RegisterData): F[SignupResult] + + def newInvite(cfg: Config)(password: Password): F[NewInviteResult] +} + +object OSignup { + + def apply[F[_]:Effect](store: Store[F]): Resource[F, OSignup[F]] = + Resource.pure(new OSignup[F] { + + def newInvite(cfg: Config)(password: Password): F[NewInviteResult] = { + if (cfg.mode == Config.Mode.Invite) { + if (cfg.newInvitePassword.isEmpty || cfg.newInvitePassword != password) NewInviteResult.passwordMismatch.pure[F] + else store.transact(RInvitation.insertNew).map(ri => NewInviteResult.success(ri.id)) + } else { + Effect[F].pure(NewInviteResult.invitationClosed) + } + } + + def register(cfg: Config)(data: RegisterData): F[SignupResult] = { + cfg.mode match { + case Config.Mode.Open => + addUser(data).map(SignupResult.fromAddResult) + + case Config.Mode.Closed => + SignupResult.signupClosed.pure[F] + + case Config.Mode.Invite => + data.invite match { + case Some(inv) => + for { + now <- Timestamp.current[F] + min = now.minus(cfg.inviteTime) + ok <- store.transact(RInvitation.useInvite(inv, min)) + res <- if (ok) addUser(data).map(SignupResult.fromAddResult) + else SignupResult.invalidInvitationKey.pure[F] + } yield res + case None => + SignupResult.invalidInvitationKey.pure[F] + } + } + } + + private def addUser(data: RegisterData): F[AddResult] = { + def toRecords: F[(RCollective, RUser)] = + for { + id2 <- Ident.randomId[F] + now <- Timestamp.current[F] + c = RCollective(data.collName, CollectiveState.Active, Language.German, now) + u = RUser(id2, data.login, data.collName, PasswordCrypt.crypt(data.password), UserState.Active, None, 0, None, now) + } yield (c, u) + + def insert(coll: RCollective, user: RUser): ConnectionIO[Int] = { + for { + n1 <- RCollective.insert(coll) + n2 <- RUser.insert(user) + } yield n1 + n2 + } + + def collectiveExists: ConnectionIO[Boolean] = + RCollective.existsById(data.collName) + + val msg = s"The collective '${data.collName}' already exists." + for { + cu <- toRecords + save <- store.add(insert(cu._1, cu._2), collectiveExists) + } yield save.fold(identity, _.withMsg(msg), identity) + } + }) +} diff --git a/modules/backend/src/main/scala/docspell/backend/signup/SignupResult.scala b/modules/backend/src/main/scala/docspell/backend/signup/SignupResult.scala new file mode 100644 index 00000000..e8230059 --- /dev/null +++ b/modules/backend/src/main/scala/docspell/backend/signup/SignupResult.scala @@ -0,0 +1,28 @@ +package docspell.backend.signup + +import docspell.store.AddResult + +sealed trait SignupResult { + +} + +object SignupResult { + + case object CollectiveExists extends SignupResult + case object InvalidInvitationKey extends SignupResult + case object SignupClosed extends SignupResult + case class Failure(ex: Throwable) extends SignupResult + case object Success extends SignupResult + + def collectiveExists: SignupResult = CollectiveExists + def invalidInvitationKey: SignupResult = InvalidInvitationKey + def signupClosed: SignupResult = SignupClosed + def failure(ex: Throwable): SignupResult = Failure(ex) + def success: SignupResult = Success + + def fromAddResult(ar: AddResult): SignupResult = ar match { + case AddResult.Success => Success + case AddResult.Failure(ex) => Failure(ex) + case AddResult.EntityExists(_) => CollectiveExists + } +} diff --git a/modules/common/src/main/scala/docspell/common/AccountId.scala b/modules/common/src/main/scala/docspell/common/AccountId.scala new file mode 100644 index 00000000..1618bf5e --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/AccountId.scala @@ -0,0 +1,35 @@ +package docspell.common + +case class AccountId(collective: Ident, user: Ident) { + + def asString = + s"${collective.id}/${user.id}" +} + +object AccountId { + private[this] val sepearatorChars: String = "/\\:" + + def parse(str: String): Either[String, AccountId] = { + val input = str.replaceAll("\\s+", "").trim + val invalid: Either[String, AccountId] = + Left(s"Cannot parse account id: $str") + + def parse0(sep: Char): Either[String, AccountId] = + input.indexOf(sep.toInt) match { + case n if n > 0 && input.length > 2 => + val coll = input.substring(0, n) + val user = input.substring(n + 1) + Ident.fromString(coll). + flatMap(collId => Ident.fromString(user). + map(userId => AccountId(collId, userId))) + case _ => + invalid + } + + val separated = sepearatorChars.foldRight(invalid) { (c, v) => + v.orElse(parse0(c)) + } + + separated.orElse(Ident.fromString(str).map(id => AccountId(id, id))) + } +} diff --git a/modules/common/src/main/scala/docspell/common/Banner.scala b/modules/common/src/main/scala/docspell/common/Banner.scala new file mode 100644 index 00000000..d84cd00d --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/Banner.scala @@ -0,0 +1,34 @@ +package docspell.common + +case class Banner( component: String + , version: String + , gitHash: Option[String] + , jdbcUrl: LenientUri + , configFile: Option[String] + , appId: Ident + , baseUrl: LenientUri) { + + private val banner = + """______ _ _ + || _ \ | | | + || | | |___ ___ ___ _ __ ___| | | + || | | / _ \ / __/ __| '_ \ / _ \ | | + || |/ / (_) | (__\__ \ |_) | __/ | | + ||___/ \___/ \___|___/ .__/ \___|_|_| + | | | + |""".stripMargin + + s""" |_| v$version (#${gitHash.map(_.take(8)).getOrElse("")})""" + + def render(prefix: String): String = { + val text = banner.split('\n').toList ++ List( + s"<< $component >>" + , s"Id: ${appId.id}" + , s"Base-Url: ${baseUrl.asString}" + , s"Database: ${jdbcUrl.asString}" + , s"Config: ${configFile.getOrElse("")}" + , "" + ) + + text.map(line => s"$prefix $line").mkString("\n") + } +} diff --git a/modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala b/modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala new file mode 100644 index 00000000..4967e661 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/BaseJsonCodecs.scala @@ -0,0 +1,16 @@ +package docspell.common + +import java.time.Instant + +import io.circe._ + +object BaseJsonCodecs { + + implicit val encodeInstantEpoch: Encoder[Instant] = + Encoder.encodeJavaLong.contramap(_.toEpochMilli) + + implicit val decodeInstantEpoch: Decoder[Instant] = + Decoder.decodeLong.map(Instant.ofEpochMilli) + + +} diff --git a/modules/common/src/main/scala/docspell/common/CollectiveState.scala b/modules/common/src/main/scala/docspell/common/CollectiveState.scala new file mode 100644 index 00000000..00e7fed1 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/CollectiveState.scala @@ -0,0 +1,52 @@ +package docspell.common + +import io.circe.{Decoder, Encoder} + +sealed trait CollectiveState +object CollectiveState { + val all = List(Active, ReadOnly, Closed, Blocked) + + /** A normal active collective */ + case object Active extends CollectiveState + + /** A collective may be readonly in cases it is implicitly closed + * (e.g. no payment). Users can still see there data and + * download, but have no write access. */ + case object ReadOnly extends CollectiveState + + /** A collective that has been explicitely closed. */ + case object Closed extends CollectiveState + + /** A collective blocked by a super user, usually some emergency + * action. */ + case object Blocked extends CollectiveState + + + def fromString(s: String): Either[String, CollectiveState] = + s.toLowerCase match { + case "active" => Right(Active) + case "readonly" => Right(ReadOnly) + case "closed" => Right(Closed) + case "blocked" => Right(Blocked) + case _ => Left(s"Unknown state: $s") + } + + def unsafe(str: String): CollectiveState = + fromString(str).fold(sys.error, identity) + + def asString(state: CollectiveState): String = state match { + case Active => "active" + case Blocked => "blocked" + case Closed => "closed" + case ReadOnly => "readonly" + } + + + + implicit val collectiveStateEncoder: Encoder[CollectiveState] = + Encoder.encodeString.contramap(CollectiveState.asString) + + implicit val collectiveStateDecoder: Decoder[CollectiveState] = + Decoder.decodeString.emap(CollectiveState.fromString) + +} \ No newline at end of file diff --git a/modules/common/src/main/scala/docspell/common/ContactKind.scala b/modules/common/src/main/scala/docspell/common/ContactKind.scala new file mode 100644 index 00000000..54ed6958 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/ContactKind.scala @@ -0,0 +1,44 @@ +package docspell.common + +import io.circe.{Decoder, Encoder} + +sealed trait ContactKind { self: Product => + + def asString: String = self.productPrefix +} + +object ContactKind { + val all = List() + + case object Phone extends ContactKind + case object Mobile extends ContactKind + case object Fax extends ContactKind + case object Email extends ContactKind + case object Docspell extends ContactKind + case object Website extends ContactKind + + def fromString(s: String): Either[String, ContactKind] = + s.toLowerCase match { + case "phone" => Right(Phone) + case "mobile" => Right(Mobile) + case "fax" => Right(Fax) + case "email" => Right(Email) + case "docspell" => Right(Docspell) + case "website" => Right(Website) + case _ => Left(s"Not a state value: $s") + } + + def unsafe(str: String): ContactKind = + fromString(str).fold(sys.error, identity) + + def asString(s: ContactKind): String = + s.asString.toLowerCase + + + implicit val contactKindEncoder: Encoder[ContactKind] = + Encoder.encodeString.contramap(_.asString) + + implicit val contactKindDecoder: Decoder[ContactKind] = + Decoder.decodeString.emap(ContactKind.fromString) + +} diff --git a/modules/common/src/main/scala/docspell/common/Direction.scala b/modules/common/src/main/scala/docspell/common/Direction.scala new file mode 100644 index 00000000..a96d8338 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/Direction.scala @@ -0,0 +1,42 @@ +package docspell.common + +import io.circe.{Decoder, Encoder} + +sealed trait Direction { + self: Product => + + def name: String = + productPrefix.toLowerCase +} + +object Direction { + + case object Incoming extends Direction + case object Outgoing extends Direction + + def incoming: Direction = Incoming + def outgoing: Direction = Outgoing + + def parse(str: String): Either[String, Direction] = + str.toLowerCase match { + case "incoming" => Right(Incoming) + case "outgoing" => Right(Outgoing) + case _ => Left(s"No direction: $str") + } + + def unsafe(str: String): Direction = + parse(str).fold(sys.error, identity) + + def isIncoming(dir: Direction): Boolean = + dir == Direction.Incoming + + def isOutgoing(dir: Direction): Boolean = + dir == Direction.Outgoing + + implicit val directionEncoder: Encoder[Direction] = + Encoder.encodeString.contramap(_.name) + + implicit val directionDecoder: Decoder[Direction] = + Decoder.decodeString.emap(Direction.parse) + +} diff --git a/modules/common/src/main/scala/docspell/common/Duration.scala b/modules/common/src/main/scala/docspell/common/Duration.scala new file mode 100644 index 00000000..16b0c3e8 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/Duration.scala @@ -0,0 +1,54 @@ +package docspell.common + +import cats.implicits._ +import scala.concurrent.duration.{FiniteDuration, Duration => SDur} +import java.time.{Duration => JDur} +import java.util.concurrent.TimeUnit + +import cats.effect.Sync + +case class Duration(nanos: Long) { + + def millis: Long = nanos / 1000000 + + def seconds: Long = millis / 1000 + + def toScala: FiniteDuration = + FiniteDuration(nanos, TimeUnit.NANOSECONDS) + + def toJava: JDur = + JDur.ofNanos(nanos) + + def formatExact: String = + s"$millis ms" +} + +object Duration { + + def apply(d: SDur): Duration = + Duration(d.toNanos) + + def apply(d: JDur): Duration = + Duration(d.toNanos) + + def seconds(n: Long): Duration = + apply(JDur.ofSeconds(n)) + + def millis(n: Long): Duration = + apply(JDur.ofMillis(n)) + + def minutes(n: Long): Duration = + apply(JDur.ofMinutes(n)) + + def hours(n: Long): Duration = + apply(JDur.ofHours(n)) + + def nanos(n: Long): Duration = + Duration(n) + + def stopTime[F[_]: Sync]: F[F[Duration]] = + for { + now <- Timestamp.current[F] + end = Timestamp.current[F] + } yield end.map(e => Duration.millis(e.toMillis - now.toMillis)) +} diff --git a/modules/common/src/main/scala/docspell/common/IdRef.scala b/modules/common/src/main/scala/docspell/common/IdRef.scala new file mode 100644 index 00000000..d33c8535 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/IdRef.scala @@ -0,0 +1,16 @@ +package docspell.common + +import io.circe._ +import io.circe.generic.semiauto._ + +case class IdRef(id: Ident, name: String) { + +} + +object IdRef { + + implicit val jsonEncoder: Encoder[IdRef] = + deriveEncoder[IdRef] + implicit val jsonDecoder: Decoder[IdRef] = + deriveDecoder[IdRef] +} \ No newline at end of file diff --git a/modules/common/src/main/scala/docspell/common/Ident.scala b/modules/common/src/main/scala/docspell/common/Ident.scala new file mode 100644 index 00000000..11cee38d --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/Ident.scala @@ -0,0 +1,57 @@ +package docspell.common + +import java.security.SecureRandom +import java.util.UUID + +import cats.effect.Sync +import io.circe.{Decoder, Encoder} +import scodec.bits.ByteVector + +case class Ident(id: String) { +} + +object Ident { + + val chars: Set[Char] = (('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ "-_").toSet + + def randomUUID[F[_]: Sync]: F[Ident] = + Sync[F].delay(unsafe(UUID.randomUUID.toString)) + + def randomId[F[_]: Sync]: F[Ident] = Sync[F].delay { + val random = new SecureRandom() + val buffer = new Array[Byte](32) + random.nextBytes(buffer) + unsafe(ByteVector.view(buffer).toBase58.grouped(11).mkString("-")) + } + + def apply(str: String): Either[String, Ident] = + fromString(str) + + def fromString(s: String): Either[String, Ident] = + if (s.forall(chars.contains)) Right(new Ident(s)) + else Left(s"Invalid identifier: $s. Allowed chars: ${chars.mkString}") + + def fromBytes(bytes: ByteVector): Ident = + unsafe(bytes.toBase58) + + def fromByteArray(bytes: Array[Byte]): Ident = + fromBytes(ByteVector.view(bytes)) + + def unsafe(s: String): Ident = + fromString(s) match { + case Right(id) => id + case Left(err) => sys.error(err) + } + + def unapply(arg: String): Option[Ident] = + fromString(arg).toOption + + + + implicit val encodeIdent: Encoder[Ident] = + Encoder.encodeString.contramap(_.id) + + implicit val decodeIdent: Decoder[Ident] = + Decoder.decodeString.emap(Ident.fromString) + +} diff --git a/modules/common/src/main/scala/docspell/common/ItemState.scala b/modules/common/src/main/scala/docspell/common/ItemState.scala new file mode 100644 index 00000000..4f4375d2 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/ItemState.scala @@ -0,0 +1,35 @@ +package docspell.common + +import io.circe.{Decoder, Encoder} + +sealed trait ItemState { self: Product => + + final def name: String = + productPrefix.toLowerCase +} + +object ItemState { + + case object Premature extends ItemState + case object Processing extends ItemState + case object Created extends ItemState + case object Confirmed extends ItemState + + def fromString(str: String): Either[String, ItemState] = + str.toLowerCase match { + case "premature" => Right(Premature) + case "processing" => Right(Processing) + case "created" => Right(Created) + case "confirmed" => Right(Confirmed) + case _ => Left(s"Invalid item state: $str") + } + + def unsafe(str: String): ItemState = + fromString(str).fold(sys.error, identity) + + implicit val jsonDecoder: Decoder[ItemState] = + Decoder.decodeString.emap(fromString) + implicit val jsonEncoder: Encoder[ItemState] = + Encoder.encodeString.contramap(_.name) +} + diff --git a/modules/common/src/main/scala/docspell/common/JobState.scala b/modules/common/src/main/scala/docspell/common/JobState.scala new file mode 100644 index 00000000..e274546a --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/JobState.scala @@ -0,0 +1,70 @@ +package docspell.common + +import io.circe.{Decoder, Encoder} + +sealed trait JobState { self: Product => + def name: String = + productPrefix.toLowerCase +} + +object JobState { + + /** Waiting for being executed. */ + case object Waiting extends JobState { + } + + /** A scheduler has picked up this job and will pass it to the next + * free slot. */ + case object Scheduled extends JobState { + } + + /** Is currently executing */ + case object Running extends JobState { + } + + /** Finished with failure and is being retried. */ + case object Stuck extends JobState { + } + + /** Finished finally with a failure */ + case object Failed extends JobState { + } + + /** Finished by cancellation. */ + case object Cancelled extends JobState { + } + + /** Finished with success */ + case object Success extends JobState { + } + + val all: Set[JobState] = Set(Waiting, Scheduled, Running, Stuck, Failed, Cancelled, Success) + val queued: Set[JobState] = Set(Waiting, Scheduled, Stuck) + val done: Set[JobState] = Set(Failed, Cancelled, Success) + + def parse(str: String): Either[String, JobState] = + str.toLowerCase match { + case "waiting" => Right(Waiting) + case "scheduled" => Right(Scheduled) + case "running" => Right(Running) + case "stuck" => Right(Stuck) + case "failed" => Right(Failed) + case "cancelled" => Right(Cancelled) + case "success" => Right(Success) + case _ => Left(s"Not a job state: $str") + } + + def unsafe(str: String): JobState = + parse(str).fold(sys.error, identity) + + def asString(state: JobState): String = + state.name + + + implicit val jobStateEncoder: Encoder[JobState] = + Encoder.encodeString.contramap(_.name) + + implicit val jobStateDecoder: Decoder[JobState] = + Decoder.decodeString.emap(JobState.parse) + +} diff --git a/modules/common/src/main/scala/docspell/common/Language.scala b/modules/common/src/main/scala/docspell/common/Language.scala new file mode 100644 index 00000000..f82b3622 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/Language.scala @@ -0,0 +1,46 @@ +package docspell.common + +import io.circe.{Decoder, Encoder} + +sealed trait Language { self: Product => + + final def name: String = + productPrefix.toLowerCase + + def iso2: String + + def iso3: String + + private[common] def allNames = + Set(name, iso3, iso2) +} + +object Language { + + case object German extends Language { + val iso2 = "de" + val iso3 = "deu" + } + + case object English extends Language { + val iso2 = "en" + val iso3 = "eng" + } + + val all: List[Language] = List(German, English) + + def fromString(str: String): Either[String, Language] = { + val lang = str.toLowerCase + all.find(_.allNames.contains(lang)). + toRight(s"Unsupported or invalid language: $str") + } + + def unsafe(str: String): Language = + fromString(str).fold(sys.error, identity) + + + implicit val jsonDecoder: Decoder[Language] = + Decoder.decodeString.emap(fromString) + implicit val jsonEncoder: Encoder[Language] = + Encoder.encodeString.contramap(_.iso3) +} diff --git a/modules/common/src/main/scala/docspell/common/LenientUri.scala b/modules/common/src/main/scala/docspell/common/LenientUri.scala new file mode 100644 index 00000000..6754e013 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/LenientUri.scala @@ -0,0 +1,186 @@ +package docspell.common + +import java.net.URL + +import fs2.Stream +import cats.implicits._ +import cats.data.NonEmptyList +import cats.effect.{Blocker, ContextShift, Sync} +import docspell.common.LenientUri.Path +import io.circe.{Decoder, Encoder} + +/** A URI. + * + * It is not compliant to rfc3986, but covers most use cases in a convenient way. + */ +case class LenientUri(scheme: NonEmptyList[String] + , authority: Option[String] + , path: LenientUri.Path + , query: Option[String] + , fragment: Option[String]) { + + def /(segment: String): LenientUri = + copy(path = path / segment) + + def ++ (np: Path): LenientUri = + copy(path = np.segments.foldLeft(path)(_ / _)) + + def ++ (np: String): LenientUri = { + val rel = LenientUri.stripLeading(np, '/') + ++(LenientUri.unsafe(s"a:$rel").path) + } + + def toJavaUrl: Either[String, URL] = + Either.catchNonFatal(new URL(asString)).left.map(_.getMessage) + + def readURL[F[_]: Sync : ContextShift](chunkSize: Int, blocker: Blocker): Stream[F, Byte] = + Stream.emit(Either.catchNonFatal(new URL(asString))). + covary[F]. + rethrow. + flatMap(url => fs2.io.readInputStream(Sync[F].delay(url.openStream()), chunkSize, blocker, true)) + + def asString: String = { + val schemePart = scheme.toList.mkString(":") + val authPart = authority.map(a => s"//$a").getOrElse("") + val pathPart = path.asString + val queryPart = query.map(q => s"?$q").getOrElse("") + val fragPart = fragment.map(f => s"#$f").getOrElse("") + s"$schemePart:$authPart$pathPart$queryPart$fragPart" + } +} + +object LenientUri { + + sealed trait Path { + def segments: List[String] + def isRoot: Boolean + def isEmpty: Boolean + def /(segment: String): Path + def asString: String + } + case object RootPath extends Path { + val segments = Nil + val isRoot = true + val isEmpty = false + def /(seg: String): Path = + NonEmptyPath(NonEmptyList.of(seg)) + def asString = "/" + } + case object EmptyPath extends Path { + val segments = Nil + val isRoot = false + val isEmpty = true + def /(seg: String): Path = + NonEmptyPath(NonEmptyList.of(seg)) + def asString = "" + } + case class NonEmptyPath(segs: NonEmptyList[String]) extends Path { + def segments = segs.toList + val isEmpty = false + val isRoot = false + def /(seg: String): Path = + copy(segs = segs.append(seg)) + def asString = segs.head match { + case "." => segments.map(percentEncode).mkString("/") + case ".." => segments.map(percentEncode).mkString("/") + case _ => "/" + segments.map(percentEncode).mkString("/") + } + } + + def unsafe(str: String): LenientUri = + parse(str).fold(sys.error, identity) + + def fromJava(u: URL): LenientUri = + unsafe(u.toExternalForm) + + def parse(str: String): Either[String, LenientUri] = { + def makePath(str: String): Path = str.trim match { + case "/" => RootPath + case "" => EmptyPath + case _ => NonEmptyList.fromList(stripLeading(str, '/').split('/').toList.map(percentDecode)) match { + case Some(nl) => NonEmptyPath(nl) + case None => sys.error(s"Invalid url: $str") + } + } + + def makeNonEmpty(str: String): Option[String] = + Option(str).filter(_.nonEmpty) + def makeScheme(s: String): Option[NonEmptyList[String]] = + NonEmptyList.fromList(s.split(':').toList.filter(_.nonEmpty).map(_.toLowerCase)) + + def splitPathQF(pqf: String): (Path, Option[String], Option[String]) = + pqf.indexOf('?') match { + case -1 => + pqf.indexOf('#') match { + case -1 => + (makePath(pqf), None, None) + case n => + (makePath(pqf.substring(0, n)), None, makeNonEmpty(pqf.substring(n + 1))) + } + case n => + pqf.indexOf('#', n) match { + case -1 => + (makePath(pqf.substring(0, n)), makeNonEmpty(pqf.substring(n+1)), None) + case k => + (makePath(pqf.substring(0, n)), makeNonEmpty(pqf.substring(n+1, k)), makeNonEmpty(pqf.substring(k+1))) + } + } + + str.split("//", 2) match { + case Array(p0, p1) => + // scheme:scheme:authority/path + val scheme = makeScheme(p0) + val (auth, pathQF) = p1.indexOf('/') match { + case -1 => (Some(p1), "") + case n => (Some(p1.substring(0, n)), p1.substring(n)) + } + val (path, query, frag) = splitPathQF(pathQF) + scheme match { + case None => + Left(s"No scheme found: $str") + case Some(nl) => + Right(LenientUri(nl, auth, path, query, frag)) + } + case Array(p0) => + // scheme:scheme:path + p0.lastIndexOf(':') match { + case -1 => + Left(s"No scheme found: $str") + case n => + val scheme = makeScheme(p0.substring(0, n)) + val (path, query, frag) = splitPathQF(p0.substring(n + 1)) + scheme match { + case None => + Left(s"No scheme found: $str") + case Some(nl) => + Right(LenientUri(nl, None, path, query, frag)) + } + } + } + } + + private[this] val delims: Set[Char] = ":/?#[]@".toSet + + private def percentEncode(s: String): String = + s.flatMap(c => if (delims.contains(c)) s"%${c.toInt.toHexString}" else c.toString) + + private def percentDecode(s: String): String = + if (!s.contains("%")) s + else s.foldLeft(("", "")) { case ((acc, res), c) => + if (acc.length == 2) ("", res :+ Integer.parseInt(acc.drop(1) :+ c, 16).toChar) + else if (acc.startsWith("%")) (acc :+ c, res) + else if (c == '%') ("%", res) + else (acc, res :+ c) + }._2 + + private def stripLeading(s: String, c: Char): String = + if (s.length > 0 && s.charAt(0) == c) s.substring(1) + else s + + + implicit val encodeLenientUri: Encoder[LenientUri] = + Encoder.encodeString.contramap(_.asString) + + implicit val decodeLenientUri: Decoder[LenientUri] = + Decoder.decodeString.emap(LenientUri.parse) +} diff --git a/modules/common/src/main/scala/docspell/common/LogLevel.scala b/modules/common/src/main/scala/docspell/common/LogLevel.scala new file mode 100644 index 00000000..6cd5cbc8 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/LogLevel.scala @@ -0,0 +1,44 @@ +package docspell.common + +import io.circe.{Decoder, Encoder} + +sealed trait LogLevel { self: Product => + def toInt: Int + final def name: String = + productPrefix.toLowerCase +} + +object LogLevel { + + case object Debug extends LogLevel { val toInt = 0 } + case object Info extends LogLevel { val toInt = 1 } + case object Warn extends LogLevel { val toInt = 2 } + case object Error extends LogLevel { val toInt = 3 } + + def fromInt(n: Int): LogLevel = + n match { + case 0 => Debug + case 1 => Info + case 2 => Warn + case 3 => Error + case _ => Debug + } + + def fromString(str: String): Either[String, LogLevel] = + str.toLowerCase match { + case "debug" => Right(Debug) + case "info" => Right(Info) + case "warn" => Right(Warn) + case "warning" => Right(Warn) + case "error" => Right(Error) + case _ => Left(s"Invalid log-level: $str") + } + + def unsafeString(str: String): LogLevel = + fromString(str).fold(sys.error, identity) + + implicit val jsonDecoder: Decoder[LogLevel] = + Decoder.decodeString.emap(fromString) + implicit val jsonEncoder: Encoder[LogLevel] = + Encoder.encodeString.contramap(_.name) +} \ No newline at end of file diff --git a/modules/common/src/main/scala/docspell/common/MetaProposal.scala b/modules/common/src/main/scala/docspell/common/MetaProposal.scala new file mode 100644 index 00000000..6c35cf9b --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/MetaProposal.scala @@ -0,0 +1,44 @@ +package docspell.common + +import cats.data.NonEmptyList +import docspell.common.MetaProposal.Candidate +import io.circe._ +import io.circe.generic.semiauto._ + +case class MetaProposal(proposalType: MetaProposalType, values: NonEmptyList[Candidate]) { + + def addIdRef(refs: Seq[Candidate]): MetaProposal = + copy(values = MetaProposal.flatten(values ++ refs.toList)) + + def isSingleValue: Boolean = + values.tail.isEmpty + + def isMultiValue: Boolean = + !isSingleValue + + def size: Int = + values.size +} + +object MetaProposal { + + case class Candidate(ref: IdRef, origin: Set[NerLabel]) + object Candidate { + implicit val jsonEncoder: Encoder[Candidate] = + deriveEncoder[Candidate] + implicit val jsonDecoder: Decoder[Candidate] = + deriveDecoder[Candidate] + } + + def flatten(s: NonEmptyList[Candidate]): NonEmptyList[Candidate] = { + def append(list: List[Candidate]): Candidate = + list.reduce((l0, l1) => l0.copy(origin = l0.origin ++ l1.origin)) + val grouped = s.toList.groupBy(_.ref.id) + NonEmptyList.fromListUnsafe(grouped.values.toList.map(append)) + } + + implicit val jsonDecoder: Decoder[MetaProposal] = + deriveDecoder[MetaProposal] + implicit val jsonEncoder: Encoder[MetaProposal] = + deriveEncoder[MetaProposal] +} diff --git a/modules/common/src/main/scala/docspell/common/MetaProposalList.scala b/modules/common/src/main/scala/docspell/common/MetaProposalList.scala new file mode 100644 index 00000000..c2343aa0 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/MetaProposalList.scala @@ -0,0 +1,82 @@ +package docspell.common + +import cats.data.NonEmptyList +import cats.kernel.Monoid +import docspell.common.MetaProposal.Candidate +import io.circe._ +import io.circe.generic.semiauto._ + +case class MetaProposalList private (proposals: List[MetaProposal]) { + + def isEmpty: Boolean = proposals.isEmpty + def nonEmpty: Boolean = proposals.nonEmpty + + def hasResults(mt: MetaProposalType, mts: MetaProposalType*): Boolean = { + (mts :+ mt).map(mtp => proposals.exists(_.proposalType == mtp)). + reduce(_ && _) + } + + def hasResultsAll: Boolean = + proposals.map(_.proposalType).toSet == MetaProposalType.all.toSet + + def getTypes: Set[MetaProposalType] = + proposals.foldLeft(Set.empty[MetaProposalType])(_ + _.proposalType) + + def fillEmptyFrom(ml: MetaProposalList): MetaProposalList = { + val list = ml.proposals.foldLeft(proposals){ (mine, mp) => + if (hasResults(mp.proposalType)) mine + else mp :: mine + } + new MetaProposalList(list) + } + + def find(mpt: MetaProposalType): Option[MetaProposal] = + proposals.find(_.proposalType == mpt) + +} + +object MetaProposalList { + val empty = MetaProposalList(Nil) + + def apply(lmp: List[MetaProposal]): MetaProposalList = + flatten(lmp.map(m => new MetaProposalList(List(m)))) + + def of(mps: MetaProposal*): MetaProposalList = + flatten(mps.toList.map(mp => MetaProposalList(List(mp)))) + + def from(mt: MetaProposalType, label: NerLabel)(refs: Seq[IdRef]): MetaProposalList = + fromSeq1(mt, refs.map(ref => Candidate(ref, Set(label)))) + + def fromSeq1(mt: MetaProposalType, refs: Seq[Candidate]): MetaProposalList = + NonEmptyList.fromList(refs.toList). + map(nl => MetaProposalList.of(MetaProposal(mt, nl))). + getOrElse(empty) + + def fromMap(m: Map[MetaProposalType, MetaProposal]): MetaProposalList = { + new MetaProposalList(m.toList.map({ case (k, v) => v.copy(proposalType = k) })) + } + + def flatten(ml: Seq[MetaProposalList]): MetaProposalList = { + val init: Map[MetaProposalType, MetaProposal] = Map.empty + + def updateMap(map: Map[MetaProposalType, MetaProposal], mp: MetaProposal): Map[MetaProposalType, MetaProposal] = + map.get(mp.proposalType) match { + case Some(mp0) => map.updated(mp.proposalType, mp0.addIdRef(mp.values.toList)) + case None => map.updated(mp.proposalType, mp) + } + + val merged = ml.foldLeft(init) { (map, el) => + el.proposals.foldLeft(map)(updateMap) + } + + fromMap(merged) + } + + implicit val jsonEncoder: Encoder[MetaProposalList] = + deriveEncoder[MetaProposalList] + implicit val jsonDecoder: Decoder[MetaProposalList] = + deriveDecoder[MetaProposalList] + + implicit val metaProposalListMonoid: Monoid[MetaProposalList] = + Monoid.instance(empty, (m0, m1) => flatten(Seq(m0, m1))) +} diff --git a/modules/common/src/main/scala/docspell/common/MetaProposalType.scala b/modules/common/src/main/scala/docspell/common/MetaProposalType.scala new file mode 100644 index 00000000..93dfb168 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/MetaProposalType.scala @@ -0,0 +1,41 @@ +package docspell.common + +import io.circe._ + +sealed trait MetaProposalType { self: Product => + + final def name: String = + productPrefix.toLowerCase +} + +object MetaProposalType { + + case object CorrOrg extends MetaProposalType + case object CorrPerson extends MetaProposalType + case object ConcPerson extends MetaProposalType + case object ConcEquip extends MetaProposalType + case object DocDate extends MetaProposalType + case object DueDate extends MetaProposalType + + val all: List[MetaProposalType] = + List(CorrOrg, CorrPerson, ConcPerson, ConcEquip) + + def fromString(str: String): Either[String, MetaProposalType] = + str.toLowerCase match { + case "corrorg" => Right(CorrOrg) + case "corrperson" => Right(CorrPerson) + case "concperson" => Right(ConcPerson) + case "concequip" => Right(ConcEquip) + case "docdate" => Right(DocDate) + case "duedate" => Right(DueDate) + case _ => Left(s"Invalid item-proposal-type: $str") + } + + def unsafe(str: String): MetaProposalType = + fromString(str).fold(sys.error, identity) + + implicit val jsonDecoder: Decoder[MetaProposalType] = + Decoder.decodeString.emap(fromString) + implicit val jsonEncoder: Encoder[MetaProposalType] = + Encoder.encodeString.contramap(_.name) +} diff --git a/modules/common/src/main/scala/docspell/common/MimeType.scala b/modules/common/src/main/scala/docspell/common/MimeType.scala new file mode 100644 index 00000000..b111dd96 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/MimeType.scala @@ -0,0 +1,62 @@ +package docspell.common + +import docspell.common.syntax.all._ +import io.circe.{Decoder, Encoder} + +/** A MIME Type impl with just enough features for the use here. + */ +case class MimeType(primary: String, sub: String) { + + def asString: String = + s"$primary/$sub" + + def matches(other: MimeType): Boolean = + primary == other.primary && + (sub == other.sub || sub == "*" ) +} + +object MimeType { + + def application(sub: String): MimeType = + MimeType("application", partFromString(sub).throwLeft) + + def text(sub: String): MimeType = + MimeType("text", partFromString(sub).throwLeft) + + def image(sub: String): MimeType = + MimeType("image", partFromString(sub).throwLeft) + + private[this] val validChars: Set[Char] = (('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ "*-").toSet + + def parse(str: String): Either[String, MimeType] = { + str.indexOf('/') match { + case -1 => Left(s"Invalid MIME type: $str") + case n => + for { + prim <- partFromString(str.substring(0, n)) + sub <- partFromString(str.substring(n + 1)) + } yield MimeType(prim.toLowerCase, sub.toLowerCase) + } + } + + def unsafe(str: String): MimeType = + parse(str).throwLeft + + private def partFromString(s: String): Either[String, String] = + if (s.forall(validChars.contains)) Right(s) + else Left(s"Invalid identifier: $s. Allowed chars: ${validChars.mkString}") + + val octetStream = application("octet-stream") + val pdf = application("pdf") + val png = image("png") + val jpeg = image("jpeg") + val tiff = image("tiff") + val html = text("html") + val plain = text("plain") + + implicit val jsonEncoder: Encoder[MimeType] = + Encoder.encodeString.contramap(_.asString) + + implicit val jsonDecoder: Decoder[MimeType] = + Decoder.decodeString.emap(parse) +} diff --git a/modules/common/src/main/scala/docspell/common/NerDateLabel.scala b/modules/common/src/main/scala/docspell/common/NerDateLabel.scala new file mode 100644 index 00000000..3e350e62 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/NerDateLabel.scala @@ -0,0 +1,7 @@ +package docspell.common + +import java.time.LocalDate + +case class NerDateLabel(date: LocalDate, label: NerLabel) { + +} diff --git a/modules/common/src/main/scala/docspell/common/NerLabel.scala b/modules/common/src/main/scala/docspell/common/NerLabel.scala new file mode 100644 index 00000000..27ee7a62 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/NerLabel.scala @@ -0,0 +1,13 @@ +package docspell.common + +import io.circe.generic.semiauto._ +import io.circe.{Decoder, Encoder} + +case class NerLabel(label: String, tag: NerTag, startPosition: Int, endPosition: Int) { + +} + +object NerLabel { + implicit val jsonEncoder: Encoder[NerLabel] = deriveEncoder[NerLabel] + implicit val jsonDecoder: Decoder[NerLabel] = deriveDecoder[NerLabel] +} diff --git a/modules/common/src/main/scala/docspell/common/NerTag.scala b/modules/common/src/main/scala/docspell/common/NerTag.scala new file mode 100644 index 00000000..39413ccc --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/NerTag.scala @@ -0,0 +1,43 @@ +package docspell.common + +import io.circe.{Decoder, Encoder} + +sealed trait NerTag { self: Product => + + final def name: String = + productPrefix.toLowerCase +} + +object NerTag { + + case object Organization extends NerTag + case object Person extends NerTag + case object Location extends NerTag + case object Misc extends NerTag + case object Email extends NerTag + case object Website extends NerTag + case object Date extends NerTag + + val all: List[NerTag] = List(Organization, Person, Location) + + def fromString(str: String): Either[String, NerTag] = + str.toLowerCase match { + case "organization" => Right(Organization) + case "person" => Right(Person) + case "location" => Right(Location) + case "misc" => Right(Misc) + case "email" => Right(Email) + case "website" => Right(Website) + case "date" => Right(Date) + case _ => Left(s"Invalid ner tag: $str") + } + + def unsafe(str: String): NerTag = + fromString(str).fold(sys.error, identity) + + + implicit val jsonDecoder: Decoder[NerTag] = + Decoder.decodeString.emap(fromString) + implicit val jsonEncoder: Encoder[NerTag] = + Encoder.encodeString.contramap(_.name) +} diff --git a/modules/common/src/main/scala/docspell/common/NodeType.scala b/modules/common/src/main/scala/docspell/common/NodeType.scala new file mode 100644 index 00000000..b060f100 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/NodeType.scala @@ -0,0 +1,25 @@ +package docspell.common + +sealed trait NodeType { self: Product => + + final def name: String = + self.productPrefix.toLowerCase + +} + +object NodeType { + + case object Restserver extends NodeType + case object Joex extends NodeType + + def fromString(str: String): Either[String, NodeType] = + str.toLowerCase match { + case "restserver" => Right(Restserver) + case "joex" => Right(Joex) + case _ => Left(s"Invalid node type: $str") + } + + def unsafe(str: String): NodeType = + fromString(str).fold(sys.error, identity) + +} diff --git a/modules/common/src/main/scala/docspell/common/Password.scala b/modules/common/src/main/scala/docspell/common/Password.scala new file mode 100644 index 00000000..73ba9b02 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/Password.scala @@ -0,0 +1,27 @@ +package docspell.common + +import io.circe.{Decoder, Encoder} + +final class Password(val pass: String) extends AnyVal { + + def isEmpty: Boolean= pass.isEmpty + + override def toString: String = + if (pass.isEmpty) "" else "***" + +} + +object Password { + + val empty = Password("") + + def apply(pass: String): Password = + new Password(pass) + + implicit val passwordEncoder: Encoder[Password] = + Encoder.encodeString.contramap(_.pass) + + implicit val passwordDecoder: Decoder[Password] = + Decoder.decodeString.map(Password(_)) + +} diff --git a/modules/common/src/main/scala/docspell/common/Priority.scala b/modules/common/src/main/scala/docspell/common/Priority.scala new file mode 100644 index 00000000..9d1db712 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/Priority.scala @@ -0,0 +1,48 @@ +package docspell.common + +import cats.implicits._ +import cats.Order +import io.circe.{Decoder, Encoder} + +sealed trait Priority { self: Product => + + final def name: String = + productPrefix.toLowerCase +} + +object Priority { + + case object High extends Priority + + case object Low extends Priority + + + def fromString(str: String): Either[String, Priority] = + str.toLowerCase match { + case "high" => Right(High) + case "low" => Right(Low) + case _ => Left(s"Invalid priority: $str") + } + + def unsafe(str: String): Priority = + fromString(str).fold(sys.error, identity) + + + def fromInt(n: Int): Priority = + if (n <= toInt(Low)) Low + else High + + def toInt(p: Priority): Int = + p match { + case Low => 0 + case High => 10 + } + + implicit val priorityOrder: Order[Priority] = + Order.by[Priority, Int](toInt) + + implicit val jsonEncoder: Encoder[Priority] = + Encoder.encodeString.contramap(_.name) + implicit val jsonDecoder: Decoder[Priority] = + Decoder.decodeString.emap(fromString) +} diff --git a/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala new file mode 100644 index 00000000..808449dd --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/ProcessItemArgs.scala @@ -0,0 +1,47 @@ +package docspell.common + +import io.circe._, io.circe.generic.semiauto._ +import docspell.common.syntax.all._ +import ProcessItemArgs._ + +case class ProcessItemArgs(meta: ProcessMeta, files: List[File]) { + + def makeSubject: String = { + files.flatMap(_.name) match { + case Nil => s"${meta.sourceAbbrev}: No files" + case n :: Nil => n + case n1 :: n2 :: Nil => s"$n1, $n2" + case more => s"${files.size} files from ${meta.sourceAbbrev}" + } + } + +} + +object ProcessItemArgs { + + val taskName = Ident.unsafe("process-item") + + case class ProcessMeta( collective: Ident + , language: Language + , direction: Option[Direction] + , sourceAbbrev: String + , validFileTypes: Seq[MimeType]) + + object ProcessMeta { + implicit val jsonEncoder: Encoder[ProcessMeta] = deriveEncoder[ProcessMeta] + implicit val jsonDecoder: Decoder[ProcessMeta] = deriveDecoder[ProcessMeta] + } + + case class File(name: Option[String], fileMetaId: Ident) + object File { + implicit val jsonEncoder: Encoder[File] = deriveEncoder[File] + implicit val jsonDecoder: Decoder[File] = deriveDecoder[File] + } + + implicit val jsonEncoder: Encoder[ProcessItemArgs] = deriveEncoder[ProcessItemArgs] + implicit val jsonDecoder: Decoder[ProcessItemArgs] = deriveDecoder[ProcessItemArgs] + + def parse(str: String): Either[Throwable, ProcessItemArgs] = + str.parseJsonAs[ProcessItemArgs] + +} diff --git a/modules/common/src/main/scala/docspell/common/ThreadFactories.scala b/modules/common/src/main/scala/docspell/common/ThreadFactories.scala new file mode 100644 index 00000000..c1ab24df --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/ThreadFactories.scala @@ -0,0 +1,20 @@ +package docspell.common + +import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.{Executors, ThreadFactory} + +object ThreadFactories { + + def ofName(prefix: String): ThreadFactory = + new ThreadFactory { + + val counter = new AtomicLong(0) + + override def newThread(r: Runnable): Thread = { + val t = Executors.defaultThreadFactory().newThread(r) + t.setName(s"$prefix-${counter.getAndIncrement()}") + t + } + } + +} diff --git a/modules/common/src/main/scala/docspell/common/Timestamp.scala b/modules/common/src/main/scala/docspell/common/Timestamp.scala new file mode 100644 index 00000000..b35d5210 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/Timestamp.scala @@ -0,0 +1,41 @@ +package docspell.common + +import java.time.{Instant, LocalDate, ZoneId} + +import cats.effect.Sync +import io.circe.{Decoder, Encoder} + +case class Timestamp(value: Instant) { + + def toMillis: Long = value.toEpochMilli + + def toSeconds: Long = value.toEpochMilli / 1000L + + def minus(d: Duration): Timestamp = + Timestamp(value.minusNanos(d.nanos)) + + def minusHours(n: Long): Timestamp = + Timestamp(value.minusSeconds(n * 60 * 60)) + + def toDate: LocalDate = + value.atZone(ZoneId.of("UTC")).toLocalDate + + def asString: String = value.toString +} + +object Timestamp { + + val Epoch = Timestamp(Instant.EPOCH) + + def current[F[_]: Sync]: F[Timestamp] = + Sync[F].delay(Timestamp(Instant.now)) + + + + implicit val encodeTimestamp: Encoder[Timestamp] = + BaseJsonCodecs.encodeInstantEpoch.contramap(_.value) + + implicit val decodeTimestamp: Decoder[Timestamp] = + BaseJsonCodecs.decodeInstantEpoch.map(Timestamp(_)) + +} diff --git a/modules/common/src/main/scala/docspell/common/UserState.scala b/modules/common/src/main/scala/docspell/common/UserState.scala new file mode 100644 index 00000000..9424a46c --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/UserState.scala @@ -0,0 +1,37 @@ +package docspell.common + +import io.circe.{Decoder, Encoder} + +sealed trait UserState +object UserState { + val all = List(Active, Disabled) + + /** An active or enabled user. */ + case object Active extends UserState + + /** The user is blocked by an admin. */ + case object Disabled extends UserState + + + def fromString(s: String): Either[String, UserState] = + s.toLowerCase match { + case "active" => Right(Active) + case "disabled" => Right(Disabled) + case _ => Left(s"Not a state value: $s") + } + + def unsafe(str: String): UserState = + fromString(str).fold(sys.error, identity) + + def asString(s: UserState): String = s match { + case Active => "active" + case Disabled => "disabled" + } + + implicit val userStateEncoder: Encoder[UserState] = + Encoder.encodeString.contramap(UserState.asString) + + implicit val userStateDecoder: Decoder[UserState] = + Decoder.decodeString.emap(UserState.fromString) + +} \ No newline at end of file diff --git a/modules/common/src/main/scala/docspell/common/pureconfig/Implicits.scala b/modules/common/src/main/scala/docspell/common/pureconfig/Implicits.scala new file mode 100644 index 00000000..4d48454e --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/pureconfig/Implicits.scala @@ -0,0 +1,35 @@ +package docspell.common.pureconfig + +import docspell.common._ +import _root_.pureconfig._ +import _root_.pureconfig.error.{CannotConvert, FailureReason} +import scodec.bits.ByteVector + +import scala.reflect.ClassTag + +object Implicits { + implicit val lenientUriReader: ConfigReader[LenientUri] = + ConfigReader[String].emap(reason(LenientUri.parse)) + + implicit val durationReader: ConfigReader[Duration] = + ConfigReader[scala.concurrent.duration.Duration].map(sd => Duration(sd)) + + implicit val passwordReader: ConfigReader[Password] = + ConfigReader[String].map(Password(_)) + + implicit val mimeTypeReader: ConfigReader[MimeType] = + ConfigReader[String].emap(reason(MimeType.parse)) + + implicit val identReader: ConfigReader[Ident] = + ConfigReader[String].emap(reason(Ident.fromString)) + + implicit val byteVectorReader: ConfigReader[ByteVector] = + ConfigReader[String].emap(reason(str => { + if (str.startsWith("hex:")) ByteVector.fromHex(str.drop(4)).toRight("Invalid hex value.") + else if (str.startsWith("b64:")) ByteVector.fromBase64(str.drop(4)).toRight("Invalid Base64 string.") + else ByteVector.fromHex(str).toRight("Invalid hex value.") + })) + + def reason[A: ClassTag](f: String => Either[String, A]): String => Either[FailureReason, A] = + in => f(in).left.map(str => CannotConvert(in, implicitly[ClassTag[A]].runtimeClass.toString, str)) +} \ No newline at end of file diff --git a/modules/common/src/main/scala/docspell/common/syntax/EitherSyntax.scala b/modules/common/src/main/scala/docspell/common/syntax/EitherSyntax.scala new file mode 100644 index 00000000..282f9c32 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/syntax/EitherSyntax.scala @@ -0,0 +1,21 @@ +package docspell.common.syntax + +trait EitherSyntax { + + implicit final class LeftStringEitherOps[A](e: Either[String, A]) { + def throwLeft: A = e match { + case Right(a) => a + case Left(err) => sys.error(err) + } + } + + implicit final class ThrowableLeftEitherOps[A](e: Either[Throwable, A]) { + def throwLeft: A = e match { + case Right(a) => a + case Left(err) => throw err + } + } + +} + +object EitherSyntax extends EitherSyntax \ No newline at end of file diff --git a/modules/common/src/main/scala/docspell/common/syntax/LoggerSyntax.scala b/modules/common/src/main/scala/docspell/common/syntax/LoggerSyntax.scala new file mode 100644 index 00000000..35d8ce4d --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/syntax/LoggerSyntax.scala @@ -0,0 +1,35 @@ +package docspell.common.syntax + +import cats.effect.Sync +import fs2.Stream +import org.log4s.Logger + +trait LoggerSyntax { + + implicit final class LoggerOps(logger: Logger) { + + def ftrace[F[_]: Sync](msg: => String): F[Unit] = + Sync[F].delay(logger.trace(msg)) + + def fdebug[F[_]: Sync](msg: => String): F[Unit] = + Sync[F].delay(logger.debug(msg)) + + def sdebug[F[_]: Sync](msg: => String): Stream[F, Nothing] = + Stream.eval(fdebug(msg)).drain + + def finfo[F[_]: Sync](msg: => String): F[Unit] = + Sync[F].delay(logger.info(msg)) + + def sinfo[F[_]: Sync](msg: => String): Stream[F, Nothing] = + Stream.eval(finfo(msg)).drain + + def fwarn[F[_]: Sync](msg: => String): F[Unit] = + Sync[F].delay(logger.warn(msg)) + + def ferror[F[_]: Sync](msg: => String): F[Unit] = + Sync[F].delay(logger.error(msg)) + + def ferror[F[_]: Sync](ex: Throwable)(msg: => String): F[Unit] = + Sync[F].delay(logger.error(ex)(msg)) + } +} diff --git a/modules/common/src/main/scala/docspell/common/syntax/StreamSyntax.scala b/modules/common/src/main/scala/docspell/common/syntax/StreamSyntax.scala new file mode 100644 index 00000000..58f6eb26 --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/syntax/StreamSyntax.scala @@ -0,0 +1,24 @@ +package docspell.common.syntax + +import cats.effect.Sync +import fs2.Stream +import cats.implicits._ +import io.circe._ +import io.circe.parser._ + +trait StreamSyntax { + + implicit class StringStreamOps[F[_]](s: Stream[F, String]) { + + def parseJsonAs[A](implicit d: Decoder[A], F: Sync[F]): F[Either[Throwable, A]] = + s.fold("")(_ + _). + compile.last. + map(optStr => for { + str <- optStr.map(_.trim).toRight(new Exception("Empty string cannot be parsed into a value")) + json <- parse(str).leftMap(_.underlying) + value <- json.as[A] + } yield value) + + } + +} diff --git a/modules/common/src/main/scala/docspell/common/syntax/StringSyntax.scala b/modules/common/src/main/scala/docspell/common/syntax/StringSyntax.scala new file mode 100644 index 00000000..d14f602e --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/syntax/StringSyntax.scala @@ -0,0 +1,21 @@ +package docspell.common.syntax + +import cats.implicits._ +import io.circe.Decoder +import io.circe.parser._ + +trait StringSyntax { + + implicit class EvenMoreStringOps(s: String) { + + def asNonBlank: Option[String] = + Option(s).filter(_.trim.nonEmpty) + + def parseJsonAs[A](implicit d: Decoder[A]): Either[Throwable, A] = + for { + json <- parse(s).leftMap(_.underlying) + value <- json.as[A] + } yield value + } + +} diff --git a/modules/common/src/main/scala/docspell/common/syntax/package.scala b/modules/common/src/main/scala/docspell/common/syntax/package.scala new file mode 100644 index 00000000..af61799d --- /dev/null +++ b/modules/common/src/main/scala/docspell/common/syntax/package.scala @@ -0,0 +1,10 @@ +package docspell.common + +package object syntax { + + object all extends EitherSyntax + with StreamSyntax + with StringSyntax + with LoggerSyntax + +} diff --git a/modules/joex/src/main/resources/logback.xml b/modules/joex/src/main/resources/logback.xml new file mode 100644 index 00000000..c33ec1f7 --- /dev/null +++ b/modules/joex/src/main/resources/logback.xml @@ -0,0 +1,14 @@ + + + true + + + [%thread] %highlight(%-5level) %cyan(%logger{15}) - %msg %n + + + + + + + + diff --git a/modules/joex/src/main/resources/reference.conf b/modules/joex/src/main/resources/reference.conf index adf3979d..79b68912 100644 --- a/modules/joex/src/main/resources/reference.conf +++ b/modules/joex/src/main/resources/reference.conf @@ -1,3 +1,131 @@ docspell.joex { + # This is the id of this node. If you run more than one server, you + # have to make sure to provide unique ids per node. + app-id = "joex1" + + + # This is the base URL this application is deployed to. This is used + # to register this joex instance such that docspell rest servers can + # reach them + base-url = "http://localhost:7878" + + # Where the REST server binds to. + # + # JOEX provides a very simple REST interface to inspect its state. + bind { + address = "localhost" + port = 7878 + } + + # The database connection. + # + # By default a H2 file-based database is configured. You can provide + # a postgresql or mariadb connection here. When using H2 use the + # PostgreSQL compatibility mode and AUTO_SERVER feature. + # + # It must be the same connection as the rest server is using. + jdbc { + url = "jdbc:h2://"${java.io.tmpdir}"/docspell-demo.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_SERVER=TRUE" + user = "sa" + password = "" + } + + # Configuration for the job scheduler. + scheduler { + + # Each scheduler needs a unique name. This defaults to the node + # name, which must be unique, too. + name = ${docspell.joex.app-id} + + # Number of processing allowed in parallel. + pool-size = 2 + + # A counting scheme determines the ratio of how high- and low-prio + # jobs are run. For example: 4,1 means run 4 high prio jobs, then + # 1 low prio and then start over. + counting-scheme = "4,1" + + # How often a failed job should be retried until it enters failed + # state. If a job fails, it becomes "stuck" and will be retried + # after a delay. + retries = 5 + + # The delay until the next try is performed for a failed job. This + # delay is increased exponentially with the number of retries. + retry-delay = "1 minute" + + # The queue size of log statements from a job. + log-buffer-size = 500 + + # If no job is left in the queue, the scheduler will wait until a + # notify is requested (using the REST interface). To also retry + # stuck jobs, it will notify itself periodically. + wakeup-period = "30 minutes" + } + + # Configuration of text extraction + # + # Extracting text currently only work for image and pdf files. It + # will first runs ghostscript to create a gray image from a + # pdf. Then unpaper is run to optimize the image for the upcoming + # ocr, which will be done by tesseract. All these programs must be + # available in your PATH or the absolute path can be specified + # below. + extraction { + allowed-content-types = [ "application/pdf", "image/jpeg", "image/png" ] + + # Defines what pages to process. If a PDF with 600 pages is + # submitted, it is probably not necessary to scan through all of + # them. This would take a long time and occupy resources for no + # value. The first few pages should suffice. The default is first + # 10 pages. + # + # If you want all pages being processed, set this number to -1. + # + # Note: if you change the ghostscript command below, be aware that + # this setting (if not -1) will add another parameter to the + # beginning of the command. + page-range { + begin = 10 + } + + # The ghostscript command. + ghostscript { + command { + program = "gs" + args = [ "-dNOPAUSE" + , "-dBATCH" + , "-dSAFER" + , "-sDEVICE=tiffscaled8" + , "-sOutputFile={{outfile}}" + , "{{infile}}" + ] + timeout = "5 minutes" + } + working-dir = ${java.io.tmpdir}"/docspell-extraction" + } + + # The unpaper command. + unpaper { + command { + program = "unpaper" + args = [ "{{infile}}", "{{outfile}}" ] + timeout = "5 minutes" + } + } + + # The tesseract command. + tesseract { + command { + program = "tesseract" + args = ["{{file}}" + , "stdout" + , "-l" + , "{{lang}}" + ] + timeout = "5 minutes" + } + } + } } \ No newline at end of file diff --git a/modules/joex/src/main/scala/docspell/joex/Config.scala b/modules/joex/src/main/scala/docspell/joex/Config.scala index 022c5d72..037f6cbf 100644 --- a/modules/joex/src/main/scala/docspell/joex/Config.scala +++ b/modules/joex/src/main/scala/docspell/joex/Config.scala @@ -1,18 +1,21 @@ package docspell.joex +import docspell.common.{Ident, LenientUri} +import docspell.joex.scheduler.SchedulerConfig import docspell.store.JdbcConfig +import docspell.text.ocr.{Config => OcrConfig} -case class Config(id: String +case class Config(appId: Ident + , baseUrl: LenientUri , bind: Config.Bind , jdbc: JdbcConfig + , scheduler: SchedulerConfig + , extraction: OcrConfig ) object Config { - - - val default: Config = - Config("testid", Config.Bind("localhost", 7878), JdbcConfig("", "", "")) - + val postgres = JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev") + val h2 = JdbcConfig(LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"), "sa", "") case class Bind(address: String, port: Int) } diff --git a/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala b/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala new file mode 100644 index 00000000..8ed1e1d2 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/ConfigFile.scala @@ -0,0 +1,20 @@ +package docspell.joex + +import docspell.common.pureconfig.Implicits._ +import _root_.pureconfig._ +import _root_.pureconfig.generic.auto._ +import docspell.joex.scheduler.CountingScheme + +object ConfigFile { + import Implicits._ + + def loadConfig: Config = + ConfigSource.default.at("docspell.joex").loadOrThrow[Config] + + + object Implicits { + implicit val countingSchemeReader: ConfigReader[CountingScheme] = + ConfigReader[String].emap(reason(CountingScheme.readString)) + + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/JoexApp.scala b/modules/joex/src/main/scala/docspell/joex/JoexApp.scala index 246dc5d2..0bcbba25 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexApp.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexApp.scala @@ -1,6 +1,23 @@ package docspell.joex +import docspell.common.Ident +import docspell.joex.scheduler.Scheduler +import docspell.store.records.RJobLog + trait JoexApp[F[_]] { def init: F[Unit] + + def scheduler: Scheduler[F] + + def findLogs(jobId: Ident): F[Vector[RJobLog]] + + /** Shuts down the job executor. + * + * It will immediately stop taking new jobs, waiting for currently + * running jobs to complete normally (i.e. running jobs are not + * canceled). After this completed, the webserver stops and the + * main loop will exit. + */ + def initShutdown: F[Unit] } diff --git a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala index a4ea36f4..8d0c6551 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexAppImpl.scala @@ -1,16 +1,56 @@ package docspell.joex +import cats.implicits._ import cats.effect._ +import docspell.common.{Ident, NodeType, ProcessItemArgs} +import docspell.joex.process.ItemHandler +import docspell.joex.scheduler.{JobTask, Scheduler, SchedulerBuilder} +import docspell.store.Store +import docspell.store.ops.ONode +import docspell.store.records.RJobLog +import fs2.concurrent.SignallingRef -final class JoexAppImpl[F[_]: Sync](cfg: Config) extends JoexApp[F] { +import scala.concurrent.ExecutionContext - def init: F[Unit] = - Sync[F].pure(()) +final class JoexAppImpl[F[_]: ConcurrentEffect : ContextShift: Timer]( cfg: Config + , nodeOps: ONode[F] + , store: Store[F] + , termSignal: SignallingRef[F, Boolean] + , val scheduler: Scheduler[F]) extends JoexApp[F] { + + def init: F[Unit] = { + val run = scheduler.start.compile.drain + for { + _ <- ConcurrentEffect[F].start(run) + _ <- scheduler.periodicAwake + _ <- nodeOps.register(cfg.appId, NodeType.Joex, cfg.baseUrl) + } yield () + } + + def findLogs(jobId: Ident): F[Vector[RJobLog]] = + store.transact(RJobLog.findLogs(jobId)) + + def shutdown: F[Unit] = + nodeOps.unregister(cfg.appId) + + def initShutdown: F[Unit] = + scheduler.shutdown(false) *> termSignal.set(true) } object JoexAppImpl { - def create[F[_]: Sync](cfg: Config): Resource[F, JoexApp[F]] = - Resource.liftF(Sync[F].pure(new JoexAppImpl(cfg))) + def create[F[_]: ConcurrentEffect : ContextShift: Timer](cfg: Config + , termSignal: SignallingRef[F, Boolean] + , connectEC: ExecutionContext + , blocker: Blocker): Resource[F, JoexApp[F]] = + for { + store <- Store.create(cfg.jdbc, connectEC, blocker) + nodeOps <- ONode(store) + sch <- SchedulerBuilder(cfg.scheduler, blocker, store). + withTask(JobTask.json(ProcessItemArgs.taskName, ItemHandler[F](cfg.extraction), ItemHandler.onCancel[F])). + resource + app = new JoexAppImpl(cfg, nodeOps, store, termSignal, sch) + appR <- Resource.make(app.init.map(_ => app))(_.shutdown) + } yield appR } diff --git a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala index 4219d15d..502ce01e 100644 --- a/modules/joex/src/main/scala/docspell/joex/JoexServer.scala +++ b/modules/joex/src/main/scala/docspell/joex/JoexServer.scala @@ -1,37 +1,48 @@ package docspell.joex import cats.effect._ +import cats.effect.concurrent.Ref +import docspell.joex.routes._ import org.http4s.server.blaze.BlazeServerBuilder import org.http4s.implicits._ import fs2.Stream - +import fs2.concurrent.SignallingRef +import org.http4s.HttpApp import org.http4s.server.middleware.Logger import org.http4s.server.Router +import scala.concurrent.ExecutionContext + object JoexServer { - def stream[F[_]: ConcurrentEffect](cfg: Config) + + private case class App[F[_]](httpApp: HttpApp[F], termSig: SignallingRef[F, Boolean], exitRef: Ref[F, ExitCode]) + + def stream[F[_]: ConcurrentEffect : ContextShift](cfg: Config, connectEC: ExecutionContext, blocker: Blocker) (implicit T: Timer[F]): Stream[F, Nothing] = { val app = for { - joexApp <- JoexAppImpl.create[F](cfg) - _ <- Resource.liftF(joexApp.init) + signal <- Resource.liftF(SignallingRef[F, Boolean](false)) + exitCode <- Resource.liftF(Ref[F].of(ExitCode.Success)) + joexApp <- JoexAppImpl.create[F](cfg, signal, connectEC, blocker) httpApp = Router( - "/api/info" -> InfoRoutes(cfg) + "/api/info" -> InfoRoutes(cfg), + "/api/v1" -> JoexRoutes(cfg, joexApp) ).orNotFound // With Middlewares in place finalHttpApp = Logger.httpApp(false, false)(httpApp) - } yield finalHttpApp + } yield App(finalHttpApp, signal, exitCode) - Stream.resource(app).flatMap(httpApp => - BlazeServerBuilder[F] - .bindHttp(cfg.bind.port, cfg.bind.address) - .withHttpApp(httpApp) - .serve + Stream.resource(app).flatMap(app => + BlazeServerBuilder[F]. + bindHttp(cfg.bind.port, cfg.bind.address). + withHttpApp(app.httpApp). + withoutBanner. + serveWhile(app.termSig, app.exitRef) ) }.drain diff --git a/modules/joex/src/main/scala/docspell/joex/Main.scala b/modules/joex/src/main/scala/docspell/joex/Main.scala index 846c9a6c..f442652b 100644 --- a/modules/joex/src/main/scala/docspell/joex/Main.scala +++ b/modules/joex/src/main/scala/docspell/joex/Main.scala @@ -1,16 +1,23 @@ package docspell.joex -import cats.effect.{ExitCode, IO, IOApp} +import cats.effect.{Blocker, ExitCode, IO, IOApp} import cats.implicits._ + import scala.concurrent.ExecutionContext import java.util.concurrent.Executors import java.nio.file.{Files, Paths} + +import docspell.common.{Banner, ThreadFactories} import org.log4s._ object Main extends IOApp { private[this] val logger = getLogger - val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(Executors.newCachedThreadPool) + val blockingEc: ExecutionContext = ExecutionContext.fromExecutor( + Executors.newCachedThreadPool(ThreadFactories.ofName("docspell-joex-blocking"))) + val blocker = Blocker.liftExecutionContext(blockingEc) + val connectEC: ExecutionContext = ExecutionContext.fromExecutorService( + Executors.newFixedThreadPool(5, ThreadFactories.ofName("docspell-joex-dbconnect"))) def run(args: List[String]) = { args match { @@ -32,7 +39,14 @@ object Main extends IOApp { } } - val cfg = Config.default - JoexServer.stream[IO](cfg).compile.drain.as(ExitCode.Success) + val cfg = ConfigFile.loadConfig + val banner = Banner("JOEX" + , BuildInfo.version + , BuildInfo.gitHeadCommit + , cfg.jdbc.url + , Option(System.getProperty("config.file")) + , cfg.appId, cfg.baseUrl) + logger.info(s"\n${banner.render("***>")}") + JoexServer.stream[IO](cfg, connectEC, blocker).compile.drain.as(ExitCode.Success) } } diff --git a/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala new file mode 100644 index 00000000..878fa519 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/process/CreateItem.scala @@ -0,0 +1,73 @@ +package docspell.joex.process + +import cats.implicits._ +import cats.effect.Sync +import fs2.Stream +import docspell.common._ +import docspell.joex.scheduler.{Context, Task} +import docspell.store.queries.QItem +import docspell.store.records.{RAttachment, RItem} + +/** + * Task that creates the item. + */ +object CreateItem { + + def apply[F[_]: Sync]: Task[F, ProcessItemArgs, ItemData] = + findExisting[F].flatMap { + case Some(ri) => Task.pure(ri) + case None => createNew[F] + } + + def createNew[F[_]: Sync]: Task[F, ProcessItemArgs, ItemData] = + Task { ctx => + val validFiles = ctx.args.meta.validFileTypes.map(_.asString).toSet + + def fileMetas(itemId: Ident, now: Timestamp) = Stream.emits(ctx.args.files). + flatMap(f => ctx.store.bitpeace.get(f.fileMetaId.id).map(fm => (f, fm))). + collect({ case (f, Some(fm)) if validFiles.contains(fm.mimetype.baseType) => (f, fm) }). + zipWithIndex. + evalMap({ case ((f, fm), index) => + Ident.randomId[F].map(id => RAttachment(id, itemId, f.fileMetaId, index.toInt, now, f.name)) + }). + compile.toVector + + val item = RItem.newItem[F](ctx.args.meta.collective + , ctx.args.makeSubject + , ctx.args.meta.sourceAbbrev + , ctx.args.meta.direction.getOrElse(Direction.Incoming) + , ItemState.Premature) + + for { + _ <- ctx.logger.info(s"Creating new item with ${ctx.args.files.size} attachment(s)") + time <- Duration.stopTime[F] + it <- item + n <- ctx.store.transact(RItem.insert(it)) + _ <- if (n != 1) storeItemError[F](ctx) else ().pure[F] + fm <- fileMetas(it.id, it.created) + k <- fm.traverse(a => ctx.store.transact(RAttachment.insert(a))) + _ <- logDifferences(ctx, fm, k.sum) + dur <- time + _ <- ctx.logger.info(s"Creating item finished in ${dur.formatExact}") + } yield ItemData(it, fm, Vector.empty, Vector.empty) + } + + def findExisting[F[_]: Sync]: Task[F, ProcessItemArgs, Option[ItemData]] = + Task { ctx => + for { + cand <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId))) + _ <- if (cand.nonEmpty) ctx.logger.warn("Found existing item with these files.") else ().pure[F] + ht <- cand.drop(1).traverse(ri => QItem.delete(ctx.store)(ri.id, ri.cid)) + _ <- if (ht.sum > 0) ctx.logger.warn(s"Removed ${ht.sum} items with same attachments") else ().pure[F] + rms <- cand.headOption.traverse(ri => ctx.store.transact(RAttachment.findByItemAndCollective(ri.id, ri.cid))) + } yield cand.headOption.map(ri => ItemData(ri, rms.getOrElse(Vector.empty), Vector.empty, Vector.empty)) + } + + private def logDifferences[F[_]: Sync](ctx: Context[F, ProcessItemArgs], saved: Vector[RAttachment], saveCount: Int): F[Unit] = + ctx.logger.info("TODO log diffs") + + private def storeItemError[F[_]: Sync](ctx: Context[F, ProcessItemArgs]): F[Unit] = { + val msg = "Inserting item failed. DB returned 0 update count!" + ctx.logger.error(msg) *> Sync[F].raiseError(new Exception(msg)) + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala b/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala new file mode 100644 index 00000000..5fc47f18 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/process/FindProposal.scala @@ -0,0 +1,181 @@ +package docspell.joex.process + +import java.time.ZoneId + +import cats.{Applicative, FlatMap} +import cats.implicits._ +import cats.effect.Sync +import docspell.common.MetaProposal.Candidate +import docspell.common._ +import docspell.joex.scheduler.{Context, Task} +import docspell.store.records.{RAttachmentMeta, REquipment, ROrganization, RPerson} +import docspell.text.contact.Domain + +/** Super simple approach to find corresponding meta data to an item + * by looking up values from NER in the users address book. + * + */ +object FindProposal { + + def apply[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] = + Task { ctx => + val rmas = data.metas.map(rm => + rm.copy(nerlabels = removeDuplicates(rm.nerlabels))) + + ctx.logger.info("Starting find-proposal") *> + rmas.traverse(rm => processAttachment(rm, data.findDates(rm), ctx).map(ml => rm.copy(proposals = ml))). + flatMap(rmv => rmv.traverse(rm => + ctx.logger.debug(s"Storing attachment proposals: ${rm.proposals}") *> + ctx.store.transact(RAttachmentMeta.updateProposals(rm.id, rm.proposals))). + map(_ => data.copy(metas = rmv))) + } + + def processAttachment[F[_]: Sync]( rm: RAttachmentMeta + , rd: Vector[NerDateLabel] + , ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = { + val finder = Finder.searchExact(ctx).next(Finder.searchFuzzy(ctx)) + List(finder.find(rm.nerlabels), makeDateProposal(rd)). + traverse(identity).map(MetaProposalList.flatten) + } + + def makeDateProposal[F[_]: Sync](dates: Vector[NerDateLabel]): F[MetaProposalList] = { + Timestamp.current[F].map { now => + val latestFirst = dates.sortWith(_.date isAfter _.date) + val nowDate = now.value.atZone(ZoneId.of("GMT")).toLocalDate + val (after, before) = latestFirst.span(ndl => ndl.date.isAfter(nowDate)) + + val dueDates = MetaProposalList.fromSeq1(MetaProposalType.DueDate, + after.map(ndl => Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label)))) + val itemDates = MetaProposalList.fromSeq1(MetaProposalType.DocDate, + before.map(ndl => Candidate(IdRef(Ident.unsafe(ndl.date.toString), ndl.date.toString), Set(ndl.label)))) + + MetaProposalList.flatten(Seq(dueDates, itemDates)) + } + } + + def removeDuplicates(labels: List[NerLabel]): List[NerLabel] = + labels.foldLeft((Set.empty[String], List.empty[NerLabel])) { case ((seen, result), el) => + if (seen.contains(el.tag.name+el.label.toLowerCase)) (seen, result) + else (seen + (el.tag.name + el.label.toLowerCase), el :: result) + }._2.sortBy(_.startPosition) + + trait Finder[F[_]] { self => + def find(labels: Seq[NerLabel]): F[MetaProposalList] + + def contraMap(f: Seq[NerLabel] => Seq[NerLabel]): Finder[F] = + labels => self.find(f(labels)) + + def filterLabels(f: NerLabel => Boolean): Finder[F] = + contraMap(_.filter(f)) + + def flatMap(f: MetaProposalList => Finder[F])(implicit F: FlatMap[F]): Finder[F] = + labels => self.find(labels).flatMap(ml => f(ml).find(labels)) + + def map(f: MetaProposalList => MetaProposalList)(implicit F: Applicative[F]): Finder[F] = + labels => self.find(labels).map(f) + + def next(f: Finder[F])(implicit F: FlatMap[F], F3: Applicative[F]): Finder[F] = + flatMap({ ml0 => + if (ml0.hasResultsAll) Finder.unit[F](ml0) + else f.map(ml1 => ml0.fillEmptyFrom(ml1)) + }) + + def nextWhenEmpty(f: Finder[F], mt0: MetaProposalType, mts: MetaProposalType*) + (implicit F: FlatMap[F], F2: Applicative[F]): Finder[F] = + flatMap(res0 => { + if (res0.hasResults(mt0, mts: _*)) Finder.unit[F](res0) + else f.map(res1 => res0.fillEmptyFrom(res1)) + }) + } + + object Finder { + def none[F[_]: Applicative]: Finder[F] = + _ => MetaProposalList.empty.pure[F] + + def unit[F[_]: Applicative](value: MetaProposalList): Finder[F] = + _ => value.pure[F] + + def searchExact[F[_]: Sync](ctx: Context[F, ProcessItemArgs]): Finder[F] = + labels => labels.toList.traverse(nl => search(nl, true, ctx)).map(MetaProposalList.flatten) + + def searchFuzzy[F[_]: Sync](ctx: Context[F, ProcessItemArgs]): Finder[F] = + labels => labels.toList.traverse(nl => search(nl, false, ctx)).map(MetaProposalList.flatten) + } + + private def search[F[_]: Sync](nt: NerLabel, exact: Boolean, ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = { + val value = + if (exact) normalizeSearchValue(nt.label) + else s"%${normalizeSearchValue(nt.label)}%" + val minLength = + if (exact) 2 else 5 + + if (value.length < minLength) { + ctx.logger.debug(s"Skipping too small value '$value' (original '${nt.label}').").map(_ => MetaProposalList.empty) + } else nt.tag match { + case NerTag.Organization => + ctx.logger.debug(s"Looking for organizations: $value") *> + ctx.store.transact(ROrganization.findLike(ctx.args.meta.collective, value)). + map(MetaProposalList.from(MetaProposalType.CorrOrg, nt)) + + case NerTag.Person => + val s1 = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, value, true)). + map(MetaProposalList.from(MetaProposalType.ConcPerson, nt)) + val s2 = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, value, false)). + map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) + ctx.logger.debug(s"Looking for persons: $value") *> (for { + ml0 <- s1 + ml1 <- s2 + } yield ml0 |+| ml1) + + case NerTag.Location => + ctx.logger.debug(s"NerTag 'Location' is currently not used. Ignoring value '$value'."). + map(_ => MetaProposalList.empty) + + case NerTag.Misc => + ctx.logger.debug(s"Looking for equipments: $value") *> + ctx.store.transact(REquipment.findLike(ctx.args.meta.collective, value)). + map(MetaProposalList.from(MetaProposalType.ConcEquip, nt)) + + case NerTag.Email => + searchContact(nt, ContactKind.Email, value, ctx) + + case NerTag.Website => + if (!exact) { + val searchString = Domain.domainFromUri(nt.label.toLowerCase). + toOption. + map(_.toPrimaryDomain.asString). + map(s => s"%$s%"). + getOrElse(value) + searchContact(nt, ContactKind.Website, searchString, ctx) + } else { + searchContact(nt, ContactKind.Website, value, ctx) + } + + case NerTag.Date => + // There is no database search required for this tag + MetaProposalList.empty.pure[F] + } + } + + private def searchContact[F[_]: Sync]( nt: NerLabel + , kind: ContactKind + , value: String + , ctx: Context[F, ProcessItemArgs]): F[MetaProposalList] = { + val orgs = ctx.store.transact(ROrganization.findLike(ctx.args.meta.collective, kind, value)). + map(MetaProposalList.from(MetaProposalType.CorrOrg, nt)) + val corrP = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, false)). + map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) + val concP = ctx.store.transact(RPerson.findLike(ctx.args.meta.collective, kind, value, true)). + map(MetaProposalList.from(MetaProposalType.CorrPerson, nt)) + + ctx.logger.debug(s"Looking with $kind: $value") *> + List(orgs, corrP, concP).traverse(identity).map(MetaProposalList.flatten) + } + + // The backslash *must* be stripped from search strings. + private [this] val invalidSearch = + "…_[]^<>=&ſ/{}*?@#$|~`+%\"';\\".toSet + + private def normalizeSearchValue(str: String): String = + str.toLowerCase.filter(c => !invalidSearch.contains(c)) +} diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala new file mode 100644 index 00000000..3092fcdb --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemData.scala @@ -0,0 +1,27 @@ +package docspell.joex.process + +import docspell.common.{Ident, NerDateLabel, NerLabel} +import docspell.joex.process.ItemData.AttachmentDates +import docspell.store.records.{RAttachment, RAttachmentMeta, RItem} + +case class ItemData( item: RItem + , attachments: Vector[RAttachment] + , metas: Vector[RAttachmentMeta] + , dateLabels: Vector[AttachmentDates]) { + + def findMeta(attachId: Ident): Option[RAttachmentMeta] = + metas.find(_.id == attachId) + + def findDates(rm: RAttachmentMeta): Vector[NerDateLabel] = + dateLabels.find(m => m.rm.id == rm.id).map(_.dates).getOrElse(Vector.empty) +} + + +object ItemData { + + case class AttachmentDates(rm: RAttachmentMeta, dates: Vector[NerDateLabel]) { + def toNerLabel: Vector[NerLabel] = + dates.map(dl => dl.label.copy(label = dl.date.toString)) + } + +} \ No newline at end of file diff --git a/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala new file mode 100644 index 00000000..67941365 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/process/ItemHandler.scala @@ -0,0 +1,62 @@ +package docspell.joex.process + +import cats.implicits._ +import cats.effect.{ContextShift, Sync} +import docspell.common.{ItemState, ProcessItemArgs} +import docspell.joex.scheduler.{Context, Task} +import docspell.store.queries.QItem +import docspell.store.records.{RItem, RJob} +import docspell.text.ocr.{Config => OcrConfig} + +object ItemHandler { + def onCancel[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] = + logWarn("Now cancelling. Deleting potentially created data."). + flatMap(_ => deleteByFileIds) + + def apply[F[_]: Sync: ContextShift](cfg: OcrConfig): Task[F, ProcessItemArgs, Unit] = + CreateItem[F]. + flatMap(itemStateTask(ItemState.Processing)). + flatMap(safeProcess[F](cfg)). + map(_ => ()) + + def itemStateTask[F[_]: Sync, A](state: ItemState)(data: ItemData): Task[F, A, ItemData] = + Task { ctx => + ctx.store.transact(RItem.updateState(data.item.id, state)).map(_ => data) + } + + def isLastRetry[F[_]: Sync, A](ctx: Context[F, A]): F[Boolean] = + for { + current <- ctx.store.transact(RJob.getRetries(ctx.jobId)) + last = ctx.config.retries == current.getOrElse(0) + } yield last + + + def safeProcess[F[_]: Sync: ContextShift](cfg: OcrConfig)(data: ItemData): Task[F, ProcessItemArgs, ItemData] = + Task(isLastRetry[F, ProcessItemArgs] _).flatMap { + case true => + ProcessItem[F](cfg)(data). + attempt.flatMap({ + case Right(d) => + Task.pure(d) + case Left(ex) => + logWarn[F]("Processing failed on last retry. Creating item but without proposals."). + flatMap(_ => itemStateTask(ItemState.Created)(data)). + andThen(_ => Sync[F].raiseError(ex)) + }) + case false => + ProcessItem[F](cfg)(data). + flatMap(itemStateTask(ItemState.Created)) + } + + def deleteByFileIds[F[_]: Sync: ContextShift]: Task[F, ProcessItemArgs, Unit] = + Task { ctx => + for { + items <- ctx.store.transact(QItem.findByFileIds(ctx.args.files.map(_.fileMetaId))) + _ <- ctx.logger.info(s"Deleting items ${items.map(_.id.id)}") + _ <- items.traverse(i => QItem.delete(ctx.store)(i.id, ctx.args.meta.collective)) + } yield () + } + + private def logWarn[F[_]](msg: => String): Task[F, ProcessItemArgs, Unit] = + Task(_.logger.warn(msg)) +} diff --git a/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala b/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala new file mode 100644 index 00000000..81d09236 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/process/LinkProposal.scala @@ -0,0 +1,71 @@ +package docspell.joex.process + +import cats.implicits._ +import cats.effect.Sync +import docspell.common._ +import docspell.joex.scheduler.{Context, Task} +import docspell.store.records.RItem + +object LinkProposal { + + def apply[F[_]: Sync](data: ItemData): Task[F, ProcessItemArgs, ItemData] = + Task { ctx => + val proposals = MetaProposalList.flatten(data.metas.map(_.proposals)) + + ctx.logger.info(s"Starting linking proposals") *> + MetaProposalType.all. + traverse(applyValue(data, proposals, ctx)). + map(result => ctx.logger.info(s"Results from proposal processing: $result")). + map(_ => data) + } + + def applyValue[F[_]: Sync](data: ItemData, proposalList: MetaProposalList, ctx: Context[F, ProcessItemArgs])(mpt: MetaProposalType): F[Result] = { + proposalList.find(mpt) match { + case None => + Result.noneFound(mpt).pure[F] + case Some(a) if a.isSingleValue => + ctx.logger.info(s"Found one candidate for ${a.proposalType}") *> + setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id). + map(_ => Result.single(mpt)) + case Some(a) => + ctx.logger.info(s"Found many (${a.size}, ${a.values.map(_.ref.id.id)}) candidates for ${a.proposalType}. Setting first.") *> + setItemMeta(data.item.id, ctx, a.proposalType, a.values.head.ref.id). + map(_ => Result.multiple(mpt)) + } + } + + def setItemMeta[F[_]: Sync](itemId: Ident, ctx: Context[F, ProcessItemArgs], mpt: MetaProposalType, value: Ident): F[Int] = + mpt match { + case MetaProposalType.CorrOrg => + ctx.logger.debug(s"Updating item organization with: ${value.id}") *> + ctx.store.transact(RItem.updateCorrOrg(itemId, ctx.args.meta.collective, Some(value))) + case MetaProposalType.ConcPerson => + ctx.logger.debug(s"Updating item concerning person with: $value") *> + ctx.store.transact(RItem.updateConcPerson(itemId, ctx.args.meta.collective, Some(value))) + case MetaProposalType.CorrPerson => + ctx.logger.debug(s"Updating item correspondent person with: $value") *> + ctx.store.transact(RItem.updateCorrPerson(itemId, ctx.args.meta.collective, Some(value))) + case MetaProposalType.ConcEquip => + ctx.logger.debug(s"Updating item concerning equipment with: $value") *> + ctx.store.transact(RItem.updateConcEquip(itemId, ctx.args.meta.collective, Some(value))) + case MetaProposalType.DocDate => + ctx.logger.debug(s"Not linking document date suggestion ${value.id}").map(_ => 0) + case MetaProposalType.DueDate => + ctx.logger.debug(s"Not linking document date suggestion ${value.id}").map(_ => 0) + } + + + sealed trait Result { + def proposalType: MetaProposalType + } + object Result { + + case class NoneFound(proposalType: MetaProposalType) extends Result + case class SingleResult(proposalType: MetaProposalType) extends Result + case class MultipleResult(proposalType: MetaProposalType) extends Result + + def noneFound(proposalType: MetaProposalType): Result = NoneFound(proposalType) + def single(proposalType: MetaProposalType): Result = SingleResult(proposalType) + def multiple(proposalType: MetaProposalType): Result = MultipleResult(proposalType) + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala new file mode 100644 index 00000000..d74a7d77 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/process/ProcessItem.scala @@ -0,0 +1,19 @@ +package docspell.joex.process + +import cats.effect.{ContextShift, Sync} +import docspell.common.ProcessItemArgs +import docspell.joex.scheduler.Task +import docspell.text.ocr.{Config => OcrConfig} + +object ProcessItem { + + def apply[F[_]: Sync: ContextShift](cfg: OcrConfig)(item: ItemData): Task[F, ProcessItemArgs, ItemData] = + TextExtraction(cfg, item). + flatMap(Task.setProgress(25)). + flatMap(TextAnalysis[F]). + flatMap(Task.setProgress(50)). + flatMap(FindProposal[F]). + flatMap(Task.setProgress(75)). + flatMap(LinkProposal[F]). + flatMap(Task.setProgress(99)) +} diff --git a/modules/joex/src/main/scala/docspell/joex/process/TestTasks.scala b/modules/joex/src/main/scala/docspell/joex/process/TestTasks.scala new file mode 100644 index 00000000..d97709f9 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/process/TestTasks.scala @@ -0,0 +1,39 @@ +package docspell.joex.process + +import cats.implicits._ +import cats.effect.Sync +import docspell.common.ProcessItemArgs +import docspell.common.syntax.all._ +import docspell.joex.scheduler.Task +import org.log4s._ + +object TestTasks { + private [this] val logger = getLogger + + def success[F[_]]: Task[F, ProcessItemArgs, Unit] = + Task { ctx => + ctx.logger.info(s"Running task now: ${ctx.args}") + } + + def failing[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] = + Task { ctx => + ctx.logger.info(s"Failing the task run :(").map(_ => + sys.error("Oh, cannot extract gold from this document") + ) + } + + def longRunning[F[_]: Sync]: Task[F, ProcessItemArgs, Unit] = + Task { ctx => + logger.fwarn(s"${Thread.currentThread()} From executing long running task") >> + ctx.logger.info(s"${Thread.currentThread()} Running task now: ${ctx.args}") >> + sleep(2400) >> + ctx.logger.debug("doing things") >> + sleep(2400) >> + ctx.logger.debug("doing more things") >> + sleep(2400) >> + ctx.logger.info("doing more things") + } + + private def sleep[F[_]:Sync](ms: Long): F[Unit] = + Sync[F].delay(Thread.sleep(ms)) +} diff --git a/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala b/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala new file mode 100644 index 00000000..64bafbcd --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/process/TextAnalysis.scala @@ -0,0 +1,49 @@ +package docspell.joex.process + +import cats.implicits._ +import cats.effect.Sync +import docspell.common.{Duration, Language, NerLabel, ProcessItemArgs} +import docspell.joex.process.ItemData.AttachmentDates +import docspell.joex.scheduler.Task +import docspell.store.records.RAttachmentMeta +import docspell.text.contact.Contact +import docspell.text.date.DateFind +import docspell.text.nlp.StanfordNerClassifier + +object TextAnalysis { + + def apply[F[_]: Sync](item: ItemData): Task[F, ProcessItemArgs, ItemData] = + Task { ctx => + for { + _ <- ctx.logger.info("Starting text analysis") + s <- Duration.stopTime[F] + t <- item.metas.toList.traverse(annotateAttachment[F](ctx.args.meta.language)) + _ <- ctx.logger.debug(s"Storing tags: ${t.map(_._1.copy(content = None))}") + _ <- t.traverse(m => ctx.store.transact(RAttachmentMeta.updateLabels(m._1.id, m._1.nerlabels))) + e <- s + _ <- ctx.logger.info(s"Text-Analysis finished in ${e.formatExact}") + v = t.toVector + } yield item.copy(metas = v.map(_._1), dateLabels = v.map(_._2)) + } + + def annotateAttachment[F[_]: Sync](lang: Language)(rm: RAttachmentMeta): F[(RAttachmentMeta, AttachmentDates)] = + for { + list0 <- stanfordNer[F](lang, rm) + list1 <- contactNer[F](rm) + dates <- dateNer[F](rm, lang) + } yield (rm.copy(nerlabels = (list0 ++ list1 ++ dates.toNerLabel).toList), dates) + + def stanfordNer[F[_]: Sync](lang: Language, rm: RAttachmentMeta): F[Vector[NerLabel]] = Sync[F].delay { + rm.content.map(StanfordNerClassifier.nerAnnotate(lang)).getOrElse(Vector.empty) + } + + def contactNer[F[_]: Sync](rm: RAttachmentMeta): F[Vector[NerLabel]] = Sync[F].delay { + rm.content.map(Contact.annotate).getOrElse(Vector.empty) + } + + def dateNer[F[_]: Sync](rm: RAttachmentMeta, lang: Language): F[AttachmentDates] = Sync[F].delay { + AttachmentDates(rm, rm.content.map(txt => DateFind.findDates(txt, lang).toVector).getOrElse(Vector.empty)) + } + + +} diff --git a/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala new file mode 100644 index 00000000..9d7bb565 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/process/TextExtraction.scala @@ -0,0 +1,45 @@ +package docspell.joex.process + +import bitpeace.RangeDef +import cats.implicits._ +import cats.effect.{Blocker, ContextShift, Sync} +import docspell.common.{Duration, Language, ProcessItemArgs} +import docspell.joex.scheduler.{Context, Task} +import docspell.store.Store +import docspell.store.records.{RAttachment, RAttachmentMeta} +import docspell.text.ocr.{TextExtract, Config => OcrConfig} + +object TextExtraction { + + def apply[F[_]: Sync : ContextShift](cfg: OcrConfig, item: ItemData): Task[F, ProcessItemArgs, ItemData] = + Task { ctx => + for { + _ <- ctx.logger.info("Starting text extraction") + start <- Duration.stopTime[F] + txt <- item.attachments.traverse(extractTextToMeta(ctx, cfg, ctx.args.meta.language)) + _ <- ctx.logger.debug("Storing extracted texts") + _ <- txt.toList.traverse(rm => ctx.store.transact(RAttachmentMeta.upsert(rm))) + dur <- start + _ <- ctx.logger.info(s"Text extraction finished in ${dur.formatExact}") + } yield item.copy(metas = txt) + } + + def extractTextToMeta[F[_]: Sync : ContextShift](ctx: Context[F, _], cfg: OcrConfig, lang: Language)(ra: RAttachment): F[RAttachmentMeta] = + for { + _ <- ctx.logger.debug(s"Extracting text for attachment ${ra.name}") + dst <- Duration.stopTime[F] + txt <- extractText(cfg, lang, ctx.store, ctx.blocker)(ra) + meta = RAttachmentMeta.empty(ra.id).copy(content = txt.map(_.trim).filter(_.nonEmpty)) + est <- dst + _ <- ctx.logger.debug(s"Extracting text for attachment ${ra.name} finished in ${est.formatExact}") + } yield meta + + def extractText[F[_]: Sync : ContextShift](ocrConfig: OcrConfig, lang: Language, store: Store[F], blocker: Blocker)(ra: RAttachment): F[Option[String]] = { + val data = store.bitpeace.get(ra.fileId.id). + unNoneTerminate. + through(store.bitpeace.fetchData2(RangeDef.all)) + + TextExtract.extract(data, blocker, lang.iso3, ocrConfig). + compile.last + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/InfoRoutes.scala b/modules/joex/src/main/scala/docspell/joex/routes/InfoRoutes.scala similarity index 79% rename from modules/joex/src/main/scala/docspell/joex/InfoRoutes.scala rename to modules/joex/src/main/scala/docspell/joex/routes/InfoRoutes.scala index f0060ce8..e187c987 100644 --- a/modules/joex/src/main/scala/docspell/joex/InfoRoutes.scala +++ b/modules/joex/src/main/scala/docspell/joex/routes/InfoRoutes.scala @@ -1,14 +1,12 @@ -package docspell.joex +package docspell.joex.routes -import cats.effect._ -import org.http4s._ +import cats.effect.Sync +import docspell.joex.{BuildInfo, Config} +import docspell.joexapi.model.VersionInfo import org.http4s.HttpRoutes import org.http4s.dsl.Http4sDsl import org.http4s.circe.CirceEntityEncoder._ -import docspell.joexapi.model._ -import docspell.joex.BuildInfo - object InfoRoutes { def apply[F[_]: Sync](cfg: Config): HttpRoutes[F] = { diff --git a/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala b/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala new file mode 100644 index 00000000..0e177d2d --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/routes/JoexRoutes.scala @@ -0,0 +1,59 @@ +package docspell.joex.routes + +import cats.implicits._ +import cats.effect._ +import docspell.common.{Duration, Ident, Timestamp} +import docspell.joex.{Config, JoexApp} +import docspell.joexapi.model._ +import docspell.store.records.{RJob, RJobLog} +import org.http4s.HttpRoutes +import org.http4s.dsl.Http4sDsl +import org.http4s.circe.CirceEntityEncoder._ + +object JoexRoutes { + + def apply[F[_]: ConcurrentEffect: Timer](cfg: Config, app: JoexApp[F]): HttpRoutes[F] = { + val dsl = new Http4sDsl[F]{} + import dsl._ + HttpRoutes.of[F] { + case POST -> Root / "notify" => + for { + _ <- app.scheduler.notifyChange + resp <- Ok(BasicResult(true, "Scheduler notified.")) + } yield resp + + case GET -> Root / "running" => + for { + jobs <- app.scheduler.getRunning + jj = jobs.map(mkJob) + resp <- Ok(JobList(jj.toList)) + } yield resp + + case POST -> Root / "shutdownAndExit" => + for { + _ <- ConcurrentEffect[F].start(Timer[F].sleep(Duration.seconds(1).toScala) *> app.initShutdown) + resp <- Ok(BasicResult(true, "Shutdown initiated.")) + } yield resp + + case GET -> Root / "job" / Ident(id) => + for { + optJob <- app.scheduler.getRunning.map(_.find(_.id == id)) + optLog <- optJob.traverse(j => app.findLogs(j.id)) + jAndL = for { job <- optJob; log <- optLog } yield mkJobLog(job, log) + resp <- jAndL.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found"))) + } yield resp + + case POST -> Root / "job" / Ident(id) / "cancel" => + for { + flag <- app.scheduler.requestCancel(id) + resp <- Ok(BasicResult(flag, if (flag) "Cancel request submitted" else "Job not found")) + } yield resp + } + } + + def mkJob(j: RJob): Job = + Job(j.id, j.subject, j.submitted, j.priority, j.retries, j.progress, j.started.getOrElse(Timestamp.Epoch)) + + def mkJobLog(j: RJob, jl: Vector[RJobLog]): JobAndLog = + JobAndLog(mkJob(j), jl.map(r => JobLogEvent(r.created, r.level, r.message)).toList) +} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala new file mode 100644 index 00000000..9f4188fa --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/Context.scala @@ -0,0 +1,69 @@ +package docspell.joex.scheduler + +import cats.Functor +import cats.effect.{Blocker, Concurrent} +import cats.implicits._ +import docspell.common.Ident +import docspell.store.Store +import docspell.store.records.RJob +import docspell.common.syntax.all._ +import org.log4s.{Logger => _, _} + +trait Context[F[_], A] { self => + + def jobId: Ident + + def args: A + + def config: SchedulerConfig + + def logger: Logger[F] + + def setProgress(percent: Int): F[Unit] + + def store: Store[F] + + def blocker: Blocker + + def map[C](f: A => C)(implicit F: Functor[F]): Context[F, C] = + new Context.ContextImpl[F, C](f(args), logger, store, blocker, config, jobId) +} + +object Context { + private [this] val log = getLogger + + def create[F[_]: Functor, A]( job: RJob + , arg: A + , config: SchedulerConfig + , log: Logger[F] + , store: Store[F] + , blocker: Blocker): Context[F, A] = + new ContextImpl(arg, log, store, blocker, config, job.id) + + def apply[F[_]: Concurrent, A]( job: RJob + , arg: A + , config: SchedulerConfig + , logSink: LogSink[F] + , blocker: Blocker + , store: Store[F]): F[Context[F, A]] = + for { + _ <- log.ftrace("Creating logger for task run") + logger <- Logger(job.id, job.info, config.logBufferSize, logSink) + _ <- log.ftrace("Logger created, instantiating context") + ctx = create[F, A](job, arg, config, logger, store, blocker) + } yield ctx + + private final class ContextImpl[F[_]: Functor, A]( val args: A + , val logger: Logger[F] + , val store: Store[F] + , val blocker: Blocker + , val config: SchedulerConfig + , val jobId: Ident) + extends Context[F,A] { + + def setProgress(percent: Int): F[Unit] = { + val pval = math.min(100, math.max(0, percent)) + store.transact(RJob.setProgress(jobId, pval)).map(_ => ()) + } + } +} \ No newline at end of file diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/CountingScheme.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/CountingScheme.scala new file mode 100644 index 00000000..3c7771a2 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/CountingScheme.scala @@ -0,0 +1,40 @@ +package docspell.joex.scheduler + +import cats.implicits._ +import docspell.common.Priority + +/** A counting scheme to indicate a ratio between scheduling high and + * low priority jobs. + * + * For example high=4, low=1 means: ”schedule 4 high priority jobs + * and then 1 low priority job“. + */ +case class CountingScheme(high: Int, low: Int, counter: Int = 0) { + + def nextPriority: (CountingScheme, Priority) = { + if (counter <= 0) (increment, Priority.High) + else { + val rest = counter % (high + low) + if (rest < high) (increment, Priority.High) + else (increment, Priority.Low) + } + } + + def increment: CountingScheme = + copy(counter = counter + 1) +} + +object CountingScheme { + + def writeString(cs: CountingScheme): String = + s"${cs.high},${cs.low}" + + def readString(str: String): Either[String, CountingScheme] = + str.split(',') match { + case Array(h, l) => + Either.catchNonFatal(CountingScheme(h.toInt, l.toInt)). + left.map(_.getMessage) + case _ => + Left(s"Invalid counting scheme: $str") + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTask.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTask.scala new file mode 100644 index 00000000..43377d8e --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTask.scala @@ -0,0 +1,33 @@ +package docspell.joex.scheduler + +import cats.implicits._ +import cats.effect.Sync +import docspell.common.Ident +import docspell.common.syntax.all._ +import io.circe.Decoder + +/** + * Binds a Task to a name. This is required to lookup the code based + * on the taskName in the RJob data and to execute it given the + * arguments that have to be read from a string. + * + * Since the scheduler only has a string for the task argument, this + * only works for Task impls that accept a string. There is a + * convenience constructor that uses circe to decode json into some + * type A. + */ +case class JobTask[F[_]](name: Ident, task: Task[F, String, Unit], onCancel: Task[F, String, Unit]) + +object JobTask { + + def json[F[_]: Sync, A](name: Ident, task: Task[F, A, Unit], onCancel: Task[F, A, Unit]) + (implicit D: Decoder[A]): JobTask[F] = { + val convert: String => F[A] = + str => str.parseJsonAs[A] match { + case Right(a) => a.pure[F] + case Left(ex) => Sync[F].raiseError(new Exception(s"Cannot parse task arguments: $str", ex)) + } + + JobTask(name, task.contramap(convert), onCancel.contramap(convert)) + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskRegistry.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskRegistry.scala new file mode 100644 index 00000000..7ef84425 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/JobTaskRegistry.scala @@ -0,0 +1,26 @@ +package docspell.joex.scheduler + +import docspell.common.Ident + +/** + * This is a mapping from some identifier to a task. This is used by + * the scheduler to lookup an implementation using the taskName field + * of the RJob database record. + */ +final class JobTaskRegistry[F[_]](tasks: Map[Ident, JobTask[F]]) { + + def withTask(task: JobTask[F]): JobTaskRegistry[F] = + JobTaskRegistry(tasks.updated(task.name, task)) + + def find(taskName: Ident): Option[JobTask[F]] = + tasks.get(taskName) +} + +object JobTaskRegistry { + + def apply[F[_]](map: Map[Ident, JobTask[F]]): JobTaskRegistry[F] = + new JobTaskRegistry[F](map) + + def empty[F[_]]: JobTaskRegistry[F] = apply(Map.empty) + +} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/LogEvent.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/LogEvent.scala new file mode 100644 index 00000000..be3a5ff2 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/LogEvent.scala @@ -0,0 +1,25 @@ +package docspell.joex.scheduler + +import cats.implicits._ +import docspell.common._ +import cats.effect.Sync + +case class LogEvent( jobId: Ident + , jobInfo: String + , time: Timestamp + , level: LogLevel + , msg: String + , ex: Option[Throwable] = None) { + + def logLine: String = + s">>> ${time.asString} $level $jobInfo: $msg" + +} + +object LogEvent { + + def create[F[_]: Sync](jobId: Ident, jobInfo: String, level: LogLevel, msg: String): F[LogEvent] = + Timestamp.current[F].map(now => LogEvent(jobId, jobInfo, now, level, msg)) + + +} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/LogSink.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/LogSink.scala new file mode 100644 index 00000000..b81aa818 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/LogSink.scala @@ -0,0 +1,59 @@ +package docspell.joex.scheduler + +import cats.implicits._ +import cats.effect.{Concurrent, Sync} +import fs2.{Pipe, Stream} +import org.log4s.{LogLevel => _, _} +import docspell.common._ +import docspell.common.syntax.all._ +import docspell.store.Store +import docspell.store.records.RJobLog + +trait LogSink[F[_]] { + + def receive: Pipe[F, LogEvent, Unit] + +} + +object LogSink { + private[this] val logger = getLogger + + def apply[F[_]](sink: Pipe[F, LogEvent, Unit]): LogSink[F] = + new LogSink[F] { + val receive = sink + } + + def logInternal[F[_]: Sync](e: LogEvent): F[Unit] = + e.level match { + case LogLevel.Info => + logger.finfo(e.logLine) + case LogLevel.Debug => + logger.fdebug(e.logLine) + case LogLevel.Warn => + logger.fwarn(e.logLine) + case LogLevel.Error => + e.ex match { + case Some(exc) => + logger.ferror(exc)(e.logLine) + case None => + logger.ferror(e.logLine) + } + } + + def printer[F[_]: Sync]: LogSink[F] = + LogSink(_.evalMap(e => logInternal(e))) + + def db[F[_]: Sync](store: Store[F]): LogSink[F] = + LogSink(_.evalMap(ev => for { + id <- Ident.randomId[F] + joblog = RJobLog(id, ev.jobId, ev.level, ev.time, ev.msg + ev.ex.map(th => ": "+ th.getMessage).getOrElse("")) + _ <- logInternal(ev) + _ <- store.transact(RJobLog.insert(joblog)) + } yield ())) + + def dbAndLog[F[_]: Concurrent](store: Store[F]): LogSink[F] = { + val s: Stream[F, Pipe[F, LogEvent, Unit]] = + Stream.emits(Seq(printer[F].receive, db[F](store).receive)) + LogSink(Pipe.join(s)) + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/Logger.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/Logger.scala new file mode 100644 index 00000000..1b250f4d --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/Logger.scala @@ -0,0 +1,49 @@ +package docspell.joex.scheduler + +import cats.implicits._ +import cats.effect.{Concurrent, Sync} +import docspell.common._ +import fs2.concurrent.Queue + +trait Logger[F[_]] { + + def trace(msg: => String): F[Unit] + def debug(msg: => String): F[Unit] + def info(msg: => String): F[Unit] + def warn(msg: => String): F[Unit] + def error(ex: Throwable)(msg: => String): F[Unit] + def error(msg: => String): F[Unit] + +} + +object Logger { + + def create[F[_]: Sync](jobId: Ident, jobInfo: String, q: Queue[F, LogEvent]): Logger[F] = + new Logger[F] { + def trace(msg: => String): F[Unit] = + LogEvent.create[F](jobId, jobInfo, LogLevel.Debug, msg).flatMap(q.enqueue1) + + def debug(msg: => String): F[Unit] = + LogEvent.create[F](jobId, jobInfo, LogLevel.Debug, msg).flatMap(q.enqueue1) + + def info(msg: => String): F[Unit] = + LogEvent.create[F](jobId, jobInfo, LogLevel.Info, msg).flatMap(q.enqueue1) + + def warn(msg: => String): F[Unit] = + LogEvent.create[F](jobId, jobInfo, LogLevel.Warn, msg).flatMap(q.enqueue1) + + def error(ex: Throwable)(msg: => String): F[Unit] = + LogEvent.create[F](jobId, jobInfo, LogLevel.Error, msg).map(le => le.copy(ex = Some(ex))).flatMap(q.enqueue1) + + def error(msg: => String): F[Unit] = + LogEvent.create[F](jobId, jobInfo, LogLevel.Error, msg).flatMap(q.enqueue1) + } + + def apply[F[_]: Concurrent](jobId: Ident, jobInfo: String, bufferSize: Int, sink: LogSink[F]): F[Logger[F]] = + for { + q <- Queue.circularBuffer[F, LogEvent](bufferSize) + log = create(jobId, jobInfo, q) + fib <- Concurrent[F].start(q.dequeue.through(sink.receive).compile.drain) + } yield log + +} \ No newline at end of file diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/Scheduler.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/Scheduler.scala new file mode 100644 index 00000000..32341cfd --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/Scheduler.scala @@ -0,0 +1,33 @@ +package docspell.joex.scheduler + +import cats.effect.{Fiber, Timer} +import fs2.Stream +import docspell.common.Ident +import docspell.store.records.RJob + +trait Scheduler[F[_]] { + + def config: SchedulerConfig + + def getRunning: F[Vector[RJob]] + + def requestCancel(jobId: Ident): F[Boolean] + + def notifyChange: F[Unit] + + def start: Stream[F, Nothing] + + /** Requests to shutdown the scheduler. + * + * The scheduler will not take any new jobs from the queue. If + * there are still running jobs, it waits for them to complete. + * when the cancelAll flag is set to true, it cancels all running + * jobs. + * + * The returned F[Unit] can be evaluated to wait for all that to + * complete. + */ + def shutdown(cancelAll: Boolean): F[Unit] + + def periodicAwake(implicit T: Timer[F]): F[Fiber[F, Unit]] +} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerBuilder.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerBuilder.scala new file mode 100644 index 00000000..7e7a9ef5 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerBuilder.scala @@ -0,0 +1,66 @@ +package docspell.joex.scheduler + +import cats.implicits._ +import cats.effect.concurrent.Semaphore +import cats.effect.{Blocker, ConcurrentEffect, ContextShift, Resource} +import docspell.store.Store +import docspell.store.queue.JobQueue +import fs2.concurrent.SignallingRef + +case class SchedulerBuilder[F[_]: ConcurrentEffect : ContextShift]( + config: SchedulerConfig + , tasks: JobTaskRegistry[F] + , store: Store[F] + , blocker: Blocker + , queue: Resource[F, JobQueue[F]] + , logSink: LogSink[F]) { + + def withConfig(cfg: SchedulerConfig): SchedulerBuilder[F] = + copy(config = cfg) + + def withTaskRegistry(reg: JobTaskRegistry[F]): SchedulerBuilder[F] = + copy(tasks = reg) + + def withTask[A](task: JobTask[F]): SchedulerBuilder[F] = + withTaskRegistry(tasks.withTask(task)) + + def withQueue(queue: Resource[F, JobQueue[F]]): SchedulerBuilder[F] = + SchedulerBuilder[F](config, tasks, store, blocker, queue, logSink) + + def withBlocker(blocker: Blocker): SchedulerBuilder[F] = + copy(blocker = blocker) + + def withLogSink(sink: LogSink[F]): SchedulerBuilder[F] = + copy(logSink = sink) + + + def serve: Resource[F, Scheduler[F]] = + resource.evalMap(sch => ConcurrentEffect[F].start(sch.start.compile.drain).map(_ => sch)) + + def resource: Resource[F, Scheduler[F]] = { + val scheduler = for { + jq <- queue + waiter <- Resource.liftF(SignallingRef(true)) + state <- Resource.liftF(SignallingRef(SchedulerImpl.emptyState[F])) + perms <- Resource.liftF(Semaphore(config.poolSize.toLong)) + } yield new SchedulerImpl[F](config, blocker, jq, tasks, store, logSink, state, waiter, perms) + + scheduler.evalTap(_.init). + map(s => s: Scheduler[F]) + } + +} + +object SchedulerBuilder { + + def apply[F[_]: ConcurrentEffect : ContextShift]( config: SchedulerConfig + , blocker: Blocker + , store: Store[F]): SchedulerBuilder[F] = + new SchedulerBuilder[F](config + , JobTaskRegistry.empty[F] + , store + , blocker + , JobQueue(store) + , LogSink.db[F](store)) + +} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerConfig.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerConfig.scala new file mode 100644 index 00000000..2aec4240 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerConfig.scala @@ -0,0 +1,25 @@ +package docspell.joex.scheduler + +import docspell.common._ + +case class SchedulerConfig( name: Ident + , poolSize: Int + , countingScheme: CountingScheme + , retries: Int + , retryDelay: Duration + , logBufferSize: Int + , wakeupPeriod: Duration + ) + +object SchedulerConfig { + + val default = SchedulerConfig( + name = Ident.unsafe("default-scheduler") + , poolSize = 2 // math.max(2, Runtime.getRuntime.availableProcessors / 2) + , countingScheme = CountingScheme(2, 1) + , retries = 5 + , retryDelay = Duration.seconds(30) + , logBufferSize = 500 + , wakeupPeriod = Duration.minutes(10) + ) +} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala new file mode 100644 index 00000000..20546f8e --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/SchedulerImpl.scala @@ -0,0 +1,227 @@ +package docspell.joex.scheduler + +import fs2.Stream +import cats.implicits._ +import cats.effect.concurrent.Semaphore +import docspell.common._ +import docspell.common.syntax.all._ +import docspell.store.queue.JobQueue +import docspell.store.records.RJob +import fs2.concurrent.SignallingRef +import cats.effect._ +import org.log4s._ +import SchedulerImpl._ +import docspell.store.Store +import docspell.store.queries.QJob + +final class SchedulerImpl[F[_]: ConcurrentEffect : ContextShift](val config: SchedulerConfig + , blocker: Blocker + , queue: JobQueue[F] + , tasks: JobTaskRegistry[F] + , store: Store[F] + , logSink: LogSink[F] + , state: SignallingRef[F, State[F]] + , waiter: SignallingRef[F, Boolean] + , permits: Semaphore[F]) extends Scheduler[F] { + + private [this] val logger = getLogger + + /** + * On startup, get all jobs in state running from this scheduler + * and put them into waiting state, so they get picked up again. + */ + def init: F[Unit] = + QJob.runningToWaiting(config.name, store) + + def periodicAwake(implicit T: Timer[F]): F[Fiber[F, Unit]] = + ConcurrentEffect[F].start(Stream.awakeEvery[F](config.wakeupPeriod.toScala). + evalMap(_ => logger.fdebug("Periodic awake reached") *> notifyChange).compile.drain) + + def getRunning: F[Vector[RJob]] = + state.get.flatMap(s => QJob.findAll(s.getRunning, store)) + + def requestCancel(jobId: Ident): F[Boolean] = + state.get.flatMap(_.cancelRequest(jobId) match { + case Some(ct) => ct.map(_ => true) + case None => logger.fwarn(s"Job ${jobId.id} not found, cannot cancel.").map(_ => false) + }) + + def notifyChange: F[Unit] = + waiter.update(b => !b) + + def shutdown(cancelAll: Boolean): F[Unit] = { + val doCancel = + state.get. + flatMap(_.cancelTokens.values.toList.traverse(identity)). + map(_ => ()) + + val runShutdown = + state.modify(_.requestShutdown) *> (if (cancelAll) doCancel else ().pure[F]) + + val wait = Stream.eval(runShutdown). + evalMap(_ => logger.finfo("Scheduler is shutting down now.")). + flatMap(_ => Stream.eval(state.get) ++ Stream.suspend(state.discrete.takeWhile(_.getRunning.nonEmpty))). + flatMap(state => { + if (state.getRunning.isEmpty) Stream.eval(logger.finfo("No jobs running.")) + else Stream.eval(logger.finfo(s"Waiting for ${state.getRunning.size} jobs to finish.")) ++ + Stream.emit(state) + }) + + (wait.drain ++ Stream.emit(())).compile.lastOrError + } + + def start: Stream[F, Nothing] = + logger.sinfo("Starting scheduler") ++ + mainLoop + + def mainLoop: Stream[F, Nothing] = { + val body: F[Boolean] = + for { + _ <- permits.available.flatMap(a => logger.fdebug(s"Try to acquire permit ($a free)")) + _ <- permits.acquire + _ <- logger.fdebug("New permit acquired") + down <- state.get.map(_.shutdownRequest) + rjob <- if (down) logger.finfo("") *> permits.release *> (None: Option[RJob]).pure[F] + else queue.nextJob(group => state.modify(_.nextPrio(group, config.countingScheme)), config.name, config.retryDelay) + _ <- logger.fdebug(s"Next job found: ${rjob.map(_.info)}") + _ <- rjob.map(execute).getOrElse(permits.release) + } yield rjob.isDefined + + Stream.eval(state.get.map(_.shutdownRequest)). + evalTap(if (_) logger.finfo[F]("Stopping main loop due to shutdown request.") else ().pure[F]). + flatMap(if (_) Stream.empty else Stream.eval(body)). + flatMap({ + case true => + mainLoop + case false => + logger.sdebug(s"Waiting for notify") ++ + waiter.discrete.take(2).drain ++ + logger.sdebug(s"Notify signal, going into main loop") ++ + mainLoop + }) + } + + def execute(job: RJob): F[Unit] = { + val task = for { + jobtask <- tasks.find(job.task).toRight(s"This executor cannot run tasks with name: ${job.task}") + } yield jobtask + + task match { + case Left(err) => + logger.ferror(s"Unable to start a task for job ${job.info}: $err") + case Right(t) => + for { + _ <- logger.fdebug(s"Creating context for job ${job.info} to run $t") + ctx <- Context[F, String](job, job.args, config, logSink, blocker, store) + jot = wrapTask(job, t.task, ctx) + tok <- forkRun(job, jot.run(ctx), t.onCancel.run(ctx), ctx) + _ <- state.modify(_.addRunning(job, tok)) + } yield () + } + } + + def onFinish(job: RJob, finalState: JobState): F[Unit] = + for { + _ <- logger.fdebug(s"Job ${job.info} done $finalState. Releasing resources.") + _ <- permits.release *> permits.available.flatMap(a => logger.fdebug(s"Permit released ($a free)")) + _ <- state.modify(_.removeRunning(job)) + _ <- QJob.setFinalState(job.id, finalState, store) + } yield () + + def onStart(job: RJob): F[Unit] = + QJob.setRunning(job.id, config.name, store) //also increments retries if current state=stuck + + def wrapTask(job: RJob, task: Task[F, String, Unit], ctx: Context[F, String]): Task[F, String, Unit] = { + task.mapF(fa => onStart(job) *> logger.fdebug("Starting task now") *> blocker.blockOn(fa)). + mapF(_.attempt.flatMap({ + case Right(()) => + logger.info(s"Job execution successful: ${job.info}") + ctx.logger.info("Job execution successful") *> + (JobState.Success: JobState).pure[F] + case Left(ex) => + state.get.map(_.wasCancelled(job)).flatMap { + case true => + logger.error(ex)(s"Job ${job.info} execution failed (cancel = true)") + ctx.logger.error(ex)("Job execution failed (cancel = true)") *> + (JobState.Cancelled: JobState).pure[F] + case false => + QJob.exceedsRetries(job.id, config.retries, store).flatMap { + case true => + logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded.") + ctx.logger.error(ex)(s"Job ${job.info} execution failed. Retries exceeded."). + map(_ => JobState.Failed: JobState) + case false => + logger.error(ex)(s"Job ${job.info} execution failed. Retrying later.") + ctx.logger.error(ex)(s"Job ${job.info} execution failed. Retrying later."). + map(_ => JobState.Stuck: JobState) + } + } + })). + mapF(_.attempt.flatMap { + case Right(jstate) => + onFinish(job, jstate) + case Left(ex) => + logger.error(ex)(s"Error happened during post-processing of ${job.info}!") + // we don't know the real outcome here… + // since tasks should be idempotent, set it to stuck. if above has failed, this might fail anyways + onFinish(job, JobState.Stuck) + }) + } + + def forkRun(job: RJob, code: F[Unit], onCancel: F[Unit], ctx: Context[F, String]): F[F[Unit]] = { + val bfa = blocker.blockOn(code) + logger.fdebug(s"Forking job ${job.info}") *> + ConcurrentEffect[F].start(bfa). + map(fiber => + logger.fdebug(s"Cancelling job ${job.info}") *> + fiber.cancel *> + onCancel.attempt.map({ + case Right(_) => () + case Left(ex) => + logger.error(ex)(s"Task's cancelling code failed. Job ${job.info}.") + () + }) *> + state.modify(_.markCancelled(job)) *> + onFinish(job, JobState.Cancelled) *> + ctx.logger.warn("Job has been cancelled.") *> + logger.fdebug(s"Job ${job.info} has been cancelled.")) + } +} + +object SchedulerImpl { + + def emptyState[F[_]]: State[F] = + State(Map.empty, Set.empty, Map.empty, false) + + case class State[F[_]]( counters: Map[Ident, CountingScheme] + , cancelled: Set[Ident] + , cancelTokens: Map[Ident, CancelToken[F]] + , shutdownRequest: Boolean) { + + def nextPrio(group: Ident, initial: CountingScheme): (State[F], Priority) = { + val (cs, prio) = counters.getOrElse(group, initial).nextPriority + (copy(counters = counters.updated(group, cs)), prio) + } + + def addRunning(job: RJob, token: CancelToken[F]): (State[F], Unit) = + (State(counters, cancelled, cancelTokens.updated(job.id, token), shutdownRequest), ()) + + def removeRunning(job: RJob): (State[F], Unit) = + (copy(cancelled = cancelled - job.id, cancelTokens = cancelTokens.removed(job.id)), ()) + + def markCancelled(job: RJob): (State[F], Unit) = + (copy(cancelled = cancelled + job.id), ()) + + def wasCancelled(job: RJob): Boolean = + cancelled.contains(job.id) + + def cancelRequest(id: Ident): Option[F[Unit]] = + cancelTokens.get(id) + + def getRunning: Seq[Ident] = + cancelTokens.keys.toSeq + + def requestShutdown: (State[F], Unit) = + (copy(shutdownRequest = true), ()) + } +} diff --git a/modules/joex/src/main/scala/docspell/joex/scheduler/Task.scala b/modules/joex/src/main/scala/docspell/joex/scheduler/Task.scala new file mode 100644 index 00000000..ec041277 --- /dev/null +++ b/modules/joex/src/main/scala/docspell/joex/scheduler/Task.scala @@ -0,0 +1,55 @@ +package docspell.joex.scheduler + +import cats.implicits._ +import cats.{Applicative, ApplicativeError, FlatMap, Functor} +import cats.data.Kleisli +import cats.effect.Sync + +/** + * The code that is executed by the scheduler + */ +trait Task[F[_], A, B] { + + def run(ctx: Context[F, A]): F[B] + + def map[C](f: B => C)(implicit F: Functor[F]): Task[F, A, C] = + Task(Task.toKleisli(this).map(f)) + + def flatMap[C](f: B => Task[F, A, C])(implicit F: FlatMap[F]): Task[F, A, C] = + Task(Task.toKleisli(this).flatMap(a => Task.toKleisli(f(a)))) + + def andThen[C](f: B => F[C])(implicit F: FlatMap[F]): Task[F, A, C] = + Task(Task.toKleisli(this).andThen(f)) + + def mapF[C](f: F[B] => F[C]): Task[F, A, C] = + Task(Task.toKleisli(this).mapF(f)) + + def attempt(implicit F: ApplicativeError[F,Throwable]): Task[F, A, Either[Throwable, B]] = + mapF(_.attempt) + + def contramap[C](f: C => F[A])(implicit F: FlatMap[F]): Task[F, C, B] = { + ctxc: Context[F, C] => f(ctxc.args).flatMap(a => run(ctxc.map(_ => a))) + } +} + +object Task { + + def pure[F[_]: Applicative, A, B](b: B): Task[F, A, B] = + Task(_ => b.pure[F]) + + def of[F[_], A, B](b: F[B]): Task[F, A, B] = + Task(_ => b) + + def apply[F[_], A, B](f: Context[F, A] => F[B]): Task[F, A, B] = + (ctx: Context[F, A]) => f(ctx) + + def apply[F[_], A, B](k: Kleisli[F, Context[F, A], B]): Task[F, A, B] = + c => k.run(c) + + + def toKleisli[F[_], A, B](t: Task[F, A, B]): Kleisli[F, Context[F, A], B] = + Kleisli(t.run) + + def setProgress[F[_]: Sync, A, B](n: Int)(data: B): Task[F, A, B] = + Task(_.setProgress(n).map(_ => data)) +} diff --git a/modules/joex/src/test/scala/docspell/joex/scheduler/CountingSchemeSpec.scala b/modules/joex/src/test/scala/docspell/joex/scheduler/CountingSchemeSpec.scala new file mode 100644 index 00000000..a84c634c --- /dev/null +++ b/modules/joex/src/test/scala/docspell/joex/scheduler/CountingSchemeSpec.scala @@ -0,0 +1,15 @@ +package docspell.joex.scheduler + +import docspell.common.Priority +import minitest.SimpleTestSuite + +object CountingSchemeSpec extends SimpleTestSuite { + + test("counting") { + val cs = CountingScheme(2,1) + val list = List.iterate(cs.nextPriority, 6)(_._1.nextPriority).map(_._2) + val expect = List(Priority.High, Priority.High, Priority.Low) + assertEquals(list, expect ++ expect) + } + +} diff --git a/modules/joexapi/src/main/resources/joex-openapi.yml b/modules/joexapi/src/main/resources/joex-openapi.yml index 75a113d9..9b5c1f7d 100644 --- a/modules/joexapi/src/main/resources/joex-openapi.yml +++ b/modules/joexapi/src/main/resources/joex-openapi.yml @@ -9,9 +9,162 @@ servers: description: Current host paths: + /api/info: + get: + tag: [ Api Info ] + summary: Get basic information about this software. + description: | + Returns the version and project name and other properties of the build. + responses: + 200: + content: + application/json: + schema: + $ref: "#/components/schemas/VersionInfo" + /api/v1/notify: + post: + tag: [ Job Executor ] + summary: Notify the job executor. + description: | + Notifies the job executor to wake up and look for jobs in th queue. + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /api/v1/running: + get: + tag: [ Job Executor ] + summary: Get a list of currently executing jobs. + description: | + Returns all jobs this executor is currently executing. + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/JobList" + /api/v1/shutdownAndExit: + post: + tag: [ Job Executor ] + summary: Stops this component and exits. + description: | + Gracefully stops the scheduler and also stops the process. + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /api/v1/job/{id}: + get: + tag: [ Current Jobs ] + summary: Get a job by its id. + description: | + Returns details about a job given the id. + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/JobAndLogs" + /api/v1/job/{id}/cancel: + post: + tag: [ Current Jobs ] + summary: Request to cancel a running job. + description: | + Requests to cancel the running job. This will try to cancel + the execution but it is not guaranteed that it can immediately + abort. The job is then removed from the queue. + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" components: schemas: + JobAndLog: + description: | + Some more details about the job. + required: + - job + - logs + properties: + job: + $ref: "#/components/schemas/Job" + logs: + type: array + items: + $ref: "#/components/schemas/JobLogEvent" + JobLogEvent: + description: | + A log output line. + required: + - time + - level + - message + properties: + time: + description: DateTime + type: integer + format: date-time + level: + type: string + format: loglevel + message: + type: string + JobList: + description: | + A list of jobs. + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/Job" + Job: + description: | + Data about a running job. + required: + - id + - name + - submitted + - priority + - retries + - progress + - started + properties: + id: + type: string + format: ident + name: + type: string + submitted: + description: DateTime + type: integer + format: date-time + priority: + type: integer + format: priority + retries: + type: integer + format: int32 + progress: + type: integer + format: int32 + started: + description: DateTime + type: integer + format: date-time VersionInfo: description: | Information about the software. @@ -33,3 +186,14 @@ components: type: string gitVersion: type: string + BasicResult: + description: | + Some basic result of an operation. + required: + - success + - message + properties: + success: + type: boolean + message: + type: string diff --git a/modules/microsite/src/main/resources/microsite/css/docspell.css b/modules/microsite/src/main/resources/microsite/css/docspell.css new file mode 100644 index 00000000..3c1c8e2f --- /dev/null +++ b/modules/microsite/src/main/resources/microsite/css/docspell.css @@ -0,0 +1,18 @@ +.jumbotron { + background: url(../img/back-master-small.jpg); + background-repeat: no-repeat; + background-size: 100% 800px; +} + +.content-wrapper h1, .h1 { + border-bottom: 1px solid #d8dfe5; + padding-bottom: 0.8rem; +} + +body { + font-size: 1.75em; +} + +h4 { + text-decoration: underline; +} diff --git a/modules/microsite/src/main/resources/microsite/data/menu.yml b/modules/microsite/src/main/resources/microsite/data/menu.yml new file mode 100644 index 00000000..78c014a9 --- /dev/null +++ b/modules/microsite/src/main/resources/microsite/data/menu.yml @@ -0,0 +1,48 @@ +options: + - title: Home + url: index.html + + - title: Getit + url: getit.html + + - title: Documentation + url: doc.html + + nested_options: + - title: Installation + url: doc/install.html + + - title: Configuring + url: doc/configure.html + + - title: Adding Meta Data + url: doc/metadata.html + + - title: Uploads + url: doc/uploading.html + + - title: Processing Queue + url: doc/processing.html + + - title: Find and Review + url: doc/curate.html + + - title: Joex + url: doc/joex.html + + - title: Development + url: dev.html + + nested_options: + - tite: ADRs + url: dev/adr.html + + - title: Api + url: api.html + + nested_options: + - title: REST Api Doc + url: openapi/docspell-openapi.html + + - title: REST OpenApi Spec + url: openapi/docspell-openapi.yml diff --git a/modules/microsite/src/main/resources/microsite/img/back-master-small.jpg b/modules/microsite/src/main/resources/microsite/img/back-master-small.jpg new file mode 100644 index 00000000..71fc1bff Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/back-master-small.jpg differ diff --git a/modules/microsite/src/main/resources/microsite/img/docspell-curate-1.jpg b/modules/microsite/src/main/resources/microsite/img/docspell-curate-1.jpg new file mode 100644 index 00000000..f33b6b16 Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/docspell-curate-1.jpg differ diff --git a/modules/microsite/src/main/resources/microsite/img/docspell-curate-2.jpg b/modules/microsite/src/main/resources/microsite/img/docspell-curate-2.jpg new file mode 100644 index 00000000..b21f14cb Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/docspell-curate-2.jpg differ diff --git a/modules/microsite/src/main/resources/microsite/img/docspell-curate-3.jpg b/modules/microsite/src/main/resources/microsite/img/docspell-curate-3.jpg new file mode 100644 index 00000000..4af82363 Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/docspell-curate-3.jpg differ diff --git a/modules/microsite/src/main/resources/microsite/img/docspell-curate-4.jpg b/modules/microsite/src/main/resources/microsite/img/docspell-curate-4.jpg new file mode 100644 index 00000000..a320d1d3 Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/docspell-curate-4.jpg differ diff --git a/modules/microsite/src/main/resources/microsite/img/docspell-curate-5.jpg b/modules/microsite/src/main/resources/microsite/img/docspell-curate-5.jpg new file mode 100644 index 00000000..dbc1a6a9 Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/docspell-curate-5.jpg differ diff --git a/modules/microsite/src/main/resources/microsite/img/docspell-curate-6.jpg b/modules/microsite/src/main/resources/microsite/img/docspell-curate-6.jpg new file mode 100644 index 00000000..f0e26769 Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/docspell-curate-6.jpg differ diff --git a/modules/microsite/src/main/resources/microsite/img/docspell-demo.gif b/modules/microsite/src/main/resources/microsite/img/docspell-demo.gif new file mode 100644 index 00000000..b7ca2be9 Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/docspell-demo.gif differ diff --git a/modules/microsite/src/main/resources/microsite/img/docspell-try.gif b/modules/microsite/src/main/resources/microsite/img/docspell-try.gif new file mode 100644 index 00000000..2f4ce6ce Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/docspell-try.gif differ diff --git a/modules/microsite/src/main/resources/microsite/img/favicon.png b/modules/microsite/src/main/resources/microsite/img/favicon.png new file mode 120000 index 00000000..add0937c --- /dev/null +++ b/modules/microsite/src/main/resources/microsite/img/favicon.png @@ -0,0 +1 @@ +../../../../../../webapp/src/main/webjar/favicon/android-icon-96x96.png \ No newline at end of file diff --git a/modules/microsite/src/main/resources/microsite/img/navbar_brand.png b/modules/microsite/src/main/resources/microsite/img/navbar_brand.png new file mode 100644 index 00000000..f41a3178 Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/navbar_brand.png differ diff --git a/modules/microsite/src/main/resources/microsite/img/navbar_brand2x.png b/modules/microsite/src/main/resources/microsite/img/navbar_brand2x.png new file mode 100644 index 00000000..f41a3178 Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/navbar_brand2x.png differ diff --git a/modules/microsite/src/main/resources/microsite/img/processing-queue.jpg b/modules/microsite/src/main/resources/microsite/img/processing-queue.jpg new file mode 100644 index 00000000..aca9407c Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/processing-queue.jpg differ diff --git a/modules/microsite/src/main/resources/microsite/img/sidebar_brand.png b/modules/microsite/src/main/resources/microsite/img/sidebar_brand.png new file mode 100644 index 00000000..f41a3178 Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/sidebar_brand.png differ diff --git a/modules/microsite/src/main/resources/microsite/img/sidebar_brand2x.png b/modules/microsite/src/main/resources/microsite/img/sidebar_brand2x.png new file mode 100644 index 00000000..f41a3178 Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/sidebar_brand2x.png differ diff --git a/modules/microsite/src/main/resources/microsite/img/sources-form.jpg b/modules/microsite/src/main/resources/microsite/img/sources-form.jpg new file mode 100644 index 00000000..685f0c0f Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/sources-form.jpg differ diff --git a/modules/microsite/src/main/resources/microsite/img/wand-white.png b/modules/microsite/src/main/resources/microsite/img/wand-white.png new file mode 100644 index 00000000..c670c27b Binary files /dev/null and b/modules/microsite/src/main/resources/microsite/img/wand-white.png differ diff --git a/modules/microsite/src/main/tut/api.md b/modules/microsite/src/main/tut/api.md new file mode 100644 index 00000000..bc9a2511 --- /dev/null +++ b/modules/microsite/src/main/tut/api.md @@ -0,0 +1,92 @@ +--- +layout: docs +position: 5 +title: Api +--- + +# {{page.title}} + +Docspell is designed as a REST server that uses JSON to exchange +data. The REST api can be used to integrate docspell into your +workflow. + +[Docspell REST Api Doc](openapi/docspell-openapi.html) + +The "raw" `openapi.yml` specification file can be found +[here](openapi/docspell-openapi.yml). + +The routes can be divided into protected and unprotected routes. The +unprotected, or open routes are at `/open/*` wihle the protected +routes are at `/sec/*`. Open routes don't require authenticated access +and can be used by any user. The protected routes require an +authenticated user. + +## Authentication + +The unprotected route `/open/auth/login` can be used to login with +account name and password. The response contains a token that can be +used for accessing protected routes. The token is only valid for a +restricted time which can be configured (default is 5 minutes). + +New tokens can be generated using an existing valid token and the +protected route `/sec/auth/session`. This will return the same +response as above, giving a new token. + +This token can be added to requests in two ways: as a cookie header or +a "normal" http header. If a cookie header is used, the cookie name +must be `docspell_auth` and a custom header must be named +`X-Docspell-Auth`. + +## Live Api + +Besides the statically generated documentation at this site, the rest +server provides a swagger generated api documenation, that allows +playing around with the api. It requires a running docspell rest +server. If it is deployed at `http://localhost:7880`, then check this +url: + +``` +http://localhost:7880/app/doc +``` + +## Examples + +These examples use the great command line tool +[curl](https://curl.haxx.se/). + +### Login + +``` +$ curl -X POST -d '{"account": "smith", "password": "test"}' http://localhost:7880/api/v1/open/auth/login +{"collective":"smith" +,"user":"smith" +,"success":true +,"message":"Login successful" +,"token":"1568142350115-ZWlrZS9laWtl-$2a$10$rGZUFDAVNIKh4Tj6u6tlI.-O2euwCvmBT0TlyDmIHR1ZsLQPAI=" +,"validMs":300000 +} +``` + +### Get new token + +``` +$ curl -XPOST -H 'X-Docspell-Auth: 1568142350115-ZWlrZS9laWtl-$2a$10$rGZUFDAVNIKh4Tj6u6tlI.-O2euwCvmBT0TlyDmIHR1ZsLQPAI=' http://localhost:7880/api/v1/sec/auth/session +{"collective":"smith" +,"user":"smith" +,"success":true +,"message":"Login successful" +,"token":"1568142446077-ZWlrZS9laWtl-$2a$10$3B0teJ9rMpsBJPzHfZZPoO-WeA1bkfEONBN8fyzWE8DeaAHtUc=" +,"validMs":300000 +} +``` + +### Get some insights + +``` +$ curl -H 'X-Docspell-Auth: 1568142446077-ZWlrZS9laWtl-$2a$10$3B0teJ9rMpsBJPzHfZZPoO-WeA1bkfEONBN8fyzWE8DeaAHtUc=' http://localhost:7880/api/v1/sec/collective/insights +{"incomingCount":3 +,"outgoingCount":1 +,"itemSize":207310 +,"tagCloud":{"items":[]} +} +``` diff --git a/modules/microsite/src/main/tut/demo.md b/modules/microsite/src/main/tut/demo.md new file mode 100644 index 00000000..3568f211 --- /dev/null +++ b/modules/microsite/src/main/tut/demo.md @@ -0,0 +1,16 @@ +--- +layout: home +position: 2 +section: demo +title: Demo +technologies: + - first: ["Scala + Elm", "Backend is in Scala with Cats/Fs2, Webapp in Elm"] + - second: ["Unpaper + Tesseract", "Text is extracted using OCR provided by tesseract"] + - third: ["Stanford NLP", "Documents are analyzed using Stanford NLP classifiers"] +--- + +# {{ page.title }} + + + + diff --git a/modules/microsite/src/main/tut/dev.md b/modules/microsite/src/main/tut/dev.md new file mode 100644 index 00000000..4efb1b98 --- /dev/null +++ b/modules/microsite/src/main/tut/dev.md @@ -0,0 +1,86 @@ +--- +layout: docs +title: Development +--- + + +# {{page.title}} + + +## Building + +[Sbt](https://scala-sbt.org) is used to build the application. Clone +the sources and run: + +- `make` to compile all sources (Elm + Scala) +- `make-zip` to create zip packages +- `make-deb` to create debian packages + +The zip files can be found afterwards in: + +``` +modules/restserver/target/universal +modules/joex/target/universal +``` + + +## Starting Servers with `reStart` + +When developing, it's very convenient to use the [revolver sbt +plugin](https://github.com/spray/sbt-revolver). Start the sbt console +and then run: + +``` +sbt:docspell-root> restserver/reStart +``` + +This starts a REST server. Once this started up, type: + +``` +sbt:docspell-root> joex/reStart +``` + +if also a joex component is required. Prefixing the commads with `~`, +results in recompile+restart once a source file is modified. + + +## Custom config file + +The sbt build is setup such that a file `dev.conf` in the root of the +source tree is picked up as config file, if it exists. So you can +create a custom config file for development. For example, a custom +database for development may be setup this way: + +``` +#jdbcurl = "jdbc:h2:///home/dev/workspace/projects/docspell/local/docspell-demo.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_SERVER=TRUE" +jdbcurl = "jdbc:postgresql://localhost:5432/docspelldev" +#jdbcurl = "jdbc:mariadb://localhost:3306/docspelldev" + +docspell.server { + backend { + jdbc { + url = ${jdbcurl} + user = "dev" + password = "dev" + } + } +} + +docspell.joex { + jdbc { + url = ${jdbcurl} + user = "dev" + password = "dev" + } + scheduler { + pool-size = 1 + } +} +``` + +## ADRs + +Some early information about certain details can be found in the few +[ADR](https://adr.github.io/) that exist: + +- [ADRs](dev/adr.html) diff --git a/modules/microsite/src/main/tut/dev/adr.md b/modules/microsite/src/main/tut/dev/adr.md new file mode 100644 index 00000000..22481e3f --- /dev/null +++ b/modules/microsite/src/main/tut/dev/adr.md @@ -0,0 +1,12 @@ +--- +layout: docs +title: ADRs +--- + +# ADR + +- [0001 Components](adr/0001_components.html) +- [0002 Component Interaction](adr/0002_component_interaction.html) +- [0003 Encryption](adr/0003_encryption.html) +- [0004 ISO8601 vs Unix](adr/0004_iso8601vsEpoch.html) +- [0005 Job Executor](adr/0005_job-executor.html) diff --git a/doc/adr/0000_use_markdown_architectural_decision_records.md b/modules/microsite/src/main/tut/dev/adr/0000_use_markdown_architectural_decision_records.md similarity index 100% rename from doc/adr/0000_use_markdown_architectural_decision_records.md rename to modules/microsite/src/main/tut/dev/adr/0000_use_markdown_architectural_decision_records.md diff --git a/doc/adr/0001_components.md b/modules/microsite/src/main/tut/dev/adr/0001_components.md similarity index 68% rename from doc/adr/0001_components.md rename to modules/microsite/src/main/tut/dev/adr/0001_components.md index 11da91eb..4511cd4a 100644 --- a/doc/adr/0001_components.md +++ b/modules/microsite/src/main/tut/dev/adr/0001_components.md @@ -1,3 +1,8 @@ +--- +layout: docs +title: Components +--- + # Components ## Context and Problem Statement @@ -7,9 +12,6 @@ goal is to be able to have multiple rest servers/webapps and multiple document processor components working togehter. -## Considered Options - - ## Decision Outcome The following are the "main" modules. There may be more helper modules @@ -29,32 +31,20 @@ on the `store` module. It provides the code for all tasks that can be submitted as jobs. If no jobs are in the queue, the joex "sleeps" and must be waked via an external request. -The main reason for this module is to provide the document processing -code. +It provides the document processing code. It provides a http rest server to get insight into the joex state and also to be notified for new jobs. ### backend -This is the heart of the application. It provides all the logic, -except document processing, as a set of "operations". An operation can -be directly mapped to a rest endpoint. An operation is roughly this: - -``` -A -> F[Either[E, B]] -``` - -First, it can fail and so there is some sort of either type to encode -failure. It also is inside a `F` context, since it may run impure -code, e.g. database calls. The input value `A` can be obtained by -amending the user input from a rest call with additional data from the -database corresponding to the current user (for example the public key -or some preference setting). +It provides all the logic, except document processing, as a set of +"operations". An operation can be directly mapped to a rest +endpoint. It is designed as a library. -### rest spec +### rest api This module contains the specification for the rest server as an `openapi.yml` file. It is packaged as a scala library that also diff --git a/doc/adr/0002_component_interaction.md b/modules/microsite/src/main/tut/dev/adr/0002_component_interaction.md similarity index 57% rename from doc/adr/0002_component_interaction.md rename to modules/microsite/src/main/tut/dev/adr/0002_component_interaction.md index ecd54f92..63e572be 100644 --- a/doc/adr/0002_component_interaction.md +++ b/modules/microsite/src/main/tut/dev/adr/0002_component_interaction.md @@ -1,3 +1,8 @@ +--- +layout: docs +title: Component Interaction +--- + # Component Interaction ## Context and Problem Statement @@ -14,7 +19,8 @@ are multiple document processors. These processes must communicate: ## Considered Options -1. JMS: Message Broker as another active component +1. JMS (ActiveMQ or similiar): Message Broker as another active + component 2. Akka: using a cluster 3. DB: Register with "call back urls" @@ -24,7 +30,8 @@ Choosing option 3: DB as central synchronisation point. The reason is that this is the simplest solution and doesn't require external libraries or more processes. The other options seem too big -of a weapon for the task at hand. +of a weapon for the task at hand. They are both large components +itself and require more knowledge to use them efficiently. It works roughly like this: @@ -47,41 +54,12 @@ It works roughly like this: ### Negative Consequences -- all components must have db access. this also is a negative point, +- all components must have db access. this also is a security con, because if one of those processes is hacked, db access is - possible. and it simply is another dependency that may not be - required for the document processors -- the document processors cannot be in a untrusted environment - (untrusted from the db's point of view). it would be for example - possible to create personal processors that only receive your own - jobs… + possible. and it simply is another dependency that is not really + required for the joex component +- the joex component cannot be in an untrusted environment (untrusted + from the db's point of view). For example, it is not possible to + create "personal joex" that only receive your own jobs… - in order to know if a component is really active, one must run a ping against the call-back url - -## Pros and Cons of the Options - -### JMS Message Broker - -- pro: offers publish-subscribe out of the box -- con: another central point of failure -- con: requires setup and maintenance -- con: complexity of whole app is strongly increased, there are now at - least 3 processes - -### Akka Cluster - -- pro: publish subscribe -- pro: no central component or separate process -- con: only works reliably in a "real cluster", where 3 nodes is a - minimum. Thus it wouldn't allow a light-weight setup of the - application -- con: introduces a new technology that is not easy to understand and - maintain (the cluster, gossip protocol etc) requires to be "good at - akka" - -### DB Sync - -- pro: simple and intuitive -- pro: no one more central point of failure -- pro: requires no additional knowledge or setup -- cons: all components require db access diff --git a/modules/microsite/src/main/tut/dev/adr/0003_encryption.md b/modules/microsite/src/main/tut/dev/adr/0003_encryption.md new file mode 100644 index 00000000..1f2bd054 --- /dev/null +++ b/modules/microsite/src/main/tut/dev/adr/0003_encryption.md @@ -0,0 +1,95 @@ +--- +layout: docs +title: Encryption +--- + +# Encryption + + +## Context and Problem Statement + +Since docspell may store important documents, it should be possible to +encrypt them on the server. It should be (almost) transparent to the +user, for example, a user must be able to login and download a file in +clear form. That is, the server must also decrypt them. + +Then all users of a collective should have access to the files. This +requires to share the key among users of a collective. + +But, even when files are encrypted, the associated meta data is not! +So especially access to the database would allow to see tags, +associated persons and correspondents of documents. + +So in short, encryption means: + +- file contents (the blobs and extracted text) is encrypted +- metadata is not +- secret keys are stored at the server (protected by a passphrase), + such that files can be downloaded in clear form + + +## Decision Drivers + +* major driver is to provide most possible privacy for users +* even at the expense of less features; currently I think that the + associated meta data is enough for finding documents (i.e. full text + search is not needed) + +## Considered Options + +It is clear, that only blobs (file contents) can be encrypted, but not +the associated metadata. And the extracted text must be encrypted, +too, obviously. + + +### Public Key Encryption (PKE) + +With PKE that the server can automatically encrypt files using +publicly available key data. It wouldn't require a user to provide a +passphrase for encryption, only for decryption. + +This would allows for first processing files (extracting text, doing +text analyisis) and encrypting them (and the text) afterwards. + +The public and secret keys are stored at the database. The secret key +must be protected. This can be done by encrypting the passphrase to +the secret key using each users login password. If a user logs in, he +or she must provide the correct password. Using this password, the +private key can be unlocked. This requires to store the private key +passphrase encrypted with every users password in the database. So the +whole security then depends on users password quality. + +There are plenty of other difficulties with this approach (how about +password change, new secret keys, adding users etc). + +Using this kind of encryption would protect the data against offline +attacks and also for accidental leakage (for example, if a bug in the +software would access a file of another user). + + +### No Encryption + +If only blobs are encrypted, against which type of attack would it +provide protection? + +The users must still trust the server. First, in order to provide the +wanted features (document processing), the server must see the file +contents. Then, it will receive and serve files in clear form, so it +has access to them anyways. + +With that in mind, the "only" feature is to protect against "stolen +database" attacks. If the database is somehow leaked, the attackers +would only see the metadata, but not real documents. It also protects +against leakage, maybe caused by a pogramming error. + +But the downside is, that it increases complexity *a lot*. And since +this is a personal tool for personal use, is it worth the effort? + + +## Decision Outcome + +No encryption, because of its complexity. + +For now, this tool is only meant for "self deployment" and personal +use. If this changes or there is enough time, this decision should be +reconsidered. diff --git a/modules/microsite/src/main/tut/dev/adr/0004_iso8601vsEpoch.md b/modules/microsite/src/main/tut/dev/adr/0004_iso8601vsEpoch.md new file mode 100644 index 00000000..dde7b048 --- /dev/null +++ b/modules/microsite/src/main/tut/dev/adr/0004_iso8601vsEpoch.md @@ -0,0 +1,42 @@ +--- +layout: docs +title: ISO8601 vs Millis +--- + +# ISO8601 vs Millis as Date-Time transfer + +## Context and Problem Statement + +The question is whether the REST Api should return an ISO8601 +formatted string in UTC timezone, or the unix time (number of +milliseconds since 1970-01-01). + +There is quite some controversy about it. + +- +- + +In my opinion, the ISO8601 format (always UTC) is better. The reason +is the better readability. But elm folks are on the other side: + +- +- + +One can convert from an ISO8601 date-time string in UTC time into the +epoch millis and vice versa. So it is the same to me. There is no less +information in a ISO8601 string than in the epoch millis. + +To avoid confusion, all date/time values should use the same encoding. + +## Decision Outcome + +I go with the epoch time. Every timestamp/date-time values is +transfered as Unix timestamp. + +Reasons: + +- the Elm application needs to frequently calculate with these values + to render the current waiting time etc. This is better if there are + numbers without requiring to parse dates first +- Since the UI is written with Elm, it's probably good to adopt their + style diff --git a/modules/microsite/src/main/tut/dev/adr/0005_job-executor.md b/modules/microsite/src/main/tut/dev/adr/0005_job-executor.md new file mode 100644 index 00000000..43acb50d --- /dev/null +++ b/modules/microsite/src/main/tut/dev/adr/0005_job-executor.md @@ -0,0 +1,136 @@ +--- +layout: docs +title: Joex - Job Executor +--- + +# Job Executor + +## Context and Problem Statement + +Docspell is a multi-user application. When processing user's +documents, there must be some thought on how to distribute all the +processing jobs on a much more restricted set of resources. There +maybe 100 users but only 4 cores that can process documents at a +time. Doing simply FIFO is not enough since it provides an unfair +distribution. The first user who submits 20 documents will then occupy +all cores for quite some time and all other users would need to wait. + +This tries to find a more fair distribution among the users (strictly +meaning collectives here) of docspell. + +The job executor is a separate component that will run in its own +process. It takes the next job from the "queue" and executes the +associated task. This is used to run the document processing jobs +(text extraction, text analysis etc). + +1. The task execution should survive restarts. State and task code + must be recreated from some persisted state. + +2. The processing should be fair with respect to collectives. + +3. It must be possible to run many job executors, possibly on + different machines. This can be used to quickly enable more + processing power and removing it once the peak is over. + +4. Task execution can fail and it should be able to retry those + tasks. Reasons are that errors may be temporarily (for example + talking to a third party service), and to enable repairing without + stopping the job executor. Some errors might be easily repaired (a + program was not installed or whatever). In such a case it is good + to know that the task will be retried later. + +## Considered Options + +In contrast to other ADRs this is just some sketching of thoughts for +the current implementation. + +1. Job description are serialized and written to the database into a + table. This becomes the queue. Tasks are identified by names and a + job executor implementation must have a map of names to code to + lookup the task to perform. The tasks arguments are serialized into + a string and written to the database. Tasks must decode the + string. This can be conveniently done using JSON and the provided + circe decoders. + +2. To provide a fair execution jobs are organized into groups. When a + new job is requested from the queue, first a group is selected + using a round-robin strategy. This should ensure good enough + fairness among groups. A group maps to a collective. Within a + group, a job is selected based on priority, submitted time (fifo) + and job state (see notes about stuck jobs). + +3. Allowing multiple job executors means that getting the next job can + fail due to simultaneous running transactions. It is retried until + it succeeds. Taking a job puts in into _scheduled_ state. Each job + executor has a unique (manually supplied) id and jobs are marked + with that id once it is handed to the executor. + +4. When a task fails, its state is updated to state _stuck_. Stuck + jobs are retried in the future. The queue prefers to return stuck + jobs that are due at the specific point in time ignoring the + priority hint. + +### More Details + +A job has these properties + +- id (something random) +- group +- taskname (to choose task to run) +- submitted-date +- worker (the id of the job executor) +- state, one of: waiting, scheduled, running, stuck, cancelled, + failed, success + - waiting: job has been inserted into the queue + - scheduled: job has been handed over to some executore and is + marked with the job executor id + - running: a task is currently executing + - stuck: a task has failed and is being retried eventually + - cancelled: task has finished and there was a cancel request + - failed: task has failed, execeeded the retries + - success: task has completed successfully + +The queue has a `take` or `nextJob` operation that takes the worker-id +and a priority hint and goes roughly like this: + +- select the next group using round-robin strategy +- select all jobs with that group, where + - state is stuck and waiting time has elapsed + - state is waiting and have the given priority if possible +- jobs are ordered by submitted time, but stuck jobs whose waiting + time elapsed are preferred + +There are two priorities within a group: high and low. A configured +counting scheme determines when to select certain priority. For +example, counting scheme of `(2,1)` would select two high priority +jobs and then 1 low priority job. The `take` operation tries to prefer +this priority but falls back to the other if no job with this priority +is available. + +A group corresponds to a collective. Then all collectives get +(roughly) equal treatment. + +Once there are no jobs in the queue the executor goes into sleep and +must be waked to run again. If a job is submitted, the executors are +notified. + +### Stuck Jobs + +A job is going into _stuck_ state, if the task has failed. In this +state, the task is rerun after a while until a maximum retry count is +reached. + +The problem is how to notify all executors when the waiting time has +elapsed. If one executor puts a job into stuck state, it means that +all others should start looking into the queue again after `x` +minutes. It would be possible to tell all existing executors to +schedule themselves to wake up in the future, but this would miss all +executors that show up later. + +The waiting time is increased exponentially after each retry (`2 ^ +retry`) and it is meant as the minimum waiting time. So it is ok if +all executors wakeup periodically and check for new work. Most of the +time this should not be necessary and is just a fallback if only stuck +jobs are in the queue and nothing is submitted for a long time. If the +system is used, jobs get submitted once in a while and would awake all +executors. diff --git a/doc/adr/template.md b/modules/microsite/src/main/tut/dev/adr/template.md similarity index 100% rename from doc/adr/template.md rename to modules/microsite/src/main/tut/dev/adr/template.md diff --git a/modules/microsite/src/main/tut/doc.md b/modules/microsite/src/main/tut/doc.md new file mode 100644 index 00000000..8bd5167b --- /dev/null +++ b/modules/microsite/src/main/tut/doc.md @@ -0,0 +1,99 @@ +--- +layout: docs +position: 4 +title: Documentation +--- + +# {{page.title}} + +Docspell assists in organizing large amounts of PDF files that are +typically scanned paper documents. You can associate tags, set +correspondends, what a document is concerned with, a name, a date and +some more. If your documents are associated with this meta data, you +should be able to quickly find them later using the search +feature. But adding this manually to each document is a tedious +task. What if most of it could be attached automatically? + +## How it works + +Documents have two main properties: a correspondent (sender or +receiver that is not you) and something the document is about. Usually +it is about a person or some thing – maybe your car, or contracts +concerning some familiy member, etc. + +1. You maintain a kind of address book. It should list all possible + correspondents and the concerning people/things. This grows + incrementally with each new unknown document. +2. When docspell analyzes a document, it tries to find matches within + your address book. It can detect the correspondent and a concerning + person or thing. It will then associate this data to your + documents. +3. You can inspect what docspell has done and correct it. If docspell + has found multiple suggestions, they will be shown for you to + select one. If it is not correctly associated, very often the + correct one is just one click away. + +The set of meta data that docspell uses to draw suggestions from, must +be maintained manually. But usually, this data doesn't grow as fast as +the documents. After a while there is a quite complete address book +and only once in a while it has to be revisited. + + +## Terms + +In order to better understand these pages, some terms should be +explained first. + +### Item + +An **Item** is roughly your (pdf) document, only that an item may span +multiple files, which are called **attachments**. And an item has +**meta data** associated: + +- a **correspondent**: the other side of the communication. It can be + an organization or a person. +- a **concerning person** or **equipment**: a person or thing that + this item is about. Maybe it is an insurance contract about your + car. +- **tag**: an item can be tagged with custom tags. A tag can have a + *category*. This is intended for grouping tags, for example a + category `doctype` could be used to group tags like `bill`, + `contract`, `receipt` etc. Usually an item is not tagged with more + than one tag of a category. +- a **item date**: this is the date of the document – if this is not + set, the created date of the item is used. +- a **due date**: an optional date indicating that something has to be + done (e.g. paying a bill, submitting it) about this item until this + date +- a **direction**: one of "incoming" or "outgoing" +- a **name**: some item name, defaults to the file name of the + attachments +- some **notes**: arbitraty descriptive text. You can use markdown + here, which is appropriately formatted in the web application. + +### Collective + +The users of the application are part of a **collective**. A +**collective** is a group of users that share access to the same +items. The account name is therefore comprised of a *collective name* +and a *user name*. + +All users of a collective are equal; they have same permissions to +access all items. The items don't belong to a user, but to the +collective. + +That means, to identify yourself when signing in, you have to give the +collective name and your user name. By default it is separated by a +slash `/`, for example `smith/john`. If your user name is the same as +the collective name, you can omit one; so `smith/smith` can be +abbreviated to just `smith`. + + +## Limitations + +* Docspell currently supports only PDF files. +* The PDF view relies on the browsers capabilities. Sadly, not all + browsers can display PDF files. Some may require extra plugins. And + it's especially sad, that mobile browsers wont't display the + files. It works with the major desktop browsers (firefox, chromium), + though. diff --git a/modules/microsite/src/main/tut/doc/configure.md b/modules/microsite/src/main/tut/doc/configure.md new file mode 100644 index 00000000..3eafed62 --- /dev/null +++ b/modules/microsite/src/main/tut/doc/configure.md @@ -0,0 +1,261 @@ +--- +layout: docs +title: Configuring +--- + +# {{ page.title }} + +Docspell's executable can take one argument – a configuration file. If +that is not given, the defaults are used. The config file overrides +default values, so only values that differ from the defaults are +necessary. + +This applies to the restserver and the joex as well. + +## Important Config Options + +The configuration of both components uses separate namespaces. The +configuration for the REST server is below `docspell.server`, while +the one for joex is below `docspell.joex`. + +### JDBC + +This configures the connection to the database. This has to be +specified for the rest server and joex. By default, a H2 database in +the current `/tmp` directory is configured. + +The config looks like this (both components): + +``` +docspell.joex.jdbc { + url = ... + user = ... + password = ... +} + +docspell.server.backend.jdbc { + url = ... + user = ... + password = ... +} +``` + +The `url` is the connection to the database. It must start with +`jdbc`, followed by name of the database. The rest is specific to the +database used: it is either a path to a file for H2 or a host/database +url for MariaDB and PostgreSQL. + +When using H2, the user is `sa`, the password can be empty and the url +must include these options: + +``` +;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_SERVER=TRUE +``` + +#### Examples + +PostgreSQL: +``` +url = "jdbc:postgresql://localhost:5432/docspelldb" +``` + +MariaDB: +``` +url = "jdbc:mariadb://localhost:3306/docspelldb" +``` + +H2 +``` +url = "jdbc:h2:///path/to/a/file.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_SERVER=TRUE" +``` + +### Bind + +The host and port the http server binds to. This applies to both +components. The joex component also exposes a small REST api to +inspect its state and notify the scheduler. + +``` +docspell.server.bind { + address = localhost + port = 7880 +} +docspell.joex.bind { + address = localhost + port = 7878 +} +``` + +By default, it binds to `localhost` and some predefined port. This +must be changed, if components are on different machines. + +### baseurl + +The base url is an important setting that defines the http URL where +the corresponding component can be reached. It applies to both +components. For a joex component, the url must be resolvable from a +REST server component. The REST server also uses this url to create +absolute urls and to configure the authenication cookie. + +By default it is build using the information from the `bind` setting. + + +``` +docspell.server.baseurl = ... +docspell.joex.baseurl = ... +``` + +#### Examples + +``` +docspell.server.baseurl = "https://docspell.example.com" +docspell.joex.baseurl = "http://192.168.101.10" +``` + + +### app-id + +The `app-id` is the identifier of the corresponding instance. It *must +be unique* for all instances. By default the REST server uses `rest1` +and joex `joex1`. It is recommended to overwrite this setting to have +an explicit and stable identifier. + +``` +docspell.server.app-id = "rest1" +docspell.joex.app-id = "joex1" +``` + +### registration options + +This defines if and how new users can create accounts. There are 3 +options: + +- *closed* no new user can sign up +- *open* new users can sign up +- *invite* new users can sign up but require an invitation key + +This applies only to the REST sevrer component. + +``` +docspell.server.signup { + mode = "open" + + # If mode == 'invite', a password must be provided to generate + # invitation keys. It must not be empty. + new-invite-password = "" + + # If mode == 'invite', this is the period an invitation token is + # considered valid. + invite-time = "3 days" +} +``` + +The mode `invite` is intended to open the application only to some +users. The admin can create these invitation keys and distribute them +to the desired people. For this, the `new-invite-password` must be +given. The idea is that only the person who installs docspell knows +this. If it is not set, then invitation won't work. New invitation +keys can be generated from within the web application or via REST +calls (using `curl`, for example). + +``` +curl -X POST -d '{"password":"blabla"}' "http://localhost:7880/api/v1/open/signup/newinvite" +``` + +### Authentication + +Authentication works in two ways: + +- with an account-name / password pair +- with an authentication token + +The initial authentication must occur with an accountname/password +pair. This will generate an authentication token which is valid for a +some time. Subsequent calls to secured routes can use this token. The +token can be given as a normal http header or via a cookie header. + +These settings apply only to the REST server. + +``` +docspell.server.auth { + server-secret = "hex:caffee" # or "b64:Y2FmZmVlCg==" + session-valid = "5 minutes" +} +``` + +The `server-secret` is used to sign the token. If multiple REST +servers are deployed, all must share the same server secret. Otherwise +tokens from one instance are not valid on another instance. The secret +can be given as Base64 encoded string or in hex form. Use the prefix +`hex:` and `b64:`, respectively. + +The `session-valid` deterimens how long a token is valid. This can be +just some minutes, the web application obtains new ones +periodically. So a short time is recommended. + + +## File Format + +The format of the configuration files can be +[HOCON](https://github.com/lightbend/config/blob/master/HOCON.md#hocon-human-optimized-config-object-notation), +JSON or whatever the used [config +library](https://github.com/lightbend/config) understands. The default +values below are in HOCON format, which is recommended, since it +allows comments and has some [advanced +features](https://github.com/lightbend/config/blob/master/README.md#features-of-hocon). Please +refer to their documentation for more on this. + +Here are the default configurations. + + +## Default Config + +### Rest Server + +``` +{% include server.conf %} +``` + +### Joex + +``` +{% include joex.conf %} +``` + +## Logging + +By default, docspell logs to stdout. This works well, when managed by +systemd or other inits. Logging is done by +[logback](https://logback.qos.ch/). Please refer to its documentation +for how to configure logging. + +If you created your logback config file, it can be added as argument +to the executable using this syntax: + +``` +/path/to/docspell -Dlogback.configurationFile=/path/to/your/logging-config-file +``` + +To get started, the default config looks like this: + +``` xml + + + true + + + [%thread] %highlight(%-5level) %cyan(%logger{15}) - %msg %n + + + + + + + + +``` + +The `` means, that only log statements with level +"INFO" will be printed. But the `` above says, that for loggers with name "docspell" +statements with level "DEBUG" will be printed, too. diff --git a/modules/microsite/src/main/tut/doc/curate.md b/modules/microsite/src/main/tut/doc/curate.md new file mode 100644 index 00000000..855c51c5 --- /dev/null +++ b/modules/microsite/src/main/tut/doc/curate.md @@ -0,0 +1,77 @@ +--- +layout: docs +title: Find and Review +--- + +# {{page.title}} + +Curating the items meta data helps finding them later. This page +describes how you can quickly go through those items and correct or +amend with existing data. + +## Select New items + +After files have been uploaded and the job executor created the +corresponding items, they will show up on the main page. All items, +the job executor has created are initially marked as *New*. The option +*only New* in the left search menu can be used to select only new +items: + +
+ +
+ + +## Check selected items + +Then you can go through all new items and check their metadata: Click +on the first item to open the detail view. This shows the documents +and the meta data in the header. + +
+ +
+ + +## Modify if necessary + +To change something, click the *Edit* button in the menu above the +document view. This will open a form next to your documents. You can +compare the data with the documents and change as you like. Since the +item status is *New*, you'll see the suggestions docspell found during +processing. If there were multiple candidates, you can select another +one by clicking its name in the suggestion list. + +
+ +
+ + +When you change something in the form, it is immediatly applied. Only +when changing text fields, a click on the *Save* symbol next to the +field is required. + + +## Confirm + +If everything looks good, click the *Confirm* button to confirm the +current data. The *New* status goes away and also the suggestions are +hidden in this state. You can always go back by clicking the +*Unconfirm* button. + +
+ +
+ + +## Proceed with next item + +To look at the next item in the search results, click the *Next* +button in the menu (next to the *Edit* button). Clicking next, will +keep the current view, so you can continue checking the data. If you +are on the last item, the view switches to the listing view when +clicking *Next*. + +
+ +
diff --git a/modules/microsite/src/main/tut/doc/install.md b/modules/microsite/src/main/tut/doc/install.md new file mode 100644 index 00000000..fa8f665c --- /dev/null +++ b/modules/microsite/src/main/tut/doc/install.md @@ -0,0 +1,218 @@ +--- +layout: docs +title: Installation +--- + +# {{ page.title }} + +This page contains detailed installation instructions. For a quick +start, refer to [this page](../getit.html). + +Docspell has been developed and tested on a GNU/Linux system. It may +run on Windows and MacOS machines, too (ghostscript and tesseract are +available on these systems). But I've never tried. + +Docspell consists of two components that are started in separate +processes: + +1. *REST Server* This is the main application, providing the REST Api + and the web application. +2. *Joex* (job executor) This is the component that does the document + processing. + +They can run on multiple machines. All REST server and Joex instances +should be on the same network. It is not strictly required that they +can reach each other, but the components can then notify themselves +about new or done work. + +While this is possible, the simple setup is to start both components +once on the same machine. + +The [download page](https://github.com/eikek/docspell/releases) +provides pre-compiled packages and the [development page](dev.html) +contains build instructions. + + +## Prerequisites + +The two components have one prerequisite in common: they both require +Java to run. While this is the only requirement for the *REST server*, +the *Joex* components requires some more external programs. + +### Java + +Very often, Java is already installed. You can check this by opening a +terminal and typing `java -version`. Otherwise install Java using your +package manager or see [this site](https://adoptopenjdk.net/) for +other options. + +It is enough to install the JRE. The JDK is required, if you want to +build docspell from source. + +Docspell has been tested with Java version 1.8 (or sometimes referred +to as JRE 8 and JDK 8, respectively). The pre-build packages are also +build using JDK 8. But a later version of Java should work as well. + +The next tools are only required on machines running the *Joex* +component. + +### External Tools for Joex + +- [Ghostscript](http://pages.cs.wisc.edu/~ghost/) (the `gs` command) + is used to extract/convert PDF files into images that are then fed + to ocr. It is available on most GNU/Linux distributions. +- [Unpaper](https://github.com/Flameeyes/unpaper) is a program that + pre-processes images to yield better results when doing ocr. If this + is not installed, docspell tries without it. However, it is + recommended to install, because it [improves text + extraction](https://github.com/tesseract-ocr/tesseract/wiki/ImproveQuality) + (at the expense of a longer runtime). +- [Tesseract](https://github.com/tesseract-ocr/tesseract) is the tool + doing the OCR (converts images into text). It is a widely used open + source OCR engine. Tesseract 3 and 4 should work with docspell; you + can adopt the command line in the configuration file, if necessary. + + +### Example Debian + +On Debian this should install all joex requirements: + +``` bash +sudo apt-get install ghostscript tesseract-ocr tesseract-ocr-deu tesseract-ocr-eng unpaper +``` + +## Database + +Both components must have access to a SQL database. Docspell has +support these databases: + +- PostreSQL +- MariaDB +- H2 + +The H2 database is an interesting option for personal and mid-size +setups, as it requires no additional work. It is integrated into +docspell and works really well. It is also configured as the default +database. + +For large installations, PostgreSQL or MariaDB is recommended. Create +a database and a user with enough privileges (read, write, create +table) to that database. + +When using H2, make sure that all components access the same database +– the jdbc url must point to the same file. Then, it is important to +add the options +`;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_SERVER=TRUE` at the end +of the url. See the [default config](configure.html) for an example. + + +## Installing from ZIP files + +After extracting the zip files, you'll find a start script in the +`bin/` folder. + + +## Installing from DEB packages + +The DEB packages can be installed on Debian, or Debian based Distros: + +``` bash +$ sudo dpkg -i docspell*.deb +``` + +Then the start scripts are in your `$PATH`. Run `docspell-restserver` +or `docspell-joex` from a terminal window. + +The packages come with a systemd unit file that will be installed to +autostart the services. + + +## Running + +Run the start script (in the corresponding `bin/` directory when using +the zip files): + +``` +$ ./docspell-restserver*/bin/docspell-restserver +$ ./docspell-joex*/bin/docspell-joex +``` + +This will startup both components using the default configuration. The +configuration should be adopted to your needs. For example, the +database connection is configured to use a H2 database in the `/tmp` +directory. Please refer to the [configuration page](configure.html) +for how to create a custom config file. Once you have your config +file, simply pass it as argument to the command: + +``` +$ ./docspell-restserver*/bin/docspell-restserver /path/to/server-config.conf +$ ./docspell-joex*/bin/docspell-joex /path/to/joex-config.conf +``` + +After starting the rest server, you can reach the web application at +path `/app/index.html`, so using default values it would be +`http://localhost:7880/app/index.html`. + +You should be able to create a new account and sign in. Check the +[configuration page](configure.html) to further customize docspell. + + +### Options + +The start scripts support some options to configure the JVM. One often +used setting is the maximum heap size of the JVM. By default, java +determines it based on properties of the current machine. You can +specify it by given java startup options to the command: + +``` +$ ./docspell-restserver*/bin/docspell-restserver -J-Xmx1G -- /path/to/server-config.conf +``` + +This would limit the maximum heap to 1GB. The double slash separates +internal options and the arguments to the program. Another frequently +used option is to change the default temp directory. Usually it is +`/tmp`, but it may be desired to have a dedicated temp directory, +which can be configured: + +``` +$ ./docspell-restserver*/bin/docspell-restserver -J-Xmx1G -Djava.io.tmpdir=/path/to/othertemp -- /path/to/server-config.conf +``` + +The command: + +``` +$ ./docspell-restserver*/bin/docspell-restserver -h +``` + +gives an overview of supported options. + + +## Raspberry Pi, and similiar + +Both component can run next to each other on a raspberry pi or +similiar device. + + +### REST Server + +The REST server component runs very well on the Raspberry Pi and +similiar devices. It doesn't require much resources, because the heavy +work is done by the joex components. + + +### Joex + +Running the joex component on the Raspberry Pi is possible, but will +result in long processing times. Tested on a RPi model 3 (4 cores, 1G +RAM) processing a PDF (scanned with 300dpi) with two pages took +9:52. You can speed it up considerably by uninstalling the `unpaper` +command, because this step takes quite long. This, of course, reduces +the quality of OCR. But without `unpaper` the same sample pdf was then +processed in 1:24, a speedup of 8 minutes. + +You should limit the joex pool size to 1 and, depending on your model +and the amount of RAM, set a heap size of at least 500M +(`-J-Xmx500M`). + +For personal setups, when you don't need the processing results asap, +this can work well enough. diff --git a/modules/microsite/src/main/tut/doc/joex.md b/modules/microsite/src/main/tut/doc/joex.md new file mode 100644 index 00000000..96bca7b0 --- /dev/null +++ b/modules/microsite/src/main/tut/doc/joex.md @@ -0,0 +1,155 @@ +--- +layout: docs +title: Joex +--- + +# {{ page.title }} + +Joex is short for *Job Executor* and it is the component managing long +running tasks in docspell. One of these long running tasks is the file +processing task. + +One joex component handles the processing of all files of all +collectives/users. It requires much more resources than the rest +server component. Therefore the number of jobs that can run in +parallel is limited with respect to the hardware it is running on. + +For larger installations, it is probably better to run several joex +components on different machines. That works out of the box, as long +as all components point to the same database and use different +`app-id`s (see [configuring docspell](./configure.html)). + +When files are submitted to docspell, they are stored in the database +and all known joex components are notified about new work. Then they +compete on getting the next job from the queue. After a job finishes +and no job is waiting in the queue, joex will sleep until notified +again. It will also periodically notify itself as a fallback. + +## Scheduler and Queue + +The scheduler is the part that runs and monitors the long running +jobs. It works together with the job queue, which defines what job to +take next. + +To create a somewhat fair distribution among multiple collectives, a +collective is first chosen in a simple round-robin way. Then a job +from this collective is chosen by priority. + +There are only two priorities: low and high. A simple *counting +scheme* determines if a low prio or high prio job is selected +next. The default is `4, 1`, meaning to first select 4 high priority +jobs and then 1 low priority job, then starting over. If no such job +exists, its falls back to the other priority. + +The priority can be set on a *Source* (see +[uploads](uploading.html)). Uploading through the web application will +always use priority *high*. The idea is that while logged in, jobs are +more important that those submitted when not logged in. + + +## Scheduler Config + +The relevant part of the config file regarding the scheduler is shown +below with some explanations. + +``` +docspell.joex { + # other settings left out for brevity + + scheduler { + + # Number of processing allowed in parallel. + pool-size = 2 + + # A counting scheme determines the ratio of how high- and low-prio + # jobs are run. For example: 4,1 means run 4 high prio jobs, then + # 1 low prio and then start over. + counting-scheme = "4,1" + + # How often a failed job should be retried until it enters failed + # state. If a job fails, it becomes "stuck" and will be retried + # after a delay. + retries = 5 + + # The delay until the next try is performed for a failed job. This + # delay is increased exponentially with the number of retries. + retry-delay = "1 minute" + + # The queue size of log statements from a job. + log-buffer-size = 500 + + # If no job is left in the queue, the scheduler will wait until a + # notify is requested (using the REST interface). To also retry + # stuck jobs, it will notify itself periodically. + wakeup-period = "30 minutes" + } +} +``` + +The `pool-size` setting deterimens how many jobs run in parallel. You +need to play with this setting on your machine to find an optimal +value. + +The `counting-scheme` determines for all collectives how to select +between high and low priority jobs; as explained above. It is +currently not possible to define that per collective. + +If a job fails, it will be set to *stuck* state and retried by the +scheduler. The `retries` setting defines how many times a job is +retried until it enters the final *failed* state. The scheduler waits +some time until running the next try. This delay is given by +`retry-delay`. This is the initial delay, the time until the first +re-try (the second attempt). This time increases exponentially with +the number of retries. + +The jobs will log about what they do, which is picked up and stored +into the database asynchronously. The log events are buffered in a +queue and another thread will consume this queue and store them in the +database. The `log-buffer-size` determines the size of the queue. + +At last, there is a `wakeup-period` that determines at what interval +the joex component notifies itself to look for new jobs. If jobs get +stuck, and joex is not notified externally it could miss to +retry. Also, since networks are not reliable, a notification may not +reach a joex component. This periodic wakup is just to ensure that +jobs are eventually run. + + +## Starting on demand + +The job executor and rest server can be started multiple times. This +is especially useful for the job executor. For example, when +submitting a lot of files in a short time, you can simply startup more +job executors on other computers on your network. Maybe use your +laptop to help with processing for a while. + +You have to make sure, that all connect to the same database, and that +all have unique `app-id`s. + +Once the files have been processced you can stop the additional +executors. + +## Shutting down + +If a job executor is sleeping and not executing any jobs, you can just +quit using SIGTERM or `Ctrl-C` when running in a terminal. But if +there are jobs currently executing, it is advisable to initiate a +graceful shutdown. The job executor will then stop taking new jobs +from the queue but it will wait until all running jobs have completed +before shutting down. + +This can be done by sending a http POST request to the api of this job +executor: + +``` +curl -XPOST "http://localhost:7878/api/v1/shutdownAndExit" +``` + +If joex receives this request it will immediately stop taking new jobs +and it will quit when all running jobs are done. + +If a job executor gets terminated while there are running jobs, the +jobs are still in the current state marked to be executed by this job +executor. In order to fix this, start the job executor again. It will +search all jobs that are marked with its id and put them back into +waiting state. Then send a graceful shutdown request as shown above. diff --git a/modules/microsite/src/main/tut/doc/metadata.md b/modules/microsite/src/main/tut/doc/metadata.md new file mode 100644 index 00000000..730e0bf6 --- /dev/null +++ b/modules/microsite/src/main/tut/doc/metadata.md @@ -0,0 +1,87 @@ +--- +layout: docs +title: Adding Meta Data +--- + +# {{ page.title }} + +## Meta Data + +The processing can be controlled implicitely by the provided meta +data. The *Meta Data* page allows to manage this meta data. You can +create the following: + +- Tags +- Organizations +- Persons +- Equipments + +### Tags + +Items can be tagged with multiple custom tags (aka labels). This +allows to describe many different workflows people may have with their +documents. + +A tag can have a *category*. This is meant to group tags together. For +example, you may want to have a tag category *doctype* that is +comprised of tags like *bill*, *contract*, *receipt* and so on. Or for +workflows, a tag category *state* may exist that includes tags like +*Todo* or *Waiting*. Or you can tag items with user names to provide +"assignment" semantics. Docspell doesn't propose any workflow, but it +can help to implement some. + +The tags are *not* taken into account when processing. Docspell will +not automatically associate tags to your items. The tags are only +meant to be used manually. + + +### Organization and Person + +The organization entity represents an non-personal (organization or +company) correspondent of an item. Docspell will choose one or more +organizations when processing documents and associate the "best" match +with your item. + +The person entitiy can appear in two roles: It may be a correspondent +or the person an item is about. So a person is either a correspondent +or a concerning person. Docspell can not know which person is which, +therefore you need to tell this by checking the box "Use for +concerning person suggestion only". If this is checked, docspell will +use this person only to suggest a concerning person. Otherwise the +person is used only for correspondent suggestions. + +Document processing uses the following properties: + +- name +- websites +- e-mails + +The website an e-mails can be added as contact information. If these +three are present, you should get good matches from docspell. All +other fields of an organization and person are not used during +document processing. They might be useful when using this as a real +address book. + + +### Equipment + +The equipment entity is almost like a tag. In fact, it could be +replaced by a tag with a specific known category. The difference is +that docspell will try to find a match and associate it with your +item. The equipment represents non-personal things that an item is +about. Examples are: bills or insurances for *cars*, contracts for +*houses* or *flats*. + +Equipments don't have contact information, so the only property that +is used to find matches during document processing is its name. + + +## Document Language + +An important setting is the language of your documents. This helps OCR +and text analysis. You can select between English and German +currently. + +Go to the *Collective Settings* page and click *Document +Language*. This will set the lanugage for all your documents. It is +not (yet) possible to specify it when uploading. diff --git a/modules/microsite/src/main/tut/doc/processing.md b/modules/microsite/src/main/tut/doc/processing.md new file mode 100644 index 00000000..fc7ef681 --- /dev/null +++ b/modules/microsite/src/main/tut/doc/processing.md @@ -0,0 +1,40 @@ +--- +layout: docs +title: Processing Queue +--- + +# {{ page.title }} + + +The page *Processing Queue* shows the current state of document +processing for your uploads. + +At the top of the page a list of running jobs is shown. Below that, +the left column shows jobs that wait to be picked up by the job +executor. On the right are finished jobs. The number of finished jobs +is cut to some maximum and is also restricted by a date range. The +page refreshes itself automatically to show the progress. + +Example screenshot: + +
+ +
+ +You can cancel running jobs or remove waiting ones from the queue. If +you click on the small file symbol on finished jobs, you can inspect +its log messages again. A running job displays the job executor id +that executes the job. + +Currently the job queue executes just the document processing tasks, +but it may be used for other long running tasks in the future. + +Since job executors are shared among all collectives, it may happen +that a job is some time waiting until it is picked up by a job +executor. You can always start more job executors to help out. + +If a job fails, it is retried after some time. Only if it fails too +often (can be configured), it then is finished with *failed* state. If +processing finally fails, the item is still created, just without +suggestions. But if processing is cancelled by the user, the item is +not created. diff --git a/modules/microsite/src/main/tut/doc/uploading.md b/modules/microsite/src/main/tut/doc/uploading.md new file mode 100644 index 00000000..dbed6b38 --- /dev/null +++ b/modules/microsite/src/main/tut/doc/uploading.md @@ -0,0 +1,130 @@ +--- +layout: docs +title: Uploads +--- + +# {{page.title}} + + +This page describes, how files can get into docspell. Technically, +there is just one way: via http multipart/form-data requests. + + +## Authenticated Upload + +From within the web application there is the "Upload Files" +page. There you can select multiple files to upload. You can also +specify whether these files should become one item or if every file is +a separate item. + +When you click "Submit" the files are uploaded and stored in the +database. Then the job executor(s) are notified which immediately +start processing them. + +Go to the top-right menu and click "Processing Queue" to see the +current state. + +This obviously requires an authenticated user. While this is handy for +ad-hoc uploads, it is very inconvenient for automating it by custom +scripts. For this the next variant exists. + +## Anonymous Upload + +It is also possible to upload files without authentication. This +should make tools that interact with docspell much easier to write. + + +### Creating Anonymous Uploads + +Go to "Collective Settings" and then to the "Source" tab. A *Source* +identifies an endpoint where files can be uploaded +anonymously. Creating a new source creates a long unique id which is +part on an url that can be used to upload files. You can choose any +time to deactivate or delete the source at which point uploading is +not possible anymore. The idea is to give this URL away safely. You +can delete it any time and no passwords or secrets are visible, even +your username is not visible. + +Example screenshot: + +
+ +
+ +This example shows a source with name "test". It defines two urls: + +- `/app/index.html#/upload/` +- `/api/v1/open/upload/item/` + +The first points to a web page where everyone could upload files into +your account. You could give this url to people for sending files +directly into your docspell. + +The second url is the API url, which accepts the requests to upload +files (which is used by the first url). + +For example, this url can be used to upload files with curl: + +``` bash +$ curl -XPOST -F file=@test.pdf http://localhost:7880/api/v1/open/upload/item/5DxhjkvWf9S-CkWqF3Kr892-WgoCspFWDo7-XBykwCyAUxQ +{"success":true,"message":"Files submitted."} +``` + +You could add more `-F file=@/path/to/your/file.pdf` to upload +multiple files (note, the `@` is required by curl, so it knows that +the following is a file). + +When files are uploaded to an source endpoint, the items resulting +from this uploads are marked with the name of the source. So you know +which source an item originated. + +If files are uploaded using the web applications *Upload files* page, +the source is implicitly set to `webapp`. If you also want to let +docspell count the files uploaded through the web interface, just +create a source (can be inactive) with that name (`webapp`). + + +## The Request + +This gives more details about the request for uploads. It is a http +`multipart/form-data` request, with two possible fields: + +- meta +- file + +The `file` field can appear multiple times and is required at least +once. It is the part containing the file to upload. + +The `meta` part is completely optional and can define additional meta +data, that docspell uses to create items from the given files. It +allows to transfer structured information together with the +unstructured binary files. + +The `meta` content must be `application/json` containing this +structure: + +``` +{ multiple: Bool +, direction: Maybe String +} +``` + +The `multiple` property is by default `true`. It means that each file +in the upload request corresponds to a single item. An upload with 5 +files will result in 5 items created. If it is `false`, then docspell +will create just one item, that will then contain all files. + +Furthermore, the direction of the document (one of `incoming` or +`outgoing`) can be given. It is optional, it can be left out or +`null`. + +This kind of request is very common and most programming languages +have support for this. For example, here is another curl command +uploading two files with meta data: + +``` +curl -XPOST -F meta='{"multiple":false, "direction": "outgoing"}' \ + -F file=@letter-en-source.pdf \ + -F file=@letter-de-source.pdf \ + http://localhost:7880/api/v1/open/upload/item/5DxhjkvWf9S-CkWqF3Kr892-WgoCspFWDo7-XBykwCyAUxQ +``` diff --git a/modules/microsite/src/main/tut/getit.md b/modules/microsite/src/main/tut/getit.md new file mode 100644 index 00000000..b268963c --- /dev/null +++ b/modules/microsite/src/main/tut/getit.md @@ -0,0 +1,55 @@ +--- +layout: home +position: 3 +section: quickstart +title: Quickstart +technologies: + - first: ["Scala + Elm", "Backend is in Scala with Cats/Fs2, Webapp in Elm"] + - second: ["Unpaper + Tesseract", "Text is extracted using OCR provided by tesseract"] + - third: ["Stanford NLP", "Documents are analyzed using Stanford NLP classifiers"] +--- + +## Download + +You can download pre-compiled binaries from the [Release +Page](https://github.com/eikek/docspell/releases). There are `deb` +packages and a generic zip files. + +You need to download the two files: + +- [docspell-restserver-{{site.version}}.zip](https://github.com/eikek/docspell/releases/download/v{{site.version}}/docspell-restserver-{{site.version}}.zip) +- [docspell-joex-{{site.version}}.zip](https://github.com/eikek/docspell/releases/download/v{{site.version}}/docspell-joex-{{site.version}}.zip) + + +## Prerequisite + +Install Java (use your package manager or look +[here](https://adoptopenjdk.net/)), +[tesseract](https://github.com/tesseract-ocr/tesseract), +[ghostscript](http://pages.cs.wisc.edu/~ghost/) and possibly +[unpaper](https://github.com/Flameeyes/unpaper). The last is not +really required, but improves OCR. + + +## Running + +1. Unzip both files: + ``` bash + $ unzip docspell-*.zip + ``` +2. Open two terminal windows and navigate to the the directory + containing the zip files. +3. Start both components executing: + ``` bash + $ ./docspell-restserver*/bin/docspell-restserver + ``` + in one terminal and + ``` bash + $ ./docspell-joex*/bin/docspell-joex + ``` + in the other. +4. Point your browser to: +5. Register a new account, sign in and try it. + +Check the [documentation](doc.html) for more information on how to use +docspell. diff --git a/modules/microsite/src/main/tut/index.md b/modules/microsite/src/main/tut/index.md new file mode 100644 index 00000000..6225ccfa --- /dev/null +++ b/modules/microsite/src/main/tut/index.md @@ -0,0 +1,49 @@ +--- +layout: home +position: 1 +section: home +title: Home +technologies: + - first: ["Scala + Elm", "Backend is in Scala with Cats/Fs2, Webapp in Elm"] + - second: ["Unpaper + Tesseract", "Text is extracted using OCR provided by tesseract"] + - third: ["Stanford NLP", "Documents are analyzed using Stanford NLP classifiers"] +--- + +# A Document Organizer + +Docspell is a simple tool to cope with your piles of (digitized) paper +documents. You'll need a scanner to convert your papers into PDF +files. Docspell can then assist in organizing the resulting PDF files +easily. Its main goal is to efficiently support two major use cases: + +1. **Stowing documents away**: Most of the time documents are received + or created. It should be *fast* to stow them away, knowing that + they can be found if necessary. + + Upload the PDF files to docspell. Docspell finds meta data and will + link them to your document, automatically. There may be false + positives, so a short review is recommended. Though even if not, + the results are not that bad. +2. **Finding them**: If there is a document needed, you can search for + it. Usually, restricting to a date range and a correspondent will + result in only a few documents to sift through. Alternatively, you + can add your own tags, names etc to better match your workflow. + +The meta data that docspell uses is provided by you. You need to +maintain a list of correspondents and maybe other things you want +docspell to draw suggestions from. So if a new document arrives (from +an unknown correspondent) then you would add a new entry to your meta +data and link it manually to the document. But the next time, docspell +will do it for you. + +Docspell is *not* a document management system. There exists a lot of +these systems that have much more features. Docspell's focus is around +the two use cases described above, which already is quite useful. + +Checkout the quick [demo](demo.html) to get a first impression and the +[quickstart](getit.html) page if you want to try it out. + +## License + +This project is distributed under the +[GPLv3](http://www.gnu.org/licenses/gpl-3.0.html) diff --git a/modules/restapi/src/main/resources/docspell-openapi.yml b/modules/restapi/src/main/resources/docspell-openapi.yml index 995df6c0..e7f20e65 100644 --- a/modules/restapi/src/main/resources/docspell-openapi.yml +++ b/modules/restapi/src/main/resources/docspell-openapi.yml @@ -3,14 +3,30 @@ openapi: 3.0.0 info: title: Docspell version: 0.1.0-SNAPSHOT + description: | + This is the remote API to Docspell, a personal document organizer. servers: - url: /api/v1 description: Current host paths: + /api/info/version: + get: + tags: [ Information ] + summary: Get version information. + description: | + Returns information about this software. + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/VersionInfo" /open/auth/login: post: + tags: [ Authentication ] summary: Authenticate with account name and password. description: | Authenticate with account name and password. The account name @@ -31,8 +47,142 @@ paths: application/json: schema: $ref: "#/components/schemas/AuthResult" + /open/upload/item/{id}: + post: + tags: [ Upload ] + summary: Upload files to docspell. + description: | + Upload a file to docspell for processing. The id is a *source + id* configured by a collective. Files are submitted for + processing which eventually resuts in an item in the inbox of + the corresponding collective. + + The request must be a `multipart/form-data` request, where the + first part has name `meta`, is optional and may contain upload + metadata as JSON. Checkout the structure `ItemUploadMeta` at + the end if it is not shown here. Other parts specify the + files. Multiple files can be specified, but at least on is + required. + + The upload meta data can be used to tell, whether multiple + files are one item, or if each file should become a single + item. By default, each file will be a one item. + + Only certain file types are supported: + + * application/pdf + + Support for more types might be added. + parameters: + - $ref: "#/components/parameters/id" + requestBody: + content: + multipart/form-data: + schema: + type: object + properties: + meta: + $ref: "#/components/schemas/ItemUploadMeta" + file: + type: array + items: + type: string + format: binary + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/upload: + post: + tags: [ Upload ] + summary: Upload files to docspell. + description: | + Upload files to docspell for processing. This route is meant + for authenticated users that upload files to their account. + + Everything else is the same as with the + `/open/upload/item/{id}` endpoint. + + The request must be a "multipart/form-data" request, where the + first part is optional and may contain upload metadata as + JSON. Other parts specify the files. Multiple files can be + specified, but at least on is required. + + The upload meta data can be used to tell, whether multiple + files are one item, or if each file should become a single + item. By default, each file will be a one item. + + Only certain file types are supported: + + * application/pdf + + Support for more types might be added. + requestBody: + content: + multipart/form-data: + schema: + type: object + properties: + meta: + $ref: "#/components/schemas/ItemUploadMeta" + file: + type: array + items: + type: string + format: binary + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /open/signup/register: + post: + tags: [ Registration ] + summary: Register a new account. + description: | + Create a new account by creating a collective and user. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/Registration" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /open/signup/newinvite: + post: + tags: [ Registration ] + summary: Generate a new invite. + description: | + When signup mode is set to "invite", docspell requires an + invitation key when signing up. These keys can be created + here. Creating such keys requires an admin role, and since + docspell has no such concept, a password from the + configuration file is required for this action. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/GenInvite" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/InviteResult" /sec/auth/session: post: + tags: [ Authentication ] summary: Authentication with a token description: | Authenticate with a token. This can be used to get a new @@ -48,6 +198,7 @@ paths: $ref: "#/components/schemas/AuthResult" /sec/auth/logout: post: + tags: [ Authentication ] summary: Logout. description: | This route informs the server about a logout. This is not @@ -57,9 +208,1801 @@ paths: responses: 200: description: Ok + /sec/tag: + get: + tags: [ Tags ] + summary: Get a list of tags + description: | + Return a list of all configured tags. + security: + - authTokenHeader: [] + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/TagList" + post: + tags: [ Tags ] + summary: Create a new tag. + description: | + Create a new tag. If a tag with this name already exists, an + error is returned. The id in the input structure is ignored. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/Tag" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + put: + tags: [ Tags ] + summary: Change an existing tag. + description: | + Changes an existing tag. The tag is looked up by its id and + all properties are changed as given. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/Tag" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/tag/{id}: + delete: + tags: [ Tags ] + summary: Delete a tag. + description: | + Deletes a tag. This also removes this tags from all items. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/organization: + get: + tags: [ Organization ] + summary: Get a list of organizations. + description: | + Return a list of all organizations. Only name and id are returned. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/full" + responses: + 200: + description: Ok + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/ReferenceList" + - $ref: "#/components/schemas/OrganizationList" + post: + tags: [ Organization ] + summary: Create a new organization. + description: | + Create a new organizaion. If an organization with this name already exists, an + error is returned. The id attribute of the request structure is ignored. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/Organization" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + put: + tags: [ Organization ] + summary: Change an existing organization. + description: | + Changes an existing organization. The organization is looked up by its id and + all properties are changed as given. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/Organization" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/organization/{id}: + get: + tags: [ Organization ] + summary: Get a list of organizations. + description: | + Return details about an organization. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/Organization" + delete: + tags: [ Organization ] + summary: Delete a organization by its id. + description: | + Deletes an organization. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/person: + get: + tags: [ Person ] + summary: Get a list of persons. + description: | + Return a list of all persons. Only name and id are returned. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/full" + responses: + 200: + description: Ok + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/ReferenceList" + - $ref: "#/components/schemas/PersonList" + post: + tags: [ Person ] + summary: Create a new person. + description: | + Create a new organizaion. If an person with this name already exists, an + error is returned. The id attribute of the request structure is ignored. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/Person" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + put: + tags: [ Person ] + summary: Change an existing person. + description: | + Changes an existing person. The person is looked up by its id and + all properties are changed as given. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/Person" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/person/{id}: + get: + tags: [ Person ] + summary: Get person details. + description: | + Return details about an person. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/Person" + delete: + tags: [ Person ] + summary: Delete a person by its id. + description: | + Deletes an person. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/equipment: + get: + tags: [ Equipment ] + summary: Get a list of equipments + description: | + Return a list of all configured equipments. + security: + - authTokenHeader: [] + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/EquipmentList" + post: + tags: [ Equipment ] + summary: Create a new equipment. + description: | + Create a new equipment. If a equipment with this name already + exists, an error is returned. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/Equipment" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + put: + tags: [ Equipment ] + summary: Change an existing equipment. + description: | + Changes an existing equipment. The equipment is looked up by + its id and all properties are changed as given. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/Equipment" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/equipment/{id}: + delete: + tags: [ Equipment ] + summary: Delete a equipment. + description: | + Deletes a equipment. This also removes this equipments from + all items. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/collective: + get: + tags: [ Collective ] + summary: Get information about your collective + description: | + Return some information about the current collective. + security: + - authTokenHeader: [] + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/Collective" + /sec/collective/settings: + get: + tags: [ Collective ] + summary: Get collective settings + description: | + Return the settings of a collective. + security: + - authTokenHeader: [] + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/CollectiveSettings" + post: + tags: [ Collective ] + summary: Set document language of the collective + description: | + Updates settings for a collective, which currently is just the + document language. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/CollectiveSettings" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/Collective" + /sec/collective/insights: + get: + tags: [ Collective ] + summary: Get some insights regarding your items. + description: | + Returns some information about how many items there are, how + much space they occupy etc. + security: + - authTokenHeader: [] + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/ItemInsights" + /sec/user: + get: + tags: [ Collective ] + summary: Get a list of collective users. + description: | + Return a list of all users of the collective. + security: + - authTokenHeader: [] + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/UserList" + post: + tags: [ Collective ] + summary: Create a new collective user. + description: | + Create a new collective user. If a user with this name already + exists, an error is returned. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/User" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + put: + tags: [ Collective ] + summary: Change an existing user. + description: | + Changes an existing user. The user is looked up by + its id and all properties are changed as given. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/User" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/user/{id}: + delete: + tags: [ Collective ] + summary: Delete a user. + description: | + Deletes a user. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/user/changePassword: + post: + tags: [ Collective ] + summary: Change the password. + description: | + Allows users to change their password. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/PasswordChange" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/source: + get: + tags: [ Source ] + summary: Get a list of sources + description: | + Return a list of all configured sources. + security: + - authTokenHeader: [] + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/SourceList" + post: + tags: [ Source ] + summary: Create a new source. + description: | + Create a new source. If a source with this name already + exists, an error is returned. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/Source" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + put: + tags: [ Source ] + summary: Change an existing source. + description: | + Changes an existing source. The source is looked up by + its id and all properties are changed as given. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/Source" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/source/{id}: + delete: + tags: [ Source ] + summary: Delete a source. + description: | + Deletes a source. This also removes this sources from + all items. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/item/search: + post: + tags: [ Item ] + summary: Search for items. + description: | + Search for items given a search form. The results are grouped + by month by default. + security: + - authTokenHeader: [] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ItemSearch" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/ItemLightList" + /sec/item/{id}: + get: + tags: [ Item ] + summary: Get details about an item. + description: | + Get detailed information about an item. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/ItemDetail" + delete: + tags: [ Item ] + summary: Delete an item. + description: | + Delete an item and all its data permanently. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/item/{id}/tags: + post: + tags: [ Item ] + summary: Set new set of tags. + description: | + Update the tags associated to an item. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ReferenceList" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/item/{id}/direction: + post: + tags: [ Item ] + summary: Set the direction of an item. + description: | + Update the direction of an item. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DirectionValue" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/item/{id}/corrOrg: + post: + tags: [ Item ] + summary: Set the correspondent organization of an item. + description: | + Update the correspondent organization of an item. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/OptionalId" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/item/{id}/corrPerson: + post: + tags: [ Item ] + summary: Set the correspondent person of an item. + description: | + Update the correspondent person of an item. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/OptionalId" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/item/{id}/concPerson: + post: + tags: [ Item ] + summary: Set the concerning person of an item. + description: | + Update the concerning person of an item. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/OptionalId" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/item/{id}/concEquipment: + post: + tags: [ Item ] + summary: Set the concering equipment of an item. + description: | + Update the concering equipment of an item. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/OptionalId" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/item/{id}/notes: + post: + tags: [ Item ] + summary: Set notes of an item. + description: | + Update the notes of an item. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/OptionalText" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/item/{id}/name: + post: + tags: [ Item ] + summary: Set the name of an item. + description: | + Update the name of an item. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/OptionalText" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/item/{id}/confirm: + post: + tags: [ Item ] + summary: Confirms the current meta data of an item. + description: | + An item is initially in state "created". The user can confirm + the associated data to put it in state "confirmed". + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/item/{id}/unconfirm: + post: + tags: [ Item ] + summary: Puts an item back to created state. + description: | + If an item is confirmed it can be set back to created to + appear as new. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/item/{id}/date: + post: + tags: [ Item ] + summary: Sets the item date. + description: | + Sets the date of an item. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/OptionalDate" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/item/{id}/duedate: + post: + tags: [ Item ] + summary: Sets the items due date. + description: | + Sets the due date of an item. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/OptionalDate" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" + /sec/item/{id}/proposals: + get: + tags: [ Item ] + summary: Get a list of proposals for this item. + description: | + During text processing, a list of possible meta data has been + extracted from each attachment that may be a match to this + item. This is returned here, without duplicates. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/ItemProposals" + + /sec/attachment/{id}: + get: + tags: [ Attachment ] + summary: Get an attachment file. + description: | + Get the binary file belonging to the attachment with the given id. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/octet-stream: + schema: + type: string + format: binary + /sec/attachment/{id}/meta: + get: + tags: [ Attachment ] + summary: Get the attachment's meta data. + description: | + Get meta data for this attachment. The meta data has been + extracted from the contents. + security: + - authTokenHeader: [] + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/AttachmentMeta" + /sec/queue/state: + get: + tags: [ Job Queue ] + summary: Get complete state of job queue. + description: | + Get the current state of the job qeue. The job qeue contains + all processing tasks and other long-running operations. All + users/collectives share processing resources. + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/JobQueueState" + /sec/queue/{id}/cancel: + post: + tags: [ Job Queue ] + summary: Cancel a job. + description: | + Tries to cancel a job and remove it from the queue. If the job + is running, a cancel request is send to the corresponding joex + instance. Otherwise the job is removed from the queue. + parameters: + - $ref: "#/components/parameters/id" + responses: + 200: + description: Ok + content: + application/json: + schema: + $ref: "#/components/schemas/BasicResult" components: schemas: + GenInvite: + description: | + A request to generate a new invitation key. + required: + - password + properties: + password: + type: string + format: password + InviteResult: + description: | + The result when requesting new invitation keys. + required: + - success + - message + properties: + success: + type: boolean + message: + type: string + key: + type: string + format: ident + ItemInsights: + description: | + Information about the items in docspell. + required: + - incomingCount + - outgoingCount + - itemSize + - tagCloud + properties: + incomingCount: + type: integer + format: int32 + outgoingCount: + type: integer + format: int32 + itemSize: + type: integer + format: int64 + tagCloud: + $ref: "#/components/schemas/TagCloud" + TagCloud: + description: | + A tag "cloud" + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/NameCount" + NameCount: + description: | + Generic structure for counting something. + required: + - name + - count + properties: + name: + type: string + count: + type: integer + format: int32 + AttachmentMeta: + description: | + Extracted meta data of an attachment. + required: + - content + - labels + - proposals + properties: + content: + type: string + labels: + type: array + items: + $ref: "#/components/schemas/Label" + proposals: + $ref: "#/components/schemas/ItemProposals" + ItemProposals: + description: | + Metadata that has been suggested to an item. + required: + - corrOrg + - corrPerson + - concPerson + - concEquipment + - itemDate + - dueDate + properties: + corrOrg: + type: array + items: + $ref: "#/components/schemas/IdName" + corrPerson: + type: array + items: + $ref: "#/components/schemas/IdName" + concPerson: + type: array + items: + $ref: "#/components/schemas/IdName" + concEquipment: + type: array + items: + $ref: "#/components/schemas/IdName" + itemDate: + type: array + items: + type: integer + format: date-time + dueDate: + type: array + items: + type: integer + format: date-time + Label: + description: | + Extracted label. + required: + - labelType + - label + - beginPos + - endPos + properties: + labelType: + type: string + format: nertag + label: + type: string + beginPos: + type: integer + format: int32 + endPos: + type: integer + format: int32 + OptionalDate: + description: | + Structure for sending an optional datetime value. + properties: + date: + type: integer + format: date-time + OptionalText: + description: | + Structure for sending optional text. + properties: + text: + type: string + OptionalId: + description: | + Structure for sending optional ids. + properties: + id: + type: string + format: ident + DirectionValue: + description: | + A direction. + required: + - direction + properties: + direction: + type: string + format: direction + ItemDetail: + description: | + Detailed information about an item. + required: + - id + - direction + - name + - source + - state + - created + - updated + - attachments + - tags + properties: + id: + type: string + format: ident + direction: + type: string + format: direction + enum: + - incoming + - outgoing + name: + type: string + source: + type: string + state: + type: string + format: itemstate + created: + type: integer + format: date-time + updated: + type: integer + format: date-time + itemDate: + type: integer + format: date-time + corrOrg: + $ref: "#/components/schemas/IdName" + corrPerson: + $ref: "#/components/schemas/IdName" + concPerson: + $ref: "#/components/schemas/IdName" + concEquipment: + $ref: "#/components/schemas/IdName" + inReplyTo: + $ref: "#/components/schemas/IdName" + dueDate: + type: integer + format: date-time + notes: + type: string + attachments: + type: array + items: + $ref: "#/components/schemas/Attachment" + tags: + type: array + items: + $ref: "#/components/schemas/Tag" + Attachment: + description: | + Information about an attachment to an item. + required: + - id + - size + - contentType + properties: + id: + type: string + format: ident + name: + type: string + size: + type: integer + format: int64 + contentType: + type: string + format: mimetype + Registration: + description: | + Data for registering a new account. + required: + - collectiveName + - login + - password + properties: + collectiveName: + type: string + format: ident + login: + type: string + format: ident + password: + type: string + format: password + invite: + type: string + format: ident + JobQueueState: + description: | + Contains all information about the job queue. + required: + - progress + - completed + - queued + properties: + progress: + type: array + items: + $ref: "#/components/schemas/JobDetail" + completed: + type: array + items: + $ref: "#/components/schemas/JobDetail" + queued: + type: array + items: + $ref: "#/components/schemas/JobDetail" + JobDetail: + description: | + Details about a job. + required: + - id + - name + - submitted + - priority + - state + - retries + - logs + - progress + properties: + id: + type: string + format: ident + name: + type: string + submitted: + description: DateTime + type: integer + format: date-time + priority: + type: string + format: priority + state: + type: string + format: jobstate + enum: + - waiting + - scheduled + - running + - stuck + - failed + - canceled + - success + retries: + type: integer + format: int32 + logs: + type: array + items: + $ref: "#/components/schemas/JobLogEvent" + progress: + type: integer + format: int32 + worker: + type: string + format: ident + started: + description: DateTime + type: integer + format: date-time + finished: + type: integer + format: date-time + JobLogEvent: + description: | + A log output line. + required: + - time + - level + - message + properties: + time: + description: DateTime + type: integer + format: date-time + level: + type: string + format: loglevel + message: + type: string + PasswordChange: + description: | + Change the password, by given the old and new one. + required: + - currentPassword + - newPassword + properties: + currentPassword: + type: string + format: password + newPassword: + type: string + format: password + UserList: + description: | + A list of users. + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/User" + User: + description: | + A user of a collective. + required: + - login + - state + - loginCount + - created + properties: + login: + type: string + format: ident + state: + type: string + format: userstate + enum: + - active + - disabled + password: + type: string + format: password + email: + type: string + lastLogin: + description: DateTime + type: integer + format: date-time + loginCount: + type: integer + format: int32 + created: + description: DateTime + type: integer + format: date-time + ItemUploadMeta: + description: | + Meta information for an item upload. The user can specify some + structured information with a binary file. + + Additional metadata is not required. However, you have to + specifiy whether the corresponding files should become one + single item or if an item is created for each file. + required: + - multiple + properties: + multiple: + type: boolean + default: true + direction: + type: string + format: direction + Collective: + description: | + Information about a collective. + required: + - id + - state + - created + properties: + id: + type: string + format: ident + state: + type: string + format: collectivestate + created: + description: DateTime + type: integer + format: date-time + CollectiveSettings: + description: | + Settings for a collective. + required: + - language + properties: + language: + type: string + format: language + SourceList: + description: | + A list of sources. + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/Source" + Source: + description: | + Data about a Source. A source defines the endpoint where + docspell receives files. + required: + - id + - abbrev + - counter + - enabled + - priority + - created + properties: + id: + type: string + format: ident + abbrev: + type: string + description: + type: string + counter: + type: integer + format: int32 + enabled: + type: boolean + priority: + type: string + format: priority + created: + description: DateTime + type: integer + format: date-time + EquipmentList: + description: | + A list of equipments. + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/Equipment" + Equipment: + description: | + Some "thing" that occurs in documents. + required: + - id + - name + - created + properties: + id: + type: string + format: ident + name: + type: string + created: + description: DateTime + type: integer + format: date-time + ReferenceList: + description: + Listing of items. + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/IdName" + Person: + description: | + Basic information about a person. + required: + - id + - name + - address + - contacts + - created + - concerning + properties: + id: + type: string + format: ident + name: + type: string + address: + $ref: "#/components/schemas/Address" + contacts: + type: array + items: + $ref: "#/components/schemas/Contact" + notes: + type: string + concerning: + type: boolean + description: | + Whether this person should be used to create suggestions + for the "concerning person" association. + created: + description: DateTime + type: integer + format: date-time + PersonList: + description: | + A list of persons. + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/Person" + Organization: + description: | + An organisation. + required: + - id + - name + - address + - contacts + - created + properties: + id: + type: string + format: ident + name: + type: string + address: + $ref: "#/components/schemas/Address" + contacts: + type: array + items: + $ref: "#/components/schemas/Contact" + notes: + type: string + created: + description: DateTime + type: integer + format: date-time + OrganizationList: + description: | + A list of full organization values. + required: + - items + properties: + items: + type: array + items: + $ref: "#/components/schemas/Organization" + Address: + description: | + Postal address information. + required: + - street + - zip + - city + - country + properties: + street: + type: string + zip: + type: string + city: + type: string + country: + type: string + Contact: + description: | + Contact information. + required: + - id + - value + - kind + properties: + id: + type: string + format: ident + value: + type: string + kind: + type: string + format: contactkind + enum: + - phone + - mobile + - website + - fax + - docspell + - email + ItemLightList: + description: | + A list of item details. + required: + - groups + properties: + groups: + type: array + items: + $ref: "#/components/schemas/ItemLightGroup" + ItemLightGroup: + description: | + A group of items. + required: + - name + - items + properties: + name: + type: string + items: + type: array + items: + $ref: "#/components/schemas/ItemLight" + ItemSearch: + description: | + A structure for a search form. + required: + - tagsInclude + - tagsExclude + - inbox + properties: + tagsInclude: + type: array + items: + type: string + format: ident + tagsExclude: + type: array + items: + type: string + format: ident + inbox: + type: boolean + direction: + type: string + format: direction + enum: + - incoming + - outgoing + name: + type: string + corrOrg: + type: string + format: ident + corrPerson: + type: string + format: ident + concPerson: + type: string + format: ident + concEquip: + type: string + format: ident + dateFrom: + type: integer + format: date-time + dateUntil: + type: integer + format: date-time + dueDateFrom: + type: integer + format: date-time + dueDateUntil: + type: integer + format: date-time + ItemLight: + description: | + An item with only a few important properties. + required: + - id + - name + - state + - date + - source + - fileCount + properties: + id: + type: string + format: ident + name: + type: string + state: + type: string + format: itemstate + date: + type: integer + format: date-time + dueDate: + type: integer + format: date-time + source: + type: string + direction: + type: string + enum: + - incoming + - outgoing + corrOrg: + $ref: "#/components/schemas/IdName" + corrPerson: + $ref: "#/components/schemas/IdName" + concPerson: + $ref: "#/components/schemas/IdName" + concEquip: + $ref: "#/components/schemas/IdName" + fileCount: + type: integer + format: int32 + IdName: + description: | + The identifier and a human readable name of some entity. + required: + - id + - name + properties: + id: + type: string + format: ident + name: + type: string + BasicResult: + description: | + Some basic result of an operation. + required: + - success + - message + properties: + success: + type: boolean + message: + type: string + Tag: + description: | + A tag used to annotate items. A tag may have a category which + groups tags together. + required: + - id + - name + - created + properties: + id: + type: string + format: ident + name: + type: string + category: + type: string + created: + type: integer + format: date-time + TagList: + description: | + A list of tags. + required: + - count + - items + properties: + count: + type: integer + format: int32 + items: + type: array + items: + $ref: '#/components/schemas/Tag' UserPass: description: | Account name and password. @@ -125,3 +2068,18 @@ components: type: apiKey in: header name: X-Docspell-Auth + parameters: + id: + name: id + in: path + description: A identifier + required: true + schema: + type: string + full: + name: full + in: query + description: Whether to list full data or just name and id. + required: false + schema: + type: boolean diff --git a/modules/restserver/src/main/resources/reference.conf b/modules/restserver/src/main/resources/reference.conf index 78391234..34f3e0f2 100644 --- a/modules/restserver/src/main/resources/reference.conf +++ b/modules/restserver/src/main/resources/reference.conf @@ -1,3 +1,87 @@ -docspell.restserver { +docspell.server { + # This is shown in the top right corner of the web application + app-name = "Docspell" + + # This is the id of this node. If you run more than one server, you + # have to make sure to provide unique ids per node. + app-id = "rest1" + + # This is the base URL this application is deployed to. This is used + # to create absolute URLs and to configure the cookie. + base-url = "http://localhost:7880" + + # Where the server binds to. + bind { + address = "localhost" + port = 7880 + } + + # Authentication. + auth { + + # The secret for this server that is used to sign the authenicator + # tokens. If multiple servers are running, all must share the same + # secret. You can use base64 or hex strings (prefix with b64: and + # hex:, respectively) + server-secret = "hex:caffee" + + # How long an authentication token is valid. The web application + # will get a new one periodically. + session-valid = "5 minutes" + } + + # Configuration for the backend. + backend { + + # The database connection. + # + # By default a H2 file-based database is configured. You can + # provide a postgresql or mariadb connection here. When using H2 + # use the PostgreSQL compatibility mode and AUTO_SERVER feature. + jdbc { + url = "jdbc:h2://"${java.io.tmpdir}"/docspell-demo.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;AUTO_SERVER=TRUE" + user = "sa" + password = "" + } + + # Configuration for registering new users. + signup { + + # The mode defines if new users can signup or not. It can have + # three values: + # + # - open: every new user can sign up + # - invite: new users can sign up only if they provide a correct + # invitation key. Invitation keys can be generated by the + # server. + # - closed: signing up is disabled. + mode = "open" + + # If mode == 'invite', a password must be provided to generate + # invitation keys. It must not be empty. + new-invite-password = "" + + # If mode == 'invite', this is the period an invitation token is + # considered valid. + invite-time = "3 days" + } + + files { + # Defines the chunk size used to store bytes. This will affect + # the memory footprint when uploading and downloading files. At + # most this amount is loaded into RAM for down- and uploading. + # + # It also defines the chunk size used for the blobs inside the + # database. + chunk-size = 524288 + + # The file content types that are considered valid. Docspell + # will only pass these files to processing. The processing code + # itself has also checks for which files are supported and which + # not. This affects the uploading part and is a first check to + # avoid that 'bad' files get into the system. + valid-mime-types = [ "application/pdf" ] + } + } } \ No newline at end of file diff --git a/modules/restserver/src/main/scala/docspell/restserver/Config.scala b/modules/restserver/src/main/scala/docspell/restserver/Config.scala index 41b86264..14640661 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/Config.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/Config.scala @@ -1,19 +1,33 @@ package docspell.restserver +import docspell.backend.auth.Login +import docspell.backend.signup.{Config => SignupConfig} import docspell.store.JdbcConfig +import docspell.backend.{Config => BackendConfig} +import docspell.common._ +import scodec.bits.ByteVector case class Config(appName: String - , baseUrl: String + , appId: Ident + , baseUrl: LenientUri , bind: Config.Bind - , jdbc: JdbcConfig + , backend: BackendConfig + , auth: Login.Config ) object Config { - + val postgres = JdbcConfig(LenientUri.unsafe("jdbc:postgresql://localhost:5432/docspelldev"), "dev", "dev") + val h2 = JdbcConfig(LenientUri.unsafe("jdbc:h2:./target/docspelldev.db;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE"), "sa", "") val default: Config = - Config("Docspell", "http://localhost:7880", Config.Bind("localhost", 7880), JdbcConfig("", "", "")) - + Config("Docspell" + , Ident.unsafe("restserver1") + , LenientUri.unsafe("http://localhost:7880") + , Config.Bind("localhost", 7880) + , BackendConfig(postgres + , SignupConfig(SignupConfig.invite, Password("testpass"), Duration.hours(5 * 24)) + , BackendConfig.Files(512 * 1024, List(MimeType.pdf))) + , Login.Config(ByteVector.fromValidHex("caffee"), Duration.minutes(2))) case class Bind(address: String, port: Int) } diff --git a/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala b/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala new file mode 100644 index 00000000..f0ad397d --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/ConfigFile.scala @@ -0,0 +1,18 @@ +package docspell.restserver + +import docspell.common.pureconfig.Implicits._ +import docspell.backend.signup.{Config => SignupConfig} +import _root_.pureconfig._ +import _root_.pureconfig.generic.auto._ + +object ConfigFile { + import Implicits._ + + def loadConfig: Config = + ConfigSource.default.at("docspell.server").loadOrThrow[Config] + + object Implicits { + implicit val signupModeReader: ConfigReader[SignupConfig.Mode] = + ConfigReader[String].emap(reason(SignupConfig.Mode.fromString)) + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/Main.scala b/modules/restserver/src/main/scala/docspell/restserver/Main.scala index ca76ca40..7d8ddb7a 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/Main.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/Main.scala @@ -2,17 +2,24 @@ package docspell.restserver import cats.effect._ import cats.implicits._ + import scala.concurrent.ExecutionContext import java.util.concurrent.Executors import java.nio.file.{Files, Paths} + +import docspell.common.{Banner, ThreadFactories} import org.log4s._ object Main extends IOApp { private[this] val logger = getLogger - val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(Executors.newCachedThreadPool) + val blockingEc: ExecutionContext = ExecutionContext.fromExecutor(Executors.newCachedThreadPool( + ThreadFactories.ofName("docspell-restserver-blocking"))) val blocker = Blocker.liftExecutionContext(blockingEc) + val connectEC: ExecutionContext = ExecutionContext.fromExecutorService(Executors.newFixedThreadPool(5, + ThreadFactories.ofName("docspell-dbconnect"))) + def run(args: List[String]) = { args match { case file :: Nil => @@ -33,7 +40,14 @@ object Main extends IOApp { } } - val cfg = Config.default - RestServer.stream[IO](cfg, blocker).compile.drain.as(ExitCode.Success) + val cfg = ConfigFile.loadConfig + val banner = Banner("REST Server" + , BuildInfo.version + , BuildInfo.gitHeadCommit + , cfg.backend.jdbc.url + , Option(System.getProperty("config.file")) + , cfg.appId, cfg.baseUrl) + logger.info(s"\n${banner.render("***>")}") + RestServer.stream[IO](cfg, connectEC, blockingEc, blocker).compile.drain.as(ExitCode.Success) } } diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestApp.scala b/modules/restserver/src/main/scala/docspell/restserver/RestApp.scala index dc9b3987..7c8edf35 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestApp.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestApp.scala @@ -1,6 +1,12 @@ package docspell.restserver +import docspell.backend.BackendApp + trait RestApp[F[_]] { def init: F[Unit] + + def config: Config + + def backend: BackendApp[F] } diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala index 2a9e50ef..aed1b38b 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestAppImpl.scala @@ -1,16 +1,28 @@ package docspell.restserver +import cats.implicits._ import cats.effect._ +import docspell.backend.BackendApp +import docspell.common.NodeType -final class RestAppImpl[F[_]: Sync](cfg: Config) extends RestApp[F] { +import scala.concurrent.ExecutionContext + +final class RestAppImpl[F[_]: Sync](val config: Config, val backend: BackendApp[F]) extends RestApp[F] { def init: F[Unit] = - Sync[F].pure(()) + backend.node.register(config.appId, NodeType.Restserver, config.baseUrl) + def shutdown: F[Unit] = + backend.node.unregister(config.appId) } object RestAppImpl { - def create[F[_]: Sync](cfg: Config): Resource[F, RestApp[F]] = - Resource.liftF(Sync[F].pure(new RestAppImpl(cfg))) + def create[F[_]: ConcurrentEffect: ContextShift](cfg: Config, connectEC: ExecutionContext, httpClientEc: ExecutionContext, blocker: Blocker): Resource[F, RestApp[F]] = + for { + backend <- BackendApp(cfg.backend, connectEC, httpClientEc, blocker) + app = new RestAppImpl[F](cfg, backend) + appR <- Resource.make(app.init.map(_ => app))(_.shutdown) + } yield appR + } diff --git a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala index e16a2270..f28cbf34 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/RestServer.scala @@ -1,42 +1,69 @@ package docspell.restserver import cats.effect._ +import docspell.backend.auth.AuthToken import org.http4s.server.blaze.BlazeServerBuilder import org.http4s.implicits._ import fs2.Stream - import org.http4s.server.middleware.Logger import org.http4s.server.Router - import docspell.restserver.webapp._ +import docspell.restserver.routes._ +import org.http4s.HttpRoutes + +import scala.concurrent.ExecutionContext object RestServer { - def stream[F[_]: ConcurrentEffect](cfg: Config, blocker: Blocker) + def stream[F[_]: ConcurrentEffect](cfg: Config, connectEC: ExecutionContext, httpClientEc: ExecutionContext, blocker: Blocker) (implicit T: Timer[F], CS: ContextShift[F]): Stream[F, Nothing] = { val app = for { - restApp <- RestAppImpl.create[F](cfg) - _ <- Resource.liftF(restApp.init) + restApp <- RestAppImpl.create[F](cfg, connectEC, httpClientEc, blocker) httpApp = Router( - "/api/info" -> InfoRoutes(cfg), + "/api/info" -> routes.InfoRoutes(cfg), + "/api/v1/open/" -> openRoutes(cfg, restApp), + "/api/v1/sec/" -> Authenticate(restApp.backend.login, cfg.auth) { + token => securedRoutes(cfg, restApp, token) + }, "/app/assets" -> WebjarRoutes.appRoutes[F](blocker, cfg), "/app" -> TemplateRoutes[F](blocker, cfg) ).orNotFound - // With Middlewares in place - finalHttpApp = Logger.httpApp(false, false)(httpApp) + finalHttpApp = Logger.httpApp(logHeaders = false, logBody = false)(httpApp) } yield finalHttpApp - Stream.resource(app).flatMap(httpApp => - BlazeServerBuilder[F] - .bindHttp(cfg.bind.port, cfg.bind.address) - .withHttpApp(httpApp) - .serve + BlazeServerBuilder[F]. + bindHttp(cfg.bind.port, cfg.bind.address). + withHttpApp(httpApp). + withoutBanner. + serve) + }.drain + + + def securedRoutes[F[_]: Effect](cfg: Config, restApp: RestApp[F], token: AuthToken): HttpRoutes[F] = + Router( + "auth" -> LoginRoutes.session(restApp.backend.login, cfg), + "tag" -> TagRoutes(restApp.backend, cfg, token), + "equipment" -> EquipmentRoutes(restApp.backend, cfg, token), + "organization" -> OrganizationRoutes(restApp.backend, cfg, token), + "person" -> PersonRoutes(restApp.backend, cfg, token), + "source" -> SourceRoutes(restApp.backend, cfg, token), + "user" -> UserRoutes(restApp.backend, cfg, token), + "collective" -> CollectiveRoutes(restApp.backend, cfg, token), + "queue" -> JobQueueRoutes(restApp.backend, cfg, token), + "item" -> ItemRoutes(restApp.backend, cfg, token), + "attachment" -> AttachmentRoutes(restApp.backend, cfg, token), + "upload" -> UploadRoutes.secured(restApp.backend, cfg, token) ) - }.drain + def openRoutes[F[_]: Effect](cfg: Config, restApp: RestApp[F]): HttpRoutes[F] = + Router( + "auth" -> LoginRoutes.login(restApp.backend.login, cfg), + "signup" -> RegisterRoutes(restApp.backend, cfg), + "upload" -> UploadRoutes.open(restApp.backend, cfg) + ) } diff --git a/modules/restserver/src/main/scala/docspell/restserver/auth/CookieData.scala b/modules/restserver/src/main/scala/docspell/restserver/auth/CookieData.scala new file mode 100644 index 00000000..862aeada --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/auth/CookieData.scala @@ -0,0 +1,36 @@ +package docspell.restserver.auth + +import org.http4s._ +import org.http4s.util._ +import docspell.backend.auth._ +import docspell.common.AccountId +import docspell.restserver.Config + +case class CookieData(auth: AuthToken) { + def accountId: AccountId = auth.account + def asString: String = auth.asString + + def asCookie(cfg: Config): ResponseCookie = { + val domain = "" //cfg.baseUrl.hostAndPort + val sec = false //cfg.baseUrl.protocol.exists(_.endsWith("s")) + ResponseCookie(CookieData.cookieName, asString, domain = Some(domain), path = Some("/api/v1"), httpOnly = true, secure = sec) + } +} +object CookieData { + val cookieName = "docspell_auth" + val headerName = "X-Docspell-Auth" + + def authenticator[F[_]](r: Request[F]): Either[String, String] = + fromCookie(r) orElse fromHeader(r) + + def fromCookie[F[_]](req: Request[F]): Either[String, String] = { + for { + header <- headers.Cookie.from(req.headers).toRight("Cookie parsing error") + cookie <- header.values.toList.find(_.name == cookieName).toRight("Couldn't find the authcookie") + } yield cookie.content + } + + def fromHeader[F[_]](req: Request[F]): Either[String, String] = { + req.headers.get(CaseInsensitiveString(headerName)).map(_.value).toRight("Couldn't find an authenticator") + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala new file mode 100644 index 00000000..2beecc99 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/conv/Conversions.scala @@ -0,0 +1,337 @@ +package docspell.restserver.conv + +import java.time.{LocalDate, ZoneId} + +import fs2.Stream +import cats.implicits._ +import cats.effect.{Effect, Sync} +import docspell.common._ +import docspell.common.syntax.all._ +import docspell.restapi.model._ +import docspell.store.records._ +import Conversions._ +import bitpeace.FileMeta +import docspell.backend.ops.OCollective.{InsightData, PassChangeResult} +import docspell.backend.ops.OJob.JobCancelResult +import docspell.backend.ops.OUpload.{UploadData, UploadMeta, UploadResult} +import docspell.backend.ops.{OItem, OJob, OOrganization, OUpload} +import docspell.store.AddResult +import org.http4s.multipart.Multipart +import org.http4s.headers.`Content-Type` +import org.log4s.Logger + +trait Conversions { + + // insights + def mkItemInsights(d: InsightData): ItemInsights = + ItemInsights(d.incoming, d.outgoing, d.bytes, TagCloud(d.tags.toList.map(p => NameCount(p._1, p._2)))) + + // attachment meta + def mkAttachmentMeta(rm: RAttachmentMeta): AttachmentMeta = + AttachmentMeta(rm.content.getOrElse("") + , rm.nerlabels.map(nl => Label(nl.tag, nl.label, nl.startPosition, nl.endPosition)) + , mkItemProposals(rm.proposals)) + + + // item proposal + def mkItemProposals(ml: MetaProposalList): ItemProposals = { + def get(mpt: MetaProposalType) = + ml.find(mpt). + map(mp => mp.values.toList.map(_.ref).map(mkIdName)). + getOrElse(Nil) + def getDates(mpt: MetaProposalType): List[Timestamp] = + ml.find(mpt). + map(mp => mp.values.toList. + map(cand => cand.ref.id.id). + flatMap(str => Either.catchNonFatal(LocalDate.parse(str)).toOption). + map(_.atTime(12, 0).atZone(ZoneId.of("GMT"))). + map(zdt => Timestamp(zdt.toInstant))). + getOrElse(Nil). + distinct. + take(5) + + ItemProposals( + corrOrg = get(MetaProposalType.CorrOrg), + corrPerson = get(MetaProposalType.CorrPerson), + concPerson = get(MetaProposalType.ConcPerson), + concEquipment = get(MetaProposalType.ConcEquip), + itemDate = getDates(MetaProposalType.DocDate), + dueDate = getDates(MetaProposalType.DueDate) + ) + } + + // item detail + def mkItemDetail(data: OItem.ItemData): ItemDetail = + ItemDetail(data.item.id + , data.item.direction + , data.item.name + , data.item.source + , data.item.state + , data.item.created + , data.item.updated + , data.item.itemDate + , data.corrOrg.map(o => IdName(o.oid, o.name)) + , data.corrPerson.map(p => IdName(p.pid, p.name)) + , data.concPerson.map(p => IdName(p.pid, p.name)) + , data.concEquip.map(e => IdName(e.eid, e.name)) + , data.inReplyTo.map(mkIdName) + , data.item.dueDate + , data.item.notes + , data.attachments.map((mkAttachment _).tupled).toList + , data.tags.map(mkTag).toList) + + def mkAttachment(ra: RAttachment, m: FileMeta): Attachment = + Attachment(ra.id, ra.name, m.length, MimeType.unsafe(m.mimetype.asString)) + + // item list + + def mkQuery(m: ItemSearch, coll: Ident): OItem.Query = + OItem.Query(coll + , m.name + , if (m.inbox) Seq(ItemState.Created) else Seq(ItemState.Created, ItemState.Confirmed) + , m.direction + , m.corrPerson + , m.corrOrg + , m.concPerson + , m.concEquip + , m.tagsInclude.map(Ident.unsafe) + , m.tagsExclude.map(Ident.unsafe) + , m.dateFrom + , m.dateUntil + , m.dueDateFrom + , m.dueDateUntil + ) + + def mkItemList(v: Vector[OItem.ListItem]): ItemLightList = { + val groups = v.groupBy(item => item.date.toDate.toString.substring(0, 7)) + + def mkGroup(g: (String, Vector[OItem.ListItem])): ItemLightGroup = + ItemLightGroup(g._1, g._2.map(mkItemLight).toList) + + val gs = groups.map(mkGroup _).toList.sortWith((g1, g2) => g1.name.compareTo(g2.name) >= 0) + ItemLightList(gs) + } + + def mkItemLight(i: OItem.ListItem): ItemLight = + ItemLight(i.id, i.name, i.state, i.date, i.dueDate, i.source, i.direction.name.some, i.corrOrg.map(mkIdName), + i.corrPerson.map(mkIdName), i.concPerson.map(mkIdName), i.concEquip.map(mkIdName), i.fileCount) + + // job + def mkJobQueueState(state: OJob.CollectiveQueueState): JobQueueState = { + def desc(f: JobDetail => Option[Timestamp])(j1: JobDetail, j2: JobDetail): Boolean = { + val t1 = f(j1).getOrElse(Timestamp.Epoch) + val t2 = f(j2).getOrElse(Timestamp.Epoch) + t1.value.isAfter(t2.value) + } + def asc(f: JobDetail => Option[Timestamp])(j1: JobDetail, j2: JobDetail): Boolean = { + val t1 = f(j1).getOrElse(Timestamp.Epoch) + val t2 = f(j2).getOrElse(Timestamp.Epoch) + t1.value.isBefore(t2.value) + } + JobQueueState(state.running.map(mkJobDetail).toList.sortWith(asc(_.started)) + , state.done.map(mkJobDetail).toList.sortWith(desc(_.finished)) + , state.queued.map(mkJobDetail).toList.sortWith(asc(_.submitted.some))) + } + + def mkJobDetail(jd: OJob.JobDetail): JobDetail = + JobDetail(jd.job.id + , jd.job.subject + , jd.job.submitted + , jd.job.priority + , jd.job.state + , jd.job.retries + , jd.logs.map(mkJobLog).toList + , jd.job.progress + , jd.job.worker + , jd.job.started + , jd.job.finished) + + def mkJobLog(jl: RJobLog): JobLogEvent = + JobLogEvent(jl.created, jl.level, jl.message) + + // upload + def readMultipart[F[_]: Effect](mp: Multipart[F], logger: Logger, prio: Priority, validFileTypes: Seq[MimeType]): F[UploadData[F]] = { + def parseMeta(body: Stream[F, Byte]): F[ItemUploadMeta] = { + body.through(fs2.text.utf8Decode). + parseJsonAs[ItemUploadMeta]. + map(_.fold(ex => { + logger.error(ex)("Reading upload metadata failed.") + throw ex + }, identity)) + } + + val meta: F[(Boolean, UploadMeta)] = mp.parts.find(_.name.exists(_ equalsIgnoreCase "meta")). + map(p => parseMeta(p.body)). + map(fm => fm.map(m => (m.multiple, UploadMeta(m.direction, "webapp", validFileTypes)))). + getOrElse((true, UploadMeta(None, "webapp", validFileTypes)).pure[F]) + + val files = mp.parts. + filter(p => p.name.forall(s => !s.equalsIgnoreCase("meta"))). + map(p => OUpload.File(p.filename, p.headers.get(`Content-Type`).map(fromContentType), p.body)) + for { + metaData <- meta + _ <- Effect[F].delay(logger.debug(s"Parsed upload meta data: $metaData")) + tracker <- Ident.randomId[F] + } yield UploadData(metaData._1, metaData._2, files, prio, Some(tracker)) + } + + // organization and person + def mkOrg(v: OOrganization.OrgAndContacts): Organization = { + val ro = v.org + Organization(ro.oid, ro.name, Address(ro.street, ro.zip, ro.city, ro.country), + v.contacts.map(mkContact).toList, ro.notes, ro.created) + } + + def newOrg[F[_]: Sync](v: Organization, cid: Ident): F[OOrganization.OrgAndContacts] = { + def contacts(oid: Ident) = + v.contacts.traverse(c => newContact(c, oid.some, None)) + for { + now <- Timestamp.current[F] + oid <- Ident.randomId[F] + cont <- contacts(oid) + org = ROrganization(oid, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, now) + } yield OOrganization.OrgAndContacts(org, cont) + } + + def changeOrg[F[_]: Sync](v: Organization, cid: Ident): F[OOrganization.OrgAndContacts] = { + def contacts(oid: Ident) = + v.contacts.traverse(c => newContact(c, oid.some, None)) + for { + cont <- contacts(v.id) + org = ROrganization(v.id, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.created) + } yield OOrganization.OrgAndContacts(org, cont) + } + + def mkPerson(v: OOrganization.PersonAndContacts): Person = { + val ro = v.person + Person(ro.pid, ro.name, Address(ro.street, ro.zip, ro.city, ro.country), + v.contacts.map(mkContact).toList, ro.notes, ro.concerning, ro.created) + } + + def newPerson[F[_]: Sync](v: Person, cid: Ident): F[OOrganization.PersonAndContacts] = { + def contacts(pid: Ident) = + v.contacts.traverse(c => newContact(c, None, pid.some)) + for { + now <- Timestamp.current[F] + pid <- Ident.randomId[F] + cont <- contacts(pid) + org = RPerson(pid, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.concerning, now) + } yield OOrganization.PersonAndContacts(org, cont) + } + + def changePerson[F[_]: Sync](v: Person, cid: Ident): F[OOrganization.PersonAndContacts] = { + def contacts(pid: Ident) = + v.contacts.traverse(c => newContact(c, None, pid.some)) + for { + cont <- contacts(v.id) + org = RPerson(v.id, cid, v.name, v.address.street, v.address.zip, v.address.city, v.address.country, v.notes, v.concerning, v.created) + } yield OOrganization.PersonAndContacts(org, cont) + } + + // contact + def mkContact(rc: RContact): Contact = + Contact(rc.contactId, rc.value, rc.kind) + + def newContact[F[_]: Sync](c: Contact, oid: Option[Ident], pid: Option[Ident]): F[RContact] = + timeId.map { case (id, now) => + RContact(id, c.value, c.kind, pid, oid, now) + } + + // users + def mkUser(ru: RUser): User = + User(ru.login, ru.state, None, ru.email, ru.lastLogin, ru.loginCount, ru.created) + + def newUser[F[_]: Sync](u: User, cid: Ident): F[RUser] = + timeId.map { case (id, now) => + RUser(id, u.login, cid, u.password.getOrElse(Password.empty), u.state, u.email, u.loginCount, u.lastLogin, u.created) + } + + def changeUser(u: User, cid: Ident): RUser = + RUser(Ident.unsafe(""), u.login, cid, u.password.getOrElse(Password.empty), u.state, u.email, u.loginCount, u.lastLogin, u.created) + + // tags + + def mkTag(rt: RTag): Tag = + Tag(rt.tagId, rt.name, rt.category, rt.created) + + def newTag[F[_]: Sync](t: Tag, cid: Ident): F[RTag] = + timeId.map { case (id, now) => + RTag(id, cid, t.name, t.category, now) + } + + def changeTag(t: Tag, cid: Ident): RTag = + RTag(t.id, cid, t.name, t.category, t.created) + + + // sources + + def mkSource(s: RSource): Source = + Source(s.sid, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created) + + def newSource[F[_]: Sync](s: Source, cid: Ident): F[RSource] = + timeId.map({ case (id, now) => + RSource(id, cid, s.abbrev, s.description, 0, s.enabled, s.priority, now) + }) + + def changeSource[F[_]: Sync](s: Source, coll: Ident): RSource = + RSource(s.id, coll, s.abbrev, s.description, s.counter, s.enabled, s.priority, s.created) + + // equipment + def mkEquipment(re: REquipment): Equipment = + Equipment(re.eid, re.name, re.created) + + def newEquipment[F[_]: Sync](e: Equipment, cid: Ident): F[REquipment] = + timeId.map({ case (id, now) => + REquipment(id, cid, e.name, now) + }) + + def changeEquipment(e: Equipment, cid: Ident): REquipment = + REquipment(e.id, cid, e.name, e.created) + + // idref + + def mkIdName(ref: IdRef): IdName = + IdName(ref.id, ref.name) + + // basic result + + def basicResult(cr: JobCancelResult): BasicResult = + cr match { + case JobCancelResult.JobNotFound => BasicResult(false, "Job not found") + case JobCancelResult.CancelRequested => BasicResult(true, "Cancel was requested at the job executor") + case JobCancelResult.Removed => BasicResult(true, "The job has been removed from the queue.") + } + + def basicResult(ar: AddResult, successMsg: String): BasicResult = ar match { + case AddResult.Success => BasicResult(true, successMsg) + case AddResult.EntityExists(msg) => BasicResult(false, msg) + case AddResult.Failure(ex) => BasicResult(false, s"Internal error: ${ex.getMessage}") + } + + def basicResult(ur: OUpload.UploadResult): BasicResult = ur match { + case UploadResult.Success => BasicResult(true, "Files submitted.") + case UploadResult.NoFiles => BasicResult(false, "There were no files to submit.") + case UploadResult.NoSource => BasicResult(false, "The source id is not valid.") + } + + def basicResult(cr: PassChangeResult): BasicResult = cr match { + case PassChangeResult.Success => BasicResult(true, "Password changed.") + case PassChangeResult.UpdateFailed => BasicResult(false, "The database update failed.") + case PassChangeResult.PasswordMismatch => BasicResult(false, "The current password is incorrect.") + case PassChangeResult.UserNotFound => BasicResult(false, "User not found.") + } + + // MIME Type + + def fromContentType(header: `Content-Type`): MimeType = + MimeType(header.mediaType.mainType, header.mediaType.subType) +} + +object Conversions extends Conversions { + + private def timeId[F[_]: Sync]: F[(Ident, Timestamp)] = + for { + id <- Ident.randomId[F] + now <- Timestamp.current + } yield (id, now) +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/http4s/ResponseGenerator.scala b/modules/restserver/src/main/scala/docspell/restserver/http4s/ResponseGenerator.scala new file mode 100644 index 00000000..629a22f4 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/http4s/ResponseGenerator.scala @@ -0,0 +1,34 @@ +package docspell.restserver.http4s + +import cats.Applicative +import org.http4s.{EntityEncoder, Header, Response} +import org.http4s.dsl.Http4sDsl + +trait ResponseGenerator[F[_]] { + self: Http4sDsl[F] => + + + implicit final class EitherResponses[A,B](e: Either[A, B]) { + def toResponse(headers: Header*) + (implicit F: Applicative[F] + , w0: EntityEncoder[F, A] + , w1: EntityEncoder[F, B]): F[Response[F]] = + e.fold( + a => UnprocessableEntity(a), + b => Ok(b) + ) + } + + implicit final class OptionResponse[A](o: Option[A]) { + def toResponse(headers: Header*) + (implicit F: Applicative[F] + , w0: EntityEncoder[F, A]): F[Response[F]] = + o.map(a => Ok(a)).getOrElse(NotFound()) + } + +} + +object ResponseGenerator { + + +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/AttachmentRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/AttachmentRoutes.scala new file mode 100644 index 00000000..d1f8629f --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/AttachmentRoutes.scala @@ -0,0 +1,63 @@ +package docspell.restserver.routes + +import cats.data.NonEmptyList +import cats.effect._ +import cats.implicits._ +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.backend.ops.OItem +import docspell.common.Ident +import org.http4s.{Header, HttpRoutes, MediaType, Response} +import org.http4s.dsl.Http4sDsl +import org.http4s.headers._ +import org.http4s.circe.CirceEntityEncoder._ +import docspell.restapi.model._ +import docspell.restserver.Config +import docspell.restserver.conv.Conversions +import org.http4s.headers.ETag.EntityTag + +object AttachmentRoutes { + + def apply[F[_]: Effect](backend: BackendApp[F], cfg: Config, user: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F]{} + import dsl._ + + def makeByteResp(data: OItem.AttachmentData[F]): F[Response[F]] = { + val mt = MediaType.unsafeParse(data.meta.mimetype.asString) + val cntLen: Header = `Content-Length`.unsafeFromLong(data.meta.length) + val eTag: Header = ETag(data.meta.checksum) + val disp: Header = `Content-Disposition`("inline", Map("filename" -> data.ra.name.getOrElse(""))) + Ok(data.data.take(data.meta.length)). + map(r => r.withContentType(`Content-Type`(mt)). + withHeaders(cntLen, eTag, disp)) + } + + HttpRoutes.of { + case req @ GET -> Root / Ident(id) => + for { + fileData <- backend.item.findAttachment(id, user.account.collective) + inm = req.headers.get(`If-None-Match`).flatMap(_.tags) + matches = matchETag(fileData, inm) + resp <- if (matches) NotModified() + else fileData.map(makeByteResp).getOrElse(NotFound(BasicResult(false, "Not found"))) + } yield resp + + case GET -> Root / Ident(id) / "meta" => + for { + rm <- backend.item.findAttachmentMeta(id, user.account.collective) + md = rm.map(Conversions.mkAttachmentMeta) + resp <- md.map(Ok(_)).getOrElse(NotFound(BasicResult(false, "Not found."))) + } yield resp + } + } + + private def matchETag[F[_]]( fileData: Option[OItem.AttachmentData[F]] + , noneMatch: Option[NonEmptyList[EntityTag]]): Boolean = + (fileData, noneMatch) match { + case (Some(fd), Some(nm)) => + fd.meta.checksum == nm.head.tag + case _ => + false + } + +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/Authenticate.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/Authenticate.scala new file mode 100644 index 00000000..56bc08d3 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/Authenticate.scala @@ -0,0 +1,54 @@ +package docspell.restserver.routes + +import cats.data._ +import cats.effect._ +import cats.implicits._ +import docspell.backend.auth._ +import docspell.restserver.auth._ +import org.http4s._ +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.dsl.Http4sDsl +import org.http4s.server._ + +object Authenticate { + + def authenticateRequest[F[_]: Effect](auth: String => F[Login.Result])(req: Request[F]): F[Login.Result] = + CookieData.authenticator(req) match { + case Right(str) => auth(str) + case Left(err) => Login.Result.invalidAuth.pure[F] + } + + + def of[F[_]: Effect](S: Login[F], cfg: Login.Config)(pf: PartialFunction[AuthedRequest[F, AuthToken], F[Response[F]]]): HttpRoutes[F] = { + val dsl: Http4sDsl[F] = new Http4sDsl[F] {} + import dsl._ + + val authUser = getUser[F](S.loginSession(cfg)) + + val onFailure: AuthedRoutes[String, F] = + Kleisli(req => OptionT.liftF(Forbidden(req.authInfo))) + + val middleware: AuthMiddleware[F, AuthToken] = + AuthMiddleware(authUser, onFailure) + + middleware(AuthedRoutes.of(pf)) + } + + def apply[F[_]: Effect](S: Login[F], cfg: Login.Config)(f: AuthToken => HttpRoutes[F]): HttpRoutes[F] = { + val dsl: Http4sDsl[F] = new Http4sDsl[F] {} + import dsl._ + + val authUser = getUser[F](S.loginSession(cfg)) + + val onFailure: AuthedRoutes[String, F] = + Kleisli(req => OptionT.liftF(Forbidden(req.authInfo))) + + val middleware: AuthMiddleware[F, AuthToken] = + AuthMiddleware(authUser, onFailure) + + middleware(AuthedRoutes(authReq => f(authReq.authInfo).run(authReq.req))) + } + + private def getUser[F[_]: Effect](auth: String => F[Login.Result]): Kleisli[F, Request[F], Either[String, AuthToken]] = + Kleisli(r => authenticateRequest(auth)(r).map(_.toEither)) +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/CollectiveRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/CollectiveRoutes.scala new file mode 100644 index 00000000..863ceecc --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/CollectiveRoutes.scala @@ -0,0 +1,52 @@ +package docspell.restserver.routes + +import cats.effect._ +import cats.implicits._ +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.restapi.model._ +import docspell.restserver.Config +import docspell.restserver.conv.Conversions +import docspell.restserver.http4s.ResponseGenerator +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.circe.CirceEntityDecoder._ +import org.http4s.dsl.Http4sDsl + +object CollectiveRoutes { + + def apply[F[_]: Effect](backend: BackendApp[F], cfg: Config, user: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] with ResponseGenerator[F] {} + import dsl._ + + HttpRoutes.of { + case GET -> Root / "insights" => + for { + ins <- backend.collective.insights(user.account.collective) + resp <- Ok(Conversions.mkItemInsights(ins)) + } yield resp + + case req@POST -> Root / "settings" => + for { + settings <- req.as[CollectiveSettings] + res <- backend.collective.updateLanguage(user.account.collective, settings.language) + resp <- Ok(Conversions.basicResult(res, "Language updated.")) + } yield resp + + case GET -> Root / "settings" => + for { + collDb <- backend.collective.find(user.account.collective) + sett = collDb.map(c => CollectiveSettings(c.language)) + resp <- sett.toResponse() + } yield resp + + case GET -> Root => + for { + collDb <- backend.collective.find(user.account.collective) + coll = collDb.map(c => Collective(c.id, c.state, c.created)) + resp <- coll.toResponse() + } yield resp + } + } + +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/EquipmentRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/EquipmentRoutes.scala new file mode 100644 index 00000000..ced18702 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/EquipmentRoutes.scala @@ -0,0 +1,52 @@ +package docspell.restserver.routes + +import cats.effect._ +import cats.implicits._ +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.common.Ident +import docspell.restapi.model._ +import docspell.restserver.Config +import docspell.restserver.conv.Conversions._ +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityDecoder._ +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.dsl.Http4sDsl + +object EquipmentRoutes { + + def apply[F[_]: Effect](backend: BackendApp[F], cfg: Config, user: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F]{} + import dsl._ + + HttpRoutes.of { + case GET -> Root => + for { + data <- backend.equipment.findAll(user.account) + resp <- Ok(EquipmentList(data.map(mkEquipment).toList)) + } yield resp + + case req @ POST -> Root => + for { + data <- req.as[Equipment] + equip <- newEquipment(data, user.account.collective) + res <- backend.equipment.add(equip) + resp <- Ok(basicResult(res, "Equipment created")) + } yield resp + + case req @ PUT -> Root => + for { + data <- req.as[Equipment] + equip = changeEquipment(data, user.account.collective) + res <- backend.equipment.update(equip) + resp <- Ok(basicResult(res, "Equipment updated.")) + } yield resp + + case DELETE -> Root / Ident(id) => + for { + del <- backend.equipment.delete(id, user.account.collective) + resp <- Ok(basicResult(del, "Equipment deleted.")) + } yield resp + } + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/InfoRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/InfoRoutes.scala similarity index 78% rename from modules/restserver/src/main/scala/docspell/restserver/InfoRoutes.scala rename to modules/restserver/src/main/scala/docspell/restserver/routes/InfoRoutes.scala index cde78464..2b79c026 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/InfoRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/InfoRoutes.scala @@ -1,13 +1,11 @@ -package docspell.restserver +package docspell.restserver.routes -import cats.effect._ -import org.http4s._ +import cats.effect.Sync +import docspell.restapi.model.VersionInfo +import docspell.restserver.{BuildInfo, Config} +import org.http4s.circe.CirceEntityEncoder._ import org.http4s.HttpRoutes import org.http4s.dsl.Http4sDsl -import org.http4s.circe.CirceEntityEncoder._ - -import docspell.restapi.model._ -import docspell.restserver.BuildInfo object InfoRoutes { diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala new file mode 100644 index 00000000..ad70e74e --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ItemRoutes.scala @@ -0,0 +1,142 @@ +package docspell.restserver.routes + +import cats.effect._ +import cats.implicits._ +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.common.{Ident, ItemState} +import org.http4s.HttpRoutes +import org.http4s.dsl.Http4sDsl +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.circe.CirceEntityDecoder._ +import docspell.restapi.model._ +import docspell.restserver.Config +import docspell.common.syntax.all._ +import docspell.restserver.conv.Conversions +import org.log4s._ + +object ItemRoutes { + private[this] val logger = getLogger + + def apply[F[_]: Effect](backend: BackendApp[F], cfg: Config, user: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F]{} + import dsl._ + + HttpRoutes.of { + case req @ POST -> Root / "search" => + for { + mask <- req.as[ItemSearch] + _ <- logger.ftrace(s"Got search mask: $mask") + query = Conversions.mkQuery(mask, user.account.collective) + _ <- logger.ftrace(s"Running query: $query") + items <- backend.item.findItems(query, 100) + resp <- Ok(Conversions.mkItemList(items)) + } yield resp + + case GET -> Root / Ident(id) => + for { + item <- backend.item.findItem(id, user.account.collective) + result = item.map(Conversions.mkItemDetail) + resp <- result.map(r => Ok(r)).getOrElse(NotFound(BasicResult(false, "Not found."))) + } yield resp + + case req@POST -> Root / Ident(id) / "confirm" => + for { + res <- backend.item.setState(id, ItemState.Confirmed, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Item data confirmed")) + } yield resp + + case req@POST -> Root / Ident(id) / "unconfirm" => + for { + res <- backend.item.setState(id, ItemState.Created, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Item back to created.")) + } yield resp + + case req@POST -> Root / Ident(id) / "tags" => + for { + tags <- req.as[ReferenceList].map(_.items) + res <- backend.item.setTags(id, tags.map(_.id), user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Tags updated")) + } yield resp + + case req@POST -> Root / Ident(id) / "direction" => + for { + dir <- req.as[DirectionValue] + res <- backend.item.setDirection(id, dir.direction, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Direction updated")) + } yield resp + + case req@POST -> Root / Ident(id) / "corrOrg" => + for { + idref <- req.as[OptionalId] + res <- backend.item.setCorrOrg(id, idref.id, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Correspondent organization updated")) + } yield resp + + case req@POST -> Root / Ident(id) / "corrPerson" => + for { + idref <- req.as[OptionalId] + res <- backend.item.setCorrPerson(id, idref.id, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Correspondent person updated")) + } yield resp + + case req@POST -> Root / Ident(id) / "concPerson" => + for { + idref <- req.as[OptionalId] + res <- backend.item.setConcPerson(id, idref.id, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Concerned person updated")) + } yield resp + + case req@POST -> Root / Ident(id) / "concEquipment" => + for { + idref <- req.as[OptionalId] + res <- backend.item.setConcEquip(id, idref.id, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated")) + } yield resp + + case req@POST -> Root / Ident(id) / "notes" => + for { + text <- req.as[OptionalText] + res <- backend.item.setNotes(id, text.text, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated")) + } yield resp + + case req@POST -> Root / Ident(id) / "name" => + for { + text <- req.as[OptionalText] + res <- backend.item.setName(id, text.text.getOrElse(""), user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Concerned equipment updated")) + } yield resp + + case req@POST -> Root / Ident(id) / "duedate" => + for { + date <- req.as[OptionalDate] + _ <- logger.fdebug(s"Setting item due date to ${date.date}") + res <- backend.item.setItemDueDate(id, date.date, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Item due date updated")) + } yield resp + + case req@POST -> Root / Ident(id) / "date" => + for { + date <- req.as[OptionalDate] + _ <- logger.fdebug(s"Setting item date to ${date.date}") + res <- backend.item.setItemDate(id, date.date, user.account.collective) + resp <- Ok(Conversions.basicResult(res, "Item date updated")) + } yield resp + + case GET -> Root / Ident(id) / "proposals" => + for { + ml <- backend.item.getProposals(id, user.account.collective) + ip = Conversions.mkItemProposals(ml) + resp <- Ok(ip) + } yield resp + + case DELETE -> Root / Ident(id) => + for { + n <- backend.item.delete(id, user.account.collective) + res = BasicResult(n > 0, if (n > 0) "Item deleted" else "Item deletion failed.") + resp <- Ok(res) + } yield resp + } + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/JobQueueRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/JobQueueRoutes.scala new file mode 100644 index 00000000..d7951460 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/JobQueueRoutes.scala @@ -0,0 +1,36 @@ +package docspell.restserver.routes + +import cats.effect._ +import cats.implicits._ +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.common.Ident +import docspell.restserver.Config +import docspell.restserver.conv.Conversions +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.dsl.Http4sDsl + +object JobQueueRoutes { + + def apply[F[_]: Effect](backend: BackendApp[F], cfg: Config, user: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] {} + import dsl._ + + HttpRoutes.of { + case GET -> Root / "state" => + for { + js <- backend.job.queueState(user.account.collective, 200) + res = Conversions.mkJobQueueState(js) + resp <- Ok(res) + } yield resp + + case POST -> Root / Ident(id) / "cancel" => + for { + result <- backend.job.cancelJob(id, user.account.collective) + resp <- Ok(Conversions.basicResult(result)) + } yield resp + } + } + +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/LoginRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/LoginRoutes.scala new file mode 100644 index 00000000..9773c865 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/LoginRoutes.scala @@ -0,0 +1,58 @@ +package docspell.restserver.routes + +import cats.effect._ +import cats.implicits._ +import docspell.backend.auth._ +import docspell.restapi.model._ +import docspell.restserver._ +import docspell.restserver.auth._ +import org.http4s._ +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.circe.CirceEntityDecoder._ +import org.http4s.dsl.Http4sDsl + +object LoginRoutes { + + def login[F[_]: Effect](S: Login[F], cfg: Config): HttpRoutes[F] = { + val dsl: Http4sDsl[F] = new Http4sDsl[F] {} + import dsl._ + + HttpRoutes.of[F] { + case req@POST -> Root / "login" => + for { + up <- req.as[UserPass] + res <- S.loginUserPass(cfg.auth)(Login.UserPass(up.account, up.password)) + resp <- makeResponse(dsl, cfg, res, up.account) + } yield resp + } + } + + def session[F[_]: Effect](S: Login[F], cfg: Config): HttpRoutes[F] = { + val dsl: Http4sDsl[F] = new Http4sDsl[F] {} + import dsl._ + + HttpRoutes.of[F] { + case req @ POST -> Root / "session" => + Authenticate.authenticateRequest(S.loginSession(cfg.auth))(req). + flatMap(res => makeResponse(dsl, cfg, res, "")) + + case POST -> Root / "logout" => + Ok().map(_.addCookie(ResponseCookie(CookieData.cookieName, "", maxAge = Some(-1)))) + } + } + + def makeResponse[F[_]: Effect](dsl: Http4sDsl[F], cfg: Config, res: Login.Result, account: String): F[Response[F]] = { + import dsl._ + res match { + case Login.Result.Ok(token) => + for { + cd <- AuthToken.user(token.account, cfg.auth.serverSecret).map(CookieData.apply) + resp <- Ok(AuthResult(token.account.collective.id, token.account.user.id, true, "Login successful", Some(cd.asString), cfg.auth.sessionValid.millis)). + map(_.addCookie(cd.asCookie(cfg))) + } yield resp + case _ => + Ok(AuthResult("", account, false, "Login failed.", None, 0L)) + } + } + +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/OrganizationRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/OrganizationRoutes.scala new file mode 100644 index 00000000..31bb782c --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/OrganizationRoutes.scala @@ -0,0 +1,61 @@ +package docspell.restserver.routes + +import cats.effect._ +import cats.implicits._ +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import org.http4s.HttpRoutes +import org.http4s.dsl.Http4sDsl +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.circe.CirceEntityDecoder._ +import docspell.restapi.model._ +import docspell.restserver.Config +import docspell.restserver.conv.Conversions._ +import ParamDecoder._ +import docspell.common.Ident + +object OrganizationRoutes { + + def apply[F[_]: Effect](backend: BackendApp[F], cfg: Config, user: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F]{} + import dsl._ + + HttpRoutes.of { + case GET -> Root :? FullQueryParamMatcher(full) => + if (full.getOrElse(false)) { + for { + data <- backend.organization.findAllOrg(user.account) + resp <- Ok(OrganizationList(data.map(mkOrg).toList)) + } yield resp + } else { + for { + data <- backend.organization.findAllOrgRefs(user.account) + resp <- Ok(ReferenceList(data.map(mkIdName).toList)) + } yield resp + } + + case req @ POST -> Root => + for { + data <- req.as[Organization] + newOrg <- newOrg(data, user.account.collective) + added <- backend.organization.addOrg(newOrg) + resp <- Ok(basicResult(added, "New organization saved.")) + } yield resp + + case req @ PUT -> Root => + for { + data <- req.as[Organization] + upOrg <- changeOrg(data, user.account.collective) + update <- backend.organization.updateOrg(upOrg) + resp <- Ok(basicResult(update, "Organization updated.")) + } yield resp + + case DELETE -> Root / Ident(id) => + for { + delOrg <- backend.organization.deleteOrg(id, user.account.collective) + resp <- Ok(basicResult(delOrg, "Organization deleted.")) + } yield resp + } + } + +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/ParamDecoder.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/ParamDecoder.scala new file mode 100644 index 00000000..3516c4c5 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/ParamDecoder.scala @@ -0,0 +1,14 @@ +package docspell.restserver.routes + +import org.http4s.QueryParamDecoder +import org.http4s.dsl.impl.OptionalQueryParamDecoderMatcher + +object ParamDecoder { + + implicit val booleanDecoder: QueryParamDecoder[Boolean] = + QueryParamDecoder.fromUnsafeCast(qp => Option(qp.value).exists(_ equalsIgnoreCase "true"))("Boolean") + + object FullQueryParamMatcher extends OptionalQueryParamDecoderMatcher[Boolean]("full") + + +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/PersonRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/PersonRoutes.scala new file mode 100644 index 00000000..d49747e9 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/PersonRoutes.scala @@ -0,0 +1,65 @@ +package docspell.restserver.routes + +import cats.effect._ +import cats.implicits._ +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import org.http4s.HttpRoutes +import org.http4s.dsl.Http4sDsl +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.circe.CirceEntityDecoder._ +import docspell.restapi.model._ +import docspell.restserver.Config +import docspell.restserver.conv.Conversions._ +import docspell.common.Ident +import docspell.common.syntax.all._ +import ParamDecoder._ +import org.log4s._ + +object PersonRoutes { + private[this] val logger = getLogger + + def apply[F[_]: Effect](backend: BackendApp[F], cfg: Config, user: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F]{} + import dsl._ + + HttpRoutes.of { + case GET -> Root :? FullQueryParamMatcher(full) => + if (full.getOrElse(false)) { + for { + data <- backend.organization.findAllPerson(user.account) + resp <- Ok(PersonList(data.map(mkPerson).toList)) + } yield resp + } else { + for { + data <- backend.organization.findAllPersonRefs(user.account) + resp <- Ok(ReferenceList(data.map(mkIdName).toList)) + } yield resp + } + + case req @ POST -> Root => + for { + data <- req.as[Person] + newPer <- newPerson(data, user.account.collective) + added <- backend.organization.addPerson(newPer) + resp <- Ok(basicResult(added, "New person saved.")) + } yield resp + + case req @ PUT -> Root => + for { + data <- req.as[Person] + upPer <- changePerson(data, user.account.collective) + update <- backend.organization.updatePerson(upPer) + resp <- Ok(basicResult(update, "Person updated.")) + } yield resp + + case DELETE -> Root / Ident(id) => + for { + _ <- logger.fdebug(s"Deleting person ${id.id}") + delOrg <- backend.organization.deletePerson(id, user.account.collective) + resp <- Ok(basicResult(delOrg, "Person deleted.")) + } yield resp + } + } + +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/RegisterRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/RegisterRoutes.scala new file mode 100644 index 00000000..0b425075 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/RegisterRoutes.scala @@ -0,0 +1,68 @@ +package docspell.restserver.routes + +import cats.effect._ +import cats.implicits._ +import docspell.backend.BackendApp +import docspell.backend.ops.OCollective.RegisterData +import docspell.backend.signup.{NewInviteResult, SignupResult} +import docspell.restapi.model._ +import docspell.restserver.Config +import docspell.restserver.http4s.ResponseGenerator +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.circe.CirceEntityDecoder._ +import org.http4s.dsl.Http4sDsl +import org.log4s._ + +object RegisterRoutes { + private[this] val logger = getLogger + + def apply[F[_]: Effect](backend: BackendApp[F], cfg: Config): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] with ResponseGenerator[F] {} + import dsl._ + + HttpRoutes.of { + case req @ POST -> Root / "register" => + for { + data <- req.as[Registration] + res <- backend.signup.register(cfg.backend.signup)(convert(data)) + resp <- Ok(convert(res)) + } yield resp + + case req@ POST -> Root / "newinvite" => + for { + data <- req.as[GenInvite] + res <- backend.signup.newInvite(cfg.backend.signup)(data.password) + resp <- Ok(convert(res)) + } yield resp + } + } + + def convert(r: NewInviteResult): InviteResult = r match { + case NewInviteResult.Success(id) => + InviteResult(true, "New invitation created.", Some(id)) + case NewInviteResult.InvitationDisabled => + InviteResult(false, "Signing up is not enabled for invitations.", None) + case NewInviteResult.PasswordMismatch => + InviteResult(false, "Password is invalid.", None) + } + + + def convert(r: SignupResult): BasicResult = r match { + case SignupResult.CollectiveExists => + BasicResult(false, "A collective with this name already exists.") + case SignupResult.InvalidInvitationKey => + BasicResult(false, "Invalid invitation key.") + case SignupResult.SignupClosed => + BasicResult(false, "Sorry, registration is closed.") + case SignupResult.Failure(ex) => + logger.error(ex)("Error signing up") + BasicResult(false, s"Internal error: ${ex.getMessage}") + case SignupResult.Success => + BasicResult(true, "Signup successful") + } + + + def convert(r: Registration): RegisterData = + RegisterData(r.collectiveName, r.login, r.password, r.invite) +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/SourceRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/SourceRoutes.scala new file mode 100644 index 00000000..13e80289 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/SourceRoutes.scala @@ -0,0 +1,54 @@ +package docspell.restserver.routes + +import cats.effect._ +import cats.implicits._ +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.common.Ident +import docspell.restapi.model._ +import docspell.restserver.Config +import docspell.restserver.conv.Conversions._ +import docspell.restserver.http4s.ResponseGenerator +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityDecoder._ +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.dsl.Http4sDsl + +object SourceRoutes { + + def apply[F[_]: Effect](backend: BackendApp[F], cfg: Config, user: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] with ResponseGenerator[F] {} + import dsl._ + + HttpRoutes.of { + case GET -> Root => + for { + all <- backend.source.findAll(user.account) + res <- Ok(SourceList(all.map(mkSource).toList)) + } yield res + + case req @ POST -> Root => + for { + data <- req.as[Source] + src <- newSource(data, user.account.collective) + added <- backend.source.add(src) + resp <- Ok(basicResult(added, "Source added.")) + } yield resp + + case req @ PUT -> Root => + for { + data <- req.as[Source] + src = changeSource(data, user.account.collective) + updated <- backend.source.update(src) + resp <- Ok(basicResult(updated, "Source updated.")) + } yield resp + + case DELETE -> Root / Ident(id) => + for { + del <- backend.source.delete(id, user.account.collective) + resp <- Ok(basicResult(del, "Source deleted.")) + } yield resp + } + } + +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/TagRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/TagRoutes.scala new file mode 100644 index 00000000..7cc2879d --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/TagRoutes.scala @@ -0,0 +1,54 @@ +package docspell.restserver.routes + +import cats.effect._ +import cats.implicits._ +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.common.Ident +import docspell.restapi.model._ +import docspell.restserver.Config +import docspell.restserver.conv.Conversions._ +import docspell.restserver.http4s.ResponseGenerator +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityDecoder._ +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.dsl.Http4sDsl + +object TagRoutes { + + def apply[F[_]: Effect](backend: BackendApp[F], cfg: Config, user: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] with ResponseGenerator[F] {} + import dsl._ + + HttpRoutes.of { + case GET -> Root => + for { + all <- backend.tag.findAll(user.account) + resp <- Ok(TagList(all.size, all.map(mkTag).toList)) + } yield resp + + case req @ POST -> Root => + for { + data <- req.as[Tag] + tag <- newTag(data, user.account.collective) + res <- backend.tag.add(tag) + resp <- Ok(basicResult(res, "Tag successfully created.")) + } yield resp + + case req @ PUT -> Root => + for { + data <- req.as[Tag] + tag = changeTag(data, user.account.collective) + res <- backend.tag.update(tag) + resp <- Ok(basicResult(res, "Tag successfully updated.")) + } yield resp + + case DELETE -> Root / Ident(id) => + for { + del <- backend.tag.delete(id, user.account.collective) + resp <- Ok(basicResult(del, "Tag successfully deleted.")) + } yield resp + } + } + +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala new file mode 100644 index 00000000..eb9bf6cc --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/UploadRoutes.scala @@ -0,0 +1,51 @@ +package docspell.restserver.routes + +import cats.effect._ +import cats.implicits._ +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.common.{Ident, Priority} +import docspell.restserver.Config +import docspell.restserver.conv.Conversions._ +import docspell.restserver.http4s.ResponseGenerator +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.EntityDecoder._ +import org.http4s.dsl.Http4sDsl +import org.http4s.multipart.Multipart +import org.log4s._ + +object UploadRoutes { + private[this] val logger = getLogger + + def secured[F[_]: Effect](backend: BackendApp[F], cfg: Config, user: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] with ResponseGenerator[F] {} + import dsl._ + + HttpRoutes.of { + case req @ POST -> Root / "item" => + for { + multipart <- req.as[Multipart[F]] + updata <- readMultipart(multipart, logger, Priority.High, cfg.backend.files.validMimeTypes) + result <- backend.upload.submit(updata, user.account) + res <- Ok(basicResult(result)) + } yield res + + } + } + + def open[F[_]: Effect](backend: BackendApp[F], cfg: Config): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] with ResponseGenerator[F] {} + import dsl._ + + HttpRoutes.of { + case req @ POST -> Root / "item" / Ident(id)=> + for { + multipart <- req.as[Multipart[F]] + updata <- readMultipart(multipart, logger, Priority.Low, cfg.backend.files.validMimeTypes) + result <- backend.upload.submit(updata, id) + res <- Ok(basicResult(result)) + } yield res + } + } +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/routes/UserRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/routes/UserRoutes.scala new file mode 100644 index 00000000..cfe26fcb --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/routes/UserRoutes.scala @@ -0,0 +1,61 @@ +package docspell.restserver.routes + +import cats.effect._ +import cats.implicits._ +import docspell.backend.BackendApp +import docspell.backend.auth.AuthToken +import docspell.common.Ident +import docspell.restapi.model._ +import docspell.restserver.Config +import docspell.restserver.conv.Conversions._ +import docspell.restserver.http4s.ResponseGenerator +import org.http4s.HttpRoutes +import org.http4s.circe.CirceEntityDecoder._ +import org.http4s.circe.CirceEntityEncoder._ +import org.http4s.dsl.Http4sDsl + +object UserRoutes { + + def apply[F[_]: Effect](backend: BackendApp[F], cfg: Config, user: AuthToken): HttpRoutes[F] = { + val dsl = new Http4sDsl[F] with ResponseGenerator[F] {} + import dsl._ + + HttpRoutes.of { + case req @ POST -> Root / "changePassword" => + for { + data <- req.as[PasswordChange] + res <- backend.collective.changePassword(user.account, data.currentPassword, data.newPassword) + resp <- Ok(basicResult(res)) + } yield resp + + case GET -> Root => + for { + all <- backend.collective.listUser(user.account.collective) + res <- Ok(UserList(all.map(mkUser).toList)) + } yield res + + case req @ POST -> Root => + for { + data <- req.as[User] + nuser <- newUser(data, user.account.collective) + added <- backend.collective.add(nuser) + resp <- Ok(basicResult(added, "User created.")) + } yield resp + + case req @ PUT -> Root => + for { + data <- req.as[User] + nuser = changeUser(data, user.account.collective) + update <- backend.collective.update(nuser) + resp <- Ok(basicResult(update, "User updated.")) + } yield resp + + case DELETE -> Root / Ident(id) => + for { + ar <- backend.collective.deleteUser(id, user.account.collective) + resp <- Ok(basicResult(ar, "User deleted.")) + } yield resp + } + } + +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala b/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala new file mode 100644 index 00000000..18cb7b70 --- /dev/null +++ b/modules/restserver/src/main/scala/docspell/restserver/webapp/Flags.scala @@ -0,0 +1,26 @@ +package docspell.restserver.webapp + +import _root_.io.circe._ +import _root_.io.circe.generic.semiauto._ +import docspell.common.LenientUri +import docspell.restserver.Config +import docspell.backend.signup.{Config => SignupConfig} +import yamusca.imports._ +import yamusca.implicits._ + +case class Flags(appName: String, baseUrl: LenientUri, signupMode: SignupConfig.Mode) + +object Flags { + def apply(cfg: Config): Flags = + Flags(cfg.appName, cfg.baseUrl, cfg.backend.signup.mode) + + implicit val jsonEncoder: Encoder[Flags] = + deriveEncoder[Flags] + + implicit def yamuscaSignupModeConverter: ValueConverter[SignupConfig.Mode] = + ValueConverter.of(m => Value.fromString(m.name)) + implicit def yamuscaUriConverter: ValueConverter[LenientUri] = + ValueConverter.of(uri => Value.fromString(uri.asString)) + implicit def yamuscaValueConverter: ValueConverter[Flags] = + ValueConverter.deriveConverter[Flags] +} diff --git a/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala index 09d4935c..f603a3d1 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/webapp/TemplateRoutes.scala @@ -7,9 +7,7 @@ import org.http4s._ import org.http4s.headers._ import org.http4s.HttpRoutes import org.http4s.dsl.Http4sDsl -import org.slf4j._ -import _root_.io.circe._ -import _root_.io.circe.generic.semiauto._ +import org.log4s._ import _root_.io.circe.syntax._ import yamusca.imports._ import yamusca.implicits._ @@ -19,7 +17,7 @@ import java.util.concurrent.atomic.AtomicReference import docspell.restserver.{BuildInfo, Config} object TemplateRoutes { - private[this] val logger = LoggerFactory.getLogger(getClass) + private[this] val logger = getLogger val `text/html` = new MediaType("text", "html") @@ -78,27 +76,16 @@ object TemplateRoutes { object DocData { def apply(cfg: Config): DocData = - DocData("/app/assets" + Webjars.swaggerui, s"/app/assets/${BuildInfo.name}/${BuildInfo.version}/openapi.yml") + DocData("/app/assets" + Webjars.swaggerui, s"/app/assets/${BuildInfo.name}/${BuildInfo.version}/docspell-openapi.yml") implicit def yamuscaValueConverter: ValueConverter[DocData] = ValueConverter.deriveConverter[DocData] } - case class Flags(appName: String, baseUrl: String) - - object Flags { - def apply(cfg: Config): Flags = - Flags(cfg.appName, cfg.baseUrl) - - implicit val jsonEncoder: Encoder[Flags] = - deriveEncoder[Flags] - implicit def yamuscaValueConverter: ValueConverter[Flags] = - ValueConverter.deriveConverter[Flags] - } - case class IndexData(flags: Flags , cssUrls: Seq[String] , jsUrls: Seq[String] + , faviconBase: String , appExtraJs: String , flagsJson: String) @@ -115,9 +102,9 @@ object TemplateRoutes { "/app/assets" + Webjars.semanticui + "/semantic.min.js", s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell-app.js" ) - , - s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.js" - , Flags(cfg).asJson.spaces2 ) + , s"/app/assets/docspell-webapp/${BuildInfo.version}/favicon" + , s"/app/assets/docspell-webapp/${BuildInfo.version}/docspell.js" + , Flags(cfg).asJson.spaces2 ) implicit def yamuscaValueConverter: ValueConverter[IndexData] = ValueConverter.deriveConverter[IndexData] diff --git a/modules/restserver/src/main/scala/docspell/restserver/webapp/WebjarRoutes.scala b/modules/restserver/src/main/scala/docspell/restserver/webapp/WebjarRoutes.scala index b7c5ee6d..38c2ed8e 100644 --- a/modules/restserver/src/main/scala/docspell/restserver/webapp/WebjarRoutes.scala +++ b/modules/restserver/src/main/scala/docspell/restserver/webapp/WebjarRoutes.scala @@ -22,7 +22,7 @@ object WebjarRoutes { } def assetFilter(asset: WebjarAsset): Boolean = - List(".js", ".css", ".html", ".jpg", ".png", ".eot", ".woff", ".woff2", ".svg", ".otf", ".ttf", ".yml"). + List(".js", ".css", ".html", ".json", ".jpg", ".png", ".eot", ".woff", ".woff2", ".svg", ".otf", ".ttf", ".yml", ".xml"). exists(e => asset.asset.endsWith(e)) } diff --git a/modules/restserver/src/main/templates/doc.html b/modules/restserver/src/main/templates/doc.html index 72820155..9ec499a2 100644 --- a/modules/restserver/src/main/templates/doc.html +++ b/modules/restserver/src/main/templates/doc.html @@ -2,7 +2,7 @@ - Swagger UI + Docspell Swagger UI diff --git a/modules/restserver/src/main/templates/index.html b/modules/restserver/src/main/templates/index.html index 2ff3ee4f..48f46736 100644 --- a/modules/restserver/src/main/templates/index.html +++ b/modules/restserver/src/main/templates/index.html @@ -1,8 +1,27 @@ + + + + + + + + + + + + + + + + + + + {{ flags.appName }} {{# cssUrls }} diff --git a/modules/store/src/main/resources/db/migration/mariadb/V1.0.0__initial_database.sql b/modules/store/src/main/resources/db/migration/mariadb/V1.0.0__initial_database.sql new file mode 100644 index 00000000..21591a01 --- /dev/null +++ b/modules/store/src/main/resources/db/migration/mariadb/V1.0.0__initial_database.sql @@ -0,0 +1,210 @@ +CREATE TABLE `filemeta` ( + `id` varchar(254) not null primary key, + `timestamp` varchar(40) not null, + `mimetype` varchar(254) not null, + `length` bigint not null, + `checksum` varchar(254) not null, + `chunks` int not null, + `chunksize` int not null +); + +CREATE TABLE `filechunk` ( + fileId varchar(254) not null, + chunkNr int not null, + chunkLength int not null, + chunkData mediumblob not null, + primary key (fileId, chunkNr) +); + +CREATE TABLE `collective` ( + `cid` varchar(254) not null primary key, + `state` varchar(254) not null, + `doclang` varchar(254) not null, + `created` timestamp not null +); + +CREATE TABLE `user_` ( + `uid` varchar(254) not null primary key, + `login` varchar(254) not null, + `cid` varchar(254) not null, + `password` varchar(254) not null, + `state` varchar(254) not null, + `email` varchar(254), + `logincount` int not null, + `lastlogin` timestamp, + `created` timestamp not null, + unique (`cid`, `login`), + foreign key (`cid`) references `collective`(`cid`) +); + +CREATE TABLE `invitation` ( + `id` varchar(254) not null primary key, + `created` timestamp not null +); + +CREATE TABLE `source` ( + `sid` varchar(254) not null primary key, + `cid` varchar(254) not null, + `abbrev` varchar(254) not null, + `description` text, + `counter` int not null, + `enabled` boolean not null, + `priority` int not null, + `created` timestamp not null, + unique (`cid`, `abbrev`), + foreign key (`cid`) references `collective`(`cid`) +); + +CREATE TABLE `organization` ( + `oid` varchar(254) not null primary key, + `cid` varchar(254) not null, + `name` varchar(254) not null, + `street` varchar(254), + `zip` varchar(254), + `city` varchar(254), + `country` varchar(254), + `notes` text, + `created` timestamp not null, + unique (`cid`, `name`), + foreign key (`cid`) references `collective`(`cid`) +); + +CREATE TABLE `person` ( + `pid` varchar(254) not null primary key, + `cid` varchar(254) not null, + `name` varchar(254) not null, + `street` varchar(254), + `zip` varchar(254), + `city` varchar(254), + `country` varchar(254), + `notes` text, + `concerning` boolean not null, + `created` varchar(30) not null, + unique (`cid`, `name`), + foreign key (`cid`) references `collective`(`cid`) +); + +CREATE TABLE `contact` ( + `contactid` varchar(254) not null primary key, + `value` varchar(254) not null, + `kind` varchar(254) not null, + `pid` varchar(254), + `oid` varchar(254), + `created` timestamp not null, + foreign key (`pid`) references `person`(`pid`), + foreign key (`oid`) references `organization`(`oid`) +); + +CREATE TABLE `equipment` ( + `eid` varchar(254) not null primary key, + `cid` varchar(254) not null, + `name` varchar(254) not null, + `created` timestamp not null, + unique (`cid`,`eid`), + foreign key (`cid`) references `collective`(`cid`) +); + +CREATE TABLE `item` ( + `itemid` varchar(254) not null primary key, + `cid` varchar(254) not null, + `name` varchar(254) not null, + `itemdate` timestamp, + `source` varchar(254) not null, + `incoming` boolean not null, + `state` varchar(254) not null, + `corrorg` varchar(254), + `corrperson` varchar(254), + `concperson` varchar(254), + `concequipment` varchar(254), + `inreplyto` varchar(254), + `duedate` timestamp, + `notes` text, + `created` timestamp not null, + `updated` timestamp not null, + foreign key (`inreplyto`) references `item`(`itemid`), + foreign key (`corrorg`) references `organization`(`oid`), + foreign key (`corrperson`) references `person`(`pid`), + foreign key (`concperson`) references `person`(`pid`), + foreign key (`concequipment`) references `equipment`(`eid`), + foreign key (`cid`) references `collective`(`cid`) +); + +CREATE TABLE `attachment` ( + `attachid` varchar(254) not null primary key, + `itemid` varchar(254) not null, + `filemetaid` varchar(254) not null, + `position` int not null, + `created` timestamp not null, + `name` varchar(254), + foreign key (`itemid`) references `item`(`itemid`), + foreign key (`filemetaid`) references `filemeta`(`id`) +); + +CREATE TABLE `attachmentmeta` ( + `attachid` varchar(254) not null primary key, + `content` text, + `nerlabels` text, + `itemproposals` text, + foreign key (`attachid`) references `attachment`(`attachid`) +); + +CREATE TABLE `tag` ( + `tid` varchar(254) not null primary key, + `cid` varchar(254) not null, + `name` varchar(254) not null, + `category` varchar(254), + `created` timestamp not null, + unique (`cid`, `name`), + foreign key (`cid`) references `collective`(`cid`) +); + +CREATE TABLE `tagitem` ( + `tagitemid` varchar(254) not null primary key, + `itemid` varchar(254) not null, + `tid` varchar(254) not null, + unique (`itemid`, `tid`), + foreign key (`itemid`) references `item`(`itemid`), + foreign key (`tid`) references `tag`(`tid`) +); + +CREATE TABLE `job` ( + `jid` varchar(254) not null primary key, + `task` varchar(254) not null, + `group_` varchar(254) not null, + `args` text not null, + `subject` varchar(254) not null, + `submitted` timestamp not null, + `submitter` varchar(254) not null, + `priority` int not null, + `state` varchar(254) not null, + `retries` int not null, + `progress` int not null, + `tracker` varchar(254), + `worker` varchar(254), + `started` timestamp, + `finished` timestamp, + `startedmillis` bigint +); + +CREATE TABLE `joblog` ( + `id` varchar(254) not null primary key, + `jid` varchar(254) not null, + `level` varchar(254) not null, + `created` timestamp not null, + `message` text not null, + foreign key (`jid`) references `job`(`jid`) +); + +CREATE TABLE `jobgroupuse` ( + `groupid` varchar(254) not null, + `workerid` varchar(254) not null, + primary key (`groupid`, `workerid`) +); + +CREATE TABLE `node` ( + `id` varchar(254) not null, + `type` varchar(254) not null, + `url` varchar(254) not null, + `updated` timestamp not null, + `created` timestamp not null +) diff --git a/modules/store/src/main/resources/db/migration/postgresql/V1.0.0__initial_database.sql b/modules/store/src/main/resources/db/migration/postgresql/V1.0.0__initial_database.sql new file mode 100644 index 00000000..504a7229 --- /dev/null +++ b/modules/store/src/main/resources/db/migration/postgresql/V1.0.0__initial_database.sql @@ -0,0 +1,211 @@ + +CREATE TABLE "filemeta" ( + "id" varchar(254) not null primary key, + "timestamp" varchar(40) not null, + "mimetype" varchar(254) not null, + "length" bigint not null, + "checksum" varchar(254) not null, + "chunks" int not null, + "chunksize" int not null +); + +CREATE TABLE "filechunk" ( + fileId varchar(254) not null, + chunkNr int not null, + chunkLength int not null, + chunkData bytea not null, + primary key (fileId, chunkNr) +); + +CREATE TABLE "collective" ( + "cid" varchar(254) not null primary key, + "state" varchar(254) not null, + "doclang" varchar(254) not null, + "created" timestamp not null +); + +CREATE TABLE "user_" ( + "uid" varchar(254) not null primary key, + "login" varchar(254) not null, + "cid" varchar(254) not null, + "password" varchar(254) not null, + "state" varchar(254) not null, + "email" varchar(254), + "logincount" int not null, + "lastlogin" timestamp, + "created" timestamp not null, + unique ("cid", "login"), + foreign key ("cid") references "collective"("cid") +); + +CREATE TABLE "invitation" ( + "id" varchar(254) not null primary key, + "created" timestamp not null +); + +CREATE TABLE "source" ( + "sid" varchar(254) not null primary key, + "cid" varchar(254) not null, + "abbrev" varchar(254) not null, + "description" text, + "counter" int not null, + "enabled" boolean not null, + "priority" int not null, + "created" timestamp not null, + unique ("cid", "abbrev"), + foreign key ("cid") references "collective"("cid") +); + +CREATE TABLE "organization" ( + "oid" varchar(254) not null primary key, + "cid" varchar(254) not null, + "name" varchar(254) not null, + "street" varchar(254), + "zip" varchar(254), + "city" varchar(254), + "country" varchar(254), + "notes" text, + "created" timestamp not null, + unique ("cid", "name"), + foreign key ("cid") references "collective"("cid") +); + +CREATE TABLE "person" ( + "pid" varchar(254) not null primary key, + "cid" varchar(254) not null, + "name" varchar(254) not null, + "street" varchar(254), + "zip" varchar(254), + "city" varchar(254), + "country" varchar(254), + "notes" text, + "concerning" boolean not null, + "created" varchar(30) not null, + unique ("cid", "name"), + foreign key ("cid") references "collective"("cid") +); + +CREATE TABLE "contact" ( + "contactid" varchar(254) not null primary key, + "value" varchar(254) not null, + "kind" varchar(254) not null, + "pid" varchar(254), + "oid" varchar(254), + "created" timestamp not null, + foreign key ("pid") references "person"("pid"), + foreign key ("oid") references "organization"("oid") +); + +CREATE TABLE "equipment" ( + "eid" varchar(254) not null primary key, + "cid" varchar(254) not null, + "name" varchar(254) not null, + "created" timestamp not null, + unique ("cid","eid"), + foreign key ("cid") references "collective"("cid") +); + +CREATE TABLE "item" ( + "itemid" varchar(254) not null primary key, + "cid" varchar(254) not null, + "name" varchar(254) not null, + "itemdate" timestamp, + "source" varchar(254) not null, + "incoming" boolean not null, + "state" varchar(254) not null, + "corrorg" varchar(254), + "corrperson" varchar(254), + "concperson" varchar(254), + "concequipment" varchar(254), + "inreplyto" varchar(254), + "duedate" timestamp, + "notes" text, + "created" timestamp not null, + "updated" timestamp not null, + foreign key ("inreplyto") references "item"("itemid"), + foreign key ("corrorg") references "organization"("oid"), + foreign key ("corrperson") references "person"("pid"), + foreign key ("concperson") references "person"("pid"), + foreign key ("concequipment") references "equipment"("eid"), + foreign key ("cid") references "collective"("cid") +); + +CREATE TABLE "attachment" ( + "attachid" varchar(254) not null primary key, + "itemid" varchar(254) not null, + "filemetaid" varchar(254) not null, + "position" int not null, + "created" timestamp not null, + "name" varchar(254), + foreign key ("itemid") references "item"("itemid"), + foreign key ("filemetaid") references "filemeta"("id") +); + +CREATE TABLE "attachmentmeta" ( + "attachid" varchar(254) not null primary key, + "content" text, + "nerlabels" text, + "itemproposals" text, + foreign key ("attachid") references "attachment"("attachid") +); + +CREATE TABLE "tag" ( + "tid" varchar(254) not null primary key, + "cid" varchar(254) not null, + "name" varchar(254) not null, + "category" varchar(254), + "created" timestamp not null, + unique ("cid", "name"), + foreign key ("cid") references "collective"("cid") +); + +CREATE TABLE "tagitem" ( + "tagitemid" varchar(254) not null primary key, + "itemid" varchar(254) not null, + "tid" varchar(254) not null, + unique ("itemid", "tid"), + foreign key ("itemid") references "item"("itemid"), + foreign key ("tid") references "tag"("tid") +); + +CREATE TABLE "job" ( + "jid" varchar(254) not null primary key, + "task" varchar(254) not null, + "group_" varchar(254) not null, + "args" text not null, + "subject" varchar(254) not null, + "submitted" timestamp not null, + "submitter" varchar(254) not null, + "priority" int not null, + "state" varchar(254) not null, + "retries" int not null, + "progress" int not null, + "tracker" varchar(254), + "worker" varchar(254), + "started" timestamp, + "finished" timestamp, + "startedmillis" bigint +); + +CREATE TABLE "joblog" ( + "id" varchar(254) not null primary key, + "jid" varchar(254) not null, + "level" varchar(254) not null, + "created" timestamp not null, + "message" text not null, + foreign key ("jid") references "job"("jid") +); + +CREATE TABLE "jobgroupuse" ( + "groupid" varchar(254) not null, + "workerid" varchar(254) not null, + primary key ("groupid", "workerid") +); + +CREATE TABLE "node" ( + "id" varchar(254) not null, + "type" varchar(254) not null, + "url" varchar(254) not null, + "updated" timestamp not null, + "created" timestamp not null +) diff --git a/modules/store/src/main/scala/docspell/store/AddResult.scala b/modules/store/src/main/scala/docspell/store/AddResult.scala new file mode 100644 index 00000000..f1dc79de --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/AddResult.scala @@ -0,0 +1,43 @@ +package docspell.store + +import AddResult._ + +sealed trait AddResult { + def toEither: Either[Throwable, Unit] + def isSuccess: Boolean + + def fold[A](fa: Success.type => A, fb: EntityExists => A, fc: Failure => A): A + + def isError: Boolean = + !isSuccess +} + +object AddResult { + + def fromUpdate(e: Either[Throwable, Int]): AddResult = + e.fold(Failure, n => if (n > 0) Success else Failure(new Exception("No rows updated"))) + + case object Success extends AddResult { + def toEither = Right(()) + val isSuccess = true + def fold[A](fa: Success.type => A, fb: EntityExists => A, fc: Failure => A): A = + fa(this) + } + + case class EntityExists(msg: String) extends AddResult { + def toEither = Left(new Exception(msg)) + val isSuccess = false + def fold[A](fa: Success.type => A, fb: EntityExists => A, fc: Failure => A): A = + fb(this) + + def withMsg(msg: String): EntityExists = + EntityExists(msg) + } + + case class Failure(ex: Throwable) extends AddResult { + def toEither = Left(ex) + val isSuccess = false + def fold[A](fa: Success.type => A, fb: EntityExists => A, fc: Failure => A): A = + fc(this) + } +} diff --git a/modules/store/src/main/scala/docspell/store/JdbcConfig.scala b/modules/store/src/main/scala/docspell/store/JdbcConfig.scala index e5afc6b3..ff5960d5 100644 --- a/modules/store/src/main/scala/docspell/store/JdbcConfig.scala +++ b/modules/store/src/main/scala/docspell/store/JdbcConfig.scala @@ -1,6 +1,8 @@ package docspell.store -case class JdbcConfig(url: String +import docspell.common.LenientUri + +case class JdbcConfig(url: LenientUri , user: String , password: String ) { @@ -24,13 +26,16 @@ case class JdbcConfig(url: String sys.error("No JDBC url specified") } + override def toString: String = + s"JdbcConfig($url, $user, ***)" } object JdbcConfig { - private[this] val jdbcRegex = "jdbc\\:([^\\:]+)\\:.*".r - def extractDbmsName(jdbcUrl: String): Option[String] = - jdbcUrl match { - case jdbcRegex(n) => Some(n.toLowerCase) - case _ => None + def extractDbmsName(jdbcUrl: LenientUri): Option[String] = + jdbcUrl.scheme.head match { + case "jdbc" => + jdbcUrl.scheme.tail.headOption + case _ => + None } } diff --git a/modules/store/src/main/scala/docspell/store/Store.scala b/modules/store/src/main/scala/docspell/store/Store.scala new file mode 100644 index 00000000..fbc541bd --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/Store.scala @@ -0,0 +1,42 @@ +package docspell.store + +import bitpeace.Bitpeace +import fs2._ +import cats.effect.{Blocker, ContextShift, Effect, Resource} +import docspell.store.impl.StoreImpl +import doobie._ +import doobie.hikari.HikariTransactor + +import scala.concurrent.ExecutionContext + +trait Store[F[_]] { + + def transact[A](prg: ConnectionIO[A]): F[A] + + def transact[A](prg: Stream[ConnectionIO, A]): Stream[F, A] + + def bitpeace: Bitpeace[F] + + def add(insert: ConnectionIO[Int], exists: ConnectionIO[Boolean]): F[AddResult] +} + +object Store { + + def create[F[_]: Effect: ContextShift](jdbc: JdbcConfig + , connectEC: ExecutionContext + , blocker: Blocker): Resource[F, Store[F]] = { + + val hxa = HikariTransactor.newHikariTransactor[F](jdbc.driverClass + , jdbc.url.asString + , jdbc.user + , jdbc.password + , connectEC + , blocker) + + for { + xa <- hxa + st = new StoreImpl[F](jdbc, xa) + _ <- Resource.liftF(st.migrate) + } yield st + } +} diff --git a/modules/store/src/main/scala/docspell/store/impl/Column.scala b/modules/store/src/main/scala/docspell/store/impl/Column.scala new file mode 100644 index 00000000..1fadcdbe --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/impl/Column.scala @@ -0,0 +1,90 @@ +package docspell.store.impl + +import doobie._, doobie.implicits._ +import docspell.store.impl.DoobieSyntax._ + +case class Column(name: String, ns: String = "", alias: String = "") { + + val f = { + val col = + if (ns.isEmpty) Fragment.const(name) + else Fragment.const(ns + "." + name) + if (alias.isEmpty) col + else col ++ fr"as" ++ Fragment.const(alias) + } + + def lowerLike[A: Put](value: A): Fragment = + fr"lower(" ++ f ++ fr") LIKE $value" + + def like[A: Put](value: A): Fragment = + f ++ fr"LIKE $value" + + def is[A: Put](value: A): Fragment = + f ++ fr" = $value" + + def is[A: Put](ov: Option[A]): Fragment = ov match { + case Some(v) => f ++ fr" = $v" + case None => fr"is null" + } + + def is(c: Column): Fragment = + f ++ fr"=" ++ c.f + + def isNull: Fragment = + f ++ fr"is null" + + def isNotNull: Fragment = + f ++ fr"is not null" + + def isIn(values: Seq[Fragment]): Fragment = + f ++ fr"IN (" ++ commas(values) ++ fr")" + + def isOrDiscard[A: Put](value: Option[A]): Fragment = + value match { + case Some(v) => is(v) + case None => Fragment.empty + } + + def isOneOf[A: Put](values: Seq[A]): Fragment = { + val vals = values.map(v => sql"$v") + isIn(vals) + } + + def isNotOneOf[A: Put](values: Seq[A]): Fragment = { + val vals = values.map(v => sql"$v") + sql"(" ++ f ++ fr"is null or" ++ f ++ fr"not IN (" ++ commas(vals) ++ sql"))" + } + + def isGt[A: Put](a: A): Fragment = + f ++ fr"> $a" + + def isGt(c: Column): Fragment = + f ++ fr">" ++ c.f + + def isLt[A: Put](a: A): Fragment = + f ++ fr"< $a" + + def isLt(c: Column): Fragment = + f ++ fr"<" ++ c.f + + def setTo[A: Put](value: A): Fragment = + is(value) + + def setTo[A: Put](va: Option[A]): Fragment = + f ++ fr" = $va" + + def ++(next: Fragment): Fragment = + f.++(next) + + def prefix(ns: String): Column = + Column(name, ns) + + def as(alias: String): Column = + Column(name, ns, alias) + + def desc: Fragment = + f ++ fr"desc" + def asc: Fragment = + f ++ fr"asc" + +} diff --git a/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala b/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala new file mode 100644 index 00000000..96a83f3d --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/impl/DoobieMeta.scala @@ -0,0 +1,90 @@ +package docspell.store.impl + +import java.time.format.DateTimeFormatter +import java.time.{Instant, LocalDate} + +import docspell.common.Timestamp + +import docspell.common._ +import doobie._ +import doobie.util.log.Success +import io.circe.{Decoder, Encoder} +import docspell.common.syntax.all._ + +trait DoobieMeta { + + implicit val sqlLogging = LogHandler({ + case e @ Success(_, _, _, _) => + DoobieMeta.logger.trace("SQL " + e) + case e => + DoobieMeta.logger.error(s"SQL Failure: $e") + }) + + def jsonMeta[A](implicit d: Decoder[A], e: Encoder[A]): Meta[A] = + Meta[String].imap(str => str.parseJsonAs[A].fold(ex => throw ex, identity))(a => e.apply(a).noSpaces) + + implicit val metaCollectiveState: Meta[CollectiveState] = + Meta[String].imap(CollectiveState.unsafe)(CollectiveState.asString) + + implicit val metaUserState: Meta[UserState] = + Meta[String].imap(UserState.unsafe)(UserState.asString) + + implicit val metaPassword: Meta[Password] = + Meta[String].imap(Password(_))(_.pass) + + implicit val metaIdent: Meta[Ident] = + Meta[String].imap(Ident.unsafe)(_.id) + + implicit val metaContactKind: Meta[ContactKind] = + Meta[String].imap(ContactKind.unsafe)(_.asString) + + implicit val metaTimestamp: Meta[Timestamp] = + Meta[Instant].imap(Timestamp(_))(_.value) + + implicit val metaJobState: Meta[JobState] = + Meta[String].imap(JobState.unsafe)(_.name) + + implicit val metaDirection: Meta[Direction] = + Meta[Boolean].imap(flag => if (flag) Direction.Incoming: Direction else Direction.Outgoing: Direction)(d => Direction.isIncoming(d)) + + implicit val metaPriority: Meta[Priority] = + Meta[Int].imap(Priority.fromInt)(Priority.toInt) + + implicit val metaLogLevel: Meta[LogLevel] = + Meta[String].imap(LogLevel.unsafeString)(_.name) + + implicit val metaLenientUri: Meta[LenientUri] = + Meta[String].imap(LenientUri.unsafe)(_.asString) + + implicit val metaNodeType: Meta[NodeType] = + Meta[String].imap(NodeType.unsafe)(_.name) + + implicit val metaLocalDate: Meta[LocalDate] = + Meta[String].imap(str => LocalDate.parse(str))(_.format(DateTimeFormatter.ISO_DATE)) + + implicit val metaItemState: Meta[ItemState] = + Meta[String].imap(ItemState.unsafe)(_.name) + + implicit val metNerTag: Meta[NerTag] = + Meta[String].imap(NerTag.unsafe)(_.name) + + implicit val metaNerLabel: Meta[NerLabel] = + jsonMeta[NerLabel] + + implicit val metaNerLabelList: Meta[List[NerLabel]] = + jsonMeta[List[NerLabel]] + + implicit val metaItemProposal: Meta[MetaProposal] = + jsonMeta[MetaProposal] + + implicit val metaItemProposalList: Meta[MetaProposalList] = + jsonMeta[MetaProposalList] + + implicit val metaLanguage: Meta[Language] = + Meta[String].imap(Language.unsafe)(_.iso3) +} + +object DoobieMeta extends DoobieMeta { + import org.log4s._ + private val logger = getLogger +} diff --git a/modules/store/src/main/scala/docspell/store/impl/DoobieSyntax.scala b/modules/store/src/main/scala/docspell/store/impl/DoobieSyntax.scala new file mode 100644 index 00000000..7ea864cc --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/impl/DoobieSyntax.scala @@ -0,0 +1,91 @@ +package docspell.store.impl + +import docspell.common.Timestamp +import doobie._ +import doobie.implicits._ + +trait DoobieSyntax { + + def coalesce(f0: Fragment, fs: Fragment*): Fragment = + sql" coalesce(" ++ commas(f0 :: fs.toList) ++ sql") " + + def power2(c: Column): Fragment = + sql"power(2," ++ c.f ++ sql")" + + def commas(fs: Seq[Fragment]): Fragment = + fs.reduce(_ ++ Fragment.const(",") ++ _) + + def commas(fa: Fragment, fas: Fragment*): Fragment = + commas(fa :: fas.toList) + + def and(fs: Seq[Fragment]): Fragment = + Fragment.const(" (") ++ fs.filter(f => !isEmpty(f)).reduce(_ ++ Fragment.const(" AND ") ++ _) ++ Fragment.const(") ") + + def and(f0: Fragment, fs: Fragment*): Fragment = + and(f0 :: fs.toList) + + def or(fs: Seq[Fragment]): Fragment = + Fragment.const(" (") ++ fs.reduce(_ ++ Fragment.const(" OR ") ++ _) ++ Fragment.const(") ") + def or(f0: Fragment, fs: Fragment*): Fragment = + or(f0 :: fs.toList) + + def where(fa: Fragment): Fragment = + if (isEmpty(fa)) Fragment.empty + else Fragment.const(" WHERE ") ++ fa + + def orderBy(fa: Fragment): Fragment = + Fragment.const(" ORDER BY ") ++ fa + + def orderBy(c0: Fragment, cs: Fragment*): Fragment = + fr"ORDER BY" ++ commas(c0 :: cs.toList) + + def updateRow(table: Fragment, where: Fragment, setter: Fragment): Fragment = + Fragment.const("UPDATE ") ++ table ++ Fragment.const(" SET ") ++ setter ++ this.where(where) + + def insertRow(table: Fragment, cols: List[Column], vals: Fragment): Fragment = + Fragment.const("INSERT INTO ") ++ table ++ Fragment.const(" (") ++ + commas(cols.map(_.f)) ++ Fragment.const(") VALUES (") ++ vals ++ Fragment.const(")") + + def insertRows(table: Fragment, cols: List[Column], vals: List[Fragment]): Fragment = + Fragment.const("INSERT INTO ") ++ table ++ Fragment.const(" (") ++ + commas(cols.map(_.f)) ++ Fragment.const(") VALUES ") ++ commas(vals.map(f => sql"(" ++ f ++ sql")")) + + + def selectSimple(cols: Seq[Column], table: Fragment, where: Fragment): Fragment = + selectSimple(commas(cols.map(_.f)), table, where) + + def selectSimple(cols: Fragment, table: Fragment, where: Fragment): Fragment = + Fragment.const("SELECT ") ++ cols ++ + Fragment.const(" FROM ") ++ table ++ this.where(where) + + def selectDistinct(cols: Seq[Column], table: Fragment, where: Fragment): Fragment = + Fragment.const("SELECT DISTINCT(") ++ commas(cols.map(_.f)) ++ + Fragment.const(") FROM ") ++ table ++ this.where(where) + + +// def selectJoinCollective(cols: Seq[Column], fkCid: Column, table: Fragment, wh: Fragment): Fragment = +// selectSimple(cols.map(_.prefix("a")) +// , table ++ fr"a," ++ RCollective.table ++ fr"b" +// , if (isEmpty(wh)) fkCid.prefix("a") is RCollective.Columns.id.prefix("b") +// else and(wh, fkCid.prefix("a") is RCollective.Columns.id.prefix("b"))) + + def selectCount(col: Column, table: Fragment, where: Fragment): Fragment = + Fragment.const("SELECT COUNT(") ++ col.f ++ Fragment.const(") FROM ") ++ table ++ this.where(where) + + def deleteFrom(table: Fragment, where: Fragment): Fragment = { + fr"DELETE FROM" ++ table ++ this.where(where) + } + + def withCTE(ps: (String, Fragment)*): Fragment = { + val subsel: Seq[Fragment] = ps.map(p => Fragment.const(p._1) ++ fr"AS (" ++ p._2 ++ fr")") + fr"WITH" ++ commas(subsel) + } + + def isEmpty(fragment: Fragment): Boolean = + Fragment.empty.toString() == fragment.toString() + + def currentTime: ConnectionIO[Timestamp] = + Timestamp.current[ConnectionIO] +} + +object DoobieSyntax extends DoobieSyntax diff --git a/modules/store/src/main/scala/docspell/store/impl/Implicits.scala b/modules/store/src/main/scala/docspell/store/impl/Implicits.scala new file mode 100644 index 00000000..ac357814 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/impl/Implicits.scala @@ -0,0 +1,5 @@ +package docspell.store.impl + + +object Implicits extends DoobieMeta + with DoobieSyntax diff --git a/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala b/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala new file mode 100644 index 00000000..8889abc4 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/impl/StoreImpl.scala @@ -0,0 +1,37 @@ +package docspell.store.impl + +import bitpeace.{Bitpeace, BitpeaceConfig, TikaMimetypeDetect} +import cats.effect.Effect +import cats.implicits._ +import docspell.common.Ident +import docspell.store.migrate.FlywayMigrate +import docspell.store.{AddResult, JdbcConfig, Store} +import doobie._ +import doobie.implicits._ + +final class StoreImpl[F[_]: Effect](jdbc: JdbcConfig, xa: Transactor[F]) extends Store[F] { + val bitpeaceCfg = BitpeaceConfig("filemeta", "filechunk", TikaMimetypeDetect, Ident.randomId[F].map(_.id)) + + def migrate: F[Int] = + FlywayMigrate.run[F](jdbc) + + def transact[A](prg: doobie.ConnectionIO[A]): F[A] = + prg.transact(xa) + + def transact[A](prg: fs2.Stream[doobie.ConnectionIO, A]): fs2.Stream[F, A] = + prg.transact(xa) + + def bitpeace: Bitpeace[F] = + Bitpeace(bitpeaceCfg, xa) + + def add(insert: ConnectionIO[Int], exists: ConnectionIO[Boolean]): F[AddResult] = { + for { + save <- transact(insert).attempt + exist <- save.swap.traverse(ex => transact(exists).map(b => (ex, b))) + } yield exist.swap match { + case Right(_) => AddResult.Success + case Left((_, true)) => AddResult.EntityExists("Adding failed, because the entity already exists.") + case Left((ex, _)) => AddResult.Failure(ex) + } + } +} diff --git a/modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala b/modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala new file mode 100644 index 00000000..5c327f3b --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/migrate/FlywayMigrate.scala @@ -0,0 +1,32 @@ +package docspell.store.migrate + +import cats.effect.Sync +import docspell.store.JdbcConfig +import org.flywaydb.core.Flyway +import org.log4s._ + +object FlywayMigrate { + private[this] val logger = getLogger + + def run[F[_]: Sync](jdbc: JdbcConfig): F[Int] = Sync[F].delay { + logger.info("Running db migrations...") + val locations = jdbc.dbmsName match { + case Some(dbtype) => + val name = if (dbtype == "h2") "postgresql" else dbtype + List("classpath:db/migration/common", s"classpath:db/migration/${name}") + case None => + logger.warn(s"Cannot read database name from jdbc url: ${jdbc.url}. Go with H2") + List("classpath:db/migration/common", "classpath:db/h2") + } + + logger.info(s"Using migration locations: $locations") + val fw = Flyway.configure(). + cleanDisabled(true). + dataSource(jdbc.url.asString, jdbc.user, jdbc.password). + locations(locations: _*). + load() + + fw.repair() + fw.migrate() + } +} diff --git a/modules/store/src/main/scala/docspell/store/ops/ONode.scala b/modules/store/src/main/scala/docspell/store/ops/ONode.scala new file mode 100644 index 00000000..2d18a6e5 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/ops/ONode.scala @@ -0,0 +1,36 @@ +package docspell.store.ops + +import cats.effect.{Effect, Resource} +import cats.implicits._ +import docspell.common.syntax.all._ +import docspell.common.{Ident, LenientUri, NodeType} +import docspell.store.Store +import docspell.store.records.RNode +import org.log4s._ + +trait ONode[F[_]] { + + def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] + + def unregister(appId: Ident): F[Unit] +} + +object ONode { + private[this] val logger = getLogger + + def apply[F[_] : Effect](store: Store[F]): Resource[F, ONode[F]] = + Resource.pure(new ONode[F] { + + def register(appId: Ident, nodeType: NodeType, uri: LenientUri): F[Unit] = + for { + node <- RNode(appId, nodeType, uri) + _ <- logger.finfo(s"Registering node $node") + _ <- store.transact(RNode.set(node)) + } yield () + + def unregister(appId: Ident): F[Unit] = + logger.finfo(s"Unregister app ${appId.id}") *> + store.transact(RNode.delete(appId)).map(_ => ()) + }) + +} diff --git a/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala new file mode 100644 index 00000000..9b9f7731 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/QAttachment.scala @@ -0,0 +1,70 @@ +package docspell.store.queries + +import fs2.Stream +import cats.implicits._ +import cats.effect.Sync +import doobie._ +import doobie.implicits._ +import docspell.common.{Ident, MetaProposalList} +import docspell.store.Store +import docspell.store.impl.Implicits._ +import docspell.store.records.{RAttachment, RAttachmentMeta, RItem} + +object QAttachment { + + def deleteById[F[_]: Sync](store: Store[F])(attachId: Ident, coll: Ident): F[Int] = { + for { + raOpt <- store.transact(RAttachment.findByIdAndCollective(attachId, coll)) + n <- raOpt.traverse(_ => store.transact(RAttachment.delete(attachId))) + f <- Stream.emit(raOpt). + unNoneTerminate. + map(_.fileId.id). + flatMap(store.bitpeace.delete). + compile.last + } yield n.getOrElse(0) + f.map(_ => 1).getOrElse(0) + } + + def deleteAttachment[F[_]: Sync](store: Store[F])(ra: RAttachment): F[Int] = { + for { + n <- store.transact(RAttachment.delete(ra.id)) + f <- Stream.emit(ra.fileId.id). + flatMap(store.bitpeace.delete). + compile.last + } yield n + f.map(_ => 1).getOrElse(0) + } + + def deleteItemAttachments[F[_]: Sync](store: Store[F])(itemId: Ident, coll: Ident): F[Int] = { + for { + ras <- store.transact(RAttachment.findByItemAndCollective(itemId, coll)) + ns <- ras.traverse(deleteAttachment[F](store)) + } yield ns.sum + } + + def getMetaProposals(itemId: Ident, coll: Ident): ConnectionIO[MetaProposalList] = { + val AC = RAttachment.Columns + val MC = RAttachmentMeta.Columns + val IC = RItem.Columns + + val q = fr"SELECT" ++ MC.proposals.prefix("m").f ++ fr"FROM" ++ RAttachmentMeta.table ++ fr"m" ++ + fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.id.prefix("a").is(MC.id.prefix("m")) ++ + fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++ + fr"WHERE" ++ and(AC.itemId.prefix("a").is(itemId), IC.cid.prefix("i").is(coll)) + + for { + ml <- q.query[MetaProposalList].to[Vector] + } yield MetaProposalList.flatten(ml) + } + + def getAttachmentMeta(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachmentMeta]] = { + val AC = RAttachment.Columns + val MC = RAttachmentMeta.Columns + val IC = RItem.Columns + + val q = fr"SELECT" ++ commas(MC.all.map(_.prefix("m").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++ + fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++ + fr"INNER JOIN" ++ RAttachmentMeta.table ++ fr"m ON" ++ AC.id.prefix("a").is(MC.id.prefix("m")) ++ + fr"WHERE" ++ and(AC.id.prefix("a") is attachId, IC.cid.prefix("i") is collective) + + q.query[RAttachmentMeta].option + } +} diff --git a/modules/store/src/main/scala/docspell/store/queries/QCollective.scala b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala new file mode 100644 index 00000000..4e3661a2 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/QCollective.scala @@ -0,0 +1,45 @@ +package docspell.store.queries + +import doobie._ +import doobie.implicits._ +import docspell.common.{Direction, Ident} +import docspell.store.impl.Implicits._ +import docspell.store.records.{RAttachment, RItem, RTag, RTagItem} + +object QCollective { + + case class InsightData( incoming: Int + , outgoing: Int + , bytes: Long + , tags: Map[String, Int]) + + def getInsights(coll: Ident): ConnectionIO[InsightData] = { + val IC = RItem.Columns + val AC = RAttachment.Columns + val TC = RTag.Columns + val RC = RTagItem.Columns + val q0 = selectCount(IC.id, RItem.table, and(IC.cid is coll, IC.incoming is Direction.incoming)). + query[Int].unique + val q1 = selectCount(IC.id, RItem.table, and(IC.cid is coll, IC.incoming is Direction.outgoing)). + query[Int].unique + + val q2 = fr"SELECT sum(m.length) FROM" ++ RItem.table ++ fr"i" ++ + fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++ + fr"INNER JOIN filemeta m ON m.id =" ++ AC.fileId.prefix("a").f ++ + fr"WHERE" ++ IC.cid.is(coll) + + val q3 = fr"SELECT" ++ commas(TC.name.prefix("t").f,fr"count(" ++ RC.itemId.prefix("r").f ++ fr")") ++ + fr"FROM" ++ RTagItem.table ++ fr"r" ++ + fr"INNER JOIN" ++ RTag.table ++ fr"t ON" ++ RC.tagId.prefix("r").is(TC.tid.prefix("t")) ++ + fr"WHERE" ++ TC.cid.prefix("t").is(coll) ++ + fr"GROUP BY" ++ TC.name.prefix("t").f + + for { + n0 <- q0 + n1 <- q1 + n2 <- q2.query[Option[Long]].unique + n3 <- q3.query[(String, Int)].to[Vector] + } yield InsightData(n0, n1, n2.getOrElse(0), Map.from(n3)) + } + +} diff --git a/modules/store/src/main/scala/docspell/store/queries/QItem.scala b/modules/store/src/main/scala/docspell/store/queries/QItem.scala new file mode 100644 index 00000000..f314a7fc --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/QItem.scala @@ -0,0 +1,204 @@ +package docspell.store.queries + +import bitpeace.FileMeta +import cats.implicits._ +import cats.effect.Sync +import fs2.Stream +import doobie._ +import doobie.implicits._ +import docspell.common.{IdRef, _} +import docspell.store.Store +import docspell.store.records._ +import docspell.store.impl.Implicits._ +import org.log4s._ + +object QItem { + private [this] val logger = getLogger + + case class ItemData( item: RItem + , corrOrg: Option[ROrganization] + , corrPerson: Option[RPerson] + , concPerson: Option[RPerson] + , concEquip: Option[REquipment] + , inReplyTo: Option[IdRef] + , tags: Vector[RTag] + , attachments: Vector[(RAttachment, FileMeta)]) { + + def filterCollective(coll: Ident): Option[ItemData] = + if (item.cid == coll) Some(this) else None + } + + def findItem(id: Ident): ConnectionIO[Option[ItemData]] = { + val IC = RItem.Columns.all.map(_.prefix("i")) + val OC = ROrganization.Columns.all.map(_.prefix("o")) + val P0C = RPerson.Columns.all.map(_.prefix("p0")) + val P1C = RPerson.Columns.all.map(_.prefix("p1")) + val EC = REquipment.Columns.all.map(_.prefix("e")) + val ICC = List(RItem.Columns.id, RItem.Columns.name).map(_.prefix("ref")) + + val cq = selectSimple(IC ++ OC ++ P0C ++ P1C ++ EC ++ ICC, RItem.table ++ fr"i", Fragment.empty) ++ + fr"LEFT JOIN" ++ ROrganization.table ++ fr"o ON" ++ RItem.Columns.corrOrg.prefix("i").is(ROrganization.Columns.oid.prefix("o")) ++ + fr"LEFT JOIN" ++ RPerson.table ++ fr"p0 ON" ++ RItem.Columns.corrPerson.prefix("i").is(RPerson.Columns.pid.prefix("p0")) ++ + fr"LEFT JOIN" ++ RPerson.table ++ fr"p1 ON" ++ RItem.Columns.concPerson.prefix("i").is(RPerson.Columns.pid.prefix("p1")) ++ + fr"LEFT JOIN" ++ REquipment.table ++ fr"e ON" ++ RItem.Columns.concEquipment.prefix("i").is(REquipment.Columns.eid.prefix("e")) ++ + fr"LEFT JOIN" ++ RItem.table ++ fr"ref ON" ++ RItem.Columns.inReplyTo.prefix("i").is(RItem.Columns.id.prefix("ref")) ++ + fr"WHERE" ++ RItem.Columns.id.prefix("i").is(id) + + val q = cq.query[(RItem, Option[ROrganization], Option[RPerson], Option[RPerson], Option[REquipment], Option[IdRef])].option + val attachs = RAttachment.findByItemWithMeta(id) + + val tags = RTag.findByItem(id) + + for { + data <- q + att <- attachs + ts <- tags + } yield data.map(d => ItemData(d._1, d._2, d._3, d._4, d._5, d._6, ts, att)) + } + + + case class ListItem( id: Ident + , name: String + , state: ItemState + , date: Timestamp + , dueDate: Option[Timestamp] + , source: String + , direction: Direction + , created: Timestamp + , fileCount: Int + , corrOrg: Option[IdRef] + , corrPerson: Option[IdRef] + , concPerson: Option[IdRef] + , concEquip: Option[IdRef]) + + case class Query( collective: Ident + , name: Option[String] + , states: Seq[ItemState] + , direction: Option[Direction] + , corrPerson: Option[Ident] + , corrOrg: Option[Ident] + , concPerson: Option[Ident] + , concEquip: Option[Ident] + , tagsInclude: List[Ident] + , tagsExclude: List[Ident] + , dateFrom: Option[Timestamp] + , dateTo: Option[Timestamp] + , dueDateFrom: Option[Timestamp] + , dueDateTo: Option[Timestamp]) + + def findItems(q: Query): Stream[ConnectionIO, ListItem] = { + val IC = RItem.Columns + val AC = RAttachment.Columns + val PC = RPerson.Columns + val OC = ROrganization.Columns + val EC = REquipment.Columns + val itemCols = IC.all + val personCols = List(RPerson.Columns.pid, RPerson.Columns.name) + val orgCols = List(ROrganization.Columns.oid, ROrganization.Columns.name) + val equipCols = List(REquipment.Columns.eid, REquipment.Columns.name) + + val finalCols = commas(IC.id.prefix("i").f + , IC.name.prefix("i").f + , IC.state.prefix("i").f + , coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) + , IC.dueDate.prefix("i").f + , IC.source.prefix("i").f + , IC.incoming.prefix("i").f + , IC.created.prefix("i").f + , fr"COALESCE(a.num, 0)" + , OC.oid.prefix("o0").f + , OC.name.prefix("o0").f + , PC.pid.prefix("p0").f + , PC.name.prefix("p0").f + , PC.pid.prefix("p1").f + , PC.name.prefix("p1").f + , EC.eid.prefix("e1").f + , EC.name.prefix("e1").f + ) + + val withItem = selectSimple(itemCols, RItem.table, IC.cid is q.collective) + val withPerson = selectSimple(personCols, RPerson.table, PC.cid is q.collective) + val withOrgs = selectSimple(orgCols, ROrganization.table, OC.cid is q.collective) + val withEquips = selectSimple(equipCols, REquipment.table, EC.cid is q.collective) + val withAttach = fr"SELECT COUNT(" ++ AC.id.f ++ fr") as num, " ++ AC.itemId.f ++ + fr"from" ++ RAttachment.table ++ fr"GROUP BY (" ++ AC.itemId.f ++ fr")" + + val query = withCTE("items" -> withItem + , "persons" -> withPerson + , "orgs" -> withOrgs + , "equips" -> withEquips + , "attachs" -> withAttach) ++ + fr"SELECT DISTINCT" ++ finalCols ++ fr" FROM items i" ++ + fr"LEFT JOIN attachs a ON" ++ IC.id.prefix("i").is(AC.itemId.prefix("a")) ++ + fr"LEFT JOIN persons p0 ON" ++ IC.corrPerson.prefix("i").is(PC.pid.prefix("p0")) ++ // i.corrperson = p0.pid" ++ + fr"LEFT JOIN orgs o0 ON" ++ IC.corrOrg.prefix("i").is(OC.oid.prefix("o0")) ++ // i.corrorg = o0.oid" ++ + fr"LEFT JOIN persons p1 ON" ++ IC.concPerson.prefix("i").is(PC.pid.prefix("p1")) ++ // i.concperson = p1.pid" ++ + fr"LEFT JOIN equips e1 ON" ++ IC.concEquipment.prefix("i").is(EC.eid.prefix("e1")) // i.concequipment = e1.eid" + + // inclusive tags are AND-ed + val tagSelectsIncl = q.tagsInclude.map(tid => + selectSimple(List(RTagItem.Columns.itemId), RTagItem.table, RTagItem.Columns.tagId is tid)). + map(f => sql"(" ++ f ++ sql") ") + + // exclusive tags are OR-ed + val tagSelectsExcl = + if (q.tagsExclude.isEmpty) Fragment.empty + else selectSimple(List(RTagItem.Columns.itemId), RTagItem.table, RTagItem.Columns.tagId isOneOf q.tagsExclude) + + val name = q.name.map(queryWildcard) + val cond = and( + IC.cid.prefix("i") is q.collective, + IC.state.prefix("i") isOneOf q.states, + IC.incoming.prefix("i") isOrDiscard q.direction, + name.map(n => IC.name.prefix("i").lowerLike(n)).getOrElse(Fragment.empty), + RPerson.Columns.pid.prefix("p0") isOrDiscard q.corrPerson, + ROrganization.Columns.oid.prefix("o0") isOrDiscard q.corrOrg, + RPerson.Columns.pid.prefix("p1") isOrDiscard q.concPerson, + REquipment.Columns.eid.prefix("e1") isOrDiscard q.concEquip, + if (q.tagsInclude.isEmpty) Fragment.empty + else IC.id.prefix("i") ++ sql" IN (" ++ tagSelectsIncl.reduce(_ ++ fr"INTERSECT" ++ _) ++ sql")", + if (q.tagsExclude.isEmpty) Fragment.empty + else IC.id.prefix("i").f ++ sql" NOT IN (" ++ tagSelectsExcl ++ sql")", + q.dateFrom.map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr">= $d").getOrElse(Fragment.empty), + q.dateTo.map(d => coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"<= $d").getOrElse(Fragment.empty), + q.dueDateFrom.map(d => IC.dueDate.prefix("i").isGt(d)).getOrElse(Fragment.empty), + q.dueDateTo.map(d => IC.dueDate.prefix("i").isLt(d)).getOrElse(Fragment.empty) + ) + + val order = orderBy(coalesce(IC.itemDate.prefix("i").f, IC.created.prefix("i").f) ++ fr"DESC") + val frag = query ++ fr"WHERE" ++ cond ++ order + logger.trace(s"List items: $frag") + frag.query[ListItem].stream + } + + + def delete[F[_]: Sync](store: Store[F])(itemId: Ident, collective: Ident): F[Int] = + for { + tn <- store.transact(RTagItem.deleteItemTags(itemId)) + rn <- QAttachment.deleteItemAttachments(store)(itemId, collective) + n <- store.transact(RItem.deleteByIdAndCollective(itemId, collective)) + } yield tn + rn + n + + def findByFileIds(fileMetaIds: List[Ident]): ConnectionIO[Vector[RItem]] = { + val IC = RItem.Columns + val AC = RAttachment.Columns + val q = fr"SELECT DISTINCT" ++ commas(IC.all.map(_.prefix("i").f)) ++ fr"FROM" ++ RItem.table ++ fr"i" ++ + fr"INNER JOIN" ++ RAttachment.table ++ fr"a ON" ++ AC.itemId.prefix("a").is(IC.id.prefix("i")) ++ + fr"WHERE" ++ AC.fileId.isOneOf(fileMetaIds) ++ orderBy(IC.created.prefix("i").asc) + + q.query[RItem].to[Vector] + } + + private def queryWildcard(value: String): String = { + def prefix(n: String) = + if (n.startsWith("*")) s"%${n.substring(1)}" + else n + + def suffix(n: String) = + if (n.endsWith("*")) s"${n.dropRight(1)}%" + else n + + prefix(suffix(value)) + } + +} diff --git a/modules/store/src/main/scala/docspell/store/queries/QJob.scala b/modules/store/src/main/scala/docspell/store/queries/QJob.scala new file mode 100644 index 00000000..c5be9dda --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/QJob.scala @@ -0,0 +1,184 @@ +package docspell.store.queries + +import cats.effect.Effect +import cats.implicits._ +import docspell.common._ +import docspell.common.syntax.all._ +import docspell.store.Store +import docspell.store.impl.Implicits._ +import docspell.store.records.{RJob, RJobGroupUse, RJobLog} +import doobie._ +import doobie.implicits._ +import fs2.Stream +import org.log4s._ + +object QJob { + private [this] val logger = getLogger + + def takeNextJob[F[_]: Effect](store: Store[F])(priority: Ident => F[Priority], worker: Ident, retryPause: Duration): F[Option[RJob]] = { + Stream.range(0, 10). + evalMap(n => takeNextJob1(store)(priority, worker, retryPause, n)). + evalTap({ x => + if (x.isLeft) logger.fdebug[F]("Cannot mark job, probably due to concurrent updates. Will retry.") + else ().pure[F] + }). + find(_.isRight). + flatMap({ + case Right(job) => + Stream.emit(job) + case Left(_) => + Stream.eval(logger.fwarn[F]("Cannot mark job, even after retrying. Give up.")).map(_ => None) + }). + compile.last.map(_.flatten) + } + + private def takeNextJob1[F[_]: Effect](store: Store[F])( priority: Ident => F[Priority] + , worker: Ident + , retryPause: Duration + , currentTry: Int): F[Either[Unit, Option[RJob]]] = { + //if this fails, we have to restart takeNextJob + def markJob(job: RJob): F[Either[Unit, RJob]] = + store.transact(for { + n <- RJob.setScheduled(job.id, worker) + _ <- if (n == 1) RJobGroupUse.setGroup(RJobGroupUse(worker, job.group)) + else 0.pure[ConnectionIO] + } yield if (n == 1) Right(job) else Left(())) + + for { + _ <- logger.ftrace[F](s"About to take next job (worker ${worker.id}), try $currentTry") + now <- Timestamp.current[F] + group <- store.transact(selectNextGroup(worker, now, retryPause)) + _ <- logger.ftrace[F](s"Choose group ${group.map(_.id)}") + prio <- group.map(priority).getOrElse((Priority.Low: Priority).pure[F]) + _ <- logger.ftrace[F](s"Looking for job of prio $prio") + job <- group.map(g => store.transact(selectNextJob(g, prio, retryPause, now))).getOrElse((None: Option[RJob]).pure[F]) + _ <- logger.ftrace[F](s"Found job: ${job.map(_.info)}") + res <- job.traverse(j => markJob(j)) + } yield res.map(_.map(_.some)).getOrElse { + if (group.isDefined) Left(()) // if a group was found, but no job someone else was faster + else Right(None) + } + } + + def selectNextGroup(worker: Ident, now: Timestamp, initialPause: Duration): ConnectionIO[Option[Ident]] = { + val JC = RJob.Columns + val waiting: JobState = JobState.Waiting + val stuck: JobState = JobState.Stuck + val jgroup = JC.group.prefix("a") + val jstate = JC.state.prefix("a") + val ugroup = RJobGroupUse.Columns.group.prefix("b") + val uworker = RJobGroupUse.Columns.worker.prefix("b") + + val stuckTrigger = coalesce(JC.startedmillis.prefix("a").f, sql"${now.toMillis}") ++ + fr"+" ++ power2(JC.retries.prefix("a")) ++ fr"* ${initialPause.millis}" + + val stateCond = or(jstate is waiting, and(jstate is stuck, stuckTrigger ++ fr"< ${now.toMillis}")) + + val sql1 = fr"SELECT" ++ jgroup.f ++ fr"as g FROM" ++ RJob.table ++ fr"a" ++ + fr"INNER JOIN" ++ RJobGroupUse.table ++ fr"b ON" ++ jgroup.isGt(ugroup) ++ + fr"WHERE" ++ and(uworker is worker, stateCond) ++ + fr"LIMIT 1" //LIMIT is not sql standard, but supported by h2,mariadb and postgres + val sql2 = fr"SELECT min(" ++ jgroup.f ++ fr") as g FROM" ++ RJob.table ++ fr"a" ++ + fr"WHERE" ++ stateCond + + val union = sql"SELECT g FROM ((" ++ sql1 ++ sql") UNION ALL (" ++ sql2 ++ sql")) as t0 WHERE g is not null" + + union.query[Ident].to[List].map(_.headOption) // either one or two results, but may be empty if RJob table is empty + } + + def selectNextJob(group: Ident, prio: Priority, initialPause: Duration, now: Timestamp): ConnectionIO[Option[RJob]] = { + val JC = RJob.Columns + val psort = + if (prio == Priority.High) JC.priority.desc + else JC.priority.asc + val waiting: JobState = JobState.Waiting + val stuck: JobState = JobState.Stuck + + val stuckTrigger = coalesce(JC.startedmillis.f, sql"${now.toMillis}") ++ fr"+" ++ power2(JC.retries) ++ fr"* ${initialPause.millis}" + val sql = selectSimple(JC.all, RJob.table, + and(JC.group is group, or(JC.state is waiting, and(JC.state is stuck, stuckTrigger ++ fr"< ${now.toMillis}")))) ++ + orderBy(JC.state.asc, psort, JC.submitted.asc) ++ + fr"LIMIT 1" + + sql.query[RJob].option + } + + def setCancelled[F[_]: Effect](id: Ident, store: Store[F]): F[Unit] = + for { + now <- Timestamp.current[F] + _ <- store.transact(RJob.setCancelled(id, now)) + } yield () + + def setFailed[F[_]: Effect](id: Ident, store: Store[F]): F[Unit] = + for { + now <- Timestamp.current[F] + _ <- store.transact(RJob.setFailed(id, now)) + } yield () + + def setSuccess[F[_]: Effect](id: Ident, store: Store[F]): F[Unit] = + for { + now <- Timestamp.current[F] + _ <- store.transact(RJob.setSuccess(id, now)) + } yield () + + def setStuck[F[_]: Effect](id: Ident, store: Store[F]): F[Unit] = + for { + now <- Timestamp.current[F] + _ <- store.transact(RJob.setStuck(id, now)) + } yield () + + def setRunning[F[_]: Effect](id: Ident, workerId: Ident, store: Store[F]): F[Unit] = + for { + now <- Timestamp.current[F] + _ <- store.transact(RJob.setRunning(id, workerId, now)) + } yield () + + def setFinalState[F[_]: Effect](id: Ident, state: JobState, store: Store[F]): F[Unit] = + state match { + case JobState.Success => + setSuccess(id, store) + case JobState.Failed => + setFailed(id, store) + case JobState.Cancelled => + setCancelled(id, store) + case JobState.Stuck => + setStuck(id, store) + case _ => + logger.ferror[F](s"Invalid final state: $state.") + } + + def exceedsRetries[F[_]: Effect](id: Ident, max: Int, store: Store[F]): F[Boolean] = + store.transact(RJob.getRetries(id)).map(n => n.forall(_ >= max)) + + def runningToWaiting[F[_]: Effect](workerId: Ident, store: Store[F]): F[Unit] = { + store.transact(RJob.setRunningToWaiting(workerId)).map(_ => ()) + } + + def findAll[F[_]: Effect](ids: Seq[Ident], store: Store[F]): F[Vector[RJob]] = + store.transact(RJob.findFromIds(ids)) + + def queueStateSnapshot(collective: Ident): Stream[ConnectionIO, (RJob, Vector[RJobLog])] = { + val JC = RJob.Columns + val waiting: Set[JobState] = Set(JobState.Waiting, JobState.Stuck, JobState.Scheduled) + val running: Set[JobState] = Set(JobState.Running) + val done = JobState.all.diff(waiting).diff(running) + + def selectJobs(now: Timestamp): Stream[ConnectionIO, RJob] = { + val refDate = now.minusHours(24) + val sql = selectSimple(JC.all, RJob.table, + and(JC.group is collective, + or(and(JC.state.isOneOf(done.toSeq), JC.submitted isGt refDate) + , JC.state.isOneOf((running ++ waiting).toSeq)))) + (sql ++ orderBy(JC.submitted.desc)).query[RJob].stream + } + + def selectLogs(job: RJob): ConnectionIO[Vector[RJobLog]] = + RJobLog.findLogs(job.id) + + for { + now <- Stream.eval(Timestamp.current[ConnectionIO]) + job <- selectJobs(now) + res <- Stream.eval(selectLogs(job)) + } yield (job, res) + } +} diff --git a/modules/store/src/main/scala/docspell/store/queries/QLogin.scala b/modules/store/src/main/scala/docspell/store/queries/QLogin.scala new file mode 100644 index 00000000..4525c6d9 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/QLogin.scala @@ -0,0 +1,36 @@ +package docspell.store.queries + +import docspell.common._ +import docspell.store.impl.Implicits._ +import docspell.store.records.RCollective.{Columns => CC} +import docspell.store.records.RUser.{Columns => UC} +import docspell.store.records.{RCollective, RUser} +import doobie._ +import doobie.implicits._ +import org.log4s._ + +object QLogin { + private[this] val logger = getLogger + + case class Data( account: AccountId + , password: Password + , collectiveState: CollectiveState + , userState: UserState) + + def findUser(acc: AccountId): ConnectionIO[Option[Data]] = { + val ucid = UC.cid.prefix("u") + val login = UC.login.prefix("u") + val pass = UC.password.prefix("u") + val ustate = UC.state.prefix("u") + val cstate = CC.state.prefix("c") + val ccid = CC.id.prefix("c") + + val sql = selectSimple( + List(ucid,login,pass,cstate,ustate), + RUser.table ++ fr"u, " ++ RCollective.table ++ fr"c", + and(ucid is ccid, login is acc.user, ucid is acc.collective)) + + logger.trace(s"SQL : $sql") + sql.query[Data].option + } +} diff --git a/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala b/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala new file mode 100644 index 00000000..361c5338 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queries/QOrganization.scala @@ -0,0 +1,86 @@ +package docspell.store.queries + +import fs2._ +import cats.implicits._ +import doobie._ +import docspell.common._ +import docspell.store.{AddResult, Store} +import docspell.store.impl.Column +import docspell.store.records.ROrganization.{Columns => OC} +import docspell.store.records.RPerson.{Columns => PC} +import docspell.store.records._ + +object QOrganization { + + def findOrgAndContact(coll: Ident, order: OC.type => Column): Stream[ConnectionIO, (ROrganization, Vector[RContact])] = { + ROrganization.findAll(coll, order). + evalMap(ro => RContact.findAllOrg(ro.oid).map(cs => (ro, cs))) + } + def findPersonAndContact(coll: Ident, order: PC.type => Column): Stream[ConnectionIO, (RPerson, Vector[RContact])] = { + RPerson.findAll(coll, order). + evalMap(ro => RContact.findAllPerson(ro.pid).map(cs => (ro, cs))) + } + + def addOrg[F[_]](org: ROrganization, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = { + val insert = for { + n <- ROrganization.insert(org) + cs <- contacts.toList.traverse(RContact.insert) + } yield n + cs.sum + + val exists = ROrganization.existsByName(cid, org.name) + + store => store.add(insert, exists) + } + + def addPerson[F[_]](person: RPerson, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = { + val insert = for { + n <- RPerson.insert(person) + cs <- contacts.toList.traverse(RContact.insert) + } yield n + cs.sum + + val exists = RPerson.existsByName(cid, person.name) + + store => store.add(insert, exists) + } + + def updateOrg[F[_]](org: ROrganization, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = { + val insert = for { + n <- ROrganization.update(org) + d <- RContact.deleteOrg(org.oid) + cs <- contacts.toList.traverse(RContact.insert) + } yield n + cs.sum + d + + val exists = ROrganization.existsByName(cid, org.name) + + store => store.add(insert, exists) + } + + def updatePerson[F[_]](person: RPerson, contacts: Seq[RContact], cid: Ident): Store[F] => F[AddResult] = { + val insert = for { + n <- RPerson.update(person) + d <- RContact.deletePerson(person.pid) + cs <- contacts.toList.traverse(RContact.insert) + } yield n + cs.sum + d + + val exists = RPerson.existsByName(cid, person.name) + + store => store.add(insert, exists) + } + + def deleteOrg(orgId: Ident, collective: Ident): ConnectionIO[Int] = { + for { + n0 <- RItem.removeCorrOrg(collective, orgId) + n1 <- RContact.deleteOrg(orgId) + n2 <- ROrganization.delete(orgId, collective) + } yield n0 + n1 + n2 + } + + def deletePerson(personId: Ident, collective: Ident): ConnectionIO[Int] = { + for { + n0 <- RItem.removeCorrPerson(collective, personId) + n1 <- RItem.removeConcPerson(collective, personId) + n2 <- RContact.deletePerson(personId) + n3 <- RPerson.delete(personId, collective) + } yield n0 + n1 + n2 + n3 + } +} diff --git a/modules/store/src/main/scala/docspell/store/queue/JobQueue.scala b/modules/store/src/main/scala/docspell/store/queue/JobQueue.scala new file mode 100644 index 00000000..8134e5a1 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/queue/JobQueue.scala @@ -0,0 +1,46 @@ +package docspell.store.queue + +import cats.implicits._ +import cats.effect.{Effect, Resource} +import docspell.common._ +import docspell.common.syntax.all._ +import docspell.store.Store +import docspell.store.queries.QJob +import docspell.store.records.RJob +import org.log4s._ + +trait JobQueue[F[_]] { + + def insert(job: RJob): F[Unit] + + def insertAll(jobs: Seq[RJob]): F[Unit] + + def nextJob(prio: Ident => F[Priority], worker: Ident, retryPause: Duration): F[Option[RJob]] +} + +object JobQueue { + private[this] val logger = getLogger + + def apply[F[_] : Effect](store: Store[F]): Resource[F, JobQueue[F]] = + Resource.pure(new JobQueue[F] { + + def nextJob(prio: Ident => F[Priority], worker: Ident, retryPause: Duration): F[Option[RJob]] = + logger.fdebug("Select next job") *> QJob.takeNextJob(store)(prio, worker, retryPause) + + def insert(job: RJob): F[Unit] = + store.transact(RJob.insert(job)). + flatMap({ n => + if (n != 1) Effect[F].raiseError(new Exception(s"Inserting job failed. Update count: $n")) + else ().pure[F] + }) + + def insertAll(jobs: Seq[RJob]): F[Unit] = + jobs.toList.traverse(j => insert(j).attempt). + map(_.foreach { + case Right(()) => + case Left(ex) => + logger.error(ex)("Could not insert job. Skipping it.") + }) + + }) +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachment.scala b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala new file mode 100644 index 00000000..5399d929 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RAttachment.scala @@ -0,0 +1,71 @@ +package docspell.store.records + +import bitpeace.FileMeta +import doobie._ +import doobie.implicits._ +import docspell.common._ +import docspell.store.impl._ +import docspell.store.impl.Implicits._ + +case class RAttachment( id: Ident + , itemId: Ident + , fileId: Ident + , position: Int + , created: Timestamp + , name: Option[String]) { + +} + +object RAttachment { + + val table = fr"attachment" + + object Columns { + val id = Column("attachid") + val itemId = Column("itemid") + val fileId = Column("filemetaid") + val position = Column("position") + val created = Column("created") + val name = Column("name") + val all = List(id, itemId, fileId, position, created, name) + } + import Columns._ + + def insert(v: RAttachment): ConnectionIO[Int] = + insertRow(table, all, fr"${v.id},${v.itemId},${v.fileId.id},${v.position},${v.created},${v.name}").update.run + + def findById(attachId: Ident): ConnectionIO[Option[RAttachment]] = + selectSimple(all, table, id is attachId).query[RAttachment].option + + def findByIdAndCollective(attachId: Ident, collective: Ident): ConnectionIO[Option[RAttachment]] = { + selectSimple(all.map(_.prefix("a")), table ++ fr"a," ++ RItem.table ++ fr"i", and( + fr"a.itemid = i.itemid", + id.prefix("a") is attachId, + RItem.Columns.cid.prefix("i") is collective + )).query[RAttachment].option + } + + def findByItem(id: Ident): ConnectionIO[Vector[RAttachment]] = + selectSimple(all, table, itemId is id).query[RAttachment].to[Vector] + + def findByItemAndCollective(id: Ident, coll: Ident): ConnectionIO[Vector[RAttachment]] = { + val q = selectSimple(all.map(_.prefix("a")), table ++ fr"a", Fragment.empty) ++ + fr"INNER JOIN" ++ RItem.table ++ fr"i ON" ++ RItem.Columns.id.prefix("i").is(itemId.prefix("a")) ++ + fr"WHERE" ++ and(itemId.prefix("a").is(id), RItem.Columns.cid.prefix("i").is(coll)) + q.query[RAttachment].to[Vector] + } + + def findByItemWithMeta(id: Ident): ConnectionIO[Vector[(RAttachment, FileMeta)]] = { + import bitpeace.sql._ + + val q = fr"SELECT a.*,m.* FROM" ++ table ++ fr"a, filemeta m WHERE a.filemetaid = m.id AND a.itemid = $id ORDER BY a.position ASC" + q.query[(RAttachment, FileMeta)].to[Vector] + } + + def delete(attachId: Ident): ConnectionIO[Int] = + for { + n0 <- RAttachmentMeta.delete(attachId) + n1 <- deleteFrom(table, id is attachId).update.run + } yield n0 + n1 + +} diff --git a/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala new file mode 100644 index 00000000..27f4fc55 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RAttachmentMeta.scala @@ -0,0 +1,62 @@ +package docspell.store.records + +import cats.implicits._ +import doobie._ +import doobie.implicits._ +import docspell.common._ +import docspell.store.impl._ +import docspell.store.impl.Implicits._ + +case class RAttachmentMeta(id: Ident + , content: Option[String] + , nerlabels: List[NerLabel] + , proposals: MetaProposalList) { + +} + +object RAttachmentMeta { + def empty(attachId: Ident) = RAttachmentMeta(attachId, None, Nil, MetaProposalList.empty) + + val table = fr"attachmentmeta" + + object Columns { + val id = Column("attachid") + val content = Column("content") + val nerlabels = Column("nerlabels") + val proposals = Column("itemproposals") + val all = List(id, content, nerlabels, proposals) + } + import Columns._ + + def insert(v: RAttachmentMeta): ConnectionIO[Int] = + insertRow(table, all, fr"${v.id},${v.content},${v.nerlabels},${v.proposals}").update.run + + def exists(attachId: Ident): ConnectionIO[Boolean] = + selectCount(id, table, id is attachId).query[Int].unique.map(_ > 0) + + def upsert(v: RAttachmentMeta): ConnectionIO[Int] = + for { + n0 <- update(v) + n1 <- if (n0 == 0) insert(v) else n0.pure[ConnectionIO] + } yield n1 + + def update(v: RAttachmentMeta): ConnectionIO[Int] = + updateRow(table, id is v.id, commas( + content setTo v.content, + nerlabels setTo v.nerlabels, + proposals setTo v.proposals + )).update.run + + def updateLabels(mid: Ident, labels: List[NerLabel]): ConnectionIO[Int] = + updateRow(table, id is mid, commas( + nerlabels setTo labels + )).update.run + + def updateProposals(mid: Ident, plist: MetaProposalList): ConnectionIO[Int] = + updateRow(table, id is mid, commas( + proposals setTo plist + )).update.run + + def delete(attachId: Ident): ConnectionIO[Int] = + deleteFrom(table, id is attachId).update.run +} diff --git a/modules/store/src/main/scala/docspell/store/records/RCollective.scala b/modules/store/src/main/scala/docspell/store/records/RCollective.scala new file mode 100644 index 00000000..1d32b647 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RCollective.scala @@ -0,0 +1,68 @@ +package docspell.store.records + +import docspell.common._ +import docspell.store.impl.Column +import docspell.store.impl.Implicits._ +import doobie._ +import doobie.implicits._ +import fs2.Stream + +case class RCollective( id: Ident + , state: CollectiveState + , language: Language + , created: Timestamp) + +object RCollective { + + val table = fr"collective" + + object Columns { + + val id = Column("cid") + val state = Column("state") + val language = Column("doclang") + val created = Column("created") + + val all = List(id, state, language, created) + } + + import Columns._ + + def insert(value: RCollective): ConnectionIO[Int] = { + val sql = insertRow(table, Columns.all, fr"${value.id},${value.state},${value.language},${value.created}") + sql.update.run + } + + def update(value: RCollective): ConnectionIO[Int] = { + val sql = updateRow(table, id is value.id, commas( + state setTo value.state + )) + sql.update.run + } + + def findLanguage(cid: Ident): ConnectionIO[Option[Language]] = + selectSimple(List(language), table, id is cid).query[Option[Language]].unique + + def updateLanguage(cid: Ident, lang: Language): ConnectionIO[Int] = + updateRow(table, id is cid, language setTo lang).update.run + + def findById(cid: Ident): ConnectionIO[Option[RCollective]] = { + val sql = selectSimple(all, table, id is cid) + sql.query[RCollective].option + } + + def existsById(cid: Ident): ConnectionIO[Boolean] = { + val sql = selectCount(id, table, id is cid) + sql.query[Int].unique.map(_ > 0) + } + + def findAll(order: Columns.type => Column): ConnectionIO[Vector[RCollective]] = { + val sql = selectSimple(all, table, Fragment.empty) ++ orderBy(order(Columns).f) + sql.query[RCollective].to[Vector] + } + + def streamAll(order: Columns.type => Column): Stream[ConnectionIO, RCollective] = { + val sql = selectSimple(all, table, Fragment.empty) ++ orderBy(order(Columns).f) + sql.query[RCollective].stream + } +} diff --git a/modules/store/src/main/scala/docspell/store/records/RContact.scala b/modules/store/src/main/scala/docspell/store/records/RContact.scala new file mode 100644 index 00000000..ecb09d95 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RContact.scala @@ -0,0 +1,73 @@ +package docspell.store.records + +import doobie._, doobie.implicits._ +import docspell.common._ +import docspell.store.impl._ +import docspell.store.impl.Implicits._ + +case class RContact( + contactId: Ident + , value: String + , kind: ContactKind + , personId: Option[Ident] + , orgId: Option[Ident] + , created: Timestamp) { + +} + +object RContact { + + val table = fr"contact" + + object Columns { + val contactId = Column("contactid") + val value = Column("value") + val kind = Column("kind") + val personId = Column("pid") + val orgId = Column("oid") + val created = Column("created") + val all = List(contactId, value,kind, personId, orgId, created) + } + + import Columns._ + + def insert(v: RContact): ConnectionIO[Int] = { + val sql = insertRow(table, all, + fr"${v.contactId},${v.value},${v.kind},${v.personId},${v.orgId},${v.created}") + sql.update.run + } + + def update(v: RContact): ConnectionIO[Int] = { + val sql = updateRow(table, contactId is v.contactId, commas( + value setTo v.value, + kind setTo v.kind, + personId setTo v.personId, + orgId setTo v.orgId + )) + sql.update.run + } + + def delete(v: RContact): ConnectionIO[Int] = + deleteFrom(table, contactId is v.contactId).update.run + + def deleteOrg(oid: Ident): ConnectionIO[Int] = + deleteFrom(table, orgId is oid).update.run + + def deletePerson(pid: Ident): ConnectionIO[Int] = + deleteFrom(table, personId is pid).update.run + + def findById(id: Ident): ConnectionIO[Option[RContact]] = { + val sql = selectSimple(all, table, contactId is id) + sql.query[RContact].option + } + + def findAllPerson(pid: Ident): ConnectionIO[Vector[RContact]] = { + val sql = selectSimple(all, table, personId is pid) + sql.query[RContact].to[Vector] + } + + def findAllOrg(oid: Ident): ConnectionIO[Vector[RContact]] = { + val sql = selectSimple(all, table, orgId is oid) + sql.query[RContact].to[Vector] + } +} diff --git a/modules/store/src/main/scala/docspell/store/records/REquipment.scala b/modules/store/src/main/scala/docspell/store/records/REquipment.scala new file mode 100644 index 00000000..d4384f5e --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/REquipment.scala @@ -0,0 +1,65 @@ +package docspell.store.records + +import doobie._, doobie.implicits._ +import docspell.common._ +import docspell.store.impl._ +import docspell.store.impl.Implicits._ + +case class REquipment( + eid: Ident + , cid: Ident + , name: String + , created: Timestamp) { + +} + +object REquipment { + + val table = fr"equipment" + + object Columns { + val eid = Column("eid") + val cid = Column("cid") + val name = Column("name") + val created = Column("created") + val all = List(eid,cid,name,created) + } + import Columns._ + + def insert(v: REquipment): ConnectionIO[Int] = { + val sql = insertRow(table, all, + fr"${v.eid},${v.cid},${v.name},${v.created}") + sql.update.run + } + + def update(v: REquipment): ConnectionIO[Int] = { + val sql = updateRow(table, and(eid is v.eid, cid is v.cid), commas( + cid setTo v.cid, + name setTo v.name + )) + sql.update.run + } + + def existsByName(coll: Ident, ename: String): ConnectionIO[Boolean] = { + val sql = selectCount(eid, table, and(cid is coll, name is ename)) + sql.query[Int].unique.map(_ > 0) + } + + def findById(id: Ident): ConnectionIO[Option[REquipment]] = { + val sql = selectSimple(all, table, eid is id) + sql.query[REquipment].option + } + + def findAll(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[REquipment]] = { + val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f) + sql.query[REquipment].to[Vector] + } + + def findLike(coll: Ident, equipName: String): ConnectionIO[Vector[IdRef]] = + selectSimple(List(eid, name), table, and(cid is coll, + name.lowerLike(equipName))). + query[IdRef].to[Vector] + + def delete(id: Ident, coll: Ident): ConnectionIO[Int] = + deleteFrom(table, and(eid is id, cid is coll)).update.run +} diff --git a/modules/store/src/main/scala/docspell/store/records/RInvitation.scala b/modules/store/src/main/scala/docspell/store/records/RInvitation.scala new file mode 100644 index 00000000..be7e49fe --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RInvitation.scala @@ -0,0 +1,53 @@ +package docspell.store.records + +import cats.implicits._ +import cats.effect.Sync +import doobie._ +import doobie.implicits._ +import docspell.common._ +import docspell.store.impl._ +import docspell.store.impl.Implicits._ + +case class RInvitation(id: Ident, created: Timestamp) { + +} + +object RInvitation { + + val table = fr"invitation" + + object Columns { + val id = Column("id") + val created = Column("created") + val all = List(id, created) + } + import Columns._ + + def generate[F[_]: Sync]: F[RInvitation] = + for { + c <- Timestamp.current[F] + i <- Ident.randomId[F] + } yield RInvitation(i, c) + + def insert(v: RInvitation): ConnectionIO[Int] = + insertRow(table, all, fr"${v.id},${v.created}").update.run + + def insertNew: ConnectionIO[RInvitation] = + generate[ConnectionIO]. + flatMap(v => insert(v).map(_ => v)) + + def findById(invite: Ident): ConnectionIO[Option[RInvitation]] = + selectSimple(all, table, id is invite).query[RInvitation].option + + def delete(invite: Ident): ConnectionIO[Int] = + deleteFrom(table, id is invite).update.run + + def useInvite(invite: Ident, minCreated: Timestamp): ConnectionIO[Boolean] = { + val get = selectCount(id, table, and(id is invite, created isGt minCreated)). + query[Int].unique + for { + inv <- get + _ <- delete(invite) + } yield inv > 0 + } +} diff --git a/modules/store/src/main/scala/docspell/store/records/RItem.scala b/modules/store/src/main/scala/docspell/store/records/RItem.scala new file mode 100644 index 00000000..99bd2d87 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RItem.scala @@ -0,0 +1,163 @@ +package docspell.store.records + +import cats.implicits._ +import cats.effect.Sync +import doobie._ +import doobie.implicits._ +import docspell.common._ +import docspell.store.impl._ +import docspell.store.impl.Implicits._ + +case class RItem( id: Ident + , cid: Ident + , name: String + , itemDate: Option[Timestamp] + , source: String + , direction: Direction + , state: ItemState + , corrOrg: Option[Ident] + , corrPerson: Option[Ident] + , concPerson: Option[Ident] + , concEquipment: Option[Ident] + , inReplyTo: Option[Ident] + , dueDate: Option[Timestamp] + , created: Timestamp + , updated: Timestamp + , notes: Option[String]) { + +} + +object RItem { + + def newItem[F[_]: Sync](cid: Ident, name: String, source: String, direction: Direction, state: ItemState): F[RItem] = + for { + now <- Timestamp.current[F] + id <- Ident.randomId[F] + } yield RItem(id, cid, name, None, source, direction, state, None, None, None, None, None, None, now, now, None) + + val table = fr"item" + + object Columns { + val id = Column("itemid") + val cid = Column("cid") + val name = Column("name") + val itemDate = Column("itemdate") + val source = Column("source") + val incoming = Column("incoming") + val state = Column("state") + val corrOrg = Column("corrorg") + val corrPerson = Column("corrperson") + val concPerson = Column("concperson") + val concEquipment = Column("concequipment") + val inReplyTo = Column("inreplyto") + val dueDate = Column("duedate") + val created = Column("created") + val updated = Column("updated") + val notes = Column("notes") + val all = List(id, cid, name, itemDate, source, incoming, state, corrOrg, + corrPerson, concPerson, concEquipment, inReplyTo, dueDate, created, updated, notes) + } + import Columns._ + + def insert(v: RItem): ConnectionIO[Int] = + insertRow(table, all, fr"${v.id},${v.cid},${v.name},${v.itemDate},${v.source},${v.direction},${v.state}," ++ + fr"${v.corrOrg},${v.corrPerson},${v.concPerson},${v.concEquipment},${v.inReplyTo},${v.dueDate}," ++ + fr"${v.created},${v.updated},${v.notes}").update.run + + def getCollective(itemId: Ident): ConnectionIO[Option[Ident]] = + selectSimple(List(cid), table, id is itemId).query[Ident].option + + def updateState(itemId: Ident, itemState: ItemState): ConnectionIO[Int] = + for { + t <- currentTime + n <- updateRow(table, id is itemId, commas(state setTo itemState, updated setTo t)).update.run + } yield n + + def updateStateForCollective(itemId: Ident, itemState: ItemState, coll: Ident): ConnectionIO[Int] = + for { + t <- currentTime + n <- updateRow(table, and(id is itemId, cid is coll), commas(state setTo itemState, updated setTo t)).update.run + } yield n + + def updateDirection(itemId: Ident, coll: Ident, dir: Direction): ConnectionIO[Int] = + for { + t <- currentTime + n <- updateRow(table, and(id is itemId, cid is coll), commas(incoming setTo dir, updated setTo t)).update.run + } yield n + + + def updateCorrOrg(itemId: Ident, coll: Ident, org: Option[Ident]): ConnectionIO[Int] = + for { + t <- currentTime + n <- updateRow(table, and(id is itemId, cid is coll), commas(corrOrg setTo org, updated setTo t)).update.run + } yield n + + def removeCorrOrg(coll: Ident, currentOrg: Ident): ConnectionIO[Int] = + for { + t <- currentTime + n <- updateRow(table, and(cid is coll, corrOrg is Some(currentOrg)), commas(corrOrg setTo(None: Option[Ident]), updated setTo t)).update.run + } yield n + + def updateCorrPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] = + for { + t <- currentTime + n <- updateRow(table, and(id is itemId, cid is coll), commas(corrPerson setTo person, updated setTo t)).update.run + } yield n + + def removeCorrPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] = + for { + t <- currentTime + n <- updateRow(table, and(cid is coll, corrPerson is Some(currentPerson)), commas(corrPerson setTo(None: Option[Ident]), updated setTo t)).update.run + } yield n + + def updateConcPerson(itemId: Ident, coll: Ident, person: Option[Ident]): ConnectionIO[Int] = + for { + t <- currentTime + n <- updateRow(table, and(id is itemId, cid is coll), commas(concPerson setTo person, updated setTo t)).update.run + } yield n + + def removeConcPerson(coll: Ident, currentPerson: Ident): ConnectionIO[Int] = + for { + t <- currentTime + n <- updateRow(table, and(cid is coll, concPerson is Some(currentPerson)), commas(concPerson setTo(None: Option[Ident]), updated setTo t)).update.run + } yield n + + def updateConcEquip(itemId: Ident, coll: Ident, equip: Option[Ident]): ConnectionIO[Int] = + for { + t <- currentTime + n <- updateRow(table, and(id is itemId, cid is coll), commas(concEquipment setTo equip, updated setTo t)).update.run + } yield n + + def removeConcEquip(coll: Ident, currentEquip: Ident): ConnectionIO[Int] = + for { + t <- currentTime + n <- updateRow(table, and(cid is coll, concEquipment is Some(currentEquip)), commas(concPerson setTo(None: Option[Ident]), updated setTo t)).update.run + } yield n + + def updateNotes(itemId: Ident, coll: Ident, text: Option[String]): ConnectionIO[Int] = + for { + t <- currentTime + n <- updateRow(table, and(id is itemId, cid is coll), commas(notes setTo text, updated setTo t)).update.run + } yield n + + def updateName(itemId: Ident, coll: Ident, itemName: String): ConnectionIO[Int] = + for { + t <- currentTime + n <- updateRow(table, and(id is itemId, cid is coll), commas(name setTo itemName, updated setTo t)).update.run + } yield n + + def updateDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] = + for { + t <- currentTime + n <- updateRow(table, and(id is itemId, cid is coll), commas(itemDate setTo date, updated setTo t)).update.run + } yield n + + def updateDueDate(itemId: Ident, coll: Ident, date: Option[Timestamp]): ConnectionIO[Int] = + for { + t <- currentTime + n <- updateRow(table, and(id is itemId, cid is coll), commas(dueDate setTo date, updated setTo t)).update.run + } yield n + + def deleteByIdAndCollective(itemId: Ident, coll: Ident): ConnectionIO[Int] = + deleteFrom(table, and(id is itemId, cid is coll)).update.run +} diff --git a/modules/store/src/main/scala/docspell/store/records/RJob.scala b/modules/store/src/main/scala/docspell/store/records/RJob.scala new file mode 100644 index 00000000..d3247fb4 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RJob.scala @@ -0,0 +1,165 @@ +package docspell.store.records + +import cats.effect.Sync +import doobie._ +import doobie.implicits._ +import docspell.common._ +import docspell.store.impl.Column +import docspell.store.impl.Implicits._ +import io.circe.Encoder + +case class RJob(id: Ident + , task: Ident + , group: Ident + , args: String + , subject: String + , submitted: Timestamp + , submitter: Ident + , priority: Priority + , state: JobState + , retries: Int + , progress: Int + , tracker: Option[Ident] + , worker: Option[Ident] + , started: Option[Timestamp] + , finished: Option[Timestamp]) { + + def info: String = + s"${id.id.substring(0, 9)}.../${group.id}/${task.id}/$priority" +} + +object RJob { + + def newJob[A](id: Ident + , task: Ident + , group: Ident + , args: A + , subject: String + , submitted: Timestamp + , submitter: Ident + , priority: Priority + , tracker: Option[Ident])(implicit E: Encoder[A]): RJob = + RJob(id, task, group, E(args).noSpaces, subject, submitted, submitter, priority, JobState.Waiting, 0, 0, tracker, None, None, None) + + val table = fr"job" + + object Columns { + val id = Column("jid") + val task = Column("task") + val group = Column("group_") + val args = Column("args") + val subject = Column("subject") + val submitted = Column("submitted") + val submitter = Column("submitter") + val priority = Column("priority") + val state = Column("state") + val retries = Column("retries") + val progress = Column("progress") + val tracker = Column("tracker") + val worker = Column("worker") + val started = Column("started") + val startedmillis = Column("startedmillis") + val finished = Column("finished") + val all = List(id,task,group,args,subject,submitted,submitter,priority,state,retries,progress,tracker,worker,started,finished) + } + + import Columns._ + + def insert(v: RJob): ConnectionIO[Int] = { + val smillis = v.started.map(_.toMillis) + val sql = insertRow(table, all ++ List(startedmillis), + fr"${v.id},${v.task},${v.group},${v.args},${v.subject},${v.submitted},${v.submitter},${v.priority},${v.state},${v.retries},${v.progress},${v.tracker},${v.worker},${v.started},${v.finished},$smillis") + sql.update.run + } + + def findFromIds(ids: Seq[Ident]): ConnectionIO[Vector[RJob]] = { + if (ids.isEmpty) Sync[ConnectionIO].pure(Vector.empty[RJob]) + else selectSimple(all, table, id isOneOf ids).query[RJob].to[Vector] + } + + def findByIdAndGroup(jobId: Ident, jobGroup: Ident): ConnectionIO[Option[RJob]] = + selectSimple(all, table, and(id is jobId, group is jobGroup)).query[RJob].option + + def setRunningToWaiting(workerId: Ident): ConnectionIO[Int] = { + val states: Seq[JobState] = List(JobState.Running, JobState.Scheduled) + updateRow(table, and(worker is workerId, state isOneOf states), + state setTo (JobState.Waiting: JobState)).update.run + } + + def incrementRetries(jobid: Ident): ConnectionIO[Int] = + updateRow(table, and(id is jobid, state is (JobState.Stuck: JobState)), + retries.f ++ fr"=" ++ retries.f ++ fr"+ 1").update.run + + def setRunning(jobId: Ident, workerId: Ident, now: Timestamp): ConnectionIO[Int] = + updateRow(table, id is jobId, commas( + state setTo (JobState.Running: JobState), + started setTo now, + startedmillis setTo now.toMillis, + worker setTo workerId + )).update.run + + def setWaiting(jobId: Ident): ConnectionIO[Int] = + updateRow(table, id is jobId, commas( + state setTo (JobState.Waiting: JobState), + started setTo (None: Option[Timestamp]), + startedmillis setTo (None: Option[Long]), + finished setTo (None: Option[Timestamp]) + )).update.run + + def setScheduled(jobId: Ident, workerId: Ident): ConnectionIO[Int] = { + for { + _ <- incrementRetries(jobId) + n <- updateRow(table, and(id is jobId, or(worker isNull, worker is workerId), state isOneOf Seq[JobState](JobState.Waiting, JobState.Stuck)), commas( + state setTo (JobState.Scheduled: JobState), + worker setTo workerId + )).update.run + } yield n + } + + def setSuccess(jobId: Ident, now: Timestamp): ConnectionIO[Int] = + updateRow(table, id is jobId, commas( + state setTo (JobState.Success: JobState), + finished setTo now + )).update.run + + def setStuck(jobId: Ident, now: Timestamp): ConnectionIO[Int] = + updateRow(table, id is jobId, commas( + state setTo (JobState.Stuck: JobState), + finished setTo now + )).update.run + + def setFailed(jobId: Ident, now: Timestamp): ConnectionIO[Int] = + updateRow(table, id is jobId, commas( + state setTo (JobState.Failed: JobState), + finished setTo now + )).update.run + + def setCancelled(jobId: Ident, now: Timestamp): ConnectionIO[Int] = + updateRow(table, id is jobId, commas( + state setTo (JobState.Cancelled: JobState), + finished setTo now + )).update.run + + def getRetries(jobId: Ident): ConnectionIO[Option[Int]] = + selectSimple(List(retries), table, id is jobId).query[Int].option + + def setProgress(jobId: Ident, perc: Int): ConnectionIO[Int] = + updateRow(table, id is jobId, progress setTo perc).update.run + + def selectWaiting: ConnectionIO[Option[RJob]] = { + val sql = selectSimple(all, table, state is (JobState.Waiting: JobState)) + sql.query[RJob].to[Vector].map(_.headOption) + } + + def selectGroupInState(states: Seq[JobState]): ConnectionIO[Vector[Ident]] = { + val sql = selectDistinct(List(group), table, state isOneOf states) ++ orderBy(group.f) + sql.query[Ident].to[Vector] + } + + def delete(jobId: Ident): ConnectionIO[Int] = { + for { + n0 <- RJobLog.deleteAll(jobId) + n1 <- deleteFrom(table, id is jobId).update.run + } yield n0 + n1 + } +} diff --git a/modules/store/src/main/scala/docspell/store/records/RJobGroupUse.scala b/modules/store/src/main/scala/docspell/store/records/RJobGroupUse.scala new file mode 100644 index 00000000..a06bb16e --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RJobGroupUse.scala @@ -0,0 +1,37 @@ +package docspell.store.records + +import cats.implicits._ +import doobie._ +import doobie.implicits._ +import docspell.common._ +import docspell.store.impl.Column +import docspell.store.impl.Implicits._ + +case class RJobGroupUse(groupId: Ident, workerId: Ident) { + +} + +object RJobGroupUse { + + val table = fr"jobgroupuse" + + object Columns { + val group = Column("groupid") + val worker = Column("workerid") + val all = List(group, worker) + } + import Columns._ + + def insert(v: RJobGroupUse): ConnectionIO[Int] = + insertRow(table, all, fr"${v.groupId},${v.workerId}").update.run + + def updateGroup(v: RJobGroupUse): ConnectionIO[Int] = + updateRow(table, worker is v.workerId, group setTo v.groupId).update.run + + def setGroup(v: RJobGroupUse): ConnectionIO[Int] = { + updateGroup(v).flatMap(n => if (n > 0) n.pure[ConnectionIO] else insert(v)) + } + + def findGroup(workerId: Ident): ConnectionIO[Option[Ident]] = + selectSimple(List(group), table, worker is workerId).query[Ident].option +} diff --git a/modules/store/src/main/scala/docspell/store/records/RJobLog.scala b/modules/store/src/main/scala/docspell/store/records/RJobLog.scala new file mode 100644 index 00000000..e3aad8c8 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RJobLog.scala @@ -0,0 +1,39 @@ +package docspell.store.records + +import doobie._ +import doobie.implicits._ +import docspell.common._ +import docspell.store.impl.Column +import docspell.store.impl.Implicits._ + +case class RJobLog( id: Ident + , jobId: Ident + , level: LogLevel + , created: Timestamp + , message: String) { + +} + +object RJobLog { + + val table = fr"joblog" + + object Columns { + val id = Column("id") + val jobId = Column("jid") + val level = Column("level") + val created = Column("created") + val message = Column("message") + val all = List(id, jobId, level, created, message) + } + import Columns._ + + def insert(v: RJobLog): ConnectionIO[Int] = + insertRow(table, all, fr"${v.id},${v.jobId},${v.level},${v.created},${v.message}").update.run + + def findLogs(id: Ident): ConnectionIO[Vector[RJobLog]] = + (selectSimple(all, table, jobId is id) ++ orderBy(created.asc)).query[RJobLog].to[Vector] + + def deleteAll(job: Ident): ConnectionIO[Int] = + deleteFrom(table, jobId is job).update.run +} diff --git a/modules/store/src/main/scala/docspell/store/records/RNode.scala b/modules/store/src/main/scala/docspell/store/records/RNode.scala new file mode 100644 index 00000000..ec42265c --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RNode.scala @@ -0,0 +1,57 @@ +package docspell.store.records + +import cats.effect.Sync +import cats.implicits._ +import doobie._ +import doobie.implicits._ +import docspell.common._ +import docspell.store.impl.Column +import docspell.store.impl.Implicits._ + + +case class RNode(id: Ident, nodeType: NodeType, url: LenientUri, updated: Timestamp, created: Timestamp) { + +} + +object RNode { + + def apply[F[_]: Sync](id: Ident, nodeType: NodeType, uri: LenientUri): F[RNode] = + Timestamp.current[F].map(now => RNode(id, nodeType, uri, now, now)) + + val table = fr"node" + + object Columns { + val id = Column("id") + val nodeType = Column("type") + val url = Column("url") + val updated = Column("updated") + val created = Column("created") + val all = List(id,nodeType,url,updated,created) + } + import Columns._ + + def insert(v: RNode): ConnectionIO[Int] = + insertRow(table, all, fr"${v.id},${v.nodeType},${v.url},${v.updated},${v.created}").update.run + + def update(v: RNode): ConnectionIO[Int] = + updateRow(table, id is v.id, commas( + nodeType setTo v.nodeType, + url setTo v.url, + updated setTo v.updated + )).update.run + + def set(v: RNode): ConnectionIO[Int] = + for { + n <- update(v) + k <- if (n == 0) insert(v) else 0.pure[ConnectionIO] + } yield n + k + + def delete(appId: Ident): ConnectionIO[Int] = + (fr"DELETE FROM" ++ table ++ where(id is appId)).update.run + + def findAll(nt: NodeType): ConnectionIO[Vector[RNode]] = + selectSimple(all, table, nodeType is nt).query[RNode].to[Vector] + + def findById(nodeId: Ident): ConnectionIO[Option[RNode]] = + selectSimple(all, table, id is nodeId).query[RNode].option +} diff --git a/modules/store/src/main/scala/docspell/store/records/ROrganization.scala b/modules/store/src/main/scala/docspell/store/records/ROrganization.scala new file mode 100644 index 00000000..d09127fb --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/ROrganization.scala @@ -0,0 +1,103 @@ +package docspell.store.records + +import fs2.Stream +import doobie._ +import doobie.implicits._ +import docspell.common.{IdRef, _} +import docspell.store.impl._ +import docspell.store.impl.Implicits._ + +case class ROrganization( + oid: Ident + , cid: Ident + , name: String + , street: String + , zip: String + , city: String + , country: String + , notes: Option[String] + , created: Timestamp) { + +} + +object ROrganization { + + val table = fr"organization" + + object Columns { + val oid = Column("oid") + val cid = Column("cid") + val name = Column("name") + val street = Column("street") + val zip = Column("zip") + val city = Column("city") + val country = Column("country") + val notes = Column("notes") + val created = Column("created") + val all = List(oid, cid, name, street, zip, city, country, notes, created) + } + + import Columns._ + + def insert(v: ROrganization): ConnectionIO[Int] = { + val sql = insertRow(table, all, + fr"${v.oid},${v.cid},${v.name},${v.street},${v.zip},${v.city},${v.country},${v.notes},${v.created}") + sql.update.run + } + + def update(v: ROrganization): ConnectionIO[Int] = { + val sql = updateRow(table, and(oid is v.oid, cid is v.cid), commas( + cid setTo v.cid, + name setTo v.name, + street setTo v.street, + zip setTo v.zip, + city setTo v.city, + country setTo v.country, + notes setTo v.notes + )) + sql.update.run + } + + def existsByName(coll: Ident, oname: String): ConnectionIO[Boolean] = + selectCount(oid, table, and(cid is coll, name is oname)).query[Int].unique.map(_ > 0) + + def findById(id: Ident): ConnectionIO[Option[ROrganization]] = { + val sql = selectSimple(all, table, cid is id) + sql.query[ROrganization].option + } + + def find(coll: Ident, orgName: String): ConnectionIO[Option[ROrganization]] = { + val sql = selectSimple(all, table, and(cid is coll, name is orgName)) + sql.query[ROrganization].option + } + + def findLike(coll: Ident, orgName: String): ConnectionIO[Vector[IdRef]] = + selectSimple(List(oid, name), table, and(cid is coll, + name.lowerLike(orgName))). + query[IdRef].to[Vector] + + def findLike(coll: Ident, contactKind: ContactKind, value: String): ConnectionIO[Vector[IdRef]] = { + val CC = RContact.Columns + val q = fr"SELECT DISTINCT" ++ commas(oid.prefix("o").f, name.prefix("o").f) ++ + fr"FROM" ++ table ++ fr"o" ++ + fr"INNER JOIN" ++ RContact.table ++ fr"c ON" ++ CC.orgId.prefix("c").is(oid.prefix("o")) ++ + fr"WHERE" ++ and(cid.prefix("o").is(coll) + , CC.kind.prefix("c") is contactKind + , CC.value.prefix("c").lowerLike(value)) + + q.query[IdRef].to[Vector] + } + + def findAll(coll: Ident, order: Columns.type => Column): Stream[ConnectionIO, ROrganization] = { + val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f) + sql.query[ROrganization].stream + } + + def findAllRef(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[IdRef]] = { + val sql = selectSimple(List(oid, name), table, cid is coll) ++ orderBy(order(Columns).f) + sql.query[IdRef].to[Vector] + } + + def delete(id: Ident, coll: Ident): ConnectionIO[Int] = + deleteFrom(table, and(oid is id, cid is coll)).update.run +} diff --git a/modules/store/src/main/scala/docspell/store/records/RPerson.scala b/modules/store/src/main/scala/docspell/store/records/RPerson.scala new file mode 100644 index 00000000..3941d441 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RPerson.scala @@ -0,0 +1,108 @@ +package docspell.store.records + +import fs2.Stream +import doobie._ +import doobie.implicits._ +import docspell.common.{IdRef, _} +import docspell.store.impl._ +import docspell.store.impl.Implicits._ + +case class RPerson( + pid: Ident + , cid: Ident + , name: String + , street: String + , zip: String + , city: String + , country: String + , notes: Option[String] + , concerning: Boolean + , created: Timestamp) { + +} + +object RPerson { + + val table = fr"person" + + object Columns { + val pid = Column("pid") + val cid = Column("cid") + val name = Column("name") + val street = Column("street") + val zip = Column("zip") + val city = Column("city") + val country = Column("country") + val notes = Column("notes") + val concerning = Column("concerning") + val created = Column("created") + val all = List(pid, cid, name, street, zip, city, country, notes, concerning, created) + } + + import Columns._ + + def insert(v: RPerson): ConnectionIO[Int] = { + val sql = insertRow(table, all, + fr"${v.pid},${v.cid},${v.name},${v.street},${v.zip},${v.city},${v.country},${v.notes},${v.concerning},${v.created}") + sql.update.run + } + + def update(v: RPerson): ConnectionIO[Int] = { + val sql = updateRow(table, and(pid is v.pid, cid is v.cid), commas( + cid setTo v.cid, + name setTo v.name, + street setTo v.street, + zip setTo v.zip, + city setTo v.city, + country setTo v.country, + concerning setTo v.concerning, + notes setTo v.notes + )) + sql.update.run + } + + def existsByName(coll: Ident, pname: String): ConnectionIO[Boolean] = + selectCount(pid, table, and(cid is coll, name is pname)).query[Int].unique.map(_ > 0) + + def findById(id: Ident): ConnectionIO[Option[RPerson]] = { + val sql = selectSimple(all, table, cid is id) + sql.query[RPerson].option + } + + def find(coll: Ident, personName: String): ConnectionIO[Option[RPerson]] = { + val sql = selectSimple(all, table, and(cid is coll, name is personName)) + sql.query[RPerson].option + } + + def findLike(coll: Ident, personName: String, concerningOnly: Boolean): ConnectionIO[Vector[IdRef]] = + selectSimple(List(pid, name), table, and(cid is coll, + concerning is concerningOnly, + name.lowerLike(personName))). + query[IdRef].to[Vector] + + def findLike(coll: Ident, contactKind: ContactKind, value: String, concerningOnly: Boolean): ConnectionIO[Vector[IdRef]] = { + val CC = RContact.Columns + val q = fr"SELECT DISTINCT" ++ commas(pid.prefix("p").f, name.prefix("p").f) ++ + fr"FROM" ++ table ++ fr"p" ++ + fr"INNER JOIN" ++ RContact.table ++ fr"c ON" ++ CC.personId.prefix("c").is(pid.prefix("p")) ++ + fr"WHERE" ++ and(cid.prefix("p").is(coll) + , CC.kind.prefix("c") is contactKind + , concerning.prefix("p") is concerningOnly + , CC.value.prefix("c").lowerLike(value)) + + q.query[IdRef].to[Vector] + } + + def findAll(coll: Ident, order: Columns.type => Column): Stream[ConnectionIO, RPerson] = { + val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f) + sql.query[RPerson].stream + } + + def findAllRef(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[IdRef]] = { + val sql = selectSimple(List(pid, name), table, cid is coll) ++ orderBy(order(Columns).f) + sql.query[IdRef].to[Vector] + } + + def delete(personId: Ident, coll: Ident): ConnectionIO[Int] = + deleteFrom(table, and(pid is personId, cid is coll)).update.run +} diff --git a/modules/store/src/main/scala/docspell/store/records/RSource.scala b/modules/store/src/main/scala/docspell/store/records/RSource.scala new file mode 100644 index 00000000..c4a99421 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RSource.scala @@ -0,0 +1,87 @@ +package docspell.store.records + +import doobie._ +import doobie.implicits._ +import docspell.common._ +import docspell.store.impl._ +import docspell.store.impl.Implicits._ + +case class RSource( + sid: Ident + , cid: Ident + , abbrev: String + , description: Option[String] + , counter: Int + , enabled: Boolean + , priority: Priority + , created: Timestamp) { + +} + +object RSource { + + val table = fr"source" + + object Columns { + + val sid = Column("sid") + val cid = Column("cid") + val abbrev = Column("abbrev") + val description = Column("description") + val counter = Column("counter") + val enabled = Column("enabled") + val priority = Column("priority") + val created = Column("created") + + val all = List(sid,cid,abbrev,description,counter,enabled,priority,created) + } + + import Columns._ + + def insert(v: RSource): ConnectionIO[Int] = { + val sql = insertRow(table, all, + fr"${v.sid},${v.cid},${v.abbrev},${v.description},${v.counter},${v.enabled},${v.priority},${v.created}") + sql.update.run + } + + def updateNoCounter(v: RSource): ConnectionIO[Int] = { + val sql = updateRow(table, and(sid is v.sid, cid is v.cid), commas( + cid setTo v.cid, + abbrev setTo v.abbrev, + description setTo v.description, + enabled setTo v.enabled, + priority setTo v.priority + )) + sql.update.run + } + + def incrementCounter(source: String, coll: Ident): ConnectionIO[Int] = + updateRow(table, and(abbrev is source, cid is coll), counter.f ++ fr"=" ++ counter.f ++ fr"+ 1").update.run + + def existsById(id: Ident): ConnectionIO[Boolean] = { + val sql = selectCount(sid, table, sid is id) + sql.query[Int].unique.map(_ > 0) + } + + def existsByAbbrev(coll: Ident, abb: String): ConnectionIO[Boolean] = { + val sql = selectCount(sid, table, and(cid is coll, abbrev is abb)) + sql.query[Int].unique.map(_ > 0) + } + + + def find(id: Ident): ConnectionIO[Option[RSource]] = { + val sql = selectSimple(all, table, sid is id) + sql.query[RSource].option + } + + def findCollective(sourceId: Ident): ConnectionIO[Option[Ident]] = + selectSimple(List(cid), table, sid is sourceId).query[Ident].option + + def findAll(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[RSource]] = { + val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f) + sql.query[RSource].to[Vector] + } + + def delete(sourceId: Ident, coll: Ident): ConnectionIO[Int] = + deleteFrom(table, and(sid is sourceId, cid is coll)).update.run +} diff --git a/modules/store/src/main/scala/docspell/store/records/RTag.scala b/modules/store/src/main/scala/docspell/store/records/RTag.scala new file mode 100644 index 00000000..246febf0 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RTag.scala @@ -0,0 +1,76 @@ +package docspell.store.records + +import doobie._, doobie.implicits._ +import docspell.common._ +import docspell.store.impl._ +import docspell.store.impl.Implicits._ + +case class RTag( + tagId: Ident + , collective: Ident + , name: String + , category: Option[String] + , created: Timestamp) { + +} + +object RTag { + + val table = fr"tag" + + object Columns { + val tid = Column("tid") + val cid = Column("cid") + val name = Column("name") + val category = Column("category") + val created = Column("created") + val all = List(tid,cid,name,category,created) + } + import Columns._ + + def insert(v: RTag): ConnectionIO[Int] = { + val sql = insertRow(table, all, + fr"${v.tagId},${v.collective},${v.name},${v.category},${v.created}") + sql.update.run + } + + def update(v: RTag): ConnectionIO[Int] = { + val sql = updateRow(table, and(tid is v.tagId, cid is v.collective), commas( + cid setTo v.collective, + name setTo v.name, + category setTo v.category + )) + sql.update.run + } + + def findById(id: Ident): ConnectionIO[Option[RTag]] = { + val sql = selectSimple(all, table, tid is id) + sql.query[RTag].option + } + + def findByIdAndCollective(id: Ident, coll: Ident): ConnectionIO[Option[RTag]] = { + val sql = selectSimple(all, table, and(tid is id, cid is coll)) + sql.query[RTag].option + } + + def existsByName(tag: RTag): ConnectionIO[Boolean] = { + val sql = selectCount(tid, table, and(cid is tag.collective, name is tag.name, category is tag.category)) + sql.query[Int].unique.map(_ > 0) + } + + def findAll(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[RTag]] = { + val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f) + sql.query[RTag].to[Vector] + } + + def findByItem(itemId: Ident): ConnectionIO[Vector[RTag]] = { + val rcol = all.map(_.prefix("t")) + (selectSimple(rcol, table ++ fr"t," ++ RTagItem.table ++ fr"i", and( + RTagItem.Columns.itemId.prefix("i") is itemId, + RTagItem.Columns.tagId.prefix("i").is(tid.prefix("t")) + )) ++ orderBy(name.prefix("t").asc)).query[RTag].to[Vector] + } + + def delete(tagId: Ident, coll: Ident): ConnectionIO[Int] = + deleteFrom(table, and(tid is tagId, cid is coll)).update.run +} diff --git a/modules/store/src/main/scala/docspell/store/records/RTagItem.scala b/modules/store/src/main/scala/docspell/store/records/RTagItem.scala new file mode 100644 index 00000000..68c8f75e --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RTagItem.scala @@ -0,0 +1,42 @@ +package docspell.store.records + +import cats.implicits._ +import doobie._ +import doobie.implicits._ +import docspell.common._ +import docspell.store.impl._ +import docspell.store.impl.Implicits._ + +case class RTagItem(tagItemId: Ident, itemId: Ident, tagId: Ident) { + +} + +object RTagItem { + + val table = fr"tagitem" + + object Columns { + val tagItemId = Column("tagitemid") + val itemId = Column("itemid") + val tagId = Column("tid") + val all = List(tagItemId, itemId, tagId) + } + import Columns._ + + def insert(v: RTagItem): ConnectionIO[Int] = + insertRow(table, all, fr"${v.tagItemId},${v.itemId},${v.tagId}").update.run + + def deleteItemTags(item: Ident): ConnectionIO[Int] = + deleteFrom(table, itemId is item).update.run + + def deleteTag(tid: Ident): ConnectionIO[Int] = + deleteFrom(table, tagId is tid).update.run + + def insertItemTags(item: Ident, tags: Seq[Ident]): ConnectionIO[Int] = { + for { + tagValues <- tags.toList.traverse(id => Ident.randomId[ConnectionIO].map(rid => RTagItem(rid, item, id))) + tagFrag = tagValues.map(v => fr"${v.tagItemId},${v.itemId},${v.tagId}") + ins <- insertRows(table, all, tagFrag).update.run + } yield ins + } +} diff --git a/modules/store/src/main/scala/docspell/store/records/RUser.scala b/modules/store/src/main/scala/docspell/store/records/RUser.scala new file mode 100644 index 00000000..bbfa3c38 --- /dev/null +++ b/modules/store/src/main/scala/docspell/store/records/RUser.scala @@ -0,0 +1,99 @@ +package docspell.store.records + +import doobie._, doobie.implicits._ +import docspell.common._ +import docspell.store.impl._ +import docspell.store.impl.Implicits._ + +case class RUser( + uid: Ident + , login: Ident + , cid: Ident + , password: Password + , state: UserState + , email: Option[String] + , loginCount: Int + , lastLogin: Option[Timestamp] + , created: Timestamp) { + +} + +object RUser { + + val table = fr"user_" + + object Columns { + val uid = Column("uid") + val cid = Column("cid") + val login = Column("login") + val password = Column("password") + val state = Column("state") + val email = Column("email") + val loginCount = Column("logincount") + val lastLogin = Column("lastlogin") + val created = Column("created") + + val all = List(uid + ,login + ,cid + ,password + ,state + ,email + ,loginCount + ,lastLogin + ,created) + } + + import Columns._ + + def insert(v: RUser): ConnectionIO[Int] = { + val sql = insertRow(table, Columns.all, + fr"${v.uid},${v.login},${v.cid},${v.password},${v.state},${v.email},${v.loginCount},${v.lastLogin},${v.created}") + sql.update.run + } + + def update(v: RUser): ConnectionIO[Int] = { + val sql = updateRow(table, and(login is v.login, cid is v.cid), commas( + state setTo v.state, + email setTo v.email, + loginCount setTo v.loginCount, + lastLogin setTo v.lastLogin + )) + sql.update.run + } + + def exists(loginName: Ident): ConnectionIO[Boolean] = { + selectCount(uid, table, login is loginName).query[Int].unique.map(_ > 0) + } + + def findByAccount(aid: AccountId): ConnectionIO[Option[RUser]] = { + val sql = selectSimple(all, table, and(cid is aid.collective, login is aid.user)) + sql.query[RUser].option + } + + def findById(userId: Ident): ConnectionIO[Option[RUser]] = { + val sql = selectSimple(all, table, uid is userId) + sql.query[RUser].option + } + + def findAll(coll: Ident, order: Columns.type => Column): ConnectionIO[Vector[RUser]] = { + val sql = selectSimple(all, table, cid is coll) ++ orderBy(order(Columns).f) + sql.query[RUser].to[Vector] + } + + def updateLogin(accountId: AccountId): ConnectionIO[Int] = + currentTime.flatMap(t => updateRow(table + , and(cid is accountId.collective, login is accountId.user) + , commas( + loginCount.f ++ fr"=" ++ loginCount.f ++ fr"+ 1", + lastLogin setTo t + )).update.run) + + def updatePassword(accountId: AccountId, hashedPass: Password): ConnectionIO[Int] = + updateRow(table + , and(cid is accountId.collective, login is accountId.user) + , password setTo hashedPass).update.run + + def delete(user: Ident, coll: Ident): ConnectionIO[Int] = + deleteFrom(table, and(cid is coll, login is user)).update.run +} diff --git a/modules/text/src/main/scala/docspell/text/contact/Contact.scala b/modules/text/src/main/scala/docspell/text/contact/Contact.scala new file mode 100644 index 00000000..8ad7829d --- /dev/null +++ b/modules/text/src/main/scala/docspell/text/contact/Contact.scala @@ -0,0 +1,51 @@ +package docspell.text.contact + +import fs2.Stream +import cats.implicits._ +import docspell.common.{Ident, LenientUri, NerLabel, NerTag} +import docspell.text.split.TextSplitter + +object Contact { + private[this] val protocols = Set("ftp", "http", "https") + + def annotate(text: String): Vector[NerLabel] = + TextSplitter.splitToken[Nothing](text, " \t\r\n".toSet). + map({ token => + if (isEmailAddress(token.value)) NerLabel(token.value, NerTag.Email, token.begin, token.end).some + else if (isWebsite(token.value)) NerLabel(token.value, NerTag.Website, token.begin, token.end).some + else None + }). + flatMap(_.map(Stream.emit).getOrElse(Stream.empty)). + toVector + + + def isEmailAddress(str: String): Boolean = { + val atIdx = str.indexOf('@') + if (atIdx <= 0 || str.indexOf('@', atIdx + 1) > 0) false + else { + val name = str.substring(0, atIdx) + val dom = str.substring(atIdx + 1) + Domain.isDomain(dom) && name.forall(c => !c.isWhitespace) + } + } + + def isWebsite(str: String): Boolean = + LenientUri.parse(str). + toOption. + map(uri => protocols.contains(uri.scheme.head)). + getOrElse(Domain.isDomain(str)) + + def isDocspellOpenUpload(str: String): Boolean = { + def isUploadPath(p: LenientUri.Path): Boolean = + p match { + case LenientUri.RootPath => false + case LenientUri.EmptyPath => false + case LenientUri.NonEmptyPath(segs) => + Ident.fromString(segs.last).isRight && + segs.init.takeRight(3) == List("open", "upload", "item") + } + LenientUri.parse(str). + toOption. + exists(uri => protocols.contains(uri.scheme.head) && isUploadPath(uri.path)) + } +} diff --git a/modules/text/src/main/scala/docspell/text/contact/Domain.scala b/modules/text/src/main/scala/docspell/text/contact/Domain.scala new file mode 100644 index 00000000..5b62aba9 --- /dev/null +++ b/modules/text/src/main/scala/docspell/text/contact/Domain.scala @@ -0,0 +1,40 @@ +package docspell.text.contact + +import cats.data.NonEmptyList +import docspell.common.LenientUri + +case class Domain(labels: NonEmptyList[String], tld: String) { + + def asString: String = + labels.toList.mkString(".") + tld + + def toPrimaryDomain: Domain = + if (labels.tail.isEmpty) this + else Domain(NonEmptyList.of(labels.last), tld) +} + +object Domain { + + def domainFromUri(uri: String): Either[String, Domain] = + LenientUri.parse(if (uri.contains("://")) uri else s"http://$uri"). + flatMap(uri => uri.authority.toRight("Uri has no authoriry part")). + flatMap(auth => parse(auth)) + + def parse(str: String): Either[String, Domain] = { + Tld.findTld(str). + map(tld => (str.dropRight(tld.length), tld)). + map({ case (names, tld) => + names.split('.').toList match { + case Nil => Left(s"Not a domain: $str") + case segs if segs.forall(label => + label.trim.nonEmpty && label.forall(c => c.isLetter || c.isDigit || c == '-')) => + Right(Domain(NonEmptyList.fromListUnsafe(segs), tld)) + case _ => Left(s"Not a domain: $str") + } + }). + getOrElse(Left(s"Not a domain $str")) + } + + def isDomain(str: String): Boolean = + parse(str).isRight +} diff --git a/modules/text/src/main/scala/docspell/text/contact/Tld.scala b/modules/text/src/main/scala/docspell/text/contact/Tld.scala new file mode 100644 index 00000000..f8caa6b3 --- /dev/null +++ b/modules/text/src/main/scala/docspell/text/contact/Tld.scala @@ -0,0 +1,83 @@ +package docspell.text.contact + +private[text] object Tld { + + def findTld(str: String): Option[String] = + known.find(str.endsWith) + + def endsWithTld(str: String): Boolean = + findTld(str).isDefined + + /** + * Some selected TLDs. + */ + private [this] val known = List( + ".com", + ".org", + ".net", + ".int", + ".edu", + ".gov", + ".mil", + ".ad", + ".ae", + ".al", + ".am", + ".ar", + ".as", + ".at", + ".au", + ".ax", + ".ba", + ".bd", + ".be", + ".bg", + ".br", + ".by", + ".bz", + ".ca", + ".cc", + ".ch", + ".cn", + ".co", + ".cu", + ".cx", + ".cy", + ".de", + ".dk", + ".dj", + ".ee", + ".eu", + ".fi", + ".fr", + ".gr", + ".hk", + ".hr", + ".hu", + ".ie", + ".il", + ".io", + ".is", + ".ir", + ".it", + ".jp", + ".li", + ".lt", + ".mt", + ".no", + ".nz", + ".pl", + ".pt", + ".ru", + ".rs", + ".se", + ".si", + ".sk", + ".th", + ".ua", + ".uk", + ".us", + ".ws" + ) + +} diff --git a/modules/text/src/main/scala/docspell/text/date/DateFind.scala b/modules/text/src/main/scala/docspell/text/date/DateFind.scala new file mode 100644 index 00000000..ff011e7f --- /dev/null +++ b/modules/text/src/main/scala/docspell/text/date/DateFind.scala @@ -0,0 +1,125 @@ +package docspell.text.date + +import fs2._ +import java.time.LocalDate + +import docspell.common.{Language, NerDateLabel, NerLabel, NerTag} +import docspell.text.split.{TextSplitter, Word} + +import scala.util.Try + +object DateFind { + + def findDates(text: String, lang: Language): Stream[Pure, NerDateLabel] = { + TextSplitter.splitToken(text, " \t.,\n\r/".toSet). + sliding(3). + filter(_.length == 3). + map(q => SimpleDate.fromParts(q.toList, lang). + map(sd => NerDateLabel(sd.toLocalDate, + NerLabel(text.substring(q(0).begin, q(2).end), NerTag.Date, q(0).begin, q(1).end)))). + collect({ case Some(d) => d }) + } + + + private case class SimpleDate(year: Int, month: Int, day: Int) { + def toLocalDate: LocalDate = + LocalDate.of(if (year < 100) 1900 + year else year, month, day) + } + + private object SimpleDate { + val p0 = readYear >> readMonth >> readDay map { + case ((y, m), d) => SimpleDate(y, m, d) + } + val p1 = readDay >> readMonth >> readYear map { + case ((d, m), y) => SimpleDate(y, m, d) + } + val p2 = readMonth >> readDay >> readYear map { + case ((m, d), y) => SimpleDate(y, m, d) + } + + // ymd ✔, ydm, dmy ✔, dym, myd, mdy ✔ + def fromParts(parts: List[Word], lang: Language): Option[SimpleDate] = { + val p = lang match { + case Language.English => p2.or(p0).or(p1) + case Language.German => p1.or(p0).or(p2) + } + p.read(parts).toOption + } + + + def readYear: Reader[Int] = { + Reader.readFirst(w => w.value.length match { + case 2 => Try(w.value.toInt).filter(n => n >= 0).toOption + case 4 => Try(w.value.toInt).filter(n => n > 1000).toOption + case _ => None + }) + } + + def readMonth: Reader[Int] = + Reader.readFirst(w => Some(months.indexWhere(_.contains(w.value))).filter(_ > 0).map(_ + 1)) + + def readDay: Reader[Int] = + Reader.readFirst(w => Try(w.value.toInt).filter(n => n > 0 && n <= 31).toOption) + + case class Reader[A](read: List[Word] => Result[A]) { + def >>[B](next: Reader[B]): Reader[(A, B)] = + Reader(read.andThen(_.next(next))) + + def map[B](f: A => B): Reader[B] = + Reader(read.andThen(_.map(f))) + + def or(other: Reader[A]): Reader[A] = + Reader(words => read(words) match { + case Result.Failure => other.read(words) + case s @ Result.Success(_, _) => s + }) + } + + object Reader { + def fail[A]: Reader[A] = + Reader(_ => Result.Failure) + + def readFirst[A](f: Word => Option[A]): Reader[A] = + Reader({ + case Nil => Result.Failure + case a :: as => f(a).map(value => Result.Success(value, as)).getOrElse(Result.Failure) + }) + } + + + sealed trait Result[+A] { + def toOption: Option[A] + def map[B](f: A => B): Result[B] + def next[B](r: Reader[B]): Result[(A, B)] + } + + object Result { + final case class Success[A](value: A, rest: List[Word]) extends Result[A] { + val toOption = Some(value) + def map[B](f: A => B): Result[B] = Success(f(value), rest) + def next[B](r: Reader[B]): Result[(A, B)] = + r.read(rest).map(b => (value, b)) + } + final case object Failure extends Result[Nothing] { + val toOption = None + def map[B](f: Nothing => B): Result[B] = this + def next[B](r: Reader[B]): Result[(Nothing, B)] = this + } + } + + private val months = List( + List("jan", "january", "januar", "01"), + List("feb", "february", "februar", "02"), + List("mar", "march", "märz", "marz", "03"), + List("apr", "april", "04"), + List("may", "mai", "05"), + List("jun", "june", "juni", "06"), + List("jul", "july", "juli", "07"), + List("aug", "august", "08"), + List("sep", "september", "09"), + List("oct", "october", "oktober", "10"), + List("nov", "november", "11"), + List("dec", "december", "dezember", "12") + ) + } +} diff --git a/modules/text/src/main/scala/docspell/text/nlp/StanfordNerClassifier.scala b/modules/text/src/main/scala/docspell/text/nlp/StanfordNerClassifier.scala new file mode 100644 index 00000000..5d825541 --- /dev/null +++ b/modules/text/src/main/scala/docspell/text/nlp/StanfordNerClassifier.scala @@ -0,0 +1,56 @@ +package docspell.text.nlp + +import java.util.zip.GZIPInputStream + +import docspell.common.{Language, NerLabel, NerTag} +import edu.stanford.nlp.ie.AbstractSequenceClassifier +import edu.stanford.nlp.ie.crf.CRFClassifier +import edu.stanford.nlp.ling.{CoreAnnotations, CoreLabel} + +import scala.jdk.CollectionConverters._ +import org.log4s._ + +import java.net.URL +import scala.util.Using + +object StanfordNerClassifier { + private [this] val logger = getLogger + + lazy val germanNerClassifier = makeClassifier(Language.German) + lazy val englishNerClassifier = makeClassifier(Language.English) + + def nerAnnotate(lang: Language)(text: String): Vector[NerLabel] = { + val nerClassifier = lang match { + case Language.English => englishNerClassifier + case Language.German => germanNerClassifier + } + nerClassifier.classify(text).asScala.flatMap(a => a.asScala). + collect(Function.unlift(label => { + val tag = label.get(classOf[CoreAnnotations.AnswerAnnotation]) + NerTag.fromString(Option(tag).getOrElse("")).toOption. + map(t => NerLabel(label.word(), t, label.beginPosition(), label.endPosition())) + })). + toVector + } + + private def makeClassifier(lang: Language): AbstractSequenceClassifier[CoreLabel] = { + logger.info(s"Creating ${lang.name} Stanford NLP NER classifier...") + val ner = classifierResource(lang) + Using(new GZIPInputStream(ner.openStream())) { in => + CRFClassifier.getClassifier(in).asInstanceOf[AbstractSequenceClassifier[CoreLabel]] + }.fold(throw _, identity) + } + + private def classifierResource(lang: Language): URL = { + def check(u: URL): URL = + if (u == null) sys.error(s"NER model url not found for language ${lang.name}") + else u + + check(lang match { + case Language.German => + getClass.getResource("/edu/stanford/nlp/models/ner/german.conll.germeval2014.hgc_175m_600.crf.ser.gz") + case Language.English => + getClass.getResource("/edu/stanford/nlp/models/ner/english.all.3class.distsim.crf.ser.gz") + }) + } +} diff --git a/modules/text/src/main/scala/docspell/text/ocr/Config.scala b/modules/text/src/main/scala/docspell/text/ocr/Config.scala new file mode 100644 index 00000000..42fe7706 --- /dev/null +++ b/modules/text/src/main/scala/docspell/text/ocr/Config.scala @@ -0,0 +1,66 @@ +package docspell.text.ocr + +import java.nio.file.{Path, Paths} + +import docspell.common._ + +case class Config( + allowedContentTypes: Set[MimeType] + , ghostscript: Config.Ghostscript + , pageRange: Config.PageRange + , unpaper: Config.Unpaper + , tesseract: Config.Tesseract +) { + + def isAllowed(mt: MimeType): Boolean = + allowedContentTypes contains mt +} + +object Config { + case class PageRange(begin: Int) + + case class Command(program: String, args: Seq[String], timeout: Duration) { + + def mapArgs(f: String => String): Command = + Command(program, args map f, timeout) + + def toCmd: List[String] = + program :: args.toList + + lazy val cmdString: String = + toCmd.mkString(" ") + } + + case class Ghostscript(command: Command, workingDir: Path) + case class Tesseract(command: Command) + case class Unpaper(command: Command) + + val default = Config( + allowedContentTypes = Set( + MimeType.pdf, + MimeType.png, + MimeType.jpeg, + MimeType.tiff + ), + pageRange = PageRange(10), + ghostscript = Ghostscript( + Command("gs", Seq("-dNOPAUSE" + , "-dBATCH" + , "-dSAFER" + , "-sDEVICE=tiffscaled8" + , "-sOutputFile={{outfile}}" + , "{{infile}}"), + Duration.seconds(30)), + Paths.get(System.getProperty("java.io.tmpdir")). + resolve("docspell-extraction")), + unpaper = Unpaper(Command("unpaper" + , Seq("{{infile}}", "{{outfile}}") + , Duration.seconds(30))), + tesseract = Tesseract( + Command("tesseract", Seq("{{file}}" + , "stdout" + , "-l" + , "{{lang}}"), + Duration.minutes(1))) + ) +} diff --git a/modules/text/src/main/scala/docspell/text/ocr/File.scala b/modules/text/src/main/scala/docspell/text/ocr/File.scala new file mode 100644 index 00000000..91bc5dcf --- /dev/null +++ b/modules/text/src/main/scala/docspell/text/ocr/File.scala @@ -0,0 +1,56 @@ +package docspell.text.ocr + +import cats.implicits._ +import scala.jdk.CollectionConverters._ +import java.io.IOException +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.{FileVisitResult, Files, Path, SimpleFileVisitor} +import java.util.concurrent.atomic.AtomicInteger + +import cats.effect.Sync +import fs2.Stream + +object File { + + def mkDir[F[_]: Sync](dir: Path): F[Path] = + Sync[F].delay(Files.createDirectories(dir)) + + def mkTempDir[F[_]: Sync](parent: Path, prefix: String): F[Path] = + mkDir(parent).map(p => Files.createTempDirectory(p, prefix)) + + def deleteDirectory[F[_]: Sync](dir: Path): F[Int] = Sync[F].delay { + val count = new AtomicInteger(0) + Files.walkFileTree(dir, new SimpleFileVisitor[Path]() { + override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = { + Files.deleteIfExists(file) + count.incrementAndGet() + FileVisitResult.CONTINUE + } + override def postVisitDirectory(dir: Path, e: IOException): FileVisitResult = + Option(e) match { + case Some(ex) => throw ex + case None => + Files.deleteIfExists(dir) + FileVisitResult.CONTINUE + } + }) + count.get + } + + def deleteFile[F[_]: Sync](file: Path): F[Unit] = + Sync[F].delay(Files.deleteIfExists(file)).map(_ => ()) + + def delete[F[_]: Sync](path: Path): F[Int] = + if (Files.isDirectory(path)) deleteDirectory(path) + else deleteFile(path).map(_ => 1) + + def withTempDir[F[_]: Sync, A](parent: Path, prefix: String) + (f: Path => Stream[F, A]): Stream[F, A] = + Stream.bracket(mkTempDir(parent, prefix))(p => delete(p).map(_ => ())).flatMap(f) + + def listFiles[F[_]: Sync](pred: Path => Boolean, dir: Path): F[List[Path]] = Sync[F].delay { + val javaList = Files.list(dir).filter(p => pred(p)).collect(java.util.stream.Collectors.toList()) + javaList.asScala.toList.sortBy(_.getFileName.toString) + } + +} diff --git a/modules/text/src/main/scala/docspell/text/ocr/MimeTypeHint.scala b/modules/text/src/main/scala/docspell/text/ocr/MimeTypeHint.scala new file mode 100644 index 00000000..497b1f33 --- /dev/null +++ b/modules/text/src/main/scala/docspell/text/ocr/MimeTypeHint.scala @@ -0,0 +1,9 @@ +package docspell.text.ocr + +case class MimeTypeHint(filename: Option[String], advertised: Option[String]) { + +} + +object MimeTypeHint { + val none = MimeTypeHint(None, None) +} diff --git a/modules/text/src/main/scala/docspell/text/ocr/Ocr.scala b/modules/text/src/main/scala/docspell/text/ocr/Ocr.scala new file mode 100644 index 00000000..1cc402c3 --- /dev/null +++ b/modules/text/src/main/scala/docspell/text/ocr/Ocr.scala @@ -0,0 +1,148 @@ +package docspell.text.ocr + +import java.nio.file.Path + +import cats.effect.{Blocker, ContextShift, Sync} +import fs2.Stream +import org.log4s._ + +object Ocr { + private[this] val logger = getLogger + + /** Extract the text of all pages in the given pdf file. + */ + def extractPdf[F[_]: Sync: ContextShift](pdf: Stream[F, Byte], blocker: Blocker, lang: String, config: Config): Stream[F, String] = + File.withTempDir(config.ghostscript.workingDir, "extractpdf") { wd => + runGhostscript(pdf, config, wd, blocker). + flatMap({ tmpImg => + runTesseractFile(tmpImg, blocker, lang, config) + }). + fold1(_ + "\n\n\n" + _) + } + + /** Extract the text from the given image file + */ + def extractImage[F[_]: Sync: ContextShift](img: Stream[F, Byte], blocker: Blocker, lang: String, config: Config): Stream[F, String] = + runTesseractStdin(img, blocker, lang, config) + + + def extractPdFFile[F[_]: Sync: ContextShift](pdf: Path, blocker: Blocker, lang: String, config: Config): Stream[F, String] = + File.withTempDir(config.ghostscript.workingDir, "extractpdf") { wd => + runGhostscriptFile(pdf, config.ghostscript.command, wd, blocker). + flatMap({ tif => + runTesseractFile(tif, blocker, lang, config) + }). + fold1(_ + "\n\n\n" + _) + } + + def extractImageFile[F[_]: Sync: ContextShift](img: Path, blocker: Blocker, lang: String, config: Config): Stream[F, String] = + runTesseractFile(img, blocker, lang, config) + + /** Run ghostscript to extract all pdf pages into tiff files. The + * files are stored to a temporary location on disk and returned. + */ + private[text] def runGhostscript[F[_]: Sync: ContextShift]( + pdf: Stream[F, Byte] + , cfg: Config + , wd: Path + , blocker: Blocker): Stream[F, Path] = { + val xargs = + if (cfg.pageRange.begin > 0) s"-dLastPage=${cfg.pageRange.begin}" +: cfg.ghostscript.command.args + else cfg.ghostscript.command.args + val cmd = cfg.ghostscript.command.copy(args = xargs).mapArgs(replace(Map( + "{{infile}}" -> "-", + "{{outfile}}" -> "%d.tif" + ))) + SystemCommand.execSuccess(cmd, blocker, wd = Some(wd), stdin = pdf). + evalMap({ _ => + File.listFiles(pathEndsWith(".tif"), wd) + }). + flatMap(fs => Stream.emits(fs)) + } + + /** Run ghostscript to extract all pdf pages into tiff files. The + * files are stored to a temporary location on disk and returned. + */ + private[text] def runGhostscriptFile[F[_]: Sync: ContextShift]( + pdf: Path + , ghostscript: Config.Command + , wd: Path, blocker: Blocker): Stream[F, Path] = { + val cmd = ghostscript.mapArgs(replace(Map( + "{{infile}}" -> pdf.toAbsolutePath.toString, + "{{outfile}}" -> "%d.tif" + ))) + SystemCommand.execSuccess[F](cmd, blocker, wd = Some(wd)). + evalMap({ _ => + File.listFiles(pathEndsWith(".tif"), wd) + }). + flatMap(fs => Stream.emits(fs)) + } + + private def pathEndsWith(ext: String): Path => Boolean = + p => p.getFileName.toString.endsWith(ext) + + /** Run unpaper to optimize the image for ocr. The + * files are stored to a temporary location on disk and returned. + */ + private[text] def runUnpaperFile[F[_]: Sync: ContextShift](img: Path + , unpaper: Config.Command + , wd: Path, blocker: Blocker): Stream[F, Path] = { + val targetFile = img.resolveSibling("u-"+ img.getFileName.toString).toAbsolutePath + val cmd = unpaper.mapArgs(replace(Map( + "{{infile}}" -> img.toAbsolutePath.toString, + "{{outfile}}" -> targetFile.toString + ))) + SystemCommand.execSuccess[F](cmd, blocker, wd = Some(wd)). + map(_ => targetFile). + handleErrorWith(th => { + logger.warn(s"Unpaper command failed: ${th.getMessage}. Using input file for text extraction.") + Stream.emit(img) + }) + } + + /** Run tesseract on the given image file and return the extracted + * text. + */ + private[text] def runTesseractFile[F[_]: Sync: ContextShift]( + img: Path + , blocker: Blocker + , lang: String + , config: Config): Stream[F, String] = { + // tesseract cannot cope with absolute filenames + // so use the parent as working dir + runUnpaperFile(img, config.unpaper.command, img.getParent, blocker). + flatMap(uimg => { + val cmd = config.tesseract.command.mapArgs(replace(Map( + "{{file}}" -> uimg.getFileName.toString + , "{{lang}}" -> fixLanguage(lang)))) + SystemCommand.execSuccess[F](cmd, blocker, wd = Some(uimg.getParent)).map(_.stdout) + }) + } + + + /** Run tesseract on the given image file and return the extracted + * text. + */ + private[text] def runTesseractStdin[F[_]: Sync: ContextShift]( + img: Stream[F, Byte] + , blocker: Blocker + , lang: String + , config: Config): Stream[F, String] = { + val cmd = config.tesseract.command.mapArgs(replace(Map( + "{{file}}" -> "stdin" + , "{{lang}}" -> fixLanguage(lang)))) + SystemCommand.execSuccess(cmd, blocker, stdin = img).map(_.stdout) + } + + private def replace(repl: Map[String, String]): String => String = + s => repl.foldLeft(s) { case (res, (k, v)) => + res.replace(k, v) + } + + private def fixLanguage(lang: String): String = + lang match { + case "de" => "deu" + case "en" => "eng" + case l => l + } +} diff --git a/modules/text/src/main/scala/docspell/text/ocr/SystemCommand.scala b/modules/text/src/main/scala/docspell/text/ocr/SystemCommand.scala new file mode 100644 index 00000000..630941e8 --- /dev/null +++ b/modules/text/src/main/scala/docspell/text/ocr/SystemCommand.scala @@ -0,0 +1,72 @@ +package docspell.text.ocr + +import java.io.InputStream +import java.nio.file.Path +import java.util.concurrent.TimeUnit +import cats.implicits._ +import cats.effect.{Blocker, ContextShift, Sync} +import fs2.{Stream, io, text} +import org.log4s.getLogger +import scala.jdk.CollectionConverters._ +import docspell.common.syntax.all._ + +object SystemCommand { + + private[this] val logger = getLogger + + final case class Result(rc: Int, stdout: String, stderr: String) + + def exec[F[_]: Sync: ContextShift]( cmd: Config.Command + , blocker: Blocker + , wd: Option[Path] = None + , stdin: Stream[F, Byte] = Stream.empty): Stream[F, Result] = + startProcess(cmd, wd){ proc => + Stream.eval { + for { + _ <- writeToProcess(stdin, proc, blocker) + term <- Sync[F].delay(proc.waitFor(cmd.timeout.seconds, TimeUnit.SECONDS)) + _ <- if (term) logger.fdebug(s"Command `${cmd.cmdString}` finished: ${proc.exitValue}") + else logger.fwarn(s"Command `${cmd.cmdString}` did not finish in ${cmd.timeout.formatExact}!") + _ <- if (!term) timeoutError(proc, cmd) else Sync[F].pure(()) + out <- if (term) inputStreamToString(proc.getInputStream, blocker) else Sync[F].pure("") + err <- if (term) inputStreamToString(proc.getErrorStream, blocker) else Sync[F].pure("") + } yield Result(proc.exitValue, out, err) + } + } + + def execSuccess[F[_]: Sync: ContextShift](cmd: Config.Command, blocker: Blocker, wd: Option[Path] = None, stdin: Stream[F, Byte] = Stream.empty): Stream[F, Result] = + exec(cmd, blocker, wd, stdin).flatMap { r => + if (r.rc != 0) Stream.raiseError[F](new Exception(s"Command `${cmd.cmdString}` returned non-zero exit code ${r.rc}. Stderr: ${r.stderr}")) + else Stream.emit(r) + } + + private def startProcess[F[_]: Sync,A](cmd: Config.Command, wd: Option[Path])(f: Process => Stream[F,A]): Stream[F, A] = { + val log = logger.fdebug(s"Running external command: ${cmd.cmdString}") + val proc = log *> Sync[F].delay { + val pb = new ProcessBuilder(cmd.toCmd.asJava) + wd.map(_.toFile).foreach(pb.directory) + pb.start() + } + Stream.bracket(proc)(p => logger.fdebug(s"Closing process: `${cmd.cmdString}`").map { _ => + p.destroy() + }).flatMap(f) + } + + private def inputStreamToString[F[_]: Sync: ContextShift](in: InputStream, blocker: Blocker): F[String] = + io.readInputStream(Sync[F].pure(in), 16 * 1024, blocker, closeAfterUse = false). + through(text.utf8Decode). + chunks. + map(_.toVector.mkString). + fold1(_ + _). + compile.last. + map(_.getOrElse("")) + + private def writeToProcess[F[_]: Sync: ContextShift](data: Stream[F, Byte], proc: Process, blocker: Blocker): F[Unit] = + data.through(io.writeOutputStream(Sync[F].delay(proc.getOutputStream), blocker)). + compile.drain + + private def timeoutError[F[_]: Sync](proc: Process, cmd: Config.Command): F[Unit] = + Sync[F].delay(proc.destroyForcibly()).attempt *> { + Sync[F].raiseError(new Exception(s"Command `${cmd.cmdString}` timed out (${cmd.timeout.formatExact})")) + } +} diff --git a/modules/text/src/main/scala/docspell/text/ocr/TextExtract.scala b/modules/text/src/main/scala/docspell/text/ocr/TextExtract.scala new file mode 100644 index 00000000..b25de191 --- /dev/null +++ b/modules/text/src/main/scala/docspell/text/ocr/TextExtract.scala @@ -0,0 +1,30 @@ +package docspell.text.ocr + +import cats.effect.{Blocker, ContextShift, Sync} +import docspell.common.MimeType +import fs2.Stream + +object TextExtract { + + def extract[F[_]: Sync: ContextShift](in: Stream[F, Byte], blocker: Blocker, lang: String, config: Config): Stream[F, String] = + extractOCR(in, blocker, lang, config) + + def extractOCR[F[_]: Sync: ContextShift](in: Stream[F, Byte], blocker: Blocker, lang: String, config: Config): Stream[F, String] = + Stream.eval(TikaMimetype.detect(in)). + flatMap({ + case mt if !config.isAllowed(mt) => + raiseError(s"File `$mt` not allowed") + + case MimeType.pdf => + Ocr.extractPdf(in, blocker, lang, config) + + case mt if mt.primary == "image" => + Ocr.extractImage(in, blocker, lang, config) + + case mt => + raiseError(s"File `$mt` not supported") + }) + + private def raiseError[F[_]: Sync](msg: String): Stream[F, Nothing] = + Stream.raiseError[F](new Exception(msg)) +} diff --git a/modules/text/src/main/scala/docspell/text/ocr/TikaMimetype.scala b/modules/text/src/main/scala/docspell/text/ocr/TikaMimetype.scala new file mode 100644 index 00000000..faa987ed --- /dev/null +++ b/modules/text/src/main/scala/docspell/text/ocr/TikaMimetype.scala @@ -0,0 +1,45 @@ +package docspell.text.ocr + +import cats.implicits._ +import cats.effect.Sync +import docspell.common.MimeType +import fs2.Stream +import org.apache.tika.config.TikaConfig +import org.apache.tika.metadata.{HttpHeaders, Metadata, TikaMetadataKeys} +import org.apache.tika.mime.MediaType + +object TikaMimetype { + private val tika = new TikaConfig().getDetector + + private def convert(mt: MediaType): MimeType = + Option(mt).map(_.toString). + map(MimeType.parse). + flatMap(_.toOption). + map(normalize). + getOrElse(MimeType.octetStream) + + private def makeMetadata(hint: MimeTypeHint): Metadata = { + val md = new Metadata + hint.filename. + foreach(md.set(TikaMetadataKeys.RESOURCE_NAME_KEY, _)) + hint.advertised. + foreach(md.set(HttpHeaders.CONTENT_TYPE, _)) + md + } + + private def normalize(in: MimeType): MimeType = in match { + case MimeType(_, sub) if sub contains "xhtml" => + MimeType.html + case _ => in + } + + private def fromBytes(bv: Array[Byte], hint: MimeTypeHint): MimeType = { + convert(tika.detect(new java.io.ByteArrayInputStream(bv), makeMetadata(hint))) + } + + def detect[F[_]: Sync](data: Stream[F, Byte]): F[MimeType] = + data.take(1024). + compile.toVector. + map(bytes => fromBytes(bytes.toArray, MimeTypeHint.none)) + +} diff --git a/modules/text/src/main/scala/docspell/text/split/TextSplitter.scala b/modules/text/src/main/scala/docspell/text/split/TextSplitter.scala new file mode 100644 index 00000000..cd8918bb --- /dev/null +++ b/modules/text/src/main/scala/docspell/text/split/TextSplitter.scala @@ -0,0 +1,30 @@ +package docspell.text.split + +import fs2.Stream + +/** Splits text into words. + * + */ +object TextSplitter { + private[this] val trimChars = + ".,…_[]^!<>=&ſ/{}*?()-:#$|~`+%\\\"'; \t\r\n".toSet + + def split[F[_]](str: String, sep: Set[Char], start: Int = 0): Stream[F, Word] = { + val indexes = sep.map(c => str.indexOf(c.toInt)).filter(_ >= 0) + val index = if (indexes.isEmpty) - 1 else indexes.min + + if (index < 0) Stream.emit(Word(str, start, start + str.length)) + else if (index == 0) split(str.substring(1), sep, start + 1) + else Stream.emit(Word(str.substring(0, index), start, start + index)) ++ + Stream.suspend(split(str.substring(index + 1), sep, start + index + 1)) + } + + + def splitToken[F[_]](str: String, sep: Set[Char], start: Int = 0): Stream[F, Word] = { + split(str, sep, start). + map(w => w.trim(trimChars)). + filter(_.nonEmpty). + map(_.toLower) + } + +} diff --git a/modules/text/src/main/scala/docspell/text/split/Word.scala b/modules/text/src/main/scala/docspell/text/split/Word.scala new file mode 100644 index 00000000..15587f7d --- /dev/null +++ b/modules/text/src/main/scala/docspell/text/split/Word.scala @@ -0,0 +1,32 @@ +package docspell.text.split + +case class Word(value: String, begin: Int, end: Int) { + def isEmpty: Boolean = value.isEmpty + def nonEmpty: Boolean = !isEmpty + def length : Int = value.length + + def trimLeft(chars: Set[Char]): Word = { + val v = value.dropWhile(chars.contains) + if (v == value) this + else Word(v, begin + length - v.length, end) + } + + def trimRight(chars: Set[Char]): Word = { + @annotation.tailrec + def findIndex(n: Int = length - 1): Int = + if (n < 0 || !chars.contains(value.charAt(n))) n + else findIndex(n - 1) + + val index = findIndex() + if (index == length - 1) this + else if (index < 0) Word("", begin, begin + 1) + else Word(value.substring(0, index + 1), begin, end - index) + } + + def trim(chars: Set[Char]): Word = + trimLeft(chars).trimRight(chars) + + def toLower: Word = + copy(value = value.toLowerCase) + +} diff --git a/modules/text/src/test/resources/letter-de-source.pdf b/modules/text/src/test/resources/letter-de-source.pdf new file mode 100644 index 00000000..d724839a Binary files /dev/null and b/modules/text/src/test/resources/letter-de-source.pdf differ diff --git a/modules/text/src/test/resources/letter-en-source.pdf b/modules/text/src/test/resources/letter-en-source.pdf new file mode 100644 index 00000000..b11fff43 Binary files /dev/null and b/modules/text/src/test/resources/letter-en-source.pdf differ diff --git a/modules/text/src/test/resources/logback.xml b/modules/text/src/test/resources/logback.xml new file mode 100644 index 00000000..5b0b6a44 --- /dev/null +++ b/modules/text/src/test/resources/logback.xml @@ -0,0 +1,14 @@ + + + true + + + [%thread] %highlight(%-5level) %cyan(%logger{15}) - %msg %n + + + + + + + + diff --git a/modules/text/src/test/scala/docspell/text/TestFiles.scala b/modules/text/src/test/scala/docspell/text/TestFiles.scala new file mode 100644 index 00000000..90b426f4 --- /dev/null +++ b/modules/text/src/test/scala/docspell/text/TestFiles.scala @@ -0,0 +1,94 @@ +package docspell.text + +import cats.effect.{Blocker, IO} +import docspell.common.LenientUri +import fs2.Stream + +import scala.concurrent.ExecutionContext + +object TestFiles { + val blocker = Blocker.liftExecutionContext(ExecutionContext.global) + implicit val CS = IO.contextShift(ExecutionContext.global) + + + val letterSourceDE: Stream[IO, Byte] = + LenientUri.fromJava(getClass.getResource("/letter-de-source.pdf")). + readURL[IO](16 * 1024, blocker) + + val letterSourceEN: Stream[IO, Byte] = + LenientUri.fromJava(getClass.getResource("/letter-en-source.pdf")). + readURL[IO](16 * 1024, blocker) + + + val letterDEText = """Max Mustermann + | + |Lilienweg 21 + | + |12345 Nebendorf + | + |E-Mail: max.muster@gmail.com + | + |Max Mustermann, Lilienweg 21, 12345 Nebendorf + | + |EasyCare AG + |Abteilung Buchhaltung + |Ackerweg 12 + | + |12346 Ulmen + | + |Nebendorf, 3. September 2019 + |Sehr geehrte Damen und Herren, + | + |hiermit kündige ich meine Mitgliedschaft in der Kranken- und Pflegeversicherung zum + |nächstmöglichen Termin. + | + |Bitte senden Sie mir innerhalb der gesetzlichen Frist von 14 Tagen eine Kündigungsbe- + |stätigung zu. + | + |Vielen Dank im Vorraus! + | + |Mit freundlichen Grüßen + | + |Max Mustermann + |""".stripMargin.trim + + val letterENText = """Derek Jeter + | + |123 Elm Ave. + | + |Treesville, ON MI1N 2P3 + |November 7, 2016 + | + |Derek Jeter, 123 Elm Ave., Treesville, ON M1N 2P3, November 7, 2016 + | + |Mr. M. Leat + | + |Chief of Syrup Production + |Old Sticky Pancake Company + |456 Maple Lane + | + |Forest, ON 7TW8 9Y0 + | + |Hemptown, September 3, 2019 + |Dear Mr. Leaf, + | + |Let me begin by thanking you for your past contributions to our Little League baseball + |team. Your sponsorship aided in the purchase of ten full uniforms and several pieces of + |baseball equipment for last year’s season. + | + |Next month, our company is planning an employee appreciation pancake breakfast hon- + |oring retired employees for their past years of service and present employees for their + |loyalty and dedication in spite of the current difficult economic conditions. + | + |We would like to place an order with your company for 25 pounds of pancake mix and + |five gallons of maple syrup. We hope you will be able to provide these products in the + |bulk quantities we require. + | + |As you are a committed corporate sponsor and long-time associate, we hope that you + |will be able to join us for breakfast on December 12, 2016. + | + |Respectfully yours, + | + |Derek Jeter + |""".stripMargin.trim +} diff --git a/modules/text/src/test/scala/docspell/text/contact/ContactAnnotateSpec.scala b/modules/text/src/test/scala/docspell/text/contact/ContactAnnotateSpec.scala new file mode 100644 index 00000000..fbb1c5a1 --- /dev/null +++ b/modules/text/src/test/scala/docspell/text/contact/ContactAnnotateSpec.scala @@ -0,0 +1,32 @@ +package docspell.text.contact + +import docspell.common.{NerLabel, NerTag} +import minitest.SimpleTestSuite + +object ContactAnnotateSpec extends SimpleTestSuite { + + test("find email") { + + val text = + """An email address such as John.Smith@example.com is made up + |of a local-part, an @ symbol, then a case-insensitive domain. + |Although the standard requires[1] the local part to be + |case-sensitive, it also urges that receiving hosts deliver + |messages in a case-independent fashion,[2] e.g., that the mail + |system at example.com treat John.Smith as equivalent to + |john.smith; some mail systems even treat them as equivalent + |to johnsmith.[3] Mail systems often limit their users' choice + |of name to a subset of the technically valid characters, and + |in some cases also limit which addresses it is possible to + |send mail to.""".stripMargin + + val labels = Contact.annotate(text) + assertEquals(labels.size, 2) + assertEquals(labels(0), + NerLabel("john.smith@example.com", NerTag.Email, 25, 47)) + assertEquals(text.substring(25, 47).toLowerCase, "john.smith@example.com") + assertEquals(labels(1), + NerLabel("example.com", NerTag.Website, 308, 319)) + assertEquals(text.substring(308, 319).toLowerCase, "example.com") + } +} diff --git a/modules/text/src/test/scala/docspell/text/date/DateFindSpec.scala b/modules/text/src/test/scala/docspell/text/date/DateFindSpec.scala new file mode 100644 index 00000000..a974fa22 --- /dev/null +++ b/modules/text/src/test/scala/docspell/text/date/DateFindSpec.scala @@ -0,0 +1,14 @@ +package docspell.text.date + +import docspell.common.Language +import docspell.text.TestFiles +import minitest._ + +object DateFindSpec extends SimpleTestSuite { + + test("find simple dates") { + + //println(DateFind.findDates(TestFiles.letterDEText, Language.German).toVector) + println(DateFind.findDates(TestFiles.letterENText, Language.English).toVector) + } +} diff --git a/modules/text/src/test/scala/docspell/text/nlp/TextAnalyserSuite.scala b/modules/text/src/test/scala/docspell/text/nlp/TextAnalyserSuite.scala new file mode 100644 index 00000000..b73471e7 --- /dev/null +++ b/modules/text/src/test/scala/docspell/text/nlp/TextAnalyserSuite.scala @@ -0,0 +1,52 @@ +package docspell.text.nlp + +import docspell.common.{Language, NerLabel, NerTag} +import docspell.text.TestFiles +import minitest.SimpleTestSuite + +object TextAnalyserSuite extends SimpleTestSuite { + + test("find english ner labels") { + val labels = StanfordNerClassifier.nerAnnotate(Language.English)(TestFiles.letterENText) + val expect = Vector(NerLabel("Derek",NerTag.Person,0,5) + , NerLabel("Jeter",NerTag.Person,6,11) + , NerLabel("Treesville",NerTag.Person,27,37) + , NerLabel("Derek",NerTag.Person,69,74) + , NerLabel("Jeter",NerTag.Person,75,80) + , NerLabel("Treesville",NerTag.Location,96,106) + , NerLabel("M.",NerTag.Person,142,144) + , NerLabel("Leat",NerTag.Person,145,149) + , NerLabel("Syrup",NerTag.Organization,160,165) + , NerLabel("Production",NerTag.Organization,166,176) + , NerLabel("Old",NerTag.Organization,177,180) + , NerLabel("Sticky",NerTag.Organization,181,187) + , NerLabel("Pancake",NerTag.Organization,188,195) + , NerLabel("Company",NerTag.Organization,196,203) + , NerLabel("Maple",NerTag.Location,208,213) + , NerLabel("Lane",NerTag.Location,214,218) + , NerLabel("Forest",NerTag.Location,220,226) + , NerLabel("Hemptown",NerTag.Location,241,249) + , NerLabel("Little",NerTag.Organization,349,355) + , NerLabel("League",NerTag.Organization,356,362) + , NerLabel("Derek",NerTag.Person,1119,1124) + , NerLabel("Jeter",NerTag.Person,1125,1130)) + assertEquals(labels, expect) + } + + test("find german ner labels") { + val labels = StanfordNerClassifier.nerAnnotate(Language.German)(TestFiles.letterDEText) + val expect = Vector(NerLabel("Max", NerTag.Person, 0, 3) + , NerLabel("Mustermann", NerTag.Person, 4, 14) + , NerLabel("Lilienweg", NerTag.Location, 16, 25) + , NerLabel("Max", NerTag.Person, 77, 80) + , NerLabel("Mustermann", NerTag.Person, 81, 91) + , NerLabel("Lilienweg", NerTag.Location, 93, 102) + , NerLabel("EasyCare", NerTag.Organization, 124, 132) + , NerLabel("AG", NerTag.Organization, 133, 135) + , NerLabel("Ackerweg", NerTag.Location, 158, 166) + , NerLabel("Nebendorf", NerTag.Location, 184, 193) + , NerLabel("Max", NerTag.Person, 505, 508) + , NerLabel("Mustermann", NerTag.Person, 509, 519)) + assertEquals(labels, expect) + } +} diff --git a/modules/text/src/test/scala/docspell/text/ocr/TextExtractionSuite.scala b/modules/text/src/test/scala/docspell/text/ocr/TextExtractionSuite.scala new file mode 100644 index 00000000..40b46a77 --- /dev/null +++ b/modules/text/src/test/scala/docspell/text/ocr/TextExtractionSuite.scala @@ -0,0 +1,25 @@ +package docspell.text.ocr + +import cats.effect.IO +import docspell.text.TestFiles +import minitest.SimpleTestSuite + +object TextExtractionSuite extends SimpleTestSuite { + import TestFiles._ + + test("extract english pdf") { + ignore() + val text = TextExtract.extract[IO](letterSourceEN, blocker, "eng", Config.default). + compile.lastOrError.unsafeRunSync() + println(text) + } + + test("extract german pdf") { + ignore() + val expect = TestFiles.letterDEText + val extract = TextExtract.extract[IO](letterSourceDE, blocker, "deu", Config.default). + compile.lastOrError.unsafeRunSync() + + assertEquals(extract.trim, expect.trim) + } +} diff --git a/modules/text/src/test/scala/docspell/text/split/TestSplitterSpec.scala b/modules/text/src/test/scala/docspell/text/split/TestSplitterSpec.scala new file mode 100644 index 00000000..004d329c --- /dev/null +++ b/modules/text/src/test/scala/docspell/text/split/TestSplitterSpec.scala @@ -0,0 +1,24 @@ +package docspell.text.split + +import minitest._ + +object TestSplitterSpec extends SimpleTestSuite { + + test("simple splitting") { + val text = """hiermit kündige ich meine Mitgliedschaft in der Kranken- und + |Pflegeversicherung zum nächstmöglichen Termin. + | + |Bitte senden Sie mir innerhalb der gesetzlichen Frist von 14 Tagen + |eine Kündigungsbestätigung zu. + | + |Vielen Dank im Vorraus!""".stripMargin + + val words = TextSplitter.splitToken(text, " \t\r\n".toSet).toVector + + + assertEquals(words.size, 31) + assertEquals(words(13), Word("bitte", 109, 114)) + assertEquals(text.substring(109, 114).toLowerCase, "bitte") + } + +} diff --git a/modules/webapp/src/main/elm/Api.elm b/modules/webapp/src/main/elm/Api.elm index 1eac72ce..caa3e60e 100644 --- a/modules/webapp/src/main/elm/Api.elm +++ b/modules/webapp/src/main/elm/Api.elm @@ -3,11 +3,99 @@ module Api exposing (..) import Http import Task import Util.Http as Http2 +import Util.File +import Json.Encode as JsonEncode +import File exposing (File) import Data.Flags exposing (Flags) -import Api.Model.UserPass exposing (UserPass) import Api.Model.AuthResult exposing (AuthResult) +import Api.Model.BasicResult exposing (BasicResult) +import Api.Model.Collective exposing (Collective) +import Api.Model.CollectiveSettings exposing (CollectiveSettings) +import Api.Model.DirectionValue exposing (DirectionValue) +import Api.Model.Equipment exposing (Equipment) +import Api.Model.EquipmentList exposing (EquipmentList) +import Api.Model.GenInvite exposing (GenInvite) +import Api.Model.InviteResult exposing (InviteResult) +import Api.Model.ItemDetail exposing (ItemDetail) +import Api.Model.ItemInsights exposing (ItemInsights) +import Api.Model.ItemLightList exposing (ItemLightList) +import Api.Model.ItemProposals exposing (ItemProposals) +import Api.Model.ItemSearch exposing (ItemSearch) +import Api.Model.ItemUploadMeta exposing (ItemUploadMeta) +import Api.Model.JobQueueState exposing (JobQueueState) +import Api.Model.OptionalDate exposing (OptionalDate) +import Api.Model.OptionalId exposing (OptionalId) +import Api.Model.OptionalText exposing (OptionalText) +import Api.Model.Organization exposing (Organization) +import Api.Model.OrganizationList exposing (OrganizationList) +import Api.Model.PasswordChange exposing (PasswordChange) +import Api.Model.Person exposing (Person) +import Api.Model.PersonList exposing (PersonList) +import Api.Model.ReferenceList exposing (ReferenceList) +import Api.Model.Registration exposing (Registration) +import Api.Model.Source exposing (Source) +import Api.Model.SourceList exposing (SourceList) +import Api.Model.Tag exposing (Tag) +import Api.Model.TagList exposing (TagList) +import Api.Model.User exposing (User) +import Api.Model.UserList exposing (UserList) +import Api.Model.UserPass exposing (UserPass) import Api.Model.VersionInfo exposing (VersionInfo) +upload: Flags -> Maybe String -> ItemUploadMeta -> List File -> (String -> (Result Http.Error BasicResult) -> msg) -> List (Cmd msg) +upload flags sourceId meta files receive = + let + metaStr = JsonEncode.encode 0 (Api.Model.ItemUploadMeta.encode meta) + mkReq file = + let + fid = Util.File.makeFileId file + path = Maybe.map ((++) "/api/v1/open/upload/item/") sourceId + |> Maybe.withDefault "/api/v1/sec/upload/item" + in + Http2.authPostTrack + { url = flags.config.baseUrl ++ path + , account = getAccount flags + , body = Http.multipartBody <| + [Http.stringPart "meta" metaStr, Http.filePart "file[]" file] + , expect = Http.expectJson (receive fid) Api.Model.BasicResult.decoder + , tracker = fid + } + in + List.map mkReq files + +uploadSingle: Flags -> Maybe String -> ItemUploadMeta -> String -> List File -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +uploadSingle flags sourceId meta track files receive = + let + metaStr = JsonEncode.encode 0 (Api.Model.ItemUploadMeta.encode meta) + fileParts = List.map (\f -> Http.filePart "file[]" f) files + allParts = (Http.stringPart "meta" metaStr) :: fileParts + path = Maybe.map ((++) "/api/v1/open/upload/item/") sourceId + |> Maybe.withDefault "/api/v1/sec/upload/item" + in + Http2.authPostTrack + { url = flags.config.baseUrl ++ path + , account = getAccount flags + , body = Http.multipartBody allParts + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + , tracker = track + } + +register: Flags -> Registration -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +register flags reg receive = + Http.post + { url = flags.config.baseUrl ++ "/api/v1/open/signup/register" + , body = Http.jsonBody (Api.Model.Registration.encode reg) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +newInvite: Flags -> GenInvite -> ((Result Http.Error InviteResult) -> msg) -> Cmd msg +newInvite flags req receive = + Http.post + { url = flags.config.baseUrl ++ "/api/v1/open/signup/newinvite" + , body = Http.jsonBody (Api.Model.GenInvite.encode req) + , expect = Http.expectJson receive Api.Model.InviteResult.decoder + } + login: Flags -> UserPass -> ((Result Http.Error AuthResult) -> msg) -> Cmd msg login flags up receive = Http.post @@ -48,7 +136,7 @@ refreshSession flags receive = if acc.success && acc.validMs > 30000 then let - delay = acc.validMs - 30000 |> toFloat + delay = Debug.log "Refresh session in " (acc.validMs - 30000) |> toFloat in Http2.executeIn delay receive (refreshSessionTask flags) else Cmd.none @@ -67,6 +155,452 @@ refreshSessionTask flags = , timeout = Nothing } + +getInsights: Flags -> ((Result Http.Error ItemInsights) -> msg) -> Cmd msg +getInsights flags receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/collective/insights" + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.ItemInsights.decoder + } + +getCollective: Flags -> ((Result Http.Error Collective) -> msg) -> Cmd msg +getCollective flags receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/collective" + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.Collective.decoder + } + +getCollectiveSettings: Flags -> ((Result Http.Error CollectiveSettings) -> msg) -> Cmd msg +getCollectiveSettings flags receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/collective/settings" + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.CollectiveSettings.decoder + } + +setCollectiveSettings: Flags -> CollectiveSettings -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +setCollectiveSettings flags settings receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/collective/settings" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.CollectiveSettings.encode settings) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +-- Tags + +getTags: Flags -> ((Result Http.Error TagList) -> msg) -> Cmd msg +getTags flags receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/tag" + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.TagList.decoder + } + +postTag: Flags -> Tag -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +postTag flags tag receive = + let + params = + { url = flags.config.baseUrl ++ "/api/v1/sec/tag" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.Tag.encode tag) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + in + if tag.id == "" then Http2.authPost params + else Http2.authPut params + +deleteTag: Flags -> String -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +deleteTag flags tag receive = + Http2.authDelete + { url = flags.config.baseUrl ++ "/api/v1/sec/tag/" ++ tag + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +-- Equipments + +getEquipments: Flags -> ((Result Http.Error EquipmentList) -> msg) -> Cmd msg +getEquipments flags receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/equipment" + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.EquipmentList.decoder + } + +postEquipment: Flags -> Equipment -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +postEquipment flags equip receive = + let + params = + { url = flags.config.baseUrl ++ "/api/v1/sec/equipment" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.Equipment.encode equip) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + in + if equip.id == "" then Http2.authPost params + else Http2.authPut params + +deleteEquip: Flags -> String -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +deleteEquip flags equip receive = + Http2.authDelete + { url = flags.config.baseUrl ++ "/api/v1/sec/equipment/" ++ equip + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + + +-- Organization + +getOrgLight: Flags -> ((Result Http.Error ReferenceList) -> msg) -> Cmd msg +getOrgLight flags receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/organization" + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.ReferenceList.decoder + } + +getOrganizations: Flags -> ((Result Http.Error OrganizationList) -> msg) -> Cmd msg +getOrganizations flags receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/organization?full=true" + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.OrganizationList.decoder + } + +postOrg: Flags -> Organization -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +postOrg flags org receive = + let + params = + { url = flags.config.baseUrl ++ "/api/v1/sec/organization" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.Organization.encode org) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + in + if org.id == "" then Http2.authPost params + else Http2.authPut params + +deleteOrg: Flags -> String -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +deleteOrg flags org receive = + Http2.authDelete + { url = flags.config.baseUrl ++ "/api/v1/sec/organization/" ++ org + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + + +-- Person + + +getPersonsLight: Flags -> ((Result Http.Error ReferenceList) -> msg) -> Cmd msg +getPersonsLight flags receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/person?full=false" + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.ReferenceList.decoder + } + +getPersons: Flags -> ((Result Http.Error PersonList) -> msg) -> Cmd msg +getPersons flags receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/person?full=true" + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.PersonList.decoder + } + +postPerson: Flags -> Person -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +postPerson flags person receive = + let + params = + { url = flags.config.baseUrl ++ "/api/v1/sec/person" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.Person.encode person) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + in + if person.id == "" then Http2.authPost params + else Http2.authPut params + +deletePerson: Flags -> String -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +deletePerson flags person receive = + Http2.authDelete + { url = flags.config.baseUrl ++ "/api/v1/sec/person/" ++ person + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +--- Sources + +getSources: Flags -> ((Result Http.Error SourceList) -> msg) -> Cmd msg +getSources flags receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/source" + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.SourceList.decoder + } + +postSource: Flags -> Source -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +postSource flags source receive = + let + params = + { url = flags.config.baseUrl ++ "/api/v1/sec/source" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.Source.encode source) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + in + if source.id == "" then Http2.authPost params + else Http2.authPut params + +deleteSource: Flags -> String -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +deleteSource flags src receive = + Http2.authDelete + { url = flags.config.baseUrl ++ "/api/v1/sec/source/" ++ src + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +-- Users + +getUsers: Flags -> ((Result Http.Error UserList) -> msg) -> Cmd msg +getUsers flags receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/user" + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.UserList.decoder + } + +postNewUser: Flags -> User -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +postNewUser flags user receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/user" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.User.encode user) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +putUser: Flags -> User -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +putUser flags user receive = + Http2.authPut + { url = flags.config.baseUrl ++ "/api/v1/sec/user" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.User.encode user) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +changePassword: Flags -> PasswordChange -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +changePassword flags cp receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/user/changePassword" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.PasswordChange.encode cp) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +deleteUser: Flags -> String -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +deleteUser flags user receive = + Http2.authDelete + { url = flags.config.baseUrl ++ "/api/v1/sec/user/" ++ user + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + + +-- Job Queue + +cancelJob: Flags -> String -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +cancelJob flags jobid receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/queue/" ++ jobid ++ "/cancel" + , account = getAccount flags + , body = Http.emptyBody + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +getJobQueueState: Flags -> ((Result Http.Error JobQueueState) -> msg) -> Cmd msg +getJobQueueState flags receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/queue/state" + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.JobQueueState.decoder + } + + +getJobQueueStateIn: Flags -> Float -> ((Result Http.Error JobQueueState) -> msg) -> Cmd msg +getJobQueueStateIn flags delay receive = + case flags.account of + Just acc -> + if acc.success && delay > 100 + then + let + _ = Debug.log "Refresh job qeue state in " delay + in + Http2.executeIn delay receive (getJobQueueStateTask flags) + else Cmd.none + Nothing -> + Cmd.none + +getJobQueueStateTask: Flags -> Task.Task Http.Error JobQueueState +getJobQueueStateTask flags = + Http2.authTask + { url = flags.config.baseUrl ++ "/api/v1/sec/queue/state" + , method = "GET" + , headers = [] + , account = getAccount flags + , body = Http.emptyBody + , resolver = Http2.jsonResolver Api.Model.JobQueueState.decoder + , timeout = Nothing + } + +-- Item + +itemSearch: Flags -> ItemSearch -> ((Result Http.Error ItemLightList) -> msg) -> Cmd msg +itemSearch flags search receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/item/search" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.ItemSearch.encode search) + , expect = Http.expectJson receive Api.Model.ItemLightList.decoder + } + +itemDetail: Flags -> String -> ((Result Http.Error ItemDetail) -> msg) -> Cmd msg +itemDetail flags id receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ id + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.ItemDetail.decoder + } + +setTags: Flags -> String -> ReferenceList -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +setTags flags item tags receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ item ++ "/tags" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.ReferenceList.encode tags) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +setDirection: Flags -> String -> DirectionValue -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +setDirection flags item dir receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ item ++ "/direction" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.DirectionValue.encode dir) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +setCorrOrg: Flags -> String -> OptionalId -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +setCorrOrg flags item id receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ item ++ "/corrOrg" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.OptionalId.encode id) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +setCorrPerson: Flags -> String -> OptionalId -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +setCorrPerson flags item id receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ item ++ "/corrPerson" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.OptionalId.encode id) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +setConcPerson: Flags -> String -> OptionalId -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +setConcPerson flags item id receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ item ++ "/concPerson" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.OptionalId.encode id) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +setConcEquip: Flags -> String -> OptionalId -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +setConcEquip flags item id receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ item ++ "/concEquipment" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.OptionalId.encode id) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +setItemName: Flags -> String -> OptionalText -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +setItemName flags item text receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ item ++ "/name" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.OptionalText.encode text) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +setItemNotes: Flags -> String -> OptionalText -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +setItemNotes flags item text receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ item ++ "/notes" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.OptionalText.encode text) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +setItemDate: Flags -> String -> OptionalDate -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +setItemDate flags item date receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ item ++ "/date" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.OptionalDate.encode date) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +setItemDueDate: Flags -> String -> OptionalDate -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +setItemDueDate flags item date receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ item ++ "/duedate" + , account = getAccount flags + , body = Http.jsonBody (Api.Model.OptionalDate.encode date) + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +setConfirmed: Flags -> String -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +setConfirmed flags item receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ item ++ "/confirm" + , account = getAccount flags + , body = Http.emptyBody + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +setUnconfirmed: Flags -> String -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +setUnconfirmed flags item receive = + Http2.authPost + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ item ++ "/unconfirm" + , account = getAccount flags + , body = Http.emptyBody + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +deleteItem: Flags -> String -> ((Result Http.Error BasicResult) -> msg) -> Cmd msg +deleteItem flags item receive = + Http2.authDelete + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ item + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.BasicResult.decoder + } + +getItemProposals: Flags -> String -> ((Result Http.Error ItemProposals) -> msg) -> Cmd msg +getItemProposals flags item receive = + Http2.authGet + { url = flags.config.baseUrl ++ "/api/v1/sec/item/" ++ item ++ "/proposals" + , account = getAccount flags + , expect = Http.expectJson receive Api.Model.ItemProposals.decoder + } + + +-- Helper + getAccount: Flags -> AuthResult getAccount flags = Maybe.withDefault Api.Model.AuthResult.empty flags.account diff --git a/modules/webapp/src/main/elm/App/Data.elm b/modules/webapp/src/main/elm/App/Data.elm index c3dc99d3..24bffa02 100644 --- a/modules/webapp/src/main/elm/App/Data.elm +++ b/modules/webapp/src/main/elm/App/Data.elm @@ -10,6 +10,13 @@ import Api.Model.AuthResult exposing (AuthResult) import Page exposing (Page(..)) import Page.Home.Data import Page.Login.Data +import Page.ManageData.Data +import Page.CollectiveSettings.Data +import Page.UserSettings.Data +import Page.Queue.Data +import Page.Register.Data +import Page.Upload.Data +import Page.NewInvite.Data type alias Model = { flags: Flags @@ -18,19 +25,38 @@ type alias Model = , version: VersionInfo , homeModel: Page.Home.Data.Model , loginModel: Page.Login.Data.Model + , manageDataModel: Page.ManageData.Data.Model + , collSettingsModel: Page.CollectiveSettings.Data.Model + , userSettingsModel: Page.UserSettings.Data.Model + , queueModel: Page.Queue.Data.Model + , registerModel: Page.Register.Data.Model + , uploadModel: Page.Upload.Data.Model + , newInviteModel: Page.NewInvite.Data.Model + , navMenuOpen: Bool + , subs: Sub Msg } init: Key -> Url -> Flags -> Model init key url flags = let - page = Page.fromUrl url |> Maybe.withDefault HomePage + page = Page.fromUrl url + |> Maybe.withDefault (defaultPage flags) in { flags = flags , key = key , page = page , version = Api.Model.VersionInfo.empty , homeModel = Page.Home.Data.emptyModel - , loginModel = Page.Login.Data.empty + , loginModel = Page.Login.Data.emptyModel + , manageDataModel = Page.ManageData.Data.emptyModel + , collSettingsModel = Page.CollectiveSettings.Data.emptyModel + , userSettingsModel = Page.UserSettings.Data.emptyModel + , queueModel = Page.Queue.Data.emptyModel + , registerModel = Page.Register.Data.emptyModel + , uploadModel = Page.Upload.Data.emptyModel + , newInviteModel = Page.NewInvite.Data.emptyModel + , navMenuOpen = False + , subs = Sub.none } type Msg @@ -39,7 +65,30 @@ type Msg | VersionResp (Result Http.Error VersionInfo) | HomeMsg Page.Home.Data.Msg | LoginMsg Page.Login.Data.Msg + | ManageDataMsg Page.ManageData.Data.Msg + | CollSettingsMsg Page.CollectiveSettings.Data.Msg + | UserSettingsMsg Page.UserSettings.Data.Msg + | QueueMsg Page.Queue.Data.Msg + | RegisterMsg Page.Register.Data.Msg + | UploadMsg Page.Upload.Data.Msg + | NewInviteMsg Page.NewInvite.Data.Msg | Logout | LogoutResp (Result Http.Error ()) | SessionCheckResp (Result Http.Error AuthResult) - | SetPage Page + | ToggleNavMenu + +isSignedIn: Flags -> Bool +isSignedIn flags = + flags.account + |> Maybe.map .success + |> Maybe.withDefault False + +checkPage: Flags -> Page -> Page +checkPage flags page = + if Page.isSecured page && isSignedIn flags then page + else if Page.isOpen page then page + else Page.loginPage page + +defaultPage: Flags -> Page +defaultPage flags = + if isSignedIn flags then HomePage else (LoginPage Nothing) diff --git a/modules/webapp/src/main/elm/App/Update.elm b/modules/webapp/src/main/elm/App/Update.elm index 094b71d0..b7a3249f 100644 --- a/modules/webapp/src/main/elm/App/Update.elm +++ b/modules/webapp/src/main/elm/App/Update.elm @@ -12,46 +12,94 @@ import Page.Home.Data import Page.Home.Update import Page.Login.Data import Page.Login.Update +import Page.ManageData.Data +import Page.ManageData.Update +import Page.CollectiveSettings.Data +import Page.CollectiveSettings.Update +import Page.UserSettings.Data +import Page.UserSettings.Update +import Page.Queue.Data +import Page.Queue.Update +import Page.Register.Data +import Page.Register.Update +import Page.Upload.Data +import Page.Upload.Update +import Page.NewInvite.Data +import Page.NewInvite.Update +import Util.Update update: Msg -> Model -> (Model, Cmd Msg) update msg model = + let + (m, c, s) = updateWithSub msg model + in + ({m|subs = s}, c) + +updateWithSub: Msg -> Model -> (Model, Cmd Msg, Sub Msg) +updateWithSub msg model = case msg of HomeMsg lm -> - updateHome lm model + updateHome lm model |> noSub LoginMsg lm -> - updateLogin lm model + updateLogin lm model |> noSub - SetPage p -> - ( {model | page = p } - , Cmd.none - ) + ManageDataMsg lm -> + updateManageData lm model |> noSub + + CollSettingsMsg m -> + updateCollSettings m model |> noSub + + UserSettingsMsg m -> + updateUserSettings m model |> noSub + + QueueMsg m -> + updateQueue m model |> noSub + + RegisterMsg m -> + updateRegister m model |> noSub + + UploadMsg m -> + updateUpload m model + + NewInviteMsg m -> + updateNewInvite m model |> noSub VersionResp (Ok info) -> - ({model|version = info}, Cmd.none) + ({model|version = info}, Cmd.none) |> noSub VersionResp (Err err) -> - (model, Cmd.none) + (model, Cmd.none, Sub.none) Logout -> - (model, Api.logout model.flags LogoutResp) + (model + , Cmd.batch + [ Api.logout model.flags LogoutResp + , Ports.removeAccount () + ] + , Sub.none) + LogoutResp _ -> - ({model|loginModel = Page.Login.Data.empty}, Ports.removeAccount (Page.pageToString HomePage)) + ({model|loginModel = Page.Login.Data.emptyModel}, Page.goto (LoginPage Nothing), Sub.none) + SessionCheckResp res -> case res of Ok lr -> let - newFlags = Data.Flags.withAccount model.flags lr - refresh = Api.refreshSession newFlags SessionCheckResp + newFlags = if lr.success then Data.Flags.withAccount model.flags lr + else Data.Flags.withoutAccount model.flags + command = if lr.success then Api.refreshSession newFlags SessionCheckResp + else Cmd.batch [Ports.removeAccount (), Page.goto (Page.loginPage model.page)] in - if (lr.success) then ({model|flags = newFlags}, refresh) - else (model, Ports.removeAccount (Page.pageToString LoginPage)) - Err _ -> (model, Ports.removeAccount (Page.pageToString LoginPage)) + ({model | flags = newFlags}, command, Sub.none) + Err _ -> + (model, Cmd.batch [Ports.removeAccount (), Page.goto (Page.loginPage model.page)], Sub.none) NavRequest req -> case req of Internal url -> let + newPage = Page.fromUrl url isCurrent = Page.fromUrl url |> Maybe.map (\p -> p == model.page) |> @@ -59,25 +107,89 @@ update msg model = in ( model , if isCurrent then Cmd.none else Nav.pushUrl model.key (Url.toString url) + , Sub.none ) External url -> ( model , Nav.load url + , Sub.none ) NavChange url -> let - page = Page.fromUrl url |> Maybe.withDefault HomePage + page = Page.fromUrl url + |> Maybe.withDefault (defaultPage model.flags) + check = checkPage model.flags page (m, c) = initPage model page in - ( { m | page = page }, c ) + if check == page then ( { m | page = page }, c, Sub.none ) + else (model, Page.goto check, Sub.none) + ToggleNavMenu -> + ({model | navMenuOpen = not model.navMenuOpen }, Cmd.none, Sub.none) + + +updateNewInvite: Page.NewInvite.Data.Msg -> Model -> (Model, Cmd Msg) +updateNewInvite lmsg model = + let + (lm, lc) = Page.NewInvite.Update.update model.flags lmsg model.newInviteModel + in + ( {model | newInviteModel = lm } + , Cmd.map NewInviteMsg lc + ) + +updateUpload: Page.Upload.Data.Msg -> Model -> (Model, Cmd Msg, Sub Msg) +updateUpload lmsg model = + let + (lm, lc, ls) = Page.Upload.Update.update (Page.uploadId model.page) model.flags lmsg model.uploadModel + in + ( { model | uploadModel = lm } + , Cmd.map UploadMsg lc + , Sub.map UploadMsg ls + ) + +updateRegister: Page.Register.Data.Msg -> Model -> (Model, Cmd Msg) +updateRegister lmsg model = + let + (lm, lc) = Page.Register.Update.update model.flags lmsg model.registerModel + in + ( { model | registerModel = lm } + , Cmd.map RegisterMsg lc + ) + +updateQueue: Page.Queue.Data.Msg -> Model -> (Model, Cmd Msg) +updateQueue lmsg model = + let + (lm, lc) = Page.Queue.Update.update model.flags lmsg model.queueModel + in + ( { model | queueModel = lm } + , Cmd.map QueueMsg lc + ) + + +updateUserSettings: Page.UserSettings.Data.Msg -> Model -> (Model, Cmd Msg) +updateUserSettings lmsg model = + let + (lm, lc) = Page.UserSettings.Update.update model.flags lmsg model.userSettingsModel + in + ( { model | userSettingsModel = lm } + , Cmd.map UserSettingsMsg lc + ) + +updateCollSettings: Page.CollectiveSettings.Data.Msg -> Model -> (Model, Cmd Msg) +updateCollSettings lmsg model = + let + (lm, lc) = Page.CollectiveSettings.Update.update model.flags lmsg model.collSettingsModel + in + ( { model | collSettingsModel = lm } + , Cmd.map CollSettingsMsg lc + ) updateLogin: Page.Login.Data.Msg -> Model -> (Model, Cmd Msg) updateLogin lmsg model = let - (lm, lc, ar) = Page.Login.Update.update model.flags lmsg model.loginModel + (lm, lc, ar) = Page.Login.Update.update (Page.loginPageReferrer model.page) model.flags lmsg model.loginModel newFlags = Maybe.map (Data.Flags.withAccount model.flags) ar |> Maybe.withDefault model.flags in @@ -94,13 +206,52 @@ updateHome lmsg model = , Cmd.map HomeMsg lc ) +updateManageData: Page.ManageData.Data.Msg -> Model -> (Model, Cmd Msg) +updateManageData lmsg model = + let + (lm, lc) = Page.ManageData.Update.update model.flags lmsg model.manageDataModel + in + ({ model | manageDataModel = lm } + ,Cmd.map ManageDataMsg lc + ) initPage: Model -> Page -> (Model, Cmd Msg) initPage model page = case page of HomePage -> - (model, Cmd.none) -{-- updateHome Page.Home.Data.GetBasicStats model --} + Util.Update.andThen1 + [updateHome Page.Home.Data.Init + ,updateQueue Page.Queue.Data.StopRefresh + ] model - LoginPage -> - (model, Cmd.none) + LoginPage _ -> + updateQueue Page.Queue.Data.StopRefresh model + + ManageDataPage -> + updateQueue Page.Queue.Data.StopRefresh model + + CollectiveSettingPage -> + Util.Update.andThen1 + [updateQueue Page.Queue.Data.StopRefresh + ,updateCollSettings Page.CollectiveSettings.Data.Init + ] model + + UserSettingPage -> + updateQueue Page.Queue.Data.StopRefresh model + + QueuePage -> + updateQueue Page.Queue.Data.Init model + + RegisterPage -> + updateQueue Page.Queue.Data.StopRefresh model + + UploadPage _ -> + updateQueue Page.Queue.Data.StopRefresh model + + NewInvitePage -> + updateQueue Page.Queue.Data.StopRefresh model + + +noSub: (Model, Cmd Msg) -> (Model, Cmd Msg, Sub Msg) +noSub (m, c) = + (m, c, Sub.none) diff --git a/modules/webapp/src/main/elm/App/View.elm b/modules/webapp/src/main/elm/App/View.elm index 81254579..03069cc7 100644 --- a/modules/webapp/src/main/elm/App/View.elm +++ b/modules/webapp/src/main/elm/App/View.elm @@ -4,19 +4,39 @@ import Html exposing (..) import Html.Attributes exposing (..) import Html.Events exposing (onClick) +import Page import App.Data exposing (..) +import Data.Flags exposing (Flags) import Page exposing (Page(..)) import Page.Home.View import Page.Login.View +import Page.ManageData.View +import Page.CollectiveSettings.View +import Page.UserSettings.View +import Page.Queue.View +import Page.Register.View +import Page.Upload.View +import Page.NewInvite.View view: Model -> Html Msg view model = case model.page of - LoginPage -> + LoginPage _ -> loginLayout model + RegisterPage -> + registerLayout model + NewInvitePage -> + newInviteLayout model _ -> defaultLayout model +registerLayout: Model -> Html Msg +registerLayout model = + div [class "register-layout"] + [ (viewRegister model) + , (footer model) + ] + loginLayout: Model -> Html Msg loginLayout model = div [class "login-layout"] @@ -24,32 +44,81 @@ loginLayout model = , (footer model) ] +newInviteLayout: Model -> Html Msg +newInviteLayout model = + div [class "newinvite-layout"] + [ (viewNewInvite model) + , (footer model) + ] + defaultLayout: Model -> Html Msg defaultLayout model = div [class "default-layout"] - [ div [class "ui fixed top sticky attached large menu black-bg"] + [ div [class "ui fixed top sticky attached large menu top-menu"] [div [class "ui fluid container"] [ a [class "header item narrow-item" ,Page.href HomePage ] - [i [classList [("lemon outline icon", True) + [i [classList [("umbrella icon", True) ]] [] ,text model.flags.config.appName] , (loginInfo model) ] ] - , div [ class "ui fluid container main-content" ] + , div [ class "main-content" ] [ (case model.page of HomePage -> viewHome model - LoginPage -> + LoginPage _ -> viewLogin model + ManageDataPage -> + viewManageData model + CollectiveSettingPage -> + viewCollectiveSettings model + UserSettingPage -> + viewUserSettings model + QueuePage -> + viewQueue model + RegisterPage -> + viewRegister model + UploadPage mid -> + viewUpload mid model + NewInvitePage -> + viewNewInvite model ) ] , (footer model) ] +viewNewInvite: Model -> Html Msg +viewNewInvite model = + Html.map NewInviteMsg (Page.NewInvite.View.view model.flags model.newInviteModel) + +viewUpload: (Maybe String) ->Model -> Html Msg +viewUpload mid model = + Html.map UploadMsg (Page.Upload.View.view mid model.uploadModel) + +viewRegister: Model -> Html Msg +viewRegister model = + Html.map RegisterMsg (Page.Register.View.view model.flags model.registerModel) + +viewQueue: Model -> Html Msg +viewQueue model = + Html.map QueueMsg (Page.Queue.View.view model.queueModel) + +viewUserSettings: Model -> Html Msg +viewUserSettings model = + Html.map UserSettingsMsg (Page.UserSettings.View.view model.userSettingsModel) + +viewCollectiveSettings: Model -> Html Msg +viewCollectiveSettings model = + Html.map CollSettingsMsg (Page.CollectiveSettings.View.view model.flags model.collSettingsModel) + +viewManageData: Model -> Html Msg +viewManageData model = + Html.map ManageDataMsg (Page.ManageData.View.view model.manageDataModel) + viewLogin: Model -> Html Msg viewLogin model = Html.map LoginMsg (Page.Login.View.view model.loginModel) @@ -59,29 +128,87 @@ viewHome model = Html.map HomeMsg (Page.Home.View.view model.homeModel) +menuEntry: Model -> Page -> List (Html Msg) -> Html Msg +menuEntry model page children = + a [classList [("icon item", True) + ,("active", model.page == page) + ] + , Page.href page] + children + loginInfo: Model -> Html Msg loginInfo model = div [class "right menu"] (case model.flags.account of Just acc -> - [a [class "item" - ] - [text "Profile" + [div [class "ui dropdown icon link item" + , onClick ToggleNavMenu ] - ,a [class "item" - ,Page.href model.page - ,onClick Logout - ] - [text "Logout " - ,text (acc.collective ++ "/" ++ acc.user) + [i [class "ui bars icon"][] + ,div [classList [("left menu", True) + ,("transition visible", model.navMenuOpen) + ] + ] + [menuEntry model HomePage + [i [class "umbrella icon"][] + ,text "Items" + ] + ,div [class "divider"][] + ,menuEntry model CollectiveSettingPage + [i [class "users circle icon"][] + ,text "Collective Settings" + ] + ,menuEntry model UserSettingPage + [i [class "user circle icon"][] + ,text "User Settings" + ] + ,div [class "divider"][] + ,menuEntry model ManageDataPage + [i [class "cubes icon"][] + ,text "Manage Data" + ] + ,div [class "divider"][] + ,menuEntry model (UploadPage Nothing) + [i [class "upload icon"][] + ,text "Upload files" + ] + ,menuEntry model QueuePage + [i [class "tachometer alternate icon"][] + ,text "Procesing Queue" + ] + ,div [classList [("divider", True) + ,("invisible", model.flags.config.signupMode /= "invite") + ]] + [] + ,a [classList [("icon item", True) + ,("invisible", model.flags.config.signupMode /= "invite") + ] + , Page.href NewInvitePage + ] + [i [class "key icon"][] + ,text "New Invites" + ] + ,div [class "divider"][] + ,a [class "icon item" + ,href "" + ,onClick Logout] + [i [class "sign-out icon"][] + ,text "Logout" + ] + ] ] ] Nothing -> [a [class "item" - ,Page.href LoginPage + ,Page.href (Page.loginPage model.page) ] [text "Login" ] + ,a [class "item" + ,Page.href RegisterPage + ] + [text "Register" + ] ] ) diff --git a/modules/webapp/src/main/elm/Comp/AddressForm.elm b/modules/webapp/src/main/elm/Comp/AddressForm.elm new file mode 100644 index 00000000..26784231 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/AddressForm.elm @@ -0,0 +1,129 @@ +module Comp.AddressForm exposing ( Model + , emptyModel + , Msg(..) + , view + , update + , getAddress) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onInput) +import Data.Flags exposing (Flags) +import Api.Model.Address exposing (Address) +import Comp.Dropdown +import Util.List + +type alias Model = + { address: Address + , street: String + , zip: String + , city: String + , country: Comp.Dropdown.Model Country + } + +type alias Country = + { code: String + , label: String + } + +countries: List Country +countries = + [ Country "DE" "Germany" + , Country "CH" "Switzerland" + , Country "GB" "Great Britain" + , Country "ES" "Spain" + , Country "AU" "Austria" + ] + +emptyModel: Model +emptyModel = + { address = Api.Model.Address.empty + , street = "" + , zip = "" + , city = "" + , country = Comp.Dropdown.makeSingleList + { makeOption = \c -> { value = c.code, text = c.label } + , placeholder = "Select Country" + , options = countries + , selected = Nothing + } + } + + +getAddress: Model -> Address +getAddress model = + { street = model.street + , zip = model.zip + , city = model.city + , country = Comp.Dropdown.getSelected model.country |> List.head |> Maybe.map .code |> Maybe.withDefault "" + } + +type Msg + = SetStreet String + | SetCity String + | SetZip String + | SetAddress Address + | CountryMsg (Comp.Dropdown.Msg Country) + +update: Msg -> Model -> (Model, Cmd Msg) +update msg model = + case msg of + SetAddress a -> + let + selection = Util.List.find (\c -> c.code == a.country) countries + |> Maybe.map List.singleton + |> Maybe.withDefault [] + (m2, c2) = Comp.Dropdown.update (Comp.Dropdown.SetSelection selection) model.country + in + ({model | address = a, street = a.street, city = a.city, zip = a.zip, country = m2 }, Cmd.map CountryMsg c2) + + SetStreet n -> + ({model | street = n}, Cmd.none) + + SetCity c -> + ({model | city = c }, Cmd.none) + + SetZip z -> + ({model | zip = z }, Cmd.none) + + CountryMsg m -> + let + (m1, c1) = Comp.Dropdown.update m model.country + in + ({model | country = m1}, Cmd.map CountryMsg c1) + +view: Model -> Html Msg +view model = + div [class "ui form"] + [div [class "field" + ] + [label [][text "Street"] + ,input [type_ "text" + ,onInput SetStreet + ,placeholder "Street" + ,value model.street + ][] + ] + ,div [class "field" + ] + [label [][text "Zip Code"] + ,input [type_ "text" + ,onInput SetZip + ,placeholder "Zip" + ,value model.zip + ][] + ] + ,div [class "field" + ] + [label [][text "City"] + ,input [type_ "text" + ,onInput SetCity + ,placeholder "City" + ,value model.city + ][] + ] + ,div [class "field"] + [label [][text "Country"] + ,Html.map CountryMsg (Comp.Dropdown.view model.country) + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/ChangePasswordForm.elm b/modules/webapp/src/main/elm/Comp/ChangePasswordForm.elm new file mode 100644 index 00000000..4c0958bc --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/ChangePasswordForm.elm @@ -0,0 +1,198 @@ +module Comp.ChangePasswordForm exposing (Model + ,emptyModel + ,Msg(..) + ,update + ,view + ) +import Http +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onInput, onClick) + +import Api +import Api.Model.PasswordChange exposing (PasswordChange) +import Util.Http +import Api.Model.BasicResult exposing (BasicResult) +import Data.Flags exposing (Flags) + +type alias Model = + { current: String + , newPass1: String + , newPass2: String + , showCurrent: Bool + , showPass1: Bool + , showPass2: Bool + , errors: List String + , loading: Bool + , successMsg: String + } + +emptyModel: Model +emptyModel = + validateModel + { current = "" + , newPass1 = "" + , newPass2 = "" + , showCurrent = False + , showPass1 = False + , showPass2 = False + , errors = [] + , loading = False + , successMsg = "" + } + +type Msg + = SetCurrent String + | SetNew1 String + | SetNew2 String + | ToggleShowPass1 + | ToggleShowPass2 + | ToggleShowCurrent + | Submit + | SubmitResp (Result Http.Error BasicResult) + + +validate: Model -> List String +validate model = + List.concat + [ if model.newPass1 /= "" && model.newPass2 /= "" && model.newPass1 /= model.newPass2 + then ["New passwords do not match."] + else [] + , if model.newPass1 == "" || model.newPass2 == "" || model.current == "" + then ["Please fill in required fields."] + else [] + ] + +validateModel: Model -> Model +validateModel model = + let + err = validate model + in + {model | errors = err, successMsg = if err == [] then model.successMsg else "" } + +-- Update + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetCurrent s -> + (validateModel {model | current = s}, Cmd.none) + + SetNew1 s -> + (validateModel {model | newPass1 = s}, Cmd.none) + + SetNew2 s -> + (validateModel {model | newPass2 = s}, Cmd.none) + + ToggleShowCurrent -> + ({model | showCurrent = not model.showCurrent}, Cmd.none) + + ToggleShowPass1 -> + ({model | showPass1 = not model.showPass1}, Cmd.none) + + ToggleShowPass2 -> + ({model | showPass2 = not model.showPass2}, Cmd.none) + + + Submit -> + let + valid = validate model + cp = PasswordChange model.current model.newPass1 + in + if List.isEmpty valid then + ({model | loading = True, errors = [], successMsg = ""}, Api.changePassword flags cp SubmitResp) + else + (model, Cmd.none) + + SubmitResp (Ok res) -> + let + em = { emptyModel | errors = [], successMsg = "Password has been changed."} + in + if res.success then + (em, Cmd.none) + else + ({model | errors = [res.message], loading = False, successMsg = ""}, Cmd.none) + + SubmitResp (Err err) -> + let + str = Util.Http.errorToString err + in + ({model | errors = [str], loading = False, successMsg = ""}, Cmd.none) + + +-- View + +view: Model -> Html Msg +view model = + div [classList [("ui form", True) + ,("error", List.isEmpty model.errors |> not) + ,("success", model.successMsg /= "") + ] + ] + [div [classList [("field", True) + ,("error", model.current == "") + ] + ] + [label [][text "Current Password*"] + ,div [class "ui action input"] + [input [type_ <| if model.showCurrent then "text" else "password" + ,onInput SetCurrent + ,value model.current + ][] + ,button [class "ui icon button", onClick ToggleShowCurrent] + [i [class "eye icon"][] + ] + ] + ] + ,div [classList [("field", True) + ,("error", model.newPass1 == "") + ] + ] + [label [][text "New Password*"] + ,div [class "ui action input"] + [input [type_ <| if model.showPass1 then "text" else "password" + ,onInput SetNew1 + ,value model.newPass1 + ][] + ,button [class "ui icon button", onClick ToggleShowPass1] + [i [class "eye icon"][] + ] + ] + ] + ,div [classList [("field", True) + ,("error", model.newPass2 == "") + ] + ] + [label [][text "New Password (repeat)*"] + ,div [class "ui action input"] + [input [type_ <| if model.showPass2 then "text" else "password" + ,onInput SetNew2 + ,value model.newPass2 + ][] + ,button [class "ui icon button", onClick ToggleShowPass2] + [i [class "eye icon"][] + ] + ] + ] + ,div [class "ui horizontal divider"][] + ,div [class "ui success message"] + [text model.successMsg + ] + ,div [class "ui error message"] + [case model.errors of + a :: [] -> + text a + _ -> + ul [class "ui list"] + (List.map (\em -> li[][text em]) model.errors) + ] + ,div [class "ui horizontal divider"][] + ,button [class "ui primary button", onClick Submit] + [text "Submit" + ] + ,div [classList [("ui dimmer", True) + ,("active", model.loading) + ]] + [div [class "ui loader"][] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/ContactField.elm b/modules/webapp/src/main/elm/Comp/ContactField.elm new file mode 100644 index 00000000..7e8818a6 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/ContactField.elm @@ -0,0 +1,124 @@ +module Comp.ContactField exposing (Model + ,emptyModel + ,getContacts + ,Msg(..) + ,update + ,view + ) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onInput, onClick) +import Api.Model.Contact exposing (Contact) +import Data.ContactType exposing (ContactType) +import Comp.Dropdown + +type alias Model = + { items: List Contact + , kind: Comp.Dropdown.Model ContactType + , value: String + } + +emptyModel: Model +emptyModel = + { items = [] + , kind = Comp.Dropdown.makeSingleList + { makeOption = \ct -> { value = Data.ContactType.toString ct, text = Data.ContactType.toString ct } + , placeholder = "" + , options = Data.ContactType.all + , selected = List.head Data.ContactType.all + } + , value = "" + } + +makeModel: List Contact -> Model +makeModel contacts = + let + em = emptyModel + in + { em | items = contacts } + +getContacts: Model -> List Contact +getContacts model = + List.filter (\c -> c.value /= "") model.items + +type Msg + = SetValue String + | TypeMsg (Comp.Dropdown.Msg ContactType) + | AddContact + | Select Contact + | SetItems (List Contact) + +update: Msg -> Model -> (Model, Cmd Msg) +update msg model = + case msg of + SetItems contacts -> + ({model | items = contacts, value = "" }, Cmd.none) + + SetValue v -> + ({model | value = v}, Cmd.none) + + TypeMsg m -> + let + (m1, c1) = Comp.Dropdown.update m model.kind + in + ({model|kind = m1}, Cmd.map TypeMsg c1) + + AddContact -> + if model.value == "" then (model, Cmd.none) + else + let + kind = Comp.Dropdown.getSelected model.kind + |> List.head + |> Maybe.map Data.ContactType.toString + |> Maybe.withDefault "" + in + ({model| items = (Contact "" model.value kind) :: model.items, value = ""}, Cmd.none) + + Select contact -> + let + newItems = List.filter (\c -> c /= contact) model.items + (m1, c1) = Data.ContactType.fromString contact.kind + |> Maybe.map (\ct -> update (TypeMsg (Comp.Dropdown.SetSelection [ct])) model) + |> Maybe.withDefault (model, Cmd.none) + in + ({m1 | value = contact.value, items = newItems}, c1) + +view: Model -> Html Msg +view model = + div [] + [div [class "fields"] + [div [class "four wide field"] + [Html.map TypeMsg (Comp.Dropdown.view model.kind) + ] + ,div [class "twelve wide field"] + [div [class "ui action input"] + [input [type_ "text" + ,onInput SetValue + ,value model.value + ][] + ,a [class "ui button", onClick AddContact, href ""] + [text "Add" + ] + ] + ] + ] + ,div [classList [("field", True) + ,("invisible", List.isEmpty model.items) + ] + ] + [div [class "ui vertical secondary fluid menu"] + (List.map (renderItem model) model.items) + ] + ] + + +renderItem: Model -> Contact -> Html Msg +renderItem model contact = + div [class "link item", onClick (Select contact) ] + [i [class "delete icon"][] + ,div [class "ui blue label"] + [text contact.kind + ] + ,text contact.value + ] diff --git a/modules/webapp/src/main/elm/Comp/DatePicker.elm b/modules/webapp/src/main/elm/Comp/DatePicker.elm new file mode 100644 index 00000000..a341d14f --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/DatePicker.elm @@ -0,0 +1,65 @@ +module Comp.DatePicker exposing (..) + +import Html exposing (Html) +import DatePicker exposing (DatePicker, DateEvent, Settings) +import Date exposing (Date) +import Time exposing (Posix, Zone, utc, Month(..)) + +type alias Msg = DatePicker.Msg + +init: (DatePicker, Cmd Msg) +init = + DatePicker.init + +emptyModel: DatePicker +emptyModel = + DatePicker.initFromDate (Date.fromCalendarDate 2019 Aug 21) + +defaultSettings: Settings +defaultSettings = + let + ds = DatePicker.defaultSettings + in + {ds | changeYear = DatePicker.from 2010} + +update: Settings -> Msg -> DatePicker -> (DatePicker, DateEvent) +update settings msg model = + DatePicker.update settings msg model + +updateDefault: Msg -> DatePicker -> (DatePicker, DateEvent) +updateDefault msg model = + DatePicker.update defaultSettings msg model + + +view : Maybe Date -> Settings -> DatePicker -> Html Msg +view md settings model = + DatePicker.view md settings model + +viewTime : Maybe Int -> Settings -> DatePicker -> Html Msg +viewTime md settings model = + let + date = Maybe.map Time.millisToPosix md + |> Maybe.map (Date.fromPosix Time.utc) + in + view date settings model + +viewTimeDefault: Maybe Int -> DatePicker -> Html Msg +viewTimeDefault md model = + viewTime md defaultSettings model + + +startOfDay: Date -> Int +startOfDay date = + let + unix0 = Date.fromPosix Time.utc (Time.millisToPosix 0) + days = Date.diff Date.Days unix0 date + in + days * 24 * 60 * 60 * 1000 + +endOfDay: Date -> Int +endOfDay date = + (startOfDay date) + ((24 * 60) - 1) * 60 * 1000 + +midOfDay: Date -> Int +midOfDay date = + (startOfDay date) + (12 * 60 * 60 * 1000) diff --git a/modules/webapp/src/main/elm/Comp/Dropdown.elm b/modules/webapp/src/main/elm/Comp/Dropdown.elm new file mode 100644 index 00000000..dad3357d --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/Dropdown.elm @@ -0,0 +1,393 @@ +module Comp.Dropdown exposing ( Model + , Option + , makeModel + , makeSingle + , makeSingleList + , makeMultiple + , update + , isDropdownChangeMsg + , view + , getSelected + , Msg(..)) + +import Http +import Task +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onInput, onClick, onFocus, onBlur) +import Json.Decode as Decode +import Simple.Fuzzy +import Util.Html exposing (onKeyUp) +import Util.List + +type alias Option = + { value: String + , text: String + } + +type alias Item a = + { value: a + , option: Option + , visible: Bool + , selected: Bool + , active: Bool + } + +makeItem: Model a -> a -> Item a +makeItem model val = + { value = val + , option = model.makeOption val + , visible = True + , selected = False + , active = False + } + +type alias Model a = + { multiple: Bool + , selected: List (Item a) + , available: List (Item a) + , makeOption: a -> Option + , menuOpen: Bool + , filterString: String + , labelColor: a -> String + , searchable: Int -> Bool + , placeholder: String + } + +makeModel: + { multiple: Bool + , searchable: Int -> Bool + , makeOption: a -> Option + , labelColor: a -> String + , placeholder: String + } -> Model a +makeModel input = + { multiple = input.multiple + , searchable = input.searchable + , selected = [] + , available = [] + , makeOption = input.makeOption + , menuOpen = False + , filterString = "" + , labelColor = input.labelColor + , placeholder = input.placeholder + } + +makeSingle: + { makeOption: a -> Option + , placeholder: String + } -> Model a +makeSingle opts = + makeModel + { multiple = False + , searchable = \n -> n > 8 + , makeOption = opts.makeOption + , labelColor = \_ -> "" + , placeholder = opts.placeholder + } + +makeSingleList: + { makeOption: a -> Option + , placeholder: String + , options: List a + , selected: Maybe a + } -> Model a +makeSingleList opts = + let + m = makeSingle {makeOption = opts.makeOption, placeholder = opts.placeholder} + m2 = {m | available = List.map (makeItem m) opts.options} + m3 = Maybe.map (makeItem m2) opts.selected + |> Maybe.map (selectItem m2) + |> Maybe.withDefault m2 + in + m3 + +makeMultiple: + { makeOption: a -> Option + , labelColor: a -> String + } -> Model a +makeMultiple opts = + makeModel + { multiple = True + , searchable = \n -> n > 8 + , makeOption = opts.makeOption + , labelColor = opts.labelColor + , placeholder = "" + } + +getSelected: Model a -> List a +getSelected model = + List.map .value model.selected + +type Msg a + = SetOptions (List a) + | SetSelection (List a) + | ToggleMenu + | AddItem (Item a) + | RemoveItem (Item a) + | Filter String + | ShowMenu Bool + | KeyPress Int + +getOptions: Model a -> List (Item a) +getOptions model = + if not model.multiple && isSearchable model && model.menuOpen + then List.filter .visible model.available + else List.filter (\e -> e.visible && (not e.selected)) model.available + +isSearchable: Model a -> Bool +isSearchable model = + List.length model.available |> model.searchable + +-- Update + +deselectItem: Model a -> Item a -> Model a +deselectItem model item = + let + value = item.option.value + sel = if model.multiple then List.filter (\e -> e.option.value /= value) model.selected + else [] + + show e = if e.option.value == value then {e | selected = False } else e + avail = List.map show model.available + in + { model | selected = sel, available = avail } + +selectItem: Model a -> Item a -> Model a +selectItem model item = + let + value = item.option.value + sel = if model.multiple + then List.concat [ model.selected, [ item ] ] + else [ item ] + + hide e = if e.option.value == value + then {e | selected = True } + else if model.multiple then e else {e | selected = False} + avail = List.map hide model.available + in + { model | selected = sel, available = avail } + + +filterOptions: String -> List (Item a) -> List (Item a) +filterOptions str list = + List.map (\e -> {e | visible = Simple.Fuzzy.match str e.option.text, active = False}) list + +applyFilter: String -> Model a -> Model a +applyFilter str model = + { model | filterString = str, available = filterOptions str model.available } + + +makeNextActive: (Int -> Int) -> Model a -> Model a +makeNextActive nextEl model = + let + opts = getOptions model + current = Util.List.findIndexed .active opts + next = Maybe.map Tuple.second current + |> Maybe.map nextEl + |> Maybe.andThen (Util.List.get opts) + merge item1 item2 = { item2 | active = item1.option.value == item2.option.value } + updateModel item = { model | available = List.map (merge item) model.available, menuOpen = True } + in + case next of + Just item -> updateModel item + Nothing -> + case List.head opts of + Just item -> updateModel item + Nothing -> model + +selectActive: Model a -> Model a +selectActive model = + let + current = getOptions model |> Util.List.find .active + in + case current of + Just item -> + selectItem model item |> applyFilter "" + Nothing -> + model + +clearActive: Model a -> Model a +clearActive model = + { model | available = List.map (\e -> {e | active = False}) model.available } + + +-- TODO enhance update function to return this info +isDropdownChangeMsg: Msg a -> Bool +isDropdownChangeMsg cm = + case cm of + AddItem _ -> True + RemoveItem _ -> True + KeyPress code -> + Util.Html.intToKeyCode code + |> Maybe.map (\c -> c == Util.Html.Enter) + |> Maybe.withDefault False + _ -> False + + +update: Msg a -> Model a -> (Model a, Cmd (Msg a)) +update msg model = + case msg of + SetOptions list -> + ({model | available = List.map (makeItem model) list}, Cmd.none) + + SetSelection list -> + let + m0 = List.foldl (\item -> \m -> deselectItem m item) model model.selected + m1 = List.map (makeItem model) list + |> List.foldl (\item -> \m -> selectItem m item) m0 + in + (m1, Cmd.none) + + ToggleMenu -> + ({model | menuOpen = not model.menuOpen}, Cmd.none) + + AddItem e -> + let + m = selectItem model e |> applyFilter "" + in + ({ m | menuOpen = False }, Cmd.none) + + RemoveItem e -> + let + m = deselectItem model e |> applyFilter "" + in + ({ m | menuOpen = False }, Cmd.none) + + Filter str -> + let + m = applyFilter str model + in + ({ m | menuOpen = True}, Cmd.none) + + ShowMenu flag -> + ({ model | menuOpen = flag }, Cmd.none) + + KeyPress code -> + case Util.Html.intToKeyCode code of + Just Util.Html.Up -> + (makeNextActive (\n -> n - 1) model, Cmd.none) + Just Util.Html.Down -> + (makeNextActive ((+) 1) model, Cmd.none) + Just Util.Html.Enter -> + let + m = selectActive model + in + ({m | menuOpen = False }, Cmd.none) + _ -> + (model, Cmd.none) + + +-- View + +view: Model a -> Html (Msg a) +view model = + if model.multiple then viewMultiple model else viewSingle model + + +viewSingle: Model a -> Html (Msg a) +viewSingle model = + let + renderClosed item = + div [class "message" + ,style "display" "inline-block !important" + ,onClick ToggleMenu + ] + [i [class "delete icon", onClick (RemoveItem item)][] + ,text item.option.text + ] + renderDefault = + [ List.head model.selected |> Maybe.map renderClosed |> Maybe.withDefault (renderPlaceholder model) + , renderMenu model + ] + + + openSearch = + [ input [ class "search" + , placeholder "Search…" + , onInput Filter + , onKeyUp KeyPress + , value model.filterString + ][] + , renderMenu model + ] + in + div [classList [ ("ui search dropdown selection", True) + , ("open", model.menuOpen) + ] + ] + (List.append [ i [class "dropdown icon", onClick ToggleMenu][] + ] <| + if model.menuOpen && isSearchable model + then openSearch + else renderDefault + ) + + +viewMultiple: Model a -> Html (Msg a) +viewMultiple model = + let + renderSelectMultiple: Item a -> Html (Msg a) + renderSelectMultiple item = + div [classList [ ("ui label", True) + , (model.labelColor item.value, True) + ] + ,style "display" "inline-block !important" + ,onClick (RemoveItem item) + ] + [text item.option.text + ,i [class "delete icon"][] + ] + in + div [classList [ ("ui search dropdown multiple selection", True) + , ("open", model.menuOpen) + ] + ] + (List.concat + [ [i [class "dropdown icon", onClick ToggleMenu][] + ] + , List.map renderSelectMultiple model.selected + , if isSearchable model then + [ input [ class "search" + , placeholder "Search…" + , onInput Filter + , onKeyUp KeyPress + , value model.filterString + ][] + ] + else [] + , [ renderMenu model + ] + ]) + +renderMenu: Model a -> Html (Msg a) +renderMenu model = + div [classList [( "menu", True ) + ,( "transition visible", model.menuOpen ) + ] + ] (getOptions model |> List.map renderOption) + + + + + +renderPlaceholder: Model a -> Html (Msg a) +renderPlaceholder model = + div [classList [ ("placeholder-message", True) + , ("text", model.multiple) + ] + ,style "display" "inline-block !important" + ,onClick ToggleMenu + ] + [text model.placeholder + ] + +renderOption: Item a -> Html (Msg a) +renderOption item = + div [classList [ ("item", True) + , ("active", item.active || item.selected) + ] + ,onClick (AddItem item) + ] + [text item.option.text + ] diff --git a/modules/webapp/src/main/elm/Comp/Dropzone.elm b/modules/webapp/src/main/elm/Comp/Dropzone.elm new file mode 100644 index 00000000..84a04a38 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/Dropzone.elm @@ -0,0 +1,142 @@ +-- inspired from here: https://ellie-app.com/3T5mNms7SwKa1 +module Comp.Dropzone exposing ( view + , Settings + , defaultSettings + , update + , setActive + , Model + , init + , Msg(..) + ) +import File exposing (File) +import File.Select +import Json.Decode as D +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (..) + +type alias State = + { hover: Bool + , active: Bool + } + + +type alias Settings = + { classList: State -> List (String, Bool) + , contentTypes: List String + } + +defaultSettings: Settings +defaultSettings = + { classList = \m -> [("ui placeholder segment", True)] + , contentTypes = [ "application/pdf" ] + } + +type alias Model = + { state: State + , settings: Settings + } + +init: Settings -> Model +init settings = + { state = State False True + , settings = settings + } + +type Msg + = DragEnter + | DragLeave + | GotFiles File (List File) + | PickFiles + | SetActive Bool + +setActive: Bool -> Msg +setActive flag = + SetActive flag + +update: Msg -> Model -> (Model, Cmd Msg, List File) +update msg model = + case msg of + SetActive flag -> + let + ns = { hover = model.state.hover, active = flag } + in + ({ model | state = ns }, Cmd.none, []) + + PickFiles -> + (model, File.Select.files model.settings.contentTypes GotFiles, []) + + DragEnter -> + let + ns = {hover = True, active = model.state.active} + in + ({model| state = ns}, Cmd.none, []) + + DragLeave -> + let + ns = {hover = False, active = model.state.active} + in + ({model | state = ns}, Cmd.none, []) + + GotFiles file files -> + let + ns = {hover = False, active = model.state.active} + newFiles = if model.state.active then filterMime model.settings (file :: files) + else [] + in + ({model | state = ns}, Cmd.none, newFiles) + + + +view: Model -> Html Msg +view model = + div + [ classList (model.settings.classList model.state) + , hijackOn "dragenter" (D.succeed DragEnter) + , hijackOn "dragover" (D.succeed DragEnter) + , hijackOn "dragleave" (D.succeed DragLeave) + , hijackOn "drop" dropDecoder + ] + [div [class "ui icon header"] + [i [class "mouse pointer icon"][] + ,div [class "content"] + [text "Drop files here" + ,div [class "sub header"] + [text "PDF files only" + ] + ] + ] + ,div [class "ui horizontal divider"] + [text "Or" + ] + ,a [classList [("ui basic primary button", True) + ,("disabled", not model.state.active) + ] + , onClick PickFiles + , href ""] + [i [class "folder open icon"][] + ,text "Select ..." + ] + ] + +filterMime: Settings -> List File -> List File +filterMime settings files = + let + pred f = + List.member (File.mime f) settings.contentTypes + in + List.filter pred files + +dropDecoder : D.Decoder Msg +dropDecoder = + D.at ["dataTransfer","files"] (D.oneOrMore GotFiles File.decoder) + + +hijackOn : String -> D.Decoder msg -> Attribute msg +hijackOn event decoder = + preventDefaultOn event (D.map hijack decoder) + + +hijack : msg -> (msg, Bool) +hijack msg = + (msg, True) diff --git a/modules/webapp/src/main/elm/Comp/EquipmentForm.elm b/modules/webapp/src/main/elm/Comp/EquipmentForm.elm new file mode 100644 index 00000000..a57fcd0c --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/EquipmentForm.elm @@ -0,0 +1,62 @@ +module Comp.EquipmentForm exposing ( Model + , emptyModel + , Msg(..) + , view + , update + , isValid + , getEquipment) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onInput) +import Data.Flags exposing (Flags) +import Api.Model.Equipment exposing (Equipment) + +type alias Model = + { equipment: Equipment + , name: String + } + +emptyModel: Model +emptyModel = + { equipment = Api.Model.Equipment.empty + , name = "" + } + +isValid: Model -> Bool +isValid model = + model.name /= "" + +getEquipment: Model -> Equipment +getEquipment model = + Equipment model.equipment.id model.name model.equipment.created + +type Msg + = SetName String + | SetEquipment Equipment + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetEquipment t -> + ({model | equipment = t, name = t.name }, Cmd.none) + + SetName n -> + ({model | name = n}, Cmd.none) + + +view: Model -> Html Msg +view model = + div [class "ui form"] + [div [classList [("field", True) + ,("error", not (isValid model)) + ] + ] + [label [][text "Name*"] + ,input [type_ "text" + ,onInput SetName + ,placeholder "Name" + ,value model.name + ][] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/EquipmentManage.elm b/modules/webapp/src/main/elm/Comp/EquipmentManage.elm new file mode 100644 index 00000000..caf1a8e9 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/EquipmentManage.elm @@ -0,0 +1,206 @@ +module Comp.EquipmentManage exposing ( Model + , emptyModel + , Msg(..) + , view + , update) + +import Http +import Api +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick, onSubmit) +import Data.Flags exposing (Flags) +import Comp.EquipmentTable +import Comp.EquipmentForm +import Comp.YesNoDimmer +import Api.Model.Equipment +import Api.Model.EquipmentList exposing (EquipmentList) +import Api.Model.BasicResult exposing (BasicResult) +import Util.Maybe +import Util.Http + +type alias Model = + { tableModel: Comp.EquipmentTable.Model + , formModel: Comp.EquipmentForm.Model + , viewMode: ViewMode + , formError: Maybe String + , loading: Bool + , deleteConfirm: Comp.YesNoDimmer.Model + } + +type ViewMode = Table | Form + +emptyModel: Model +emptyModel = + { tableModel = Comp.EquipmentTable.emptyModel + , formModel = Comp.EquipmentForm.emptyModel + , viewMode = Table + , formError = Nothing + , loading = False + , deleteConfirm = Comp.YesNoDimmer.emptyModel + } + +type Msg + = TableMsg Comp.EquipmentTable.Msg + | FormMsg Comp.EquipmentForm.Msg + | LoadEquipments + | EquipmentResp (Result Http.Error EquipmentList) + | SetViewMode ViewMode + | InitNewEquipment + | Submit + | SubmitResp (Result Http.Error BasicResult) + | YesNoMsg Comp.YesNoDimmer.Msg + | RequestDelete + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + TableMsg m -> + let + (tm, tc) = Comp.EquipmentTable.update flags m model.tableModel + (m2, c2) = ({model | tableModel = tm + , viewMode = Maybe.map (\_ -> Form) tm.selected |> Maybe.withDefault Table + , formError = if Util.Maybe.nonEmpty tm.selected then Nothing else model.formError + } + , Cmd.map TableMsg tc + ) + (m3, c3) = case tm.selected of + Just equipment -> + update flags (FormMsg (Comp.EquipmentForm.SetEquipment equipment)) m2 + Nothing -> + (m2, Cmd.none) + in + (m3, Cmd.batch [c2, c3]) + + FormMsg m -> + let + (m2, c2) = Comp.EquipmentForm.update flags m model.formModel + in + ({model | formModel = m2}, Cmd.map FormMsg c2) + + LoadEquipments -> + ({model| loading = True}, Api.getEquipments flags EquipmentResp) + + EquipmentResp (Ok equipments) -> + let + m2 = {model|viewMode = Table, loading = False} + in + update flags (TableMsg (Comp.EquipmentTable.SetEquipments equipments.items)) m2 + + EquipmentResp (Err err) -> + ({model|loading = False}, Cmd.none) + + SetViewMode m -> + let + m2 = {model | viewMode = m } + in + case m of + Table -> + update flags (TableMsg Comp.EquipmentTable.Deselect) m2 + Form -> + (m2, Cmd.none) + + InitNewEquipment -> + let + nm = {model | viewMode = Form, formError = Nothing } + equipment = Api.Model.Equipment.empty + in + update flags (FormMsg (Comp.EquipmentForm.SetEquipment equipment)) nm + + Submit -> + let + equipment = Comp.EquipmentForm.getEquipment model.formModel + valid = Comp.EquipmentForm.isValid model.formModel + in if valid then + ({model|loading = True}, Api.postEquipment flags equipment SubmitResp) + else + ({model|formError = Just "Please correct the errors in the form."}, Cmd.none) + + SubmitResp (Ok res) -> + if res.success then + let + (m2, c2) = update flags (SetViewMode Table) model + (m3, c3) = update flags LoadEquipments m2 + in + ({m3|loading = False}, Cmd.batch [c2,c3]) + else + ({model | formError = Just res.message, loading = False }, Cmd.none) + + SubmitResp (Err err) -> + ({model | formError = Just (Util.Http.errorToString err), loading = False}, Cmd.none) + + RequestDelete -> + update flags (YesNoMsg Comp.YesNoDimmer.activate) model + + YesNoMsg m -> + let + (cm, confirmed) = Comp.YesNoDimmer.update m model.deleteConfirm + equip = Comp.EquipmentForm.getEquipment model.formModel + cmd = if confirmed then Api.deleteEquip flags equip.id SubmitResp else Cmd.none + in + ({model | deleteConfirm = cm}, cmd) + +view: Model -> Html Msg +view model = + if model.viewMode == Table then viewTable model + else viewForm model + +viewTable: Model -> Html Msg +viewTable model = + div [] + [button [class "ui basic button", onClick InitNewEquipment] + [i [class "plus icon"][] + ,text "Create new" + ] + ,Html.map TableMsg (Comp.EquipmentTable.view model.tableModel) + ,div [classList [("ui dimmer", True) + ,("active", model.loading) + ]] + [div [class "ui loader"][] + ] + ] + +viewForm: Model -> Html Msg +viewForm model = + let + newEquipment = model.formModel.equipment.id == "" + in + Html.form [class "ui segment", onSubmit Submit] + [Html.map YesNoMsg (Comp.YesNoDimmer.view model.deleteConfirm) + ,if newEquipment then + h3 [class "ui dividing header"] + [text "Create new equipment" + ] + else + h3 [class "ui dividing header"] + [text ("Edit equipment: " ++ model.formModel.equipment.name) + ,div [class "sub header"] + [text "Id: " + ,text model.formModel.equipment.id + ] + ] + ,Html.map FormMsg (Comp.EquipmentForm.view model.formModel) + ,div [classList [("ui error message", True) + ,("invisible", Util.Maybe.isEmpty model.formError) + ] + ] + [Maybe.withDefault "" model.formError |> text + ] + ,div [class "ui horizontal divider"][] + ,button [class "ui primary button", type_ "submit"] + [text "Submit" + ] + ,a [class "ui secondary button", onClick (SetViewMode Table), href ""] + [text "Cancel" + ] + ,if not newEquipment then + a [class "ui right floated red button", href "", onClick RequestDelete] + [text "Delete"] + else + span[][] + ,div [classList [("ui dimmer", True) + ,("active", model.loading) + ]] + [div [class "ui loader"][] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/EquipmentTable.elm b/modules/webapp/src/main/elm/Comp/EquipmentTable.elm new file mode 100644 index 00000000..c78a5f49 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/EquipmentTable.elm @@ -0,0 +1,62 @@ +module Comp.EquipmentTable exposing ( Model + , emptyModel + , Msg(..) + , view + , update) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) +import Data.Flags exposing (Flags) +import Api.Model.Equipment exposing (Equipment) + +type alias Model = + { equips: List Equipment + , selected: Maybe Equipment + } + +emptyModel: Model +emptyModel = + { equips = [] + , selected = Nothing + } + +type Msg + = SetEquipments (List Equipment) + | Select Equipment + | Deselect + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetEquipments list -> + ({model | equips = list, selected = Nothing }, Cmd.none) + + Select equip -> + ({model | selected = Just equip}, Cmd.none) + + Deselect -> + ({model | selected = Nothing}, Cmd.none) + + +view: Model -> Html Msg +view model = + table [class "ui selectable table"] + [thead [] + [tr [] + [th [][text "Name"] + ] + ] + ,tbody [] + (List.map (renderEquipmentLine model) model.equips) + ] + +renderEquipmentLine: Model -> Equipment -> Html Msg +renderEquipmentLine model equip = + tr [classList [("active", model.selected == Just equip)] + ,onClick (Select equip) + ] + [td [] + [text equip.name + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/ItemDetail.elm b/modules/webapp/src/main/elm/Comp/ItemDetail.elm new file mode 100644 index 00000000..00758504 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/ItemDetail.elm @@ -0,0 +1,832 @@ +module Comp.ItemDetail exposing ( Model + , emptyModel + , Msg(..) + , UserNav(..) + , update + , view + ) + +import Api +import Http +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick, onInput) +import Comp.Dropdown exposing (isDropdownChangeMsg) +import Comp.YesNoDimmer +import Comp.DatePicker +import DatePicker exposing (DatePicker) +import Data.Flags exposing (Flags) +import Data.Direction exposing (Direction) +import Api.Model.ItemDetail exposing (ItemDetail) +import Api.Model.Tag exposing (Tag) +import Api.Model.TagList exposing (TagList) +import Api.Model.BasicResult exposing (BasicResult) +import Api.Model.ReferenceList exposing (ReferenceList) +import Api.Model.IdName exposing (IdName) +import Api.Model.DirectionValue exposing (DirectionValue) +import Api.Model.OptionalId exposing (OptionalId) +import Api.Model.OptionalText exposing (OptionalText) +import Api.Model.OptionalDate exposing (OptionalDate) +import Api.Model.EquipmentList exposing (EquipmentList) +import Api.Model.ItemProposals exposing (ItemProposals) +import Util.Time +import Util.String +import Util.Maybe +import Util.Html +import Util.Size +import Markdown + +type alias Model = + { item: ItemDetail + , visibleAttach: Int + , menuOpen: Bool + , tagModel: Comp.Dropdown.Model Tag + , directionModel: Comp.Dropdown.Model Direction + , corrOrgModel: Comp.Dropdown.Model IdName + , corrPersonModel: Comp.Dropdown.Model IdName + , concPersonModel: Comp.Dropdown.Model IdName + , concEquipModel: Comp.Dropdown.Model IdName + , nameModel: String + , notesModel: Maybe String + , deleteConfirm: Comp.YesNoDimmer.Model + , itemDatePicker: DatePicker + , itemDate: Maybe Int + , itemProposals: ItemProposals + , dueDate: Maybe Int + , dueDatePicker: DatePicker + } + +emptyModel: Model +emptyModel = + { item = Api.Model.ItemDetail.empty + , visibleAttach = 0 + , menuOpen = False + , tagModel = Comp.Dropdown.makeMultiple + { makeOption = \tag -> { value = tag.id, text = tag.name } + , labelColor = \tag -> if Util.Maybe.nonEmpty tag.category then "basic blue" else "" + } + , directionModel = Comp.Dropdown.makeSingleList + { makeOption = \entry -> {value = Data.Direction.toString entry, text = Data.Direction.toString entry} + , options = Data.Direction.all + , placeholder = "Choose a direction…" + , selected = Nothing + } + , corrOrgModel = Comp.Dropdown.makeSingle + { makeOption = \e -> {value = e.id, text = e.name} + , placeholder = "" + } + , corrPersonModel = Comp.Dropdown.makeSingle + { makeOption = \e -> {value = e.id, text = e.name} + , placeholder = "" + } + , concPersonModel = Comp.Dropdown.makeSingle + { makeOption = \e -> {value = e.id, text = e.name} + , placeholder = "" + } + , concEquipModel = Comp.Dropdown.makeSingle + { makeOption = \e -> {value = e.id, text = e.name} + , placeholder = "" + } + , nameModel = "" + , notesModel = Nothing + , deleteConfirm = Comp.YesNoDimmer.emptyModel + , itemDatePicker = Comp.DatePicker.emptyModel + , itemDate = Nothing + , itemProposals = Api.Model.ItemProposals.empty + , dueDate = Nothing + , dueDatePicker = Comp.DatePicker.emptyModel + } + +type UserNav + = NavBack | NavPrev | NavNext | NavNone | NavNextOrBack + +noNav: (Model, Cmd Msg) -> (Model, Cmd Msg, UserNav) +noNav (model, cmd) = + (model, cmd, NavNone) + +type Msg + = ToggleMenu + | ReloadItem + | Init + | SetItem ItemDetail + | SetActiveAttachment Int + | NavClick UserNav + | TagDropdownMsg (Comp.Dropdown.Msg Tag) + | DirDropdownMsg (Comp.Dropdown.Msg Direction) + | OrgDropdownMsg (Comp.Dropdown.Msg IdName) + | CorrPersonMsg (Comp.Dropdown.Msg IdName) + | ConcPersonMsg (Comp.Dropdown.Msg IdName) + | ConcEquipMsg (Comp.Dropdown.Msg IdName) + | GetTagsResp (Result Http.Error TagList) + | GetOrgResp (Result Http.Error ReferenceList) + | GetPersonResp (Result Http.Error ReferenceList) + | GetEquipResp (Result Http.Error EquipmentList) + | SetName String + | SaveName + | SetNotes String + | SaveNotes + | ConfirmItem + | UnconfirmItem + | SetCorrOrgSuggestion IdName + | SetCorrPersonSuggestion IdName + | SetConcPersonSuggestion IdName + | SetConcEquipSuggestion IdName + | SetItemDateSuggestion Int + | SetDueDateSuggestion Int + | ItemDatePickerMsg Comp.DatePicker.Msg + | DueDatePickerMsg Comp.DatePicker.Msg + | YesNoMsg Comp.YesNoDimmer.Msg + | RequestDelete + | SaveResp (Result Http.Error BasicResult) + | DeleteResp (Result Http.Error BasicResult) + | GetItemResp (Result Http.Error ItemDetail) + | GetProposalResp (Result Http.Error ItemProposals) + | RemoveDueDate + | RemoveDate + + +-- update + +getOptions: Flags -> Cmd Msg +getOptions flags = + Cmd.batch + [ Api.getTags flags GetTagsResp + , Api.getOrgLight flags GetOrgResp + , Api.getPersonsLight flags GetPersonResp + , Api.getEquipments flags GetEquipResp + ] + +saveTags: Flags -> Model -> Cmd Msg +saveTags flags model = + let + tags = Comp.Dropdown.getSelected model.tagModel + |> List.map (\t -> IdName t.id t.name) + |> ReferenceList + in + Api.setTags flags model.item.id tags SaveResp + +setDirection: Flags -> Model -> Cmd Msg +setDirection flags model = + let + dir = Comp.Dropdown.getSelected model.directionModel |> List.head + in + case dir of + Just d -> + Api.setDirection flags model.item.id (DirectionValue (Data.Direction.toString d)) SaveResp + Nothing -> + Cmd.none + +setCorrOrg: Flags -> Model -> Maybe IdName -> Cmd Msg +setCorrOrg flags model mref = + let + idref = Maybe.map .id mref + |> OptionalId + in + Api.setCorrOrg flags model.item.id idref SaveResp + +setCorrPerson: Flags -> Model -> Maybe IdName -> Cmd Msg +setCorrPerson flags model mref = + let + idref = Maybe.map .id mref + |> OptionalId + in + Api.setCorrPerson flags model.item.id idref SaveResp + +setConcPerson: Flags -> Model -> Maybe IdName -> Cmd Msg +setConcPerson flags model mref = + let + idref = Maybe.map .id mref + |> OptionalId + in + Api.setConcPerson flags model.item.id idref SaveResp + +setConcEquip: Flags -> Model -> Maybe IdName -> Cmd Msg +setConcEquip flags model mref = + let + idref = Maybe.map .id mref + |> OptionalId + in + Api.setConcEquip flags model.item.id idref SaveResp + +setName: Flags -> Model -> Cmd Msg +setName flags model = + let + text = OptionalText (Just model.nameModel) + in + if model.nameModel == "" then Cmd.none + else Api.setItemName flags model.item.id text SaveResp + +setNotes: Flags -> Model -> Cmd Msg +setNotes flags model = + let + text = OptionalText model.notesModel + in + if model.notesModel == Nothing then Cmd.none + else Api.setItemNotes flags model.item.id text SaveResp + +setDate: Flags -> Model-> Maybe Int -> Cmd Msg +setDate flags model date = + Api.setItemDate flags model.item.id (OptionalDate date) SaveResp + +setDueDate: Flags -> Model -> Maybe Int -> Cmd Msg +setDueDate flags model date = + Api.setItemDueDate flags model.item.id (OptionalDate date) SaveResp + + +update: Flags -> Msg -> Model -> (Model, Cmd Msg, UserNav) +update flags msg model = + case msg of + Init -> + let + (dp, dpc) = Comp.DatePicker.init + in + ( {model | itemDatePicker = dp, dueDatePicker = dp} + , Cmd.batch [getOptions flags + , Cmd.map ItemDatePickerMsg dpc + , Cmd.map DueDatePickerMsg dpc + ] + , NavNone + ) + + SetItem item -> + let + (m1, c1, _) = update flags (TagDropdownMsg (Comp.Dropdown.SetSelection item.tags)) model + (m2, c2, _) = update flags (DirDropdownMsg (Comp.Dropdown.SetSelection (Data.Direction.fromString item.direction + |> Maybe.map List.singleton + |> Maybe.withDefault []))) m1 + (m3, c3, _) = update flags (OrgDropdownMsg (Comp.Dropdown.SetSelection (item.corrOrg + |> Maybe.map List.singleton + |> Maybe.withDefault []))) m2 + (m4, c4, _) = update flags (CorrPersonMsg (Comp.Dropdown.SetSelection (item.corrPerson + |> Maybe.map List.singleton + |> Maybe.withDefault []))) m3 + (m5, c5, _) = update flags (ConcPersonMsg (Comp.Dropdown.SetSelection (item.concPerson + |> Maybe.map List.singleton + |> Maybe.withDefault []))) m4 + proposalCmd = if item.state == "created" + then Api.getItemProposals flags item.id GetProposalResp + else Cmd.none + in + ({m5|item = item, nameModel = item.name, notesModel = item.notes, itemDate = item.itemDate, dueDate = item.dueDate} + ,Cmd.batch [c1, c2, c3,c4,c5, getOptions flags, proposalCmd] + ) |> noNav + + SetActiveAttachment pos -> + ({model|visibleAttach = pos}, Cmd.none, NavNone) + + NavClick nav -> + (model, Cmd.none, nav) + + ToggleMenu -> + ({model|menuOpen = not model.menuOpen}, Cmd.none, NavNone) + + ReloadItem -> + if model.item.id == "" then (model, Cmd.none, NavNone) + else (model, Api.itemDetail flags model.item.id GetItemResp, NavNone) + + TagDropdownMsg m -> + let + (m2, c2) = Comp.Dropdown.update m model.tagModel + newModel = {model|tagModel = m2} + save = if isDropdownChangeMsg m then saveTags flags newModel else Cmd.none + in + (newModel, Cmd.batch[ save, Cmd.map TagDropdownMsg c2 ], NavNone) + DirDropdownMsg m -> + let + (m2, c2) = Comp.Dropdown.update m model.directionModel + newModel = {model|directionModel = m2} + save = if isDropdownChangeMsg m then setDirection flags newModel else Cmd.none + in + (newModel, Cmd.batch [save, Cmd.map DirDropdownMsg c2 ]) |> noNav + OrgDropdownMsg m -> + let + (m2, c2) = Comp.Dropdown.update m model.corrOrgModel + newModel = {model|corrOrgModel = m2} + idref = Comp.Dropdown.getSelected m2 |> List.head + save = if isDropdownChangeMsg m then setCorrOrg flags newModel idref else Cmd.none + in + (newModel, Cmd.batch [save, Cmd.map OrgDropdownMsg c2]) |> noNav + CorrPersonMsg m -> + let + (m2, c2) = Comp.Dropdown.update m model.corrPersonModel + newModel = {model|corrPersonModel = m2} + idref = Comp.Dropdown.getSelected m2 |> List.head + save = if isDropdownChangeMsg m then setCorrPerson flags newModel idref else Cmd.none + in + (newModel, Cmd.batch [save, Cmd.map CorrPersonMsg c2]) |> noNav + ConcPersonMsg m -> + let + (m2, c2) = Comp.Dropdown.update m model.concPersonModel + newModel = {model|concPersonModel = m2} + idref = Comp.Dropdown.getSelected m2 |> List.head + save = if isDropdownChangeMsg m then setConcPerson flags newModel idref else Cmd.none + in + (newModel, Cmd.batch [save, Cmd.map ConcPersonMsg c2]) |> noNav + ConcEquipMsg m -> + let + (m2, c2) = Comp.Dropdown.update m model.concEquipModel + newModel = {model|concEquipModel = m2} + idref = Comp.Dropdown.getSelected m2 |> List.head + save = if isDropdownChangeMsg m then setConcEquip flags newModel idref else Cmd.none + in + (newModel, Cmd.batch [save, Cmd.map ConcEquipMsg c2]) |> noNav + SetName str -> + ({model|nameModel = str}, Cmd.none) |> noNav + + SaveName -> + (model, setName flags model) |> noNav + + SetNotes str -> + ({model|notesModel = if str == "" then Nothing else Just str}, Cmd.none) |> noNav + SaveNotes -> + (model, setNotes flags model) |> noNav + + ConfirmItem -> + (model, Api.setConfirmed flags model.item.id SaveResp) |> noNav + + UnconfirmItem -> + (model, Api.setUnconfirmed flags model.item.id SaveResp) |> noNav + + ItemDatePickerMsg m -> + let + (dp, event) = Comp.DatePicker.updateDefault m model.itemDatePicker + in + case event of + DatePicker.Picked date -> + let + newModel = {model|itemDatePicker = dp, itemDate = Just (Comp.DatePicker.midOfDay date)} + in + (newModel, setDate flags newModel newModel.itemDate) |> noNav + _ -> + ({model|itemDatePicker = dp}, Cmd.none) |> noNav + + RemoveDate -> + ({ model | itemDate = Nothing }, setDate flags model Nothing ) |> noNav + + DueDatePickerMsg m -> + let + (dp, event) = Comp.DatePicker.updateDefault m model.dueDatePicker + in + case event of + DatePicker.Picked date -> + let + newModel = {model|dueDatePicker = dp, dueDate = Just (Comp.DatePicker.midOfDay date)} + in + (newModel, setDueDate flags newModel newModel.dueDate) |> noNav + _ -> + ({model|dueDatePicker = dp}, Cmd.none) |> noNav + + RemoveDueDate -> + ({ model | dueDate = Nothing }, setDueDate flags model Nothing ) |> noNav + + YesNoMsg m -> + let + (cm, confirmed) = Comp.YesNoDimmer.update m model.deleteConfirm + cmd = if confirmed then Api.deleteItem flags model.item.id DeleteResp else Cmd.none + in + ({model | deleteConfirm = cm}, cmd) |> noNav + + RequestDelete -> + update flags (YesNoMsg Comp.YesNoDimmer.activate) model + + SetCorrOrgSuggestion idname -> + (model, setCorrOrg flags model (Just idname)) |> noNav + SetCorrPersonSuggestion idname -> + (model, setCorrPerson flags model (Just idname)) |> noNav + SetConcPersonSuggestion idname -> + (model, setConcPerson flags model (Just idname)) |> noNav + SetConcEquipSuggestion idname -> + (model, setConcEquip flags model (Just idname)) |> noNav + SetItemDateSuggestion date -> + (model, setDate flags model (Just date)) |> noNav + SetDueDateSuggestion date -> + (model, setDueDate flags model (Just date)) |> noNav + + GetTagsResp (Ok tags) -> + let + tagList = Comp.Dropdown.SetOptions tags.items + (m1, c1, _) = update flags (TagDropdownMsg tagList) model + in + (m1, c1) |> noNav + GetTagsResp (Err err) -> + (model, Cmd.none) |> noNav + GetOrgResp (Ok orgs) -> + let + opts = Comp.Dropdown.SetOptions orgs.items + in + update flags (OrgDropdownMsg opts) model + + GetOrgResp (Err err) -> + (model, Cmd.none) |> noNav + GetPersonResp (Ok ps) -> + let + opts = Comp.Dropdown.SetOptions ps.items + (m1, c1, _) = update flags (CorrPersonMsg opts) model + (m2, c2, _) = update flags (ConcPersonMsg opts) m1 + in + (m2, Cmd.batch [c1, c2]) |> noNav + + GetPersonResp (Err err) -> + (model, Cmd.none) |> noNav + GetEquipResp (Ok equips) -> + let + opts = Comp.Dropdown.SetOptions (List.map (\e -> IdName e.id e.name) equips.items) + in + update flags (ConcEquipMsg opts) model + + GetEquipResp (Err err) -> + (model, Cmd.none) |> noNav + SaveResp (Ok res) -> + if res.success then (model, Api.itemDetail flags model.item.id GetItemResp) |> noNav + else (model, Cmd.none) |> noNav + SaveResp (Err err) -> + (model, Cmd.none) |> noNav + DeleteResp (Ok res) -> + if res.success then (model, Cmd.none, NavNextOrBack) + else (model, Cmd.none) |> noNav + DeleteResp (Err err) -> + (model, Cmd.none) |> noNav + GetItemResp (Ok item) -> + update flags (SetItem item) model + GetItemResp (Err err) -> + (model, Cmd.none) |> noNav + + GetProposalResp (Ok ip) -> + ({model | itemProposals = ip}, Cmd.none) |> noNav + GetProposalResp (Err err) -> + (model, Cmd.none) |> noNav + +-- view + +actionInputDatePicker: DatePicker.Settings +actionInputDatePicker = + let + ds = Comp.DatePicker.defaultSettings + in + { ds | containerClassList = [("ui action input", True)] } + + +view: Model -> Html Msg +view model = + div [] + [div [classList [("ui ablue-comp menu", True) + ]] + [a [class "item", href "", onClick (NavClick NavBack)] + [i [class "arrow left icon"][] + ] + ,a [class "item", href "", onClick (NavClick NavPrev)] + [i [class "caret square left outline icon"][] + ] + ,a [class "item", href "", onClick (NavClick NavNext)] + [i [class "caret square right outline icon"][] + ] + ,a [classList [("toggle item", True) + ,("active", model.menuOpen) + ] + ,title "Expand Menu" + ,onClick ToggleMenu + ,href "" + ] + [i [class "edit icon"][] + ] + ] + ,div [class "ui grid"] + [div [classList [("six wide column", True) + ,("invisible", not model.menuOpen) + ]] + (if model.menuOpen then (renderEditMenu model) else []) + ,div [classList [("ten", model.menuOpen) + ,("sixteen", not model.menuOpen) + ,("wide column", True) + ]] + <| List.concat + [ [renderItemInfo model] + , [renderAttachmentsTabMenu model] + , renderAttachmentsTabBody model + , renderNotes model + , renderIdInfo model + ] + ] + ] + + +renderIdInfo: Model -> List (Html Msg) +renderIdInfo model = + [div [class "ui center aligned container"] + [span [class "small-info"] + [text model.item.id + ,text " • " + ,text "Created: " + ,Util.Time.formatDateTime model.item.created |> text + ,text " • " + ,text "Updated: " + ,Util.Time.formatDateTime model.item.updated |> text + ] + ] + ] + +renderNotes: Model -> List (Html Msg) +renderNotes model = + case model.item.notes of + Nothing -> [] + Just str -> + [h3 [class "ui header"] + [text "Notes" + ] + ,Markdown.toHtml [class "item-notes"] str + ] + +renderAttachmentsTabMenu: Model -> Html Msg +renderAttachmentsTabMenu model = + div [class "ui top attached tabular menu"] + (List.indexedMap (\pos -> \a -> + div [classList [("item", True) + ,("active", pos == model.visibleAttach) + ] + ,onClick (SetActiveAttachment pos) + ] + [a.name |> Maybe.withDefault "No Name" |> text + ,text " (" + ,text (Util.Size.bytesReadable Util.Size.B (toFloat a.size)) + ,text ")" + ]) + model.item.attachments) + +renderAttachmentsTabBody: Model -> List (Html Msg) +renderAttachmentsTabBody model = + List.indexedMap (\pos -> \a -> + div [classList [("ui attached tab segment", True) + ,("active", pos == model.visibleAttach) + ] + ] + [div [class "ui 4:3 embed doc-embed"] + [embed [src ("/api/v1/sec/attachment/" ++ a.id), type_ a.contentType] + [] + ] + ] + ) model.item.attachments + +renderItemInfo: Model -> Html Msg +renderItemInfo model = + let + name = div [class "item"] + [i [class (Data.Direction.iconFromString model.item.direction)][] + ,text model.item.name + ] + date = div [class "item"] + [Maybe.withDefault model.item.created model.item.itemDate + |> Util.Time.formatDate + |> text + ] + duedate = div [class "item"] + [i [class "bell icon"][] + ,Maybe.map Util.Time.formatDate model.item.dueDate + |> Maybe.withDefault "" + |> text + ] + corr = div [class "item"] + [i [class "envelope outline icon"][] + , List.filterMap identity [model.item.corrOrg, model.item.corrPerson] + |> List.map .name + |> String.join ", " + |> Util.String.withDefault "(None)" + |> text + ] + conc = div [class "item"] + [i [class "comment outline icon"][] + ,List.filterMap identity [model.item.concPerson, model.item.concEquipment] + |> List.map .name + |> String.join ", " + |> Util.String.withDefault "(None)" + |> text + ] + src = div [class "item"] + [text model.item.source + ] + in + div [class "ui fluid container"] + ([h2 [class "ui header"] + [i [class (Data.Direction.iconFromString model.item.direction)][] + ,div [class "content"] + [text model.item.name + ,div [classList [("ui teal label", True) + ,("invisible", model.item.state /= "created") + ]] + [text "New!" + ] + ,div [class "sub header"] + [div [class "ui horizontal bulleted list"] <| + List.append + [ date + , corr + , conc + , src + ] (if Util.Maybe.isEmpty model.item.dueDate then [] else [duedate]) + ] + ] + ] + ] ++ (renderTags model)) + +renderTags: Model -> List (Html Msg) +renderTags model = + case model.item.tags of + [] -> [] + _ -> + [div [class "ui right aligned fluid container"] <| + List.map + (\t -> div [classList [("ui tag label", True) + ,("blue", Util.Maybe.nonEmpty t.category) + ] + ] + [text t.name + ] + ) model.item.tags + ] + + +renderEditMenu: Model -> List (Html Msg) +renderEditMenu model = + [renderEditButtons model + ,renderEditForm model + ] + +renderEditButtons: Model -> Html Msg +renderEditButtons model = + div [class "ui top attached right aligned segment"] + [ button [classList [("ui primary button", True) + ,("invisible", model.item.state /= "created") + ] + ,onClick ConfirmItem + ] + [ i [class "check icon"][] + , text "Confirm" + ] + , button [classList [("ui primary button", True) + ,("invisible", model.item.state /= "confirmed") + ] + ,onClick UnconfirmItem + ] + [ i [class "eye slash outline icon"][] + , text "Unconfirm" + ] + , button [class "ui negative button", onClick RequestDelete] + [ i [class "delete icon"] [] + , text "Delete" + ] + ] + +renderEditForm: Model -> Html Msg +renderEditForm model = + div [class "ui attached segment"] + [Html.map YesNoMsg (Comp.YesNoDimmer.view model.deleteConfirm) + ,div [class "ui form"] + [div [class "field"] + [label [] + [i [class "tags icon"][] + ,text "Tags" + ] + ,Html.map TagDropdownMsg (Comp.Dropdown.view model.tagModel) + ] + ,div [class " field"] + [label [][text "Name"] + ,div [class "ui action input"] + [input [type_ "text", value model.nameModel, onInput SetName][] + ,button [class "ui icon button", onClick SaveName][i [class "save outline icon"][]] + ] + ] + ,div [class "field"] + [label [][text "Direction"] + ,Html.map DirDropdownMsg (Comp.Dropdown.view model.directionModel) + ] + ,div [class " field"] + [label [][text "Date"] + ,div [class "ui action input"] + [Html.map ItemDatePickerMsg (Comp.DatePicker.viewTime model.itemDate actionInputDatePicker model.itemDatePicker) + ,a [class "ui icon button", href "", onClick RemoveDate] + [i [class "trash alternate outline icon"][] + ] + ] + ,renderItemDateSuggestions model + ] + ,div [class " field"] + [label [][text "Due Date"] + ,div [class "ui action input"] + [Html.map DueDatePickerMsg (Comp.DatePicker.viewTime model.dueDate actionInputDatePicker model.dueDatePicker) + ,a [class "ui icon button", href "", onClick RemoveDueDate] + [i [class "trash alternate outline icon"][]] + ] + ,renderDueDateSuggestions model + ] + ,h4 [class "ui dividing header"] + [i [class "tiny envelope outline icon"][] + ,text "Correspondent" + ] + ,div [class "field"] + [label [][text "Organization"] + ,Html.map OrgDropdownMsg (Comp.Dropdown.view model.corrOrgModel) + ,renderOrgSuggestions model + ] + ,div [class "field"] + [label [][text "Person"] + ,Html.map CorrPersonMsg (Comp.Dropdown.view model.corrPersonModel) + ,renderCorrPersonSuggestions model + ] + ,h4 [class "ui dividing header"] + [i [class "tiny comment outline icon"][] + ,text "Concerning" + ] + ,div [class "field"] + [label [][text "Person"] + ,Html.map ConcPersonMsg (Comp.Dropdown.view model.concPersonModel) + ,renderConcPersonSuggestions model + ] + ,div [class "field"] + [label [][text "Equipment"] + ,Html.map ConcEquipMsg (Comp.Dropdown.view model.concEquipModel) + ,renderConcEquipSuggestions model + ] + ,h4 [class "ui dividing header"] + [i [class "tiny edit icon"][] + ,div [class "content"] + [text "Notes" + ,div [class "sub header"] + [a [class "ui link" + ,target "_blank" + ,href "https://guides.github.com/features/mastering-markdown" + ] + [text "Markdown" + ] + ,text " is supported" + ] + ] + ] + ,div [class "field"] + [div [class "ui action input"] + [textarea [rows 7, onInput SetNotes][Maybe.withDefault "" model.notesModel |> text] + ,button [class "ui icon button", onClick SaveNotes] + [i [class "save outline icon"][] + ] + ] + ] + ] + ] + + + +renderSuggestions: Model -> (a -> String) -> List a -> (a -> Msg) -> Html Msg +renderSuggestions model mkName idnames tagger = + div [classList [("ui secondary vertical menu", True) + ,("invisible", model.item.state /= "created") + ]] + [div [class "item"] + [div [class "header"] + [text "Suggestions" + ] + ,div [class "menu"] <| + (idnames + |> List.take 5 + |> List.map (\p -> a [class "item", href "", onClick (tagger p)][text (mkName p)])) + ] + ] + +renderOrgSuggestions: Model -> Html Msg +renderOrgSuggestions model = + renderSuggestions model + .name + (List.take 5 model.itemProposals.corrOrg) + SetCorrOrgSuggestion + +renderCorrPersonSuggestions: Model -> Html Msg +renderCorrPersonSuggestions model = + renderSuggestions model + .name + (List.take 5 model.itemProposals.corrPerson) + SetCorrPersonSuggestion + +renderConcPersonSuggestions: Model -> Html Msg +renderConcPersonSuggestions model = + renderSuggestions model + .name + (List.take 5 model.itemProposals.concPerson) + SetConcPersonSuggestion + +renderConcEquipSuggestions: Model -> Html Msg +renderConcEquipSuggestions model = + renderSuggestions model + .name + (List.take 5 model.itemProposals.concEquipment) + SetConcEquipSuggestion + +renderItemDateSuggestions: Model -> Html Msg +renderItemDateSuggestions model = + renderSuggestions model + Util.Time.formatDate + (List.take 5 model.itemProposals.itemDate) + SetItemDateSuggestion + +renderDueDateSuggestions: Model -> Html Msg +renderDueDateSuggestions model = + renderSuggestions model + Util.Time.formatDate + (List.take 5 model.itemProposals.dueDate) + SetDueDateSuggestion diff --git a/modules/webapp/src/main/elm/Comp/ItemList.elm b/modules/webapp/src/main/elm/Comp/ItemList.elm new file mode 100644 index 00000000..d7716838 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/ItemList.elm @@ -0,0 +1,235 @@ +module Comp.ItemList exposing (Model + , emptyModel + , Msg(..) + , prevItem + , nextItem + , update + , view) + +import Set exposing (Set) +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) +import Api.Model.ItemLightList exposing (ItemLightList) +import Api.Model.ItemLightGroup exposing (ItemLightGroup) +import Api.Model.ItemLight exposing (ItemLight) +import Data.Flags exposing (Flags) +import Data.Direction +import Util.List +import Util.String +import Util.Time +import Util.Maybe + +type alias Model = + { results: ItemLightList + , openGroups: Set String + } + +emptyModel: Model +emptyModel = + { results = Api.Model.ItemLightList.empty + , openGroups = Set.empty + } + +type Msg + = SetResults ItemLightList + | ToggleGroupState ItemLightGroup + | CollapseAll + | ExpandAll + | SelectItem ItemLight + +nextItem: Model -> String -> Maybe ItemLight +nextItem model id = + List.concatMap .items model.results.groups + |> Util.List.findNext (\i -> i.id == id) + +prevItem: Model -> String -> Maybe ItemLight +prevItem model id = + List.concatMap .items model.results.groups + |> Util.List.findPrev (\i -> i.id == id) + +openAllGroups: Model -> Set String +openAllGroups model = + List.foldl + (\g -> \set -> Set.insert g.name set) + model.openGroups + model.results.groups + +update: Flags -> Msg -> Model -> (Model, Cmd Msg, Maybe ItemLight) +update flags msg model = + case msg of + SetResults list -> + let + newModel = { model | results = list, openGroups = Set.empty } + in + ({newModel|openGroups = openAllGroups newModel}, Cmd.none, Nothing) + + ToggleGroupState group -> + let + m2 = if isGroupOpen model group then closeGroup model group + else openGroup model group + in + (m2, Cmd.none, Nothing) + + CollapseAll -> + ({model | openGroups = Set.empty }, Cmd.none, Nothing) + + ExpandAll -> + let + open = openAllGroups model + in + ({model | openGroups = open }, Cmd.none, Nothing) + + SelectItem item -> + (model, Cmd.none, Just item) + + +view: Model -> Html Msg +view model = + div [] + [div [class "ui ablue-comp menu"] + [div [class "right floated menu"] + [a [class "item" + ,title "Expand all" + ,onClick ExpandAll + ,href "" + ] + [i [class "double angle down icon"][] + ] + ,a [class "item" + ,title "Collapse all" + ,onClick CollapseAll + ,href "" + ] + [i [class "double angle up icon"][] + ] + ] + ] + ,div [class "ui middle aligned very relaxed divided basic list segment"] + (List.map (viewGroup model) model.results.groups) + ] + + +isGroupOpen: Model -> ItemLightGroup -> Bool +isGroupOpen model group = + Set.member group.name model.openGroups + +openGroup: Model -> ItemLightGroup -> Model +openGroup model group = + { model | openGroups = Set.insert group.name model.openGroups } + +closeGroup: Model -> ItemLightGroup -> Model +closeGroup model group = + { model | openGroups = Set.remove group.name model.openGroups } + +viewGroup: Model -> ItemLightGroup -> Html Msg +viewGroup model group = + let + groupOpen = isGroupOpen model group + children = + [i [classList [("large middle aligned icon", True) + ,("caret right", not groupOpen) + ,("caret down", groupOpen) + ]][] + ,div [class "content"] + [div [class "right floated content"] + [div [class "ui blue label"] + [List.length group.items |> String.fromInt |> text + ] + ] + ,a [class "header" + ,onClick (ToggleGroupState group) + ,href "" + ] + [text group.name + ] + ,div [class "description"] + [makeSummary group |> text + ] + ] + ] + itemTable = + div [class "ui basic content segment no-margin"] + [(renderItemTable model group.items) + ] + in + if isGroupOpen model group then + div [class "item"] + (List.append children [itemTable]) + else + div [class "item"] + children + + +renderItemTable: Model -> List ItemLight -> Html Msg +renderItemTable model items = + table [class "ui selectable padded table"] + [thead [] + [tr [] + [th [class "collapsing"][] + ,th [class "collapsing"][text "Name"] + ,th [class "collapsing"][text "Date"] + ,th [class "collapsing"][text "Source"] + ,th [][text "Correspondent"] + ,th [][text "Concerning"] + ] + ] + ,tbody[] + (List.map (renderItemLine model) items) + ] + +renderItemLine: Model -> ItemLight -> Html Msg +renderItemLine model item = + let + dirIcon = i [class (Data.Direction.iconFromMaybe item.direction)][] + corr = List.filterMap identity [item.corrOrg, item.corrPerson] + |> List.map .name + |> List.intersperse ", " + |> String.concat + conc = List.filterMap identity [item.concPerson, item.concEquip] + |> List.map .name + |> List.intersperse ", " + |> String.concat + in + tr [onClick (SelectItem item)] + [td [class "collapsing"] + [div [classList [("ui teal ribbon label", True) + ,("invisible", item.state /= "created") + ] + ][text "New" + ] + ] + ,td [class "collapsing"] + [ dirIcon + , Util.String.ellipsis 45 item.name |> text + ] + ,td [class "collapsing"] + [Util.Time.formatDateShort item.date |> text + ,span [classList [("invisible", Util.Maybe.isEmpty item.dueDate) + ] + ] + [text " " + ,div [class "ui basic label"] + [i [class "bell icon"][] + ,Maybe.map Util.Time.formatDateShort item.dueDate |> Maybe.withDefault "" |> text + ] + ] + ] + ,td [class "collapsing"][text item.source] + ,td [][text corr] + ,td [][text conc] + ] + +makeSummary: ItemLightGroup -> String +makeSummary group = + let + corrOrgs = List.filterMap .corrOrg group.items + corrPers = List.filterMap .corrPerson group.items + concPers = List.filterMap .concPerson group.items + concEqui = List.filterMap .concEquip group.items + all = List.concat [corrOrgs, corrPers, concPers, concEqui] + in + List.map .name all + |> Util.List.distinct + |> List.intersperse ", " + |> String.concat diff --git a/modules/webapp/src/main/elm/Comp/OrgForm.elm b/modules/webapp/src/main/elm/Comp/OrgForm.elm new file mode 100644 index 00000000..e25eed70 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/OrgForm.elm @@ -0,0 +1,113 @@ +module Comp.OrgForm exposing ( Model + , emptyModel + , Msg(..) + , view + , update + , isValid + , getOrg) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onInput) +import Data.Flags exposing (Flags) +import Api.Model.Organization exposing (Organization) +import Comp.AddressForm +import Comp.ContactField + +type alias Model = + { org: Organization + , name: String + , addressModel: Comp.AddressForm.Model + , contactModel: Comp.ContactField.Model + , notes: Maybe String + } + +emptyModel: Model +emptyModel = + { org = Api.Model.Organization.empty + , name = "" + , addressModel = Comp.AddressForm.emptyModel + , contactModel = Comp.ContactField.emptyModel + , notes = Nothing + } + +isValid: Model -> Bool +isValid model = + model.name /= "" + +getOrg: Model -> Organization +getOrg model = + let + o = model.org + in + { o | name = model.name + , address = Comp.AddressForm.getAddress model.addressModel + , contacts = Comp.ContactField.getContacts model.contactModel + , notes = model.notes + } + +type Msg + = SetName String + | SetOrg Organization + | AddressMsg Comp.AddressForm.Msg + | ContactMsg Comp.ContactField.Msg + | SetNotes String + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetOrg t -> + let + (m1, c1) = update flags (AddressMsg (Comp.AddressForm.SetAddress t.address)) model + (m2, c2) = update flags (ContactMsg (Comp.ContactField.SetItems t.contacts)) m1 + in + ({m2 | org = t, name = t.name, notes = t.notes }, Cmd.none) + + AddressMsg am -> + let + (m1, c1) = Comp.AddressForm.update am model.addressModel + in + ({model | addressModel = m1}, Cmd.map AddressMsg c1) + + ContactMsg m -> + let + (m1, c1) = Comp.ContactField.update m model.contactModel + in + ({model | contactModel = m1}, Cmd.map ContactMsg c1) + + SetName n -> + ({model | name = n}, Cmd.none) + + SetNotes str -> + ({model | notes = if str == "" then Nothing else Just str}, Cmd.none) + + +view: Model -> Html Msg +view model = + div [class "ui form"] + [div [classList [("field", True) + ,("error", not (isValid model)) + ] + ] + [label [][text "Name*"] + ,input [type_ "text" + ,onInput SetName + ,placeholder "Name" + ,value model.name + ][] + ] + ,h3 [class "ui dividing header"] + [text "Address" + ] + ,Html.map AddressMsg (Comp.AddressForm.view model.addressModel) + ,h3 [class "ui dividing header"] + [text "Contacts" + ] + ,Html.map ContactMsg (Comp.ContactField.view model.contactModel) + ,h3 [class "ui dividing header"] + [text "Notes" + ] + ,div [class "field"] + [textarea [onInput SetNotes][Maybe.withDefault "" model.notes |> text ] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/OrgManage.elm b/modules/webapp/src/main/elm/Comp/OrgManage.elm new file mode 100644 index 00000000..89b45b29 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/OrgManage.elm @@ -0,0 +1,206 @@ +module Comp.OrgManage exposing ( Model + , emptyModel + , Msg(..) + , view + , update) + +import Http +import Api +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick, onSubmit) +import Data.Flags exposing (Flags) +import Comp.OrgTable +import Comp.OrgForm +import Comp.YesNoDimmer +import Api.Model.Organization +import Api.Model.OrganizationList exposing (OrganizationList) +import Api.Model.BasicResult exposing (BasicResult) +import Util.Maybe +import Util.Http + +type alias Model = + { tableModel: Comp.OrgTable.Model + , formModel: Comp.OrgForm.Model + , viewMode: ViewMode + , formError: Maybe String + , loading: Bool + , deleteConfirm: Comp.YesNoDimmer.Model + } + +type ViewMode = Table | Form + +emptyModel: Model +emptyModel = + { tableModel = Comp.OrgTable.emptyModel + , formModel = Comp.OrgForm.emptyModel + , viewMode = Table + , formError = Nothing + , loading = False + , deleteConfirm = Comp.YesNoDimmer.emptyModel + } + +type Msg + = TableMsg Comp.OrgTable.Msg + | FormMsg Comp.OrgForm.Msg + | LoadOrgs + | OrgResp (Result Http.Error OrganizationList) + | SetViewMode ViewMode + | InitNewOrg + | Submit + | SubmitResp (Result Http.Error BasicResult) + | YesNoMsg Comp.YesNoDimmer.Msg + | RequestDelete + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + TableMsg m -> + let + (tm, tc) = Comp.OrgTable.update flags m model.tableModel + (m2, c2) = ({model | tableModel = tm + , viewMode = Maybe.map (\_ -> Form) tm.selected |> Maybe.withDefault Table + , formError = if Util.Maybe.nonEmpty tm.selected then Nothing else model.formError + } + , Cmd.map TableMsg tc + ) + (m3, c3) = case tm.selected of + Just org -> + update flags (FormMsg (Comp.OrgForm.SetOrg org)) m2 + Nothing -> + (m2, Cmd.none) + in + (m3, Cmd.batch [c2, c3]) + + FormMsg m -> + let + (m2, c2) = Comp.OrgForm.update flags m model.formModel + in + ({model | formModel = m2}, Cmd.map FormMsg c2) + + LoadOrgs -> + ({model| loading = True}, Api.getOrganizations flags OrgResp) + + OrgResp (Ok orgs) -> + let + m2 = {model|viewMode = Table, loading = False} + in + update flags (TableMsg (Comp.OrgTable.SetOrgs orgs.items)) m2 + + OrgResp (Err err) -> + ({model|loading = False}, Cmd.none) + + SetViewMode m -> + let + m2 = {model | viewMode = m } + in + case m of + Table -> + update flags (TableMsg Comp.OrgTable.Deselect) m2 + Form -> + (m2, Cmd.none) + + InitNewOrg -> + let + nm = {model | viewMode = Form, formError = Nothing } + org = Api.Model.Organization.empty + in + update flags (FormMsg (Comp.OrgForm.SetOrg org)) nm + + Submit -> + let + org = Comp.OrgForm.getOrg model.formModel + valid = Comp.OrgForm.isValid model.formModel + in if valid then + ({model|loading = True}, Api.postOrg flags org SubmitResp) + else + ({model|formError = Just "Please correct the errors in the form."}, Cmd.none) + + SubmitResp (Ok res) -> + if res.success then + let + (m2, c2) = update flags (SetViewMode Table) model + (m3, c3) = update flags LoadOrgs m2 + in + ({m3|loading = False}, Cmd.batch [c2,c3]) + else + ({model | formError = Just res.message, loading = False }, Cmd.none) + + SubmitResp (Err err) -> + ({model | formError = Just (Util.Http.errorToString err), loading = False}, Cmd.none) + + RequestDelete -> + update flags (YesNoMsg Comp.YesNoDimmer.activate) model + + YesNoMsg m -> + let + (cm, confirmed) = Comp.YesNoDimmer.update m model.deleteConfirm + org = Comp.OrgForm.getOrg model.formModel + cmd = if confirmed then Api.deleteOrg flags org.id SubmitResp else Cmd.none + in + ({model | deleteConfirm = cm}, cmd) + +view: Model -> Html Msg +view model = + if model.viewMode == Table then viewTable model + else viewForm model + +viewTable: Model -> Html Msg +viewTable model = + div [] + [button [class "ui basic button", onClick InitNewOrg] + [i [class "plus icon"][] + ,text "Create new" + ] + ,Html.map TableMsg (Comp.OrgTable.view model.tableModel) + ,div [classList [("ui dimmer", True) + ,("active", model.loading) + ]] + [div [class "ui loader"][] + ] + ] + +viewForm: Model -> Html Msg +viewForm model = + let + newOrg = model.formModel.org.id == "" + in + Html.form [class "ui segment", onSubmit Submit] + [Html.map YesNoMsg (Comp.YesNoDimmer.view model.deleteConfirm) + ,if newOrg then + h3 [class "ui dividing header"] + [text "Create new organization" + ] + else + h3 [class "ui dividing header"] + [text ("Edit org: " ++ model.formModel.org.name) + ,div [class "sub header"] + [text "Id: " + ,text model.formModel.org.id + ] + ] + ,Html.map FormMsg (Comp.OrgForm.view model.formModel) + ,div [classList [("ui error message", True) + ,("invisible", Util.Maybe.isEmpty model.formError) + ] + ] + [Maybe.withDefault "" model.formError |> text + ] + ,div [class "ui horizontal divider"][] + ,button [class "ui primary button", type_ "submit"] + [text "Submit" + ] + ,a [class "ui secondary button", onClick (SetViewMode Table), href ""] + [text "Cancel" + ] + ,if not newOrg then + a [class "ui right floated red button", href "", onClick RequestDelete] + [text "Delete"] + else + span[][] + ,div [classList [("ui dimmer", True) + ,("active", model.loading) + ]] + [div [class "ui loader"][] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/OrgTable.elm b/modules/webapp/src/main/elm/Comp/OrgTable.elm new file mode 100644 index 00000000..85b3ea87 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/OrgTable.elm @@ -0,0 +1,74 @@ +module Comp.OrgTable exposing ( Model + , emptyModel + , Msg(..) + , view + , update) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) +import Data.Flags exposing (Flags) +import Api.Model.Organization exposing (Organization) +import Api.Model.Address exposing (Address) +import Api.Model.Contact exposing (Contact) +import Util.Address +import Util.Contact + +type alias Model = + { equips: List Organization + , selected: Maybe Organization + } + +emptyModel: Model +emptyModel = + { equips = [] + , selected = Nothing + } + +type Msg + = SetOrgs (List Organization) + | Select Organization + | Deselect + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetOrgs list -> + ({model | equips = list, selected = Nothing }, Cmd.none) + + Select equip -> + ({model | selected = Just equip}, Cmd.none) + + Deselect -> + ({model | selected = Nothing}, Cmd.none) + + +view: Model -> Html Msg +view model = + table [class "ui selectable table"] + [thead [] + [tr [] + [th [class "collapsing"][text "Name"] + ,th [][text "Address"] + ,th [][text "Contact"] + ] + ] + ,tbody [] + (List.map (renderOrgLine model) model.equips) + ] + +renderOrgLine: Model -> Organization -> Html Msg +renderOrgLine model org = + tr [classList [("active", model.selected == Just org)] + ,onClick (Select org) + ] + [td [class "collapsing"] + [text org.name + ] + ,td [] + [Util.Address.toString org.address |> text + ] + ,td [] + [Util.Contact.toString org.contacts |> text + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/PersonForm.elm b/modules/webapp/src/main/elm/Comp/PersonForm.elm new file mode 100644 index 00000000..aaa9e77a --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/PersonForm.elm @@ -0,0 +1,129 @@ +module Comp.PersonForm exposing ( Model + , emptyModel + , Msg(..) + , view + , update + , isValid + , getPerson) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onInput, onCheck) +import Data.Flags exposing (Flags) +import Api.Model.Person exposing (Person) +import Comp.AddressForm +import Comp.ContactField +import Comp.YesNoDimmer + +type alias Model = + { org: Person + , name: String + , addressModel: Comp.AddressForm.Model + , contactModel: Comp.ContactField.Model + , notes: Maybe String + , concerning: Bool + } + +emptyModel: Model +emptyModel = + { org = Api.Model.Person.empty + , name = "" + , addressModel = Comp.AddressForm.emptyModel + , contactModel = Comp.ContactField.emptyModel + , notes = Nothing + , concerning = False + } + +isValid: Model -> Bool +isValid model = + model.name /= "" + +getPerson: Model -> Person +getPerson model = + let + o = model.org + in + { o | name = model.name + , address = Comp.AddressForm.getAddress model.addressModel + , contacts = Comp.ContactField.getContacts model.contactModel + , notes = model.notes + , concerning = model.concerning + } + +type Msg + = SetName String + | SetPerson Person + | AddressMsg Comp.AddressForm.Msg + | ContactMsg Comp.ContactField.Msg + | SetNotes String + | SetConcerning Bool + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetPerson t -> + let + (m1, c1) = update flags (AddressMsg (Comp.AddressForm.SetAddress t.address)) model + (m2, c2) = update flags (ContactMsg (Comp.ContactField.SetItems t.contacts)) m1 + in + ({m2 | org = t, name = t.name, notes = t.notes, concerning = t.concerning }, Cmd.none) + + AddressMsg am -> + let + (m1, c1) = Comp.AddressForm.update am model.addressModel + in + ({model | addressModel = m1}, Cmd.map AddressMsg c1) + + ContactMsg m -> + let + (m1, c1) = Comp.ContactField.update m model.contactModel + in + ({model | contactModel = m1}, Cmd.map ContactMsg c1) + + SetName n -> + ({model | name = n}, Cmd.none) + + SetNotes str -> + ({model | notes = if str == "" then Nothing else Just str}, Cmd.none) + + SetConcerning flag -> + ({model | concerning = not model.concerning}, Cmd.none) + + +view: Model -> Html Msg +view model = + div [class "ui form"] + [div [classList [("field", True) + ,("error", not (isValid model)) + ] + ] + [label [][text "Name*"] + ,input [type_ "text" + ,onInput SetName + ,placeholder "Name" + ,value model.name + ][] + ] + ,div [class "inline field"] + [div [class "ui checkbox"] + [input [type_ "checkbox" + , checked model.concerning + , onCheck SetConcerning][] + ,label [][text "Use for concerning person suggestion only"] + ] + ] + ,h3 [class "ui dividing header"] + [text "Address" + ] + ,Html.map AddressMsg (Comp.AddressForm.view model.addressModel) + ,h3 [class "ui dividing header"] + [text "Contacts" + ] + ,Html.map ContactMsg (Comp.ContactField.view model.contactModel) + ,h3 [class "ui dividing header"] + [text "Notes" + ] + ,div [class "field"] + [textarea [onInput SetNotes][Maybe.withDefault "" model.notes |> text ] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/PersonManage.elm b/modules/webapp/src/main/elm/Comp/PersonManage.elm new file mode 100644 index 00000000..add51e1d --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/PersonManage.elm @@ -0,0 +1,207 @@ +module Comp.PersonManage exposing ( Model + , emptyModel + , Msg(..) + , view + , update) + +import Http +import Api +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick, onSubmit) +import Data.Flags exposing (Flags) +import Comp.PersonTable +import Comp.PersonForm +import Comp.YesNoDimmer +import Api.Model.Person +import Api.Model.PersonList exposing (PersonList) +import Api.Model.BasicResult exposing (BasicResult) +import Util.Maybe +import Util.Http + +type alias Model = + { tableModel: Comp.PersonTable.Model + , formModel: Comp.PersonForm.Model + , viewMode: ViewMode + , formError: Maybe String + , loading: Bool + , deleteConfirm: Comp.YesNoDimmer.Model + } + +type ViewMode = Table | Form + +emptyModel: Model +emptyModel = + { tableModel = Comp.PersonTable.emptyModel + , formModel = Comp.PersonForm.emptyModel + , viewMode = Table + , formError = Nothing + , loading = False + , deleteConfirm = Comp.YesNoDimmer.emptyModel + } + +type Msg + = TableMsg Comp.PersonTable.Msg + | FormMsg Comp.PersonForm.Msg + | LoadPersons + | PersonResp (Result Http.Error PersonList) + | SetViewMode ViewMode + | InitNewPerson + | Submit + | SubmitResp (Result Http.Error BasicResult) + | YesNoMsg Comp.YesNoDimmer.Msg + | RequestDelete + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + TableMsg m -> + let + (tm, tc) = Comp.PersonTable.update flags m model.tableModel + (m2, c2) = ({model | tableModel = tm + , viewMode = Maybe.map (\_ -> Form) tm.selected |> Maybe.withDefault Table + , formError = if Util.Maybe.nonEmpty tm.selected then Nothing else model.formError + } + , Cmd.map TableMsg tc + ) + (m3, c3) = case tm.selected of + Just org -> + update flags (FormMsg (Comp.PersonForm.SetPerson org)) m2 + Nothing -> + (m2, Cmd.none) + in + (m3, Cmd.batch [c2, c3]) + + FormMsg m -> + let + (m2, c2) = Comp.PersonForm.update flags m model.formModel + in + ({model | formModel = m2}, Cmd.map FormMsg c2) + + LoadPersons -> + ({model| loading = True}, Api.getPersons flags PersonResp) + + PersonResp (Ok orgs) -> + let + m2 = {model|viewMode = Table, loading = False} + in + update flags (TableMsg (Comp.PersonTable.SetPersons orgs.items)) m2 + + PersonResp (Err err) -> + ({model|loading = False}, Cmd.none) + + SetViewMode m -> + let + m2 = {model | viewMode = m } + in + case m of + Table -> + update flags (TableMsg Comp.PersonTable.Deselect) m2 + Form -> + (m2, Cmd.none) + + InitNewPerson -> + let + nm = {model | viewMode = Form, formError = Nothing } + org = Api.Model.Person.empty + in + update flags (FormMsg (Comp.PersonForm.SetPerson org)) nm + + Submit -> + let + person = Comp.PersonForm.getPerson model.formModel + valid = Comp.PersonForm.isValid model.formModel + in if valid then + ({model|loading = True}, Api.postPerson flags person SubmitResp) + else + ({model|formError = Just "Please correct the errors in the form."}, Cmd.none) + + SubmitResp (Ok res) -> + if res.success then + let + (m2, c2) = update flags (SetViewMode Table) model + (m3, c3) = update flags LoadPersons m2 + in + ({m3|loading = False}, Cmd.batch [c2,c3]) + else + ({model | formError = Just res.message, loading = False }, Cmd.none) + + SubmitResp (Err err) -> + ({model | formError = Just (Util.Http.errorToString err), loading = False}, Cmd.none) + + RequestDelete -> + update flags (YesNoMsg Comp.YesNoDimmer.activate) model + + YesNoMsg m -> + let + (cm, confirmed) = Comp.YesNoDimmer.update m model.deleteConfirm + person = Comp.PersonForm.getPerson model.formModel + cmd = if confirmed then Api.deletePerson flags person.id SubmitResp else Cmd.none + in + ({model | deleteConfirm = cm}, cmd) + + +view: Model -> Html Msg +view model = + if model.viewMode == Table then viewTable model + else viewForm model + +viewTable: Model -> Html Msg +viewTable model = + div [] + [button [class "ui basic button", onClick InitNewPerson] + [i [class "plus icon"][] + ,text "Create new" + ] + ,Html.map TableMsg (Comp.PersonTable.view model.tableModel) + ,div [classList [("ui dimmer", True) + ,("active", model.loading) + ]] + [div [class "ui loader"][] + ] + ] + +viewForm: Model -> Html Msg +viewForm model = + let + newPerson = model.formModel.org.id == "" + in + Html.form [class "ui segment", onSubmit Submit] + [Html.map YesNoMsg (Comp.YesNoDimmer.view model.deleteConfirm) + ,if newPerson then + h3 [class "ui dividing header"] + [text "Create new person" + ] + else + h3 [class "ui dividing header"] + [text ("Edit org: " ++ model.formModel.org.name) + ,div [class "sub header"] + [text "Id: " + ,text model.formModel.org.id + ] + ] + ,Html.map FormMsg (Comp.PersonForm.view model.formModel) + ,div [classList [("ui error message", True) + ,("invisible", Util.Maybe.isEmpty model.formError) + ] + ] + [Maybe.withDefault "" model.formError |> text + ] + ,div [class "ui horizontal divider"][] + ,button [class "ui primary button", type_ "submit"] + [text "Submit" + ] + ,a [class "ui secondary button", onClick (SetViewMode Table), href ""] + [text "Cancel" + ] + ,if not newPerson then + a [class "ui right floated red button", href "", onClick RequestDelete] + [text "Delete"] + else + span[][] + ,div [classList [("ui dimmer", True) + ,("active", model.loading) + ]] + [div [class "ui loader"][] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/PersonTable.elm b/modules/webapp/src/main/elm/Comp/PersonTable.elm new file mode 100644 index 00000000..413bc2ef --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/PersonTable.elm @@ -0,0 +1,81 @@ +module Comp.PersonTable exposing ( Model + , emptyModel + , Msg(..) + , view + , update) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) +import Data.Flags exposing (Flags) +import Api.Model.Person exposing (Person) +import Api.Model.Address exposing (Address) +import Api.Model.Contact exposing (Contact) +import Util.Address +import Util.Contact + +type alias Model = + { equips: List Person + , selected: Maybe Person + } + +emptyModel: Model +emptyModel = + { equips = [] + , selected = Nothing + } + +type Msg + = SetPersons (List Person) + | Select Person + | Deselect + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetPersons list -> + ({model | equips = list, selected = Nothing }, Cmd.none) + + Select equip -> + ({model | selected = Just equip}, Cmd.none) + + Deselect -> + ({model | selected = Nothing}, Cmd.none) + + +view: Model -> Html Msg +view model = + table [class "ui selectable table"] + [thead [] + [tr [] + [th [class "collapsing"][text "Name"] + ,th [class "collapsing"][text "Concerning"] + ,th [][text "Address"] + ,th [][text "Contact"] + ] + ] + ,tbody [] + (List.map (renderPersonLine model) model.equips) + ] + +renderPersonLine: Model -> Person -> Html Msg +renderPersonLine model person = + tr [classList [("active", model.selected == Just person)] + ,onClick (Select person) + ] + [td [class "collapsing"] + [text person.name + ] + ,td [class "collapsing"] + [if person.concerning then + i [class "check square outline icon"][] + else + i [class "minus square outline icon"][] + ] + ,td [] + [Util.Address.toString person.address |> text + ] + ,td [] + [Util.Contact.toString person.contacts |> text + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/SearchMenu.elm b/modules/webapp/src/main/elm/Comp/SearchMenu.elm new file mode 100644 index 00000000..116910c5 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/SearchMenu.elm @@ -0,0 +1,429 @@ +module Comp.SearchMenu exposing ( Model + , emptyModel + , Msg(..) + , update + , NextState + , view + , getItemSearch + ) + +import Http +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onCheck, onInput) +import Data.Direction exposing (Direction) +import Data.Flags exposing (Flags) +import Comp.Dropdown exposing (isDropdownChangeMsg) +import Comp.DatePicker +import DatePicker exposing (DatePicker) +import Api +import Api.Model.IdName exposing (IdName) +import Api.Model.ItemSearch exposing (ItemSearch) +import Api.Model.TagList exposing (TagList) +import Api.Model.Tag exposing (Tag) +import Api.Model.Equipment exposing (Equipment) +import Api.Model.ReferenceList exposing (ReferenceList) +import Api.Model.EquipmentList exposing (EquipmentList) +import Util.Maybe +import Util.Update + +-- Data Model + +type alias Model = + { tagInclModel: Comp.Dropdown.Model Tag + , tagExclModel: Comp.Dropdown.Model Tag + , directionModel: Comp.Dropdown.Model Direction + , orgModel: Comp.Dropdown.Model IdName + , corrPersonModel: Comp.Dropdown.Model IdName + , concPersonModel: Comp.Dropdown.Model IdName + , concEquipmentModel: Comp.Dropdown.Model Equipment + , inboxCheckbox: Bool + , fromDateModel: DatePicker + , fromDate: Maybe Int + , untilDateModel: DatePicker + , untilDate: Maybe Int + , fromDueDateModel: DatePicker + , fromDueDate: Maybe Int + , untilDueDateModel: DatePicker + , untilDueDate: Maybe Int + , nameModel: Maybe String + } + + +emptyModel: Model +emptyModel = + { tagInclModel = makeTagModel + , tagExclModel = makeTagModel + , directionModel = Comp.Dropdown.makeSingleList + { makeOption = \entry -> {value = Data.Direction.toString entry, text = Data.Direction.toString entry} + , options = Data.Direction.all + , placeholder = "Choose a direction…" + , selected = Nothing + } + , orgModel = Comp.Dropdown.makeModel + { multiple = False + , searchable = \n -> n > 5 + , makeOption = \e -> {value = e.id, text = e.name} + , labelColor = \_ -> "" + , placeholder = "Choose an organization" + } + , corrPersonModel = Comp.Dropdown.makeSingle + { makeOption = \e -> {value = e.id, text = e.name} + , placeholder = "Choose a person" + } + , concPersonModel = Comp.Dropdown.makeSingle + { makeOption = \e -> {value = e.id, text = e.name} + , placeholder = "Choose a person" + } + , concEquipmentModel = Comp.Dropdown.makeModel + { multiple = False + , searchable = \n -> n > 5 + , makeOption = \e -> {value = e.id, text = e.name} + , labelColor = \_ -> "" + , placeholder = "Choosa an equipment" + } + , inboxCheckbox = False + , fromDateModel = Comp.DatePicker.emptyModel + , fromDate = Nothing + , untilDateModel = Comp.DatePicker.emptyModel + , untilDate = Nothing + , fromDueDateModel = Comp.DatePicker.emptyModel + , fromDueDate = Nothing + , untilDueDateModel = Comp.DatePicker.emptyModel + , untilDueDate = Nothing + , nameModel = Nothing + } + +type Msg + = Init + | TagIncMsg (Comp.Dropdown.Msg Tag) + | TagExcMsg (Comp.Dropdown.Msg Tag) + | DirectionMsg (Comp.Dropdown.Msg Direction) + | OrgMsg (Comp.Dropdown.Msg IdName) + | CorrPersonMsg (Comp.Dropdown.Msg IdName) + | ConcPersonMsg (Comp.Dropdown.Msg IdName) + | ConcEquipmentMsg (Comp.Dropdown.Msg Equipment) + | FromDateMsg Comp.DatePicker.Msg + | UntilDateMsg Comp.DatePicker.Msg + | FromDueDateMsg Comp.DatePicker.Msg + | UntilDueDateMsg Comp.DatePicker.Msg + | ToggleInbox + | GetTagsResp (Result Http.Error TagList) + | GetOrgResp (Result Http.Error ReferenceList) + | GetEquipResp (Result Http.Error EquipmentList) + | GetPersonResp (Result Http.Error ReferenceList) + | SetName String + + +makeTagModel: Comp.Dropdown.Model Tag +makeTagModel = + Comp.Dropdown.makeModel + { multiple = True + , searchable = \n -> n > 4 + , makeOption = \tag -> { value = tag.id, text = tag.name } + , labelColor = \tag -> if Util.Maybe.nonEmpty tag.category then "basic blue" else "" + , placeholder = "Choose a tag…" + } + +getDirection: Model -> Maybe Direction +getDirection model = + let + selection = Comp.Dropdown.getSelected model.directionModel + in + case selection of + [d] -> Just d + _ -> Nothing + +getItemSearch: Model -> ItemSearch +getItemSearch model = + let e = Api.Model.ItemSearch.empty in + { e | tagsInclude = Comp.Dropdown.getSelected model.tagInclModel |> List.map .id + , tagsExclude = Comp.Dropdown.getSelected model.tagExclModel |> List.map .id + , corrPerson = Comp.Dropdown.getSelected model.corrPersonModel |> List.map .id |> List.head + , corrOrg = Comp.Dropdown.getSelected model.orgModel |> List.map .id |> List.head + , concPerson = Comp.Dropdown.getSelected model.concPersonModel |> List.map .id |> List.head + , concEquip = Comp.Dropdown.getSelected model.concEquipmentModel |> List.map .id |> List.head + , direction = Comp.Dropdown.getSelected model.directionModel |> List.head |> Maybe.map Data.Direction.toString + , inbox = model.inboxCheckbox + , dateFrom = model.fromDate + , dateUntil = model.untilDate + , dueDateFrom = model.fromDueDate + , dueDateUntil = model.untilDueDate + , name = model.nameModel + } + +-- Update + +type alias NextState + = { modelCmd: (Model, Cmd Msg) + , stateChange: Bool + } + +noChange: (Model, Cmd Msg) -> NextState +noChange p = + NextState p False + +update: Flags -> Msg -> Model -> NextState +update flags msg model = + case msg of + Init -> + let + (dp, dpc) = Comp.DatePicker.init + in + noChange ({model|untilDateModel = dp, fromDateModel = dp, untilDueDateModel = dp, fromDueDateModel = dp} + , Cmd.batch + [Api.getTags flags GetTagsResp + ,Api.getOrgLight flags GetOrgResp + ,Api.getEquipments flags GetEquipResp + ,Api.getPersonsLight flags GetPersonResp + ,Cmd.map UntilDateMsg dpc + ,Cmd.map FromDateMsg dpc + ,Cmd.map UntilDueDateMsg dpc + ,Cmd.map FromDueDateMsg dpc + ] + ) + + GetTagsResp (Ok tags) -> + let + tagList = Comp.Dropdown.SetOptions tags.items + in + noChange <| + Util.Update.andThen1 + [ update flags (TagIncMsg tagList) >> .modelCmd + , update flags (TagExcMsg tagList) >> .modelCmd + ] + model + + GetTagsResp (Err err) -> + noChange (model, Cmd.none) + + GetEquipResp (Ok equips) -> + let + opts = Comp.Dropdown.SetOptions equips.items + in + update flags (ConcEquipmentMsg opts) model + + GetEquipResp (Err err) -> + noChange (model, Cmd.none) + + GetOrgResp (Ok orgs) -> + let + opts = Comp.Dropdown.SetOptions orgs.items + in + update flags (OrgMsg opts) model + + GetOrgResp (Err err) -> + noChange (model, Cmd.none) + + GetPersonResp (Ok ps) -> + let + opts = Comp.Dropdown.SetOptions ps.items + in + noChange <| + Util.Update.andThen1 + [ update flags (CorrPersonMsg opts) >> .modelCmd + , update flags (ConcPersonMsg opts) >> .modelCmd + ] + model + + GetPersonResp (Err err) -> + noChange (model, Cmd.none) + + TagIncMsg m -> + let + (m2, c2) = Comp.Dropdown.update m model.tagInclModel + in + NextState ({model|tagInclModel = m2}, Cmd.map TagIncMsg c2) (isDropdownChangeMsg m) + + TagExcMsg m -> + let + (m2, c2) = Comp.Dropdown.update m model.tagExclModel + in + NextState ({model|tagExclModel = m2}, Cmd.map TagExcMsg c2) (isDropdownChangeMsg m) + + DirectionMsg m -> + let + (m2, c2) = Comp.Dropdown.update m model.directionModel + in + NextState ({model|directionModel = m2}, Cmd.map DirectionMsg c2) (isDropdownChangeMsg m) + + OrgMsg m -> + let + (m2, c2) = Comp.Dropdown.update m model.orgModel + in + NextState ({model|orgModel = m2}, Cmd.map OrgMsg c2) (isDropdownChangeMsg m) + + CorrPersonMsg m -> + let + (m2, c2) = Comp.Dropdown.update m model.corrPersonModel + in + NextState ({model|corrPersonModel = m2}, Cmd.map CorrPersonMsg c2) (isDropdownChangeMsg m) + + ConcPersonMsg m -> + let + (m2, c2) = Comp.Dropdown.update m model.concPersonModel + in + NextState ({model|concPersonModel = m2}, Cmd.map ConcPersonMsg c2) (isDropdownChangeMsg m) + + ConcEquipmentMsg m -> + let + (m2, c2) = Comp.Dropdown.update m model.concEquipmentModel + in + NextState ({model|concEquipmentModel = m2}, Cmd.map ConcEquipmentMsg c2) (isDropdownChangeMsg m) + + ToggleInbox -> + let + current = model.inboxCheckbox + in + NextState ({model | inboxCheckbox = not current }, Cmd.none) True + + FromDateMsg m -> + let + (dp, event) = Comp.DatePicker.updateDefault m model.fromDateModel + nextDate = case event of + DatePicker.Picked date -> + Just (Comp.DatePicker.startOfDay date) + _ -> + Nothing + in + NextState ({model|fromDateModel = dp, fromDate = nextDate}, Cmd.none) (model.fromDate /= nextDate) + + UntilDateMsg m -> + let + (dp, event) = Comp.DatePicker.updateDefault m model.untilDateModel + nextDate = case event of + DatePicker.Picked date -> + Just (Comp.DatePicker.endOfDay date) + _ -> + Nothing + in + NextState ({model|untilDateModel = dp, untilDate = nextDate}, Cmd.none) (model.untilDate /= nextDate) + + FromDueDateMsg m -> + let + (dp, event) = Comp.DatePicker.updateDefault m model.fromDueDateModel + nextDate = case event of + DatePicker.Picked date -> + Just (Comp.DatePicker.startOfDay date) + _ -> + Nothing + in + NextState ({model|fromDueDateModel = dp, fromDueDate = nextDate}, Cmd.none) (model.fromDueDate /= nextDate) + + UntilDueDateMsg m -> + let + (dp, event) = Comp.DatePicker.updateDefault m model.untilDueDateModel + nextDate = case event of + DatePicker.Picked date -> + Just (Comp.DatePicker.endOfDay date) + _ -> + Nothing + in + NextState ({model|untilDueDateModel = dp, untilDueDate = nextDate}, Cmd.none) (model.untilDueDate /= nextDate) + + SetName str -> + let + next = if str == "" then Nothing else Just str + in + NextState ({model|nameModel = next}, Cmd.none) (model.nameModel /= next) + + +-- View + + + +view: Model -> Html Msg +view model = + div [class "ui form"] + [div [class "inline field"] + [div [class "ui checkbox"] + [input [type_ "checkbox" + , onCheck (\_ -> ToggleInbox) + , checked model.inboxCheckbox][] + ,label [][text "Only New" + ] + ] + ] + ,div [class "field"] + [label [][text "Name"] + ,input [type_ "text" + ,onInput SetName + ,model.nameModel |> Maybe.withDefault "" |> value + ][] + ,span [class "small-info"] + [text "May contain wildcard " + ,code [][text "*"] + ,text " at beginning or end" + ] + ] + ,div [class "field"] + [label [][text "Direction"] + ,Html.map DirectionMsg (Comp.Dropdown.view model.directionModel) + ] + ,h3 [class "ui header"] + [text "Tags" + ] + ,div [class "field"] + [label [][text "Include (and)"] + ,Html.map TagIncMsg (Comp.Dropdown.view model.tagInclModel) + ] + ,div [class "field"] + [label [][text "Exclude (or)"] + ,Html.map TagExcMsg (Comp.Dropdown.view model.tagExclModel) + ] + ,h3 [class "ui header"] + [ case getDirection model of + Just Data.Direction.Incoming -> text "Sender" + Just Data.Direction.Outgoing -> text "Recipient" + Nothing -> text "Correspondent" + ] + ,div [class "field"] + [label [][text "Organization"] + ,Html.map OrgMsg (Comp.Dropdown.view model.orgModel) + ] + ,div [class "field"] + [label [][text "Person"] + ,Html.map CorrPersonMsg (Comp.Dropdown.view model.corrPersonModel) + ] + ,h3 [class "ui header"] + [text "Concerned" + ] + ,div [class "field"] + [label [][text "Person"] + ,Html.map ConcPersonMsg (Comp.Dropdown.view model.concPersonModel) + ] + ,div [class "field"] + [label [][text "Equipment"] + ,Html.map ConcEquipmentMsg (Comp.Dropdown.view model.concEquipmentModel) + ] + ,h3 [class "ui header"] + [text "Date" + ] + ,div [class "fields"] + [div [class "field"] + [label [][text "From" + ] + ,Html.map FromDateMsg (Comp.DatePicker.viewTimeDefault model.fromDate model.fromDateModel) + ] + ,div [class "field"] + [label [][text "To" + ] + ,Html.map UntilDateMsg (Comp.DatePicker.viewTimeDefault model.untilDate model.untilDateModel) + ] + ] + ,h3 [class "ui header"] + [text "Due Date" + ] + ,div [class "fields"] + [div [class "field"] + [label [][text "Due From" + ] + ,Html.map FromDueDateMsg (Comp.DatePicker.viewTimeDefault model.fromDueDate model.fromDueDateModel) + ] + ,div [class "field"] + [label [][text "Due To" + ] + ,Html.map UntilDueDateMsg (Comp.DatePicker.viewTimeDefault model.untilDueDate model.untilDueDateModel) + ] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/Settings.elm b/modules/webapp/src/main/elm/Comp/Settings.elm new file mode 100644 index 00000000..79a61ff4 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/Settings.elm @@ -0,0 +1,62 @@ +module Comp.Settings exposing (..) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Data.Language exposing (Language) +import Data.Flags exposing (Flags) +import Comp.Dropdown +import Api.Model.CollectiveSettings exposing (CollectiveSettings) + +type alias Model = + { langModel: Comp.Dropdown.Model Language + , initSettings: CollectiveSettings + } + +init: CollectiveSettings -> Model +init settings = + let + lang = Data.Language.fromString settings.language |> Maybe.withDefault Data.Language.German + in + { langModel = Comp.Dropdown.makeSingleList + { makeOption = \l -> { value = Data.Language.toIso3 l, text = Data.Language.toName l } + , placeholder = "" + , options = Data.Language.all + , selected = Just lang + } + , initSettings = settings + } + +getSettings: Model -> CollectiveSettings +getSettings model = + CollectiveSettings + (Comp.Dropdown.getSelected model.langModel + |> List.head + |> Maybe.map Data.Language.toIso3 + |> Maybe.withDefault model.initSettings.language + ) + +type Msg + = LangDropdownMsg (Comp.Dropdown.Msg Language) + + +update: Flags -> Msg -> Model -> (Model, Cmd Msg, Maybe CollectiveSettings) +update flags msg model = + case msg of + LangDropdownMsg m -> + let + (m2, c2) = Comp.Dropdown.update m model.langModel + nextModel = {model|langModel = m2} + nextSettings = if Comp.Dropdown.isDropdownChangeMsg m then Just (getSettings nextModel) + else Nothing + in + (nextModel, Cmd.map LangDropdownMsg c2, nextSettings) + + +view: Model -> Html Msg +view model = + div [class "ui form"] + [div [class "field"] + [label [][text "Document Language"] + ,Html.map LangDropdownMsg (Comp.Dropdown.view model.langModel) + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/SourceForm.elm b/modules/webapp/src/main/elm/Comp/SourceForm.elm new file mode 100644 index 00000000..2f59d1a4 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/SourceForm.elm @@ -0,0 +1,168 @@ +module Comp.SourceForm exposing ( Model + , emptyModel + , Msg(..) + , view + , update + , isValid + , getSource) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onInput, onCheck) +import Data.Flags exposing (Flags) +import Data.SourceState exposing (SourceState) +import Data.Priority exposing (Priority) +import Comp.Dropdown +import Api.Model.Source exposing (Source) +import Util.Maybe + +type alias Model = + { source: Source + , abbrev: String + , description: Maybe String + , priority: Comp.Dropdown.Model Priority + , enabled: Bool + } + +emptyModel: Model +emptyModel = + { source = Api.Model.Source.empty + , abbrev = "" + , description = Nothing + , priority = Comp.Dropdown.makeSingleList + { makeOption = \p -> { text = Data.Priority.toName p, value = Data.Priority.toName p } + , placeholder = "" + , options = Data.Priority.all + , selected = Nothing + } + , enabled = False + } + +isValid: Model -> Bool +isValid model = + model.abbrev /= "" + +getSource: Model -> Source +getSource model = + let + s = model.source + in + {s | abbrev = model.abbrev + , description = model.description + , enabled = model.enabled + , priority = Comp.Dropdown.getSelected model.priority + |> List.head + |> Maybe.map Data.Priority.toName + |> Maybe.withDefault s.priority + } + + +type Msg + = SetAbbrev String + | SetSource Source + | SetDescr String + | ToggleEnabled + | PrioDropdownMsg (Comp.Dropdown.Msg Priority) + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetSource t -> + let + post = model.source + np = {post | id = t.id + , abbrev = t.abbrev + , description = t.description + , priority = t.priority + , enabled = t.enabled + } + in + ({model | source = np + , abbrev = t.abbrev + , description = t.description + , priority = Comp.Dropdown.makeSingleList + { makeOption = \p -> { text = Data.Priority.toName p, value = Data.Priority.toName p } + , placeholder = "" + , options = Data.Priority.all + , selected = Data.Priority.fromString t.priority + } + , enabled = t.enabled }, Cmd.none) + + ToggleEnabled -> + let + _ = Debug.log "got" model.enabled + in + ({model | enabled = not model.enabled}, Cmd.none) + + SetAbbrev n -> + ({model | abbrev = n}, Cmd.none) + + SetDescr d -> + ({model | description = if d /= "" then Just d else Nothing }, Cmd.none) + + PrioDropdownMsg m -> + let + (m2, c2) = Comp.Dropdown.update m model.priority + in + ({model | priority = m2 }, Cmd.map PrioDropdownMsg c2) + +view: Flags -> Model -> Html Msg +view flags model = + div [class "ui form"] + [div [classList [("field", True) + ,("error", not (isValid model)) + ] + ] + [label [][text "Abbrev*"] + ,input [type_ "text" + ,onInput SetAbbrev + ,placeholder "Abbrev" + ,value model.abbrev + ][] + ] + ,div [class "field"] + [label [][text "Description"] + ,textarea [onInput SetDescr][model.description |> Maybe.withDefault "" |> text] + ] + ,div [class "inline field"] + [div [class "ui checkbox"] + [input [type_ "checkbox" + , onCheck (\_ -> ToggleEnabled) + , checked model.enabled][] + ,label [][text "Enabled"] + ] + ] + ,div [class "field"] + [label [][text "Priority"] + ,Html.map PrioDropdownMsg (Comp.Dropdown.view model.priority) + ] + ,urlInfoMessage flags model + ] + +urlInfoMessage: Flags -> Model -> Html Msg +urlInfoMessage flags model = + div [classList [("ui info icon message", True) + ,("hidden", not model.enabled || model.source.id == "") + ]] + [i [class "info icon"][] + ,div [class "content"] + [div [class "header"] + [text "Public Uploads" + ] + ,p [][text "This source defines URLs that can be used by anyone to send files to " + ,text "you. There is a web page that you can share or tha API url can be used " + ,text "with other clients." + ] + ,dl [class "ui list"] + [dt [][text "Public Upload Page"] + ,dd [][let + url = flags.config.baseUrl ++ "/app/index.html#/upload/" ++ model.source.id + in + a [href url, target "_blank"][code [][text url]] + ] + ,dt [][text "Public API Upload URL"] + ,dd [][code [][text (flags.config.baseUrl ++ "/api/v1/open/upload/item/" ++ model.source.id)] + ] + ] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/SourceManage.elm b/modules/webapp/src/main/elm/Comp/SourceManage.elm new file mode 100644 index 00000000..f12cfa63 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/SourceManage.elm @@ -0,0 +1,207 @@ +module Comp.SourceManage exposing ( Model + , emptyModel + , Msg(..) + , view + , update) + +import Http +import Api +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick, onSubmit) +import Data.Flags exposing (Flags) +import Comp.SourceTable +import Comp.SourceForm +import Comp.YesNoDimmer +import Api.Model.Source +import Api.Model.SourceList exposing (SourceList) +import Api.Model.BasicResult exposing (BasicResult) +import Util.Maybe +import Util.Http + +type alias Model = + { tableModel: Comp.SourceTable.Model + , formModel: Comp.SourceForm.Model + , viewMode: ViewMode + , formError: Maybe String + , loading: Bool + , deleteConfirm: Comp.YesNoDimmer.Model + } + +type ViewMode = Table | Form + +emptyModel: Model +emptyModel = + { tableModel = Comp.SourceTable.emptyModel + , formModel = Comp.SourceForm.emptyModel + , viewMode = Table + , formError = Nothing + , loading = False + , deleteConfirm = Comp.YesNoDimmer.emptyModel + } + +type Msg + = TableMsg Comp.SourceTable.Msg + | FormMsg Comp.SourceForm.Msg + | LoadSources + | SourceResp (Result Http.Error SourceList) + | SetViewMode ViewMode + | InitNewSource + | Submit + | SubmitResp (Result Http.Error BasicResult) + | YesNoMsg Comp.YesNoDimmer.Msg + | RequestDelete + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + TableMsg m -> + let + (tm, tc) = Comp.SourceTable.update flags m model.tableModel + (m2, c2) = ({model | tableModel = tm + , viewMode = Maybe.map (\_ -> Form) tm.selected |> Maybe.withDefault Table + , formError = if Util.Maybe.nonEmpty tm.selected then Nothing else model.formError + } + , Cmd.map TableMsg tc + ) + (m3, c3) = case tm.selected of + Just source -> + update flags (FormMsg (Comp.SourceForm.SetSource source)) m2 + Nothing -> + (m2, Cmd.none) + in + (m3, Cmd.batch [c2, c3]) + + FormMsg m -> + let + (m2, c2) = Comp.SourceForm.update flags m model.formModel + in + ({model | formModel = m2}, Cmd.map FormMsg c2) + + LoadSources -> + ({model| loading = True}, Api.getSources flags SourceResp) + + SourceResp (Ok sources) -> + let + m2 = {model|viewMode = Table, loading = False} + in + update flags (TableMsg (Comp.SourceTable.SetSources sources.items)) m2 + + SourceResp (Err err) -> + ({model|loading = False}, Cmd.none) + + SetViewMode m -> + let + m2 = {model | viewMode = m } + in + case m of + Table -> + update flags (TableMsg Comp.SourceTable.Deselect) m2 + Form -> + (m2, Cmd.none) + + InitNewSource -> + let + nm = {model | viewMode = Form, formError = Nothing } + source = Api.Model.Source.empty + in + update flags (FormMsg (Comp.SourceForm.SetSource source)) nm + + Submit -> + let + source = Comp.SourceForm.getSource model.formModel + valid = Comp.SourceForm.isValid model.formModel + in if valid then + ({model|loading = True}, Api.postSource flags source SubmitResp) + else + ({model|formError = Just "Please correct the errors in the form."}, Cmd.none) + + SubmitResp (Ok res) -> + if res.success then + let + (m2, c2) = update flags (SetViewMode Table) model + (m3, c3) = update flags LoadSources m2 + in + ({m3|loading = False}, Cmd.batch [c2,c3]) + else + ({model | formError = Just res.message, loading = False }, Cmd.none) + + SubmitResp (Err err) -> + ({model | formError = Just (Util.Http.errorToString err), loading = False}, Cmd.none) + + RequestDelete -> + update flags (YesNoMsg Comp.YesNoDimmer.activate) model + + YesNoMsg m -> + let + (cm, confirmed) = Comp.YesNoDimmer.update m model.deleteConfirm + src = Comp.SourceForm.getSource model.formModel + cmd = if confirmed then Api.deleteSource flags src.id SubmitResp else Cmd.none + in + ({model | deleteConfirm = cm}, cmd) + +view: Flags -> Model -> Html Msg +view flags model = + if model.viewMode == Table then viewTable model + else div [](viewForm flags model) + +viewTable: Model -> Html Msg +viewTable model = + div [] + [button [class "ui basic button", onClick InitNewSource] + [i [class "plus icon"][] + ,text "Create new" + ] + ,Html.map TableMsg (Comp.SourceTable.view model.tableModel) + ,div [classList [("ui dimmer", True) + ,("active", model.loading) + ]] + [div [class "ui loader"][] + ] + ] + +viewForm: Flags -> Model -> List (Html Msg) +viewForm flags model = + let + newSource = model.formModel.source.id == "" + in + [if newSource then + h3 [class "ui top attached header"] + [text "Create new source" + ] + else + h3 [class "ui top attached header"] + [text ("Edit: " ++ model.formModel.source.abbrev) + ,div [class "sub header"] + [text "Id: " + ,text model.formModel.source.id + ] + ] + ,Html.form [class "ui attached segment", onSubmit Submit] + [Html.map YesNoMsg (Comp.YesNoDimmer.view model.deleteConfirm) + ,Html.map FormMsg (Comp.SourceForm.view flags model.formModel) + ,div [classList [("ui error message", True) + ,("invisible", Util.Maybe.isEmpty model.formError) + ] + ] + [Maybe.withDefault "" model.formError |> text + ] + ,div [class "ui horizontal divider"][] + ,button [class "ui primary button", type_ "submit"] + [text "Submit" + ] + ,a [class "ui secondary button", onClick (SetViewMode Table), href ""] + [text "Cancel" + ] + ,if not newSource then + a [class "ui right floated red button", href "", onClick RequestDelete] + [text "Delete"] + else + span[][] + ,div [classList [("ui dimmer", True) + ,("active", model.loading) + ]] + [div [class "ui loader"][] + ] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/SourceTable.elm b/modules/webapp/src/main/elm/Comp/SourceTable.elm new file mode 100644 index 00000000..89e98852 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/SourceTable.elm @@ -0,0 +1,85 @@ +module Comp.SourceTable exposing ( Model + , emptyModel + , Msg(..) + , view + , update) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) +import Data.Flags exposing (Flags) +import Data.Priority exposing (Priority) +import Api.Model.Source exposing (Source) + +type alias Model = + { sources: List Source + , selected: Maybe Source + } + +emptyModel: Model +emptyModel = + { sources = [] + , selected = Nothing + } + +type Msg + = SetSources (List Source) + | Select Source + | Deselect + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetSources list -> + ({model | sources = list, selected = Nothing }, Cmd.none) + + Select source -> + ({model | selected = Just source}, Cmd.none) + + Deselect -> + ({model | selected = Nothing}, Cmd.none) + + +view: Model -> Html Msg +view model = + table [class "ui selectable table"] + [thead [] + [tr [] + [th [class "collapsing"][text "Abbrev"] + ,th [class "collapsing"][text "Enabled"] + ,th [class "collapsing"][text "Counter"] + ,th [class "collapsing"][text "Priority"] + ,th [][text "Id"] + ] + ] + ,tbody [] + (List.map (renderSourceLine model) model.sources) + ] + +renderSourceLine: Model -> Source -> Html Msg +renderSourceLine model source = + tr [classList [("active", model.selected == Just source)] + ,onClick (Select source) + ] + [td [class "collapsing"] + [text source.abbrev + ] + ,td [class "collapsing"] + [if source.enabled then + i [class "check square outline icon"][] + else + i [class "minus square outline icon"][] + ] + ,td [class "collapsing"] + [source.counter |> String.fromInt |> text + ] + ,td [class "collapsing"] + [Data.Priority.fromString source.priority + |> Maybe.map Data.Priority.toName + |> Maybe.withDefault source.priority + |> text + ] + ,td [] + [text source.id + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/TagForm.elm b/modules/webapp/src/main/elm/Comp/TagForm.elm new file mode 100644 index 00000000..56a7d4c5 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/TagForm.elm @@ -0,0 +1,76 @@ +module Comp.TagForm exposing ( Model + , emptyModel + , Msg(..) + , view + , update + , isValid + , getTag) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onInput) +import Data.Flags exposing (Flags) +import Api.Model.Tag exposing (Tag) + +type alias Model = + { tag: Tag + , name: String + , category: Maybe String + } + +emptyModel: Model +emptyModel = + { tag = Api.Model.Tag.empty + , name = "" + , category = Nothing + } + +isValid: Model -> Bool +isValid model = + model.name /= "" + +getTag: Model -> Tag +getTag model = + Tag model.tag.id model.name model.category 0 + +type Msg + = SetName String + | SetCategory String + | SetTag Tag + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetTag t -> + ({model | tag = t, name = t.name, category = t.category }, Cmd.none) + + SetName n -> + ({model | name = n}, Cmd.none) + + SetCategory n -> + ({model | category = Just n}, Cmd.none) + + +view: Model -> Html Msg +view model = + div [class "ui form"] + [div [classList [("field", True) + ,("error", not (isValid model)) + ] + ] + [label [][text "Name*"] + ,input [type_ "text" + ,onInput SetName + ,placeholder "Name" + ,value model.name + ][] + ] + ,div [class "field"] + [label [][text "Category"] + ,input [type_ "text" + ,onInput SetCategory + ,placeholder "Category (optional)" + ,value (Maybe.withDefault "" model.category) + ][] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/TagManage.elm b/modules/webapp/src/main/elm/Comp/TagManage.elm new file mode 100644 index 00000000..42fbf9c8 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/TagManage.elm @@ -0,0 +1,206 @@ +module Comp.TagManage exposing ( Model + , emptyModel + , Msg(..) + , view + , update) + +import Http +import Api +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick, onSubmit) +import Data.Flags exposing (Flags) +import Comp.TagTable +import Comp.TagForm +import Comp.YesNoDimmer +import Api.Model.Tag +import Api.Model.TagList exposing (TagList) +import Api.Model.BasicResult exposing (BasicResult) +import Util.Maybe +import Util.Http + +type alias Model = + { tagTableModel: Comp.TagTable.Model + , tagFormModel: Comp.TagForm.Model + , viewMode: ViewMode + , formError: Maybe String + , loading: Bool + , deleteConfirm: Comp.YesNoDimmer.Model + } + +type ViewMode = Table | Form + +emptyModel: Model +emptyModel = + { tagTableModel = Comp.TagTable.emptyModel + , tagFormModel = Comp.TagForm.emptyModel + , viewMode = Table + , formError = Nothing + , loading = False + , deleteConfirm = Comp.YesNoDimmer.emptyModel + } + +type Msg + = TableMsg Comp.TagTable.Msg + | FormMsg Comp.TagForm.Msg + | LoadTags + | TagResp (Result Http.Error TagList) + | SetViewMode ViewMode + | InitNewTag + | Submit + | SubmitResp (Result Http.Error BasicResult) + | YesNoMsg Comp.YesNoDimmer.Msg + | RequestDelete + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + TableMsg m -> + let + (tm, tc) = Comp.TagTable.update flags m model.tagTableModel + (m2, c2) = ({model | tagTableModel = tm + , viewMode = Maybe.map (\_ -> Form) tm.selected |> Maybe.withDefault Table + , formError = if Util.Maybe.nonEmpty tm.selected then Nothing else model.formError + } + , Cmd.map TableMsg tc + ) + (m3, c3) = case tm.selected of + Just tag -> + update flags (FormMsg (Comp.TagForm.SetTag tag)) m2 + Nothing -> + (m2, Cmd.none) + in + (m3, Cmd.batch [c2, c3]) + + FormMsg m -> + let + (m2, c2) = Comp.TagForm.update flags m model.tagFormModel + in + ({model | tagFormModel = m2}, Cmd.map FormMsg c2) + + LoadTags -> + ({model| loading = True}, Api.getTags flags TagResp) + + TagResp (Ok tags) -> + let + m2 = {model|viewMode = Table, loading = False} + in + update flags (TableMsg (Comp.TagTable.SetTags tags.items)) m2 + + TagResp (Err err) -> + ({model|loading = False}, Cmd.none) + + SetViewMode m -> + let + m2 = {model | viewMode = m } + in + case m of + Table -> + update flags (TableMsg Comp.TagTable.Deselect) m2 + Form -> + (m2, Cmd.none) + + InitNewTag -> + let + nm = {model | viewMode = Form, formError = Nothing } + tag = Api.Model.Tag.empty + in + update flags (FormMsg (Comp.TagForm.SetTag tag)) nm + + Submit -> + let + tag = Comp.TagForm.getTag model.tagFormModel + valid = Comp.TagForm.isValid model.tagFormModel + in if valid then + ({model|loading = True}, Api.postTag flags tag SubmitResp) + else + ({model|formError = Just "Please correct the errors in the form."}, Cmd.none) + + SubmitResp (Ok res) -> + if res.success then + let + (m2, c2) = update flags (SetViewMode Table) model + (m3, c3) = update flags LoadTags m2 + in + ({m3|loading = False}, Cmd.batch [c2,c3]) + else + ({model | formError = Just res.message, loading = False }, Cmd.none) + + SubmitResp (Err err) -> + ({model | formError = Just (Util.Http.errorToString err), loading = False}, Cmd.none) + + RequestDelete -> + update flags (YesNoMsg Comp.YesNoDimmer.activate) model + + YesNoMsg m -> + let + (cm, confirmed) = Comp.YesNoDimmer.update m model.deleteConfirm + tag = Comp.TagForm.getTag model.tagFormModel + cmd = if confirmed then Api.deleteTag flags tag.id SubmitResp else Cmd.none + in + ({model | deleteConfirm = cm}, cmd) + +view: Model -> Html Msg +view model = + if model.viewMode == Table then viewTable model + else viewForm model + +viewTable: Model -> Html Msg +viewTable model = + div [] + [button [class "ui basic button", onClick InitNewTag] + [i [class "plus icon"][] + ,text "Create new" + ] + ,Html.map TableMsg (Comp.TagTable.view model.tagTableModel) + ,div [classList [("ui dimmer", True) + ,("active", model.loading) + ]] + [div [class "ui loader"][] + ] + ] + +viewForm: Model -> Html Msg +viewForm model = + let + newTag = model.tagFormModel.tag.id == "" + in + Html.form [class "ui segment", onSubmit Submit] + [Html.map YesNoMsg (Comp.YesNoDimmer.view model.deleteConfirm) + ,if newTag then + h3 [class "ui dividing header"] + [text "Create new tag" + ] + else + h3 [class "ui dividing header"] + [text ("Edit tag: " ++ model.tagFormModel.tag.name) + ,div [class "sub header"] + [text "Id: " + ,text model.tagFormModel.tag.id + ] + ] + ,Html.map FormMsg (Comp.TagForm.view model.tagFormModel) + ,div [classList [("ui error message", True) + ,("invisible", Util.Maybe.isEmpty model.formError) + ] + ] + [Maybe.withDefault "" model.formError |> text + ] + ,div [class "ui horizontal divider"][] + ,button [class "ui primary button", type_ "submit"] + [text "Submit" + ] + ,a [class "ui secondary button", onClick (SetViewMode Table), href ""] + [text "Cancel" + ] + ,if not newTag then + a [class "ui right floated red button", href "", onClick RequestDelete] + [text "Delete"] + else + span[][] + ,div [classList [("ui dimmer", True) + ,("active", model.loading) + ]] + [div [class "ui loader"][] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/TagTable.elm b/modules/webapp/src/main/elm/Comp/TagTable.elm new file mode 100644 index 00000000..bddf60d1 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/TagTable.elm @@ -0,0 +1,66 @@ +module Comp.TagTable exposing ( Model + , emptyModel + , Msg(..) + , view + , update) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) +import Data.Flags exposing (Flags) +import Api.Model.Tag exposing (Tag) + +type alias Model = + { tags: List Tag + , selected: Maybe Tag + } + +emptyModel: Model +emptyModel = + { tags = [] + , selected = Nothing + } + +type Msg + = SetTags (List Tag) + | Select Tag + | Deselect + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetTags list -> + ({model | tags = list, selected = Nothing }, Cmd.none) + + Select tag -> + ({model | selected = Just tag}, Cmd.none) + + Deselect -> + ({model | selected = Nothing}, Cmd.none) + + +view: Model -> Html Msg +view model = + table [class "ui selectable table"] + [thead [] + [tr [] + [th [][text "Name"] + ,th [][text "Category"] + ] + ] + ,tbody [] + (List.map (renderTagLine model) model.tags) + ] + +renderTagLine: Model -> Tag -> Html Msg +renderTagLine model tag = + tr [classList [("active", model.selected == Just tag)] + ,onClick (Select tag) + ] + [td [] + [text tag.name + ] + ,td [] + [Maybe.withDefault "-" tag.category |> text + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/UserForm.elm b/modules/webapp/src/main/elm/Comp/UserForm.elm new file mode 100644 index 00000000..b6ba4529 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/UserForm.elm @@ -0,0 +1,150 @@ +module Comp.UserForm exposing ( Model + , emptyModel + , Msg(..) + , view + , update + , isValid + , isNewUser + , getUser) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onInput, onCheck) +import Data.Flags exposing (Flags) +import Data.UserState exposing (UserState) +import Api.Model.User exposing (User) +import Util.Maybe +import Comp.Dropdown + +type alias Model = + { user: User + , login: String + , email: Maybe String + , state: Comp.Dropdown.Model UserState + , password: Maybe String + } + +emptyModel: Model +emptyModel = + { user = Api.Model.User.empty + , login = "" + , email = Nothing + , password = Nothing + , state = Comp.Dropdown.makeSingleList + { makeOption = \s -> { value = Data.UserState.toString s, text = Data.UserState.toString s } + , placeholder = "" + , options = Data.UserState.all + , selected = List.head Data.UserState.all + } + } + +isValid: Model -> Bool +isValid model = + if model.user.login == "" then + model.login /= "" && Util.Maybe.nonEmpty model.password + else + True + +isNewUser: Model -> Bool +isNewUser model = + model.user.login == "" + +getUser: Model -> User +getUser model = + let + s = model.user + state = Comp.Dropdown.getSelected model.state + |> List.head + |> Maybe.withDefault Data.UserState.Active + |> Data.UserState.toString + in + {s | login = model.login + , email = model.email + , state = state + , password = model.password + } + + +type Msg + = SetLogin String + | SetUser User + | SetEmail String + | StateMsg (Comp.Dropdown.Msg UserState) + | SetPassword String + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetUser t -> + let + state = Comp.Dropdown.makeSingleList + { makeOption = \s -> { value = Data.UserState.toString s, text = Data.UserState.toString s } + , placeholder = "" + , options = Data.UserState.all + , selected = Data.UserState.fromString t.state + |> Maybe.map (\u -> List.filter ((==) u) Data.UserState.all) + |> Maybe.andThen List.head + |> Util.Maybe.withDefault (List.head Data.UserState.all) + } + in + ({model | user = t + , login = t.login + , email = t.email + , password = t.password + , state = state }, Cmd.none) + + StateMsg m -> + let + (m1, c1) = Comp.Dropdown.update m model.state + in + ({model | state = m1}, Cmd.map StateMsg c1) + + SetLogin n -> + ({model | login = n}, Cmd.none) + + SetEmail e -> + ({model | email = if e == "" then Nothing else Just e }, Cmd.none) + + SetPassword p -> + ({model | password = if p == "" then Nothing else Just p}, Cmd.none) + + +view: Model -> Html Msg +view model = + div [class "ui form"] + [div [classList [("field", True) + ,("error", model.login == "") + ,("invisible", model.user.login /= "") + ] + ] + [label [][text "Login*"] + ,input [type_ "text" + ,onInput SetLogin + ,placeholder "Login" + ,value model.login + ][] + ] + ,div [class "field"] + [label [][text "E-Mail"] + ,input [ onInput SetEmail + , model.email |> Maybe.withDefault "" |> value + , placeholder "E-Mail" + ][] + ] + ,div [class "field"] + [label [][text "State"] + ,Html.map StateMsg (Comp.Dropdown.view model.state) + ] + ,div [classList [("field", True) + ,("invisible", model.user.login /= "") + ,("error", Util.Maybe.isEmpty model.password) + ] + ] + [label [][text "Password*"] + ,input [type_ "text" + , onInput SetPassword + , placeholder "Password" + , Maybe.withDefault "" model.password |> value + ][] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/UserManage.elm b/modules/webapp/src/main/elm/Comp/UserManage.elm new file mode 100644 index 00000000..7a1cb349 --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/UserManage.elm @@ -0,0 +1,205 @@ +module Comp.UserManage exposing ( Model + , emptyModel + , Msg(..) + , view + , update) + +import Http +import Api +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick, onSubmit) +import Data.Flags exposing (Flags) +import Comp.UserTable +import Comp.UserForm +import Comp.YesNoDimmer +import Api.Model.User +import Api.Model.UserList exposing (UserList) +import Api.Model.BasicResult exposing (BasicResult) +import Util.Maybe +import Util.Http + +type alias Model = + { tableModel: Comp.UserTable.Model + , formModel: Comp.UserForm.Model + , viewMode: ViewMode + , formError: Maybe String + , loading: Bool + , deleteConfirm: Comp.YesNoDimmer.Model + } + +type ViewMode = Table | Form + +emptyModel: Model +emptyModel = + { tableModel = Comp.UserTable.emptyModel + , formModel = Comp.UserForm.emptyModel + , viewMode = Table + , formError = Nothing + , loading = False + , deleteConfirm = Comp.YesNoDimmer.emptyModel + } + +type Msg + = TableMsg Comp.UserTable.Msg + | FormMsg Comp.UserForm.Msg + | LoadUsers + | UserResp (Result Http.Error UserList) + | SetViewMode ViewMode + | InitNewUser + | Submit + | SubmitResp (Result Http.Error BasicResult) + | YesNoMsg Comp.YesNoDimmer.Msg + | RequestDelete + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + TableMsg m -> + let + (tm, tc) = Comp.UserTable.update flags m model.tableModel + (m2, c2) = ({model | tableModel = tm + , viewMode = Maybe.map (\_ -> Form) tm.selected |> Maybe.withDefault Table + , formError = if Util.Maybe.nonEmpty tm.selected then Nothing else model.formError + } + , Cmd.map TableMsg tc + ) + (m3, c3) = case tm.selected of + Just user -> + update flags (FormMsg (Comp.UserForm.SetUser user)) m2 + Nothing -> + (m2, Cmd.none) + in + (m3, Cmd.batch [c2, c3]) + + FormMsg m -> + let + (m2, c2) = Comp.UserForm.update flags m model.formModel + in + ({model | formModel = m2}, Cmd.map FormMsg c2) + + LoadUsers -> + ({model| loading = True}, Api.getUsers flags UserResp) + + UserResp (Ok users) -> + let + m2 = {model|viewMode = Table, loading = False} + in + update flags (TableMsg (Comp.UserTable.SetUsers users.items)) m2 + + UserResp (Err err) -> + ({model|loading = False}, Cmd.none) + + SetViewMode m -> + let + m2 = {model | viewMode = m } + in + case m of + Table -> + update flags (TableMsg Comp.UserTable.Deselect) m2 + Form -> + (m2, Cmd.none) + + InitNewUser -> + let + nm = {model | viewMode = Form, formError = Nothing } + user = Api.Model.User.empty + in + update flags (FormMsg (Comp.UserForm.SetUser user)) nm + + Submit -> + let + user = Comp.UserForm.getUser model.formModel + valid = Comp.UserForm.isValid model.formModel + cmd = if Comp.UserForm.isNewUser model.formModel + then Api.postNewUser flags user SubmitResp + else Api.putUser flags user SubmitResp + in if valid then + ({model|loading = True}, cmd) + else + ({model|formError = Just "Please correct the errors in the form."}, Cmd.none) + + SubmitResp (Ok res) -> + if res.success then + let + (m2, c2) = update flags (SetViewMode Table) model + (m3, c3) = update flags LoadUsers m2 + in + ({m3|loading = False}, Cmd.batch [c2,c3]) + else + ({model | formError = Just res.message, loading = False }, Cmd.none) + + SubmitResp (Err err) -> + ({model | formError = Just (Util.Http.errorToString err), loading = False}, Cmd.none) + + RequestDelete -> + update flags (YesNoMsg Comp.YesNoDimmer.activate) model + + YesNoMsg m -> + let + (cm, confirmed) = Comp.YesNoDimmer.update m model.deleteConfirm + user = Comp.UserForm.getUser model.formModel + cmd = if confirmed then Api.deleteUser flags user.login SubmitResp else Cmd.none + in + ({model | deleteConfirm = cm}, cmd) + +view: Model -> Html Msg +view model = + if model.viewMode == Table then viewTable model + else viewForm model + +viewTable: Model -> Html Msg +viewTable model = + div [] + [button [class "ui basic button", onClick InitNewUser] + [i [class "plus icon"][] + ,text "Create new" + ] + ,Html.map TableMsg (Comp.UserTable.view model.tableModel) + ,div [classList [("ui dimmer", True) + ,("active", model.loading) + ]] + [div [class "ui loader"][] + ] + ] + +viewForm: Model -> Html Msg +viewForm model = + let + newUser = Comp.UserForm.isNewUser model.formModel + in + Html.form [class "ui segment", onSubmit Submit] + [Html.map YesNoMsg (Comp.YesNoDimmer.view model.deleteConfirm) + ,if newUser then + h3 [class "ui dividing header"] + [text "Create new user" + ] + else + h3 [class "ui dividing header"] + [text ("Edit user: " ++ model.formModel.user.login) + ] + ,Html.map FormMsg (Comp.UserForm.view model.formModel) + ,div [classList [("ui error message", True) + ,("invisible", Util.Maybe.isEmpty model.formError) + ] + ] + [Maybe.withDefault "" model.formError |> text + ] + ,div [class "ui horizontal divider"][] + ,button [class "ui primary button", type_ "submit"] + [text "Submit" + ] + ,a [class "ui secondary button", onClick (SetViewMode Table), href ""] + [text "Cancel" + ] + ,if not newUser then + a [class "ui right floated red button", href "", onClick RequestDelete] + [text "Delete"] + else + span[][] + ,div [classList [("ui dimmer", True) + ,("active", model.loading) + ]] + [div [class "ui loader"][] + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/UserTable.elm b/modules/webapp/src/main/elm/Comp/UserTable.elm new file mode 100644 index 00000000..fd85f7eb --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/UserTable.elm @@ -0,0 +1,83 @@ +module Comp.UserTable exposing ( Model + , emptyModel + , Msg(..) + , view + , update) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) +import Data.Flags exposing (Flags) +import Api.Model.User exposing (User) +import Util.Time exposing (formatDateTime) + +type alias Model = + { users: List User + , selected: Maybe User + } + +emptyModel: Model +emptyModel = + { users = [] + , selected = Nothing + } + +type Msg + = SetUsers (List User) + | Select User + | Deselect + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetUsers list -> + ({model | users = list, selected = Nothing }, Cmd.none) + + Select user -> + ({model | selected = Just user}, Cmd.none) + + Deselect -> + ({model | selected = Nothing}, Cmd.none) + + +view: Model -> Html Msg +view model = + table [class "ui selectable table"] + [thead [] + [tr [] + [th [class "collapsing"][text "Login"] + ,th [class "collapsing"][text "State"] + ,th [class "collapsing"][text "Email"] + ,th [class "collapsing"][text "Logins"] + ,th [class "collapsing"][text "Last Login"] + ,th [class "collapsing"][text "Created"] + ] + ] + ,tbody [] + (List.map (renderUserLine model) model.users) + ] + +renderUserLine: Model -> User -> Html Msg +renderUserLine model user = + tr [classList [("active", model.selected == Just user)] + ,onClick (Select user) + ] + [td [class "collapsing"] + [text user.login + ] + ,td [class "collapsing"] + [text user.state + ] + ,td [class "collapsing"] + [Maybe.withDefault "" user.email |> text + ] + ,td [class "collapsing"] + [String.fromInt user.loginCount |> text + ] + ,td [class "collapsing"] + [Maybe.map formatDateTime user.lastLogin |> Maybe.withDefault "" |> text + ] + ,td [class "collapsing"] + [formatDateTime user.created |> text + ] + ] diff --git a/modules/webapp/src/main/elm/Comp/YesNoDimmer.elm b/modules/webapp/src/main/elm/Comp/YesNoDimmer.elm new file mode 100644 index 00000000..5c3216db --- /dev/null +++ b/modules/webapp/src/main/elm/Comp/YesNoDimmer.elm @@ -0,0 +1,95 @@ +module Comp.YesNoDimmer exposing ( Model + , Msg(..) + , emptyModel + , update + , view + , view2 + , activate + , disable + , Settings + , defaultSettings + ) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) + +type alias Model = + { active: Bool + } + +emptyModel: Model +emptyModel = + { active = False + } + +type Msg + = Activate + | Disable + | ConfirmDelete + +type alias Settings = + { message: String + , headerIcon: String + , headerClass: String + , confirmButton: String + , cancelButton: String + , invertedDimmer: Bool + } + +defaultSettings: Settings +defaultSettings = + { message = "Delete this item permanently?" + , headerIcon = "exclamation icon" + , headerClass = "ui inverted icon header" + , confirmButton = "Yes, do it!" + , cancelButton = "No" + , invertedDimmer = False + } + + +activate: Msg +activate = Activate + +disable: Msg +disable = Disable + +update: Msg -> Model -> (Model, Bool) +update msg model = + case msg of + Activate -> + ({model | active = True}, False) + Disable -> + ({model | active = False}, False) + ConfirmDelete -> + ({model | active = False}, True) + +view: Model -> Html Msg +view model = + view2 True defaultSettings model + +view2: Bool -> Settings -> Model -> Html Msg +view2 active settings model = + div [classList [("ui dimmer", True) + ,("inverted", settings.invertedDimmer) + ,("active", (active && model.active)) + ] + ] + [div [class "content"] + [h3 [class settings.headerClass] + [if settings.headerIcon == "" then span[][] else i [class settings.headerIcon][] + ,text settings.message + ] + ] + ,div [class "content"] + [div [class "ui buttons"] + [a [class "ui primary button", onClick ConfirmDelete, href ""] + [text settings.confirmButton + ] + ,div [class "or"][] + ,a [class "ui secondary button", onClick Disable, href ""] + [text settings.cancelButton + ] + ] + ] + ] diff --git a/modules/webapp/src/main/elm/Data/ContactType.elm b/modules/webapp/src/main/elm/Data/ContactType.elm new file mode 100644 index 00000000..dc21cd8d --- /dev/null +++ b/modules/webapp/src/main/elm/Data/ContactType.elm @@ -0,0 +1,41 @@ +module Data.ContactType exposing (..) + +type ContactType + = Phone + | Mobile + | Fax + | Email + | Docspell + | Website + + +fromString: String -> Maybe ContactType +fromString str = + case String.toLower str of + "phone" -> Just Phone + "mobile" -> Just Mobile + "fax" -> Just Fax + "email" -> Just Email + "docspell" -> Just Docspell + "website" -> Just Website + _ -> Nothing + +toString: ContactType -> String +toString ct = + case ct of + Phone -> "Phone" + Mobile -> "Mobile" + Fax -> "Fax" + Email -> "Email" + Docspell -> "Docspell" + Website -> "Website" + +all: List ContactType +all = + [ Mobile + , Phone + , Email + , Website + , Fax + , Docspell + ] diff --git a/modules/webapp/src/main/elm/Data/Direction.elm b/modules/webapp/src/main/elm/Data/Direction.elm new file mode 100644 index 00000000..e4f6009e --- /dev/null +++ b/modules/webapp/src/main/elm/Data/Direction.elm @@ -0,0 +1,45 @@ +module Data.Direction exposing (..) + +type Direction + = Incoming + | Outgoing + +fromString: String -> Maybe Direction +fromString str = + case String.toLower str of + "outgoing" -> Just Outgoing + "incoming" -> Just Incoming + _ -> Nothing + +all: List Direction +all = + [ Incoming + , Outgoing + ] + +toString: Direction -> String +toString dir = + case dir of + Incoming -> "Incoming" + Outgoing -> "Outgoing" + +icon: Direction -> String +icon dir = + case dir of + Incoming -> "level down alternate icon" + Outgoing -> "level up alternate icon" + +unknownIcon: String +unknownIcon = + "question circle outline icon" + +iconFromString: String -> String +iconFromString dir = + fromString dir + |> Maybe.map icon + |> Maybe.withDefault unknownIcon + +iconFromMaybe: Maybe String -> String +iconFromMaybe ms = + Maybe.map iconFromString ms + |> Maybe.withDefault unknownIcon diff --git a/modules/webapp/src/main/elm/Data/Flags.elm b/modules/webapp/src/main/elm/Data/Flags.elm index 44ecb038..01bdc9da 100644 --- a/modules/webapp/src/main/elm/Data/Flags.elm +++ b/modules/webapp/src/main/elm/Data/Flags.elm @@ -5,6 +5,7 @@ import Api.Model.AuthResult exposing (AuthResult) type alias Config = { appName: String , baseUrl: String + , signupMode: String } type alias Flags = @@ -20,3 +21,7 @@ getToken flags = withAccount: Flags -> AuthResult -> Flags withAccount flags acc = { flags | account = Just acc } + +withoutAccount: Flags -> Flags +withoutAccount flags = + { flags | account = Nothing } diff --git a/modules/webapp/src/main/elm/Data/Language.elm b/modules/webapp/src/main/elm/Data/Language.elm new file mode 100644 index 00000000..3b29fa22 --- /dev/null +++ b/modules/webapp/src/main/elm/Data/Language.elm @@ -0,0 +1,27 @@ +module Data.Language exposing (..) + +type Language + = German + | English + +fromString: String -> Maybe Language +fromString str = + if str == "deu" || str == "de" || str == "german" then Just German + else if str == "eng" || str == "en" || str == "english" then Just English + else Nothing + +toIso3: Language -> String +toIso3 lang = + case lang of + German -> "deu" + English -> "eng" + +toName: Language -> String +toName lang = + case lang of + German -> "German" + English -> "English" + +all: List Language +all = + [ German, English ] diff --git a/modules/webapp/src/main/elm/Data/Priority.elm b/modules/webapp/src/main/elm/Data/Priority.elm new file mode 100644 index 00000000..290feef4 --- /dev/null +++ b/modules/webapp/src/main/elm/Data/Priority.elm @@ -0,0 +1,25 @@ +module Data.Priority exposing (..) + +type Priority + = High + | Low + +fromString: String -> Maybe Priority +fromString str = + let + s = String.toLower str + in + case s of + "low" -> Just Low + "high" -> Just High + _ -> Nothing + +toName: Priority -> String +toName lang = + case lang of + Low -> "Low" + High-> "High" + +all: List Priority +all = + [ Low, High ] diff --git a/modules/webapp/src/main/elm/Data/SourceState.elm b/modules/webapp/src/main/elm/Data/SourceState.elm new file mode 100644 index 00000000..78dda709 --- /dev/null +++ b/modules/webapp/src/main/elm/Data/SourceState.elm @@ -0,0 +1,24 @@ +module Data.SourceState exposing (..) + +type SourceState + = Active + | Disabled + +fromString: String -> Maybe SourceState +fromString str = + case String.toLower str of + "active" -> Just Active + "disabled" -> Just Disabled + _ -> Nothing + +all: List SourceState +all = + [ Active + , Disabled + ] + +toString: SourceState -> String +toString dir = + case dir of + Active -> "Active" + Disabled -> "Disabled" diff --git a/modules/webapp/src/main/elm/Data/UserState.elm b/modules/webapp/src/main/elm/Data/UserState.elm new file mode 100644 index 00000000..fa1c52a4 --- /dev/null +++ b/modules/webapp/src/main/elm/Data/UserState.elm @@ -0,0 +1,24 @@ +module Data.UserState exposing (..) + +type UserState + = Active + | Disabled + +fromString: String -> Maybe UserState +fromString str = + case String.toLower str of + "active" -> Just Active + "disabled" -> Just Disabled + _ -> Nothing + +all: List UserState +all = + [ Active + , Disabled + ] + +toString: UserState -> String +toString dir = + case dir of + Active -> "Active" + Disabled -> "Disabled" diff --git a/modules/webapp/src/main/elm/Main.elm b/modules/webapp/src/main/elm/Main.elm index 72f5c58c..f8d8089a 100644 --- a/modules/webapp/src/main/elm/Main.elm +++ b/modules/webapp/src/main/elm/Main.elm @@ -9,6 +9,7 @@ import Html.Events exposing (..) import Api import Ports +import Page import Data.Flags exposing (Flags) import App.Data exposing (..) import App.Update exposing (..) @@ -36,7 +37,9 @@ init : Flags -> Url -> Key -> (Model, Cmd Msg) init flags url key = let im = App.Data.init key url flags - (m, cmd) = App.Update.initPage im im.page + page = checkPage flags im.page + (m, cmd) = if im.page == page then App.Update.initPage im page + else (im, Page.goto page) sessionCheck = case m.flags.account of Just acc -> Api.loginSession flags SessionCheckResp @@ -46,8 +49,8 @@ init flags url key = viewDoc: Model -> Document Msg viewDoc model = - { title = model.flags.config.appName - , body = [ (view model) ] + { title = model.flags.config.appName ++ ": " ++ (Page.pageName model.page) + , body = [ (view model) ] } -- SUBSCRIPTIONS @@ -55,4 +58,4 @@ viewDoc model = subscriptions : Model -> Sub Msg subscriptions model = - Sub.none + model.subs diff --git a/modules/webapp/src/main/elm/Page.elm b/modules/webapp/src/main/elm/Page.elm index b1e7bbd3..c088028c 100644 --- a/modules/webapp/src/main/elm/Page.elm +++ b/modules/webapp/src/main/elm/Page.elm @@ -2,25 +2,114 @@ module Page exposing ( Page(..) , href , goto , pageToString + , pageFromString + , pageName + , loginPage + , loginPageReferrer + , uploadId , fromUrl + , isSecured + , isOpen ) import Url exposing (Url) -import Url.Parser as Parser exposing ((), Parser, oneOf, s, string) +import Url.Parser as Parser exposing ((), (), Parser, oneOf, s, string) +import Url.Parser.Query as Query import Html exposing (Attribute) import Html.Attributes as Attr import Browser.Navigation as Nav +import Util.Maybe type Page = HomePage - | LoginPage + | LoginPage (Maybe String) + | ManageDataPage + | CollectiveSettingPage + | UserSettingPage + | QueuePage + | RegisterPage + | UploadPage (Maybe String) + | NewInvitePage +isSecured: Page -> Bool +isSecured page = + case page of + HomePage -> True + LoginPage _ -> False + ManageDataPage -> True + CollectiveSettingPage -> True + UserSettingPage -> True + QueuePage -> True + RegisterPage -> False + NewInvitePage -> False + UploadPage arg -> + Util.Maybe.isEmpty arg + +isOpen: Page -> Bool +isOpen page = + not (isSecured page) + +loginPage: Page -> Page +loginPage p = + case p of + LoginPage _ -> LoginPage Nothing + _ -> LoginPage (Just (pageToString p |> String.dropLeft 2)) + + +pageName: Page -> String +pageName page = + case page of + HomePage -> "Home" + LoginPage _ -> "Login" + ManageDataPage -> "Manage Data" + CollectiveSettingPage -> "Collective Settings" + UserSettingPage -> "User Settings" + QueuePage -> "Processing" + RegisterPage -> "Register" + NewInvitePage -> "New Invite" + UploadPage arg -> + case arg of + Just _ -> "Anonymous Upload" + Nothing -> "Upload" + +loginPageReferrer: Page -> Maybe Page +loginPageReferrer page = + case page of + LoginPage r -> Maybe.andThen pageFromString r + _ -> Nothing + +uploadId: Page -> Maybe String +uploadId page = + case page of + UploadPage id -> id + _ -> Nothing + pageToString: Page -> String pageToString page = case page of HomePage -> "#/home" - LoginPage -> "#/login" + LoginPage referer -> + Maybe.map (\p -> "/" ++ p) referer + |> Maybe.withDefault "" + |> (++) "#/login" + ManageDataPage -> "#/manageData" + CollectiveSettingPage -> "#/collectiveSettings" + UserSettingPage -> "#/userSettings" + QueuePage -> "#/queue" + RegisterPage -> "#/register" + UploadPage sourceId -> + Maybe.map (\id -> "/" ++ id) sourceId + |> Maybe.withDefault "" + |> (++) "#/upload" + NewInvitePage -> "#/newinvite" + +pageFromString: String -> Maybe Page +pageFromString str = + let + url = Url.Url Url.Http "" Nothing str Nothing Nothing + in + Parser.parse parser url href: Page -> Attribute msg href page = @@ -33,9 +122,17 @@ goto page = parser: Parser (Page -> a) a parser = oneOf - [ Parser.map HomePage Parser.top - , Parser.map HomePage (s "home") - , Parser.map LoginPage (s "login") + [ Parser.map HomePage (oneOf [s"", s "home"]) + , Parser.map (\s -> LoginPage (Just s)) (s "login" string) + , Parser.map (LoginPage Nothing) (s "login") + , Parser.map ManageDataPage (s "manageData") + , Parser.map CollectiveSettingPage (s "collectiveSettings") + , Parser.map UserSettingPage (s "userSettings") + , Parser.map QueuePage (s "queue") + , Parser.map RegisterPage (s "register") + , Parser.map (\s -> UploadPage (Just s)) (s "upload" string) + , Parser.map (UploadPage Nothing) (s "upload") + , Parser.map NewInvitePage (s "newinvite") ] fromUrl : Url -> Maybe Page diff --git a/modules/webapp/src/main/elm/Page/CollectiveSettings/Data.elm b/modules/webapp/src/main/elm/Page/CollectiveSettings/Data.elm new file mode 100644 index 00000000..d8689559 --- /dev/null +++ b/modules/webapp/src/main/elm/Page/CollectiveSettings/Data.elm @@ -0,0 +1,45 @@ +module Page.CollectiveSettings.Data exposing (..) + +import Http +import Comp.SourceManage +import Comp.UserManage +import Comp.Settings +import Data.Language +import Api.Model.BasicResult exposing (BasicResult) +import Api.Model.CollectiveSettings exposing (CollectiveSettings) +import Api.Model.ItemInsights exposing (ItemInsights) + +type alias Model = + { currentTab: Maybe Tab + , sourceModel: Comp.SourceManage.Model + , userModel: Comp.UserManage.Model + , settingsModel: Comp.Settings.Model + , insights: ItemInsights + , submitResult: Maybe BasicResult + } + +emptyModel: Model +emptyModel = + { currentTab = Just InsightsTab + , sourceModel = Comp.SourceManage.emptyModel + , userModel = Comp.UserManage.emptyModel + , settingsModel = Comp.Settings.init Api.Model.CollectiveSettings.empty + , insights = Api.Model.ItemInsights.empty + , submitResult = Nothing + } + +type Tab + = SourceTab + | UserTab + | InsightsTab + | SettingsTab + +type Msg + = SetTab Tab + | SourceMsg Comp.SourceManage.Msg + | UserMsg Comp.UserManage.Msg + | SettingsMsg Comp.Settings.Msg + | Init + | GetInsightsResp (Result Http.Error ItemInsights) + | CollectiveSettingsResp (Result Http.Error CollectiveSettings) + | SubmitResp (Result Http.Error BasicResult) diff --git a/modules/webapp/src/main/elm/Page/CollectiveSettings/Update.elm b/modules/webapp/src/main/elm/Page/CollectiveSettings/Update.elm new file mode 100644 index 00000000..90fc6a6d --- /dev/null +++ b/modules/webapp/src/main/elm/Page/CollectiveSettings/Update.elm @@ -0,0 +1,82 @@ +module Page.CollectiveSettings.Update exposing (update) + +import Api +import Api.Model.BasicResult exposing (BasicResult) +import Page.CollectiveSettings.Data exposing (..) +import Data.Flags exposing (Flags) +import Data.Language +import Comp.SourceManage +import Comp.UserManage +import Comp.Settings +import Util.Http + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetTab t -> + let + m = { model | currentTab = Just t } + in + case t of + SourceTab -> + update flags (SourceMsg Comp.SourceManage.LoadSources) m + + UserTab -> + update flags (UserMsg Comp.UserManage.LoadUsers) m + + InsightsTab -> + update flags Init m + + SettingsTab -> + update flags Init m + + SourceMsg m -> + let + (m2, c2) = Comp.SourceManage.update flags m model.sourceModel + in + ({model | sourceModel = m2}, Cmd.map SourceMsg c2) + + UserMsg m -> + let + (m2, c2) = Comp.UserManage.update flags m model.userModel + in + ({model | userModel = m2}, Cmd.map UserMsg c2) + + SettingsMsg m -> + let + (m2, c2, msett) = Comp.Settings.update flags m model.settingsModel + cmd = case msett of + Nothing -> Cmd.none + Just sett -> + Api.setCollectiveSettings flags sett SubmitResp + in + ({model | settingsModel = m2, submitResult = Nothing}, Cmd.batch [cmd, Cmd.map SettingsMsg c2]) + + Init -> + ({model|submitResult = Nothing} + ,Cmd.batch + [ Api.getInsights flags GetInsightsResp + , Api.getCollectiveSettings flags CollectiveSettingsResp + ] + ) + + GetInsightsResp (Ok data) -> + ({model|insights = data}, Cmd.none) + + GetInsightsResp (Err err) -> + (model, Cmd.none) + + CollectiveSettingsResp (Ok data) -> + ({model | settingsModel = Comp.Settings.init data }, Cmd.none) + + CollectiveSettingsResp (Err err) -> + (model, Cmd.none) + + SubmitResp (Ok res) -> + ({model | submitResult = Just res}, Cmd.none) + + SubmitResp (Err err) -> + let + res = BasicResult False (Util.Http.errorToString err) + in + ({model | submitResult = Just res}, Cmd.none) diff --git a/modules/webapp/src/main/elm/Page/CollectiveSettings/View.elm b/modules/webapp/src/main/elm/Page/CollectiveSettings/View.elm new file mode 100644 index 00000000..abac4d5f --- /dev/null +++ b/modules/webapp/src/main/elm/Page/CollectiveSettings/View.elm @@ -0,0 +1,197 @@ +module Page.CollectiveSettings.View exposing (view) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) +import Api.Model.NameCount exposing (NameCount) +import Util.Html exposing (classActive) +import Data.Flags exposing (Flags) +import Page.CollectiveSettings.Data exposing (..) +import Comp.SourceManage +import Comp.UserManage +import Comp.Settings +import Util.Size +import Util.Maybe + +view: Flags -> Model -> Html Msg +view flags model = + div [class "collectivesetting-page ui padded grid"] + [div [class "four wide column"] + [h4 [class "ui top attached ablue-comp header"] + [text "Collective" + ] + ,div [class "ui attached fluid segment"] + [div [class "ui fluid vertical secondary menu"] + [div [classActive (model.currentTab == Just InsightsTab) "link icon item" + ,onClick (SetTab InsightsTab) + ] + [i [class "chart bar outline icon"][] + ,text "Insights" + ] + ,div [classActive (model.currentTab == Just SourceTab) "link icon item" + ,onClick (SetTab SourceTab) + ] + [i [class "upload icon"][] + ,text "Sources" + ] + , div [classActive (model.currentTab == Just SettingsTab) "link icon item" + ,onClick (SetTab SettingsTab) + ] + [i [class "language icon"][] + ,text "Document Language" + ] + ,div [classActive (model.currentTab == Just UserTab) "link icon item" + ,onClick (SetTab UserTab) + ] + [i [class "user icon"][] + ,text "Users" + ] + ] + ] + ] + ,div [class "twelve wide column"] + [div [class ""] + (case model.currentTab of + Just SourceTab -> viewSources flags model + Just UserTab -> viewUsers model + Just InsightsTab -> viewInsights model + Just SettingsTab -> viewSettings model + Nothing -> [] + ) + ] + ] + +viewInsights: Model -> List (Html Msg) +viewInsights model = + [h1 [class "ui header"] + [i [class "chart bar outline icon"][] + ,div [class "content"] + [text "Insights" + ] + ] + ,div [class "ui basic blue segment"] + [h4 [class "ui header"] + [text "Items" + ] + ,div [class "ui statistics"] + [div [class "ui statistic"] + [div [class "value"] + [String.fromInt (model.insights.incomingCount + model.insights.outgoingCount) |> text + ] + ,div [class "label"] + [text "Items" + ] + ] + ,div [class "ui statistic"] + [div [class "value"] + [String.fromInt model.insights.incomingCount |> text + ] + ,div [class "label"] + [text "Incoming" + ] + ] + ,div [class "ui statistic"] + [div [class "value"] + [String.fromInt model.insights.outgoingCount |> text + ] + ,div [class "label"] + [text "Outgoing" + ] + ] + ] + ] + ,div [class "ui basic blue segment"] + [h4 [class "ui header"] + [text "Size" + ] + ,div [class "ui statistics"] + [div [class "ui statistic"] + [div [class "value"] + [toFloat model.insights.itemSize |> Util.Size.bytesReadable Util.Size.B |> text + ] + ,div [class "label"] + [text "Size" + ] + ] + ] + ] + ,div [class "ui basic blue segment"] + [h4 [class "ui header"] + [text "Tags" + ] + ,div [class "ui statistics"] + (List.map makeTagStats model.insights.tagCloud.items) + ] + ] + +makeTagStats: NameCount -> Html Msg +makeTagStats nc = + div [class "ui statistic"] + [div [class "value"] + [String.fromInt nc.count |> text + ] + ,div [class "label"] + [text nc.name + ] + ] + + +viewSources: Flags -> Model -> List (Html Msg) +viewSources flags model = + [h2 [class "ui header"] + [i [class "ui upload icon"][] + ,div [class "content"] + [text "Sources" + ] + ] + ,Html.map SourceMsg (Comp.SourceManage.view flags model.sourceModel) + ] + + +viewUsers: Model -> List (Html Msg) +viewUsers model = + [h2 [class "ui header"] + [i [class "ui user icon"][] + ,div [class "content"] + [text "Users" + ] + ] + ,Html.map UserMsg (Comp.UserManage.view model.userModel) + ] + +viewSettings: Model -> List (Html Msg) +viewSettings model = + [div [class "ui grid"] + [div [class "row"] + [div [class "sixteen wide colum"] + [h2 [class "ui header"] + [i [class "ui language icon"][] + ,div [class "content"] + [text "Document Language" + ] + ] + ] + ] + ,div [class "row"] + [div [class "six wide column"] + [div [class "ui basic segment"] + [text "The language of your documents. This helps text recognition (OCR) and text analysis." + ] + ] + ] + ,div [class "row"] + [div [class "six wide column"] + [Html.map SettingsMsg (Comp.Settings.view model.settingsModel) + ,div [classList [("ui message", True) + ,("hidden", Util.Maybe.isEmpty model.submitResult) + ,("success", Maybe.map .success model.submitResult |> Maybe.withDefault False) + ,("error", Maybe.map .success model.submitResult |> Maybe.map not |> Maybe.withDefault False) + ]] + [Maybe.map .message model.submitResult + |> Maybe.withDefault "" + |> text + ] + ] + ] + ] + ] diff --git a/modules/webapp/src/main/elm/Page/Home/Data.elm b/modules/webapp/src/main/elm/Page/Home/Data.elm index c55df396..5f0d13fc 100644 --- a/modules/webapp/src/main/elm/Page/Home/Data.elm +++ b/modules/webapp/src/main/elm/Page/Home/Data.elm @@ -1,15 +1,36 @@ module Page.Home.Data exposing (..) import Http +import Comp.SearchMenu +import Comp.ItemList +import Comp.ItemDetail +import Api.Model.ItemLightList exposing (ItemLightList) +import Api.Model.ItemDetail exposing (ItemDetail) type alias Model = - { + { searchMenuModel: Comp.SearchMenu.Model + , itemListModel: Comp.ItemList.Model + , searchInProgress: Bool + , itemDetailModel: Comp.ItemDetail.Model + , viewMode: ViewMode } emptyModel: Model emptyModel = - { + { searchMenuModel = Comp.SearchMenu.emptyModel + , itemListModel = Comp.ItemList.emptyModel + , itemDetailModel = Comp.ItemDetail.emptyModel + , searchInProgress = False + , viewMode = Listing } type Msg - = Dummy + = Init + | SearchMenuMsg Comp.SearchMenu.Msg + | ItemListMsg Comp.ItemList.Msg + | ItemSearchResp (Result Http.Error ItemLightList) + | DoSearch + | ItemDetailMsg Comp.ItemDetail.Msg + | ItemDetailResp (Result Http.Error ItemDetail) + +type ViewMode = Listing | Detail diff --git a/modules/webapp/src/main/elm/Page/Home/Update.elm b/modules/webapp/src/main/elm/Page/Home/Update.elm index 6d6c0d87..e30d4b88 100644 --- a/modules/webapp/src/main/elm/Page/Home/Update.elm +++ b/modules/webapp/src/main/elm/Page/Home/Update.elm @@ -3,7 +3,98 @@ module Page.Home.Update exposing (update) import Api import Data.Flags exposing (Flags) import Page.Home.Data exposing (..) +import Comp.SearchMenu +import Comp.ItemList +import Comp.ItemDetail +import Util.Update update: Flags -> Msg -> Model -> (Model, Cmd Msg) update flags msg model = - (model, Cmd.none) + case msg of + Init -> + Util.Update.andThen1 + [ update flags (SearchMenuMsg Comp.SearchMenu.Init) + , update flags (ItemDetailMsg Comp.ItemDetail.Init) + , doSearch flags + ] + model + + SearchMenuMsg m -> + let + nextState = Comp.SearchMenu.update flags m model.searchMenuModel + newModel = {model | searchMenuModel = Tuple.first nextState.modelCmd} + (m2, c2) = if nextState.stateChange then doSearch flags newModel else (newModel, Cmd.none) + in + (m2, Cmd.batch [c2, Cmd.map SearchMenuMsg (Tuple.second nextState.modelCmd)]) + + ItemListMsg m -> + let + (m2, c2, mitem) = Comp.ItemList.update flags m model.itemListModel + cmd = case mitem of + Just item -> + Api.itemDetail flags item.id ItemDetailResp + Nothing -> + Cmd.none + in + ({model | itemListModel = m2}, Cmd.batch [ Cmd.map ItemListMsg c2, cmd ]) + + ItemSearchResp (Ok list) -> + let + m = {model|searchInProgress = False, viewMode = Listing} + in + update flags (ItemListMsg (Comp.ItemList.SetResults list)) m + + ItemSearchResp (Err err) -> + ({model|searchInProgress = False}, Cmd.none) + + DoSearch -> + doSearch flags model + + ItemDetailMsg m -> + let + (m2, c2, nav) = Comp.ItemDetail.update flags m model.itemDetailModel + newModel = {model | itemDetailModel = m2} + newCmd = Cmd.map ItemDetailMsg c2 + in + case nav of + Comp.ItemDetail.NavBack -> + doSearch flags newModel + Comp.ItemDetail.NavPrev -> + case Comp.ItemList.prevItem model.itemListModel m2.item.id of + Just n -> + (newModel, Cmd.batch [newCmd, Api.itemDetail flags n.id ItemDetailResp]) + Nothing -> + (newModel, newCmd) + Comp.ItemDetail.NavNext -> + case Comp.ItemList.nextItem model.itemListModel m2.item.id of + Just n -> + (newModel, Cmd.batch [newCmd, Api.itemDetail flags n.id ItemDetailResp]) + Nothing -> + (newModel, newCmd) + Comp.ItemDetail.NavNextOrBack -> + case Comp.ItemList.nextItem model.itemListModel m2.item.id of + Just n -> + (newModel, Cmd.batch [newCmd, Api.itemDetail flags n.id ItemDetailResp]) + Nothing -> + doSearch flags newModel + Comp.ItemDetail.NavNone -> + (newModel, newCmd) + + ItemDetailResp (Ok item) -> + let + m = {model | viewMode = Detail} + in + update flags (ItemDetailMsg (Comp.ItemDetail.SetItem item)) m + + ItemDetailResp (Err err) -> + let + _ = Debug.log "Error" err + in + (model, Cmd.none) + +doSearch: Flags -> Model -> (Model, Cmd Msg) +doSearch flags model = + let + mask = Comp.SearchMenu.getItemSearch model.searchMenuModel + in + ({model|searchInProgress = True, viewMode = Listing}, Api.itemSearch flags mask ItemSearchResp) diff --git a/modules/webapp/src/main/elm/Page/Home/View.elm b/modules/webapp/src/main/elm/Page/Home/View.elm index 35cff7ff..b75c8e12 100644 --- a/modules/webapp/src/main/elm/Page/Home/View.elm +++ b/modules/webapp/src/main/elm/Page/Home/View.elm @@ -6,18 +6,69 @@ import Html.Events exposing (onClick) import Page exposing (Page(..)) import Page.Home.Data exposing (..) +import Comp.SearchMenu +import Comp.ItemList +import Comp.ItemDetail import Data.Flags +import Util.Html exposing (onClickk) view: Model -> Html Msg view model = - div [class "home-page ui fluid grid"] - [div [class "three wide column"] - [h3 [][text "Menu"] + div [class "home-page ui padded grid"] + [div [class "four wide column"] + [div [class "ui top attached ablue-comp menu"] + [h4 [class "header item"] + [text "Search" + ] + ,div [class "right floated menu"] + [a [class "item" + ,onClick DoSearch + ,href "" + ] + [i [class "ui search icon"][] + ] + ] + ] + ,div [class "ui attached fluid segment"] + [(Html.map SearchMenuMsg (Comp.SearchMenu.view model.searchMenuModel)) + ] ] - ,div [class "seven wide column", style "border-left" "1px solid"] - [h3 [][text "List"] + ,div [class "twelve wide column"] + [case model.viewMode of + Listing -> + if model.searchInProgress then resultPlaceholder + else (Html.map ItemListMsg (Comp.ItemList.view model.itemListModel)) + Detail -> + Html.map ItemDetailMsg (Comp.ItemDetail.view model.itemDetailModel) + ] + ] + +resultPlaceholder: Html Msg +resultPlaceholder = + div [class "ui basic segment"] + [div [class "ui active inverted dimmer"] + [div [class "ui medium text loader"] + [text "Searching …" + ] ] - ,div [class "six wide column", style "border-left" "1px solid", style "height" "100vh"] - [h3 [][text "DocView"] + ,div [class "ui middle aligned very relaxed divided basic list segment"] + [div [class "item"] + [div [class "ui fluid placeholder"] + [div [class "full line"][] + ,div [class "full line"][] + ] + ] + ,div [class "item"] + [div [class "ui fluid placeholder"] + [div [class "full line"][] + ,div [class "full line"][] + ] + ] + ,div [class "item"] + [div [class "ui fluid placeholder"] + [div [class "full line"][] + ,div [class "full line"][] + ] + ] ] ] diff --git a/modules/webapp/src/main/elm/Page/Login/Data.elm b/modules/webapp/src/main/elm/Page/Login/Data.elm index f95ad50f..eb3376ea 100644 --- a/modules/webapp/src/main/elm/Page/Login/Data.elm +++ b/modules/webapp/src/main/elm/Page/Login/Data.elm @@ -1,6 +1,7 @@ module Page.Login.Data exposing (..) import Http +import Page exposing (Page(..)) import Api.Model.AuthResult exposing (AuthResult) type alias Model = @@ -9,8 +10,8 @@ type alias Model = , result: Maybe AuthResult } -empty: Model -empty = +emptyModel: Model +emptyModel = { username = "" , password = "" , result = Nothing diff --git a/modules/webapp/src/main/elm/Page/Login/Update.elm b/modules/webapp/src/main/elm/Page/Login/Update.elm index 41adf63d..4a9d3123 100644 --- a/modules/webapp/src/main/elm/Page/Login/Update.elm +++ b/modules/webapp/src/main/elm/Page/Login/Update.elm @@ -9,8 +9,8 @@ import Api.Model.UserPass exposing (UserPass) import Api.Model.AuthResult exposing (AuthResult) import Util.Http -update: Flags -> Msg -> Model -> (Model, Cmd Msg, Maybe AuthResult) -update flags msg model = +update: Maybe Page -> Flags -> Msg -> Model -> (Model, Cmd Msg, Maybe AuthResult) +update referrer flags msg model = case msg of SetUsername str -> ({model | username = str}, Cmd.none, Nothing) @@ -21,19 +21,22 @@ update flags msg model = (model, Api.login flags (UserPass model.username model.password) AuthResp, Nothing) AuthResp (Ok lr) -> - if lr.success - then ({model|result = Just lr, password = ""}, setAccount lr, Just lr) - else ({model|result = Just lr, password = ""}, Ports.removeAccount "", Just lr) + let + gotoRef = Maybe.withDefault HomePage referrer |> Page.goto + in + if lr.success + then ({model|result = Just lr, password = ""}, Cmd.batch [setAccount lr, gotoRef], Just lr) + else ({model|result = Just lr, password = ""}, Ports.removeAccount (), Just lr) AuthResp (Err err) -> let empty = Api.Model.AuthResult.empty lr = {empty|message = Util.Http.errorToString err} in - ({model|password = "", result = Just lr}, Ports.removeAccount "", Just empty) + ({model|password = "", result = Just lr}, Ports.removeAccount (), Just empty) setAccount: AuthResult -> Cmd msg setAccount result = if result.success then Ports.setAccount result - else Ports.removeAccount "" + else Ports.removeAccount () diff --git a/modules/webapp/src/main/elm/Page/Login/View.elm b/modules/webapp/src/main/elm/Page/Login/View.elm index 2c9efae4..90785ccb 100644 --- a/modules/webapp/src/main/elm/Page/Login/View.elm +++ b/modules/webapp/src/main/elm/Page/Login/View.elm @@ -3,7 +3,7 @@ module Page.Login.View exposing (view) import Html exposing (..) import Html.Attributes exposing (..) import Html.Events exposing (onClick, onInput, onSubmit) - +import Page exposing (Page(..)) import Page.Login.Data exposing (..) view: Model -> Html Msg @@ -11,31 +11,51 @@ view model = div [class "login-page"] [div [class "ui centered grid"] [div [class "row"] - [div [class "eight wide column ui segment login-view"] - [h1 [class "ui dividing header"][text "Sign in to Docspell"] - ,Html.form [class "ui large error form", onSubmit Authenticate] + [div [class "six wide column ui segment login-view"] + [h1 [class "ui center aligned icon header"] + [i [class "umbrella icon"][] + ,div [class "content"] + [text "Sign in to Docspell" + ] + ] + ,Html.form [class "ui large error raised form segment", onSubmit Authenticate] [div [class "field"] [label [][text "Username"] - ,input [type_ "text" - ,onInput SetUsername - ,value model.username - ][] + ,div [class "ui left icon input"] + [input [type_ "text" + ,onInput SetUsername + ,value model.username + ,placeholder "Collective / Login" + ,autofocus True + ][] + ,i [class "user icon"][] + ] ] ,div [class "field"] [label [][text "Password"] - ,input [type_ "password" - ,onInput SetPassword - ,value model.password - ][] + ,div [class "ui left icon input"] + [input [type_ "password" + ,onInput SetPassword + ,value model.password + ,placeholder "Password" + ][] + ,i [class "lock icon"][] + ] ] - ,button [class "ui primary button" + ,button [class "ui primary fluid button" ,type_ "submit" - ,onClick Authenticate ] [text "Login" ] ] ,(resultMessage model) + ,div[class "ui very basic right aligned segment"] + [text "No account? " + ,a [class "ui icon link", Page.href RegisterPage] + [i [class "edit icon"][] + ,text "Sign up!" + ] + ] ] ] ] diff --git a/modules/webapp/src/main/elm/Page/ManageData/Data.elm b/modules/webapp/src/main/elm/Page/ManageData/Data.elm new file mode 100644 index 00000000..bbe1def1 --- /dev/null +++ b/modules/webapp/src/main/elm/Page/ManageData/Data.elm @@ -0,0 +1,36 @@ +module Page.ManageData.Data exposing (..) + +import Comp.TagManage +import Comp.EquipmentManage +import Comp.OrgManage +import Comp.PersonManage + +type alias Model = + { currentTab: Maybe Tab + , tagManageModel: Comp.TagManage.Model + , equipManageModel: Comp.EquipmentManage.Model + , orgManageModel: Comp.OrgManage.Model + , personManageModel: Comp.PersonManage.Model + } + +emptyModel: Model +emptyModel = + { currentTab = Nothing + , tagManageModel = Comp.TagManage.emptyModel + , equipManageModel = Comp.EquipmentManage.emptyModel + , orgManageModel = Comp.OrgManage.emptyModel + , personManageModel = Comp.PersonManage.emptyModel + } + +type Tab + = TagTab + | EquipTab + | OrgTab + | PersonTab + +type Msg + = SetTab Tab + | TagManageMsg Comp.TagManage.Msg + | EquipManageMsg Comp.EquipmentManage.Msg + | OrgManageMsg Comp.OrgManage.Msg + | PersonManageMsg Comp.PersonManage.Msg diff --git a/modules/webapp/src/main/elm/Page/ManageData/Update.elm b/modules/webapp/src/main/elm/Page/ManageData/Update.elm new file mode 100644 index 00000000..51d58daa --- /dev/null +++ b/modules/webapp/src/main/elm/Page/ManageData/Update.elm @@ -0,0 +1,52 @@ +module Page.ManageData.Update exposing (update) + +import Page.ManageData.Data exposing (..) +import Data.Flags exposing (Flags) +import Comp.TagManage +import Comp.EquipmentManage +import Comp.OrgManage +import Comp.PersonManage + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetTab t -> + let + m = { model | currentTab = Just t } + in + case t of + TagTab -> + update flags (TagManageMsg Comp.TagManage.LoadTags) m + + EquipTab -> + update flags (EquipManageMsg Comp.EquipmentManage.LoadEquipments) m + + OrgTab -> + update flags (OrgManageMsg Comp.OrgManage.LoadOrgs) m + + PersonTab -> + update flags (PersonManageMsg Comp.PersonManage.LoadPersons) m + + TagManageMsg m -> + let + (m2, c2) = Comp.TagManage.update flags m model.tagManageModel + in + ({model | tagManageModel = m2}, Cmd.map TagManageMsg c2) + + EquipManageMsg m -> + let + (m2, c2) = Comp.EquipmentManage.update flags m model.equipManageModel + in + ({model | equipManageModel = m2}, Cmd.map EquipManageMsg c2) + + OrgManageMsg m -> + let + (m2, c2) = Comp.OrgManage.update flags m model.orgManageModel + in + ({model | orgManageModel = m2}, Cmd.map OrgManageMsg c2) + + PersonManageMsg m -> + let + (m2, c2) = Comp.PersonManage.update flags m model.personManageModel + in + ({model | personManageModel = m2}, Cmd.map PersonManageMsg c2) diff --git a/modules/webapp/src/main/elm/Page/ManageData/View.elm b/modules/webapp/src/main/elm/Page/ManageData/View.elm new file mode 100644 index 00000000..8eacbbe8 --- /dev/null +++ b/modules/webapp/src/main/elm/Page/ManageData/View.elm @@ -0,0 +1,104 @@ +module Page.ManageData.View exposing (view) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) + +import Util.Html exposing (classActive) +import Page.ManageData.Data exposing (..) +import Comp.TagManage +import Comp.EquipmentManage +import Comp.OrgManage +import Comp.PersonManage + +view: Model -> Html Msg +view model = + div [class "managedata-page ui padded grid"] + [div [class "four wide column"] + [h4 [class "ui top attached ablue-comp header"] + [text "Manage Data" + ] + ,div [class "ui attached fluid segment"] + [div [class "ui fluid vertical secondary menu"] + [div [classActive (model.currentTab == Just TagTab) "link icon item" + ,onClick (SetTab TagTab) + ] + [i [class "tag icon"][] + ,text "Tag" + ] + ,div [classActive (model.currentTab == Just EquipTab) "link icon item" + ,onClick (SetTab EquipTab) + ] + [i [class "box icon"][] + ,text "Equipment" + ] + ,div [classActive (model.currentTab == Just OrgTab) "link icon item" + ,onClick (SetTab OrgTab) + ] + [i [class "factory icon"][] + ,text "Organization" + ] + ,div [classActive (model.currentTab == Just PersonTab) "link icon item" + ,onClick (SetTab PersonTab) + ] + [i [class "user icon"][] + ,text "Person" + ] + ] + ] + ] + ,div [class "twelve wide column"] + [div [class ""] + (case model.currentTab of + Just TagTab -> viewTags model + Just EquipTab -> viewEquip model + Just OrgTab -> viewOrg model + Just PersonTab -> viewPerson model + Nothing -> [] + ) + ] + ] + +viewTags: Model -> List (Html Msg) +viewTags model = + [h2 [class "ui header"] + [i [class "ui tag icon"][] + ,div [class "content"] + [text "Tags" + ] + ] + ,Html.map TagManageMsg (Comp.TagManage.view model.tagManageModel) + ] + +viewEquip: Model -> List (Html Msg) +viewEquip model = + [h2 [class "ui header"] + [i [class "ui box icon"][] + ,div [class "content"] + [text "Equipment" + ] + ] + ,Html.map EquipManageMsg (Comp.EquipmentManage.view model.equipManageModel) + ] + +viewOrg: Model -> List (Html Msg) +viewOrg model = + [h2 [class "ui header"] + [i [class "ui factory icon"][] + ,div [class "content"] + [text "Organizations" + ] + ] + ,Html.map OrgManageMsg (Comp.OrgManage.view model.orgManageModel) + ] + +viewPerson: Model -> List (Html Msg) +viewPerson model = + [h2 [class "ui header"] + [i [class "ui user icon"][] + ,div [class "content"] + [text "Person" + ] + ] + ,Html.map PersonManageMsg (Comp.PersonManage.view model.personManageModel) + ] diff --git a/modules/webapp/src/main/elm/Page/NewInvite/Data.elm b/modules/webapp/src/main/elm/Page/NewInvite/Data.elm new file mode 100644 index 00000000..a9944c41 --- /dev/null +++ b/modules/webapp/src/main/elm/Page/NewInvite/Data.elm @@ -0,0 +1,39 @@ +module Page.NewInvite.Data exposing (..) + +import Http +import Api.Model.InviteResult exposing (InviteResult) + +type alias Model = + { password: String + , result: State + } + +type State + = Empty + | Failed String + | Success InviteResult + + +isFailed: State -> Bool +isFailed state = + case state of + Failed _ -> True + _ -> False + +isSuccess: State -> Bool +isSuccess state = + case state of + Success _ -> True + _ -> False + +emptyModel: Model +emptyModel = + { password = "" + , result = Empty + } + +type Msg + = SetPassword String + | GenerateInvite + | Reset + | InviteResp (Result Http.Error InviteResult) diff --git a/modules/webapp/src/main/elm/Page/NewInvite/Update.elm b/modules/webapp/src/main/elm/Page/NewInvite/Update.elm new file mode 100644 index 00000000..508b9b48 --- /dev/null +++ b/modules/webapp/src/main/elm/Page/NewInvite/Update.elm @@ -0,0 +1,27 @@ +module Page.NewInvite.Update exposing (update) + +import Api +import Data.Flags exposing (Flags) +import Page.NewInvite.Data exposing (..) +import Api.Model.GenInvite exposing (GenInvite) +import Api.Model.InviteResult +import Util.Http + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetPassword str -> + ({model|password = str}, Cmd.none) + + Reset -> + (emptyModel, Cmd.none) + + GenerateInvite -> + (model, Api.newInvite flags (GenInvite model.password) InviteResp) + + InviteResp (Ok res) -> + if res.success then ({model | result = (Success res)}, Cmd.none) + else ({model | result = (Failed res.message)}, Cmd.none) + + InviteResp (Err err) -> + ({model|result = Failed (Util.Http.errorToString err)}, Cmd.none) diff --git a/modules/webapp/src/main/elm/Page/NewInvite/View.elm b/modules/webapp/src/main/elm/Page/NewInvite/View.elm new file mode 100644 index 00000000..28776f24 --- /dev/null +++ b/modules/webapp/src/main/elm/Page/NewInvite/View.elm @@ -0,0 +1,102 @@ +module Page.NewInvite.View exposing (view) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick, onInput, onSubmit) +import Data.Flags exposing (Flags) +import Page.NewInvite.Data exposing (..) +import Api.Model.InviteResult +import Util.Maybe + +view: Flags -> Model -> Html Msg +view flags model = + div [class "newinvite-page"] + [div [class "ui centered grid"] + [div [class "row"] + [div [class "eight wide column ui segment newinvite-view"] + [h1 [class "ui cener aligned icon header"] + [i [class "umbrella icon"][] + ,text "Create new invitations" + ] + ,inviteMessage flags + ,Html.form [classList [("ui large form raised segment", True) + ,("error", isFailed model.result) + ,("success", isSuccess model.result) + ] + , onSubmit GenerateInvite] + [div [class "required field"] + [label [][text "New Invitation Password"] + ,div [class "ui left icon input"] + [input [type_ "password" + ,onInput SetPassword + ,value model.password + ,autofocus True + ][] + ,i [class "key icon"][] + ] + ] + ,button [class "ui primary button" + ,type_ "submit" + ] + [text "Submit" + ] + ,a [class "ui right floated button", href "", onClick Reset] + [text "Reset" + ] + ,resultMessage model + ] + ] + ] + ] + ] + +resultMessage: Model -> Html Msg +resultMessage model = + div [classList [("ui message", True) + ,("error", isFailed model.result) + ,("success", isSuccess model.result) + ,("hidden", model.result == Empty) + ]] + [case model.result of + Failed m -> + div [class "content"] + [div [class "header"][text "Error"] + ,p [][text m] + ] + Success r -> + div [class "content"] + [div [class "header"][text "Success"] + ,p [][text r.message] + ,p [][text "Invitation Key:"] + ,pre[][Maybe.withDefault "" r.key |> text + ] + ] + Empty -> + span[][] + ] + +inviteMessage: Flags -> Html Msg +inviteMessage flags = + div [classList [("ui message", True) + ,("hidden", flags.config.signupMode /= "invite") + ]] + [p [][text + + """Docspell requires an invite when signing up. You can + create these invites here and send them to friends so + they can signup with docspell.""" + + ] + ,p [][text + + """Each invite can only be used once. You'll need to + create one key for each person you want to invite.""" + + ] + ,p [][text + + """Creating an invite requires providing the password + from the configuration.""" + + ] + ] diff --git a/modules/webapp/src/main/elm/Page/Queue/Data.elm b/modules/webapp/src/main/elm/Page/Queue/Data.elm new file mode 100644 index 00000000..ad719f06 --- /dev/null +++ b/modules/webapp/src/main/elm/Page/Queue/Data.elm @@ -0,0 +1,79 @@ +module Page.Queue.Data exposing (..) + +import Http +import Api.Model.JobQueueState exposing (JobQueueState) +import Api.Model.JobDetail exposing (JobDetail) +import Api.Model.BasicResult exposing (BasicResult) +import Time +import Util.Duration +import Util.Maybe +import Comp.YesNoDimmer + +type alias Model = + { state: JobQueueState + , error: String + , pollingInterval: Float + , init: Bool + , stopRefresh: Bool + , currentMillis: Int + , showLog: Maybe JobDetail + , deleteConfirm: Comp.YesNoDimmer.Model + , cancelJobRequest: Maybe String + } + +emptyModel: Model +emptyModel = + { state = Api.Model.JobQueueState.empty + , error = "" + , pollingInterval = 1200 + , init = False + , stopRefresh = False + , currentMillis = 0 + , showLog = Nothing + , deleteConfirm = Comp.YesNoDimmer.emptyModel + , cancelJobRequest = Nothing + } + +type Msg + = Init + | StateResp (Result Http.Error JobQueueState) + | StopRefresh + | NewTime Time.Posix + | ShowLog JobDetail + | QuitShowLog + | RequestCancelJob JobDetail + | DimmerMsg JobDetail Comp.YesNoDimmer.Msg + | CancelResp (Result Http.Error BasicResult) + +getRunningTime: Model -> JobDetail -> Maybe String +getRunningTime model job = + let + mkTime: Int -> Int -> Maybe String + mkTime start end = + if start < end then Just <| Util.Duration.toHuman (end - start) + else Nothing + in + case (job.started, job.finished) of + (Just sn, Just fn) -> + Util.Maybe.or + [ mkTime sn fn + , mkTime sn model.currentMillis + ] + + (Just sn, Nothing) -> + mkTime sn model.currentMillis + + (Nothing, _) -> + Nothing + +getSubmittedTime: Model -> JobDetail -> Maybe String +getSubmittedTime model job = + if model.currentMillis > job.submitted then + Just <| Util.Duration.toHuman (model.currentMillis - job.submitted) + else + Nothing + +getDuration: Model -> JobDetail -> Maybe String +getDuration model job = + if job.state == "stuck" then getSubmittedTime model job + else Util.Maybe.or [ (getRunningTime model job), (getSubmittedTime model job) ] diff --git a/modules/webapp/src/main/elm/Page/Queue/Update.elm b/modules/webapp/src/main/elm/Page/Queue/Update.elm new file mode 100644 index 00000000..f91b17db --- /dev/null +++ b/modules/webapp/src/main/elm/Page/Queue/Update.elm @@ -0,0 +1,76 @@ +module Page.Queue.Update exposing (update) + +import Api +import Ports +import Page.Queue.Data exposing (..) +import Data.Flags exposing (Flags) +import Util.Http +import Time +import Task +import Comp.YesNoDimmer + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + Init -> + let + start = if model.init + then Cmd.none + else Cmd.batch + [Api.getJobQueueState flags StateResp + ,getNewTime + ] + in + ({model|init = True, stopRefresh = False}, start) + + StateResp (Ok s) -> + let + progressCmd = + List.map (\job -> Ports.setProgress (job.id, job.progress)) s.progress + _ = Debug.log "stopRefresh" model.stopRefresh + refresh = + if model.pollingInterval <= 0 || model.stopRefresh then Cmd.none + else Cmd.batch + [Api.getJobQueueStateIn flags model.pollingInterval StateResp + ,getNewTime + ] + in + ({model | state = s, stopRefresh = False}, Cmd.batch (refresh :: progressCmd)) + + StateResp (Err err) -> + ({model | error = Util.Http.errorToString err }, Cmd.none) + + StopRefresh -> + ({model | stopRefresh = True, init = False }, Cmd.none) + + NewTime t -> + ({model | currentMillis = Time.posixToMillis t}, Cmd.none) + + ShowLog job -> + ({model | showLog = Just job}, Cmd.none) + + QuitShowLog -> + ({model | showLog = Nothing}, Cmd.none) + + RequestCancelJob job -> + let + newModel = {model|cancelJobRequest = Just job.id} + in + update flags (DimmerMsg job Comp.YesNoDimmer.Activate) newModel + + DimmerMsg job m -> + let + (cm, confirmed) = Comp.YesNoDimmer.update m model.deleteConfirm + cmd = if confirmed then Api.cancelJob flags job.id CancelResp else Cmd.none + in + ({model | deleteConfirm = cm}, cmd) + + CancelResp (Ok r) -> + (model, Cmd.none) + CancelResp (Err err) -> + (model, Cmd.none) + + +getNewTime : Cmd Msg +getNewTime = + Task.perform NewTime Time.now diff --git a/modules/webapp/src/main/elm/Page/Queue/View.elm b/modules/webapp/src/main/elm/Page/Queue/View.elm new file mode 100644 index 00000000..905de30d --- /dev/null +++ b/modules/webapp/src/main/elm/Page/Queue/View.elm @@ -0,0 +1,217 @@ +module Page.Queue.View exposing (view) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) + +import Page.Queue.Data exposing (..) +import Api.Model.JobQueueState exposing (JobQueueState) +import Api.Model.JobDetail exposing (JobDetail) +import Api.Model.JobLogEvent exposing (JobLogEvent) +import Data.Priority +import Comp.YesNoDimmer +import Util.Time exposing (formatDateTime, formatIsoDateTime) +import Util.Duration + +view: Model -> Html Msg +view model = + div [class "queue-page ui grid container"] <| + List.concat + [ case model.showLog of + Just job -> + [renderJobLog job] + Nothing -> + List.map (renderProgressCard model) model.state.progress + |> List.map (\el -> div [class "row"][div [class "column"][el]]) + , [div [class "two column row"] + [renderWaiting model + ,renderCompleted model + ] + ] + ] + +renderJobLog: JobDetail -> Html Msg +renderJobLog job = + div [class "ui fluid card"] + [div [class "content"] + [i [class "delete link icon", onClick QuitShowLog][] + ,text job.name + ] + ,div [class "content"] + [div [class "job-log"] + (List.map renderLogLine job.logs) + ] + ] + + +renderWaiting: Model -> Html Msg +renderWaiting model = + div [class "column"] + [div [class "ui center aligned basic segment"] + [i [class "ui large angle double up icon"][] + ] + ,div [class "ui centered cards"] + (List.map (renderInfoCard model) model.state.queued) + ] + +renderCompleted: Model -> Html Msg +renderCompleted model = + div [class "column"] + [div [class "ui center aligned basic segment"] + [i [class "ui large angle double down icon"][] + ] + ,div [class "ui centered cards"] + (List.map (renderInfoCard model) model.state.completed) + ] + +renderProgressCard: Model -> JobDetail -> Html Msg +renderProgressCard model job = + div [class "ui fluid card"] + [div [id job.id, class "ui top attached indicating progress"] + [div [class "bar"] + [] + ] + ,Html.map (DimmerMsg job) (Comp.YesNoDimmer.view2 (model.cancelJobRequest == Just job.id) dimmerSettings model.deleteConfirm) + ,div [class "content"] + [ div [class "right floated meta"] + [div [class "ui label"] + [text job.state + ,div [class "detail"] + [Maybe.withDefault "" job.worker |> text + ] + ] + ,div [class "ui basic label"] + [i [class "clock icon"][] + ,div [class "detail"] + [getDuration model job |> Maybe.withDefault "-:-" |> text + ] + ] + ] + , i [class "asterisk loading icon"][] + , text job.name + ] + ,div [class "content"] + [div [class "job-log"] + (List.map renderLogLine job.logs) + ] + ,div [class "meta"] + [div [class "right floated"] + [button [class "ui button", onClick (RequestCancelJob job)] + [text "Cancel" + ] + ] + ] + ] + +renderLogLine: JobLogEvent -> Html Msg +renderLogLine log = + span [class (String.toLower log.level)] + [formatIsoDateTime log.time |> text + ,text ": " + ,text log.message + , br[][] + ] + +isFinal: JobDetail -> Bool +isFinal job = + case job.state of + "failed" -> True + "success" -> True + "cancelled" -> True + _ -> False + +dimmerSettings: Comp.YesNoDimmer.Settings +dimmerSettings = + let + defaults = Comp.YesNoDimmer.defaultSettings + in + { defaults | headerClass = "ui inverted header", headerIcon = "", message = "Cancel/Delete this job?"} + +renderInfoCard: Model -> JobDetail -> Html Msg +renderInfoCard model job = + div [classList [("ui fluid card", True) + ,(jobStateColor job, True) + ] + ] + [Html.map (DimmerMsg job) (Comp.YesNoDimmer.view2 (model.cancelJobRequest == Just job.id) dimmerSettings model.deleteConfirm) + ,div [class "content"] + [div [class "right floated"] + [if isFinal job || job.state == "stuck" then + span [onClick (ShowLog job)] + [i [class "file link icon", title "Show log"][] + ] + else + span[][] + ,i [class "delete link icon", title "Remove", onClick (RequestCancelJob job)][] + ] + ,if isFinal job then + span [class "invisible"][] + else + div [class "right floated"] + [div [class "meta"] + [getDuration model job |> Maybe.withDefault "-:-" |> text + ] + ] + ,i [classList [("check icon", job.state == "success") + ,("redo icon", job.state == "stuck") + ,("bolt icon", job.state == "failed") + ,("meh outline icon", job.state == "canceled") + ,("cog icon", not (isFinal job) && job.state /= "stuck") + ] + ][] + ,text job.name + ] + ,div [class "content"] + [div [class "right floated"] + [if isFinal job then + div [class ("ui basic label " ++ jobStateColor job)] + [i [class "clock icon"][] + ,div [class "detail"] + [getDuration model job |> Maybe.withDefault "-:-" |> text + ] + ] + else + span [class "invisible"][] + ,div [class ("ui basic label " ++ jobStateColor job)] + [text "Prio" + ,div [class "detail"] + [code [][Data.Priority.fromString job.priority + |> Maybe.map Data.Priority.toName + |> Maybe.withDefault job.priority + |> text + ] + ] + ] + ,div [class ("ui basic label " ++ jobStateColor job)] + [text "Retries" + ,div [class "detail"] + [job.retries |> String.fromInt |> text + ] + ] + ] + ,jobStateLabel job + ,div [class "ui basic label"] + [Util.Time.formatDateTime job.submitted |> text + ] + ] + ] + +jobStateColor: JobDetail -> String +jobStateColor job = + case job.state of + "success" -> "green" + "failed" -> "red" + "canceled" -> "orange" + "stuck" -> "purple" + "scheduled" -> "blue" + "waiting" -> "grey" + _ -> "" + +jobStateLabel: JobDetail -> Html Msg +jobStateLabel job = + let + col = jobStateColor job + in + div [class ("ui label " ++ col)] + [text job.state + ] diff --git a/modules/webapp/src/main/elm/Page/Register/Data.elm b/modules/webapp/src/main/elm/Page/Register/Data.elm new file mode 100644 index 00000000..b9b8d6c0 --- /dev/null +++ b/modules/webapp/src/main/elm/Page/Register/Data.elm @@ -0,0 +1,44 @@ +module Page.Register.Data exposing (..) + +import Http +import Api.Model.BasicResult exposing (BasicResult) + +type alias Model = + { result: Maybe BasicResult + , collId: String + , login: String + , pass1: String + , pass2: String + , showPass1: Bool + , showPass2: Bool + , errorMsg: List String + , loading: Bool + , successMsg: String + , invite: Maybe String + } + +emptyModel: Model +emptyModel = + { result = Nothing + , collId = "" + , login = "" + , pass1 = "" + , pass2 = "" + , showPass1 = False + , showPass2 = False + , errorMsg = [] + , successMsg = "" + , loading = False + , invite = Nothing + } + +type Msg + = SetCollId String + | SetLogin String + | SetPass1 String + | SetPass2 String + | SetInvite String + | RegisterSubmit + | ToggleShowPass1 + | ToggleShowPass2 + | SubmitResp (Result Http.Error BasicResult) diff --git a/modules/webapp/src/main/elm/Page/Register/Update.elm b/modules/webapp/src/main/elm/Page/Register/Update.elm new file mode 100644 index 00000000..4d78054a --- /dev/null +++ b/modules/webapp/src/main/elm/Page/Register/Update.elm @@ -0,0 +1,84 @@ +module Page.Register.Update exposing (update) + +import Api +import Api.Model.Registration exposing (Registration) +import Page.Register.Data exposing (..) +import Data.Flags exposing (Flags) +import Page exposing (Page(..)) + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + RegisterSubmit -> + case model.errorMsg of + [] -> + let + reg = { collectiveName = model.collId + , login = model.login + , password = model.pass1 + , invite = model.invite + } + in + (model, Api.register flags reg SubmitResp) + + _ -> + (model, Cmd.none) + + SetCollId str -> + let + m = {model|collId = str} + err = validateForm m + in + ({m|errorMsg = err}, Cmd.none) + + SetLogin str -> + let + m = {model|login = str} + err = validateForm m + in + ({m|errorMsg = err}, Cmd.none) + + SetPass1 str -> + let + m = {model|pass1 = str} + err = validateForm m + in + ({m|errorMsg = err}, Cmd.none) + + SetPass2 str -> + let + m = {model|pass2 = str} + err = validateForm m + in + ({m|errorMsg = err}, Cmd.none) + + SetInvite str -> + ({model | invite = if str == "" then Nothing else Just str}, Cmd.none) + + ToggleShowPass1 -> + ({model|showPass1 = not model.showPass1}, Cmd.none) + + ToggleShowPass2 -> + ({model|showPass2 = not model.showPass2}, Cmd.none) + + SubmitResp (Ok r) -> + let + m = emptyModel + cmd = if r.success then Page.goto (LoginPage Nothing) else Cmd.none + in + ({m|result = if r.success then Nothing else Just r}, cmd) + + SubmitResp (Err err) -> + (model, Cmd.none) + +validateForm: Model -> List String +validateForm model = + if model.collId == "" || + model.login == "" || + model.pass1 == "" || + model.pass2 == "" then + [ "All fields are required!"] + else if model.pass1 /= model.pass2 then + ["The passwords do not match."] + else + [] diff --git a/modules/webapp/src/main/elm/Page/Register/View.elm b/modules/webapp/src/main/elm/Page/Register/View.elm new file mode 100644 index 00000000..e7ed1c58 --- /dev/null +++ b/modules/webapp/src/main/elm/Page/Register/View.elm @@ -0,0 +1,120 @@ +module Page.Register.View exposing (view) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick, onInput, onSubmit) +import Data.Flags exposing (Flags) +import Page.Register.Data exposing (..) +import Page exposing (Page(..)) + +view: Flags -> Model -> Html Msg +view flags model = + div [class "register-page"] + [div [class "ui centered grid"] + [div [class "row"] + [div [class "six wide column ui segment register-view"] + [h1 [class "ui cener aligned icon header"] + [i [class "umbrella icon"][] + ,text "Sign up @ Docspell" + ] + ,Html.form [class "ui large error form raised segment", onSubmit RegisterSubmit] + [div [class "required field"] + [label [][text "Collective ID"] + ,div [class "ui left icon input"] + [input [type_ "text" + ,onInput SetCollId + ,value model.collId + ,autofocus True + ][] + ,i [class "users icon"][] + ] + ] + ,div [class "required field"] + [label [][text "User Login"] + ,div [class "ui left icon input"] + [input [type_ "text" + ,onInput SetLogin + ,value model.login + ][] + ,i [class "user icon"][] + ] + ] + ,div [class "required field" + ] + [label [][text "Password"] + ,div [class "ui left icon action input"] + [input [type_ <| if model.showPass1 then "text" else "password" + ,onInput SetPass1 + ,value model.pass1 + ][] + ,i [class "lock icon"][] + ,button [class "ui icon button", onClick ToggleShowPass1] + [i [class "eye icon"][] + ] + ] + ] + ,div [class "required field" + ] + [label [][text "Password (repeat)"] + ,div [class "ui left icon action input"] + [input [type_ <| if model.showPass2 then "text" else "password" + ,onInput SetPass2 + ,value model.pass2 + ][] + ,i [class "lock icon"][] + ,button [class "ui icon button", onClick ToggleShowPass2] + [i [class "eye icon"][] + ] + ] + ] + ,div [classList [("field", True) + ,("invisible", flags.config.signupMode /= "invite") + ]] + [label [][text "Invitation Key"] + ,div [class "ui left icon input"] + [input [type_ "text" + ,onInput SetInvite + ,model.invite |> Maybe.withDefault "" |> value + ][] + ,i [class "key icon"][] + ] + ] + ,button [class "ui primary button" + ,type_ "submit" + ] + [text "Submit" + ] + ] + ,(resultMessage model) + ,div [class "ui very basic right aligned segment"] + [text "Already signed up? " + ,a [class "ui link", Page.href (LoginPage Nothing)] + [i [class "sign-in icon"][] + ,text "Sign in" + ] + ] + ] + ] + ] + ] + +resultMessage: Model -> Html Msg +resultMessage model = + case model.result of + Just r -> + if r.success + then + div [class "ui success message"] + [text "Registration successful." + ] + else + div [class "ui error message"] + [text r.message + ] + + Nothing -> + if List.isEmpty model.errorMsg then + span [class "invisible"][] + else + div [class "ui error message"] + (List.map (\s -> div[][text s]) model.errorMsg) diff --git a/modules/webapp/src/main/elm/Page/Upload/Data.elm b/modules/webapp/src/main/elm/Page/Upload/Data.elm new file mode 100644 index 00000000..19ac5819 --- /dev/null +++ b/modules/webapp/src/main/elm/Page/Upload/Data.elm @@ -0,0 +1,91 @@ +module Page.Upload.Data exposing (..) + +import Http +import Set exposing (Set) +import File exposing (File) +import Api.Model.BasicResult exposing (BasicResult) +import Util.File exposing (makeFileId) +import Comp.Dropzone + +type alias Model = + { incoming: Bool + , singleItem: Bool + , files: List File + , completed: Set String + , errored: Set String + , loading: Set String + , dropzone: Comp.Dropzone.Model + } + +dropzoneSettings: Comp.Dropzone.Settings +dropzoneSettings = + let + ds = Comp.Dropzone.defaultSettings + in + {ds | classList = (\m -> [("ui attached blue placeholder segment dropzone", True) + ,("dragging", m.hover) + ,("disabled", not m.active) + ]) + } + + +emptyModel: Model +emptyModel = + { incoming = True + , singleItem = False + , files = [] + , completed = Set.empty + , errored = Set.empty + , loading = Set.empty + , dropzone = Comp.Dropzone.init dropzoneSettings + } + +type Msg + = SubmitUpload + | SingleUploadResp String (Result Http.Error BasicResult) + | GotProgress String Http.Progress + | ToggleIncoming + | ToggleSingleItem + | Clear + | DropzoneMsg Comp.Dropzone.Msg + + +isLoading: Model -> File -> Bool +isLoading model file = + Set.member (makeFileId file)model.loading + +isCompleted: Model -> File -> Bool +isCompleted model file = + Set.member (makeFileId file)model.completed + +isError: Model -> File -> Bool +isError model file = + Set.member (makeFileId file) model.errored + +isIdle: Model -> File -> Bool +isIdle model file = + not (isLoading model file || isCompleted model file || isError model file) + +uploadAllTracker: String +uploadAllTracker = + "upload-all" + +isInitial: Model -> Bool +isInitial model = + Set.isEmpty model.loading && + Set.isEmpty model.completed && + Set.isEmpty model.errored + +isDone: Model -> Bool +isDone model = + List.map makeFileId model.files + |> List.all (\id -> Set.member id model.completed || Set.member id model.errored) + +isSuccessAll: Model -> Bool +isSuccessAll model = + List.map makeFileId model.files + |> List.all (\id -> Set.member id model.completed) + +hasErrors: Model -> Bool +hasErrors model = + not (Set.isEmpty model.errored) diff --git a/modules/webapp/src/main/elm/Page/Upload/Update.elm b/modules/webapp/src/main/elm/Page/Upload/Update.elm new file mode 100644 index 00000000..5f9e6075 --- /dev/null +++ b/modules/webapp/src/main/elm/Page/Upload/Update.elm @@ -0,0 +1,94 @@ +module Page.Upload.Update exposing (update) + +import Api +import Http +import Set exposing (Set) +import Page.Upload.Data exposing (..) +import Data.Flags exposing (Flags) +import Comp.Dropzone +import File +import File.Select +import Ports +import Api.Model.ItemUploadMeta +import Util.File exposing (makeFileId) +import Util.Http + +update: (Maybe String) -> Flags -> Msg -> Model -> (Model, Cmd Msg, Sub Msg) +update sourceId flags msg model = + case msg of + ToggleIncoming -> + ({model|incoming = not model.incoming}, Cmd.none, Sub.none) + + ToggleSingleItem -> + ({model|singleItem = not model.singleItem}, Cmd.none, Sub.none) + + SubmitUpload -> + let + emptyMeta = Api.Model.ItemUploadMeta.empty + meta = {emptyMeta | multiple = not model.singleItem + , direction = if model.incoming then Just "incoming" else Just "outgoing" + } + fileids = List.map makeFileId model.files + uploads = if model.singleItem then Api.uploadSingle flags sourceId meta uploadAllTracker model.files (SingleUploadResp uploadAllTracker) + else Cmd.batch (Api.upload flags sourceId meta model.files SingleUploadResp) + tracker = if model.singleItem then Http.track uploadAllTracker (GotProgress uploadAllTracker) + else Sub.batch <| List.map (\id -> Http.track id (GotProgress id)) fileids + (cm2, _, _) = Comp.Dropzone.update (Comp.Dropzone.setActive False) model.dropzone + in + ({model|loading = Set.fromList fileids, dropzone = cm2}, uploads, tracker) + + SingleUploadResp fileid (Ok res) -> + let + compl = if res.success then setCompleted model fileid + else model.completed + errs = if not res.success then setErrored model fileid + else model.errored + load = if fileid == uploadAllTracker then Set.empty + else Set.remove fileid model.loading + in + ({model|completed = compl, errored = errs, loading = load} + , Ports.setProgress (fileid, 100), Sub.none + ) + + SingleUploadResp fileid (Err err) -> + let + _ = Debug.log "error" err + errs = setErrored model fileid + load = if fileid == uploadAllTracker then Set.empty + else Set.remove fileid model.loading + in + ({model|errored = errs, loading = load}, Cmd.none, Sub.none) + + GotProgress fileid progress -> + let + percent = case progress of + Http.Sending p -> + Http.fractionSent p + |> (*) 100 + |> round + _ -> 0 + updateBars = if percent == 0 then Cmd.none + else if model.singleItem then Ports.setAllProgress (uploadAllTracker, percent) + else Ports.setProgress (fileid, percent) + in + (model, updateBars, Sub.none) + + Clear -> + (emptyModel, Cmd.none, Sub.none) + + DropzoneMsg m -> + let + (m2, c2, files) = Comp.Dropzone.update m model.dropzone + nextFiles = List.append model.files files + in + ({model| files = nextFiles, dropzone = m2}, Cmd.map DropzoneMsg c2, Sub.none) + +setCompleted: Model -> String -> Set String +setCompleted model fileid = + if fileid == uploadAllTracker then List.map makeFileId model.files |> Set.fromList + else Set.insert fileid model.completed + +setErrored: Model -> String -> Set String +setErrored model fileid = + if fileid == uploadAllTracker then List.map makeFileId model.files |> Set.fromList + else Set.insert fileid model.errored diff --git a/modules/webapp/src/main/elm/Page/Upload/View.elm b/modules/webapp/src/main/elm/Page/Upload/View.elm new file mode 100644 index 00000000..5ed4ff5a --- /dev/null +++ b/modules/webapp/src/main/elm/Page/Upload/View.elm @@ -0,0 +1,166 @@ +module Page.Upload.View exposing (view) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick, onCheck) +import Comp.Dropzone +import File exposing (File) +import Page exposing (Page(..)) +import Page.Upload.Data exposing (..) +import Util.File exposing (makeFileId) +import Util.Maybe +import Util.Size + +view: (Maybe String) -> Model -> Html Msg +view mid model = + div [class "upload-page ui grid container"] + [div [class "row"] + [div [class "sixteen wide column"] + [div [class "ui top attached segment"] + [renderForm model + ] + ,Html.map DropzoneMsg (Comp.Dropzone.view model.dropzone) + ,div [class "ui bottom attached segment"] + [a [class "ui primary button", href "", onClick SubmitUpload] + [text "Submit" + ] + ,a [class "ui secondary button", href "", onClick Clear] + [text "Reset" + ] + ] + ] + ] + ,if isDone model && hasErrors model then renderErrorMsg model + else span[class "invisible"][] + ,if List.isEmpty model.files then span[][] + else if isSuccessAll model then renderSuccessMsg (Util.Maybe.nonEmpty mid) model + else renderUploads model + ] + + +renderErrorMsg: Model -> Html Msg +renderErrorMsg model = + div [class "row"] + [div [class "sixteen wide column"] + [div [class "ui large error message"] + [h3 [class "ui header"] + [i [class "meh outline icon"][] + ,text "Some files failed to upload" + ] + ,text "There were errors uploading some files." + ] + ] + ] + +renderSuccessMsg: Bool -> Model -> Html Msg +renderSuccessMsg public model = + div [class "row"] + [div [class "sixteen wide column"] + [div [class "ui large success message"] + [h3 [class "ui header"] + [i [class "smile outline icon"][] + ,text "All files uploaded" + ] + ,if public then p [][] else p [] + [text "Your files have been successfully uploaded. They are now being processed. Check the " + ,a [class "ui link", Page.href HomePage] + [text "Items page" + ] + ,text " later where the files will arrive eventually. Or go to the " + ,a [class "ui link", Page.href QueuePage] + [text "Processing Page" + ] + ,text " to view the current processing state." + ] + ,p [] + [text "Click " + ,a [class "ui link", href "", onClick Clear] + [text "Reset" + ] + ,text " to upload more files." + ] + ] + ] + ] + + +renderUploads: Model -> Html Msg +renderUploads model = + div [class "row"] + [div [class "sixteen wide column"] + [div [class "ui basic segment"] + [h2 [class "ui header"] + [text "Selected Files" + ] + ,div [class "ui items"] <| + if model.singleItem then + (List.map (renderFileItem model (Just uploadAllTracker)) model.files) + else + (List.map (renderFileItem model Nothing) model.files) + ] + ] + ] + + +renderFileItem: Model -> Maybe String -> File -> Html Msg +renderFileItem model mtracker file = + let + name = File.name file + size = File.size file + |> toFloat + |> Util.Size.bytesReadable Util.Size.B + in + div [class "item"] + [i [classList [("large", True) + ,("file outline icon", isIdle model file) + ,("loading spinner icon", isLoading model file) + ,("green check icon", isCompleted model file) + ,("red bolt icon", isError model file) + ]][] + ,div [class "middle aligned content"] + [div [class "header"] + [text name + ] + ,div [class "right floated meta"] + [text size + ] + ,div [class "description"] + [div [classList [("ui small indicating progress", True) + ,(uploadAllTracker, Util.Maybe.nonEmpty mtracker) + ] + , id (makeFileId file) + ] + [div [class "bar"] + [] + ] + ] + ] + ] + + +renderForm: Model -> Html Msg +renderForm model = + div [class "row"] + [Html.form [class "ui form"] + [div [class "grouped fields"] + [div [class "field"] + [div [class "ui radio checkbox"] + [input [type_ "radio", checked model.incoming, onCheck (\_ ->ToggleIncoming)][] + ,label [][text "Incoming"] + ] + ] + ,div [class "field"] + [div [class "ui radio checkbox"] + [input [type_ "radio", checked (not model.incoming), onCheck (\_ -> ToggleIncoming)][] + ,label [][text "Outgoing"] + ] + ] + ] + ,div [class "inline field"] + [div [class "ui checkbox"] + [input [type_ "checkbox", checked model.singleItem, onCheck (\_ -> ToggleSingleItem)][] + ,label [][text "All files are one single item"] + ] + ] + ] + ] diff --git a/modules/webapp/src/main/elm/Page/UserSettings/Data.elm b/modules/webapp/src/main/elm/Page/UserSettings/Data.elm new file mode 100644 index 00000000..2aecc50e --- /dev/null +++ b/modules/webapp/src/main/elm/Page/UserSettings/Data.elm @@ -0,0 +1,20 @@ +module Page.UserSettings.Data exposing (..) + +import Comp.ChangePasswordForm + +type alias Model = + { currentTab: Maybe Tab + , changePassModel: Comp.ChangePasswordForm.Model + } + +emptyModel: Model +emptyModel = + { currentTab = Nothing + , changePassModel = Comp.ChangePasswordForm.emptyModel + } + +type Tab = ChangePassTab + +type Msg + = SetTab Tab + | ChangePassMsg Comp.ChangePasswordForm.Msg diff --git a/modules/webapp/src/main/elm/Page/UserSettings/Update.elm b/modules/webapp/src/main/elm/Page/UserSettings/Update.elm new file mode 100644 index 00000000..86864206 --- /dev/null +++ b/modules/webapp/src/main/elm/Page/UserSettings/Update.elm @@ -0,0 +1,20 @@ +module Page.UserSettings.Update exposing (update) + +import Page.UserSettings.Data exposing (..) +import Data.Flags exposing (Flags) +import Comp.ChangePasswordForm + +update: Flags -> Msg -> Model -> (Model, Cmd Msg) +update flags msg model = + case msg of + SetTab t -> + let + m = { model | currentTab = Just t } + in + (m, Cmd.none) + + ChangePassMsg m -> + let + (m2, c2) = Comp.ChangePasswordForm.update flags m model.changePassModel + in + ({model | changePassModel = m2}, Cmd.map ChangePassMsg c2) diff --git a/modules/webapp/src/main/elm/Page/UserSettings/View.elm b/modules/webapp/src/main/elm/Page/UserSettings/View.elm new file mode 100644 index 00000000..120effc1 --- /dev/null +++ b/modules/webapp/src/main/elm/Page/UserSettings/View.elm @@ -0,0 +1,47 @@ +module Page.UserSettings.View exposing (view) + +import Html exposing (..) +import Html.Attributes exposing (..) +import Html.Events exposing (onClick) +import Util.Html exposing (classActive) + +import Page.UserSettings.Data exposing (..) +import Comp.ChangePasswordForm + +view: Model -> Html Msg +view model = + div [class "usersetting-page ui padded grid"] + [div [class "four wide column"] + [h4 [class "ui top attached ablue-comp header"] + [text "User" + ] + ,div [class "ui attached fluid segment"] + [div [class "ui fluid vertical secondary menu"] + [div [classActive (model.currentTab == Just ChangePassTab) "link icon item" + ,onClick (SetTab ChangePassTab) + ] + [i [class "user secret icon"][] + ,text "Change Password" + ] + ] + ] + ] + ,div [class "twelve wide column"] + [div [class ""] + (case model.currentTab of + Just ChangePassTab -> viewChangePassword model + Nothing -> [] + ) + ] + ] + +viewChangePassword: Model -> List (Html Msg) +viewChangePassword model = + [h2 [class "ui header"] + [i [class "ui user secret icon"][] + ,div [class "content"] + [text "Change Password" + ] + ] + ,Html.map ChangePassMsg (Comp.ChangePasswordForm.view model.changePassModel) + ] diff --git a/modules/webapp/src/main/elm/Ports.elm b/modules/webapp/src/main/elm/Ports.elm index edfe2b1e..6cab6915 100644 --- a/modules/webapp/src/main/elm/Ports.elm +++ b/modules/webapp/src/main/elm/Ports.elm @@ -5,4 +5,7 @@ import Api.Model.AuthResult exposing (AuthResult) port initElements: () -> Cmd msg port setAccount: AuthResult -> Cmd msg -port removeAccount: String -> Cmd msg +port removeAccount: () -> Cmd msg + +port setProgress: (String, Int) -> Cmd msg +port setAllProgress: (String, Int) -> Cmd msg diff --git a/modules/webapp/src/main/elm/Util/Address.elm b/modules/webapp/src/main/elm/Util/Address.elm new file mode 100644 index 00000000..51737202 --- /dev/null +++ b/modules/webapp/src/main/elm/Util/Address.elm @@ -0,0 +1,10 @@ +module Util.Address exposing (..) + +import Api.Model.Address exposing (Address) + +toString: Address -> String +toString a = + [ a.street, a.zip, a.city, a.country ] + |> List.filter (String.isEmpty >> not) + |> List.intersperse ", " + |> String.concat diff --git a/modules/webapp/src/main/elm/Util/Contact.elm b/modules/webapp/src/main/elm/Util/Contact.elm new file mode 100644 index 00000000..85e86160 --- /dev/null +++ b/modules/webapp/src/main/elm/Util/Contact.elm @@ -0,0 +1,9 @@ +module Util.Contact exposing (..) + +import Api.Model.Contact exposing (Contact) + +toString: List Contact -> String +toString contacts = + List.map (\c -> c.kind ++ ": " ++ c.value) contacts + |> List.intersperse ", " + |> String.concat diff --git a/modules/webapp/src/main/elm/Util/Duration.elm b/modules/webapp/src/main/elm/Util/Duration.elm new file mode 100644 index 00000000..22d778ea --- /dev/null +++ b/modules/webapp/src/main/elm/Util/Duration.elm @@ -0,0 +1,47 @@ +module Util.Duration exposing (Duration, toHuman) + +-- 486ms -> 12s -> 1:05 -> 59:45 -> 1:02:12 + +type alias Duration = Int + +toHuman: Duration -> String +toHuman dur = + fromMillis dur + + +-- implementation + +fromMillis: Int -> String +fromMillis ms = + case ms // 1000 of + 0 -> + (String.fromInt ms) ++ "ms" + n -> + fromSeconds n + +fromSeconds: Int -> String +fromSeconds sec = + case sec // 60 of + 0 -> + (String.fromInt sec) ++ "s" + n -> + let + s = sec - (n * 60) + in + (fromMinutes n) ++ ":" ++ (num s) + +fromMinutes: Int -> String +fromMinutes min = + case min // 60 of + 0 -> + (num min) + n -> + let + m = min - (n * 60) + in + (num n) ++ ":" ++ (num m) + +num: Int -> String +num n = + String.fromInt n + |> (++) (if n < 10 then "0" else "") diff --git a/modules/webapp/src/main/elm/Util/File.elm b/modules/webapp/src/main/elm/Util/File.elm new file mode 100644 index 00000000..f0785dec --- /dev/null +++ b/modules/webapp/src/main/elm/Util/File.elm @@ -0,0 +1,9 @@ +module Util.File exposing (..) + +import File exposing (File) +import Util.String + +makeFileId: File -> String +makeFileId file = + (File.name file) ++ "-" ++ (File.size file |> String.fromInt) + |> Util.String.crazyEncode diff --git a/modules/webapp/src/main/elm/Util/Html.elm b/modules/webapp/src/main/elm/Util/Html.elm new file mode 100644 index 00000000..8c58ebcc --- /dev/null +++ b/modules/webapp/src/main/elm/Util/Html.elm @@ -0,0 +1,40 @@ +module Util.Html exposing (..) + +import Html exposing (Attribute) +import Html.Attributes exposing (class) +import Html.Events exposing (on, keyCode) +import Json.Decode as Decode + +type KeyCode + = Up + | Down + | Left + | Right + | Enter + +intToKeyCode: Int -> Maybe KeyCode +intToKeyCode code = + case code of + 38 -> Just Up + 40 -> Just Down + 39 -> Just Right + 37 -> Just Left + 13 -> Just Enter + _ -> Nothing + +onKeyUp : (Int -> msg) -> Attribute msg +onKeyUp tagger = + on "keyup" (Decode.map tagger keyCode) + + +onClickk : msg -> Attribute msg +onClickk msg = + Html.Events.preventDefaultOn "click" (Decode.map alwaysPreventDefault (Decode.succeed msg)) + +alwaysPreventDefault : msg -> ( msg, Bool ) +alwaysPreventDefault msg = + ( msg, True ) + +classActive: Bool -> String -> Attribute msg +classActive active classes = + class (classes ++ (if active then " active" else "")) diff --git a/modules/webapp/src/main/elm/Util/Http.elm b/modules/webapp/src/main/elm/Util/Http.elm index ec6be5ca..959676b8 100644 --- a/modules/webapp/src/main/elm/Util/Http.elm +++ b/modules/webapp/src/main/elm/Util/Http.elm @@ -14,6 +14,7 @@ authReq: {url: String ,headers: List Http.Header ,body: Http.Body ,expect: Http.Expect msg + ,tracker: Maybe String } -> Cmd msg authReq req = Http.request @@ -23,7 +24,7 @@ authReq req = , expect = req.expect , body = req.body , timeout = Nothing - , tracker = Nothing + , tracker = req.tracker } authPost: {url: String @@ -39,6 +40,40 @@ authPost req = , expect = req.expect , method = "POST" , headers = [] + , tracker = Nothing + } + +authPostTrack: {url: String + ,account: AuthResult + ,body: Http.Body + ,expect: Http.Expect msg + ,tracker: String + } -> Cmd msg +authPostTrack req = + authReq + { url = req.url + , account = req.account + , body = req.body + , expect = req.expect + , method = "POST" + , headers = [] + , tracker = Just req.tracker + } + +authPut: {url: String + ,account: AuthResult + ,body: Http.Body + ,expect: Http.Expect msg + } -> Cmd msg +authPut req = + authReq + { url = req.url + , account = req.account + , body = req.body + , expect = req.expect + , method = "PUT" + , headers = [] + , tracker = Nothing } authGet: {url: String @@ -53,6 +88,22 @@ authGet req = , expect = req.expect , method = "GET" , headers = [] + , tracker = Nothing + } + +authDelete: {url: String + ,account: AuthResult + ,expect: Http.Expect msg + } -> Cmd msg +authDelete req = + authReq + { url = req.url + , account = req.account + , body = Http.emptyBody + , expect = req.expect + , method = "DELETE" + , headers = [] + , tracker = Nothing } diff --git a/modules/webapp/src/main/elm/Util/List.elm b/modules/webapp/src/main/elm/Util/List.elm new file mode 100644 index 00000000..99071e71 --- /dev/null +++ b/modules/webapp/src/main/elm/Util/List.elm @@ -0,0 +1,53 @@ +module Util.List exposing ( find + , findIndexed + , get + , distinct + , findNext + , findPrev + ) + +get: List a -> Int -> Maybe a +get list index = + if index < 0 then Nothing + else case list of + [] -> + Nothing + x :: xs -> + if index == 0 + then Just x + else get xs (index - 1) + +find: (a -> Bool) -> List a -> Maybe a +find pred list = + findIndexed pred list |> Maybe.map Tuple.first + +findIndexed: (a -> Bool) -> List a -> Maybe (a, Int) +findIndexed pred list = + findIndexed1 pred list 0 + +findIndexed1: (a -> Bool) -> List a -> Int -> Maybe (a, Int) +findIndexed1 pred list index = + case list of + [] -> Nothing + x :: xs -> + if pred x then Just (x, index) + else findIndexed1 pred xs (index + 1) + +distinct: List a -> List a +distinct list = + List.reverse <| + List.foldl (\a -> \r -> if (List.member a r) then r else a :: r) [] list + +findPrev: (a -> Bool) -> List a -> Maybe a +findPrev pred list = + findIndexed pred list + |> Maybe.map Tuple.second + |> Maybe.map (\i -> i - 1) + |> Maybe.andThen (get list) + +findNext: (a -> Bool) -> List a -> Maybe a +findNext pred list = + findIndexed pred list + |> Maybe.map Tuple.second + |> Maybe.map (\i -> i + 1) + |> Maybe.andThen (get list) diff --git a/modules/webapp/src/main/elm/Util/Maybe.elm b/modules/webapp/src/main/elm/Util/Maybe.elm new file mode 100644 index 00000000..e4f895a6 --- /dev/null +++ b/modules/webapp/src/main/elm/Util/Maybe.elm @@ -0,0 +1,23 @@ +module Util.Maybe exposing (..) + +nonEmpty: Maybe a -> Bool +nonEmpty ma = + Maybe.map (\_ -> True) ma + |> Maybe.withDefault False + +isEmpty: Maybe a -> Bool +isEmpty ma = + not (nonEmpty ma) + +withDefault: Maybe a -> Maybe a -> Maybe a +withDefault ma1 ma2 = + if isEmpty ma2 then ma1 else ma2 + +or: List (Maybe a) -> Maybe a +or listma = + case listma of + [] -> Nothing + el :: els -> + case el of + Just _ -> el + Nothing -> or els diff --git a/modules/webapp/src/main/elm/Util/Size.elm b/modules/webapp/src/main/elm/Util/Size.elm new file mode 100644 index 00000000..cba32c40 --- /dev/null +++ b/modules/webapp/src/main/elm/Util/Size.elm @@ -0,0 +1,24 @@ +module Util.Size exposing (..) + +type SizeUnit = G|M|K|B + +prettyNumber: Float -> String +prettyNumber n = + let + parts = String.split "." (String.fromFloat n) + in + case parts of + n0 :: d :: [] -> n0 ++ "." ++ (String.left 2 d) + _ -> String.join "." parts + +bytesReadable: SizeUnit -> Float -> String +bytesReadable unit n = + let + k = n / 1024 + num = prettyNumber n + in + case unit of + G -> num ++ "G" + M -> if k > 1 then (bytesReadable G k) else num ++ "M" + K -> if k > 1 then (bytesReadable M k) else num ++ "K" + B -> if k > 1 then (bytesReadable K k) else num ++ "B" diff --git a/modules/webapp/src/main/elm/Util/String.elm b/modules/webapp/src/main/elm/Util/String.elm new file mode 100644 index 00000000..b2a6de04 --- /dev/null +++ b/modules/webapp/src/main/elm/Util/String.elm @@ -0,0 +1,28 @@ +module Util.String exposing (..) + +import Base64 + +crazyEncode: String -> String +crazyEncode str = + let + b64 = Base64.encode str + len = String.length b64 + in + case (String.right 2 b64 |> String.toList) of + '=' :: '=' :: [] -> + (String.dropRight 2 b64) ++ "0" + + _ :: '=' :: [] -> + (String.dropRight 1 b64) ++ "1" + + _ -> + b64 + +ellipsis: Int -> String -> String +ellipsis len str = + if String.length str <= len then str + else (String.left (len - 3) str) ++ "..." + +withDefault: String -> String -> String +withDefault default str = + if str == "" then default else str diff --git a/modules/webapp/src/main/elm/Util/Time.elm b/modules/webapp/src/main/elm/Util/Time.elm new file mode 100644 index 00000000..51381277 --- /dev/null +++ b/modules/webapp/src/main/elm/Util/Time.elm @@ -0,0 +1,87 @@ +module Util.Time exposing (..) + +import DateFormat +import Time exposing (Posix, Zone, utc) + + +dateFormatter : Zone -> Posix -> String +dateFormatter = + DateFormat.format + [ DateFormat.dayOfWeekNameAbbreviated + , DateFormat.text ", " + , DateFormat.monthNameFull + , DateFormat.text " " + , DateFormat.dayOfMonthSuffix + , DateFormat.text ", " + , DateFormat.yearNumber + ] + +dateFormatterShort: Zone -> Posix -> String +dateFormatterShort = + DateFormat.format + [ DateFormat.yearNumber + , DateFormat.text "/" + , DateFormat.monthFixed + , DateFormat.text "/" + , DateFormat.dayOfMonthFixed + ] + +timeFormatter: Zone -> Posix -> String +timeFormatter = + DateFormat.format + [ DateFormat.hourMilitaryNumber + , DateFormat.text ":" + , DateFormat.minuteFixed + ] + +isoDateTimeFormatter: Zone -> Posix -> String +isoDateTimeFormatter = + DateFormat.format + [ DateFormat.yearNumber + , DateFormat.text "-" + , DateFormat.monthFixed + , DateFormat.text "-" + , DateFormat.dayOfMonthFixed + , DateFormat.text "T" + , DateFormat.hourMilitaryNumber + , DateFormat.text ":" + , DateFormat.minuteFixed + , DateFormat.text ":" + , DateFormat.secondFixed + ] + + +timeZone: Zone +timeZone = + utc + +{- Format millis into "Wed, 10. Jan 2018, 18:57" +-} +formatDateTime: Int -> String +formatDateTime millis = + (formatDate millis) ++ ", " ++ (formatTime millis) + +formatIsoDateTime: Int -> String +formatIsoDateTime millis = + Time.millisToPosix millis + |> isoDateTimeFormatter timeZone + +{- Format millis into "18:57". The current time (not the duration of + the millis). +-} +formatTime: Int -> String +formatTime millis = + Time.millisToPosix millis + |> timeFormatter timeZone + +{- Format millis into "Wed, 10. Jan 2018" +-} +formatDate: Int -> String +formatDate millis = + Time.millisToPosix millis + |> dateFormatter timeZone + +formatDateShort: Int -> String +formatDateShort millis = + Time.millisToPosix millis + |> dateFormatterShort timeZone diff --git a/modules/webapp/src/main/elm/Util/Update.elm b/modules/webapp/src/main/elm/Util/Update.elm new file mode 100644 index 00000000..32da4b33 --- /dev/null +++ b/modules/webapp/src/main/elm/Util/Update.elm @@ -0,0 +1,15 @@ +module Util.Update exposing (..) + + +andThen1: List (a -> (a, Cmd b)) -> a -> (a, Cmd b) +andThen1 fs a = + let + init = (a, []) + update el tuple = + let + (a2, c2) = el (Tuple.first tuple) + in + (a2, c2 :: (Tuple.second tuple)) + in + List.foldl update init fs + |> Tuple.mapSecond Cmd.batch diff --git a/modules/webapp/src/main/webjar/docspell.css b/modules/webapp/src/main/webjar/docspell.css index cf23206f..31452bc6 100644 --- a/modules/webapp/src/main/webjar/docspell.css +++ b/modules/webapp/src/main/webjar/docspell.css @@ -1,36 +1,226 @@ /* Docspell CSS */ +/* https://www.color-hex.com/color/f0f8ff + * https://www.color-hex.com/color-palette/1637 + */ .default-layout { background: #fff; - height: 100vh; +/* height: 100vh; */ } .default-layout .main-content { margin-top: 45px; } -.login-layout { - background: #aaa; +.default-layout .top-menu { + background: aliceblue; + box-shadow: 1px 1px 0px 0px black; +} + +.default-layout .ui.multiple.search.dropdown>input.search { + width: 3.5em; +} + +.default-layout .job-log { + background: #181819; + color: white; + font-size: smaller; + overflow: auto; + padding: 0.5em; + font-family: monospace,monospace; +} + +.default-layout .job-log>.debug { + color: rgba(255,255,255,0.5); +} +.default-layout .job-log>.error { + color: rgba(255,0,0,0.9); +} +.default-layout .job-log>.warn { + color: rgba(255,255,0,0.9); +} + +.default-layout .main-content>.grid { + margin-top: 0; +} + +.default-layout .dropzone.dragging { + background: aliceblue; +} + +.ui.search.dropdown.open { + z-index: 20; +} + +.ui.grid>.ablue.column, .ui.grid>.ablue.row, .ui.grid>.row>.ablue.column { + background: aliceblue; +} + +.ui.ablue.menu, .ui.menu .ablue.item { + background-color: aliceblue; +} + +.ui.ablue-comp.menu, .ui.menu .ablue-comp.item { + background-color: #fff7f0; +} +.ui.ablue-comp.header { + background-color: #fff7f0; +} + +.ui.ablue-shade.menu, .ui.menu .ablue-shade.item { + background-color: #d8dfe5; +} + +span.small-info { + font-size: smaller; + color: rgba(0,0,0,0.6); +} + +.placeholder-message { + color: rgba(0,0,0,0.4); +} + +.login-layout, .register-layout, .newinvite-layout { + background: #708090; height: 101vh; } -.login-layout .login-view { +.login-layout .login-view, .register-layout .register-view, .newinvite-view { background: #fff; position: relative; top: 20vh; } - .invisible { display: none !important; } +.no-margin { + margin: 0 !important; +} + +.no-border { + border: 0 !important; +} + @media (min-height: 320px) { .ui.footer { position: fixed; bottom: 0; width: 100%; text-align: center; - font-size: x-small; + font-size: smaller; + font-family: monospace; + height: 20px; } } + + + +/* Datepicker + From: https://github.com/CurrySoftware/elm-datepicker/blob/3.1.0/css/elm-datepicker.css + License: BSD-3-Clause +*/ +.elm-datepicker--container { + position: relative; } + +.elm-datepicker--input:focus { + outline: 0; } + +.elm-datepicker--picker { + position: absolute; + border: 1px solid #CCC; + z-index: 10; + background-color: white; } + +.elm-datepicker--picker-header, +.elm-datepicker--weekdays { + background: #F2F2F2; } + +.elm-datepicker--picker-header { + display: flex; + align-items: center; + background: aliceblue; } + +.elm-datepicker--prev-container, +.elm-datepicker--next-container { + flex: 0 1 auto; + cursor: pointer; } + +.elm-datepicker--month-container { + flex: 1 1 auto; + padding: 0.5em; + display: flex; + flex-direction: column; } + +.elm-datepicker--month, +.elm-datepicker--year { + flex: 1 1 auto; + cursor: default; + text-align: center; } + +.elm-datepicker--year { + font-size: 0.6em; + font-weight: 700; } + +.elm-datepicker--prev, +.elm-datepicker--next { + border: 6px solid transparent; + background-color: inherit; + display: block; + width: 0; + height: 0; + padding: 0 0.2em; } + +.elm-datepicker--prev { + border-right-color: #AAA; } + .elm-datepicker--prev:hover { + border-right-color: #BBB; } + +.elm-datepicker--next { + border-left-color: #AAA; } + .elm-datepicker--next:hover { + border-left-color: #BBB; } + +.elm-datepicker--table { + border-spacing: 0; + border-collapse: collapse; + font-size: 0.8em; } + .elm-datepicker--table td { + width: 2em; + height: 2em; + text-align: center; } + +.elm-datepicker--row { + border-top: 1px solid #F2F2F2; } + +.elm-datepicker--dow { + border-bottom: 1px solid #CCC; + cursor: default; } + +.elm-datepicker--day { + cursor: pointer; } + .elm-datepicker--day:hover { + background: #F2F2F2; } + +.elm-datepicker--disabled { + cursor: default; + color: #DDD; } + .elm-datepicker--disabled:hover { + background: inherit; } + +.elm-datepicker--picked { + color: white; + background: darkblue; } + .elm-datepicker--picked:hover { + background: darkblue; } + +.elm-datepicker--today { + font-weight: bold; } + +.elm-datepicker--other-month { + color: #AAA; } +.elm-datepicker--other-month.elm-datepicker--disabled { + color: #EEE; } +.elm-datepicker--other-month.elm-datepicker--picked { + color: white; } diff --git a/modules/webapp/src/main/webjar/docspell.js b/modules/webapp/src/main/webjar/docspell.js index 0f3ce5b9..a09bad28 100644 --- a/modules/webapp/src/main/webjar/docspell.js +++ b/modules/webapp/src/main/webjar/docspell.js @@ -6,8 +6,34 @@ var elmApp = Elm.Main.init({ }); elmApp.ports.initElements.subscribe(function() { - console.log("Initialsing elements …"); - $('.ui.dropdown').dropdown(); - $('.ui.checkbox').checkbox(); - $('.ui.accordion').accordion(); +// console.log("Initialsing elements …"); +// $('.ui.dropdown').dropdown(); +// $('.ui.checkbox').checkbox(); +// $('.ui.accordion').accordion(); +}); + +elmApp.ports.setAccount.subscribe(function(authResult) { + console.log("Add account from local storage"); + localStorage.setItem("account", JSON.stringify(authResult)); +}); + +elmApp.ports.removeAccount.subscribe(function() { + console.log("Remove account from local storage"); + localStorage.removeItem("account"); +}); + +elmApp.ports.setProgress.subscribe(function(input) { + var id = input[0]; + var percent = input[1]; + setTimeout(function () { + $("#"+id).progress({percent: percent}); + }, 100); +}); + +elmApp.ports.setAllProgress.subscribe(function(input) { + var id = input[0]; + var percent = input[1]; + setTimeout(function () { + $("."+id).progress({percent: percent}); + }, 100); }); diff --git a/modules/webapp/src/main/webjar/favicon/android-icon-144x144.png b/modules/webapp/src/main/webjar/favicon/android-icon-144x144.png new file mode 100644 index 00000000..3e2b85ac Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/android-icon-144x144.png differ diff --git a/modules/webapp/src/main/webjar/favicon/android-icon-192x192.png b/modules/webapp/src/main/webjar/favicon/android-icon-192x192.png new file mode 100644 index 00000000..116cdacb Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/android-icon-192x192.png differ diff --git a/modules/webapp/src/main/webjar/favicon/android-icon-36x36.png b/modules/webapp/src/main/webjar/favicon/android-icon-36x36.png new file mode 100644 index 00000000..839996c1 Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/android-icon-36x36.png differ diff --git a/modules/webapp/src/main/webjar/favicon/android-icon-48x48.png b/modules/webapp/src/main/webjar/favicon/android-icon-48x48.png new file mode 100644 index 00000000..3df120db Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/android-icon-48x48.png differ diff --git a/modules/webapp/src/main/webjar/favicon/android-icon-72x72.png b/modules/webapp/src/main/webjar/favicon/android-icon-72x72.png new file mode 100644 index 00000000..695e4c79 Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/android-icon-72x72.png differ diff --git a/modules/webapp/src/main/webjar/favicon/android-icon-96x96.png b/modules/webapp/src/main/webjar/favicon/android-icon-96x96.png new file mode 100644 index 00000000..1261adf1 Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/android-icon-96x96.png differ diff --git a/modules/webapp/src/main/webjar/favicon/apple-icon-114x114.png b/modules/webapp/src/main/webjar/favicon/apple-icon-114x114.png new file mode 100644 index 00000000..27d7e329 Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/apple-icon-114x114.png differ diff --git a/modules/webapp/src/main/webjar/favicon/apple-icon-120x120.png b/modules/webapp/src/main/webjar/favicon/apple-icon-120x120.png new file mode 100644 index 00000000..f4e4a248 Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/apple-icon-120x120.png differ diff --git a/modules/webapp/src/main/webjar/favicon/apple-icon-144x144.png b/modules/webapp/src/main/webjar/favicon/apple-icon-144x144.png new file mode 100644 index 00000000..3e2b85ac Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/apple-icon-144x144.png differ diff --git a/modules/webapp/src/main/webjar/favicon/apple-icon-152x152.png b/modules/webapp/src/main/webjar/favicon/apple-icon-152x152.png new file mode 100644 index 00000000..e6545fa8 Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/apple-icon-152x152.png differ diff --git a/modules/webapp/src/main/webjar/favicon/apple-icon-180x180.png b/modules/webapp/src/main/webjar/favicon/apple-icon-180x180.png new file mode 100644 index 00000000..54869a40 Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/apple-icon-180x180.png differ diff --git a/modules/webapp/src/main/webjar/favicon/apple-icon-57x57.png b/modules/webapp/src/main/webjar/favicon/apple-icon-57x57.png new file mode 100644 index 00000000..9c815ed1 Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/apple-icon-57x57.png differ diff --git a/modules/webapp/src/main/webjar/favicon/apple-icon-60x60.png b/modules/webapp/src/main/webjar/favicon/apple-icon-60x60.png new file mode 100644 index 00000000..eebea11f Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/apple-icon-60x60.png differ diff --git a/modules/webapp/src/main/webjar/favicon/apple-icon-72x72.png b/modules/webapp/src/main/webjar/favicon/apple-icon-72x72.png new file mode 100644 index 00000000..695e4c79 Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/apple-icon-72x72.png differ diff --git a/modules/webapp/src/main/webjar/favicon/apple-icon-76x76.png b/modules/webapp/src/main/webjar/favicon/apple-icon-76x76.png new file mode 100644 index 00000000..8bec684c Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/apple-icon-76x76.png differ diff --git a/modules/webapp/src/main/webjar/favicon/apple-icon-precomposed.png b/modules/webapp/src/main/webjar/favicon/apple-icon-precomposed.png new file mode 100644 index 00000000..0940deec Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/apple-icon-precomposed.png differ diff --git a/modules/webapp/src/main/webjar/favicon/apple-icon.png b/modules/webapp/src/main/webjar/favicon/apple-icon.png new file mode 100644 index 00000000..0940deec Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/apple-icon.png differ diff --git a/modules/webapp/src/main/webjar/favicon/browserconfig.xml b/modules/webapp/src/main/webjar/favicon/browserconfig.xml new file mode 100644 index 00000000..c5541482 --- /dev/null +++ b/modules/webapp/src/main/webjar/favicon/browserconfig.xml @@ -0,0 +1,2 @@ + +#ffffff \ No newline at end of file diff --git a/modules/webapp/src/main/webjar/favicon/favicon-16x16.png b/modules/webapp/src/main/webjar/favicon/favicon-16x16.png new file mode 100644 index 00000000..bdf32a64 Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/favicon-16x16.png differ diff --git a/modules/webapp/src/main/webjar/favicon/favicon-32x32.png b/modules/webapp/src/main/webjar/favicon/favicon-32x32.png new file mode 100644 index 00000000..ccada452 Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/favicon-32x32.png differ diff --git a/modules/webapp/src/main/webjar/favicon/favicon-96x96.png b/modules/webapp/src/main/webjar/favicon/favicon-96x96.png new file mode 100644 index 00000000..1261adf1 Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/favicon-96x96.png differ diff --git a/modules/webapp/src/main/webjar/favicon/favicon.ico b/modules/webapp/src/main/webjar/favicon/favicon.ico new file mode 100644 index 00000000..64ae647c Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/favicon.ico differ diff --git a/modules/webapp/src/main/webjar/favicon/manifest.json b/modules/webapp/src/main/webjar/favicon/manifest.json new file mode 100644 index 00000000..013d4a6a --- /dev/null +++ b/modules/webapp/src/main/webjar/favicon/manifest.json @@ -0,0 +1,41 @@ +{ + "name": "App", + "icons": [ + { + "src": "\/android-icon-36x36.png", + "sizes": "36x36", + "type": "image\/png", + "density": "0.75" + }, + { + "src": "\/android-icon-48x48.png", + "sizes": "48x48", + "type": "image\/png", + "density": "1.0" + }, + { + "src": "\/android-icon-72x72.png", + "sizes": "72x72", + "type": "image\/png", + "density": "1.5" + }, + { + "src": "\/android-icon-96x96.png", + "sizes": "96x96", + "type": "image\/png", + "density": "2.0" + }, + { + "src": "\/android-icon-144x144.png", + "sizes": "144x144", + "type": "image\/png", + "density": "3.0" + }, + { + "src": "\/android-icon-192x192.png", + "sizes": "192x192", + "type": "image\/png", + "density": "4.0" + } + ] +} \ No newline at end of file diff --git a/modules/webapp/src/main/webjar/favicon/ms-icon-144x144.png b/modules/webapp/src/main/webjar/favicon/ms-icon-144x144.png new file mode 100644 index 00000000..3e2b85ac Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/ms-icon-144x144.png differ diff --git a/modules/webapp/src/main/webjar/favicon/ms-icon-150x150.png b/modules/webapp/src/main/webjar/favicon/ms-icon-150x150.png new file mode 100644 index 00000000..dd66a488 Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/ms-icon-150x150.png differ diff --git a/modules/webapp/src/main/webjar/favicon/ms-icon-310x310.png b/modules/webapp/src/main/webjar/favicon/ms-icon-310x310.png new file mode 100644 index 00000000..be18b359 Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/ms-icon-310x310.png differ diff --git a/modules/webapp/src/main/webjar/favicon/ms-icon-70x70.png b/modules/webapp/src/main/webjar/favicon/ms-icon-70x70.png new file mode 100644 index 00000000..1e3b8340 Binary files /dev/null and b/modules/webapp/src/main/webjar/favicon/ms-icon-70x70.png differ diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 2e19e3f7..d5901abf 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,32 +1,66 @@ +package docspell.build + import sbt._ object Dependencies { - val BetterMonadicForVersion = "0.3.0" - val BitpeaceVersion = "0.4.0-M2" - val CirceVersion = "0.12.0-M4" - val DoobieVersion = "0.8.0-M1" + val BcryptVersion = "0.4" + val BetterMonadicForVersion = "0.3.1" + val BitpeaceVersion = "0.4.0-M2" // 0.4.0 + val CirceVersion = "0.12.1" + val DoobieVersion = "0.8.0-RC1" // 0.8.2 val FastparseVersion = "2.1.3" - val FlywayVersion = "6.0.0-beta2" - val Fs2Version = "1.1.0-M1" + val FlywayVersion = "6.0.3" + val Fs2Version = "1.1.0-M1" // 2.0.0 val H2Version = "1.4.199" - val Http4sVersion = "0.21.0-M2" + val Http4sVersion = "0.21.0-M4" // waiting for new version supporting cats2/fs2-2 val KindProjectorVersion = "0.10.3" val Log4sVersion = "1.8.2" val LogbackVersion = "1.2.3" - val MariaDbVersion = "2.4.2" - val MiniTestVersion = "2.5.0" - val PostgresVersion = "42.2.6" - val PureConfigVersion = "0.11.1" + val MariaDbVersion = "2.4.4" + val MiniTestVersion = "2.7.0" + val PostgresVersion = "42.2.8" + val PureConfigVersion = "0.12.0" val SqliteVersion = "3.28.0" - val TikaVersion = "1.20" + val StanfordNlpVersion = "3.9.2" + val TikaVersion = "1.22" val javaxMailVersion = "1.6.2" val dnsJavaVersion = "2.1.9" - val YamuscaVersion = "0.6.0-M2" + val YamuscaVersion = "0.6.0" + val stanfordNlpCore = Seq( + "edu.stanford.nlp" % "stanford-corenlp" % StanfordNlpVersion excludeAll( + ExclusionRule("com.io7m.xom", "xom"), + ExclusionRule("javax.servlet", "javax.servlet-api"), + ExclusionRule("org.apache.lucene", "lucene-queryparser"), + ExclusionRule("org.apache.lucene", "lucene-queries"), + ExclusionRule("org.apache.lucene", "lucene-analyzers-common"), + ExclusionRule("org.apache.lucene", "lucene-core"), + ExclusionRule("com.sun.xml.bind", "jaxb-impl"), + ExclusionRule("com.sun.xml.bind", "jaxb-core"), + ExclusionRule("javax.xml.bind", "jaxb-api"), + ExclusionRule("de.jollyday", "jollyday"), + ExclusionRule("com.apple", "AppleJavaExtensions"), + ExclusionRule("org.glassfish", "javax.json") + ) + ) + + val stanfordNlpModels = Seq( + "edu.stanford.nlp" % "stanford-corenlp" % StanfordNlpVersion classifier "models-german", + "edu.stanford.nlp" % "stanford-corenlp" % StanfordNlpVersion classifier "models-english" + ) + + val tika = Seq( + "org.apache.tika" % "tika-core" % TikaVersion + ) + + val bcrypt = Seq( + "org.mindrot" % "jbcrypt" % BcryptVersion + ) val fs2 = Seq( - "co.fs2" %% "fs2-core" % Fs2Version + "co.fs2" %% "fs2-core" % Fs2Version, + "co.fs2" %% "fs2-io" % Fs2Version ) val http4s = Seq( @@ -34,6 +68,10 @@ object Dependencies { "org.http4s" %% "http4s-circe" % Http4sVersion, "org.http4s" %% "http4s-dsl" % Http4sVersion, ) + + val http4sClient = Seq( + "org.http4s" %% "http4s-blaze-client" % Http4sVersion + ) val circe = Seq( "io.circe" %% "circe-generic" % CirceVersion, @@ -46,7 +84,7 @@ object Dependencies { ) val logging = Seq( - "ch.qos.logback" % "logback-classic" % LogbackVersion % Runtime + "ch.qos.logback" % "logback-classic" % LogbackVersion ) // https://github.com/melrief/pureconfig @@ -113,7 +151,7 @@ object Dependencies { val betterMonadicFor = "com.olegpy" %% "better-monadic-for" % BetterMonadicForVersion val webjars = Seq( - "swagger-ui" -> "3.22.2", + "swagger-ui" -> "3.23.8", "Semantic-UI" -> "2.4.1", "jquery" -> "3.4.1" ).map({case (a, v) => "org.webjars" % a % v }) diff --git a/project/NerModelsPlugin.scala b/project/NerModelsPlugin.scala new file mode 100644 index 00000000..4ad2b461 --- /dev/null +++ b/project/NerModelsPlugin.scala @@ -0,0 +1,65 @@ +package docspell.build + +import sbt.{Def, _} +import sbt.Keys._ + +/** Take some files from dependencies and put them into the resources + * of a local sbt project. + * + * The reason is that the stanford ner model files are very very + * large: the jar file for the english models is about 1G and the jar + * file for the german models is about 170M. But I only need one file + * that is about 60M from each jar. So just for the sake to save 1GB + * file size when packaging docspell, this ugly plugin exists…. + * + * The jar files to filter must be added to the libraryDependencies + * in config "NerModels". + */ +object NerModelsPlugin extends AutoPlugin { + + object autoImport { + val NerModels = config("NerModels") + + val nerModelsFilter = settingKey[String => Boolean]("Which files to keep.") + val nerModelsRunFilter = taskKey[Seq[File]]("Extract files from libraryDependencies") + + } + + import autoImport._ + + def nerModelSettings: Seq[Setting[_]] = Seq( + nerModelsFilter := (_ => false), + nerModelsRunFilter := { + filterArtifacts(streams.value.log + , Classpaths.managedJars(NerModels, Set("jar", "zip"), update.value) + , nerModelsFilter.value + , (Compile/resourceManaged).value) + }, + Compile / resourceGenerators += nerModelsRunFilter.taskValue + ) + + def nerClassifierSettings: Seq[Setting[_]] = Seq( + libraryDependencies ++= Dependencies.stanfordNlpModels.map(_ % NerModels), + nerModelsFilter := { + name => nerModels.exists(name.endsWith) + } + ) + + override def projectConfigurations: Seq[Configuration] = + Seq(NerModels) + + override def projectSettings: Seq[Setting[_]] = + nerModelSettings + + def filterArtifacts(logger: Logger, cp: Classpath, nameFilter: NameFilter, out: File): Seq[File] = { + logger.info(s"NerModels: Filtering artifacts...") + cp.files.flatMap(f => { + IO.unzip(f, out, nameFilter) + }) + } + + private val nerModels = List( + "german.conll.germeval2014.hgc_175m_600.crf.ser.gz", + "english.all.3class.distsim.crf.ser.gz" + ) +} diff --git a/project/build.nix b/project/build.nix new file mode 100644 index 00000000..522d61e9 --- /dev/null +++ b/project/build.nix @@ -0,0 +1,16 @@ +with import { }; +let + initScript = writeScript "docspell-build-init" '' + export LD_LIBRARY_PATH= + ${bash}/bin/bash -c sbt + ''; +in +buildFHSUserEnv { + name = "docspell-sbt"; + targetPkgs = pkgs: with pkgs; [ + netcat jdk8 wget which zsh dpkg sbt git elmPackages.elm ncurses fakeroot mc jekyll + # haskells http client needs this (to download elm packages) + iana-etc + ]; + runScript = initScript; +} diff --git a/project/plugins.sbt b/project/plugins.sbt index 723b2d44..bef27cbf 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,8 +1,8 @@ addSbtPlugin("io.get-coursier" % "sbt-coursier" % "2.0.0-RC2") -addSbtPlugin("com.github.eikek" % "sbt-openapi-schema" % "0.5.0-SNAPSHOT") +addSbtPlugin("com.github.eikek" % "sbt-openapi-schema" % "0.5.0") addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.9.0") addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0") addSbtPlugin("io.spray" % "sbt-revolver" % "0.9.1") -addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.25") -addSbtPlugin("com.jsuereth" % "sbt-pgp" % "2.0.0-M2") +addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.4.1") +addSbtPlugin("com.jsuereth" % "sbt-pgp" % "2.0.1-M3") addSbtPlugin("com.47deg" % "sbt-microsites" % "0.9.2")