[{"_1":2,"_183":-5,"_184":-5},"loaderData",{"_3":4,"_18":19,"_25":26,"_162":163},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7,"_13":14,"_15":16,"_17":11},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","theme","system","contrast","normal","dockerRebrandTour","routes/_layout",{"_20":-5,"_21":22,"_23":11,"_24":22},"username","isAskGordonEnabled",true,"showMinimalLanding","showHoneypotTrap","routes/_layout.r.$namespace.$reponame",{"_27":28,"_29":-5,"_30":11,"_31":11,"_32":33,"_34":35,"_36":-5,"_37":11,"_38":39,"_88":89,"_112":113,"_150":-5,"_151":11,"_152":22,"_153":11,"_154":11,"_155":-5,"_156":11,"_157":-5,"_158":159,"_160":114,"_161":-7},"canonicalUrl","https://hub.docker.com/r/tokenmill/beagle-dictionary-validator/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","tokenmill","reponame","beagle-dictionary-validator","cloudBuildSource","isAutobuildsAvailable","repo",{"_40":41,"_42":43,"_45":46,"_47":48,"_49":11,"_50":51,"_52":53,"_54":55,"_57":35,"_32":33,"_58":59,"_60":44,"_61":62,"_63":64,"_65":66,"_67":-5,"_68":62,"_69":70,"_71":11,"_72":33,"_73":74,"_79":11,"_80":81,"_85":33,"_86":87},"categories",[],"content_types",[44],"image","date_registered","2019-10-15T13:42:27.369605Z","description","Phrases dictionary validator.","is_private","last_updated","2020-05-08T06:37:05.09775Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[56],"application/vnd.docker.container.image.v1+json","name","pull_count",350,"repository_type","star_count",0,"status",1,"status_description","active","source","collaborator_count","full_description","\n

\n\n\n# Beagle\n\n[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)\n[![pipeline status](https://gitlab.com/tokenmill/oss/beagle/badges/master/pipeline.svg)](https://gitlab.com/tokenmill/oss/beagle/pipelines/master/latest)\n[![Maven Central](https://img.shields.io/maven-central/v/lt.tokenmill/beagle.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:%22lt.tokenmill%22%20AND%20a:%22beagle%22)\n\nBeagle is a detector of interesting things in text. Its intended use is in-stream search applications. Suppose you need to monitor a stream of text documents such as web crawl results, chat messages, or corporate documents in order to identify keywords, phrases, regexes, and [complex search queries](https://lucene.apache.org/core/2_9_4/queryparsersyntax.html) of interest. With Beagle you can quickly be up and running with such a system, allowing you to focus on productively monitoring your documents.\n\nBeagle is based on the [Lucene monitor](https://github.com/apache/lucene-solr/tree/master/lucene/monitor) library which is based on [Luwak](https://github.com/flaxsearch/luwak).\n\n## Components\n\n- [Phrase highlighter with support for](#phrase-annotator-usage):\n - case sensitivity,\n - ascii folding,\n - stemming support for various languages,\n - phrase slop,\n - synonymous phrases,\n - metadata,\n - tokenizer,\n - ensuring order of terms in a phrase with slop,\n - any combination of previously mentioned features.\n- [Java interface to the phrase highlighter](#java-interface-to-the-phrase-highlighter)\n- (alpha!) [Lucene query string support](#lucene-query-support) (interface is subject to change)\n- [Dictionary file readers (csv, json, edn)](#dictionary-readers)\n- [Dictionary validator](#dictionary-validator)\n- [Dictionary optimizer](#dictionary-optimizer)\n- [Annotation merger](#annotation-merger)\n\n## Phrase Annotator Usage\n\n```clojure\n(require '[beagle.phrases :as phrases])\n\n(let [dictionary [{:text \"to be annotated\" :id \"1\"}]\n highlighter-fn (phrases/highlighter dictionary)]\n (highlighter-fn \"before annotated to be annotated after annotated\"))\n=> ({:text \"to be annotated\", :type \"LABEL\", :dict-entry-id \"1\", :meta {}, :begin-offset 17, :end-offset 32})\n\n;; Case sensitivity is controlled per dictionary entry \n(let [dictionary [{:text \"TO BE ANNOTATED\" :id \"1\" :case-sensitive? false}]\n highlighter-fn (phrases/highlighter dictionary)]\n (highlighter-fn \"before annotated to be annotated after annotated\"))\n=> ({:text \"to be annotated\", :type \"LABEL\", :dict-entry-id \"1\", :meta {}, :begin-offset 17, :end-offset 32})\n\n;; ASCII folding is controlled per dictionary entry\n(let [dictionary [{:text \"TÖ BE ÄNNÖTÄTED\" :id \"1\" :case-sensitive? false :ascii-fold? true}]\n highlighter-fn (phrases/highlighter dictionary)]\n (highlighter-fn \"before annotated to be annotated after annotated\"))\n=> ({:text \"to be annotated\", :type \"LABEL\", :dict-entry-id \"1\", :meta {}, :begin-offset 17, :end-offset 32})\n\n;; Stemming is supported for multiple languages per dictionary entry\n(let [dictionary [{:text \"Kaunas\" :id \"1\" :stem? true :stemmer :lithuanian}]\n highlighter-fn (phrases/highlighter dictionary)]\n (highlighter-fn \"Kauno miestas\"))\n=> ({:text \"Kauno\", :type \"PHRASE\", :dict-entry-id \"1\", :meta {}, :begin-offset 0, :end-offset 5})\n\n;; Phrases also support slop (i.e. terms edit distance) per dictionary entry\n(let [txt \"before start and end after\"\n dictionary [{:text \"start end\" :id \"1\" :slop 1}]\n highlighter-fn (phrases/highlighter dictionary)]\n (highlighter-fn txt))\n=> ({:text \"start and end\", :type \"PHRASE\", :dict-entry-id \"1\", :meta {}, :begin-offset 7, :end-offset 20})\n\n;; Every phrase can specify which tokenizer to use\n(let [txt \"[URGENT!] Do this immediately!\"\n dictionary [{:text \"[URGENT!]\" :id \"a\" :tokenizer :whitespace}\n {:text \"[URGENT!]\" :id \"b\" :tokenizer :standard}]\n highlighter-fn (phrases/highlighter dictionary)]\n (clojure.pprint/pprint (highlighter-fn txt)))\n=> \n({:text \"[URGENT!]\",\n :type \"PHRASE\",\n :dict-entry-id \"a\",\n :meta {},\n :begin-offset 0,\n :end-offset 9}\n {:text \"URGENT\",\n :type \"PHRASE\",\n :dict-entry-id \"b\",\n :meta {},\n :begin-offset 1,\n :end-offset 7})\n\n;; Ensure that phrase terms are matched in the provided order\n;; e.g. NOT preserving order (default)\n(let [txt \"Mill Token\"\n dictionary [{:text \"Token Mill\" :slop 2 :in-order? false}]\n highlighter-fn (phrases/highlighter dictionary)]\n (highlighter-fn txt))\n=> [{:text \"Mill Token\" :type \"PHRASE\" :dict-entry-id \"0\" :meta {} :begin-offset 0 :end-offset 10}]\n;; e.g. Preserving order\n(let [txt \"Mill Token\"\n dictionary [{:text \"Token Mill\" :slop 2 :in-order? true}]\n highlighter-fn (phrases/highlighter dictionary)]\n (highlighter-fn txt))\n=> ()\n```\n\n## Java Interface to the Phrase Highlighter\n\nExample:\n```java\nimport lt.tokenmill.beagle.phrases.Annotation;\nimport lt.tokenmill.beagle.phrases.Annotator;\nimport lt.tokenmill.beagle.phrases.DictionaryEntry;\n\nimport java.util.Arrays;\nimport java.util.Collection;\nimport java.util.HashMap;\n\npublic class Main {\n public static void main(String[] args) {\n DictionaryEntry dictionaryEntry = new DictionaryEntry(\"test phrase\");\n Annotator annotator = new Annotator(Arrays.asList(dictionaryEntry));\n Collection annotations = annotator.annotate(\"This is my test phrase\");\n annotations.forEach(s -> System.out.println(\"Annotated: \\'\" + s.text() + \"\\' at offset: \" + s.beginOffset() + \":\" + s.endOffset()));\n }\n}\n\n// => Annotated: 'test phrase' at offset: 11:22\n```\n\nThe available options for the Java API are explained with examples in the [Java Interface for Phrase Highlighting wiki page](https://github.com/tokenmill/beagle/wiki/Java-Interface-for-Phrase-Highlighting).\n\nAll the options that are present in the Clojure interface are also available for use in Java, just convert Clojure keywords to Java strings, e.g.\n```\n:case-sensitive? => \"case-sensitive?\"\n``` \n\n### Project Setup with Maven\n\nThe library is deployed in the Maven Central Repository and you can just add the beagle dependency to your `pom.xml`:\n\n```xml\n\n lt.tokenmill\n beagle\n 0.3.1\n\n```\n\n## Lucene Query Support\n\nExamples: \n\n```clojure\n(require '[beagle.lucene-alpha :as lucene])\n\n(let [txt \"some text this other that\"\n dictionary [{:text \"this AND that\" :id \"1\" :slop 1}]\n annotator-fn (lucene/annotator dictionary)]\n (annotator-fn txt {}))\n=> ({:text \"this AND that\", :type \"QUERY\", :dict-entry-id \"1\", :meta {}})\n```\n\n## Performance\n\nThe performance was measured on a desktop PC with Ubuntu 19.04 and 8-core Ryzen 1700.\n \nThe test setup was for news articles and dictionary made up of names of city names in USA.\n\nCode and data for benchmarking and more benchmarks can be found [here](https://github.com/tokenmill/beagle-performance-benchmarks).\n\n### Single-thread\n\nAverage time spent per document ranged from 1.58 ms for dictionary of 5k phrases to 4.58 ms per document for 80k phrases.\n\n![alt text](charts/st-avg-per-doc.png)\n\nThroughput of docs analyzed ranged from 626 docs/sec for dictionary of 5k phrases to 210 docs/sec for 80k phrases.\n\n![alt text](charts/st-throughput-per-sec.png)\n\nMax time spent per document has couple of spikes when processing a document takes ~1000ms. These spikes should \nhave been caused either by GC pauses, or JVM deoptimizations. Aside from those spikes, max time ranges grows steadily\nfrom 15 ms to 72 ms as the dictionary size grows. \n\nMin time spent per document is fairly stable for any dictionary size and is about 0.45 ms. Most likely these are the\ncases when [Presearcher](https://lucene.apache.org/core/8_2_0/monitor/index.html) haven't found any candidate queries to run against the document. \n\n![alt text](charts/st-min-max-per-doc.png)\n\n### Multi-threaded\n\nUsing `core.async` pipeline time spent per single doc ranged from 3.38 ms for dictionary of 5k phrases to 15.34 ms per document for 80k phrases.\n\n![alt text](charts/mt-avg-per-doc.png)\n\nTotal time spent to process all 10k docs ranged from 2412 ms for dictionary of 5k phrases to 12595 ms per document for 80k phrases.\n\n![alt text](charts/mt-total.png)\n\nThroughput of docs analyzed ranged from 4143 docs/sec for dictionary of 5k phrases to 793 docs/sec for 80k phrases.\n\n![alt text](charts/mt-throughput-per-sec.png)\n\nMax time spent per document has risen fairy steady from 24.15 ms for dictionary of 10k phrases to 113.45 ms per document for 60k phrases.\n\nMin time spent per document varied from 0.6 ms for dictionary of 10k phrases to 1.1 ms per document for 55k phrases.\n\n![alt text](charts/mt-min-max-per-doc.png)\n\n### Conclusions about Performance\n\nProcessing of a one document on average is faster in the single-thread mode by roughly by 3x compared to multi-threaded mode but even \nin multi-threaded mode one document rarely takes more than 10 ms. \n\nIn multi-threaded mode throughput grows with the number on CPU cores almost linearly: 4143/8=518 docs per core per sec in multi-threaded mode\nwhile in single-thread mode 626 docs per core per sec.\n\n## Dictionary Readers\n\nThree file formats are supported: csv, edn, json.\n\n### CSV Dictionary Format\n\nSeparator: \",\"\nEscape: \"\\\"\"\n\nThe first line *MUST* be a header.\n\nSupported header keys: `[\"text\" \"type\" \"id\" \"synonyms\" \"case-sensitive?\" \":ascii-fold?\" \"meta\"]`\n\nOrder is not important.\n\nUnder `synonyms`, there should be a list of string separated by \";\"\nUnder `meta`, there should be a list of strings separated by \";\". Even number of strings is expected. In case of odd number, last one is ignored.\n\n## Dictionary Validator\n\nAccepts any number of dictionaries to validate as long as they are provided in pairs as '\"/path/to/dictionary/file\" \"file-type\"'\n\n### Supported File Types\n\n- csv\n- json\n- edn\n\n### Output\n\n- If any dictionary is invalid exception will be thrown with exit status 1\n\n### Usage\n\n#### Clojure\n\nTo use validator directly execute command: `clj -m beagle.validator \"/path/to/dictionary/file\" \"file-type\" \"/path/to/dictionary/file2\" \"file-type\" & ...`\n\n##### Example:\n\n```\nclj -m beagle.validator \"your-dict.csv\" \"csv\" \"your-other-dict.json\" \"json\"\n```\n\n#### Docker\n\nExample in Gitlab CI:\n\n```\nvalidate-dictionaries:\n stage: dictionary-validation\n when: always\n image: tokenmill/beagle-dictionary-validator\n script:\n - >\n dictionary-validator\n /path/to/dict.csv csv\n /path/to/dict.json json\n /path/to/dict.edn edn\n```\n\n## Dictionary Optimizer\n\nSupported optimizations:\n- Remove duplicate dictionary entries\n- Merge synonyms\n- Synonyms and text equality check\n\nThere are cases when dictionary entries can't be merged:\n- Differences in text analysis\n\nExamples:\n```clojure\n(require '[beagle.dictionary-optimizer :as optimizer])\n\n; Remove duplicates\n(let [dictionary [{:text \"TO BE ANNOTATED\" :id \"1\"}\n {:text \"TO BE ANNOTATED\"}]]\n (optimizer/optimize dictionary))\n=> ({:text \"TO BE ANNOTATED\", :id \"1\"})\n\n; Merge synonyms\n(let [dictionary [{:text \"TO BE ANNOTATED\" :synonyms [\"ONE\"]}\n {:text \"TO BE ANNOTATED\" :synonyms [\"TWO\"]}]]\n (optimizer/optimize dictionary))\n=> ({:text \"TO BE ANNOTATED\", :synonyms (\"TWO\" \"ONE\")})\n\n; Synonyms and text equality check\n(let [dictionary [{:text \"TO BE ANNOTATED\" :synonyms [\"TO BE ANNOTATED\"]}]]\n (optimizer/optimize dictionary))\n=> ({:text \"TO BE ANNOTATED\", :synonyms [\"TO BE ANNOTATED\"]})\n\n; Can't be merged because of differences in text analysis\n(let [dictionary [{:text \"TO BE ANNOTATED\" :case-sensitive? true}\n {:text \"TO BE ANNOTATED\" :case-sensitive? false}]]\n (optimizer/optimize dictionary))\n=> ({:text \"TO BE ANNOTATED\", :case-sensitive? true} {:text \"TO BE ANNOTATED\", :case-sensitive? false})\n```\n\n## Annotation Merger\n\nOnly annotations of the same type are merged.\n\nHandled cases:\n- Duplicate annotations\n- Nested annotations\n\nExamples:\n```clojure\n(require '[beagle.annotation-merger :as merger])\n\n(let [dictionary [{:text \"TEST\"}\n {:text \"This TEST is\"}]\n highlighter-fn (phrases/highlighter dictionary)\n annotations (highlighter-fn \"This TEST is\")]\n (println \"Annotations: \" annotations)\n (merger/merge-same-type-annotations annotations))\nAnnotations: ({:text TEST, :type PHRASE, :dict-entry-id 0, :meta {}, :begin-offset 5, :end-offset 9} {:text This TEST is, :type PHRASE, :dict-entry-id 1, :meta {}, :begin-offset 0, :end-offset 12})\n=> ({:text \"This TEST is\", :type \"PHRASE\", :dict-entry-id \"1\", :meta {}, :begin-offset 0, :end-offset 12})\n\n;; You can also inline the need of merging annotations\n(let [dictionary [{:text \"TEST\"}\n {:text \"This TEST is\"}]\n highlighter-fn (phrases/highlighter dictionary)]\n (highlighter-fn \"This TEST is\" {:merge-annotations? true}))\n=> ({:text \"This TEST is\", :type \"PHRASE\", :dict-entry-id \"1\", :meta {}, :begin-offset 0, :end-offset 12})\n```\n\n## License\n\nCopyright © 2019 [TokenMill UAB](http://www.tokenmill.lt).\n\nDistributed under the The Apache License, Version 2.0.\n","has_starred","hub_user","immutable_tags_settings",{"_75":11,"_76":77},"enabled","rules",[78],".*","is_automated","permissions",{"_82":22,"_83":11,"_84":11},"read","write","admin","user","storage_size",27149130,"orgData",{"_90":91,"_92":93,"_94":95,"_96":33,"_97":98,"_99":100,"_101":102,"_103":104,"_105":106,"_107":108,"_109":104,"_110":104,"_111":22},"id","1efcb1e807144eb488a9019bb870d0b6","uuid","1efcb1e8-0714-4eb4-88a9-019bb870d0b6","type","organization","orgname","full_name","TokenMill UAB","location","Vilnius, Lithuania","company","Tokenmill","profile_url","","date_joined","2019-05-23T11:28:45.916325Z","gravatar_url","https://www.gravatar.com/avatar/782f49602118372c4695cb865158e6d7?s=80&r=g&d=mm","gravatar_email","badge","is_active","recentTags",[114],{"_115":44,"_116":117,"_118":119,"_90":120,"_121":122,"_50":139,"_140":117,"_141":33,"_142":56,"_57":143,"_144":145,"_146":130,"_147":139,"_148":66,"_149":22},"content_type","creator",6447493,"full_size",10885151,73168471,"images",[123],{"_124":125,"_126":127,"_128":104,"_129":130,"_131":132,"_133":134,"_135":104,"_136":-5,"_137":119,"_63":66,"_138":-5},"architecture","amd64","digest","sha256:c4c34862deff4a457c1504ec3ed9b5a622727b716bbb62e78078ccaa3ef71fa8","features","last_pulled","2026-10-29T07:55:36.939922165Z","last_pushed","2020-05-08T06:36:54Z","os","linux","os_features","os_version","size","variant","2020-05-08T06:36:53.939051Z","last_updater","last_updater_username","media_type","latest","repository",7859884,"tag_last_pulled","tag_last_pushed","tag_status","v2","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","showSbxRunInCloud","showSbxLogoOverride","sbxTrademarkText","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"selectedTag","attestationsUrl","routes/_layout.r.$namespace.$reponame.tags",{"_38":164,"_171":172,"_158":181,"_182":11},{"_40":165,"_42":166,"_45":46,"_47":48,"_49":11,"_50":51,"_52":53,"_54":167,"_57":35,"_32":33,"_58":59,"_60":44,"_61":62,"_63":64,"_65":66,"_67":-5,"_68":62,"_69":70,"_71":11,"_72":33,"_73":168,"_79":11,"_80":170,"_85":33,"_86":87},[],[44],[56],{"_75":11,"_76":169},[78],{"_82":22,"_83":11,"_84":11},"tags",{"_173":64,"_174":-5,"_175":-5,"_176":177},"count","next","previous","results",[178],{"_115":44,"_116":117,"_118":119,"_90":120,"_121":179,"_50":139,"_140":117,"_141":33,"_142":56,"_57":143,"_144":145,"_146":130,"_147":139,"_148":66,"_149":22},[180],{"_124":125,"_126":127,"_128":104,"_129":130,"_131":132,"_133":134,"_135":104,"_136":-5,"_137":119,"_63":66,"_138":-5},[],"canDeleteTags","actionData","errors"]

tokenmill/beagle-dictionary-validator

Tags for tokenmill/beagle-dictionary-validator