[{"_1":2,"_179":-5,"_180":-5},"loaderData",{"_3":4,"_18":19,"_26":27,"_158":159},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7,"_13":14,"_15":16,"_17":11},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","theme","system","contrast","normal","dockerRebrandTour","routes/_layout",{"_20":-5,"_21":22,"_23":22,"_24":11,"_25":22},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","showHoneypotTrap","routes/_layout.r.$namespace.$reponame",{"_28":29,"_30":-5,"_31":11,"_32":11,"_33":34,"_35":36,"_37":-5,"_38":11,"_39":40,"_89":90,"_108":109,"_146":-5,"_147":11,"_148":22,"_149":11,"_150":11,"_151":-5,"_152":11,"_153":-5,"_154":155,"_156":110,"_157":-7},"canonicalUrl","https://hub.docker.com/r/mathwebsearch/mwsapi/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","mathwebsearch","reponame","mwsapi","cloudBuildSource","isAutobuildsAvailable","repo",{"_41":42,"_43":44,"_46":47,"_48":49,"_50":11,"_51":52,"_53":54,"_55":56,"_58":36,"_33":34,"_59":60,"_61":45,"_62":63,"_64":65,"_66":67,"_68":-5,"_69":63,"_70":71,"_72":11,"_73":34,"_74":75,"_80":11,"_81":82,"_86":34,"_87":88},"categories",[],"content_types",[45],"image","date_registered","2019-04-17T12:36:58.413825Z","description","","is_private","last_updated","2019-10-09T13:52:27.640581Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[57],"application/vnd.docker.container.image.v1+json","name","pull_count",429,"repository_type","star_count",0,"status",1,"status_description","active","source","collaborator_count","full_description","# mws-api\n\n[![Build Status](https://travis-ci.org/MathWebSearch/mwsapi.svg?branch=master)](https://travis-ci.org/MathWebSearch/mwsapi)\n\nA [golang](https://golang.org) library and set of tools to setup, query and maintain a MathWebSearch and Temasearch instance.\n\n## Overview\n\n- `cmd`: Implementation of commands\n - `cmd/temaquery`: Queries A joined MathWebSearch + ElasticSearch Setup\n - `cmd/mwsapid`: HTTP Daemon serving temasearch queries\n - `cmd/mwsquery`: Queries a (plain) MathWebSearch instance for MathWebSeach Queries\n - `cmd/elasticquery`: Queries a (plain) Elasticsearch instance for Tema Queries\n - `cmd/elasticsync`: Creates and maintains an Elasticsearch instance for use with Temasearch\n- `connection`: Contains Connection Code to MathWebSearch and ElasticSearch\n- `engine`: Underlying code used by commands above\n - `engine/elasticsync`: Creates and maintains an Elasticsearch instance for use with Temasearch\n - `engine/elasticengine`: ElasticSearch only queries\n - `engine/mwsengine`: MathWebSearch queries\n - `engine/temaengine`: TemaSearch Queries\n- `query`: Implements query parsing and serializing\n- `result`: Implements result parsing and serializing\n- `utils`: General utility functions\n\n## Processes\n\nIn the following we describe the basic functionality between all programs within this repository.\nThis documentation is intended to serve as an entry point, and thus does not describe all implementation details.\nThe most detailed reference is always the source code.\n\nIn principle, the source code for all commands is found in the appropriate `cmd/` subdirectory.\nExecutables can be built by either using thhe standard `go build ./cmd/$CMD` or by simply using `make $CMD`.\nThe binaries will be placed in the root directory.\n\n### MathWebSearch Query\n\nThe program in `cmd/mwsquery` can run plain MathWebSearch Queries.\nQueries are defined by the [Query Struct](mws/wrapper.go).\n\nEach Query consists of a list of MathWebSearch Expressions.\nA MathWebSearch Expression is a ContentMathML expression in XML Syntax with additional support for Query variables.\nFor this one can use `` tags to specify universal variables.\nThe text content of a `` is considered its name and qvars with the same name will match to the same expressions.\n\nEach Expression can be given as an argument to the `mwsquery` executable.\nFor example:\n\n`./mwsquery 'x'`\n\nNormal results are returned as JSON to STDOUT.\nThe results are defined by the [Result Struct](mws/result.go).\n\nAll queries are paginated -- by default they return the first 10 results.\nThe parameters `-from` and `-size` can be used to customize the result set.\n\nSometimes it is only important how many results are returned, not the results themselves.\nFor this purpose the `-count` flag can be provided.\n\nAdditionally, instead of returning the full results, sometimes it is also desired to only return the ids of each found formulae.\nThis can be useful for debugging and use inside a full TemaSearch scenario.\nTo achieve this, the `-ids` flag can be provided.\n\n### Elasticsearch Syncronization\n\nThe program in `cmd/elasticsync` creates and maintains an Elasticsearch Index for use with Tema-Search.\n\nA Temasearch Index is a set of JSON objects conforming to the [Temasearch Harvest Element Schema](tema/Element.go).\nIn the following we call each such document a _Harvest Element_.\nA group of elements belonging to a single .harvest file (which in and of itself belongs to one source file) is usually contained in one line of an elasticsearch index file ending in .json.\nFor backward compatibility, in between each lines of items in the index, an additional document containing legacy meta-information should be stored.\nThese .json files are stored within one folder on disk.\n\nIn order to make this index queryable, it needs to be kept in sync with an appropriate Elasticsearch index.\nTo achieve this one could in principle perform the following process to syncronize the disk with the index:\n\n- Delete all existing indexed documents from Elasticsearch (if any)\n- Read each `.json` file from disk and then\n- add the documents contained inside of it to Elasticsearch\n\nThis approach does not scale well with large datasets.\nHaving to delete the entire database, only to add the same content back is too slow.\n\nInstead we split the Temasearch index into into the different files and treat each file seperatly.\nWe call each file a _segment_.\nTo syncronize an updated on-disk index into Elasticsearch, we roughly do the following:\n\n- Mark all existing segments in the database as 'untouched'\n- For each segment from the ElasticSearch index to be added:\n - compute a hash of the segment\n - check if this segment with the same name is already stored in the database by comparing the hash\n - if yes, we do not need to do anything as it has not changed\n - if no, we remove the old segment documents (if any) and add the new documents belonging to this hash\n - mark the segment as 'touched' within this syncronization process\n- Delete the documents belonging to any segment still marked as 'untouched'\n\nThis process is far more efficient -- only updating documents in the database that have actually been changed.\nHowever, this process requires that two seperate ElasticSearch indexes are maintained.\nThe first index -- called tema by convention -- contains the TemaSearch Index Documents and is most obvious.\nThe second index is called tema-segments and contains a list of known segments as well as their hashes.\nAs a hash implementation we use SHA256.\n\n### Elasticsearch Query\n\nThe program in `cmd/elasticquery` can run queries against the elasticsearch part of Temasearch.\nQueries are defined by the [Query Struct](tema/query/main.go) and consist of two parameters:\n\n- Some text to search the index for\n- A list of mathwebsearch ids that were found by normal MathWebSearch\n\nA query may have both text and ids to search for, but it must not be empty.\nThese can be provided to `elasticquery` using the `text` and `ids` parameters.\nFor example:\n\n`./elasticquery -text \"Hello\" -ids 1,2,3`\n\nNormal results are returned as JSON to STDOUT.\nThe results are defined by the [Result Struct](tema/query/main.go).\n\nAll queries are paginated -- by default they return the first 10 results.\nThe parameters `-from` and `-size` can be used to customize the result set.\n\nSometimes it is only important how many results are returned, not the results themselves.\nFor this purpose the `-count` flag can be provided.\n\nInternally, each query consists of two phases:\n\n- The Document Phase. This intially queries elasticsearch to find all matching documents.\n- The Highlight Phase. For each returned document, elasticsearch is queried again to highlight matching segments.\n\nA normal query runs both phases.\nFor debugging, it is possible to only run the document phase by running the `-document-phase-only` flag.\n\n## API Daemon\n\nThe program in `cmd/mwsapid` implements an HTTP Daemon that can answer all the queries above.\nIt can be configured using the command line arguments:\n\n```\nUsage of mwsapid:\n -elastic-host string\n Host to use for elasticsearch\n -elastic-port int\n Port to use for elasticsearch (default 9200)\n -mws-host string\n Host to use for mathwebsearch. If omitted, disable mathwebsearch support\n -mws-port int\n Port to use for mathwebsearch (default 8080)\n -host string\n Host to listen on (default \"localhost\")\n -port int\n Port to listen on for queries (default 3000)\n```\n\n### General Structure\n\nThe server supports three kinds of requests -- which are described in more detail below.\nWhen using `POST` requests, all parameters should be encoded using JSON in the body.\n\nFor all requests, the server will respond with a JSON object in the body.\nBy default, this corresponds to a simple `application/json` response.\nHowever, when the URL-parameter `callback` is provided, a [`JSONP`](https://web.archive.org/web/20160304044218/http://www.json-p.org/) response is sent instead.\n\nFurthermore, the server makes use of the following status codes:\n\n| Code | Description | Meaning |\n| ---- | --------------------- | ------------------------------------------------------------------------------ |\n| 200 | OK | Request suceeded and the body will contain the desired response. |\n| 400 | Bad Request | Malformed request, this occurs if some parameters are out of range or missing. |\n| 404 | Not Found | The given request was not found or is not supported by the server. |\n| 405 | Method Not Allowed | The request method (i.e. POST or GET) is not allowed for the given request. |\n| 500 | Internal Server Error | Something went wrong when trying to answer the query. |\n\nWhen responding with a non-200 status code, the body will always contain a JSON string with a detailed error message.\nThis message is not intended for end users, instead it should be used by developers to debug the issue at hand.\n\n### Search Result Serialization\n\n#### Result\n\nA search results are represented using [the Result struct](result/result.go) as follows:\n\n| Field | Type | Optional | Description |\n| --------- | ---------------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------- |\n| kind | `string` | no | Type of response returned. One of `\"mwsd\"`, `\"tema\"`, `\"elastic\"`, `\"elastic-document\"` or `\"\"`. The `elastic` ones are currently unused. |\n| took | `number` | yes | Time it took to run the query in Nanoseconds. Optional. |\n| stats | `Dict` | yes | Time in nanoseconds spent in specific phases. Component names may change in the future. |\n| total | `number` | no | Total number of results for the query, regardless how many are returned. |\n| from | `number` | no | 0-based number this set of results starts at. |\n| size | `number` | no | Number of results returned. |\n| variables | `Array` | yes | Query Variables found within the original query. See detailed description below. |\n| ids | `Array` | yes | Internal result ids, when requested. |\n| hits | `Array` | yes | The list of matching hits. See detailed description below. |\n\n#### Hit\n\nA hit is represented using [the Hit struct](result/hit.go) as following:\n\n| Field | Type | Optional | Description |\n| -------- | -------------------- | -------- | ----------------------------------------------------------------------------------------------------- |\n| id | `string` | yes | (Possibly internal) id of this hit. |\n| url | `string` | yes | Url of this hit. |\n| xpath | `string` | yes | Xpath of the query term to the formulae referred to by this hit. |\n| element | `HarvestElement` | yes | Harvest element (`aka element`) belonging to this hit. See below for detailed description. |\n| score | `number` | yes | Score of this Hit as determined by ElasticSearch. |\n| snippets | `Array` | yes | Snipets that caused this hit to gain the score. TemaSearch only. |\n| math_ids | `Array` | no | Formulae found within this hit. See detailed description below. |\n\n#### HarvestElement\n\nAn `` element is represented using [the HarvestElement struct](result/harvest.go) as follows:\n\n| Field | Type | Optional | Description |\n| -------- | ---------------------------------------- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| metadata | `any` | no | Metadata belonging to this element. When representing a valid JSON object, returns that JSON object. When empty, an empty JSON obeject. Otherwise a string representing the XML. |\n| segment | `string` | yes | Name of the segment (i.e. `.harvest` file) this element comes from. |\n| text | `string` | no | Text contained in this document. Might contained replaced math ids. |\n| mws_id | `Dict` | yes | Information about each math identifier within this document, and a map from the internal mws ids to the appropriate formula. See below for detailed description. |\n| mws_ids | `Array` | yes | List of math identifiers in this documents. Corresponds to the keys of `mws_id`. |\n| math | `Dict` | no | Source code of replaced math elements within this document. |\n\n#### MathFormula\n\nA math formula is represented using [the MathFormula struct](result/formula.go) as following:\n\n| Field | Type | Optional | Description |\n| ------- | ---------------------- | -------- | ------------------------------------------------------------- |\n| source | `string` | yes | MathML Element Representing entire formula. |\n| durl | `string` | yes | Document URL this formula is contained in. |\n| url | `string` | yes | Local Formula Id |\n| xpath | `string` | yes | XPath from the formula to the matching subterm. |\n| subterm | `string` | yes | MathML Element representing the matching subterm. |\n| subst | `Dict` | yes | MathML Elements representing values for the subsituted terms. |\n\n#### QueryVariable\n\nA query variable is represented using [the QueryVariable struct](result/variable.go) as following:\n\n| Field | Type | Optional | Description |\n| ----- | -------- | -------- | -------------------------------------------------------- |\n| name | `string` | no | Name of this Query Variable |\n| xpath | `string` | no | XPath from the root of the query to the variable itself. |\n\n### Status Request\n\nThe Status Handler is called running a GET on `/`.\nIt takes no parameters and returns a [StatusResponse](engine/server.go) with the following structure:\n\n| Field | Type | Optional | Description |\n| ------- | -------------------- | -------- | -------------------------------------- |\n| name | `string` | no | Name of this server. Always `mwsapid`. |\n| tagline | `string` | no | Server Tagline. |\n| engines | `Dict` | no | Supported \"engines\" or routes. |\n\nExample Response:\n\n```json\n{\n \"name\": \"mwsapid\",\n \"tagline\": \"You know, for math\",\n \"engines\": {\n \"mws\": true,\n \"tema\": false\n }\n}\n```\n\n### MathWebSearch Request\n\nThe MWS Handler is called running a POST on `/mws/`.\nIt takes parameters of type [MWSAPIRequest](engine/mwsengine/handler.go).\n\n| Field | Type | Optional | Description |\n| ----------- | --------------- | -------- | --------------------------------------------------------------------------------------------------------------------------------------------- |\n| expressions | `Array` | yes | List of MathWebSearch expressions. Each should be a the body of a single \"mws:expression\" tag, using the \"mws\" and \"m\" predefined namespaces. |\n| mwsids | `boolean` | yes | When true, do not return MathWebSearch results, but only their IDs |\n| count | `boolean` | yes | When true, return only count of results, not results themselves. |\n| from | `number` | yes | Used for pagination. 0-based index to start result set at. Defaults to 0, must be >= 0. |\n| size | `number` | yes | Used for pagination. Maximum number of results to returns. Defaults to 10, must be between 0 and 100 inclusive. |\n\nFor example, when the server is running on localhost at port 3000, the following curl command could be used to make a simple request:\n\n`curl -d '{\"expressions\":[\"x\"]}' -H \"Content-Type: application/json\" -X POST http://localhost:3000/mws/`\n\nIf the count parameter is true, the response will be a single json number.\nOtherwise, the server will return a [Result](result/result.go) struct, see the above section for details on how this looks.\nThis behaviour is identical to the `mwsquery` executable.\nExample responses can be found in the [cmd/mwsquery/cmd/testdata](cmd/mwsquery/cmd/testdata) folder.\n\n### TmeaSearch Request\n\nNot yet documented.\n\n## Docker\n\nFor convenience, a Dockerfile serving the `API` daemon is provided.\nIt can be found at the automated build [mathwebsearch/mwsapi](https://hub.docker.com/r/mathwebsearch/mwsapi) on DockerHub.\nIt can be run as follows:\n\n```\ndocker run mathwebsearch/mwsapi\n```\n\nIt serves the API Daemon (see above) on port 3000 by default and can be customized using the following environment variables:\n\n- MWSAPID_HOST: Host to listen for requests. Defaults to \"0.0.0.0\".\n- MWSAPID_PORT: Port to listen for requests. Defaults to 3000.\n\n- MWSAPID_MWS_HOST: Host to expect MathWebSearch Daemon on. If omitted, MathWebSearch support is disabled.\n- MWSAPID_MWS_PORT: Port to expect MathWwebSearch Daemon on. Defaults to 8080.\n\n- MWSAPID_ELASTIC_HOST: Host to expect Elasticsearch Daemon on. If omitted, TemaSearch support is disabled.\n- MWSAPID_ELASTIC_PORT: Port to expected Elasticsearch on. Default to 9200.\n\nFurthermore, a Docker Image for elasticsync also exists.\nSee [MathWebSearch/tema-elasticsync](https://github.com/MathWebSearch/tema-elasticsync) for details.\n\n## License\n\nGPL3, see [LICENSE](LICENSE).\n","has_starred","hub_user","immutable_tags_settings",{"_76":11,"_77":78},"enabled","rules",[79],".*","is_automated","permissions",{"_83":22,"_84":11,"_85":11},"read","write","admin","user","storage_size",669180663,"orgData",{"_91":92,"_93":94,"_95":96,"_97":34,"_98":49,"_99":49,"_100":49,"_101":49,"_102":103,"_104":49,"_105":49,"_106":49,"_107":22},"id","b624cdbf371b4ed49c29e128e6ebbc73","uuid","b624cdbf-371b-4ed4-9c29-e128e6ebbc73","type","organization","orgname","full_name","location","company","profile_url","date_joined","2019-01-23T15:26:48.553065Z","gravatar_url","gravatar_email","badge","is_active","recentTags",[110],{"_111":45,"_112":113,"_114":115,"_91":116,"_117":118,"_51":135,"_136":113,"_137":34,"_138":57,"_58":139,"_140":141,"_142":126,"_143":135,"_144":67,"_145":22},"content_type","creator",5212444,"full_size",5485684,52971237,"images",[119],{"_120":121,"_122":123,"_124":49,"_125":126,"_127":128,"_129":130,"_131":49,"_132":-5,"_133":115,"_64":67,"_134":-5},"architecture","amd64","digest","sha256:9e4556ffa2ac22df0c172e3382b03d05c3d6cfc568d45538f50f575a80da4d94","features","last_pulled","2026-10-19T20:57:07.454863145Z","last_pushed","2019-10-09T13:52:28Z","os","linux","os_features","os_version","size","variant","2019-10-09T13:52:27.265585Z","last_updater","last_updater_username","media_type","latest","repository",6991582,"tag_last_pulled","tag_last_pushed","tag_status","v2","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","showSbxRunInCloud","showSbxLogoOverride","sbxTrademarkText","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"selectedTag","attestationsUrl","routes/_layout.r.$namespace.$reponame.tags",{"_39":160,"_167":168,"_154":177,"_178":11},{"_41":161,"_43":162,"_46":47,"_48":49,"_50":11,"_51":52,"_53":54,"_55":163,"_58":36,"_33":34,"_59":60,"_61":45,"_62":63,"_64":65,"_66":67,"_68":-5,"_69":63,"_70":71,"_72":11,"_73":34,"_74":164,"_80":11,"_81":166,"_86":34,"_87":88},[],[45],[57],{"_76":11,"_77":165},[79],{"_83":22,"_84":11,"_85":11},"tags",{"_169":65,"_170":-5,"_171":-5,"_172":173},"count","next","previous","results",[174],{"_111":45,"_112":113,"_114":115,"_91":116,"_117":175,"_51":135,"_136":113,"_137":34,"_138":57,"_58":139,"_140":141,"_142":126,"_143":135,"_144":67,"_145":22},[176],{"_120":121,"_122":123,"_124":49,"_125":126,"_127":128,"_129":130,"_131":49,"_132":-5,"_133":115,"_64":67,"_134":-5},[],"canDeleteTags","actionData","errors"]

mathwebsearch/mwsapi

Tags for mathwebsearch/mwsapi