[{"_1":2,"_169":-5,"_170":-5},"loaderData",{"_3":4,"_13":14,"_21":22,"_144":145},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","routes/_layout",{"_15":-5,"_16":17,"_18":17,"_19":11,"_20":11},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","showHoneypotTrap","routes/_layout.r.$namespace.$reponame",{"_23":24,"_25":-5,"_26":11,"_27":11,"_28":29,"_30":31,"_32":-5,"_33":11,"_34":35,"_84":-5,"_85":86,"_135":-5,"_136":11,"_137":17,"_138":11,"_139":-5,"_140":11,"_141":-5,"_142":143},"canonicalUrl","https://hub.docker.com/r/felix/esfdw/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","felix","reponame","esfdw","cloudBuildSource","isAutobuildsAvailable","repo",{"_36":37,"_38":39,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":51,"_53":31,"_28":29,"_54":55,"_56":40,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":58,"_65":66,"_67":11,"_68":29,"_69":70,"_75":17,"_76":77,"_81":29,"_82":83},"categories",[],"content_types",[40],"image","date_registered","2017-09-11T11:33:07.701777Z","description","Elasticsearch Foreign Data Wrapper (FDW) for Postgres\n\nBased on https://github.com/rtkwlf/esfdw","is_private","last_updated","2017-09-22T15:01:20.816261Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[52],"application/vnd.docker.container.image.v1+json","name","pull_count",437,"repository_type","star_count",0,"status",1,"status_description","active","source","collaborator_count","full_description","# This repository\nPlease note that code in this repo is for **Elasticsearch 5** only! Go upstream for something that works on lower versions!\n\n# esfdw\n\nesfdw is a PostgreSQL [foreign data wrapper](http://www.postgresql.org/docs/current/static/postgres-fdw.html) for Elasticsearch.\n\nElasticsearch is widely used for document and log data storage, in particular as part of the [ELK stack](https://www.elastic.co/webinars/introduction-elk-stack). esfdw allows PostgreSQL to be used as a query engine for data stored in Elasticsearch. Use cases include:\n * Writing SQL JOIN queries against Elasticsearch documents and letting the PostgreSQL engine do the heavy lifting\n * Running window functions on data stored in Elasticsearch\n * Applying PostgreSQL aggregations that do not currently have a native Elasticsearch equivalent\n\nesfdw depends on [Multicorn](http://multicorn.org), a PostgreSQL extension for writing foreign data wrappers in Python.\n\n## Features\n\n#### Supported functionality\n\n * SELECT queries\n \n * Converting many common PostgreSQL operators to Elasticsearch filters, which are then used in the Elasticsearch query that retrieves the documents. This greatly improves performance by reducing the amount of data that needs to be fetched from Elasticsearch into PostgreSQL. Operators currently pushed down to Elasticsearch are `=`, `<>`, `LIKE`, `<@` (range), `<`, `<=`, `>`, and `>=`.\n * Using the [`fields` query parameter](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-fields.html) to reduce the amount of data transferred between Elasticsearch and PostgreSQL\n * Specifying the document type and index or indices to use in the Elasticsearch query on a per-table basis\n * [Mostly-automatic generation](#mapping_to_schema) of `CREATE FOREIGN TABLE` DDL from Elasticsearch mappings\n * Estimating the resulting relation size so as to help the planner (`get_rel_size` Multicorn method)\n\n#### Outstanding functionality\n\nThe following is not currently supported, but contributions are always welcome!\n\n * UPDATE or DELETE statements. In other words, the Multicorn write API is not implemented.\n * Pushing down sort (e.g., `ORDER BY`) qualifiers to Elasticsearch.\n * WHERE statements with OR (see [this multicorn issue](https://github.com/Kozea/Multicorn/issues/30))\n\n#### Caveats\n\nesfdw issues Elasticsearch queries using the scan/scroll REST API. The performance, therefore, is greatly affected by the throughput and latency of the network connection between the PostgreSQL server and the Elasticsearch cluster.\n\nWhile any valid PostgreSQL query can be run, only [some operators](#pushed_operators) are pushed down to Elasticsearch. In particular, aggregations are not pushed down at all. Where the query includes components that cannot be pushed down to Elasticsearch, the requisite data set is fetched into PostgreSQL, and any necessary query processing is then performed inside the PostgreSQL engine. For example, a `SELECT COUNT(*) FROM foo WHERE col = 'value'` PostgreSQL query will be translated to an Elasticsearch query with a `term` filter for field `col` and value `value`. All documents matching this filter will be fetched from Elasticsearch and will then be counted by PostgreSQL.\n\n## Requirements\n\n * Multicorn 1.2.x and up.\n * esfdw has only been tested with Python 2.7, although there should be no major roadblocks to Python 3 support. If you try it with Python 3, let us know how it goes.\n * esfdw has been tested with Elasticsearch 1.7.x. However, the queries used by Elasticsearch are not exotic and should work with other Elasticsearch versions. Let us know if you run into problems.\n * esfdw uses the `elasticsearch-py` Elasticsearch client.\n\n## Installation\n\n * [Install Multicorn](http://multicorn.org/#idid3).\n * Install the `elasticsearch` pip. **Please ensure that you install a version that's [appropriate for your version of Elasticsearch](http://elasticsearch-py.readthedocs.org/en/master/#compatibility).**\n * `pip setup.py install` to install esfdw directly from the git repository.\n\n## Usage\n\n### Server creation\n\nExample:\n\n```sql\nCREATE EXTENSION multicorn;\nCREATE SERVER es_srv FOREIGN DATA WRAPPER multicorn OPTIONS (\n wrapper 'esfdw.ESForeignDataWrapper',\n hostname 'my.es_server.com',\n port '9200'\n);\n```\n\n##### Options\n\n * `hostname` is the hostname of the Elasticsearch server. If unspecified, it defaults to `localhost`.\n * `port` is the port number for the connection to the Elasticsearch server. If unspecified, it defaults to `9200`.\n\n### Foreign table creation\n\nIn the default implementation, a esfdw foreign table corresponds to exactly one doc_type. This doc_type is specified as a mandatory option to the `CREATE FOREIGN TABLE` statement.\n\nSimilarly, the `CREATE FOREIGN TABLE` statement also requires an option to specify the index or indices to use for the search ([multiple indices notation](https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-index.html) is supported). If more sophisticated behaviour to derive the index name is desired (e.g., dynamically deriving the index based on the constraints in a query), this can be implemented by subclassing `esfdw.ESForeignDataWrapper` and overriding the `get_index` method.\n\nExample:\n\n```sql\nCREATE FOREIGN TABLE foreign_es_table (\n col1 int,\n col2 text,\n nested_object__a text,\n nested_object__b text\n) server es_srv OPTIONS (\n index 'logstash-2015.12.*',\n doc_type 'my_log',\n column_name_translation 'true'\n);\n```\n\n##### Options\n\n * `index` is the value of the index parameter to use in Elasticsearch searches.\n * `doc_type` is the value of the doc_type parameter to use in Elasticsearch searches.\n \n * `loglevel` can be one of `DEBUG`, `INFO`, `WARNING`, `ERROR` or `CRITICAL` (see the\n [multicorn docs](http://multicorn.readthedocs.io/en/latest/implementing-tutorial.html#error-reporting)\n * Setting `debug` to `true` prints debugging information to STDOUT of the **server** process.\n\n\n#### Automatic (mostly) Elasticsearch mapping conversion to foreign table schema\n\nThe `esfdw.mapping_to_schema` module can be used to convert Elasticsearch field mappings to foreign table schema in a mostly-automatic way. Run `python -m esfdw.mapping_to_schema -h` for usage details.\n\nGiven a JSON file with Elasticsearch mappings (see https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html for the concept and https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-get-mapping.html for an explanation of how to extract from a running cluster), the script generates corresponding `CREATE FOREIGN TABLE` statements.\n\nThe script generates a foreign table per doc_type per index, with the name of the table derived from the name of the doc_type, except that dashes are replaced with underscores.\n\nColumn types are translated from the Elasticsearch equivalent and are always scalar. Nested objects are not represented as JSON; instead, a column definition is generated for every nested leaf field. Elasticsearch mappings do not contain an indication of whether the field is a list field, which means that the script cannot know when to make a column an array. The schema can be fixed up manually if array columns are desired.\n\nColumn names match the Elasticsearch field names except that [standard esfdw name translation rules are applied](#column_name_translation).\n\n### Debugging\n\nesfdw logs at the debug level the Elasticsearch queries that it issues. To see the message, you need to [configure the PostgreSQL log level](http://www.postgresql.org/docs/current/static/runtime-config-logging.html) to display DEBUG messages.\n\nFor example, to have the Elasticsearch query displayed in psql, run the following in your psql session:\n`SET client_min_messages TO DEBUG;`\n\n## Contributing\n\n1. Fork it\n2. Create your feature branch (`git checkout -b my-new-feature`)\n3. Commit your changes (`git commit -am 'Add some feature'`)\n4. Make sure the tests are passing (`python setup.py test`)\n5. Push to the branch (`git push origin my-new-feature`)\n6. Create a new Pull Request\n\nPlease include new tests with your changes! Tests live in the `tests` directory, and test files are named `_test.py`, where `` is the name of the module under test in the `esfdw` directory.\n\n# Development\nSQL statements live in `sql/`. Elasticsearch scripts and helpers can be found in `dev/`.\n\nStart it up, put some documents into Elasticsearch and look them up:\n```\ncp .env.sample .env\ncp dev/docker-compose.override.yml .\ndocker-compose up -d\n./dev/index.sh\n./dev/search.sh\n```\n\nUse the FDW:\n```\n./dev/query < sql/server.sql\n./dev/query < sql/table.sql\n./dev/query < sql/select.sql\n./dev/query < sql/json_select.sql\n./tests/integration/run.sh\n```\n","has_starred","hub_user","immutable_tags_settings",{"_71":11,"_72":73},"enabled","rules",[74],".*","is_automated","permissions",{"_78":17,"_79":11,"_80":11},"read","write","admin","user","storage_size",198435686,"orgData","recentTags",[87,125],{"_88":40,"_89":90,"_91":92,"_93":94,"_95":96,"_46":114,"_115":90,"_116":29,"_117":52,"_53":118,"_119":120,"_121":105,"_122":114,"_123":62,"_124":17},"content_type","creator",3862,"full_size",197180991,"id",15103705,"images",[97],{"_98":99,"_100":101,"_102":103,"_104":105,"_106":107,"_108":109,"_110":103,"_111":-5,"_112":92,"_59":62,"_113":-5},"architecture","amd64","digest","sha256:4d7912695f79c4c4e5fe0ec120dbe75fa30a906333ec7dc88bdb0b84a3be6a86","features","","last_pulled","2026-09-24T04:57:40.154387892Z","last_pushed","2017-09-22T15:01:20Z","os","linux","os_features","os_version","size","variant","2017-09-22T15:01:19.513776Z","last_updater","last_updater_username","media_type","latest","repository",1809544,"tag_last_pulled","tag_last_pushed","tag_status","v2",{"_88":40,"_89":90,"_91":126,"_93":127,"_95":128,"_46":133,"_115":90,"_116":29,"_117":52,"_53":134,"_119":120,"_121":131,"_122":133,"_123":62,"_124":17},197180688,15329843,[129],{"_98":99,"_100":130,"_102":103,"_104":131,"_106":132,"_108":109,"_110":103,"_111":-5,"_112":126,"_59":62,"_113":-5},"sha256:8a6842713285c9d30e4f7271dfb6b7cf3e17d1d2f2454244f8f352b75fe1c865","2026-09-17T19:24:45.495598787Z","2017-09-15T13:17:30Z","2017-09-15T13:17:29.457554Z","handle-lists","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","showSbxLogoOverride","sbxTrademarkText","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"routes/_layout.r.$namespace.$reponame.tags",{"_34":146,"_153":154,"_142":167,"_168":11},{"_36":147,"_38":148,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":149,"_53":31,"_28":29,"_54":55,"_56":40,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":58,"_65":66,"_67":11,"_68":29,"_69":150,"_75":17,"_76":152,"_81":29,"_82":83},[],[40],[52],{"_71":11,"_72":151},[74],{"_78":17,"_79":11,"_80":11},"tags",{"_155":156,"_157":-5,"_158":-5,"_159":160},"count",2,"next","previous","results",[161,164],{"_88":40,"_89":90,"_91":92,"_93":94,"_95":162,"_46":114,"_115":90,"_116":29,"_117":52,"_53":118,"_119":120,"_121":105,"_122":114,"_123":62,"_124":17},[163],{"_98":99,"_100":101,"_102":103,"_104":105,"_106":107,"_108":109,"_110":103,"_111":-5,"_112":92,"_59":62,"_113":-5},{"_88":40,"_89":90,"_91":126,"_93":127,"_95":165,"_46":133,"_115":90,"_116":29,"_117":52,"_53":134,"_119":120,"_121":131,"_122":133,"_123":62,"_124":17},[166],{"_98":99,"_100":130,"_102":103,"_104":131,"_106":132,"_108":109,"_110":103,"_111":-5,"_112":126,"_59":62,"_113":-5},[],"canDeleteTags","actionData","errors"]

felix/esfdw

Tags for felix/esfdw