[{"_1":2,"_154":-5,"_155":-5},"loaderData",{"_3":4,"_13":14,"_21":22,"_133":134},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","routes/_layout",{"_15":-5,"_16":17,"_18":17,"_19":11,"_20":17},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","showHoneypotTrap","routes/_layout.r.$namespace.$reponame",{"_23":24,"_25":-5,"_26":11,"_27":11,"_28":29,"_30":31,"_32":-5,"_33":11,"_34":35,"_84":-5,"_85":86,"_124":-5,"_125":11,"_126":17,"_127":11,"_128":-5,"_129":11,"_130":-5,"_131":132},"canonicalUrl","https://hub.docker.com/r/trozz/elastic-data-generator/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","trozz","reponame","elastic-data-generator","cloudBuildSource","isAutobuildsAvailable","repo",{"_36":37,"_38":39,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":51,"_53":31,"_28":29,"_54":55,"_56":40,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":58,"_65":66,"_67":11,"_68":29,"_69":70,"_75":11,"_76":77,"_81":29,"_82":83},"categories",[],"content_types",[40],"image","date_registered","2019-03-01T13:18:37.605418Z","description","","is_private","last_updated","2019-03-01T13:27:14.416347Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[52],"application/vnd.docker.container.image.v1+json","name","pull_count",169,"repository_type","star_count",0,"status",1,"status_description","active","source","collaborator_count","full_description","Elasticsearch For Beginners: Generate and Upload Randomized Test Data\n====\nBecause everybody loves test data.\n\n\n### Ok, so what is this thing doing?\n\n`es_test_data.py` lets you generate and upload randomized test data to your ES cluster so you can start running queries, see what performance is like, and verify your cluster is able to handle the load.
\nIt allows for easy configuring of what the test documents look like, what kind of data types they include and what the field names are called.\n\n## Cool, how do I use this? \n\n#### Prerequisites\n\nLet's assume you have an Elasticsearch cluster running. If not, [set it up locally](http://ohardt.us/es-install) and point your browser to [http://localhost:9200](http://localhost:9200) to see if it's up.\n\nPython and [Tornado](https://github.com/tornadoweb/tornado/) and [NumPy](http://www.numpy.org/) are used, run `pip install tornado numpy` to install Tornado and NumPy if you don't have it already.\n\n#### Lets get started\n\nIt's as simple as this:\n\n```\n$ python es_test_data.py --es_url=http://localhost:9200\n[I 150604 15:43:19 es_test_data:42] Trying to create index http://localhost:9200/test_data\n[I 150604 15:43:19 es_test_data:47] Guess the index exists already\n[I 150604 15:43:19 es_test_data:184] Generating 10000 docs, upload batch size is 1000\n[I 150604 15:43:19 es_test_data:62] Upload: OK - upload took: 25ms, total docs uploaded: 1000\n[I 150604 15:43:20 es_test_data:62] Upload: OK - upload took: 25ms, total docs uploaded: 2000\n[I 150604 15:43:20 es_test_data:62] Upload: OK - upload took: 19ms, total docs uploaded: 3000\n[I 150604 15:43:20 es_test_data:62] Upload: OK - upload took: 18ms, total docs uploaded: 4000\n[I 150604 15:43:20 es_test_data:62] Upload: OK - upload took: 27ms, total docs uploaded: 5000\n[I 150604 15:43:20 es_test_data:62] Upload: OK - upload took: 19ms, total docs uploaded: 6000\n[I 150604 15:43:20 es_test_data:62] Upload: OK - upload took: 15ms, total docs uploaded: 7000\n[I 150604 15:43:20 es_test_data:62] Upload: OK - upload took: 24ms, total docs uploaded: 8000\n[I 150604 15:43:20 es_test_data:62] Upload: OK - upload took: 32ms, total docs uploaded: 9000\n[I 150604 15:43:20 es_test_data:62] Upload: OK - upload took: 31ms, total docs uploaded: 10000\n[I 150604 15:43:20 es_test_data:216] Done - total docs uploaded: 10000, took 1 seconds\n[I 150604 15:43:20 es_test_data:217] Bulk upload average: 23 ms\n[I 150604 15:43:20 es_test_data:218] Bulk upload median: 24 ms\n[I 150604 15:43:20 es_test_data:219] Bulk upload 95th percentile: 31 ms\n$\n```\n \nWithout any command line options, it will generate and upload 1000 documents of the format\n\n```\n{\n \"name\":<>,\n \"age\":<>,\n \"last_updated\":<>\n}\n```\nto an Elasticsearch cluster at `http://localhost:9200` to an index called `test_data`.\n\n#### Not bad but what can I configure?\n\n`python es_test_data.py --help` gives you the full set of command line options, here are the most important ones:\n\n- `--es_url=http://localhost:9200` the base URL of your ES node, don't include the index name\n- `--count=###` number of documents to generate and upload\n- `--index_name=test_data` the name of the index to upload the data to. If it doesn't exist it'll be created with these options\n - `--num_of_shards=2` the number of shards for the index\n - `num_of_replicas=0` the number of replicas for the index\n- `--batch_size=###` we use bulk upload to send the docs to ES, this option controls how many we send at a time\n- `--force_init_index=False` if `True` it will delete and re-create the index\n- `--dict_file=filename.dic` if provided the `dict` data type will use words from the dictionary file, format is one word per line. The entire file is loaded at start-up so be careful with (very) large files. You can download wordlists e.g.. from [here](http://ohardt.us/word-lists). \n\n#### What about the document format? \n\nGlad you're asking, let's get to the doc format.
\nThe doc format is configured via `--format=<>` with the default being `name:str,age:int,last_updated:ts`.\n\nThe general syntax looks like this:\n\n`<>:<>,<>::<>, ...`\n\nFor every document, `es_test_data.py` will generate random values for each of the fields configured.\n\nCurrently supported field types are:\n\n- `bool` returns a random true or false\n- `ts` a timestamp (in milliseconds), randomly picked between now +/- 30 days\n- `ipv4` returns a random ipv4\n- `tstxt` a timestamp in the \"%Y-%m-%dT%H:%M:%S.000-0000\" format, randomly picked between now +/- 30 days\n- `int:min:max` a random integer between `min` and `max`. If `min and `max` are not provided they default to 0 and 100000\n- `str:min:max` a word ( as in, a string), made up of `min` to `max` random upper/lowercase and digit characters. If `min` and `max` are optional, defaulting to `3` and `10`\n- `words:min:max` a random number of `strs`, separated by space, `min` and `max` are optional, defaulting to '2' and `10`\n- `dict:min:max` a random number of entries from the dictionary file, separated by space, `min` and `max` are optional, defaulting to '2' and `10`\n- `text:words:min:max` a random number of words seperated by space from a given list of `-` seperated words, the words are optional defaulting to `text1` `text2` and `text3`, min and max are optional, defaulting to `1` and `1`\n\n\n\n## todo\n- document the remaining cmd line options\n- more different format types\n- ...\n\nAll suggestions, comments, ideas, pull requests are welcome!\n\n","has_starred","hub_user","immutable_tags_settings",{"_71":11,"_72":73},"enabled","rules",[74],".*","is_automated","permissions",{"_78":17,"_79":11,"_80":11},"read","write","admin","user","storage_size",390216930,"orgData","recentTags",[87],{"_88":40,"_89":90,"_91":92,"_93":94,"_95":96,"_46":113,"_114":90,"_115":29,"_116":52,"_53":117,"_118":119,"_120":104,"_121":113,"_122":62,"_123":17},"content_type","creator",37330,"full_size",390206199,"id",48372200,"images",[97],{"_98":99,"_100":101,"_102":44,"_103":104,"_105":106,"_107":108,"_109":44,"_110":-5,"_111":92,"_59":62,"_112":-5},"architecture","amd64","digest","sha256:159c7304f22d12563a1d5fed42fce6bca00adf9bf4bbcedb2e132f96b72252af","features","last_pulled","2026-09-21T20:48:06.710211657Z","last_pushed","2019-03-01T13:27:15Z","os","linux","os_features","os_version","size","variant","2019-03-01T13:27:14.061716Z","last_updater","last_updater_username","media_type","latest","repository",6762449,"tag_last_pulled","tag_last_pushed","tag_status","v2","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","showSbxLogoOverride","sbxTrademarkText","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"routes/_layout.r.$namespace.$reponame.tags",{"_34":135,"_142":143,"_131":152,"_153":11},{"_36":136,"_38":137,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":138,"_53":31,"_28":29,"_54":55,"_56":40,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":58,"_65":66,"_67":11,"_68":29,"_69":139,"_75":11,"_76":141,"_81":29,"_82":83},[],[40],[52],{"_71":11,"_72":140},[74],{"_78":17,"_79":11,"_80":11},"tags",{"_144":60,"_145":-5,"_146":-5,"_147":148},"count","next","previous","results",[149],{"_88":40,"_89":90,"_91":92,"_93":94,"_95":150,"_46":113,"_114":90,"_115":29,"_116":52,"_53":117,"_118":119,"_120":104,"_121":113,"_122":62,"_123":17},[151],{"_98":99,"_100":101,"_102":44,"_103":104,"_105":106,"_107":108,"_109":44,"_110":-5,"_111":92,"_59":62,"_112":-5},[],"canDeleteTags","actionData","errors"]

trozz/elastic-data-generator

Tags for trozz/elastic-data-generator