[{"_1":2,"_161":-5,"_162":-5},"loaderData",{"_3":4,"_17":18,"_25":26,"_140":141},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7,"_13":14,"_15":16},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","theme","system","contrast","normal","routes/_layout",{"_19":-5,"_20":21,"_22":21,"_23":11,"_24":21},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","showHoneypotTrap","routes/_layout.r.$namespace.$reponame",{"_27":28,"_29":-5,"_30":11,"_31":11,"_32":33,"_34":35,"_36":-5,"_37":11,"_38":39,"_88":-5,"_89":90,"_129":-5,"_130":11,"_131":21,"_132":11,"_133":-5,"_134":11,"_135":-5,"_136":137,"_138":91,"_139":-7},"canonicalUrl","https://hub.docker.com/r/vanatteveldt/nlpipe/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","vanatteveldt","reponame","nlpipe","cloudBuildSource","isAutobuildsAvailable","repo",{"_40":41,"_42":43,"_45":46,"_47":48,"_49":11,"_50":51,"_52":53,"_54":55,"_57":35,"_32":33,"_58":59,"_60":44,"_61":62,"_63":64,"_65":66,"_67":-5,"_68":62,"_69":70,"_71":11,"_72":33,"_73":74,"_79":21,"_80":81,"_85":33,"_86":87},"categories",[],"content_types",[44],"image","date_registered","2016-11-16T20:17:12.410109Z","description","Easy NLP processing and (caching) pipelining","is_private","last_updated","2019-03-25T17:16:42.797584Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[56],"application/vnd.docker.container.image.v1+json","name","pull_count",1313,"repository_type","star_count",0,"status",1,"status_description","active","source","collaborator_count","full_description","# NLPipe [![Build Status](https://travis-ci.org/vanatteveldt/nlpipe.png?branch=master)](https://travis-ci.org/vanatteveldt/nlpipe)\n\nClient/server based NLP Pipelining\n\nThis is a simple, filesystem-based format- and progress agnostic setup for running document processing.\nThe intended usage is to make it easy to package and distribute different parsers, preprocessors etc.,\nand call them from other programs such as R or python without worrying about dependencies, installation, etc. \n\nComponents:\n\n- Storage\n- HTTP Server\n- Client bindings\n- Workers\n\nInstall and test\n===\n\nRest-server\n---\n\nYou can install a rest-server on your system using PIP or you can use\na ready-to-run docker image. \n\n### As a Docker container\n\nDo (possibly as superuser):\n\n```{sh}\ndocker run --name nlpipe -dp 5001:5001 vanatteveldt/nlpipe\n```\n\nThis will pull the nlpipe docker image and run the nlpipe restserver on port 5001 and by default run all known worker modules. Note: The `-d` means that the docker process will be 'detached', i.e. run in the background. \n\nTo see (or *f*ollow) the logs of a running worker, use:\n\n```{sh}\ndocker logs [-f] nlpipe\n```\n\n### Directly in your computer\n\nNLPipe runs on Python version 3.something.\nTo install nlpipe locally, it is best to create a virtual environment and install nlpipe in it:\n\n```{sh}\npyvenv env\nenv/bin/pip install -e git+git://github.com/vanatteveldt/nlpipe.git#egg=nlpipe\n```\n\nNow you can run nlpipe from the created environment. e.g. to run a\nwebserver that listens http://localhost:5001 in test-mode, open a separate\nterminal, \"source\" the Python virtual environment and do e.g.:\n\n```{sh}\nenv/bin/python -m nlpipe.restserver\n```\n\nThe program prints\n\n`Running on http://localhost:5001/ (Press CTRL+C to quit)`. \n\nThe server runs until you quit this process with CTRL+C. In the mean time it prints\nlogging information in your xterm window.\n\nThe purpose of the server is only to move files around. In order to\nprocess files you need to set up a separate worker. The worker polls\nthe server for new-uploaded texts, performs a task on them and returns\nthe processed texts to the server. NLPipe supplies a\ndemo processor-module `test_upper`. To set up a worker for this module, open a\nseparate xterm, source the Python virtual environment in it and do e.g.:\n\n```{sh}\n$ env/bin/python -m nlpipe.worker http://localhost:5001 test_upper\n```\n\nThe program responds with\n`... Workers active and waiting for input`\nand keeps running. You can see that it polls the server, because the\nserver prints loads of messages like:\n\n`[2017-05-03 12:59:00,844 werkzeug INFO ] 127.0.0.1 - -\n[03/May/2017 12:59:00] \"GET /api/modules/test_upper/ HTTP/1.1\" 404 -`\n\nNote: This is not needed for the Docker server, because workers have\nbeen pre-installed there.\n\n### Test whether it works\n\nNLPipe provides a client to communicate with the server. To use it, do e.g.\n\n```{sh}\n$ env/bin/python -m nlpipe.client http://localhost:5001 test_upper process \"this is a test\"\n0x54b0c58c7ce9f2a8b551351102ee0938\n$ env/bin/python -m nlpipe.client http://localhost:5001 test_upper status 0x54b0c58c7ce9f2a8b551351102ee0938\nDONE\n$ env/bin/python -m nlpipe.client http://localhost:5001 test_upper result 0x54b0c58c7ce9f2a8b551351102ee0938\nTHIS IS A TEST\n```\n\nExplanation: The first line submits the string \"this is a test\" as a\ntask for the `test_upper` processor. The command returns an\nidentifier. The second line requests the status of the task. The\nstatus might be \"PENDING\", \"STARTED\", \"DONE\", \"ERROR\". When the task\nhas been done, the third command retrieves the processed task. \n\n\nExample Setups\n===\n\nCoreNLP lemmatize\n---\n\nTo setup corenlp lemmatize and nlpipe, use:\n\n```{sh}\n$ docker run --name corenlp -dp 9000:9000 chilland/corenlp-docker \n$ docker run --name nlpipe --link corenlp:corenlp -e \"CORENLP_HOST=http://corenlp:9000\" -dp 5001:5001 vanatteveldt/nlpipe\n```\nAnd e.g. lemmatize a test sentence:\n\n```{sh}\n$ docker exec -it nlpipe python -m nlpipe.client /tmp/nlpipe-data corenlp_lemmatize process_inline --format=csv 'this is a test'\nid,sentence,offset,word,lemma,POS,POS1,ner\n0x54b0c58c7ce9f2a8b551351102ee0938,1,0,this,this,DT,D,O\n0x54b0c58c7ce9f2a8b551351102ee0938,1,5,is,be,VBZ,V,O\n0x54b0c58c7ce9f2a8b551351102ee0938,1,8,a,a,DT,D,O\n0x54b0c58c7ce9f2a8b551351102ee0938,1,10,test,test,NN,N,O\n```\n\nDistributed setup\n---\n\nYou can setup a server on one computer and run workers on a different computer. \n\nSetting up the server without any workers:\n\n```{sh}\ndocker run --name nlpipe -dp 5001:5001 vanatteveldt/nlpipe python -m nlpipe.restserver\n```\n\nStarting a `corenlp_lemmatize` worker on a different (or the same) machine (assuming the server runs at `example.com`):\n\n```{sh}\n$ docker run --name corenlp -dp 9000:9000 chilland/corenlp-docker \ndocker run --name nlpipeworker --link corenlp:corenlp -e \"CORENLP_HOST=http://corenlp:9000\" -dp 5001:5001 vanatteveldt/nlpipe python -m nlpipe.worker http://example.com:5001 corenlp_lemmatize\n```\n\nAnd lemmatizing a document from a third machine:\n(note that using a docker is overkill here, it would be better to just use the python or R client)\n\n```{sh}\ndocker run vanatteveldt/nlpipe python -m nlpipe.client http://i.amcat.nl:5001 corenlp_lemmatize process_inline --format csv \"this is a test!\"\n```\n\nDesign\n===\n\nStorage directory layout\n---\n\nThs server uses file system to manage task queue and results cache. \nEach task (e.g. corenlp_lemmatize) contains subfolders containing the documents\n\n```\n- \n - queue\n - in_process\n - results\n - errors\n```\n\nProcess flow:\n- client puts document into `/queue`\n- worker moves a document from `/queue` to `/in_process` and gets the text\n- worker processes the document\n- worker stores the result in `/results` and removes it from `/in_process`\n- client retrieves the document from `/results`\n\nThe goal of this setup is to use the filesystem as a hierarchical database and use the UNIX atomic FS operations as a thread-safe locking/scheduling mechanism. The worker that manages to e.g. move the document from queue to in_process is the one doing the task. If two workers simultaneously select the same document to process, only the first will be able to move it, and the second will get an error from the file system and should select the next document. \n\nBefore putting a document on the queue, a client should check whether it is not already known and then create it. \nThis is not atomic, so it is possible that another thread has created the document at between checking and creating, but in that case the creation will give an error. \nIn the (unlikely) event that another thread has created the document and a worked has moved it to in_process in the interval between checking and creating a document, there is a risk that the document will be processed twice, but this should not lead to a problem except for wasted processing time. \n\n\nClient access\n---\n\nClients/workers can access the filesystem directly. \nSince it is thread safe, this is the most efficient way of \nor use the HTTP server. \n\nThe built-in HTTP server will allow access to the NLPipe service with the following REST endpoints:\n\nFrom client perspective:\n\n```\nPUT / # adds a document by hash\nPOST # adds a document, returning the hash\nHEAD / # gets status of task\nGET / # get result for task (or 404 / error)\n```\n\nFrom worker perspective:\n\n```\nGET # gets one document from task (and moves from queue to in_process)\nGET ?n=N # gets N documents from task (and moves from queue to in_process)\nPUT / # stores result \n```\n\nThere are also client bindings for the direct filesystem access (python) and for the HTTP server (python and R).\nThe python bindings are included in this repository ([nlpipe/client.py](nlpipe/client.py)). R bindings are available at [http://github.com/vanatteveldt/nlpiper](vanatteveldt/nlpiper). \n","has_starred","hub_user","immutable_tags_settings",{"_75":11,"_76":77},"enabled","rules",[78],".*","is_automated","permissions",{"_82":21,"_83":11,"_84":11},"read","write","admin","user","storage_size",7341934039,"orgData","recentTags",[91],{"_92":44,"_93":94,"_95":96,"_97":98,"_99":100,"_50":118,"_119":94,"_120":33,"_121":56,"_57":122,"_123":124,"_125":109,"_126":118,"_127":66,"_128":21},"content_type","creator",1160308,"full_size",381012030,"id",6174602,"images",[101],{"_102":103,"_104":105,"_106":107,"_108":109,"_110":111,"_112":113,"_114":107,"_115":-5,"_116":96,"_63":66,"_117":-5},"architecture","amd64","digest","sha256:77a541c36c244d5d654d5b28ff43d98d87cb8fe429c5a7d909b23e87ea866c0e","features","","last_pulled","2026-09-24T16:02:36.012440688Z","last_pushed","2019-03-25T17:16:43Z","os","linux","os_features","os_version","size","variant","2019-03-25T17:16:42.502246Z","last_updater","last_updater_username","media_type","latest","repository",1051385,"tag_last_pulled","tag_last_pushed","tag_status","v2","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","showSbxLogoOverride","sbxTrademarkText","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"selectedTag","attestationsUrl","routes/_layout.r.$namespace.$reponame.tags",{"_38":142,"_149":150,"_136":159,"_160":11},{"_40":143,"_42":144,"_45":46,"_47":48,"_49":11,"_50":51,"_52":53,"_54":145,"_57":35,"_32":33,"_58":59,"_60":44,"_61":62,"_63":64,"_65":66,"_67":-5,"_68":62,"_69":70,"_71":11,"_72":33,"_73":146,"_79":21,"_80":148,"_85":33,"_86":87},[],[44],[56],{"_75":11,"_76":147},[78],{"_82":21,"_83":11,"_84":11},"tags",{"_151":64,"_152":-5,"_153":-5,"_154":155},"count","next","previous","results",[156],{"_92":44,"_93":94,"_95":96,"_97":98,"_99":157,"_50":118,"_119":94,"_120":33,"_121":56,"_57":122,"_123":124,"_125":109,"_126":118,"_127":66,"_128":21},[158],{"_102":103,"_104":105,"_106":107,"_108":109,"_110":111,"_112":113,"_114":107,"_115":-5,"_116":96,"_63":66,"_117":-5},[],"canDeleteTags","actionData","errors"]

vanatteveldt/nlpipe

Tags for vanatteveldt/nlpipe