[{"_1":2,"_189":-5,"_190":-5},"loaderData",{"_3":4,"_13":14,"_21":22,"_164":165},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","routes/_layout",{"_15":-5,"_16":17,"_18":17,"_19":11,"_20":11},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","showHoneypotTrap","routes/_layout.r.$namespace.$reponame",{"_23":24,"_25":-5,"_26":11,"_27":11,"_28":29,"_30":31,"_32":-5,"_33":11,"_34":35,"_84":85,"_103":104,"_155":-5,"_156":11,"_157":17,"_158":11,"_159":-5,"_160":11,"_161":-5,"_162":163},"canonicalUrl","https://hub.docker.com/r/datopian/data-subscriptions/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","datopian","reponame","data-subscriptions","cloudBuildSource","isAutobuildsAvailable","repo",{"_36":37,"_38":39,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":51,"_53":31,"_28":29,"_54":55,"_56":40,"_57":58,"_59":58,"_60":61,"_62":-5,"_63":64,"_65":66,"_67":11,"_68":29,"_69":70,"_75":11,"_76":77,"_81":29,"_82":83},"categories",[],"content_types",[40],"image","date_registered","2020-04-21T08:42:36.408473Z","description","","is_private","last_updated","2023-02-03T12:34:59.437232Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[52],"application/vnd.docker.container.image.v1+json","name","pull_count",135274,"repository_type","star_count",1,"status","status_description","active","source","collaborator_count",0,"full_description","# Subscriptions and Notifications (for Datasets)\n\nData Subscriptions is a standalone microservice for CKAN that allows users to subscribe to notifications of dataset changes. Notifications are available in several forms: web, email, and SMS (in development). Users can subscribe and unsubcribe to changes, configure the medium for notifications etc. It is feature complete and actively deployed in production.\n\nThe core job story is:\n\n> When I am interested in a given dataset I want to get notifications (in a dashboard, via email etc) when that dataset is updated (or other changes happen) so that I can take action e.g. update my application, check if security issue etc\n\n## Features\n\n* Subscribe to notifications for datasets (+ (maybe) other types of events e.g. someone added or removed organization)\n * From WUI (and email for unwatch)\n* Configure medium of notification e.g. email, mobile, web\n* View notifications stream arising from my watching\n* Review my watched list (and modify it)\n* Admin can configure which datasets are watchable\n* Admin can view stats about watching\n\n### Flows\n\n* Watch (unwatch) flow\n * From WUI\n * Unwatch from email\n * Optional: initiate watch flow without being registered / logged in yet\n* Configure settings flow\n* Admin related flows\n\n## How it works\n\nFor the end-users, it works like this:\n\n[![](https://mermaid.ink/img/eyJjb2RlIjoiZ3JhcGggTFJcbiAgY3VyYXRvcigoRGF0YSBDdXJhdG9yKSlcbiAgdXNlcigoVXNlcikpXG5cbiAgc3ViZ3JhcGggY2thbltcIkNLQU4gQ2xhc3NpY1wiXVxuICAgIGRhdGFzZXRbXCJEYXRhc2V0L1Jlc291cmNlXCJdXG4gIGVuZFxuXG5cbiAgY3VyYXRvciAtLSB1cGRhdGVzIG1ldGFkYXRhIC0tPiBkYXRhc2V0XG4gIGN1cmF0b3IgLS0gdXBkYXRlcyBkYXRhIC0tPiBkYXRhc2V0XG4gIHVzZXIgLS0gc3Vic2NyaWJlcyAtLT4gZGF0YXNldFxuIiwibWVybWFpZCI6eyJ0aGVtZSI6ImRlZmF1bHQifSwidXBkYXRlRWRpdG9yIjpmYWxzZX0)](https://mermaid-js.github.io/mermaid-live-editor/#/edit/eyJjb2RlIjoiZ3JhcGggTFJcbiAgY3VyYXRvcigoRGF0YSBDdXJhdG9yKSlcbiAgdXNlcigoVXNlcikpXG5cbiAgc3ViZ3JhcGggY2thbltcIkNLQU4gQ2xhc3NpY1wiXVxuICAgIGRhdGFzZXRbXCJEYXRhc2V0L1Jlc291cmNlXCJdXG4gIGVuZFxuXG5cbiAgY3VyYXRvciAtLSB1cGRhdGVzIG1ldGFkYXRhIC0tPiBkYXRhc2V0XG4gIGN1cmF0b3IgLS0gdXBkYXRlcyBkYXRhIC0tPiBkYXRhc2V0XG4gIHVzZXIgLS0gc3Vic2NyaWJlcyAtLT4gZGF0YXNldFxuIiwibWVybWFpZCI6eyJ0aGVtZSI6ImRlZmF1bHQifSwidXBkYXRlRWRpdG9yIjpmYWxzZX0)\n\nEvery 10 minutes, it communicates with CKAN Classic to store the latest updates in a local database. Every 30 minutes, it sends an aggregated notification to users. The worker sends notifications only to users with an active subscription to datasets changed in the past minutes. You can configure all the time frequencies via environment variables.\n\n[![](https://mermaid.ink/img/eyJjb2RlIjoiZ3JhcGggTFJcbiAgc3ViZ3JhcGggZGF0YXN1YnNjcmlwdGlvbnNbXCJEYXRhIFN1YnNjcmlwdGlvbnNcIl1cbiAgICBEYXRhYmFzZVxuICAgIFdvcmtlclxuICBlbmRcblxuICBzdWJncmFwaCBja2FuY2xhc3NpY1tcIkNLQU4gQ2xhc3NpY1wiXVxuICAgIGNrYW5hcGlbXCJBUElcIl1cbiAgZW5kXG5cbiAgV29ya2VyIC0tIFwicHVsbHMgZGF0YXNldCBhY3Rpdml0eSAodXBkYXRlcylcIiAtLT4gY2thbmFwaVxuICBXb3JrZXIgLS0gc3RvcmVzIGFjdGl2aXR5IC0tPiBEYXRhYmFzZVxuICBXb3JrZXIgLS0gcHJlcGFyZXMgZW1haWwgLS0-IHNlbmRncmlkW1wiU2VuZEdyaWQ8YnI-KHRoaXJkLXBhcnR5KVwiXVxuICBXb3JrZXIgLS0gcHVsbHMgYWN0aXZpdHkgLS0-IERhdGFiYXNlXG4gIHNlbmRncmlkIC0tIHNlbmRzIGVtYWlsIC0tPiB1c2VyKChcIlVzZXJcIikpIiwibWVybWFpZCI6eyJ0aGVtZSI6ImRlZmF1bHQifSwidXBkYXRlRWRpdG9yIjpmYWxzZX0)](https://mermaid-js.github.io/mermaid-live-editor/#/edit/eyJjb2RlIjoiZ3JhcGggTFJcbiAgc3ViZ3JhcGggZGF0YXN1YnNjcmlwdGlvbnNbXCJEYXRhIFN1YnNjcmlwdGlvbnNcIl1cbiAgICBEYXRhYmFzZVxuICAgIFdvcmtlclxuICBlbmRcblxuICBzdWJncmFwaCBja2FuY2xhc3NpY1tcIkNLQU4gQ2xhc3NpY1wiXVxuICAgIGNrYW5hcGlbXCJBUElcIl1cbiAgZW5kXG5cbiAgV29ya2VyIC0tIFwicHVsbHMgZGF0YXNldCBhY3Rpdml0eSAodXBkYXRlcylcIiAtLT4gY2thbmFwaVxuICBXb3JrZXIgLS0gc3RvcmVzIGFjdGl2aXR5IC0tPiBEYXRhYmFzZVxuICBXb3JrZXIgLS0gcHJlcGFyZXMgZW1haWwgLS0-IHNlbmRncmlkW1wiU2VuZEdyaWQ8YnI-KHRoaXJkLXBhcnR5KVwiXVxuICBXb3JrZXIgLS0gcHVsbHMgYWN0aXZpdHkgLS0-IERhdGFiYXNlXG4gIHNlbmRncmlkIC0tIHNlbmRzIGVtYWlsIC0tPiB1c2VyKChcIlVzZXJcIikpIiwibWVybWFpZCI6eyJ0aGVtZSI6ImRlZmF1bHQifSwidXBkYXRlRWRpdG9yIjpmYWxzZX0)\n\nThis service also has a REST API to:\n\n- Change a subscription status.\n- Block a dataset from being subscribed.\n\n## API\n\nTo list all available routes in the app, you can run the following docker-compose command:\n\n```bash\ndocker-compose run --rm web data_subscriptions routes\n```\n\nBase URI: `/api/v1/`\n\n### Non-subscribable datasets\n\n**Endpoint: `/nonsubscribable_datasets/`**\n\nBy default, all datasets are subscribable. It is the data curator's work to disable subscriptions for a dataset.\n\nAvailable methods:\n\n- `GET` - Check if a dataset is non-subscribable.\n- `POST` - Make a dataset non-subscribable, i.e., disable subscriptions.\n- `DELETE` - Delete a dataset from list of non-subscribable datasets, i.e., make it subscribable again.\n\n### Subscription Status\n\n**Endpoint: `/subscription_status`**\n\nAvailable methods:\n\n- `POST` - Check the a user's subscription status.\n - body: `{\"dataset_id\": , \"kind\": \"DATASET\", \"user_id\": }` //for a single dataset subscription\n - body: `{\"kind\": \"NEW_DATASETS\", \"user_id\": }` //for a new dataset subscription\n\n### Subscriptions\n\n**Endpoint: `/subscription`**\n\nAvailable methods:\n\n- `POST` - Subscribe a user to notifications.\n - body: `{\"dataset_id\": , \"kind\": \"DATASET\", \"user_id\": , \"username\": , \"email\": }` //for a single dataset subscription\n - body: `{\"kind\": \"NEW_DATASETS\", \"user_id\": , \"username\": , \"email\": }` //for a new dataset subscription\n- `DELETE` - Unsubscribe a user from notifications.\n - body: `{\"dataset_id\": , \"kind\": \"DATASET\", \"user_id\": }` //for a single dataset subscription\n - body: `{\"kind\": \"NEW_DATASETS\", \"user_id\": }` //for a new dataset subscription\n\n### User\n\n**Endpoint: `/user/`**\n\nAvailable methods:\n\n- `GET` - Get list of subscriptions for a given authorized user.\n\n### Dataset\n\n**Endpoint: `/dataset/`\n\nAvailable methods:\n\n- `DELETE` - Delete the subscriptions for given dataset.\n\n### Subscription Report\n\n**Endpoint: `/stat`**\n\nAvailable methods:\n\n- `GET` - Get report of all subscribers in JSON.\n- `GET` - Get report of all subscribers in CSV.\n - param: `download=yes`\n\n## Development\n\nTo setup the project using Docker containers:\n\n```sh\n$ make setup\n```\n\nThe project is backed by a test suite, which can be run with a single command:\n\n```sh\n$ make test\n```\n\nFinally, to start the services and get to see the logs in real time:\n\n```\n$ docker-compose up\n```\n\n## Deployment\n\nWe do continuous deployment [to Heroku](https://gitlab.com/datopian/clients/data-subscriptions/-/blob/master/heroku.yml) via [GitLab CI](https://gitlab.com/datopian/clients/data-subscriptions/-/blob/master/.gitlab-ci.yml).\n\n\n# Design\n\n## Job Stories\n\n### Dataset Update Notifications via Email or SMS\n\nWhen using the data in this data portal I want to be notified (e.g. via email) when one of the key datasetsis updated so that I know to integrate that updated data with my workflow\n \n* [=> by the below: have a user account => signup, login etc\n* I want to be able to see a list of all the subscribable datasets\n* I want to be choose the datasets i subscribe to (and unsubscribe from)\n * Some datasets may not be subscribable to?? (see admin item)\n * a visual element to “subscribe to this dataset“. This visual element will describe what it means to subscribe, and, provide an action for the user to subscribe herself.\n * Only show subscribe option to logged in users (??)\n * When a registered user visits a dataset page, she sees the option to subscribe to updates for this dataset\n * Bulk unsubscribe\n* I want to be able to choose the medium of notification e.g. email or sms (webhook??)\n* What events trigger a notification? Any update event to the data (and metadata)\n * TODO: clarify if metadata included\n* Frequency control?? E.g. once a day or every event? ANS: every event\n* What is the payload? E.g. just email or attachments e.g. CSV\n* Can unsubscribe via link in email or via UI\n* Confirm their email or text subscription via email b4 being subscribed (GDPR?)\n \n### Control what datasets are subscribable to\n\nAs a sysadmin user, I want to be able to select (and deselect), from all the datasets on the portal, datasets that can be subscribed to, so that I can use this dataset level flag to build out email and text (sms) subscription services.\n \n### View Stats on Subscriptions\n\nWhen an Admin of a data portal I want to view stats on subscriptions so that I can manage this and report to other stakeholders\n\n* Filter by datasets\n* View data on specific users\n* Time they subscribed, username, and email\n \n### Create/Edit Email Template for Notifications \n\nWhen configuring the subscription system I want to be able to create and modify email template so that I can brand and customize the email user's receive\n\n* Is this per dataset? No: it’s generic for all datasets\n\n## Domain Model\n\n* **Subscription**: `tuple(account, dataset, {event-type-filter}, [frequency?], ...)`\n * `tuple(account, filter-function-on-activity-stream)`\n* **Subscription Settings**: `tuple(account, media[web|email|sms|newsletter], other options)`\n* **Account (User)**: a user for a subscription\n \nAlternatively could seem them as any filter operation on an overall activity stream => leads us into event bus type territory … \n\nIn the most general sense, A \"subscription\" (for notifications) is a function f:\n\n```\nf(event stream) = user's event stream\n```\n\nAnd notifications are:\n\n```\ng(user's event stream, settings) => emails/sms etc\n```\n\n## Components\n\n* SubscriptionDB + API\n* NotifierService e.g. bulk email/sms\n* UI\n* Notification\n* RulesDB (?) (for generating notifications)\n \nRequired but probably external\n\n* Accounts (user identifier + email + sms)\n\nNotes\n\n* I (account) subscribe to dataset (dataset) X\n* Curator updates dset (dataset) X\n* I get a notification (event)\n* Curator updates dataset (dataset) Y\n* I don't get a notification (event)\n* Curator updates dataset (dataset) X\n* I get a notification (event)\n\n## Sequence Diagrams\n\n### Subscribe (/ Unsubscribe / Change subscription) button click\n\n* TODO: add to sequence checking restrictions on which datasets you can subscribe to\n\n```mermaid\nsequenceDiagram\n\n Frontend->>API: POST /subscription?dataset=123&account=qwe&event-type=xxx\n API->>Authz: check authorization for {user, dataset}\n Authz-->>API: OK\n API->>DB: UPSERT INTO subscriptions {timestamp, account, dataset, event-type}\n API-->>Frontend: HTTP 201 CREATED\n Frontend->>Frontend: Update UI\n```\n\n### Event happens ...\n\n* curator edits a datasets and event stream has a new event (EventStream = activity stream table in CKAN Classic))\n* MetaStore->>EventStream: `INSERT INTO (dataset, eventType, message)`\n* Prior to event happening SubsNotificationService has subscribed itself to the EventStream\n\n```mermaid\nsequenceDiagram\n\n EventStream->>Notifier: POST /event\n loop for xmedium in [email, sms]\n Notifier->>SubsDB: SELECT FROM subscribers JOIN subs_settings WHERE dataset = ? AND eventType = ? AND medium = xmedium\n SubsDB-->>Notifier: returns(account[])\n Notifier-->>Settings: get Email/SMS Template\n Notifier->>Emailer/SMS: send(account.email/mobile[], msg)\n end\n```\n\n* https://github.com/ckan/ckan/blob/master/ckan/model/activity.py\n * There is a big table of all the activities ...\n * Could turn this into an event hub service (esp if we )\n * Could we use postgres listen/notify ...\n * Or we could even just do polling if time internal is not too short ...\n\n### Render subscribe UI in frontend ...\n\n* Frontend needs to check before rendering if this is a dataset that allows subscriptions\n* Frontend needs to check if user is subscribed\n* TODO.QU: where do we save whether a dataset is subscribable\n * ANS: in a separate table: `nonsubscribable_datasets`\n\n```mermaid\nsequenceDiagram\n\n Frontend->>Frontend: is user logged in?\n Frontend->>API: GET /is_subscribable?dataset=xxx?\n API->>DB: SELECT NOT xxx FROM nonsubscribable_datasets\n API-->>Frontend: yes/no [if no, done]\n Frontend->>API: GET /subscription?dataset=123&account=qwe\n API->>Authz: check authorization for {user, dataset}\n Authz-->>API: OK\n API->>DB: SELECT subscriptions {timestamp, account, dataset, event-type}\n DB-->>API: return subscriptions[]\n API-->>Frontend: return event-type or null\n Frontend->>Frontend: showSubscribeUI(current-event-type | default-event-type)\n```\n\n### User views (and changes) notification settings\n\n### User views all existing (and potential?) subscriptions (and changes)\n\n* TODO.QU: does include potential - that's weird in my view (how does it work with 200 datasets or 200k). Just let people subscribe via the dataset page\n\n### Admin views stats ...\n\n### Admin sets subscribable status\n\n\n## Appendix: How GitHub does it\n\nNB: github has got a lot more complex over the years so user flow isn't quite as smooth as it once was.\n\n```mermaid\ngraph TD\n\nroot[Profile Dropdown in navbar]\nsettings[Settings]\nnotifications[Notifications Subsection - config e.g. what medium]\nwatching[github.com/watching
list of what you are watching]\ndashboard[User dashboard
prime info is notifications]\n\nroot --> settings\nsettings --> notifications\nnotifications --> watching\nroot --> dashboard\n```\n\n### Watch / Unwatch Repo\n\n![](https://i.imgur.com/nTi0ob0.gif)\n\n\n### Notifications\n\n![](https://i.imgur.com/MDTxNos.png)\n\n\n### Settings\n\n![](https://i.imgur.com/wpn4g96.png)\n","has_starred","hub_user","immutable_tags_settings",{"_71":11,"_72":73},"enabled","rules",[74],".*","is_automated","permissions",{"_78":17,"_79":11,"_80":11},"read","write","admin","user","storage_size",1924642144,"orgData",{"_86":87,"_88":89,"_90":91,"_92":29,"_93":44,"_94":44,"_95":44,"_96":44,"_97":98,"_99":44,"_100":44,"_101":44,"_102":17},"id","8da03e66cb084fa685e78dac69c2f992","uuid","8da03e66-cb08-4fa6-85e7-8dac69c2f992","type","organization","orgname","full_name","location","company","profile_url","date_joined","2017-03-31T06:50:17.646784Z","gravatar_url","gravatar_email","badge","is_active","recentTags",[105,143],{"_106":40,"_107":108,"_109":110,"_111":112,"_86":113,"_114":115,"_46":130,"_131":132,"_133":134,"_135":52,"_53":136,"_137":138,"_139":121,"_140":130,"_141":61,"_142":17},"content_type","creator",712814,"digest","sha256:a23dd30f35d415821e10ebfcd6316f69f2a23c9edadd90366989d1789f27fc6c","full_size",406479958,96482600,"images",[116],{"_117":118,"_109":110,"_119":44,"_120":121,"_122":123,"_124":125,"_126":44,"_127":-5,"_128":112,"_59":61,"_129":-5},"architecture","amd64","features","last_pulled","2026-09-22T01:43:30.997099695Z","last_pushed","2023-02-03T12:35:00Z","os","linux","os_features","os_version","size","variant","2023-02-03T12:34:59.283157Z","last_updater",2497189,"last_updater_username","shubhammahajan","media_type","latest","repository",8879079,"tag_last_pulled","tag_last_pushed","tag_status","v2",{"_106":40,"_107":144,"_111":145,"_86":146,"_114":147,"_46":153,"_131":144,"_133":29,"_135":52,"_53":154,"_137":138,"_139":150,"_140":153,"_141":152,"_142":17},1608714,403435987,105988352,[148],{"_117":118,"_109":149,"_119":44,"_120":150,"_122":151,"_124":125,"_126":44,"_127":-5,"_128":145,"_59":152,"_129":-5},"sha256:aaf083ebcfb1c7f4ce2dfddf74535e07353d9ec98faa0a45735f98fbe0ff62ac","2026-08-23T12:48:03.876797602Z","2020-06-30T13:33:58Z","inactive","2020-06-30T13:33:57.271035Z","k8s","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","showSbxLogoOverride","sbxTrademarkText","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"routes/_layout.r.$namespace.$reponame.tags",{"_34":166,"_173":174,"_162":187,"_188":11},{"_36":167,"_38":168,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":169,"_53":31,"_28":29,"_54":55,"_56":40,"_57":58,"_59":58,"_60":61,"_62":-5,"_63":64,"_65":66,"_67":11,"_68":29,"_69":170,"_75":11,"_76":172,"_81":29,"_82":83},[],[40],[52],{"_71":11,"_72":171},[74],{"_78":17,"_79":11,"_80":11},"tags",{"_175":176,"_177":-5,"_178":-5,"_179":180},"count",2,"next","previous","results",[181,184],{"_106":40,"_107":108,"_109":110,"_111":112,"_86":113,"_114":182,"_46":130,"_131":132,"_133":134,"_135":52,"_53":136,"_137":138,"_139":121,"_140":130,"_141":61,"_142":17},[183],{"_117":118,"_109":110,"_119":44,"_120":121,"_122":123,"_124":125,"_126":44,"_127":-5,"_128":112,"_59":61,"_129":-5},{"_106":40,"_107":144,"_111":145,"_86":146,"_114":185,"_46":153,"_131":144,"_133":29,"_135":52,"_53":154,"_137":138,"_139":150,"_140":153,"_141":152,"_142":17},[186],{"_117":118,"_109":149,"_119":44,"_120":150,"_122":151,"_124":125,"_126":44,"_127":-5,"_128":145,"_59":152,"_129":-5},[],"canDeleteTags","actionData","errors"]

datopian/data-subscriptions

Tags for datopian/data-subscriptions