[{"_1":2,"_153":-5,"_154":-5},"loaderData",{"_3":4,"_13":14,"_21":22,"_132":133},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","routes/_layout",{"_15":-5,"_16":17,"_18":17,"_19":11,"_20":11},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","showHoneypotTrap","routes/_layout.r.$namespace.$reponame",{"_23":24,"_25":-5,"_26":11,"_27":11,"_28":29,"_30":31,"_32":-5,"_33":11,"_34":35,"_84":-5,"_85":86,"_125":-5,"_126":11,"_127":17,"_128":11,"_129":-5,"_130":131},"canonicalUrl","https://hub.docker.com/r/niccokunzmann/first_timer_scraper/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","niccokunzmann","reponame","first_timer_scraper","cloudBuildSource","isAutobuildsAvailable","repo",{"_36":37,"_38":39,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":51,"_53":31,"_28":29,"_54":55,"_56":40,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":58,"_65":66,"_67":11,"_68":29,"_69":70,"_75":17,"_76":77,"_81":29,"_82":83},"categories",[],"content_types",[40],"image","date_registered","2017-03-31T09:31:44.218827Z","description","Make the first-time contributors visible.","is_private","last_updated","2018-05-30T22:15:31.714135Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[52],"application/vnd.docker.container.image.v1+json","name","pull_count",2041,"repository_type","star_count",0,"status",1,"status_description","active","source","collaborator_count","full_description","first_timer_scraper\n===================\n\n[![](https://dockerbuildbadges.quelltext.eu/status.svg?repository=first_timer_scraper&organization=niccokunzmann)](https://hub.docker.com/r/niccokunzmann/first_timer_scraper/builds/)\n[![](https://firsttimers.quelltext.eu/repository/niccokunzmann/first_timer_scraper.svg)](https://firsttimers.quelltext.eu/repository/niccokunzmann/first_timer_scraper.html)\n\n[Visit the daily build][online].\n\nThis is inspired by `first-timers-only` issues:\nHow can we make it possible for new-comers to contribute to a project.\n\nThis web service tries to solve this by looking at the data:\n\n- What did first-timers contributed to?\n- Did they continue to contribute?\n- Which issues did they solve?\n- Which projects are best at welcoming new contributors?\n\nFirst Timer Contributions\n-------------------------\n\nTo find first timer pull-requests, we look at organizations and\ntheir repositories.\n\n- **What are first timer repositories?**\n - They MUST have at least two people contributing\n - They MUST have pull requests\n\n- **What is a first-timer pull-request?**\n - They MUST have at least one commit\n - In the repository history:\n - The issue commit MUST be preceded by commits of the author\n - At least one \"other author\" MUST have contributed before\n - There MUST NOT be a commit of the author before the \"other author\"\n - They may be on any branch.\n\n- **What is a first-timer issue?**\n There is no automatic linking between the pull-request and the issue.\n Thus, we must assume that one of the following takes place:\n - There is a public web page linked in the pull-request.\n This is the case with external issue trackers like with Django.\n We may assume certain issue trackers.\n - The issue is referenced as GitHub issue in the pull-request.\n - The issue is referenced by commits by the author preceding the merge commit.\n\nImplementation\n--------------\n\nFor each organization submitted:\n1. submit all repositories\n\nFor each submitted repository\n1. clone\n2. check if it can be a first-timer repository\n3. extract for all commits\n - author email\n - author name\n - commit hash\n - time\n4. Get all pull-requests\n - Get the first commit\n - Get the author, email\n - Get the GitHub user issuing this pull-request\n5. find first-timer pull-requests\n\nAPI\n---\n\n`ENDING` is either `.html` or `.json`.\n\n- General\n - `GET /` \n Show a description of the project.\n - Link to this repository#readme.\n - Link to download the code /source.\n - How to contribute.\n - What this is about.\n - `GET /source` \n Get the source code as a zip file.\n- GitHub Authentication:\n - `GET /auth`\n Show a form to register a new authentication.\n - `POST /auth`\n Add `username` and `password` to those usable to scrape GitHub.\n They will be tried and removed if invalid.\n- Organization\n - `GET /organizations` \n [schema][organizations-schema]\n [example responses][organizations-examples] \n List all organizations with links to their statuses. \n Users may be listed among them.\n Arguments: `offset=0&limit=10`\n - offset is the number of organizations to skip\n - limit is the number of organizations to display\n - `GET /organization/` \n [schema][organization-schema]\n [example responses][organization-examples] \n Get an organization and its status.\n Same as the element of `/organizations.json`\n Ending:\n - `json`\n - `html`\n - `svg` ![](https://img.shields.io/badge/First%20Timers-1-blue.svg)\n - `POST /organization` \n Submit an `organization` for scraping.\n This shows an html page with a link to the status of the organization.\n- Repository\n - `GET /repositories` \n [schema][repositories-schema]\n [example responses][repositories-examples] \n List all repositories with links to their statuses.\n See below for the content of the list.\n - `GET /repository//` \n [schema][repository-schema]\n [example responses][repository-examples] \n Get the repository and its status. \n Ending:\n - `json`\n - `html`\n - `svg` ![](https://img.shields.io/badge/First%20Timers-1-blue.svg)\n - `POST /repository` \n Submit a `repository` for scraping\n- User\n - `GET /users` \n Same as `GET /organizations`.\n - `GET /user/` \n Same as `GET /organization/`.\n - `POST /user/` \n Trace back the user's repositories to their origins.\n Submit all found organizations.\n- Issues\n - TODO: do we need an issue endpoint?\n\n### Minimal definitions\n\nWhen objects are defined, they contain minimal definitions.\nThey can be used to infer the most important data and find the full data.\nE.g. `repository[\"urls\"][\"json\"]` always points to the repository endpoint.\n\n`repository`, `user`, `organization` and `issue` have this in common:\n```\n{\n \"name\": \"\",\n \"urls\": {\n \"html\": \"\",\n \"json\": \"\",\n \"github_html\": \"\",\n \"github_api\": \"\",\n },\n \"last_update\": \"\",\n }\n}\n```\n- `last_update_time` times are given like this:\n `2011-01-26T19:01:12Z` in UTC.\n You can parse it with\n `time.strptime(\"2011-01-26T19:01:12Z\", \"%Y-%m-%dT%H:%M:%SZ\")`\n\n\n- Additionally `repository`, `issue` always define\n ```\n {\n \"full_name\": \"\"\n }\n ```\n\nCommand Line\n------------\n\n`python3 -m first_timer_scraper `\n\n- `CACHE_FOLDER` is the folder where the scraped data is stored.\n This is the cache. It is totally ok to remove all this data.\n- `SECRETS_FOLDER` is the folder where the secrets are stored.\n These are the secrets to access the GitHub API.\n- `MODEL_FOLDER` is the storage place of the model/the data base this is built.\n If you delete this, you can start scraping anew.\n \n### Installation\n\nYou need to install [Python 3](https://www.python.org/downloads/) and pip.\nUnder Ubuntu, you can do this:\n\n sudo apt-get -y install python3 python3-pip\n\nTo install all required packages, execute\n\n pip3 install --user -r requirements.txt\n\n### Windows\n\n py -3 -m pip install --user -r requirements.txt\n py -3 -m first_timer_scraper.app data secret model\n\n### Docker\n\nThis runs the docker container:\n\n docker run --rm \\\n -p 8080:8080 \\\n -v \"secret:/app/secret\" \\\n -v \"model:/app/model\" \\\n niccokunzmann/first_timer_scraper\n\nThe parameters have the following meaning:\n- The API of the first-timer-scraper is available as .\n- The secret data including passwords are stored in the folder `./secret`.\n- The model data is stored in the folder `./model`\n- `--rm` is for development purposes.\n It removes the container and all the cache when the container stops.\n\nWhen you ran the command, you can visit ,\nsubmit credentials and scrape repositories.\n\nYou can build the Docker image like this:\n\n docker build . -t niccokunzmann/first_timer_scraper\n\nDeployment\n----------\n\nDeployment takes up to 24 hours from a merge into master:\n\n1. A commit is added to [master][commits-master].\n2. An [automated build][builds] is started.\n3. The [application is deployed][online] around 4:00 GMT\n\nData Model\n----------\n\nThe data model describes what is saved when scraped.\n\n```\n{\n \"loklak\": {\n \"repos\": {\n \"loklak_server\": {\n \"first_timer_prs\": {\n \"112\": \"contributor1\"\n },\n \"last_update_requested\": \"2011-01-26T19:01:12Z\"\n }\n },\n \"last_update_requested\": \"2011-01-26T19:01:12Z\",\n \"first_timer_prs\":{}\n },\n \"contributor1\": {\n \"first_timer_prs\": {\n \"loklak/loklak_server\": {\n \"created_at\" : \"2011-01-26T19:01:12Z\"\n \"number\": 112 // lowest number wins\n \"last_update_requested\": \"2011-01-26T19:01:12Z\"\n }\n },\n \"last_update_requested\": \"2011-01-26T19:01:12Z\",\n \"repos\": {}\n }\n}\n```\n\nFurther Reading\n---------------\n\n- GitHub API:\n - Rate Limiting: https://developer.github.com/v3/#rate-limiting\n- PyGithub: https://github.com/PyGithub/PyGithub/\n\n[online]: https://firsttimers.quelltext.eu\n[commits-master]: https://github.com/niccokunzmann/first_timer_scraper/commits/master\n[builds]: https://hub.docker.com/r/niccokunzmann/first_timer_scraper/builds/\n[organizations-schema]: first_timer_scraper/tests/schemas/organizations.json\n[organizations-examples]: first_timer_scraper/tests/tests/organizations/works\n[organization-schema]: first_timer_scraper/tests/schemas/organization.json\n[organization-examples]: first_timer_scraper/tests/tests/organization/works\n[repositories-schema]: first_timer_scraper/tests/schemas/repositories.json\n[repositories-examples]: first_timer_scraper/tests/tests/repositories/works\n[repository-schema]: first_timer_scraper/tests/schemas/repository.json\n[repository-examples]: first_timer_scraper/tests/tests/repository/works\n\n","has_starred","hub_user","immutable_tags_settings",{"_71":11,"_72":73},"enabled","rules",[74],".*","is_automated","permissions",{"_78":17,"_79":11,"_80":11},"read","write","admin","user","storage_size",620826203,"orgData","recentTags",[87],{"_88":40,"_89":90,"_91":92,"_93":94,"_95":96,"_46":114,"_115":90,"_116":29,"_117":52,"_53":118,"_119":120,"_121":105,"_122":114,"_123":62,"_124":17},"content_type","creator",542821,"full_size",43897633,"id",9558775,"images",[97],{"_98":99,"_100":101,"_102":103,"_104":105,"_106":107,"_108":109,"_110":103,"_111":-5,"_112":92,"_59":62,"_113":-5},"architecture","amd64","digest","sha256:889c0bfb639a532650af43e5d6ac31d6b2a22a5f5d65be19fd21bf53c607e764","features","","last_pulled","2026-09-20T00:49:30.987613951Z","last_pushed","2018-05-30T22:15:32Z","os","linux","os_features","os_version","size","variant","2018-05-30T22:15:31.409181Z","last_updater","last_updater_username","media_type","latest","repository",1372202,"tag_last_pulled","tag_last_pushed","tag_status","v2","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"routes/_layout.r.$namespace.$reponame.tags",{"_34":134,"_141":142,"_130":151,"_152":11},{"_36":135,"_38":136,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":137,"_53":31,"_28":29,"_54":55,"_56":40,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":58,"_65":66,"_67":11,"_68":29,"_69":138,"_75":17,"_76":140,"_81":29,"_82":83},[],[40],[52],{"_71":11,"_72":139},[74],{"_78":17,"_79":11,"_80":11},"tags",{"_143":60,"_144":-5,"_145":-5,"_146":147},"count","next","previous","results",[148],{"_88":40,"_89":90,"_91":92,"_93":94,"_95":149,"_46":114,"_115":90,"_116":29,"_117":52,"_53":118,"_119":120,"_121":105,"_122":114,"_123":62,"_124":17},[150],{"_98":99,"_100":101,"_102":103,"_104":105,"_106":107,"_108":109,"_110":103,"_111":-5,"_112":92,"_59":62,"_113":-5},[],"canDeleteTags","actionData","errors"]

niccokunzmann/first_timer_scraper

Tags for niccokunzmann/first_timer_scraper