[{"_1":2,"_169":-5,"_170":-5},"loaderData",{"_3":4,"_13":14,"_21":22,"_144":145},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","routes/_layout",{"_15":-5,"_16":17,"_18":17,"_19":11,"_20":11},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","showHoneypotTrap","routes/_layout.r.$namespace.$reponame",{"_23":24,"_25":-5,"_26":11,"_27":11,"_28":29,"_30":31,"_32":-5,"_33":11,"_34":35,"_84":-5,"_85":86,"_135":-5,"_136":11,"_137":17,"_138":11,"_139":-5,"_140":11,"_141":-5,"_142":143},"canonicalUrl","https://hub.docker.com/r/cslev/doh_docker/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","cslev","reponame","doh_docker","cloudBuildSource","isAutobuildsAvailable","repo",{"_36":37,"_38":39,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":51,"_53":31,"_28":29,"_54":55,"_56":40,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":58,"_65":66,"_67":11,"_68":29,"_69":70,"_75":11,"_76":77,"_81":29,"_82":83},"categories",[],"content_types",[40],"image","date_registered","2020-03-20T07:26:46.30228Z","description","Containerized Firefox to visit the top 1M website using DNS-over-HTTPS and captures the packet trace","is_private","last_updated","2021-04-01T09:54:53.632856Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[52],"application/vnd.docker.container.image.v1+json","name","pull_count",2532,"repository_type","star_count",0,"status",1,"status_description","active","source","collaborator_count","full_description","# Dissemination\nThis repository has also been made for our [research paper](https://github.com/cslev/doh_ml/raw/main/DNS_over_HTTPS_identification.pdf) titled **Privacy of DNS-over-HTTPS: Requiem for a Dream?** to appear at [IEEE Euro S&P](http://www.ieee-security.org/TC/EuroSP2021/).\n\nWhen using the repo, please use the full reference to our paper as follows:\n### Bibtex\n```\n@inproceedings{doh_identification_ml,\n author = {Levente Csikor and Himanshu Singh and Min Suk Kang and Dinil Mon Divakaran},\n title = {{Privacy of DNS-over-HTTPS: Requiem for a Dream?}},\n booktitle = {IEEE European Symposium on Security and Privacy},\n year = {2021}\n} \n```\n### Text\n```\nL. Csikor, H. Singh, M. S. Kang and D. M. Divakaran, \"Privacy of DNS-over-HTTPS: Requiem for a Dream?\", In proceedings of IEEE European Symposium on Security and Privacy, 2021.\n```\n\n\n# What is this repository all about?\nThis containerized bundle uses Selenium and Firefox to use DNS-over-HTTPS (provided by Firefox) for name resolution when visiting the websites of Alexa's list of the top 1M websites (an older version from 2018 is part of the repository). \nFor every website visit, the browser is closed to flush the DNS cache, and the corresponding traffic trace (i.e., a pcap file), as well as the SSL keys are logged. \nThen, using these two, in this docker container, the pcap files are analyzed and the relevant information will be saved in CSV files (pcap files will be deleted afterwards as they consume a huge amount of space).\nSuch relevant information is as follows:\n - frame number\n - timestamp \n - Source IP\n - Destination IP\n - Source Port\n - Destination Port (although, it is 443 all the time)\n - Protocol (labeled by Wireshark)\n - frame length\n - payload information\n\nThe `.csv` files can be processed by the [machine learning application written for this purpose](https://github.com/cslev/doh_ml).\n\n\n## Requirements\nBeing a docker container, you have to have a running docker subsystem installed. If you have no such subsystem, first, go to [https://docs.docker.com/install/linux/docker-ce/debian/](https://docs.docker.com/install/linux/docker-ce/debian/), pick your distribution on the left hand side, and follow the instructions to install it.\n\n# Obtaining the container\n\n## Auto-builds at DockerHub\nYou can obtain an auto-built up-to-date image from [DockerHub](https://hub.docker.com/repository/docker/cslev/doh_docker).\nDo do so, just do the following:\n```\nsudo docker pull cslev/doh_docker:latest\n```\nThis will download the container and you are ready to use it (see parameters and configurations for docker-compose below).\n\n## Build your own image\nClone the repository first, then build the image.\n```\ngit clone https://github.com/cslev/doh_docker\ncd doh_docker\nsudo docker build -t cslev/doh_docker:latest -f Dockerfile .\n```\nIn the last command `-t` specifies the tag (default `latest`) used for our image! Feel free to use another tag, but to be sync with a possible future update might be coming from DockerHub later, we do not recommend to change any part of this command.\n\n\n# Running the container\nThe container and the scripts inside can be parametrized by environment variables. You can either set these variables by definint `-e VARNAME=value` everytime, or create and environment file that you pass to the container when started.\nFor simplicity, we discuss the first approach only, but let's see first the ENV variables\n\n## All Environment variables\n`DOH_DOCKER_NAME` the name of the container\n`DOH_DOCKER_RESOLVER` the resolver intended to use. `doh_docker` container supports almost all resolvers publicly available. See a complete list [here](https://raw.githubusercontent.com/cslev/doh_docker/master/source/r_config.json). This `json` file is used in the program when deciding the resolver related parameters. Default is *cloudflare*.\n`DOH_DOCKER_START` **first** domain to visit in the Alexa's domain list or whatever domain list file you use (see `DOH_DOCKER_DOMAIN_LIST`). Default is: *1*\n\n`DOH_DOCKER_END` **last** domain to visit in the Alexa's domain list or whatever domain list file you use (see `DOH_DOCKER_DOMAIN_LIST`). Default is: *5000*\n\n`DOH_DOCKER_BATCH` The desired batch size in one iteration. Default is set to *200*, which means that for every 200 website-visits, we will have a separate PCAP file for managability reasons (still not too big and can be analyzed in reasonable time). Although, at the end, they will be CSV files. We do not recommend to change this value unless you know what you are doing.\n\n`DOH_DOCKER_INTF` the used interface in the container. Default is *eth0* and if you don't have any special configuration, you don't have to change this at all.\n\n`DOH_DOCKER_DOMAIN_LIST` this is the file describing the domains to visit in the form of how the provided [top-1m.csv](https://github.com/cslev/doh_docker/blob/master/source/top-1m.csv) looks like. If you want to use your own file, the easiest way is to store it in your ``, which you will mount into the container at `/doh_project/archives`. Accordingly, you can specify the path to your domain list file as `/doh_project/archives/my_list_of_domains.csv`.\n\n`DOH_DOCKER_META` any meta information you would like to pass. This information will be appended to the final compressed file's name; again for easier identification. So, be short and meaningful. Default is *sg*; we simply use our locations as an extra information.\n\n`DOH_DOCKER_WEBPAGE_TIMEOUT` this indicates the program how many seconds it should wait for a webpage to load. Since, not all domains work (or might not be resolved by a filter), we have to have a timeout. Default is *20* (do not reduce it much more than below 15 seconds. Inside the container, the browser needs this time to effectively load all the contents.)\n\n`DOH_DOCKER_ARCHIVE_PATH` define it to the mount point, i.e., `/doh_project/archives`. This path is set by default. Only change if you intend to use different path inside the container for archives.\n\n## Running with `docker run`\nTo run the container, the command looks as follows:\n```\nsudo docker run -d --name my_doh -e DOH_DOCKER_NAME=my_doh [-e FURTHER_ENV_VARS] -v :/doh_project/archives:rw --shm-size 4g cslev/doh_docker:latest\n```\n\n`--name` just assigns a simple name to the container to make it easier (in the future) to refer to it when listing running containers via `docker ps` (instead of the docker subsystem's random image names).\n\n`--shm-size 4g` is some extra memory assignment needed for Selenium and Firefox, otherwise the whole process becomes extremely slow (if starts at all).\n\n`-v :/doh_project/archives:rw` will mount your `` to the container, and it will store the final archive there. (Yes, all results will be in a compressed archive, you can later uncompress and analyze.)\n\n`-e DOH_DOCKER_NAME=` sets environmental variable to `my_doh`. This variable is used in `.bashrc` to set `$NAME` in the promp, i.e., when logged into/attached to the container and `$NAME=cloudflare`, you will see `root@cloudflare`.\nThis is also for simplyfing things; it can help a lot to identify which one is doing what experiment as complete run with 5,000 websites can take around 24-48 hours.\n\n**Define the other environment variables if needed**\n\n## Running with `docker-compose`\nWe also provide a docker-compose YAML file to ease the management of the containers, and also let you to easily run multiple containers with one command.\nIf you are unfamiliar with docker-compose and you don't have it installed, refer to [this](https://docs.docker.com/compose/).\nIt is worth having a look...docker-compose can ease your life.\n\n### Obtain docker-compose quickly\nA quick howto for Linux. \n#### Get the correct variant for your distro:\n```\nsudo curl -L \"https://github.com/docker/compose/releases/download/1.28.2/docker-compose-$(uname -s)-$(uname -m)\" -o /usr/local/bin/docker-compose\n```\n#### Add the correct permissions\n```\nsudo chmod +x /usr/local/bin/docker-compose\n```\nAnd you are good to go :)\n\n### docker-compose\nThe repository contains a `docker-compose.yml.template` file. It looks quite simple and straightforward:\n```\n---\nversion: '3.3'\n\nservices:\n doh1:\n image: 'cslev/doh_docker:latest'\n container_name: ${R1} #rename according to you scenario for easier identification\n\n volumes:\n - './container_data/MY_CONTAINER:/doh_project/archives:rw'\n\n \n shm_size: '4g'\n\n environment:\n DOH_DOCKER_NAME: 'MY_CONTAINER' #should be same as container_name\n DOH_DOCKER_RESOLVER: 'cloudflare' # resolver to use\n DOH_DOCKER_START: '1' #first domain to visit in the domain list\n DOH_DOCKER_END: '5000' #last domain to visit in the domain list\n DOH_DOCKER_BATCH: '200' #how many domains to visit within a batch (don't change unless you know what you are doing)\n DOH_DOCKER_INTF: 'eth0' #change this if your container would not have eth0 as a default interface\n DOH_DOCKER_DOMAIN_LIST: 'top-1m.csv' #the relative path inside the container pointing to the list of domains to visit (in Alexa list style: id, domain <-one each line)\n DOH_DOCKER_META: 'sg' #any meta info to affix your final output files for easier identification\n DOH_DOCKER_WEBPAGE_TIMEOUT: '20' #how many seconds we wait for a website to load before throwing timeout error and skip\n DOH_DOCKER_ARCHIVE_PATH: '/doh_project/archives' #where to store the compressed output. Should be the same as defined by the 'volume'\n\n```\nAs you can see everything has a comment to help you defining the environment variables easily. Besides, other container related settings are defined including `shm-size` and also the volume to mount to `/doh_project/archives`.\nThe only interesting part is the variable `$R1`. \n\n`docker-compose` can use variables in the YAML files if those variables are defined in the `.env` file located in the same directory.\nIf you check the file we provided, each `RX` defines a resolver that we support.\n```\nR1=cloudflare\nR2=google\nR3=cleanbrowsing\nR4=quad9\nR5=powerdns\nR6=dohli\nR7=adguard\nR8=opendns\nR9=CZNIC\nR10=DNSlify\n...\n```\nYou can either use these variables, or you can explicity say `cloudflare` in the docker-compose.yml file instead of the `${R1}`.\nHowever, this `RX` thingy becomes handy if you simply want to change your setup after a successful run.\nFor instance, let's say your experiment with `cloudflare` is done. You just simply replace the variables to `google` in order to run the experiment by using Google's resolver via a simple `sed`:\n```\nsed -i \"s/R1/R2/g\" docker-compose.yml\n```\n\n#### Run (finally)\n```\nsudo docker-compose -f docker-compose.yml up -d\n```\n`-d` puts everything into a daemonized mode to supress container output.\nHowever, you can see after this command right away when your service is up and running.\n`-f` is only needed if your compose file has a different name than `docker-compose.yml`\n\nNote, simply multiplying the service descriptor `doh1` with different names in the docker-compose.yml, you can easily define multiple different containers, and by the `docker-compose up` command, you fire up all of them in one shot.\n\n\n# Getting the data\nThe container will exit once the data gathering is complete and all relevant data will be saved in a compressed archive, called `doh_data__.tar.gz`!\nIf you HAVE used the command above with the `-v` option, or defined the `volume` in the `docker-compose.yml` file, you will have the data on your host machine.\n\nOtherwise, to get the data we need to restart the container (as it has exited) and copy the compressed archive to the host.\n\nFirst, restart (recall the name we have set via `--name` above):\n```\nsudo docker start my_doh\n```\nCopy compressed file to the host:\n```\nsudo docker cp my_doh:/doh_project/archives/doh_data__.tar.gz ./\n```\n\n**And, we would need this file! If you consider to contribute to our project, please share your data with us **\n\nDon't forget to stop and/or remove your container if not needed anymore (i.e., if you have `doh_data__.tar.gz`):\n```\nsudo docker stop my_doh\nsudo docker rm my_doh\n```\n\n\n\n# Monitoring/Logging\nEasiest way to see whether the process is finished is to check the status of the container itself. If it is exited, then it's done.\n```\nsudo docker ps -a -f name=my_doh\nCONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES\nde77210d2748 cslev/doh_docker \"/doh_project/start_…\" 3 minutes ago Up 3 minutes my_doh\n```\nThe example above shows that the container is still running.\n\nIf a little bit more granular status update is required, we can monitor at which (number of) website our container is visiting at the moment.\n```\ndocker exec -it my_doh tail -f progess.log\n\n11 https://www.taobao.com\n12 https://www.twitter.com\n13 https://www.tmall.com\n14 https://www.google.co.jp\n15 https://www.live.com\n16 https://www.vk.com\n17 https://www.instagram.com\n18 https://www.sohu.com\nMessage: Timeout loading page after 25000ms\n\n19 https://www.sina.com.cn\n```\nAs you can see, some websites are not loaded due to a timeout!\n\n\n\n# Troubleshooting\nIt might happen that the script inside the container crashes due to several reasons (e.g., running out of memory, selenium error, which is not handled properly, etc.).\nOne of the most obvious things that can point this out is the size of the archive file! It is normally around 70-100 MB. So, if your compressed archive is below 50MB, there is a high chance that something was not working properly.\n\nThis can result in that some batches might not have finished. For instance, having a serious issue at the 45th website would mean that the first batch of 200 websites will be stopped at the 45th website, and jumps to the next batch (i.e., continues from website no. 201).\n\n## Look for batches\nBy looking into the log files, one can see when the processing has been started and what was the last website in that batch. If it is not 200,400, ..., 5000 (in case of websites between 1-5000 and batch size of 200, i.e., in the default setting), then there was an error.\n\nTo quickly check this out during processing, let's count the successful batches that ends with `00`.\n```\nsudo docker exec my_doh cat /doh_project/progress.log |grep batch -B 10 |grep ^[0-9]|awk '{print $1}'|grep 00$|wc -l\n```\nIf the result is *25*, you are good!\nThere is another problem!\n\nNote, once the measurement is done, container is stopped. If restarted afterwards, the `progess.log` SYMLINK will point to the new, almost empty file. In this case, use the explicit path to the log file, or copy-paste the archive from the container to your host, decompress it, and issue the above command on the `doh_log.log` file.\n\n## TimeoutException\nTo quickly check this out, let's count the unsuccessful connection attempts:\n```\nsudo docker exec my_doh cat /doh_project/progress.log |grep -i timeoutexception |wc -l\n```\nThis means that the preset seconds for timeout was not enough! Usually, it is not your problem, it can be the problem of the DoH resolver used. If you set timeout to a considerably long interval (e.g., 50 sec), then you can rest assured that timeouts are not because of you, but because of the DoH resolver!\n\n","has_starred","hub_user","immutable_tags_settings",{"_71":11,"_72":73},"enabled","rules",[74],".*","is_automated","permissions",{"_78":17,"_79":11,"_80":11},"read","write","admin","user","storage_size",39632481621,"orgData","recentTags",[87,125],{"_88":40,"_89":90,"_91":92,"_93":94,"_95":96,"_46":114,"_115":90,"_116":29,"_117":52,"_53":118,"_119":120,"_121":105,"_122":114,"_123":62,"_124":17},"content_type","creator",2489092,"full_size",362337918,"id",91560794,"images",[97],{"_98":99,"_100":101,"_102":103,"_104":105,"_106":107,"_108":109,"_110":103,"_111":-5,"_112":92,"_59":62,"_113":-5},"architecture","amd64","digest","sha256:caa3f3adc93b27c2fb870e7789b2636116c35250960337b641fcef5f8826be70","features","","last_pulled","2026-09-23T12:01:43.442518156Z","last_pushed","2021-04-01T08:30:28Z","os","linux","os_features","os_version","size","variant","2021-04-01T09:54:52.192099Z","last_updater","last_updater_username","media_type","latest","repository",8679371,"tag_last_pulled","tag_last_pushed","tag_status","v2",{"_88":40,"_89":90,"_91":126,"_93":127,"_95":128,"_46":133,"_115":90,"_116":29,"_117":52,"_53":134,"_119":120,"_121":131,"_122":133,"_123":62,"_124":17},317532636,97846220,[129],{"_98":99,"_100":130,"_102":103,"_104":131,"_106":132,"_108":109,"_110":103,"_111":-5,"_112":126,"_59":62,"_113":-5},"sha256:23cd9b5f0e8d78d8d64e0325e70b7e96bd9fcb93a28e8eb9a751e18e5ff482d2","2026-09-22T05:32:59.314876726Z","2020-10-14T04:57:30Z","2020-10-14T04:57:29.727863Z","debug","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","showSbxLogoOverride","sbxTrademarkText","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"routes/_layout.r.$namespace.$reponame.tags",{"_34":146,"_153":154,"_142":167,"_168":11},{"_36":147,"_38":148,"_41":42,"_43":44,"_45":11,"_46":47,"_48":49,"_50":149,"_53":31,"_28":29,"_54":55,"_56":40,"_57":58,"_59":60,"_61":62,"_63":-5,"_64":58,"_65":66,"_67":11,"_68":29,"_69":150,"_75":11,"_76":152,"_81":29,"_82":83},[],[40],[52],{"_71":11,"_72":151},[74],{"_78":17,"_79":11,"_80":11},"tags",{"_155":156,"_157":-5,"_158":-5,"_159":160},"count",2,"next","previous","results",[161,164],{"_88":40,"_89":90,"_91":92,"_93":94,"_95":162,"_46":114,"_115":90,"_116":29,"_117":52,"_53":118,"_119":120,"_121":105,"_122":114,"_123":62,"_124":17},[163],{"_98":99,"_100":101,"_102":103,"_104":105,"_106":107,"_108":109,"_110":103,"_111":-5,"_112":92,"_59":62,"_113":-5},{"_88":40,"_89":90,"_91":126,"_93":127,"_95":165,"_46":133,"_115":90,"_116":29,"_117":52,"_53":134,"_119":120,"_121":131,"_122":133,"_123":62,"_124":17},[166],{"_98":99,"_100":130,"_102":103,"_104":131,"_106":132,"_108":109,"_110":103,"_111":-5,"_112":126,"_59":62,"_113":-5},[],"canDeleteTags","actionData","errors"]

cslev/doh_docker

Tags for cslev/doh_docker