[{"_1":2,"_180":-5,"_181":-5},"loaderData",{"_3":4,"_17":18,"_25":26,"_159":160},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7,"_13":14,"_15":16},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","theme","system","contrast","normal","routes/_layout",{"_19":-5,"_20":21,"_22":21,"_23":11,"_24":21},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","showHoneypotTrap","routes/_layout.r.$namespace.$reponame",{"_27":28,"_29":-5,"_30":11,"_31":11,"_32":33,"_34":35,"_36":-5,"_37":11,"_38":39,"_88":89,"_110":111,"_148":-5,"_149":11,"_150":21,"_151":11,"_152":-5,"_153":11,"_154":-5,"_155":156,"_157":112,"_158":-7},"canonicalUrl","https://hub.docker.com/r/aryeelab/pdr/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","aryeelab","reponame","pdr","cloudBuildSource","isAutobuildsAvailable","repo",{"_40":41,"_42":43,"_45":46,"_47":48,"_49":11,"_50":51,"_52":53,"_54":55,"_57":35,"_32":33,"_58":59,"_60":44,"_61":62,"_63":64,"_65":66,"_67":-5,"_68":62,"_69":70,"_71":11,"_72":33,"_73":74,"_79":21,"_80":81,"_85":33,"_86":87},"categories",[],"content_types",[44],"image","date_registered","2018-11-28T19:27:13.856465Z","description","Calculate Percent Discordant Reads (PDR) in bisulfite sequencing.","is_private","last_updated","2019-03-21T18:02:56.0585Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[56],"application/vnd.docker.container.image.v1+json","name","pull_count",301,"repository_type","star_count",0,"status",1,"status_description","active","source","collaborator_count","full_description","\n\n# FireCloud/WDL DNA methylation workflows\n

\nThis repository https://github.com/aryeelab/dna-methylation-tools contains a suite of tools to conduct DNA methylation data analysis. It is maintained by Divy Kangeyan at the Aryee Lab\n

\n\n

\nThis platform contains publicly accessible cloud-based preprocessing and quality control pipelines that go from raw data to CpG-level methylation estimates. The technologies covered include Whole Genome Bisulfite Sequencing (WGBS), Reduced Representation Bisulfite Sequencing (RRBS) and Hybrid Selection (capture) Bisulfite Sequencing (HSBS). Leveraging the FireCloud platform allows users to:\n

\n\n

Ensure cross-platform reproducibility of analyses
Achieve scalability to large whole genome datasets with 100GB+ of raw data per sample, and to single-cell datasets with thousands of cells
Provide access to best-practice analysis pipelines
Enable integration and comparison between user-provided data and publicly available data (e.g. TCGA)

\n\n\n## Workflow steps\nAnalysis should be run in two successive processes: \n\n

Per-sample preprocessing
Aggregation and quality control analysis

\n\n## Getting started\n

\nIf you plan to use the WDL workflows in your local computing environment, then this repository should be cloned. If you are using FireCloud, then you should either clone the FireCloud workspace aryee-lab/dna-methylation or if you already have your own workspace you can import the method configuration of your interest from aryee-lab/dna-methylation workspace.\n

\n\n## Uploading the files to FireCloud\n

\nFASTQ files and target coverage files can be uploaded to FireCloud using gsutil (https://pypi.org/project/gsutil/)\n

\n\n## Data model\n\n### Participant file\n

\nBefore running the processes, you need to generate participants file and participant_set file. Each line in participant file specify a single sample. \n\nentity:participant_id column specifies the sample name \n\nbs_fastq1\t& bs_fastq2 specify the paired fastq files.\n\nIf one sample has multiple FASTQ files for different lanes or runs they can be added in the same column with comma separation. \n\n### Participant Set file\n\nEach line in participant_set file specify a set of samples that should be aggregated and analyzed together. \n \nmembership:participant_set_id specify the name of the participant set.\n \nparticipant_id column specify the names of the participants in the set. \n\nBoth of these files are tab separated text files. Examples of these files are shown in Firecloud_imports subdirectory.\n

\n\n## Alignment and methylation calling\n

\nIn order to perform alignment and methylation calling choose bismark_rrbs, bismark_wgbs or bismark_hsbs method configuration with appropriate reference genome suffix. As the name indicates bismark_rrbs is for samples that are generated from Reduced Representation Bisulfite Sequencing (RRBS) with Mspl digestion and bismark_wgbs is for data generated from Whole Genome Bisulfite Sequencing (WGBS). bismark_hsbs is for data generated from Hybrid Selection Bisulfite Sequencing (HSBS). These worflows can also combine fastq files from multiple lanes if the samples are sequenced in such a way.\n

\n\n

Upload the fastq files to the Google cloud bucket
Upload additional files such as target coverage bed file for HSBS sequencing
In the FireCloud workspace choose bismark_rrbs, bismark_wgbs or bismark_hsbs method configuration with appropriate reference genome suffix
Change other parameters according to preference
Press Launch Analysis in upper right hand corner
Choose the participants from the list of files
Click Launch

\n\n\nIf you are interested in conducting alignment and methylation calling for an entire participant set. Choose the participant set and in the box named **Define expression** type *this.participants*\n\n\nYou can observe the status of the job by going to *Monitor* tab\n\n### Alignment and methylation calling specific parameters\n```\nr1_fastq & r2_fastq\n```\n- Paired end FASTQ files\n\n```\nsamplename\n```\n- base name for a sample\n\n```\ngenome_index\n```\n- Reference genome to conduct bisulfite specicfic alignment\n\n```\nn_bp_trim_read1 & n_bp_trim_read1\n```\n- Number of bases to trim in read 1 and read 2. This is only specified for wgbs and hsbs workflows. rrbs workflow uses the --rrbs option from trimGalore.\n\n```\nchrom_sizes\n```\n- chrom_sizes file in order to generate the BIGWIG file\n\n\n\n## Aggregation and Quality Control Analysis\nAfter the alignment and methylation calling each sample will have their methylation information and metadata stored in HDF5 format\n\nIn order to aggregate all of them and obtain the quality control report\n

Choose aggregate_bismark_output method configuration with appropriate reference genome suffix
Change other parameters according to preference
Save it and press Launch Analysis
Since root entity type in aggregation step is participant set, you will choose participant_set with participants of your interest
Finally click Launch

\n\nTo check the results from any of the work flows, go to Monitor tab, click *View* in the Status columns and then click the *Workflow ID* in the bottom of the page.\n\n### Aggregation specific parameters\n```\nin_pe_reports_files\n```\n- Report files from the bismark alignment\n\n```\nin_covgz_files\n```\n- coverage output file\n\n```\nin_mbias_files\n```\n- M-bias file\n\n```\nBSGenome_targz\n```\n- BSGenome targz file\n\n```\nBSGenome_package\n```\n- Name of the BSGenome package\n\n```\nGenome_build\n```\n- Name of the genome build, e.g. mm10, hg19 etc. \n\n\n### General parameters\n```\ncpu\n```\n- Number of CPUs\n\n```\ndisks\n```\n- Name of the disk (Depends on the disk size)\n\n```\nmemory\n```\n- Memory requirement\n\n```\nmulticore\n```\n- Number of cores to use, especially during alignment\n\n```\npreemptible\n```\n- Preemptible option is to use preemptible virtual machine, if it is set to 0 the workflow runs uninterrupted. If it is set to any integer other than zero it may be interpreted that many times. However preemtible option reduces the computing cost significantly.\n\n### WDL specifications for the workflows\n\nThe per-sample preprocessing WDL workflow input parameters consist primarily of input file names and the Bismark genome index to use. A subset of the outputs from this step (Mbias files and methylated and unmethylated read coverage tables) are passed on as inputs to the subsequent aggregation and QC WDL workflow which in turn produces a QC report, aggregated methylation tables and a Bioconductor (bsseq) object. All tasks specify runtime parameters including disk size, CPU and memory requirements, and the docker image version to be executed.\n\n## scmeth\nQuality control analysis is conducted via the bioconductor package scmeth. QC analysis can be done independently with this package given that you have the appropriate objects.\n\n\n\n\n## Running the WDL workflow in local environment\n### Setup\n\n1. Clone this repository\n\n```\ngit clone aryeelab/dna-methylation-tools\n```\n\n2. Change the directory\n\n```\ncd dna-methylation-tools\n```\n\n\n3. If you want to run the examples below, download this small genome index and chrom.sizes file for mm10_chr 19: \n\n```\nwget https://storage.googleapis.com/aryeelab/bismark-index/mm10_chr19/bismark_mm10_chr19.tar.gz\n```\n\n```\nwget https://storage.googleapis.com/aryeelab/chrom.sizes/mm10_chr19.chrom.sizes\n```\n\n### Running the WDL workflow in Cromwell\n\nNote: On Mac, cromwell can be installed with the following command\n```\nbrew install cromwell\n```\n\nUsers can also use the following command in order to install cromwell from the docker image in this repo.\n```\ndocker build Docker/cromwell\n```\n\n\nFollowing commands are based on cromwell version 30. \n\n**Edit the file paths for FASTQ files, genome index, chrom.sizes file and monitoring script in the json file according to your directory paths to run the test sample.**\n\n```\n{\n \"call_bismark_pool.r1_fastq\": \"[INSERT ABSOLUTE PATH]/dna-methylation-tools/testdata/small_01_R1.fastq.gz\",\n \"call_bismark_pool.r2_fastq\": \"[INSERT ABSOLUTE PATH]/dna-methylation-tools/testdata/small_01_R2.fastq.gz\",\n \"call_bismark_pool.chrom_sizes\": \"[INSERT ABSOLUTE PATH]/dna-methylation-tools/mm10.chrom.sizes\",\n \"call_bismark_pool.monitoring_script\": \"[INSERT ABSOLUTE PATH]/dna-methylation-tools/monitor.sh\",\n \"call_bismark_pool.genome_index\": \"[INSERT ABSOLUTE PATH]/dna-methylation-tools/bismark_mm10_chr19.tar.gz\"\n \n}\n```\n\n\n#### WGBS, Paired-end reads\n```\njava -jar cromwell-30.2.jar run bismark_wgbs.wdl -i bismark_wgbs.json\n```\n\n#### RRBS, Paired-end reads\n```\njava -jar cromwell-30.2.jar run bismark_rrbs.wdl -i bismark_rrbs.json\n```\n\n\n#### HSBS, Paired-end reads\n```\njava -jar cromwell-30.2.jar run bismark_hsbs.wdl -i bismark_hsbs.json\n```\n\n#### Aggregation workflow\n```\njava -jar cromwell-30.2.jar run aggregate_bismark_output.wdl -i aggregate_bismark.json\n```\n\n#### Single-end reads (Out of date)\n```\njava -jar cromwell-30.2.jar run bsseq_preprocess_se.wdl -i sample1_se.json\n```\n\n### Cost & Time Analysis in FireCloud\nWe estimated the time and cost for Whole Genome Bisulfite Sequencing (WGBS) sample from TCGA that contains approximately 290 million reads.\n\n![](TCGA_Time_Cost_Analysis.svg)\n\nAlso following plot shows the time & cost estimates for single cell samples in non-preemptible machine\n\nFor 10 sample set: samples were sequenced with a median of 741,214 reads (range of 414,443 to 1,825,672)\n\nFor 25 sample set: samples were sequenced with a median of 1,044,720 reads (range of 10,507 to 3,644,360)\n\n![](10Sample_Time_Cost_Analysis.svg) ![](25Sample_Time_Cost_Analysis.svg)\n\nPlots below are for same sample set but in preemptible machine\n\n![](10Sample_preemptible_Time_Cost_Analysis.svg) ![](25Sample_preemptible_Time_Cost_Analysis.svg)\n\nWe also conducted similar analysis for the aggregation step\n\n![](Aggregation_10_sampleSet.svg) ![](Aggregation_25_sampleSet.svg) ![](Aggregation_100_sampleSet.svg)\n\n## TCGA data\nWe have preprocessed and made available 47 WGBS samples available from TCGA. The raw (FASTQ) data, processed data and workflows are made available in a FireCloud workspace (See https://portal.firecloud.org/#workspaces/aryee-merkin/TCGA_WGBS_hg19). Processed data can also be found in bsseq format in tcgaWGBSData.hg19 package (https://bioconductor.org/packages/release/data/experiment/html/tcgaWGBSData.hg19.html).\n\n\n## Questions and Comments\nPlease use the Github Issues tracker with any issue you face with the platform. Any specific questions or comments, contact me at divyswar01@g.harvard.edu\n","has_starred","hub_user","immutable_tags_settings",{"_75":11,"_76":77},"enabled","rules",[78],".*","is_automated","permissions",{"_82":21,"_83":11,"_84":11},"read","write","admin","user","storage_size",1849041102,"orgData",{"_90":91,"_92":93,"_94":95,"_96":33,"_97":98,"_99":100,"_101":100,"_102":103,"_104":105,"_106":100,"_107":100,"_108":100,"_109":21},"id","1925829d55ee4b3fbf72dd70d633869c","uuid","1925829d-55ee-4b3f-bf72-dd70d633869c","type","organization","orgname","full_name","Aryee Lab","location","","company","profile_url","http://aryee.mgh.harvard.edu","date_joined","2016-05-13T02:08:10.864148Z","gravatar_url","gravatar_email","badge","is_active","recentTags",[112],{"_113":44,"_114":115,"_116":117,"_90":118,"_119":120,"_50":137,"_138":115,"_139":33,"_140":56,"_57":141,"_142":143,"_144":128,"_145":137,"_146":66,"_147":21},"content_type","creator",825896,"full_size",245478189,40975314,"images",[121],{"_122":123,"_124":125,"_126":100,"_127":128,"_129":130,"_131":132,"_133":100,"_134":-5,"_135":117,"_63":66,"_136":-5},"architecture","amd64","digest","sha256:2bfe4c5052562e560b8f8be4dc11fe2a0462d6ff3ab591ac6d2867b336268e33","features","last_pulled","2026-10-04T22:19:56.705400624Z","last_pushed","2019-03-21T18:02:56Z","os","linux","os_features","os_version","size","variant","2019-03-21T18:02:55.718201Z","last_updater","last_updater_username","media_type","latest","repository",6359853,"tag_last_pulled","tag_last_pushed","tag_status","v2","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","showSbxLogoOverride","sbxTrademarkText","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"selectedTag","attestationsUrl","routes/_layout.r.$namespace.$reponame.tags",{"_38":161,"_168":169,"_155":178,"_179":11},{"_40":162,"_42":163,"_45":46,"_47":48,"_49":11,"_50":51,"_52":53,"_54":164,"_57":35,"_32":33,"_58":59,"_60":44,"_61":62,"_63":64,"_65":66,"_67":-5,"_68":62,"_69":70,"_71":11,"_72":33,"_73":165,"_79":21,"_80":167,"_85":33,"_86":87},[],[44],[56],{"_75":11,"_76":166},[78],{"_82":21,"_83":11,"_84":11},"tags",{"_170":64,"_171":-5,"_172":-5,"_173":174},"count","next","previous","results",[175],{"_113":44,"_114":115,"_116":117,"_90":118,"_119":176,"_50":137,"_138":115,"_139":33,"_140":56,"_57":141,"_142":143,"_144":128,"_145":137,"_146":66,"_147":21},[177],{"_122":123,"_124":125,"_126":100,"_127":128,"_129":130,"_131":132,"_133":100,"_134":-5,"_135":117,"_63":66,"_136":-5},[],"canDeleteTags","actionData","errors"]

aryeelab/pdr

Tags for aryeelab/pdr