Prepare ENCODE ATAC/DNase Peaks 02

Download the data

set environment

Code 
source ../run_config_project.sh
show_env
You are working on             Duke Server: RCC
BASE DIRECTORY (FD_BASE):      /data/reddylab/Kuei
REPO DIRECTORY (FD_REPO):      /data/reddylab/Kuei/repo
WORK DIRECTORY (FD_WORK):      /data/reddylab/Kuei/work
DATA DIRECTORY (FD_DATA):      /data/reddylab/Kuei/data
CONTAINER DIR. (FD_SING):      /data/reddylab/Kuei/container

You are working with           ENCODE FCC
PATH OF PROJECT (FD_PRJ):      /data/reddylab/Kuei/repo/Proj_ENCODE_FCC
PROJECT RESULTS (FD_RES):      /data/reddylab/Kuei/repo/Proj_ENCODE_FCC/results
PROJECT SCRIPTS (FD_EXE):      /data/reddylab/Kuei/repo/Proj_ENCODE_FCC/scripts
PROJECT DATA    (FD_DAT):      /data/reddylab/Kuei/repo/Proj_ENCODE_FCC/data
PROJECT NOTE    (FD_NBK):      /data/reddylab/Kuei/repo/Proj_ENCODE_FCC/notebooks
PROJECT DOCS    (FD_DOC):      /data/reddylab/Kuei/repo/Proj_ENCODE_FCC/docs
PROJECT LOG     (FD_LOG):      /data/reddylab/Kuei/repo/Proj_ENCODE_FCC/log
PROJECT REF     (FD_REF):      /data/reddylab/Kuei/repo/Proj_ENCODE_FCC/references
PROJECT IMAGE   (FP_PRJ_SIF):  /data/reddylab/Kuei/container/project/singularity_proj_encode_fcc.sif
PROJECT CONF.   (FP_CNF):      /data/reddylab/Kuei/repo/Proj_ENCODE_FCC/scripts/config_project.sh

Set global variables

Code 
TXT_FOLDER="encode_open_chromatin"

Execute

Run download script

Code 
FD_OUT=${FD_DAT}/external/${TXT_FOLDER}

cd ${FD_OUT}
chmod +x ./run_download_files.sh

./run_download_files.sh

Run checksum

Code 
FN_EXE=run_checksum_files.sh
FP_EXE=${FD_EXE}/${FN_EXE}

FD_OUT=${FD_DAT}/external/${TXT_FOLDER}
FP_INP=${FD_OUT}/checksum_md5sum.txt
FP_OUT=${FD_OUT}/checksum_results.txt

${FP_EXE} ${FP_CNF} ${FD_OUT} ${FP_INP} ${FP_OUT}
Hostname:           plp-rcc-node-25
Slurm Array Index: 
Time Stamp:         05-13-25+15:33:27

Change directory:
/data/reddylab/Kuei/repo/Proj_ENCODE_FCC/data/external/encode_open_chromatin

Checksum files...


Done!
Run Time: 7 seconds

Review

Check output files

Code 
FD_OUT=${FD_DAT}/external/${TXT_FOLDER}

cd ${FD_OUT}
ls -sh {*bed.gz,*.bw} | wc -l
ls -sh {*bed.gz,*.bw}
10
1.3M K562.hg38.ENCSR000EKS.ENCFF274YGF.DNase.bed.gz
714M K562.hg38.ENCSR000EKS.ENCFF972GVB.DNase.bw
1.8M K562.hg38.ENCSR000EOT.ENCFF185XRG.DNase.bed.gz
672M K562.hg38.ENCSR000EOT.ENCFF414OGC.DNase.bw
4.8M K562.hg38.ENCSR483RKN.ENCFF558BLC.ATAC.bed.gz
825M K562.hg38.ENCSR483RKN.ENCFF600FDO.ATAC.bw
2.7M K562.hg38.ENCSR483RKN.ENCFF925CYR.ATAC.bed.gz
6.8M K562.hg38.ENCSR868FGK.ENCFF333TAT.ATAC.bed.gz
2.8G K562.hg38.ENCSR868FGK.ENCFF357GNC.ATAC.bw
4.1M K562.hg38.ENCSR868FGK.ENCFF948AFM.ATAC.bed.gz
Code 
FD_OUT=${FD_DAT}/external/${TXT_FOLDER}
FN_OUT=K562.hg38.ENCSR483RKN.ENCFF558BLC.ATAC.bed.gz
FP_OUT=${FD_OUT}/${FN_OUT}

zcat ${FP_OUT} | head -n 3
chr1    100028923   100029241   Peak_202759 52  .   2.91347 5.24652 3.38358 74
chr1    100037575   100038998   Peak_172664 68  .   1.83651 6.83702 4.90355 60
chr1    100037575   100038998   Peak_30781  934 .   5.32597 93.45621    90.75060    988

Check checksum results

Code 
FD_OUT=${FD_DAT}/external/${TXT_FOLDER}
FP_OUT=${FD_OUT}/checksum_results.txt

cat ${FP_OUT}
cat ${FP_OUT} | grep "FAILED" && echo "FAILED" || echo "All PASSED"
K562.hg38.ENCSR000EKS.ENCFF972GVB.DNase.bw: OK
K562.hg38.ENCSR000EKS.ENCFF274YGF.DNase.bed.gz: OK
K562.hg38.ENCSR000EOT.ENCFF414OGC.DNase.bw: OK
K562.hg38.ENCSR000EOT.ENCFF185XRG.DNase.bed.gz: OK
K562.hg38.ENCSR483RKN.ENCFF600FDO.ATAC.bw: OK
K562.hg38.ENCSR483RKN.ENCFF558BLC.ATAC.bed.gz: OK
K562.hg38.ENCSR483RKN.ENCFF925CYR.ATAC.bed.gz: OK
K562.hg38.ENCSR868FGK.ENCFF357GNC.ATAC.bw: OK
K562.hg38.ENCSR868FGK.ENCFF948AFM.ATAC.bed.gz: OK
K562.hg38.ENCSR868FGK.ENCFF333TAT.ATAC.bed.gz: OK
All PASSED

Check execution log

Code 
FD_OUT=${FD_DAT}/external/${TXT_FOLDER}

head -n 10 ${FD_OUT}/run_download.log.txt
--2025-05-13 15:30:59--  https://www.encodeproject.org/files/ENCFF972GVB/@@download/ENCFF972GVB.bigWig
Resolving www.encodeproject.org (www.encodeproject.org)... 34.211.244.144
Connecting to www.encodeproject.org (www.encodeproject.org)|34.211.244.144|:443... connected.
HTTP request sent, awaiting response... 307 Temporary Redirect
Location: https://encode-public.s3.amazonaws.com/2020/11/22/953ca40d-376f-4043-8dc9-710dd64a5ea9/ENCFF972GVB.bigWig?response-content-disposition=attachment%3B%20filename%3DENCFF972GVB.bigWig&AWSAccessKeyId=ASIATGZNGCNXZYW4CKKO&Signature=FO5Mfgj23r5A6t8MWa4rZLhRDc8%3D&x-amz-security-token=IQoJb3JpZ2luX2VjEEsaCXVzLXdlc3QtMiJGMEQCIBk3ht1wXuBOY0mV2%2BeMo8mYnoKBcLNUJ2417mpMgBWFAiBlqbrBgNXV9vo7awtqpBuPWrFcmbwljHbf826BIeRgxSq8BQj0%2F%2F%2F%2F%2F%2F%2F%2F%2F%2F8BEAAaDDIyMDc0ODcxNDg2MyIMh4BXxWB6vCEm%2FvTbKpAFCDAFujPYEwvRs8y68Jn1ZxiRLvC5xRckEo2mO8cpUMjBPVf4S3%2BCBIom0n4PvOdj6loQ%2BZUcK4fs9sUTvB%2B4RO4XD%2BlNtXk7X07mPBIBfxKzAlThb6Po3Q9LKGwyGO3F1ns57RCYpoulHWAsOrO9Ixq3MKXa659ijhzW%2FKc%2Fvk4LNUQfZ%2FrnTS5a%2BABAIFT1sN2KEVrL%2Fpnnn6WU%2B6ST%2FjmvwBHFlkwcRlY8ILxAV%2Bo2H1kXPrBak0jYPUZ7Xzlo%2BLwYhow7cM57Twrvxg2pY1SC2btSurNcuiyQyBIfX8bMmkp90AeI4o49wMsdlrGcKHOs7ReLbfIlXSi%2BOMKst7hPzTzwj3r4FWAFoTbDwSui2i3wI1vFvDmxmp7erIMatpINeoUwfEulyMsKoFmnFJa0MhlmecWpflvbM91F3wmcNd8tB1nq7620Hg7kvoRwOJpUKoqBGzewRii3pk81VVxljNALdyXi8Sd5MJAKbMzjuRPKtMtrhwcoRPkWdswyalApQItJCD9UjkEVNB47gqVB%2BWBbeVLHDrjLtzs4AKoaCwv6EEvD4X9PVYZsE0rTgVRfJKzXiROkccfLfnmCvMcPHy7bwNgCfVRZLlipbUiKhmxTqVDAhijtvsFaRSD%2BgnCOE7A3683hidqYI0dgs4vQU1I%2BecYDlq%2BAtSq01gDLSSdWPot6ndMUdEeG4MGaBp49UAdHTY4NL7AgyKth7mhVfPVrf5DOFjDWyUOOtnlmMROsazqjVc66jSBg8BIO62OoCPvebsM35TxumSE91914jL7Xc0H5nstpZsX1qXCVyTc6CXmysrtXSW3t9l4QadJUTAiBOIZ1IDlzynGRON8Pry%2FUwR6lynTf4RgR0vcwvqqOwQY6sgH3Qj3QMM7W5Ds%2BH1TlV8g6vk8YV3oNgQG97XWl2mB5qG%2FrTlZsBwquG3hDsduh7YT%2FHAqQyYZEYy3dZz5QNZmIi59jngQ%2BQ6KbsLvO7n8E6OsaV5jQUiwxRqrNe0Tw6Yg%2FXVacY0%2F8ta5n8I6NSkc7dYEobUgonasBq3%2FISArPGFFjr9C9Kcl1Kr%2FAYwmbjuRcUqx5nWceIVx%2F%2BdN9IjCwuuaYXmGZlTX%2FB%2FKyD%2FSO41a9&Expires=1747294260 [following]
--2025-05-13 15:31:00--  https://encode-public.s3.amazonaws.com/2020/11/22/953ca40d-376f-4043-8dc9-710dd64a5ea9/ENCFF972GVB.bigWig?response-content-disposition=attachment%3B%20filename%3DENCFF972GVB.bigWig&AWSAccessKeyId=ASIATGZNGCNXZYW4CKKO&Signature=FO5Mfgj23r5A6t8MWa4rZLhRDc8%3D&x-amz-security-token=IQoJb3JpZ2luX2VjEEsaCXVzLXdlc3QtMiJGMEQCIBk3ht1wXuBOY0mV2%2BeMo8mYnoKBcLNUJ2417mpMgBWFAiBlqbrBgNXV9vo7awtqpBuPWrFcmbwljHbf826BIeRgxSq8BQj0%2F%2F%2F%2F%2F%2F%2F%2F%2F%2F8BEAAaDDIyMDc0ODcxNDg2MyIMh4BXxWB6vCEm%2FvTbKpAFCDAFujPYEwvRs8y68Jn1ZxiRLvC5xRckEo2mO8cpUMjBPVf4S3%2BCBIom0n4PvOdj6loQ%2BZUcK4fs9sUTvB%2B4RO4XD%2BlNtXk7X07mPBIBfxKzAlThb6Po3Q9LKGwyGO3F1ns57RCYpoulHWAsOrO9Ixq3MKXa659ijhzW%2FKc%2Fvk4LNUQfZ%2FrnTS5a%2BABAIFT1sN2KEVrL%2Fpnnn6WU%2B6ST%2FjmvwBHFlkwcRlY8ILxAV%2Bo2H1kXPrBak0jYPUZ7Xzlo%2BLwYhow7cM57Twrvxg2pY1SC2btSurNcuiyQyBIfX8bMmkp90AeI4o49wMsdlrGcKHOs7ReLbfIlXSi%2BOMKst7hPzTzwj3r4FWAFoTbDwSui2i3wI1vFvDmxmp7erIMatpINeoUwfEulyMsKoFmnFJa0MhlmecWpflvbM91F3wmcNd8tB1nq7620Hg7kvoRwOJpUKoqBGzewRii3pk81VVxljNALdyXi8Sd5MJAKbMzjuRPKtMtrhwcoRPkWdswyalApQItJCD9UjkEVNB47gqVB%2BWBbeVLHDrjLtzs4AKoaCwv6EEvD4X9PVYZsE0rTgVRfJKzXiROkccfLfnmCvMcPHy7bwNgCfVRZLlipbUiKhmxTqVDAhijtvsFaRSD%2BgnCOE7A3683hidqYI0dgs4vQU1I%2BecYDlq%2BAtSq01gDLSSdWPot6ndMUdEeG4MGaBp49UAdHTY4NL7AgyKth7mhVfPVrf5DOFjDWyUOOtnlmMROsazqjVc66jSBg8BIO62OoCPvebsM35TxumSE91914jL7Xc0H5nstpZsX1qXCVyTc6CXmysrtXSW3t9l4QadJUTAiBOIZ1IDlzynGRON8Pry%2FUwR6lynTf4RgR0vcwvqqOwQY6sgH3Qj3QMM7W5Ds%2BH1TlV8g6vk8YV3oNgQG97XWl2mB5qG%2FrTlZsBwquG3hDsduh7YT%2FHAqQyYZEYy3dZz5QNZmIi59jngQ%2BQ6KbsLvO7n8E6OsaV5jQUiwxRqrNe0Tw6Yg%2FXVacY0%2F8ta5n8I6NSkc7dYEobUgonasBq3%2FISArPGFFjr9C9Kcl1Kr%2FAYwmbjuRcUqx5nWceIVx%2F%2BdN9IjCwuuaYXmGZlTX%2FB%2FKyD%2FSO41a9&Expires=1747294260
Resolving encode-public.s3.amazonaws.com (encode-public.s3.amazonaws.com)... 52.92.202.17, 52.92.176.57, 52.92.204.249, ...
Connecting to encode-public.s3.amazonaws.com (encode-public.s3.amazonaws.com)|52.92.202.17|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 747881155 (713M) [binary/octet-stream]