三维基因组分析之Juicer

uname -a  查看内核/操作系统
#Linux chunyu-PowerEdge-R720 5.13.0-43-generic #48~20.04.1-Ubuntu SMP Thu May 12 12:59:19 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux
df -h  #硬盘信息
#文件系统                                          容量  已用  可用 已用% 挂载点
#rpool/USERDATA/lxh_gscj6g                        46T   298G  46T  1%   /home/lxh
#rpool/ROOT/ubuntu_ktv56z/usr/local               46T   3.7G  46T  1%   /usr/local
free -h  #内存信息
#              总计         已用        空闲      共享    缓冲/缓存    可用
#内存：       503Gi         260Gi       241Gi     10Mi   1.3Gi        240Gi
#交换：       2.0Gi         0B          2.0Gi
cat /proc/cpuinfo  #CPU信息
cat /proc/cpuinfo| grep "physical id"| sort| uniq| wc -l  #CPU数
cat /proc/cpuinfo | grep "cpu cores" | uniq  #每个CPU物理核心数
cat /proc/cpuinfo| grep "processor"| wc -l  #逻辑CPU数
nvidia-smi  #查看GPU信息和使用情况
cut -d: -f1 /etc/passwd  #查看系统所有用户
pstree -aps pid  #查看进程关系

#Uninstall old versions
sudo apt-get remove docker docker-engine docker.io containerd runc
sudo rm -rf /var/lib/docker
sudo rm -rf /var/lib/containerd

##Install using the repository
#Set up the repository
sudo apt-get update
sudo apt-get install \
    ca-certificates \
    curl \
    gnupg \
    lsb-release
sudo mkdir -m 0755 -p /etc/apt/keyrings
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

#Install Docker Engine
sudo apt-get update
sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
sudo docker run hello-world

#添加用户进docker组，支持非root使用
sudo gpasswd -a username docker
newgrp docker
docker run hello-world

#GPU 支持(https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#installation-guide)
distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \
      && curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
      && curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | \
            sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
            sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
sudo apt-get update
sudo apt-get install -y nvidia-container-toolkit
sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker
sudo docker run --rm --runtime=nvidia --gpus all nvidia/cuda:11.6.2-base-ubuntu20.04 nvidia-smi
# apt-get install nvidia-container-runtime

# hic-pipeline/
cd docker/hic-pipeline
docker build -t encodedcc/hic-pipeline:1.15.1 .
cd docker/delta
docker build -t encodedcc/hic-pipeline:1.15.1_delta .
cd docker/hiccups
docker build -t encodedcc/hic-pipeline:1.15.1_hiccups .

#或者从docker hub下载镜像
docker pull encodedcc/hic-pipeline:1.15.1
docker pull encodedcc/hic-pipeline:1.15.1_delta
docker pull encodedcc/hic-pipeline:1.15.1_hiccups

#列出安装的镜像
docker images
#REPOSITORY               TAG              IMAGE ID       CREATED         SIZE
#encodedcc/hic-pipeline   1.15.1_delta     41fc8e6dd680   2 hours ago     5.81GB
#encodedcc/hic-pipeline   1.15.1           95c8bf750530   3 months ago    2.21GB
#encodedcc/hic-pipeline   1.15.1_hiccups   363f736bd618   7 months ago    2.52GB
#hello-world              latest           feb5d9fea6a5   17 months ago   13.3kB

#启动容器
docker run -it encodedcc/hic-pipeline:1.15.1_hiccups /bin/bash
# -i 交互式使用
# -t 显示终端
# -v 主机路径:容器路径，实现主机和docker文件的共享，如 -v /home/lxh/Lab/ZheLiu/Hi-C/hic_run/hiccups_run/:/shared_folder
# --gpus all 将GPU添加到容器

#退出容器
exit

git clone https://github.com/ENCODE-DCC/hic-pipeline.git
cd hic-pipeline
pip install caper  #依赖caper实现，首次运行时安装
caper run hic.wdl -i tests/functional/json/test_hic.json --docker

##下载序列文件和注释文件
#对于人类和小鼠，从GENCODE (https://www.gencodegenes.org/) 下载参考基因组序列文件和注释文件
#Genome assembly GRCh38 /2023-02-08
wget -c https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_43/GRCh38.primary_assembly.genome.fa.gz  #-c，断点续传
wget https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_43/gencode.v43.annotation.gff3.gz
wget https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_43/gencode.v43.annotation.gtf.gz

#Genome assembly GRCm38 /2020-04-29
wget -c https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M25/GRCm38.primary_assembly.genome.fa.gz
wget https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M25/gencode.vM25.annotation.gff3.gz
wget https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M25/gencode.vM25.annotation.gtf.gz

##构建index文件
#GRCh38
bwa index -p GRCh38.primary_assembly.genome.fa.gz -a bwtsw /home/lxh/HiC_test/refgenomes/sequences/GRCh38/GRCh38.primary_assembly.genome.fa
#GRCm38
bwa index -p GRCm38.primary_assembly.genome.fa.gz -a bwtsw /home/lxh/HiC_test/refgenomes/sequences/GRCm38/GRCm38.primary_assembly.genome.fa


#染色体大小
samtools faidx GRCh38.primary_assembly.genome.fa
cut -f1,2 GRCh38.primary_assembly.genome.fa.fai > GRCh38.chrom.sizes
samtools faidx GRCm38.primary_assembly.genome.fa
cut -f1,2 GRCm38.primary_assembly.genome.fa.fai > GRCm38.chrom.sizes

##酶切片段文件
#HindIII
----------------
5' A^AGCTT 3'
3'  TTCGA^A 5'
----------------
#BglII
----------------
5' A^GATCT 3'
3'  TCTAG^A 5'
----------------
#DpnII
----------------
5' ^GATC 3'
3'  CTAG^ 5'
----------------
#MboI
----------------
5' ^GATC 3'
3'  CTAG^ 5'
----------------

##juicer
# 'HindIII'     : 'AAGCTT',
# 'DpnII'       : 'GATC',
# 'MboI'        : 'GATC',
# 'Sau3AI'      : 'GATC',
# 'Arima'       : [ 'GATC', 'GANTC' ]

mkdir enzymesizes && cd enzymesizes
nano restriction_site_locations.json
caper run ../make_restriction_site_locations.wdl -i restriction_site_locations.json --docker

#restriction_site_locations.json
{
  "make_restriction_site_locations.reference_fasta": "/home/lxh/HiC_test/refgenomes/sequences/GRCm38/GRCm38.primary_assembly.genome.fa",
  "make_restriction_site_locations.restriction_enzyme": "DpnII",
  "make_restriction_site_locations.assembly_name": "GRCm38"
}

conda activate hicenv  #激活conda环境
conda install -c bioconda multiqc

cd PATH_TO_RAW_DATA
mkdir fastqc
fastqc -o fastqc/ -t 24 ./*.fq.gz
multiqc .

#准备参考文件<JuicerDIR>
#Juicer 支持UCSC或者ENCODE的参考序列文件
mkdir -p references && cd references/mm10
wget -c https://www.encodeproject.org/files/ENCFF018NEO/@@download/ENCFF018NEO.tar.gz
wget https://www.encodeproject.org/files/mm10_no_alt.chrom.sizes/@@download/mm10_no_alt.chrom.sizes.tsv
tar -xzvf ENCFF018NEO.tar.gz -C .
nano restriction_site_locations.json
caper run ../../make_restriction_site_locations.wdl -i restriction_site_locations.json --docker
#run caper<workDir>
mkdir hic_run && cd hic_run
nano input.json  #json配置文件
caper run /home/lxh/HiC_test/hic-pipeline/hic.wdl -i input.json --docker
#
nohup caper run /home/lxh/HiC_test/hic-pipeline/hic.wdl -i input_EKO4d.json --docker >nohup_EKO4d.out 2>&1 &
nohup caper run /home/lxh/HiC_test/hic-pipeline/hic.wdl -i input_WT4d.json --docker >nohup_WT4d.out 2>&1 &

#hiccups分析|docker容器运行
cp /home/lxh/Lab/ZheLiu/Hi-C/hic_run/hic/fee97d45-1a06-4a50-94ee-a1db98fbf854/call-add_norm/shard-1/execution/inter_30.hic .
docker run -it -v /home/lxh/Lab/ZheLiu/Hi-C/hic_run/hic/fee97d45-1a06-4a50-94ee-a1db98fbf854/hiccups/:/shared_folder --gpus all encodedcc/hic-pipeline:1.15.1_hiccups /bin/bash
nvidia-smi  #GPU信息
#java -jar /opt/scripts/common/juicer_tools.jar hiccups -m 1024 inter_30.hic loops
java -jar /opt/scripts/common/juicer_tools.jar hiccups \
-m 1024 \
-r 5000,10000,25000,50000,100000 \
-f .1,.1,.1,.1,.1 \
-p 4,2,1,1,1 \
-i 7,5,3,3,3 \
-t 0.02,1.5,1.75,2,2 \
-d 20000,20000,50000,50000,50000 \
inter_30.hic loops_2