2026-08-02

This commit is contained in:
IvisTang 2026-08-02 14:18:07 +08:00
parent 7f4e888e6a
commit e398109a10
14 changed files with 14496 additions and 7653 deletions

2
.envrc
View File

@ -4,5 +4,3 @@ export SCRIPTS="$PROJECTHOME/99.scripts"
export PUEUE_CONFIG_PATH="$PROJECTHOME/.pueue.yml" export PUEUE_CONFIG_PATH="$PROJECTHOME/.pueue.yml"
watch_file pixi.lock watch_file pixi.lock
eval "$(pixi shell-hook)" eval "$(pixi shell-hook)"
PATH_add $SCRIPTS/bucky/bin
PATH_add $SCRIPTS/bpp/bin

4
.gitignore vendored
View File

@ -12,7 +12,11 @@
06.phylogeny_reconstruction/* 06.phylogeny_reconstruction/*
06.phylogenetic_analysis/* 06.phylogenetic_analysis/*
06.gene_trees/* 06.gene_trees/*
07.species_tree/*
07.coalescence_network/*
10.plastid/* 10.plastid/*
11.reference_v2/*
12.orthofinder/*
98.results/* 98.results/*
99.scripts/bucky/ 99.scripts/bucky/
99.scripts/phyparts/ 99.scripts/phyparts/

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,84 @@
#!/usr/bin/env python
"""
Prepare input sequence file from fasta file in a directory for BPP.
"""
import argparse
import sys
from pathlib import Path
from Bio import SeqIO
def get_fasta_files(input_dir: Path, ext=".fasta") -> list[Path]:
"""
Get a list of fasta files in the specified directory with the given extension.
Args:
input_dir (Path): Path to the directory containing fasta files.
ext (str): Extension of the fasta files.
Returns:
list[Path]: List of Paths to the fasta files.
"""
files = list(input_dir.glob(f"*{ext}"))
if not files:
print(f"No fasta files with extension '{ext}' found in directory '{input_dir}'")
sys.exit(1)
return files
def transform_fasta_to_bpp(fasta_file: Path):
"""
Transform a fasta file into BPP sequence format.
Args:
fasta_file (Path): Path to the input fasta file.
Returns:
list[str]: Lines formatted for BPP input.
"""
num_ind = 0
output_lines = []
seq_len = []
for record in SeqIO.parse(fasta_file, "fasta"):
ind_id = record.id
num_ind += 1
output_lines.append(f"^{ind_id} {str(record.seq)}\n")
seq_len.append(len(record.seq))
if len(set(seq_len)) != 1:
print(f"Error: Sequences in file '{fasta_file}' have different lengths.")
sys.exit(1)
header = f"{num_ind} {seq_len[0]}\n"
return [header, "\n", *output_lines, "\n"]
def make_bpp_seq(input_dir: Path, output_file: Path, ext=".fasta"):
"""
Create a BPP formatted sequence file from fasta files in a directory.
Args:
input_dir (Path): Path to the directory containing fasta files.
output_file (Path): Path to the output BPP formatted file.
ext (str): Extension of the fasta files.
"""
fasta_files = get_fasta_files(input_dir, ext)
with open(output_file, "w") as out_f:
for fasta_file in fasta_files:
bpp_lines = transform_fasta_to_bpp(fasta_file)
out_f.writelines(bpp_lines)
print(f"BPP formatted sequence file written to '{output_file}'")
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Create a BPP formatted sequence file from fasta files in a directory."
)
parser.add_argument(
"input_dir", type=Path, help="Directory containing fasta files."
)
parser.add_argument(
"output_file", type=Path, help="Output BPP formatted sequence file."
)
parser.add_argument(
"--ext",
type=str,
default=".fasta",
help="Extension of the fasta files (default: .fasta).",
)
args = parser.parse_args()
make_bpp_seq(args.input_dir, args.output_file, args.ext)

View File

@ -0,0 +1,89 @@
# BPP Coalescence Network Analysis Scripts
This directory contains scripts for running BPP (Bayesian Phylogenetics & Phylogeography) analysis using the A00 method for coalescence network inference.
## Files
1. `run_bpp_a00.ctl` - Control file for BPP A00 analysis
2. `run_bpp.sh` - Shell script to execute the BPP analysis
3. `generate_and_run_bpp.py` - Python script to generate control file and run BPP analysis
4. `README.md` - This file
## Prerequisites
1. BPP software must be installed and available in your PATH
2. Input sequence data in PHYLIP format
3. Imap file mapping sequences to species
4. Known species tree topology
## Usage
### Method 1: Manual Control File Editing
1. Update the control file (`run_bpp_a00.ctl`) with your specific:
- Sequence file path (`seqfile`)
- Imap file path (`Imapfile`)
- Species names, counts, and tree topology in `species&tree` block
- Number of loci (`nloci`)
- Other parameters as needed
2. Prepare your input files:
- Sequence file in PHYLIP format with multiple loci
- Imap file mapping individual sequences to species
- Ensure your species tree topology is correct
3. Run the analysis:
```bash
./run_bpp.sh
```
### Method 2: Automated Python Script (Recommended)
Use the Python script to automatically generate the control file and run the analysis:
```bash
python generate_and_run_bpp.py \
--seqfile sequences.txt \
--imapfile species_map.txt \
--species sp1 sp2 sp3 sp4 sp5 \
--nloci 100 \
--output my_analysis_result \
--threads 8 \
--burnin 20000 \
--nsample 100000 \
--sampfreq 5
```
## BPP A00 Analysis Mode
The A00 analysis mode is appropriate for your use case because:
- You have one sample per species
- You have a known species tree (systematic network)
- You want to estimate population size (θ) and divergence time (τ) parameters
In this mode:
- `speciesdelimitation = 0` - Species delimitation is fixed
- `speciestree = 0` - Species tree is fixed
- Parameters θ and τ are estimated on the fixed tree
## Key Parameters in Control File
- `thetaprior`: Inverse gamma prior for population size parameters (θ)
- `tauprior`: Inverse gamma prior for root divergence time, Dirichlet for others
- `burnin`: Number of burn-in iterations for MCMC
- `nsample`: Number of samples to collect
- `sampfreq`: Sampling frequency
- `threads`: Number of CPU threads to use
## Output Files
After successful execution, you will get:
- Main output file (default: `bpp_a00_results.txt`)
- MCMC samples file (default: `bpp_a00_results.mcmc.txt`)
- Seed information file (default: `bpp_a00_results.SeedUsed`)
## Customization
Depending on your data characteristics, you may want to adjust:
- Prior parameters for θ and τ
- MCMC settings (burnin, nsample, sampfreq)
- Threading options
- Output options in the `print` line

View File

@ -0,0 +1,8 @@
# Example Imap File Template
# This maps individual sequences to species
# Format: Individual_ID<tab>Species_Name
ind1 sp1
ind2 sp2
ind3 sp3
ind4 sp4

View File

@ -0,0 +1,18 @@
# Example Sequence File Template (PHYLIP format)
# This is a template - replace with your actual sequence data
# Locus 1
4 20
sp1^ind1 ATCGATCGAT CGATCGATCG
sp2^ind2 ATCGATCGAT CGATCGATCG
sp3^ind3 ATCGATCGAT CGATCGATCG
sp4^ind4 ATCGATCGAT CGATCGATCG
# Locus 2
4 20
sp1^ind1 CGATCGATCG ATCGATCGAT
sp2^ind2 CGATCGATCG ATCGATCGAT
sp3^ind3 CGATCGATCG ATCGATCGAT
sp4^ind4 CGATCGATCG ATCGATCGAT
# More loci...

View File

@ -0,0 +1,251 @@
#!/usr/bin/env python3
"""
Script to generate BPP A00 control file and run BPP analysis.
This script accepts command-line arguments to customize the BPP analysis,
generates a control file template with the provided parameters,
and executes the BPP analysis.
Usage:
python generate_and_run_bpp.py --seqfile SEQUENCE_FILE \
--imapfile IMAP_FILE \
--species SPECIES_NAMES \
--nloci NUMBER_OF_LOCI \
--output OUTPUT_NAME \
--threads THREADS \
--burnin BURNIN \
--nsample NSAMPLE \
--sampfreq SAMPFREQ
Parameters:
--seqfile: Path to the sequence file in PHYLIP format
--imapfile: Path to the Imap file mapping sequences to species
--species: Space-separated list of species names
--nloci: Number of loci in the dataset
--output: Base name for output files (default: bpp_a00_results)
--threads: Number of CPU threads to use (default: 4)
--burnin: Number of burn-in iterations for MCMC (default: 10000)
--nsample: Number of samples to collect (default: 50000)
--sampfreq: Sampling frequency (default: 2)
--thetaprior_alpha: Alpha parameter for theta prior (default: 3)
--thetaprior_beta: Beta parameter for theta prior (default: 0.002)
--tauprior_alpha: Alpha parameter for tau prior (default: 3)
--tauprior_beta: Beta parameter for tau prior (default: 0.004)
"""
import argparse
import os
import sys
from pathlib import Path
def generate_ctl_file(seqfile, imapfile, species_names, nloci, output_name="bpp_a00_results",
threads=4, burnin=10000, nsample=50000, sampfreq=2,
thetaprior_alpha=3, thetaprior_beta=0.002,
tauprior_alpha=3, tauprior_beta=0.004):
"""
Generate a BPP A00 control file with the specified parameters.
Args:
seqfile: Path to the sequence file in PHYLIP format
imapfile: Path to the Imap file mapping sequences to species
species_names: List of species names
nloci: Number of loci in the dataset
output_name: Base name for output files
threads: Number of CPU threads to use
burnin: Number of burn-in iterations for MCMC
nsample: Number of samples to collect
sampfreq: Sampling frequency
thetaprior_alpha: Alpha parameter for theta prior
thetaprior_beta: Beta parameter for theta prior
tauprior_alpha: Alpha parameter for tau prior
tauprior_beta: Beta parameter for tau prior
Returns:
str: Content of the control file
"""
# Convert species names to a space-separated string for the tree
species_str = " ".join(species_names)
# Create the species count list (all species have 10 individuals in this example)
species_counts = " ".join(["10"] * len(species_names))
# Create the tree structure (assuming a simple rooted tree)
# For a more complex tree, you would need to provide the full Newick format
if len(species_names) == 1:
tree_str = f"{species_names[0]};"
elif len(species_names) == 2:
tree_str = f"({species_names[0]}, {species_names[1]});"
elif len(species_names) >= 3:
# Create a simple nested tree structure
# For simplicity, we'll create a tree with the first two species as siblings,
# then the rest nested appropriately
tree_parts = []
for i in range(0, len(species_names), 2):
if i + 1 < len(species_names):
tree_parts.append(f"({species_names[i]}, {species_names[i+1]})")
else:
tree_parts.append(species_names[i])
# Build the final tree by nesting the parts
while len(tree_parts) > 1:
left = tree_parts.pop(0)
right = tree_parts.pop(0)
tree_parts.insert(0, f"({left}, {right})")
tree_str = f"{tree_parts[0]};"
else:
tree_str = "((sp1, sp2), sp3);"
# Generate the control file content
ctl_content = f"""# BPP A00 Analysis Control File
# Generated by generate_and_run_bpp.py
seed = -1
# Input files - these are set by the script
seqfile = {seqfile}
Imapfile = {imapfile}
jobname = {output_name}
# Analysis type A00: Fixed species delimitation and fixed species tree
# Estimate theta and tau parameters on a known tree
speciesdelimitation = 0 # Fixed species delimitation
speciestree = 0 # Fixed species tree
# Species tree specification
# Updated with provided species names, counts, and tree topology
species&tree = {len(species_names)} {species_str}
{species_counts}
{tree_str}
# Sequence phasing - set to 1 for unphased diploid data, 0 for phased haploid data
phase = 0 0 0 0 0
# Use sequence likelihood for inference
usedata = 1
# Number of loci to analyze - updated with provided value
nloci = {nloci}
# Clean data option - set to 1 to remove sites with ambiguity characters
cleandata = 0
# Priors for theta (population size parameters)
# Inverse gamma prior with alpha={thetaprior_alpha}, beta={thetaprior_beta} (mean={thetaprior_alpha/thetaprior_beta})
thetaprior = invgamma {thetaprior_alpha} {thetaprior_beta}
# Priors for tau (divergence times)
# Inverse gamma prior for root tau, Dirichlet for other taus
tauprior = invgamma {tauprior_alpha} {tauprior_beta}
# Fine-tune MCMC proposals automatically
finetune = 1
# Output options: MCMC samples, locus rates, heredity scalars, gene trees, substitution parameters
print = 1 0 0 0 0
# MCMC settings
burnin = {burnin} # Burn-in iterations
sampfreq = {sampfreq} # Sample every {sampfreq} iterations
nsample = {nsample} # Number of samples to collect
# Number of threads to use
threads = {threads}
# Checkpointing - save progress every 50000 iterations
checkpoint = 50000 50000
"""
return ctl_content
def main():
parser = argparse.ArgumentParser(description='Generate BPP A00 control file and run analysis')
# Required arguments
parser.add_argument('--seqfile', required=True, help='Path to the sequence file in PHYLIP format')
parser.add_argument('--imapfile', required=True, help='Path to the Imap file mapping sequences to species')
parser.add_argument('--species', nargs='+', required=True, help='Space-separated list of species names')
parser.add_argument('--nloci', type=int, required=True, help='Number of loci in the dataset')
# Optional arguments with defaults
parser.add_argument('--output', default='bpp_a00_results', help='Base name for output files (default: bpp_a00_results)')
parser.add_argument('--threads', type=int, default=4, help='Number of CPU threads to use (default: 4)')
parser.add_argument('--burnin', type=int, default=10000, help='Number of burn-in iterations for MCMC (default: 10000)')
parser.add_argument('--nsample', type=int, default=50000, help='Number of samples to collect (default: 50000)')
parser.add_argument('--sampfreq', type=int, default=2, help='Sampling frequency (default: 2)')
parser.add_argument('--thetaprior_alpha', type=float, default=3.0, help='Alpha parameter for theta prior (default: 3.0)')
parser.add_argument('--thetaprior_beta', type=float, default=0.002, help='Beta parameter for theta prior (default: 0.002)')
parser.add_argument('--tauprior_alpha', type=float, default=3.0, help='Alpha parameter for tau prior (default: 3.0)')
parser.add_argument('--tauprior_beta', type=float, default=0.004, help='Beta parameter for tau prior (default: 0.004)')
args = parser.parse_args()
# Validate inputs
if not os.path.exists(args.seqfile):
print(f"Error: Sequence file '{args.seqfile}' does not exist")
sys.exit(1)
if not os.path.exists(args.imapfile):
print(f"Error: Imap file '{args.imapfile}' does not exist")
sys.exit(1)
if len(args.species) < 1:
print("Error: At least one species name must be provided")
sys.exit(1)
# Generate the control file content
ctl_content = generate_ctl_file(
seqfile=args.seqfile,
imapfile=args.imapfile,
species_names=args.species,
nloci=args.nloci,
output_name=args.output,
threads=args.threads,
burnin=args.burnin,
nsample=args.nsample,
sampfreq=args.sampfreq,
thetaprior_alpha=args.thetaprior_alpha,
thetaprior_beta=args.thetaprior_beta,
tauprior_alpha=args.tauprior_alpha,
tauprior_beta=args.tauprior_beta
)
# Write the control file to disk
ctl_filename = f"{args.output}.ctl"
with open(ctl_filename, 'w') as f:
f.write(ctl_content)
print(f"Generated control file: {ctl_filename}")
print("\nControl file content:")
print("=" * 50)
print(ctl_content)
print("=" * 50)
# Check if BPP is installed
if not os.system('which bpp > /dev/null 2>&1') == 0:
print("Error: BPP is not installed or not in PATH")
print("Please install BPP and make sure it's accessible in your PATH")
sys.exit(1)
# Run BPP analysis
print(f"\nRunning BPP analysis with control file: {ctl_filename}")
cmd = f"bpp --cfile {ctl_filename}"
print(f"Executing: {cmd}")
# Execute the command
result = os.system(cmd)
if result == 0:
print("\nBPP analysis completed successfully!")
print(f"Output files:")
print(f" Main output: {args.output}.txt")
print(f" MCMC samples: {args.output}.mcmc.txt")
else:
print("\nError: BPP analysis failed")
sys.exit(1)
if __name__ == "__main__":
main()

View File

@ -0,0 +1,60 @@
#!/bin/bash
# Script to run BPP A00 analysis for coalescence network inference
# Set working directory
WORK_DIR=$(pwd)
SCRIPT_DIR="$(dirname "$0")"
# Check if BPP is installed and available
if ! command -v bpp &> /dev/null
then
echo "Error: BPP is not installed or not in PATH"
echo "Please install BPP and make sure it's accessible in your PATH"
exit 1
fi
# Check if control file exists
CONTROL_FILE="${SCRIPT_DIR}/run_bpp_a00.ctl"
if [ ! -f "$CONTROL_FILE" ]; then
echo "Error: Control file not found at $CONTROL_FILE"
echo "Please make sure the control file exists"
exit 1
fi
echo "Starting BPP A00 analysis..."
echo "Control file: $CONTROL_FILE"
echo "Working directory: $WORK_DIR"
# Run BPP analysis
bpp --cfile "$CONTROL_FILE"
# Check if the run was successful
if [ $? -eq 0 ]; then
echo "BPP analysis completed successfully!"
# Check for output files
JOBNAME=$(grep -E "^jobname\s*=" "$CONTROL_FILE" | sed -E 's/^jobname\s*=\s*(.*)/\1/' | tr -d ' ')
if [ -z "$JOBNAME" ]; then
JOBNAME="out"
fi
echo "Output files:"
echo " Main output: ${JOBNAME}.txt"
echo " MCMC samples: ${JOBNAME}.mcmc.txt"
if [ -f "${JOBNAME}.txt" ]; then
echo "Main output file created successfully"
else
echo "Warning: Main output file not found"
fi
if [ -f "${JOBNAME}.mcmc.txt" ]; then
echo "MCMC samples file created successfully"
else
echo "Warning: MCMC samples file not found"
fi
else
echo "Error: BPP analysis failed"
exit 1
fi

View File

@ -0,0 +1,57 @@
# BPP A00 Analysis Control File
# For coalescence network analysis with known species tree and delimitation
seed = -1
# Input files - these need to be updated with your actual file paths
seqfile = sequences.txt
Imapfile = species_map.txt
jobname = bpp_a00_results
# Analysis type A00: Fixed species delimitation and fixed species tree
# Estimate theta and tau parameters on a known tree
speciesdelimitation = 0 # Fixed species delimitation
speciestree = 0 # Fixed species tree
# Species tree specification
# Update the following with your actual species names, counts, and tree topology
species&tree = 5 sp1 sp2 sp3 sp4 sp5
10 10 10 10 10
((((sp1, sp2), sp3), sp4), sp5);
# Sequence phasing - set to 1 for unphased diploid data, 0 for phased haploid data
phase = 0 0 0 0 0
# Use sequence likelihood for inference
usedata = 1
# Number of loci to analyze - update with your actual number of loci
nloci = 100
# Clean data option - set to 1 to remove sites with ambiguity characters
cleandata = 0
# Priors for theta (population size parameters)
# Inverse gamma prior with alpha=3, beta=0.002 (mean=0.001)
thetaprior = invgamma 3 0.002
# Priors for tau (divergence times)
# Inverse gamma prior for root tau, Dirichlet for other taus
tauprior = invgamma 3 0.004
# Fine-tune MCMC proposals automatically
finetune = 1
# Output options: MCMC samples, locus rates, heredity scalars, gene trees, substitution parameters
print = 1 0 0 0 0
# MCMC settings
burnin = 10000 # Burn-in iterations
sampfreq = 2 # Sample every 2 iterations
nsample = 50000 # Number of samples to collect
# Number of threads to use
threads = 4
# Checkpointing - save progress every 50000 iterations
checkpoint = 50000 50000

View File

@ -0,0 +1,40 @@
#!/bin/bash
set -euo pipefail
cd "$PROJECTHOME"/11.reference_v2
# HG sequences name should be changed.
cd raw
seqkit replace -k HG.map.txt -p "^(\S+).*$" -r "{kv}" GCA_030763125.1_ASM3076312v1_genomic.fna -o HG.fa -K --f-use-regexp
# MG & ZG use hapA genome
grep -Fwf MG.target.txt MG.gff > MG.target.gff
grep -Fwf ZG.target.txt ZG.gff > ZG.target.gff
# Extract CDS sequences from GFF3 files using gffread
pueue add -- gffread -g raw/HN.fa -x HN.cds.fa -y HN.pep.fa raw/HN.gff3
pueue add -- gffread -g raw/HG.fa -x HG.cds.fa -y HG.pep.fa raw/Hippophae_gyantsensis.gff
pueue add -- gffread -g raw/HS.fa -x HS.cds.fa -y HS.pep.fa raw/HS.gff3
pueue add -- gffread -g raw/YN.genome.fa -x YN.cds.fa -y YN.pep.fa raw/YN.gff3
pueue add -- gffread -g raw/ZG.genome.fa -x ZG.cds.fa -y ZG.pep.fa raw/ZG.target.gff
pueue add -- gffread -g raw/ZJ.genome.fa -x ZJ.cds.fa -y ZJ.pep.fa raw/ZJ.gff
pueue add -- gffread -g raw/ZY.fa -x ZY.cds.fa -y ZY.pep.fa raw/ZY.gff3
pueue add -- gffread -g raw/Hippophae_tibetana.genomic.fasta -x HT.cds.fa -y HT.pep.fa raw/Hippophae_tibetana.mRNA.gff
pueue add -- gffread -g raw/MG.genome.fasta -x MG.cds.fa -y MG.pep.fa raw/MG.target.gff
cp raw/EA.gff3.cds EA.cds.fa
cp raw/EM.genome.cds.fasta EM.cds.fa
cp raw/EA.gff3.pep EA.pep.fa
cp raw/EM.pep.fasta EM.pep.fa
mkdir pep
mkdir cds
mv ./*.cds.fa cds
mv ./*.pep.fa pep
cd pep
primary_transcript ./ last_dot
# EM基因命名特殊性导致直接使用primary_transcript会出现问题直接复制即可
cp EM.pep.fa primary_transcripts/EM.pep.fa

View File

@ -0,0 +1,21 @@
# prepare input files for orthofinder
mkdir input_pep
cp ../11.reference_v2/pep/primary_transcripts/*.pep.fa input_pep
mkdir input_pep/core
mkdir input_pep/outgroup
mv input_pep/EA.pep.fa input_pep/outgroup
mv input_pep/EM.pep.fa input_pep/outgroup
mv input_pep/ZJ.pep.fa input_pep/outgroup
mv input_pep/*.pep.fa input_pep/core
for i in input_pep/*/*.pep.fa; do
base=$(basename "$i" .pep.fa)
echo "Processing $base"
sed -i "s/>/>${base}@/g" "$i"
done
# run orthofinder with Hippophae genomes as backbone
pueue add -- orthofinder -t 12 -a 4 -f input_pep/core -o orthofinder_out
# assign outgroups
pueue add -- orthofinder -t 12 -a 4 --assign input_pep/outgroup --core orthofinder_out/Results_Hippophae_only

15828
pixi.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -1,8 +1,8 @@
[workspace] [workspace]
authors = ["IvisTang <me@ivistang.com>"] authors = ["IvisTang <me@ivistang.com>"]
channels = [ channels = [
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/", "https://mirrors4.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/",
"https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/bioconda/", "https://mirrors4.tuna.tsinghua.edu.cn/anaconda/cloud/bioconda/",
"conda-forge", "conda-forge",
"bioconda" "bioconda"
] ]
@ -61,6 +61,15 @@ r-processx = ">=3.8.6,<4"
beagle-lib = ">=4.0.1,<5" beagle-lib = ">=4.0.1,<5"
beast = ">=10.5.0,<11" beast = ">=10.5.0,<11"
[feature.genome.dependencies]
gffread = ">=0.12.9,<0.13"
orthofinder = ">=3.1.5,<4"
seqkit = ">=2.13.0,<3"
tmux = ">=3.7b_,<4"
numpy = "<2"
[environments] [environments]
default = ["base"] base = ["base"]
beast = ["beast"] beast = ["beast"]
default = ["genome"]