module1_2/inputData/project-monocots/phyloTree.txt

 #treenode_number	window_size	genome_number	gf_number
 # Tree in Newick Format:
-#(((51364, 54711), 51051), ((25734, 33018), 33908))
-(((Spirodela_51364, Acorus_54711), Dioscorea_51051), ((Ananas_25734, Elaeis_33018), Asparagus_33908))
-#51364	54711	25734,33018,33908,51051
-#51051	51364,54711	25734,33018,33908
-#33908	25734,33018	51051,51364,54711
-#25734	33018	33908,51051,51364,54711
+((((Spirodela_51364, Acorus_54711), Dioscorea_51051), Asparagus_33908), (Ananas_25734, Elaeis_33018))

module1_2/src/Module1_2.ipynb

 %% Cell type:markdown id: tags:
 
 # RACCROCHE - Module 1 & 2
 This is a full workflow that shows methods regarding the construction of gene families, listing of candidate adjacencies and construction of ancestral contigs using Maximum Weight Matching.
 
 %% Cell type:markdown id: tags:
 
 # Section 1: Importing Libraries, Modules and Configuration File
 
 %% Cell type:code id: tags:
 
 ``` python
 # setting the Notebook Display Method
 %matplotlib inline
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 # install tqdm-joblib
 %pip install tqdm-joblib
 
 # install Bio
 %pip install Bio
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 # import libraries
 import sys
 import yaml
 import time
 import os
 import io
 
 import pandas as pd
 import seaborn as sns
 import matplotlib
 import matplotlib.pyplot as plt
 from Bio import Phylo
 from scipy.spatial.distance import squareform
 
 from tqdm import tqdm
 import tqdm_joblib
 from joblib import Parallel, delayed
 
 # import modules
 #? eventually the below list will be in your package
 from GeneFamily import GeneFamily
 from Genome import Genome
 from MWMInputTreeNode import MWMInputTreeNode
 from mwmatching import *
 from Contig import *
 ```
 
 %% Cell type:markdown id: tags:
 
 ## Parsing the Configuration File
 
 The configuration file consists of a list of input/output directories, input files and parameters to the pipeline.
 
 Specifications for Input:
 - leaf_genome_info: the input genomes with their positions on the phylogenetic tree, i.e. *Genomes.txt* <br>
 - synteny_file_name: the post-fix of the synmap output of pairwise genome comparisons; <br>
 - synteny_file_path: the path to the synmap output data <br>
 - phylo_tree_path: the path to the Newick Phylogenetic Tree structure <br>
 - jar_path: the path to the UniMoG jar file <br>
 
 __Note__: The input genomes is in the *Genomes.txt* file.
 It contains the information about input genomes, including CoGe ID, genomeName, ancestor<br>
 The phylogenetic relationship is in the *phyloTree* file.
 It includes the unrooted phylogenetic tree in newick format where each tree leaf is denoted as name_CoGeID. <br>
 Sample Newick Tree for the monocot project: ( ( (Spirodela_51364, Acorus_54711), Dioscorea_51051), ( (Ananas_25734, Elaeis_33018), Asparagus_33908) )
 
 Specifications for Output:
 - gene_list: path where the gene list output should be created <br>
 - gene_family: path where the gene family output should be created <br>
 - genomes: path where the genome string output should be created <br>
 - mwm_input_template: path and template for the output files created from MWM Input step <br>
 - mwm_output_template: path and template for the output files created from MWM <br>
 - contig_template: path and template for the output files created from constructing contigs <br>
 - dcj_output_path: path to the output file when calculating DCJ distance between ancestors <br>
 - dcj_summary_path: path to the output file containing a summary of the DCJ calculations <br><br>
 
 Global parameters:
 - min_cutoff_weight: minimum similarity cutoff weight for gene family construction, i.e. *min_cutoff_weight=65*
 - max_cutoff_weight: maximum similarity cutoff weight for gene family construction, i.e. *max_cutoff_weight=100*
 - ws: window size, i.e. *ws=7*
 - min_mwm_weight: minimum adjacency weight to be considered for maximum weight matching, i.e. *min_mwm_weight=100*
 - gf1: maximum number of genes in a gene family, i.e. *gf1=50*
 - gf2: the maximum number of genes from a genome in a gene family, i.e. *gf2=10*
 - gf3: the minimum number of genomes in a gene family, i.e. *gf3=1*
 
 %% Cell type:code id: tags:
 
 ``` python
 # specify your Project Name
 
-proj_name = "buxus"
+proj_name = "monocots"
 
 # reading from the configuration file
 config_file = "../inputData/project-" + proj_name + "/config.yaml"
 
 with open(config_file, 'r') as stream:
     parsed_config = yaml.load(stream, Loader=yaml.FullLoader)
 
 directory = parsed_config['output_path'] + parsed_config['project_name']
 os.makedirs(directory, exist_ok = True)
 
 print("Project Name and Input Directory:", parsed_config['input_path'] + parsed_config['project_name'])
 print("Project Name and Output Directory:", parsed_config['output_path'] + parsed_config['project_name'])
 print('''Please check required input data under the Input Directory:
 \t 1. Genome.txt with info about input genomes
 \t 2. phyloTree.txt with Newick Tree Structure
 \t 3. SynMap results between every pair of genomes
 ''')
 #print("Project input data directory:", parsed_config['input_file']['synteny_file_path'])
 #print("Input extant genomes info:", parsed_config['input_file']['leaf_genome_info'])
 #print("The postfix of SynMap output files:", parsed_config['input_file']['synteny_file_name'])
 ```
 
 %% Output
 
-    Project Name and Input Directory: ../inputData/project-buxus
-    Project Name and Output Directory: ../outputData/project-buxus
+    Project Name and Input Directory: ../inputData/project-monocots
+    Project Name and Output Directory: ../outputData/project-monocots
     Please check required input data under the Input Directory:
     	 1. Genome.txt with info about input genomes
     	 2. phyloTree.txt with Newick Tree Structure
     	 3. SynMap results between every pair of genomes
     
 
 %% Cell type:markdown id: tags:
 
 # Section 2: Constructing Gene Families from syntenically validated orthologs from SynMap
 In order to succesfully construct gene families, we require:
 - Successful parsing of the configuration file <br>
 - Valid parameters in the configuration file
 
 %% Cell type:markdown id: tags:
 
 ## Reading in Input Genome Data and Tree Structure Data
 Each genome is a leaf of the input phylogenetic tree. The *get_leaves_and_tree_info()* method extracts the following attribute of each genome from the input file specified by *leaf_genome_info* parameter in the configuration file.
 >1) genome ID; <br>
 >2) genome name; <br>
 >3) most recent ancestor; <br>
 >4) number of chromosomes; <br>
 >5) file name of the genome annotations; <br>
 >6) desired tree structure in Newick format.
 
 %% Cell type:code id: tags:
 
 ``` python
 # reading in input genome info
 gene_family = GeneFamily(parsed_config)
 all_leaves, median_structure, newick_structure = gene_family.get_leaves_and_tree_info()
 
 print("Extant genomes to be analyzed in this project: \n")
 
 genomes = pd.DataFrame(all_leaves)
 display = genomes.iloc[:, :3]
 
 display.columns = ['Genome ID', 'Genome Name', 'Ancestor']
 display.set_index('Genome ID', inplace = True, drop = True)
 
 display
 ```
 
 %% Output
 
     Confirm ancestors:
-    ['28620', '35405', '54057,19990,57266,57268']
-    ['57266', '57268', '54057,19990,28620,35405']
-    ['28620,35405', '57266,57268', '54057,19990']
-    ['19990', '54057', '57266,57268,28620,35405']
+    ['51364', '54711', '25734,33018,33908,51051']
+    ['51364,54711', '51051', '25734,33018,33908']
+    ['51364,54711,51051', '33908', '25734,33018']
+    ['25734', '33018', '33908,51051,51364,54711']
     
     Extant genomes to be analyzed in this project:
     
 
-                Genome Name Ancestor
+              Genome Name Ancestor
     Genome ID
-    19990             Vitis        1
-    28620         Aquilegea        4
-    35405           Nelumbo        4
-    54057        Amaranthus        1
-    57266      Tetracentron        2
-    57268             Buxus        2
+    25734          Ananas        4
+    33018          Elaeis        4
+    33908       Asparagus        3
+    51051       Dioscorea        2
+    51364       Spirodela        1
+    54711          Acorus        1
 
 %% Cell type:code id: tags:
 
 ``` python
 tree = Phylo.read(io.StringIO(newick_structure), "newick")
 
 matplotlib.rc('font', size=8)
 fig = plt.figure(figsize=(8, 5), dpi=100)
 axes = fig.add_subplot(1, 1, 1)
 axes.set_facecolor('white')
 
 Phylo.draw(tree, axes = axes)
 ```
 
 %% Output
 
-
+
 
 %% Cell type:markdown id: tags:
 
 ## Creating gene families from syntenically validated homology generated from SynMap
 
 Read in all genes for every genome, record gene-related information and use gf1 (from config file), cutoff weights in order to create gene families.
 
 %% Cell type:code id: tags:
 
 ``` python
 print(all_leaves)
 # creating gene families
 gene_family.make_gene_family(all_leaves)
 
 print(len(all_leaves))
 ```
 
 %% Output
 
-    [['19990', 'Vitis', '1', '19'], ['28620', 'Aquilegea', '4', '7'], ['35405', 'Nelumbo', '4', '8'], ['54057', 'Amaranthus', '1', '16'], ['57266', 'Tetracentron', '2', '19'], ['57268', 'Buxus', '2', '14']]
+    [['25734', 'Ananas', '4', '25'], ['33018', 'Elaeis', '4', '16'], ['33908', 'Asparagus', '3', '10'], ['51051', 'Dioscorea', '2', '21'], ['51364', 'Spirodela', '1', '20'], ['54711', 'Acorus', '1', '12']]
     6
 
 %% Cell type:markdown id: tags:
 
 ## Visualizing Gene Families
 In the configuration file *config.yaml*, there are three parameters to restrict the size of gene families:
 - gf1: the maximum number of genes in a gene family
 - gf2: the maximum number of genes from a single genome in a gene family
 - gf3: the minimum number of genomes in a gene family <br>
 
 %% Cell type:code id: tags:
 
 ``` python
 families = list(gene_family.gene_family.keys())
 
 values = list(gene_family.gene_family.values())
 lengths = [len(values[i]) for i in range(0, len(values))]
 
 print("Total number of gene familes:", len(families))
 print("The size of the largest gene family:", max(lengths))
 ```
 
 %% Output
 
-    Total number of gene familes: 13649
+    Total number of gene familes: 8790
     The size of the largest gene family: 49
 
 %% Cell type:code id: tags:
 
 ``` python
 sns.set(style="darkgrid")
 family_dict = {'label': families, 'lengths': lengths}
 pdfamily = pd.DataFrame(family_dict)
 
 fig=sns.histplot(data = pdfamily, x = "lengths", binwidth = 3).set(title = "Overview of Gene Families")
 
 plt.xlabel("Number of genes in a gene family")
 plt.ylabel("Frequency")
 plt.show(fig)
 ```
 
 %% Output
 
-
+
 
 %% Cell type:markdown id: tags:
 
 # Section 3: Representing Genomes by Gene Family Labels
 In order to succesfully separate the input data (genes) into respective chromosomes and genomes, we require:
 - Successful creation of gene families
 
 %% Cell type:markdown id: tags:
 
 ## Grouping Genes and their ordering by Chromosomes then Chromosomes by Genomes
 
 %% Cell type:code id: tags:
 
 ``` python
 # creating an instance of the Genome class
 initialize_genome = Genome(gene_family.gene_list, all_leaves, parsed_config)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 # separating each genome's genes into chromosomes
 genome = initialize_genome.get_genome_in_string()
 ```
 
 %% Cell type:markdown id: tags:
 
 ## Visualizing Genes Families in Chromosomes of extant genomes
 
 %% Cell type:code id: tags:
 
 ``` python
 genomes, chromosomes, gene_lengths = [], [], []
 genome_lengths = []
 num_chr = []
 
 for key, value in genome.items():
     current_lengths = 0
 
     for genome_id in all_leaves:
         if int(genome_id[0]) == int(key):
             genome_name = genome_id[1]
             num_chromosomes = int(genome_id[3])
             num_chr.append(int(num_chromosomes))
 
     for i, chromosome in enumerate(value):
         genomes.append(genome_name)
         gene_lengths.append(len(chromosome))
         current_lengths += 1
 
     chromosomes.extend(range(1, num_chromosomes + 1))
 
     genome_lengths.append(current_lengths)
 
 #print(chromosomes)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 length_index = 0
 
 # removing short scaffolds for visualization
 for i, lengths in enumerate(genome_lengths):
     left = length_index
     right = length_index + genome_lengths[i]
 
     gene_lengths[left:right] = sorted(gene_lengths[left:right], reverse = True)
     del gene_lengths[left + num_chr[i] : right]
     del genomes[left + num_chr[i] : right]
 
     length_index += num_chr[i]
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 print("Visualizing the number of gene families in each chromosome of each genome")
 
 data = {'genome': genomes, 'chromosome number': chromosomes, 'number of gene families': gene_lengths}
 
 pddata = pd.DataFrame(data)
 
 sns.set(style="darkgrid")
 
 g = sns.catplot(x = 'chromosome number', y = 'number of gene families', col = 'genome', data = pddata, kind="bar", aspect = .7, sharex = False, dodge = False)
 g.set_xticklabels(rotation = 30)
 ```
 
 %% Output
 
     Visualizing the number of gene families in each chromosome of each genome
 
     /Users/lij313/opt/anaconda3/lib/python3.8/site-packages/seaborn/categorical.py:3803: UserWarning: Setting `sharex=False` with `color=None` may cause different levels of the `x` variable to share colors. This will change in a future version.
       warnings.warn(msg.format("sharex", "x"), UserWarning)
 
-    <seaborn.axisgrid.FacetGrid at 0x7f9d12684490>
+    <seaborn.axisgrid.FacetGrid at 0x7fc296913190>
 
-
+
 
 %% Cell type:markdown id: tags:
 
 # Section 4: Extract gene proximity (or adjacencies) from each extant genome
 
 Identify generalized gene adjacencies with a user specified window/gap size. The size parameter (*ws*) is defined in the configuration file.
 In order to succesfully prepare adjacency data for the Maximum Weight Matching algorithm, we require:
 - Successful separation of input data
 
 %% Cell type:markdown id: tags:
 
 ## Collecting Gene Adjacencies for Maximum Weight Matching
 
 Adjacencies are extracted from gene orders in each extant genome. <br>
 A large number of (generalized) adjacencies is collected for each ancestor defined in *median_structure*.
 
 
 Number of edges are the same since the number of genes and window size is consistent for each ancestor. However, the weights will be different.
 
 %% Cell type:code id: tags:
 
 ``` python
 ### manually change the median structure to construct monoploid extant genome
-#median_structure=[['57266', '57266', '57266']]
+#median_structure=[['61837', '61837', '61837']]
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 # preparing data for maximum weight matching + maximum weight matching
 window_size = parsed_config['ws']
 min_adj_weight = parsed_config['min_mwm_weight']
 num_gene_families = len(gene_family.gene_family)
 
 input_tree_node = MWMInputTreeNode(genome, all_leaves)
 
 mwmin_output_template = parsed_config['output_file']['mwm_input_template']
 
 num_anc = 1
 mwm_inputs = [None] * len(median_structure)
 
 for i, structure in tqdm(enumerate(median_structure), total=len(mwm_inputs)):
     outfile_mwmin = directory + mwmin_output_template + str(num_anc) + ".txt"
     mwm_inputs[num_anc - 1] = input_tree_node.get_mwm_input(structure, num_gene_families, window_size, min_adj_weight, outfile_mwmin)
     num_anc += 1
 ```
 
 %% Output
 
-    100%|██████████| 4/4 [00:03<00:00,  1.27it/s]
+    100%|██████████| 4/4 [00:02<00:00,  1.53it/s]
 
 %% Cell type:markdown id: tags:
 
 # Section 5: Maximum Weight Matching
 In order to successfully complete the Maximum Weight Matching algorithm, we require:
 - Successful preparation of Maximum Weight Matching Input Data
 
 %% Cell type:markdown id: tags:
 
 ## Maximum Weight Matching
 
 %% Cell type:markdown id: tags:
 
 Produces the result of Maximum Weight Matching for each ancestor. <br>
 This step is computationally expensive. <br>
 Uses multiprocessing to speed up the process.
 
 %% Cell type:code id: tags:
 
 ``` python
 # The path of output file in general form
 mwm_output_template = parsed_config['output_file']['mwm_output_template']
 
 def mwm_matching(mwm_input, num_anc):
     outfile_mwmout = directory + mwm_output_template + str(num_anc) + ".txt"
     maxWeightMatching(mwm_input, outfile_mwmout)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 # this step is slow
 # inputs for multiprocessing
 numbers = [i for i in range(1, num_anc)]
 inputs = zip(mwm_inputs, numbers)
 
 start = time.time()
 
 # multiprocessing to make workflow faster
 with tqdm_joblib.tqdm_joblib(tqdm(desc = "Maximum Weight Matching", total = len(mwm_inputs))) as progress_bar:
     Parallel(n_jobs=(num_anc - 1))(delayed(mwm_matching)(mwm_inputs[i], numbers[i]) for i in range (0, len(mwm_inputs)))
 
 end = time.time()
 ```
 
+%% Output
+
+    Maximum Weight Matching:   0%|          | 0/4 [00:00<?, ?it/s]
+      0%|          | 0/4 [00:00<?, ?it/s]
+
 %% Cell type:code id: tags:
 
 ``` python
 print("MWM solution was computed in %d minutes" % int((end - start)/60))
 ```
 
 %% Cell type:markdown id: tags:
 
 # Section 6: Constructing Ancestral Contigs
 In order to succesfully construct ancestral contigs, we require:
 - Successful Maximum Weight Matching on the prepared input data
 
 %% Cell type:markdown id: tags:
 
 ## Creating Contigs
 
 %% Cell type:markdown id: tags:
 
 Produces the contigs for each ancestor.
 
 %% Cell type:code id: tags:
 
 ``` python
 # creating contigs from the maximum weight matching output
 contig_template = parsed_config['output_file']['contig_template']
 
 # string to use for next step
 dcj_files = []
 
 for i in tqdm(range(1, num_anc), total = len(mwm_inputs)):
     outfile_mwmout = directory + mwm_output_template + str(i) + ".txt"
     outfile_contig = directory + contig_template + str(i) + ".txt"
     dcj_files.append(outfile_contig)
 
     contig = Contig(outfile_mwmout, int(num_gene_families))
     contig.get_edge()
     list_telomeres = contig.find_telomeres()
 
     contig_list = contig.get_contigs(list_telomeres, outfile_contig, str(i))
 ```
 
-%% Output
-
-    100%|██████████| 4/4 [00:00<00:00,  6.68it/s]
-
 %% Cell type:markdown id: tags:
 
 # Section 7: Calculating DCJ Distance Between Ancestors
 In order to succesfully calculate the DCJ Distance between ancestors, we require:
 - Successful creation of ancestral contigs
 
 %% Cell type:markdown id: tags:
 
 ## Calculating DCJ Distance
 
 %% Cell type:code id: tags:
 
 ``` python
 jar_path = parsed_config['input_file']['jar_path']
 
 dcj_output = []
 
 # Calculate DCJ Distance
 print("Starting DCJ Calculations, May Take a While")
 
 for i in tqdm(range(1, num_anc)):
     for j in range(i + 1, num_anc):
         dcj_output_path = directory + parsed_config['output_file']['dcj_output_path'] + str(i) + "_" + str(j) + ".txt"
         command = "java -jar " + jar_path + "/UniMoG-java11.jar " + str(dcj_files[i - 1]) + " " + str(dcj_files[j - 1]) + " -m=1 -p >" + dcj_output_path
         dcj_output.append(dcj_output_path)
         print(command)
         print("Computing DCJ distance between ancestors", i, "and", j)
         os.system(command)
 
 print("Calculation Done")
 ```
 
-%% Output
-
-    
-      0%|          | 0/4 [00:00<?, ?it/s]
-
-    Starting DCJ Calculations, May Take a While
-    java -jar ./UniMoG-java11.jar ../outputData/project-buxus/contigAnc1.txt ../outputData/project-buxus/contigAnc2.txt -m=1 -p >../outputData/project-buxus/dcj_output1_2.txt
-    Computing DCJ distance between ancestors 1 and 2
-    java -jar ./UniMoG-java11.jar ../outputData/project-buxus/contigAnc1.txt ../outputData/project-buxus/contigAnc3.txt -m=1 -p >../outputData/project-buxus/dcj_output1_3.txt
-    Computing DCJ distance between ancestors 1 and 3
-    java -jar ./UniMoG-java11.jar ../outputData/project-buxus/contigAnc1.txt ../outputData/project-buxus/contigAnc4.txt -m=1 -p >../outputData/project-buxus/dcj_output1_4.txt
-    Computing DCJ distance between ancestors 1 and 4
-
-    
-     25%|██▌       | 1/4 [00:39<01:57, 39.25s/it]
-
-    java -jar ./UniMoG-java11.jar ../outputData/project-buxus/contigAnc2.txt ../outputData/project-buxus/contigAnc3.txt -m=1 -p >../outputData/project-buxus/dcj_output2_3.txt
-    Computing DCJ distance between ancestors 2 and 3
-    java -jar ./UniMoG-java11.jar ../outputData/project-buxus/contigAnc2.txt ../outputData/project-buxus/contigAnc4.txt -m=1 -p >../outputData/project-buxus/dcj_output2_4.txt
-    Computing DCJ distance between ancestors 2 and 4
-
-    
-     50%|█████     | 2/4 [01:06<01:04, 32.16s/it]
-
-    java -jar ./UniMoG-java11.jar ../outputData/project-buxus/contigAnc3.txt ../outputData/project-buxus/contigAnc4.txt -m=1 -p >../outputData/project-buxus/dcj_output3_4.txt
-    Computing DCJ distance between ancestors 3 and 4
-
-    100%|██████████| 4/4 [01:16<00:00, 19.01s/it]
-
-    Calculation Done
-
-    
-
 %% Cell type:code id: tags:
 
 ``` python
 dcj_summary = directory + parsed_config['output_file']['dcj_summary_path']
 dcj_matrix = []
 
 with open(dcj_summary, 'w') as dcj_summary_file:
     for i in range(len(median_structure)):
             dcj_summary_file.write("Median Structure for Ancestor " + str((i + 1)) + ": " + "%s" % median_structure[i] + "\n")
 
     for file in dcj_output:
         path = file
 
         with open(path, 'r') as dcj_file:
             dcj_info = dcj_file.readlines()
 
         dcj_summary_file.write(dcj_info[0])
         print(dcj_info[0])
 
         distance = int((dcj_info[0].split())[-1])
         dcj_matrix.append(distance)
 
 print("Summay of pairwise DCJ distance -- Done")
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 matrix_indices = [("Ancestor " + str(i)) for i in range (1, num_anc)]
 dcj_matrix_summary = pd.DataFrame(squareform(dcj_matrix), index=matrix_indices, columns=matrix_indices)
 dcj_matrix_summary.style.set_caption("DCJ Distance Summary Symmetric Matrix")
 dcj_matrix_summary
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 ```