Updated Britten data set; restructured connectome dataset page

.gitignore

@@ -685,3 +685,21 @@ __pycache__
 /docs/assets/Brittin2021*
 /docs/*_Brittin2021_data*.md
 /clean.sh
+/docs/assets/Test_Nonpharyngeal_Chemical_Exc.json
+/docs/assets/Test_Nonpharyngeal_Chemical_Exc.png
+/docs/assets/Test_Nonpharyngeal_Chemical_Exc_graph.json
+/docs/assets/Test_Nonpharyngeal_Chemical_Exc_graph.png
+/docs/assets/Test_Nonpharyngeal_Chemical_Exc_hiveplot.json
+/docs/assets/Test_Nonpharyngeal_Chemical_Exc_hiveplot.png
+/docs/assets/Test_Nonpharyngeal_Chemical_Inh.json
+/docs/assets/Test_Nonpharyngeal_Chemical_Inh.png
+/docs/assets/Test_Nonpharyngeal_Chemical_Inh_graph.json
+/docs/assets/Test_Nonpharyngeal_Chemical_Inh_graph.png
+/docs/assets/Test_Nonpharyngeal_Chemical_Inh_hiveplot.json
+/docs/assets/Test_Nonpharyngeal_Chemical_Inh_hiveplot.png
+/docs/assets/Test_Nonpharyngeal_Electrical.json
+/docs/assets/Test_Nonpharyngeal_Electrical.png
+/docs/assets/Test_Nonpharyngeal_Electrical_graph.json
+/docs/assets/Test_Nonpharyngeal_Electrical_graph.png
+/docs/assets/Test_Nonpharyngeal_Electrical_hiveplot.json
+/docs/assets/Test_Nonpharyngeal_Electrical_hiveplot.png

cect/Analysis.py

@@ -0,0 +1,143 @@
+import sys
+import numpy as np
+
+from cect.Cells import get_contralateral_neuron
+
+
+def test_bilaterals():
+    from cect.White_whole import get_instance
+
+    """
+    from cect.Cook2019MaleReader import get_instance
+    from cect.Cook2019HermReader import get_instance
+    from cect.WitvlietDataReader1 import get_instance
+    from cect.WitvlietDataReader8 import get_instance
+    from cect.TestDataReader import get_instance
+    from cect.SpreadsheetDataReader import get_instance
+    from cect.Cook2020DataReader import get_instance
+    from cect.BrittinDataReader import get_instance
+    from cect.WormNeuroAtlasMAReader import get_instance
+    from cect.WormNeuroAtlasFuncReader import get_instance
+    from cect.RipollSanchezDataReader import get_instance
+    from cect.ConnectomeView import PHARYNX_VIEW as view
+    from cect.ConnectomeView import SOCIAL_VIEW as view
+    from cect.ConnectomeView import NEURONS_VIEW as view
+    from cect.ConnectomeView import RAW_VIEW as view"""
+    from cect.ConnectomeView import NONPHARYNGEAL_NEURONS_VIEW as view
+
+    print(
+        "NOTE: For the sake of this paper, we excluded the pharyngeal neurons from the connectome data for both genders due to their distinction from the somatic nervous system."
+    )
+
+    cds = get_instance()
+
+    cds2 = cds.get_connectome_view(view)
+
+    synclass = "Chemical Inh"
+    synclass = "Chemical Exc" if "Raw" not in view.name else "Chemical"
+
+    # synclass = "Acetylcholine"
+    # synclass = "Chemical"
+    # synclass = "Electrical"
+
+    print(cds2.summary())
+
+    print("Keys: %s, plotting: %s" % (view.synclass_sets.keys(), synclass))
+
+    def array_info(conn_array):
+        nonzero = np.count_nonzero(conn_array)
+        print(
+            "- Connection - shape: %s, %i non-zero entries, %i total\n%s\n"
+            % (
+                conn_array.shape,
+                nonzero,
+                np.sum(conn_array),
+                conn_array,
+            )
+        )
+
+    dim = cds2.connections[synclass].shape[0]
+
+    new_conn_array = np.zeros([dim, dim], dtype=np.float64)
+    array_info(new_conn_array)
+
+    for synclass in [
+        "Chemical",
+        "Chemical Exc",
+        "Chemical Inh",
+        "Electrical",
+        "Contact",
+        "Functional",
+        "Extrasynaptic",
+    ]:
+        print("   Adding conns of type: %s" % synclass)
+        if synclass in cds2.connections:
+            conns_cs = cds2.connections[synclass]
+
+            array_info(conns_cs)
+            for i in range(len(conns_cs)):
+                for j in range(len(conns_cs[i])):
+                    if (
+                        i != j
+                    ):  #  Kim et al 2024: Self-connections are treated as nonexistent (∀𝐴𝑖𝑖 = 0).
+                        new_conn_array[i, j] = (
+                            new_conn_array[i, j] or conns_cs[i, j] > 0
+                        )
+
+    amal = "CS+GJ"
+    print("Amalgamated array:")
+    array_info(new_conn_array)
+
+    # fig = cds2.to_plotly_hive_plot_fig(list(view.synclass_sets.keys())[0], view)
+    # fig = cds2.to_plotly_graph_fig(synclass, view)
+    # fig = cds2.to_plotly_matrix_fig(list(view.synclass_sets.keys())[0], view)
+
+    conn_count = 0
+    symm_conn_count = 0
+
+    POS_ZERO_NEG_COLORMAP2 = ["red", "pink", "white", "lightblue", "blue"]
+    POS_ZERO_COLORMAP2 = ["white", "lightblue", "blue"]
+
+    colormap = POS_ZERO_COLORMAP2
+
+    scaled_conn_array = np.array(new_conn_array)
+
+    for i in range(len(new_conn_array)):
+        pre = cds2.nodes[i]
+        pre_ = get_contralateral_neuron(pre)
+        for j in range(len(new_conn_array[i])):
+            post = cds2.nodes[j]
+            post_ = get_contralateral_neuron(post)
+            w = new_conn_array[i][j]
+            if w != 0:
+                print(f"Connection {conn_count}:\t{pre}->{post} ({w})")
+                assert pre is not post
+                w_ = new_conn_array[cds2.nodes.index(pre_), cds2.nodes.index(post_)]
+                print(f"   - Mirror:\t{pre_}->{post_} ({w_})\n")
+                symm_conn_count += w_
+                if w_ == 0:
+                    scaled_conn_array[i][j] = 0.5
+                    scaled_conn_array[
+                        cds2.nodes.index(pre_), cds2.nodes.index(post_)
+                    ] = -1
+                    colormap = POS_ZERO_NEG_COLORMAP2
+                conn_count += 1
+
+    print(
+        f"Of {(len(new_conn_array)**2)} possible edges, {conn_count} are connected, {symm_conn_count} are mirrored {'%.2f'%(100*symm_conn_count/conn_count)}% "
+    )
+
+    cds2.connections[amal] = scaled_conn_array
+
+    # import plotly.io as pio
+    # pio.renderers.default = "browser"
+    if "-nogui" not in sys.argv:
+        # cds2.connection_number_plot
+
+        fig = cds2.to_plotly_matrix_fig(amal, view, colormap, bold_bilaterals=True)
+
+        fig.show()
+
+
+if __name__ == "__main__":
+    test_bilaterals()

cect/BrittinDataReader.py

@@ -12,6 +12,8 @@
 
 from cect.ConnectomeDataset import ConnectomeDataset
 from cect.ConnectomeDataset import get_dataset_source_on_github
+from cect.Cells import is_one_of_bilateral_pair
+from cect.Cells import get_contralateral_neuron
 
 import os
 from openpyxl import load_workbook
@@ -40,7 +42,7 @@ def __init__(self, reference_graph):
 
         cells, neuron_conns = self.read_data()
         for conn in neuron_conns:
-            self.add_connection_info(conn)
+            self.add_connection_info(conn, check_overwritten_connections=True)
 
     def read_data(self):
         cells = []
@@ -50,11 +52,14 @@ def read_data(self):
 
         sheet = wb.get_sheet_by_name(self.reference_graph)
 
+        bilaterals = []
+        single_cells = []
+
         print_("Opened sheet %s in Excel file: %s" % (sheet, filename))
-        print(dir(sheet))
+        # print(dir(sheet))
 
         for row in sheet.rows:
-            print(row[0].value)
+            # print(row[0].value)
             if "cell_1" not in row[0].value:
                 delta = int(row[3].value)
                 if delta == 4:
@@ -64,15 +69,40 @@ def read_data(self):
                     syntype = "Contact"
                     synclass = "%s%s" % (self.reference_graph, row[3].value)
                     synclass = "Contact"
+
                     ci = ConnectionInfo(pre, post, num, syntype, synclass)
-                    print("Adding  %s" % ci)
+                    conns.append(ci)
+                    ci = ConnectionInfo(post, pre, num, syntype, synclass)
                     conns.append(ci)
 
                     if pre not in cells:
                         cells.append(pre)
                     if post not in cells:
                         cells.append(post)
 
+                    pre_ = get_contralateral_neuron(pre)
+                    post_ = get_contralateral_neuron(post)
+                    ci_ = ConnectionInfo(pre_, post_, num, syntype, synclass)
+                    conns.append(ci_)
+                    ci_ = ConnectionInfo(post_, pre_, num, syntype, synclass)
+                    conns.append(ci_)
+
+                    if pre_ not in cells:
+                        cells.append(pre_)
+                    if post_ not in cells:
+                        cells.append(post_)
+
+        for cell in cells:
+            if is_one_of_bilateral_pair(cell):
+                bilaterals.append(cell)
+            else:
+                single_cells.append(cell)
+
+        print_(
+            "Finished processing %i cells, %i bilateral and %i single cells: %s"
+            % (len(cells), len(bilaterals), len(single_cells), sorted(single_cells))
+        )
+
         return cells, conns
 
 

cect/CellInfo.py

@@ -247,6 +247,7 @@ def generate_cell_info_pages(connectomes):
         )
 
         reference_cs = "Cook2019Male" if is_male_specific_cell(cell) else "Cook2019Herm"
+        # reference_cs = "White_whole"
 
         reference_gj = reference_cs
         reference_mono = "Bentley2016_MA"
@@ -262,8 +263,7 @@ def generate_cell_info_pages(connectomes):
         conns_to_pep = "???"
         conns_from_func = "???"
         conns_to_func = "???"
-        conns_from_cont = "???"
-        conns_to_cont = "???"
+        conns_cont = "???"
         conns_gj = "???"
 
         tables_md = ""
@@ -303,7 +303,7 @@ def generate_cell_info_pages(connectomes):
                     if cds_name == reference_func:
                         conns_to_func = _get_top_list(conns, max_conn_cells)
                     if cds_name == reference_cont:
-                        conns_to_cont = _get_top_list(conns, max_conn_cells)
+                        conns_cont = _get_top_list(conns, max_conn_cells)
 
                     for c in conns:
                         cc = get_cell_internal_link(
@@ -352,7 +352,7 @@ def generate_cell_info_pages(connectomes):
                         if cds_name == reference_func:
                             conns_from_func = _get_top_list(conns, max_conn_cells)
                         if cds_name == reference_cont:
-                            conns_from_cont = _get_top_list(conns, max_conn_cells)
+                            pass  # same as to...
 
                         for c in conns:
                             cc = get_cell_internal_link(
@@ -382,24 +382,26 @@ def generate_cell_info_pages(connectomes):
 
 <table style="width:700px">
 <tr>
-    <td><b><a href="#chemical-synaptic-connections-to-{cell.lower()}">Chemical</a></b></td>
+    <td><b><a href="#electrical-synaptic-connections-fromto-{cell.lower()}" title="Electrical connectivity from {reference_cs}">Electrical</a></b></td> <td colspan="5" align="middle">{conns_gj}</td> 
+</tr><tr>
+    <td>&nbsp;</td> <td colspan="5" align="middle">\u2195</td> 
+</tr><tr>
+    <td><b><a href="#chemical-synaptic-connections-to-{cell.lower()}" title="Chemical synaptic connectivity from {reference_cs}">Chemical</a></b></td>
     <td style="width:40%">{conns_to_cs}</td>
     <td style="width:5%" style="vertical-align:bottom;text-align:center;">\u2198</td>
     <td rowspan="5" style="vertical-align:middle;text-align:center;"><b>{cell_link}</b></td>
     <td style="width:5%" style="vertical-align:bottom;text-align:center;">\u2197</td>
     <td style="width:40%">{conns_from_cs}</td>
 </tr><tr>
-    <td><b><a href="#monoaminergic-connections-to-{cell.lower()}">Monoaminergic</a></b></td><td>{conns_to_mono}</td><td align="middle">→</td><td align="middle">→</td><td>{conns_from_mono}</td>
-</tr><tr>
-    <td><b><a href="#peptidergic-connections-to-{cell.lower()}">Peptidergic</a></b></td>  <td>{conns_to_pep}</td><td align="middle">→</td><td align="middle">→</td><td>{conns_from_pep}</td>
+    <td><b><a href="#monoaminergic-connections-to-{cell.lower()}" title="Monoaminergic connectivity from {reference_mono}">Monoaminergic</a></b></td><td>{conns_to_mono}</td><td align="middle">→</td><td align="middle">→</td><td>{conns_from_mono}</td>
 </tr><tr>
-    <td><b><a href="#functional-connections-to-{cell.lower()}">Functional</a></b></td>   <td>{conns_to_func}</td><td align="middle">→</td><td align="middle">→</td><td>{conns_from_func}</td>
+    <td><b><a href="#peptidergic-connections-to-{cell.lower()}" title="Peptidergic connectivity from {reference_pep}">Peptidergic</a></b></td>  <td>{conns_to_pep}</td><td align="middle">→</td><td align="middle">→</td><td>{conns_from_pep}</td>
 </tr><tr>
-    <td><b><a href="#membrane-contacts-to-{cell.lower()}">Contactome</a></b></td>   <td>{conns_to_cont}</td><td align="middle">\u2197</td><td align="middle">\u2198</td><td>{conns_from_cont}</td>
+    <td><b><a href="#functional-connections-to-{cell.lower()}" title="Functional connectivity from {reference_func}">Functional</a></b></td>   <td>{conns_to_func}</td><td align="middle">→</td><td align="middle">→</td><td>{conns_from_func}</td>
 </tr><tr>
     <td>&nbsp;</td> <td colspan="5" align="middle">\u2195</td> 
 </tr><tr>
-    <td><b><a href="#electrical-synaptic-connections-fromto-{cell.lower()}">Electrical</a></b></td> <td colspan="5" align="middle">{conns_gj}</td> 
+    <td><b><a href="#membrane-contacts-to-{cell.lower()}" title="Contactome from {reference_cont}">Contactomic</a></b></td>  <td colspan="5" align="middle">{conns_cont}</td>
 </tr>
 </table>