Lots more info on cell types

cect/Comparison.py

@@ -2,16 +2,17 @@
 
 from cect.ConnectomeReader import analyse_connections
 from cect.ConnectomeReader import check_neurons
-from cect.Cells import get_cell_link
+from cect.Cells import get_cell_internal_link
 from cect import print_
 import json
 
 import sys
 import numpy as np
 
+import importlib
+
 all_data = {}
 
-quick = len(sys.argv) > 1 and eval(sys.argv[1])
 
 reader_pages = {
     "TestData": "Test_data",
@@ -39,24 +40,6 @@
 ]
 
 
-readers = {
-    "SSData": ["cect.SpreadsheetDataReader", None],
-    "UpdSSData": ["cect.UpdatedSpreadsheetDataReader", None],
-    "UpdSSData2": ["cect.UpdatedSpreadsheetDataReader2", None],
-    "Varshney": ["cect.VarshneyDataReader", "Varshney_2011"],
-    "White_L4": ["cect.White_L4", "White_1986"],
-    "White_whole": ["cect.White_whole", "White_1986"],
-    "TestData": ["cect.TestDataReader", None],
-    "Cook2020": ["cect.Cook2020DataReader", "Cook_2020"],
-}
-if not quick:
-    readers["White_A"] = ["cect.White_A", "White_1986"]
-    readers["Witvliet1"] = ["cect.WitvlietDataReader1", "Witvliet_2021"]
-    readers["Witvliet2"] = ["cect.WitvlietDataReader2", "Witvliet_2021"]
-    readers["WormNeuroAtlas"] = ["cect.WormNeuroAtlasReader", "Randi_2023"]
-    readers["Cook2019Herm"] = ["cect.Cook2019HermReader", "Cook_2019"]
-
-
 def shorten_neurotransmitter(nt):
     return (
         nt.replace("Acetylcholine", "ACh")
@@ -72,7 +55,7 @@ def _format_json(json_str):
     return json.dumps(json.loads(json_str), sort_keys=True, indent=2)
 
 
-def get_2d_graph_markdown(reader_name, view_name, connectome, synclass, indent="    "):
+def get_2d_graph_markdown(reader_name, view, connectome, synclass, indent="    "):
     view_name = view.name
 
     fig = connectome.to_plotly_graph_fig(synclass, view)
@@ -130,168 +113,206 @@ def get_matrix_markdown(reader_name, view_name, connectome, synclass, indent="
     )
 
 
-for reader_name, reader_info in readers.items():
-    reader = reader_info[0]
-    decription_page = reader_info[1] if len(reader_info) > 1 else None
+def generate_comparison_page(quick: bool):
+    connectomes = {}
 
-    print_("\n****** Importing dataset %s using %s ******" % (reader_name, reader))
+    readers = {
+        "SSData": ["cect.SpreadsheetDataReader", None],
+        "UpdSSData": ["cect.UpdatedSpreadsheetDataReader", None],
+        "UpdSSData2": ["cect.UpdatedSpreadsheetDataReader2", None],
+        "Varshney": ["cect.VarshneyDataReader", "Varshney_2011"],
+        "White_whole": ["cect.White_whole", "White_1986"],
+        "TestData": ["cect.TestDataReader", None],
+        "Cook2020": ["cect.Cook2020DataReader", "Cook_2020"],
+    }
+    if not quick:
+        readers["White_A"] = ["cect.White_A", "White_1986"]
+        readers["White_L4"] = ["cect.White_L4", "White_1986"]
+        readers["Witvliet1"] = ["cect.WitvlietDataReader1", "Witvliet_2021"]
+        readers["Witvliet2"] = ["cect.WitvlietDataReader2", "Witvliet_2021"]
+        readers["WormNeuroAtlas"] = ["cect.WormNeuroAtlasReader", "Randi_2023"]
+        readers["Cook2019Herm"] = ["cect.Cook2019HermReader", "Cook_2019"]
 
-    exec("from %s import read_data, read_muscle_data, READER_DESCRIPTION" % reader)
-    cells, neuron_conns = read_data(include_nonconnected_cells=True)
+    for reader_name, reader_info in readers.items():
+        reader = reader_info[0]
+        decription_page = reader_info[1] if len(reader_info) > 1 else None
 
-    preferred, not_in_preferred, missing_preferred = check_neurons(cells)
+        print_("\n****** Importing dataset %s using %s ******" % (reader_name, reader))
 
-    neuron_nts = {}
+        reader_module = importlib.import_module(reader)
 
-    connectome = None
-    try:
-        exec("from %s import get_instance" % reader)
-        connectome = get_instance()
-        print_("Adding full connectome info")
-    except:
-        print_("NOT adding full connectome info")
+        # exec("from %s import read_data, read_muscle_data, READER_DESCRIPTION" % reader)
+        cells, neuron_conns = reader_module.read_data(include_nonconnected_cells=True)
+
+        preferred, not_in_preferred, missing_preferred = check_neurons(cells)
+
+        neuron_nts = {}
         connectome = None
 
-    for c in neuron_conns:
-        nt = c.synclass
-        if len(nt) == 0:
-            nt = "**MISSING**"
+        try:
+            connectome = reader_module.get_instance()
+            print_("Adding full connectome info: %s" % connectome)
 
-        if not nt in neuron_nts:
-            neuron_nts[nt] = 0
+        except:
+            print_("NOT adding full connectome info")
+            connectome = None
 
-        neuron_nts[nt] += 1
+        for c in neuron_conns:
+            nt = c.synclass
+            if len(nt) == 0:
+                nt = "**MISSING**"
 
-    nts_info = ""
-    for nt in sorted(neuron_nts.keys()):
-        nts_info += "%s (%i)<br/>" % (shorten_neurotransmitter(nt), neuron_nts[nt])
+            if not nt in neuron_nts:
+                neuron_nts[nt] = 0
 
-    neurons2muscles, muscles, muscle_conns = read_muscle_data()
+            neuron_nts[nt] += 1
 
-    muscle_nts = {}
-    for c in muscle_conns:
-        nt = c.synclass
-        if len(nt) == 0:
-            nt = "**MISSING**"
+        nts_info = ""
+        for nt in sorted(neuron_nts.keys()):
+            nts_info += "%s (%i)<br/>" % (shorten_neurotransmitter(nt), neuron_nts[nt])
 
-        if not nt in muscle_nts:
-            muscle_nts[nt] = 0
+        neurons2muscles, muscles, muscle_conns = reader_module.read_muscle_data()
 
-        muscle_nts[nt] += 1
+        muscle_nts = {}
+        for c in muscle_conns:
+            nt = c.synclass
+            if len(nt) == 0:
+                nt = "**MISSING**"
 
-    m_nts_info = ""
-    for nt in sorted(muscle_nts):
-        m_nts_info += "%s (%i)<br/>" % (shorten_neurotransmitter(nt), muscle_nts[nt])
+            if not nt in muscle_nts:
+                muscle_nts[nt] = 0
 
-    ref = (
-        "[%s](%s.md)" % (reader_name, reader_pages[reader_name])
-        if reader_name in reader_pages
-        else reader_name
-    )
+            muscle_nts[nt] += 1
 
-    if reader_name in reader_pages:
-        matrix_filename = "docs/%s.md" % reader_pages[reader_name]
-        graph_filename = "docs/%s_graph.md" % reader_pages[reader_name]
+        m_nts_info = ""
+        for nt in sorted(muscle_nts):
+            m_nts_info += "%s (%i)<br/>" % (
+                shorten_neurotransmitter(nt),
+                muscle_nts[nt],
+            )
 
-        for filename in [matrix_filename, graph_filename]:
-            with open(filename, "w") as f:
-                matrix = filename == matrix_filename
+        ref = (
+            "[%s](%s.md)" % (reader_name, reader_pages[reader_name])
+            if reader_name in reader_pages
+            else reader_name
+        )
 
-                f.write("## %s\n" % reader_name)
-                if decription_page is not None:
-                    f.write(
-                        "[Source publication of dataset](%s.md)\n\n" % decription_page
-                    )
+        if reader_name in reader_pages:
+            connectomes[reader_name] = connectome
+
+            matrix_filename = "docs/%s.md" % reader_pages[reader_name]
+            graph_filename = "docs/%s_graph.md" % reader_pages[reader_name]
+
+            for filename in [graph_filename, matrix_filename]:
+                with open(filename, "w") as f:
+                    matrix = filename == matrix_filename
 
-                f.write("%s\n\n" % READER_DESCRIPTION)
+                    f.write("## %s\n" % reader_name)
+                    if decription_page is not None:
+                        f.write(
+                            "[Source publication of dataset](%s.md)\n\n"
+                            % decription_page
+                        )
 
-                f.write(
-                    "[View as matrix](%s.md){ .md-button } [View as graph](%s_graph.md){ .md-button }\n\n"
-                    % (reader_pages[reader_name], reader_pages[reader_name])
-                )
+                    f.write("%s\n\n" % reader_module.READER_DESCRIPTION)
+
+                    f.write(
+                        "[View as graph](%s_graph.md){ .md-button } [View as matrix](%s.md){ .md-button }\n\n"
+                        % (reader_pages[reader_name], reader_pages[reader_name])
+                    )
 
-                if connectome is not None:
-                    from ConnectomeView import ALL_VIEWS
+                    if connectome is not None:
+                        from cect.ConnectomeView import ALL_VIEWS
 
-                    indent = "    "
+                        indent = "    "
 
-                    for view in ALL_VIEWS:
-                        cv = connectome.get_connectome_view(view)
+                        for view in ALL_VIEWS:
+                            cv = connectome.get_connectome_view(view)
 
-                        f.write('=== "%s"\n' % view.name)
+                            f.write('=== "%s"\n' % view.name)
 
-                        for sc in view.synclass_sets:
-                            f.write(indent + '=== "%s"\n' % sc)
+                            for sc in view.synclass_sets:
+                                f.write(indent + '=== "%s"\n' % sc)
 
-                            if matrix:
-                                f.write(
-                                    get_matrix_markdown(
-                                        reader_name,
-                                        view.name,
-                                        cv,
-                                        sc,
-                                        indent=indent + indent,
+                                if matrix:
+                                    f.write(
+                                        get_matrix_markdown(
+                                            reader_name,
+                                            view.name,
+                                            cv,
+                                            sc,
+                                            indent=indent + indent,
+                                        )
                                     )
-                                )
-
-                            else:
-                                f.write(
-                                    get_2d_graph_markdown(
-                                        reader_name,
-                                        view,
-                                        cv,
-                                        sc,
-                                        indent=indent + indent,
+
+                                else:
+                                    f.write(
+                                        get_2d_graph_markdown(
+                                            reader_name,
+                                            view,
+                                            cv,
+                                            sc,
+                                            indent=indent + indent,
+                                        )
                                     )
-                                )
-
-                cell_types = {
-                    "Neurons": preferred,
-                    "Missing neurons": missing_preferred,
-                    "Muscles": muscles,
-                    "Other cells": not_in_preferred,
-                }
-
-                for t in cell_types:
-                    f.write("\n### %s (%i)\n" % (t, len(cell_types[t])))
-                    if len(cell_types[t]) > 0:
-                        f.write("<details><summary>Full list of %s</summary>\n" % t)
-                        ss = sorted(cell_types[t])
-                        for n in ss:
-                            f.write("%s" % (get_cell_link(n, True)))
-                            if n is not ss[-1]:
-                                f.write(" | ")
-
-                        f.write("\n</details>\n")
-
-            print_("Written page: %s" % filename)
-
-    all_data[ref] = [
-        len(preferred),
-        len(missing_preferred),
-        len(not_in_preferred),
-        len(muscles),
-        len(neuron_conns),
-        len(neurons2muscles),
-        len(muscle_conns),
-        nts_info,
-        m_nts_info,
-    ]
-
-print_("\nFinished loading all the data from the readers!")
-
-import pandas as pd
-import numpy as np
 
-df_all = pd.DataFrame(all_data).transpose()
-# df_all.set_index("Values")
+                    cell_types = {
+                        "Neurons": preferred,
+                        "Missing neurons": missing_preferred,
+                        "Muscles": muscles,
+                        "Other cells": not_in_preferred,
+                    }
+
+                    for t in cell_types:
+                        f.write("\n### %s (%i)\n" % (t, len(cell_types[t])))
+                        if len(cell_types[t]) > 0:
+                            f.write("<details><summary>Full list of %s</summary>\n" % t)
+                            ss = sorted(cell_types[t])
+                            for n in ss:
+                                f.write("%s" % (get_cell_internal_link(n, True)))
+                                if n is not ss[-1]:
+                                    f.write(" | ")
+
+                            f.write("\n</details>\n")
+
+                print_("Written page: %s" % filename)
+
+        all_data[ref] = [
+            len(preferred),
+            len(missing_preferred),
+            len(not_in_preferred),
+            len(muscles),
+            len(neuron_conns),
+            len(neurons2muscles),
+            len(muscle_conns),
+            nts_info,
+            m_nts_info,
+        ]
+
+    print_("\nFinished loading all the data from the readers!")
+
+    import pandas as pd
+    import numpy as np
+
+    df_all = pd.DataFrame(all_data).transpose()
+    # df_all.set_index("Values")
+
+    # h = HTML(df_all.to_html(escape=False, index=False))
+
+    mk = df_all.to_markdown()
+
+    filename = "docs/Comparison.md"
+    with open(filename, "w") as f:
+        f.write(mk)
+
+    print_("Written page: %s" % filename)
+
+    return connectomes
 
-# h = HTML(df_all.to_html(escape=False, index=False))
 
-mk = df_all.to_markdown()
+if __name__ == "__main__":
+    quick = len(sys.argv) > 1 and eval(sys.argv[1])
 
-filename = "docs/Comparison.md"
-with open(filename, "w") as f:
-    f.write(mk)
+    connectomes = generate_comparison_page(quick)
 
-print_("Written page: %s" % filename)
+    print("Finished. All loaded connectomes:\n%s" % connectomes)