Pandas better integration

src/Global_Var.py

@@ -48,6 +48,7 @@ class Global_Var():
         self.Max = 0
 
         self.LayersNeuronsInfo = []
+        self.NeuronsSize = None
         self.Layer_Neuron = None
         self.NeuronsNbr = 0
         self.LayersNbr = 0
@@ -175,7 +176,7 @@ class Global_Var():
 
         return x, y
 
-    def createHeatMap(self, x, y, data):
+    def createHeatMap(self, x, y, data, depth):
         """ Create a heatmap from a set of synapses values .
 
         Args:
@@ -185,10 +186,21 @@ class Global_Var():
         Returns:
             Array: Heatmap vector
         """
+        #data = data[data.C == 0]
         heatmap = np.zeros((x, y))
-        heatmap[:] = None
+        heatmap[:] = -1
+        data = data.to_numpy()
+
+        if(depth == 0): # single depth
+            for d in data:
+                heatmap[d[2][0]][d[2][1]] = d[2][2]
+        else: # multiple dimensions
             for d in data:
-            heatmap[d[0]][d[1]] = d[2]
+                if(heatmap[d[2][0]][d[2][1]] == -1):
+                    heatmap[d[2][0]][d[2][1]] = d[2][2]
+                else:
+                    if(heatmap[d[2][0]][d[2][1]] < d[2][2]):
+                        heatmap[d[2][0]][d[2][1]] = -d[2][2]
 
         return heatmap
 

src/Modules/General/spark.py

 """ Spark pre-processing operations.
 """
 
+from operator import contains
 from bson.json_util import dumps
 from bson.json_util import loads
 import traceback
@@ -39,11 +40,14 @@ class spark(sparkOp):
             for l in LN:
                 if(l == "Input"):
                     self.g.Input = LN[l]
-                else:
+                elif("pool" not in l):
                     self.g.LayersNeuronsInfo.append(
                         {"layer": l, "neuronNbr": int(LN[l])})
                     self.g.NeuronsNbr += int(LN[l])
                     self.g.LayersNbr += 1
+            
+            self.g.NeuronsSize = pymongo.collection.Collection(self.g.db, 'synapseWeight').find_one(sort=[("index.x", -1),("index.y", -1)])["index"]
+
             # get date & time of the simulation
             self.g.Date = data["T"]
             # calculate simulation time

src/Modules/Synapse/callbacks.py

@@ -354,21 +354,14 @@ class callbacks(callbacksOp):
                 html component that contains the global heatmaps
             """
 
-            heatMapX = list(data["index"].max())[0] + 1
-            heatMapY = list(data["index"].max())[1] + 1
+            heatMapX = max(list(data["x"])) + 1
+            heatMapY = max(list(data["y"])) + 1
+            depth = max(list(data["C"]))
             
-            dfTo = [i["To"] for i in data["_id"]]
-            dfC = [i["C"] for i in data["_id"]]
-
-            data["To"] = dfTo
-            data["C"] = dfC
-
-            data = data.drop("_id", 1)
             data = data.sort_values(["To", "C"])
-            data = data[["V", "index", "To"]]
 
-            data["data"] = [[i["x"], i["y"], v] for i, v in zip(data["index"], data["V"])]
-            data = data[["To", "data"]]
+            data["data"] = [[x, y, v] for x, y, v in zip(data["x"], data["y"], data["V"])]
+            data = data[["To", "C", "data"]]
 
             for i in g.LayersNeuronsInfo:
                 if(i["layer"] == selectedLayer):
@@ -385,10 +378,10 @@ class callbacks(callbacksOp):
                     if(index < neuronsNbr):
                         fig.add_trace(
                             go.Heatmap(
-                                zmin=0,
+                                zmin=-1,
                                 zmax=1,
                                 z=g.createHeatMap(
-                                    heatMapX, heatMapY, data[data.To == index]["data"].to_numpy()),
+                                    heatMapX, heatMapY, data[data.To == index].sort_values(by=['C']),depth),
                                 colorscale='jet',
                                 name=str(index)),
                             row=xx, col=yy)
@@ -583,10 +576,6 @@ class callbacks(callbacksOp):
                 heatmap content (data and layout)
             """
             try:
-                #if(data != []):
-                #    print("*")
-                #    heatMapX = list(data["index"].max())[0] + 1
-                #    heatMapY = list(data["index"].max())[1] + 1
 
                 heatMapWithIndexs = []
                 layout = go.Layout(
@@ -600,11 +589,13 @@ class callbacks(callbacksOp):
                 if isOn:
                     if data != None:
                         data.sort(key=lambda d: d["C"])
+                        depth = data[-1]["C"]
                         group = groupby(data, key=lambda d: d["C"])
 
-                        # add neuron idexes to show heatmap correctly
+                        # add neuron indexes to show heatmap correctly
                         data = [item for item in (
                             max(v, key=lambda k: k["V"]) for k, v in group)]
+
                         # update heatmap in heatmaps
                         if str(index) not in heatmaps:
                             heatmaps[str(index)] = np.zeros(
@@ -621,7 +612,7 @@ class callbacks(callbacksOp):
 
                         return {'data': [
                             go.Heatmap(
-                                z=g.createHeatMap(heatMapX, heatMapY, heatMapWithIndexs),
+                                z=g.createHeatMap(heatMapX, heatMapY, pd.DataFrame(heatMapWithIndexs), depth),
                                 zmin=0,
                                 zmax=1,
                                 colorscale="jet")],
@@ -679,7 +670,6 @@ class callbacks(callbacksOp):
 
             # ToJson----------------------
             SynapseWeight = loads(dumps(SynapseWeight))
-
             # ----------------------------
 
             if not SynapseWeight:
@@ -702,7 +692,7 @@ class callbacks(callbacksOp):
                 .option("pipeline","[{$match: { L: {$eq: '"+layer+"'}}}]")
 
             df = df.load()
-            
+            df = df.select("_id.To","_id.C","L","T","V","_id.index.x","_id.index.y")
             # ToJson----------------------
             globalSynapseWeights = df.toPandas()
             # ----------------------------

src/Modules/Synapse/layout.py

@@ -67,23 +67,23 @@ class layout(layoutOp):
                         "fontWeight": "500", "marginLeft": "20px", "height": "36px",
                         "backgroundColor":"rgb(68, 71, 99)","borderColor":"rgb(68, 71, 99)"}),
                         html.Div([
-                        dbc.Input(type="number",id="HeatMapX",value=28, min=0, step=1,
+                        dbc.Input(type="number",id="HeatMapX",value=self.g.NeuronsSize["x"]+1, min=0, step=1,
                         style={"width":"80px","marginLeft":"10px","textAlign":"center"}),
                         html.P("X",style={"padding":"5px"}),
-                        dbc.Input(type="number",id="HeatMapY",value=28, min=0, step=1,
+                        dbc.Input(type="number",id="HeatMapY",value=self.g.NeuronsSize["y"]+1, min=0, step=1,
                         style={"width":"80px","textAlign":"center"})], className="d-flex",style={"marginLeft":"3px"})          
                         ,html.Div(id='clear-Synapse',children="False", style={'display': 'none'})
                         ,html.Div(id='display-Synapse',children="False", style={'display': 'none'})
                     ], className="d-flex"),
                         dbc.Card( dbc.CardBody([dbc.Card([dbc.CardHeader(
                         dbc.Button( "Layer HeatMap", color="none",
-                        id=f"group-GlobalHeatMapAreaSynapse-toggle",
+                        id="group-GlobalHeatMapAreaSynapse-toggle",
                         style={"width":"100%","height":"100%","borderTop":"2px solid rgb(68, 71, 99)","padding":"10px"}
                         ),style={"padding":"0px",}),
                         dbc.Collapse( dbc.CardBody([
                         html.Div(id={'type': "GlobalHeatMapAreaSynapse"}, children=[], 
                         style={"textAlign": "-webkit-center"})]),
-                        id=f"collapse-GlobalHeatMapAreaSynapse")])])),
+                        id="collapse-GlobalHeatMapAreaSynapse")])])),
                         html.Div(id={'type': "GraphsAreaSynapse"}, children=[], 
                         style={"textAlign": "-webkit-center", "paddingTop": "10px"})]),
                     ], style={"textAlign": "center", "padding": "10px"}

src/Modules/Synapse/spark.py

@@ -35,7 +35,7 @@ class spark(sparkOp):
                 # --------------------------------------------------
                 df = self.g.sparkSession.read.format("com.mongodb.spark.sql") \
                     .option("spark.mongodb.input.uri", self.MONGODBURL + self.g.name + "."+self.DOCUMENT_NAME+"?authSource=admin&readPreference=primaryPreferred") \
-                    .option("pipeline", "[{ $sort: { T: 1 } },{$group : { _id : {To:'$To', C:'$C'}, L : { $last: '$L'}, T : { $last: '$T'},V : { $last: '$V'},index : { $last: '$index'} } }]")
+                    .option("pipeline", "[{ $sort: { T: 1 } },{$group : { _id : {To:'$To', C:'$C', index:'$index' }, L : { $last: '$L'}, T : { $last: '$T'},V : { $last: '$V'} } }]")
 
                 df = df.load()