Skip to content
Snippets Groups Projects
Commit cdbfb58c authored by Hammouda Elbez's avatar Hammouda Elbez :computer:
Browse files

Merge branch 'v0.351' into 'master' 

V0.351

See merge request !22
parents 34edcd63 59cf456a
Branches
No related tags found
1 merge request!22V0.351
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 294 additions and 250 deletions
assets/app.css
+ 4
− 0
View file @ cdbfb58c
......@@ -78,3 +78,7 @@ button:focus{
color: rgb(68, 71, 99);
background-color: rgba(68, 71, 99, 0.116);
}
.Select-control{
width: 100% !important;
}
\ No newline at end of file
src/Global_Var.py
+ 18
− 20
View file @ cdbfb58c
......@@ -37,25 +37,27 @@ class Global_Var():
Input = 0
Date = ""
Accuracy = "0"
Labels = None
finalLabels = None
labelsExistance = False
oldIdSpike = None
def __init__(self):
""" Initialize all the variables used in analysis
"""
stepMax = 0
Max = 0
LayersNeuronsInfo = []
Layer_Neuron = None
NeuronsNbr = 0
LayersNbr = 0
Dataset = ""
Input = 0
Date = ""
Accuracy = "0"
Labels = None
oldIdSpike = None
self.stepMax = 0
self.Max = 0
self.LayersNeuronsInfo = []
self.Layer_Neuron = None
self.NeuronsNbr = 0
self.LayersNbr = 0
self.Dataset = ""
self.Input = 0
self.Date = ""
self.Accuracy = "0"
self.finalLabels = None
self.labelsExistance = False
self.oldIdSpike = None
# MongoDB connection ---------------------------------------------
......@@ -139,11 +141,6 @@ class Global_Var():
col.create_index([("To", 1)])
print("Synapses index done")
# Check if indexing took a lot of time, we reboot MongoDB (to free memory)
#if (int(round(time.time() * 1000))-millis) > 10000:
# print("Restarting Mongo")
# os.system('sudo service mongod restart')
def getLabelTime(self, step, value):
""" Return a string that represent time .
......@@ -192,6 +189,7 @@ class Global_Var():
heatmap[:] = None
for d in data:
heatmap[d[0]][d[1]] = d[2]
return heatmap
def createVerticalHeatMap(self, data):
......@@ -256,7 +254,7 @@ class Global_Var():
conf.setMaster('local[*]')
conf.setAppName(self.name)
conf.set("spark.executor.instances", "4")
conf.set("spark.executor.instances", "8")
conf.set("spark.executor.memory", "8g")
conf.set("spark.sql.execution.arrow.pyspark.enabled", "true")
......
src/Modules/General/callbacks.py
+ 189
− 170
View file @ cdbfb58c
This diff is collapsed.
src/Modules/General/layout.py
+ 26
− 15
View file @ cdbfb58c
......@@ -17,16 +17,17 @@ class layout():
# InfoGraph Axis -------------------------------------------------
oldSliderValue = 0
generalGraphFilterOld = []
generalLayerFilterOld = []
xAxisLabel = deque(maxlen=100)
InfoGraphX = deque(maxlen=100)
SpikeGraphY = deque(maxlen=100)
SynapseGraphY = deque(maxlen=100)
PotentialGraphY = deque(maxlen=100)
SpikeGraphY = dict()
SynapseGraphY = dict()
PotentialGraphY = dict()
LossGraphY = deque(maxlen=100)
MaxSpike = 0
MaxPotential = 0
MaxSynapse = 0
MaxSpike = dict()
MaxPotential = dict()
MaxSynapse = dict()
# LabelPie Data --------------------------------------------------
Label = [[], []]
Max = 0
......@@ -67,11 +68,12 @@ class layout():
self.SpikeGraphY.clear()
self.LossGraphY.clear()
self.SynapseGraphY.clear()
self.PotentialGraphY.clear()
self.xAxisLabel.clear()
self.Label.clear()
self.MaxPotential = 0
self.MaxSpike = 0
self.MaxSynapse = 0
self.MaxPotential = dict()
self.MaxSpike = dict()
self.MaxSynapse = dict()
self.Max = 0
self.Nodes = []
self.Edges = []
......@@ -128,16 +130,25 @@ class layout():
size=30,
color="#28a745",
style={"marginLeft": "10px"}
), dcc.Dropdown(
),
# Graphs filter
dcc.Dropdown(
id='GeneralGraphFilter',
options=[{'label': "Spikes", 'value': "Spikes"}, {'label': "Synapses update", 'value': "Synapses"}, {
'label': "Neurons potential update", 'value': "Potentials"}],
options=[{'label': "Spikes", 'value': "Spikes"}, {'label': "Synapses activity", 'value': "Synapses"}, {
'label': "Neurons potential", 'value': "Potentials"}],
value=['Spikes'],
multi=True,
style={'width': '80%', "marginLeft": "10px", "textAlign": "start"})], className="row", style={"paddingLeft": "20px"})
style={"minWidth":"20%","marginLeft": "10px", "textAlign": "start"}),
# Layers filter
dcc.Dropdown(
id='GeneralLayerFilter',
options=[{'label': str(i), 'value': str(i)} for i in (i for i in g.Layer_Neuron if i != "Input")],
value=[str(i) for i in (i for i in g.Layer_Neuron if i != "Input")],
multi=True,
style={"minWidth":"20%","marginLeft": "15px", "textAlign": "start"})], className="row", style={"paddingLeft": "20px",'width': '100%'})
], className="col-12")
], className="row"),
html.Div([dcc.Graph(id='general-graph', animate=False, config={"displaylogo": False})])], className="col-lg-9 col-sm-12 col-xs-12" if(g.Labels != None) else "col-lg-12 col-sm-12 col-xs-12"),
html.Div([dcc.Graph(id='general-graph', animate=False, config={"displaylogo": False})])], className="col-lg-9 col-sm-12 col-xs-12" if(g.labelsExistance) else "col-lg-12 col-sm-12 col-xs-12"),
html.Div([
html.Div([
html.P("Inputs", style={
......@@ -149,7 +160,7 @@ class layout():
color="#28a745",
style={"marginLeft": "10px"}
)], className="row"),
dcc.Graph(id='label-graph', animate=False, config={"displaylogo": False})], className="col-lg-3 col-sm-12 col-xs-12") if(g.Labels != None) else []], className="row")
dcc.Graph(id='label-graph', animate=False, config={"displaylogo": False})], className="col-lg-3 col-sm-12 col-xs-12") if(g.labelsExistance) else []], className="row")
], style={"padding": "5px"})), label="General information", value="General information"),
dcc.Tab(dbc.Card(
dbc.CardBody([
......
src/Modules/General/spark.py
+ 3
− 2
View file @ cdbfb58c
......@@ -63,6 +63,7 @@ def preProcessing(g):
M = max(M, pymongo.collection.Collection(
g.db, 'synapseWeight').find_one(sort=[("T", -1)])["T"])
if ('labels' in g.db.list_collection_names()):
g.labelsExistance = True
M = max(M, pymongo.collection.Collection(
g.db, 'labels').find_one(sort=[("T", -1)])["T"])
else:
......@@ -77,12 +78,12 @@ def preProcessing(g):
try:
data = next(col, None)
g.Accuracy = data["MaxS"]
g.Labels = data["NLabel"]
g.finalLabels = data["NLabel"]
except Exception as e:
g.Accuracy = "--"
print("No label or accuracy recorded")
g.Labels = None
g.finalLabels = None
pass
# ------------------------------------------------------------
......
src/Modules/Neuron/callbacks.py
+ 6
− 6
View file @ cdbfb58c
......@@ -209,7 +209,7 @@ class callbacks():
"["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]")
output = [self.spikeCountDrawGraph(selectedItem["index"], data, super.xAxisSpikeNbrGraph[selectedItem["index"]], super.xAxisSpikeNbrLabel[selectedItem["index"]], super.yAxisSpikeNbrGraph, isOnSpike),
self.classDrawGraph(selectedItem["index"], data, isOnAcc, g.Labels)]
self.classDrawGraph(selectedItem["index"], data, isOnAcc, g.finalLabels)]
return output
else:
......@@ -225,7 +225,7 @@ class callbacks():
"["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]")
output = [self.spikeCountDrawGraph(selectedItem["index"], data, super.xAxisSpikeNbrGraph[selectedItem["index"]], super.xAxisSpikeNbrLabel[selectedItem["index"]], super.yAxisSpikeNbrGraph, isOnSpike),
self.classDrawGraph(selectedItem["index"], data, isOnAcc, g.Labels)]
self.classDrawGraph(selectedItem["index"], data, isOnAcc, g.finalLabels)]
return output
else:
......@@ -244,7 +244,7 @@ class callbacks():
super.xAxisSpikeNbrLabel[selectedItem["index"]].append("["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]")
output = [self.spikeCountDrawGraph(selectedItem["index"], data, super.xAxisSpikeNbrGraph[selectedItem["index"]], super.xAxisSpikeNbrLabel[selectedItem["index"]], super.yAxisSpikeNbrGraph, isOnSpike),
self.classDrawGraph(selectedItem["index"], data, isOnAcc, g.Labels)]
self.classDrawGraph(selectedItem["index"], data, isOnAcc, g.finalLabels)]
return output
else:
......@@ -363,7 +363,7 @@ class callbacks():
html.Div("−", className="btn-danger btn-circle", style={"fontWeight": "500","fontSize":"16px"}, id={"index": str(index), "type": "DeleteComponent"})], className="row", style={"height": "35px"}),
dcc.Graph(id={"index": str(index), "type": "FreqGraph", "layer": layer, "neuron": neuron}, style={
"width": "100%", "height": "290px"}, className="col-12", animate=False, config={"displaylogo": False}),
], className="col-lg-4 col-sm-12 col-xs-12" if(g.Labels != None) else "col-lg-6 col-sm-6 col-xs-6"),
], className="col-lg-4 col-sm-12 col-xs-12" if(g.finalLabels != None) else "col-lg-6 col-sm-6 col-xs-6"),
html.Div([
html.Div([html.P("Potential", style={"textAlign": "start", "marginLeft": "20px", "marginTop": "6px", "fontSize": "13px"}),
daq.PowerButton(
......@@ -376,7 +376,7 @@ class callbacks():
)], className="row", style={"height": "35px"}),
dcc.Graph(id={"index": str(index), "type": "PotentialGraph", "layer": layer, "neuron": neuron}, style={
"width": "100%", "height": "290px"}, className="col-12", animate=False, config={"displaylogo": False}),
], className="col-lg-4 col-sm-6 col-xs-6" if(g.Labels != None) else "col-lg-6 col-sm-6 col-xs-6"),
], className="col-lg-4 col-sm-6 col-xs-6" if(g.finalLabels != None) else "col-lg-6 col-sm-6 col-xs-6"),
html.Div([
html.Div([html.P("Class", style={"textAlign": "start", "marginLeft": "20px", "marginTop": "6px", "fontSize": "13px"}),
daq.PowerButton(
......@@ -389,7 +389,7 @@ class callbacks():
html.Div([html.P(layer+" Neuron "+neuron, style={"fontSize": "13px", "paddingTop": "6px", "marginRight": "10px"})],
className="row", style={"direction": "rtl", "width": "70%"})], className="row", style={"height": "35px"}),
dcc.Graph(id={"index": str(index), "type": "AccuracyGraph", "layer": layer, "neuron": neuron}, style={
"width": "100%", "height": "290px"}, className="col-12", animate=False, config={"displaylogo": False})], className="col-lg-4 col-sm-6 col-xs-6" if(g.Labels != None) else "", style={} if(g.Labels != None) else {'display': 'none'})],className="row")], style={"background": "rgb(242, 248, 255)", "paddingBottom": "10px", "marginBottom": "10px", "margin": "5px", "borderRadius": "10px"}, id="VisComponent"+str(index))
"width": "100%", "height": "290px"}, className="col-12", animate=False, config={"displaylogo": False})], className="col-lg-4 col-sm-6 col-xs-6" if(g.finalLabels != None) else "", style={} if(g.finalLabels != None) else {'display': 'none'})],className="row")], style={"background": "rgb(242, 248, 255)", "paddingBottom": "10px", "marginBottom": "10px", "margin": "5px", "borderRadius": "10px"}, id="VisComponent"+str(index))
def addSpaceHolder(self,globalVar,app):
""" Adds a space holder area when no graphs are selected.
......
src/Modules/Neuron/layout.py
+ 5
− 5
View file @ cdbfb58c
......@@ -82,7 +82,7 @@ class layout():
"fontWeight": "500", "marginLeft": "20px", "height": "36px", "backgroundColor": "rgb(68, 71, 99)", "borderColor": "rgb(68, 71, 99)"}), html.Div(id='clear-Neuron', children="False", style={'display': 'none'}), html.Div(id='display-Neuron', children="False", style={'display': 'none'})
], className="row"),
html.Div(id={'type': "GraphsAreaNeuron"}, children=[html.Div(id={'type': "OutputNeurons"}, children=[dcc.Graph(id="SpikePerNeuronFreq", figure=self.SpikePerNeuron3D(g), config={"displaylogo": False}, className="col-6"),
dcc.Graph(id="SpikePerNeuronNbr", animate=False, config={"displaylogo": False}, className="col-6")], className="row")], style={"textAlign": "-webkit-center", "paddingTop": "10px"}) if(g.Labels != None) else html.Div(id={'type': "GraphsAreaNeuron"}, children=[], style={"textAlign": "-webkit-center", "paddingTop": "10px"})
dcc.Graph(id="SpikePerNeuronNbr", animate=False, config={"displaylogo": False}, className="col-6")], className="row")], style={"textAlign": "-webkit-center", "paddingTop": "10px"}) if(g.finalLabels != None) else html.Div(id={'type': "GraphsAreaNeuron"}, children=[], style={"textAlign": "-webkit-center", "paddingTop": "10px"})
])
], style={"textAlign": "center", "padding": "10px"}
))
......@@ -107,7 +107,7 @@ class layout():
Returns:
the 3D graph
"""
if(g.Labels == None):
if(g.finalLabels == None):
return {'data': [],
'layout': {'margin': {'l': 0, 'r': 0, 't': 30, 'b': 0},
'scene': {
......@@ -126,7 +126,7 @@ class layout():
xx = [N["i"]["N"] for N in data]
yy = [(count["count"]/total) for count in data]
zz = [int(c["Label"]) for i, c in product(data, g.Labels) if (
zz = [int(c["Label"]) for i, c in product(data, g.finalLabels) if (
int(c["N"]) == int(i["i"]["N"]) and c["L"] == i["i"]["L"])]
labels = list(dict.fromkeys(zz))
......@@ -179,13 +179,13 @@ class layout():
Returns:
graph of filtered class neurons activity
"""
if(filteredClass == None or g.Labels == None):
if(filteredClass == None or g.finalLabels == None):
return {'data': [],
'layout': {'margin': {'l': 0, 'r': 0, 't': 30, 'b': 0}}}
else:
data = self.SpikePerNeuron
data = [d for d, l in product(data, g.Labels) if (
data = [d for d, l in product(data, g.finalLabels) if (
int(l["N"]) == d['i']['N'] and int(l["Label"]) == filteredClass["z"])]
xx = [N["i"]["N"] for N in data]
yy = [(count["count"]) for count in data]
......
src/Modules/Synapse/callbacks.py
+ 35
− 30
View file @ cdbfb58c
......@@ -3,6 +3,7 @@
Dash callbacks are the responsible on updating graphs each step.
"""
import time
from dash.dependencies import Input, Output, State, MATCH, ALL
import dash
import pymongo
......@@ -86,7 +87,7 @@ class callbacks():
data = self.getGlobalSynapseWeights(g,selectedLayer)
super.globalHeatMap[selectedLayer] = data
return [graphsArea, self.AddGlobalHeatmap(g, selectedLayer, data, HeatMapX, HeatMapY) if data != [] else []]
return [graphsArea, self.AddGlobalHeatmap(g, selectedLayer, data, HeatMapX, HeatMapY) if not data.empty else []]
else:
if(context[0]['prop_id'] != "."):
# Delete item
......@@ -347,26 +348,29 @@ class callbacks():
Args:
g (Global_Var): reference to access global variables
selectedLayer (String): layer name
data (list): synapses weights of all neurons in the layer
data (dataframe): synapses weights of all neurons in the layer
heatMapX (int): X dimension for heatmaps
heatMapY (int): Y dimension for heatmaps
Returns:
html component that contains the global heatmaps
"""
df = pd.DataFrame(data)
dfTo = [i["To"] for i in df["_id"]]
dfC = [i["C"] for i in df["_id"]]
df["To"] = dfTo
df["C"] = dfC
heatMapX = list(data["index"].max())[0] + 1
heatMapY = list(data["index"].max())[1] + 1
df = df.drop("_id", 1)
df = df.sort_values(["To", "C"])
df = df[["V", "index", "To"]]
dfTo = [i["To"] for i in data["_id"]]
dfC = [i["C"] for i in data["_id"]]
df["data"] = [[i["x"], i["y"], v] for i, v in zip(df["index"], df["V"])]
df = df[["To", "data"]]
data["To"] = dfTo
data["C"] = dfC
data = data.drop("_id", 1)
data = data.sort_values(["To", "C"])
data = data[["V", "index", "To"]]
data["data"] = [[i["x"], i["y"], v] for i, v in zip(data["index"], data["V"])]
data = data[["To", "data"]]
for i in g.LayersNeuronsInfo:
if(i["layer"] == selectedLayer):
......@@ -386,17 +390,18 @@ class callbacks():
zmin=0,
zmax=1,
z=g.createHeatMap(
heatMapX, heatMapY, df[df.To == index]["data"].to_numpy()),
heatMapX, heatMapY, data[data.To == index]["data"].to_numpy()),
colorscale='jet',
name=str(index)),
row=xx, col=yy)
fig.update_yaxes(autorange="reversed", row=xx, col=yy)
fig.update_layout(height=80, width=80)
index += 1
fig.update_layout(height=800, width=800, title_text=selectedLayer)
fig.update_layout(height=1200, width=1200, title_text=selectedLayer)
fig.update_traces(showscale=False)
return html.Div([dcc.Graph(id="GlobalHeatmap", figure=fig, config={"displaylogo": False})])
return html.Div([dcc.Graph(id="GlobalHeatmap", figure=fig, config={"displaylogo": False}, style={'overflowY': 'scroll', 'height': 500})])
def synapseFreqDrawGraph(self, g, index, data, xAxis, xAxisLabel, yAxisList, HeatMapSynapseFreqGraph, isOn):
""" Create scatter plot with vertical heatmap for selected neuron synapses.
......@@ -574,6 +579,11 @@ class callbacks():
heatmap content (data and layout)
"""
try:
#if(data != []):
# print("*")
# heatMapX = list(data["index"].max())[0] + 1
# heatMapY = list(data["index"].max())[1] + 1
heatMapWithIndexs = []
layout = go.Layout(
margin={'l': 0, 'r': 0, 't': 0, 'b': 25},
......@@ -654,12 +664,12 @@ class callbacks():
# MongoDB---------------------
col = pymongo.collection.Collection(g.db, 'synapseWeight')
SynapseWeight = col.aggregate([
{"$match": {"$and": [
{"T": {'$gt': timestamp, '$lte': (timestamp+g.updateInterval)}},
{"L": {'$in': layer}},
{"To": {'$in': neuron}}
]}
{"To": {'$in': neuron}},
{"T": {'$gt': timestamp, '$lte': (timestamp+g.updateInterval)}} ]}
}])
# ToJson----------------------
......@@ -682,20 +692,15 @@ class callbacks():
final synapses weights
"""
# MongoDB---------------------
col = pymongo.collection.Collection(g.db, 'synapseWeightFinal')
df = g.sparkSession.read.format("com.mongodb.spark.sql") \
.option("spark.mongodb.input.uri", g.MONGODBURL + g.name + ".synapseWeightFinal?authSource=admin&readPreference=primaryPreferred") \
.option("pipeline","[{$match: { L: {$eq: '"+layer+"'}}}]")
globalSynapseWeights = col.aggregate([{"$match": {"_id.L": {'$in': [layer]}}},
{"$group": {
"_id": {"To": "$To", "C": "$C"},
"T": {"$last": "$T"},
"index": {"$last": "$index"},
"V": {"$last": "$V"}}
}], allowDiskUse=True)
df = df.load()
# ToJson----------------------
globalSynapseWeights = loads(dumps(globalSynapseWeights))
globalSynapseWeights = df.toPandas()
# ----------------------------
return globalSynapseWeights
except Exception as e:
......
src/Modules/Synapse/layout.py
+ 1
− 1
View file @ cdbfb58c
......@@ -83,7 +83,7 @@ class layout():
),style={"padding":"0px",}),
dbc.Collapse( dbc.CardBody([
html.Div(id={'type': "GlobalHeatMapAreaSynapse"}, children=[],
style={"textAlign": "-webkit-center", "paddingTop": "10px"})]),
style={"textAlign": "-webkit-center"})]),
id=f"collapse-GlobalHeatMapAreaSynapse")])])),
html.Div(id={'type': "GraphsAreaSynapse"}, children=[],
style={"textAlign": "-webkit-center", "paddingTop": "10px"})]),
......
src/Modules/Synapse/spark.py
+ 7
− 1
View file @ cdbfb58c
......@@ -37,7 +37,7 @@ def preProcessing(g):
# --------------------------------------------------
df = g.sparkSession.read.format("com.mongodb.spark.sql") \
.option("spark.mongodb.input.uri", MONGODB_URL + g.name + "."+DOCUMENT_NAME+"?authSource=admin&readPreference=primaryPreferred") \
.option("pipeline", "[{ $sort: { T: 1 } },{$group : { _id : {To:'$To', C:'$C', L:'$L'}, T : { $last: '$T'},V : { $last: '$V'},index : { $last: '$index'} } }]")
.option("pipeline", "[{ $sort: { T: 1 } },{$group : { _id : {To:'$To', C:'$C'}, L : { $last: '$L'}, T : { $last: '$T'},V : { $last: '$V'},index : { $last: '$index'} } }]")
df = df.load()
......@@ -53,6 +53,12 @@ def preProcessing(g):
col = pymongo.collection.Collection(g.db, OUTPUT_DOCUMENT_NAME_FINAL)
col.create_index([("_id.L", 1)])
col.create_index([("_id", 1)])
col.create_index([("_id.To", 1),("_id.C", 1)])
col.create_index([("T", 1)])
col.create_index([("index", 1)])
col.create_index([("V", 1)])
# --------------------------------------------------------
else:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment