Skip to content
Snippets Groups Projects
Commit 73c861cc authored by Elbez Hammouda's avatar Elbez Hammouda
Browse files

Structure updated

parent de6a7f82
Branches
No related tags found
1 merge request!23V0.351
Showing
with 2260 additions and 858 deletions
File added
This diff is collapsed.
...@@ -31,7 +31,7 @@ mongo_exists = False ...@@ -31,7 +31,7 @@ mongo_exists = False
app_vis = None app_vis = None
users = dict() users = dict()
g = Global_Var() g = Global_Var()
__name__ = "__main__"
url = g.config.VS2N_HOST url = g.config.VS2N_HOST
port = g.config.VS2N_PORT port = g.config.VS2N_PORT
...@@ -245,7 +245,7 @@ def Module(n, g): ...@@ -245,7 +245,7 @@ def Module(n, g):
n (String): module name n (String): module name
g (Global_Var): reference to access global variables g (Global_Var): reference to access global variables
""" """
importlib.import_module("src.Modules." + n + ".spark").process(g) importlib.import_module("src.Modules." + n + ".spark").init(g)
def LoadingModules(): def LoadingModules():
...@@ -280,13 +280,13 @@ def processing(): ...@@ -280,13 +280,13 @@ def processing():
continue continue
# Loading Modules to Dashboard # Loading Modules to Dashboard
layout().Vis(app_vis, g) layout().load(app_vis, g)
return "done" return "done"
# Create dash app instance ------------------------------------------- # Create dash app instance -------------------------------------------
app_vis = dash.Dash( app_vis = dash.Dash(
'__name__', __name__,
requests_pathname_prefix='/vis/', requests_pathname_prefix='/vis/',
external_stylesheets=[dbc.themes.BOOTSTRAP], external_stylesheets=[dbc.themes.BOOTSTRAP],
suppress_callback_exceptions=True, suppress_callback_exceptions=True,
......
...@@ -12,13 +12,13 @@ import plotly.graph_objects as go ...@@ -12,13 +12,13 @@ import plotly.graph_objects as go
from dash.exceptions import PreventUpdate from dash.exceptions import PreventUpdate
from plotly.subplots import make_subplots from plotly.subplots import make_subplots
from dash.dependencies import Input, Output, State, MATCH from dash.dependencies import Input, Output, State, MATCH
from src.templates.callbacksOp import callbacksOp
class callbacks(callbacksOp):
class callbacks():
""" Callbacks class """ Callbacks class
""" """
def init(self, super, app, g): def __init__(self, super, app, g):
""" Initialize the callback . """ Initialize the callback .
Args: Args:
......
...@@ -9,9 +9,9 @@ from .callbacks import callbacks ...@@ -9,9 +9,9 @@ from .callbacks import callbacks
import dash_core_components as dcc import dash_core_components as dcc
import dash_html_components as html import dash_html_components as html
import dash_bootstrap_components as dbc import dash_bootstrap_components as dbc
from src.templates.layoutOp import layoutOp
class layout(layoutOp):
class layout():
""" Layout class """ Layout class
""" """
# InfoGraph Axis ------------------------------------------------- # InfoGraph Axis -------------------------------------------------
...@@ -78,7 +78,7 @@ class layout(): ...@@ -78,7 +78,7 @@ class layout():
self.Nodes = [] self.Nodes = []
self.Edges = [] self.Edges = []
def Vis(self, app, g): def Vis(self):
""" Create layer components """ Create layer components
Args: Args:
...@@ -92,16 +92,16 @@ class layout(): ...@@ -92,16 +92,16 @@ class layout():
# Table of network information ------------------------------- # Table of network information -------------------------------
tablecontent = [ tablecontent = [
html.Tr([html.Td("Neurons", style={"width": "25%", "fontWeight": "500"}), html.Td(g.NeuronsNbr, style={"width": "25%"}), html.Td( html.Tr([html.Td("Neurons", style={"width": "25%", "fontWeight": "500"}), html.Td(self.g.NeuronsNbr, style={"width": "25%"}), html.Td(
"Layers", style={"width": "25%", "fontWeight": "500"}), html.Td(g.LayersNbr, style={"width": "25%"})]), "Layers", style={"width": "25%", "fontWeight": "500"}), html.Td(self.g.LayersNbr, style={"width": "25%"})]),
html.Tr([html.Td("Input", style={"width": "25%", "fontWeight": "500"}), html.Td(g.Input, style={"width": "25%"}), html.Td( html.Tr([html.Td("Input", style={"width": "25%", "fontWeight": "500"}), html.Td(self.g.Input, style={"width": "25%"}), html.Td(
"Dataset", style={"width": "25%", "fontWeight": "500"}), html.Td(g.Dataset, style={"width": "25%"})]), "Dataset", style={"width": "25%", "fontWeight": "500"}), html.Td(self.g.Dataset, style={"width": "25%"})]),
html.Tr([html.Td("Simulation Date", style={"width": "25%", "fontWeight": "500"}), html.Td(g.Date, style={"width": "25%"}), html.Td( html.Tr([html.Td("Simulation Date", style={"width": "25%", "fontWeight": "500"}), html.Td(self.g.Date, style={"width": "25%"}), html.Td(
"Accuracy", style={"width": "25%", "fontWeight": "500"}), html.Td(str(g.Accuracy)+" %", style={"width": "25%"})]) "Accuracy", style={"width": "25%", "fontWeight": "500"}), html.Td(str(self.g.Accuracy)+" %", style={"width": "25%"})])
] ]
# Generate 2D View ------------------------------------------- # Generate 2D View -------------------------------------------
self.generate2DView(self.Nodes, self.Edges, g) self.generate2DView(self.Nodes, self.Edges, self.g)
# Tabs content ----------------------------------------------- # Tabs content -----------------------------------------------
info_vis = dbc.Card( info_vis = dbc.Card(
...@@ -143,14 +143,14 @@ class layout(): ...@@ -143,14 +143,14 @@ class layout():
dcc.Dropdown( dcc.Dropdown(
id='GeneralLayerFilter', id='GeneralLayerFilter',
options=[{'label': str(i), 'value': str(i)} for i in ( options=[{'label': str(i), 'value': str(i)} for i in (
i for i in g.Layer_Neuron if i != "Input")], i for i in self.g.Layer_Neuron if i != "Input")],
value=[str(i) for i in ( value=[str(i) for i in (
i for i in g.Layer_Neuron if i != "Input")], i for i in self.g.Layer_Neuron if i != "Input")],
multi=True, multi=True,
style={"minWidth": "20%", "marginLeft": "15px", "textAlign": "start"})], className="row", style={"paddingLeft": "20px", 'width': '100%'}) style={"minWidth": "20%", "marginLeft": "15px", "textAlign": "start"})], className="row", style={"paddingLeft": "20px", 'width': '100%'})
], className="col-12") ], className="col-12")
], className="row"), ], className="row"),
html.Div([dcc.Graph(id='general-graph', animate=False, config={"displaylogo": False})])], className="col-lg-9 col-sm-12 col-xs-12" if(g.labelsExistance) else "col-lg-12 col-sm-12 col-xs-12"), html.Div([dcc.Graph(id='general-graph', animate=False, config={"displaylogo": False})])], className="col-lg-9 col-sm-12 col-xs-12" if(self.g.labelsExistance) else "col-lg-12 col-sm-12 col-xs-12"),
html.Div([ html.Div([
html.Div([ html.Div([
html.P("Inputs", style={ html.P("Inputs", style={
...@@ -162,7 +162,7 @@ class layout(): ...@@ -162,7 +162,7 @@ class layout():
color="#28a745", color="#28a745",
style={"marginLeft": "10px"} style={"marginLeft": "10px"}
)], className="row"), )], className="row"),
dcc.Graph(id='label-graph', animate=False, config={"displaylogo": False})], className="col-lg-3 col-sm-12 col-xs-12") if(g.labelsExistance) else []], className="row") dcc.Graph(id='label-graph', animate=False, config={"displaylogo": False})], className="col-lg-3 col-sm-12 col-xs-12") if(self.g.labelsExistance) else []], className="row")
], style={"padding": "5px"})), label="General information", value="General information"), ], style={"padding": "5px"})), label="General information", value="General information"),
dcc.Tab(dbc.Card( dcc.Tab(dbc.Card(
dbc.CardBody([ dbc.CardBody([
...@@ -203,15 +203,15 @@ class layout(): ...@@ -203,15 +203,15 @@ class layout():
tabs = [dcc.Tab(info_vis, label="General")] tabs = [dcc.Tab(info_vis, label="General")]
# Add active modules # Add active modules
for m in [i for i in g.modules if i != "General"]: for m in [i for i in self.g.modules if i != "General"]:
try: try:
tabs.append(dcc.Tab(importlib.import_module( tabs.append(dcc.Tab(importlib.import_module(
".Modules."+m+".layout", package="src").layout().Vis(app, g), label=m)) ".Modules."+m+".layout", package="src").layout().load(self.app, self.g), label=m))
except Exception as e: except Exception as e:
print("Tabs appending:"+str(e)) print("Tabs appending:"+str(e))
# App layout # App layout
app.layout = html.Div([ self.app.layout = html.Div([
html.Nav(className="navbar sticky-top navbar-dark", children=[ html.Nav(className="navbar sticky-top navbar-dark", children=[
dbc.Row([ dbc.Row([
html.Div([html.A("VS2N", className="navbar-brand", href="#")], className="col-12", style={ html.Div([html.A("VS2N", className="navbar-brand", href="#")], className="col-12", style={
...@@ -245,7 +245,7 @@ class layout(): ...@@ -245,7 +245,7 @@ class layout():
html.Span("Update Interval (s)", html.Span("Update Interval (s)",
className="input-group-text") className="input-group-text")
], className="input-group-prepend"), ], className="input-group-prepend"),
dbc.Input(type="number", id="interval", value=g.updateInterval, min=0.005, dbc.Input(type="number", id="interval", value=self.g.updateInterval, min=0.005,
max=180, step=0.005, style={"width": "30%", "textAlign": "center"}) max=180, step=0.005, style={"width": "30%", "textAlign": "center"})
], className="input-group col-md-4 col-sm-4 col-lg-4", style={"height": "38px", "paddingTop": "12px"}) ], className="input-group col-md-4 col-sm-4 col-lg-4", style={"height": "38px", "paddingTop": "12px"})
], className="d-flex justify-content-center"), dbc.Col( ], className="d-flex justify-content-center"), dbc.Col(
...@@ -254,7 +254,7 @@ class layout(): ...@@ -254,7 +254,7 @@ class layout():
dcc.Slider( dcc.Slider(
id='vis-slider', id='vis-slider',
min=0, min=0,
max=g.stepMax, max=self.g.stepMax,
step=1, step=1,
value=0, value=0,
)], style={"textAlign": "start", "padding": "5px"})], )], style={"textAlign": "start", "padding": "5px"})],
...@@ -265,4 +265,4 @@ class layout(): ...@@ -265,4 +265,4 @@ class layout():
), html.Div(id='v-step', children="0", style={'display': 'none'}), html.Div(id='clear', children="False", style={'display': 'none'})]) ), html.Div(id='v-step', children="0", style={'display': 'none'}), html.Div(id='clear', children="False", style={'display': 'none'})])
# load callbacks # load callbacks
callbacks().init(self, app, g) callbacks(self,self.app, self.g)
...@@ -5,89 +5,85 @@ from bson.json_util import dumps ...@@ -5,89 +5,85 @@ from bson.json_util import dumps
from bson.json_util import loads from bson.json_util import loads
import traceback import traceback
import pymongo import pymongo
from src.templates.sparkOp import sparkOp
def init(g):
spark(g)
def process(g): class spark(sparkOp):
""" Start processing and reduce module count . def __init__(self, g):
module = "General"
Args: document = ""
g (Global_Var): reference to access global variables output_document = ""
""" super().__init__(g, module, document, output_document)
preProcessing(g)
g.modulesNbr = g.modulesNbr - 1
# Spark operations and pre-processing -------------------------------- # Spark operations and pre-processing --------------------------------
def preProcessing(self):
def preProcessing(g):
""" Apache Spark pre-processing. """ Apache Spark pre-processing.
Args:
g (Global_Var): reference to access global variables
""" """
MODULE_NAME = "General"
try: try:
# initial variables # initial variables
g.LayersNeuronsInfo = [] self.g.LayersNeuronsInfo = []
g.NeuronsNbr = 0 self.g.NeuronsNbr = 0
g.LayersNbr = 0 self.g.LayersNbr = 0
# get info collection # get info collection
col = pymongo.collection.Collection(g.db, 'info').find() col = pymongo.collection.Collection(self.g.db, 'info').find()
data = next(col, None) data = next(col, None)
# get dataset name # get dataset name
g.Dataset = data["D"] self.g.Dataset = data["D"]
# get network layers and neurons # get network layers and neurons
LN = data["L:N"] LN = data["L:N"]
LN = loads(dumps(LN)) LN = loads(dumps(LN))
g.Layer_Neuron = LN self.g.Layer_Neuron = LN
for l in LN: for l in LN:
if(l == "Input"): if(l == "Input"):
g.Input = LN[l] self.g.Input = LN[l]
else: else:
g.LayersNeuronsInfo.append( self.g.LayersNeuronsInfo.append(
{"layer": l, "neuronNbr": int(LN[l])}) {"layer": l, "neuronNbr": int(LN[l])})
g.NeuronsNbr += int(LN[l]) self.g.NeuronsNbr += int(LN[l])
g.LayersNbr += 1 self.g.LayersNbr += 1
# get date & time of the simulation # get date & time of the simulation
g.Date = data["T"] self.g.Date = data["T"]
# calculate simulation time # calculate simulation time
try: try:
M = 0 M = 0
if ('spikes' in g.db.list_collection_names()): if ('spikes' in self.g.db.list_collection_names()):
M = max(M, pymongo.collection.Collection( M = max(M, pymongo.collection.Collection(
g.db, 'spikes').find_one(sort=[("T", -1)])["T"]) self.g.db, 'spikes').find_one(sort=[("T", -1)])["T"])
if ('potential' in g.db.list_collection_names()): if ('potential' in self.g.db.list_collection_names()):
M = max(M, pymongo.collection.Collection( M = max(M, pymongo.collection.Collection(
g.db, 'potential').find_one(sort=[("T", -1)])["T"]) self.g.db, 'potential').find_one(sort=[("T", -1)])["T"])
if ('synapseWeight' in g.db.list_collection_names()): if ('synapseWeight' in self.g.db.list_collection_names()):
M = max(M, pymongo.collection.Collection( M = max(M, pymongo.collection.Collection(
g.db, 'synapseWeight').find_one(sort=[("T", -1)])["T"]) self.g.db, 'synapseWeight').find_one(sort=[("T", -1)])["T"])
if ('labels' in g.db.list_collection_names()): if ('labels' in self.g.db.list_collection_names()):
g.labelsExistance = True self.g.labelsExistance = True
M = max(M, pymongo.collection.Collection( M = max(M, pymongo.collection.Collection(
g.db, 'labels').find_one(sort=[("T", -1)])["T"]) self.g.db, 'labels').find_one(sort=[("T", -1)])["T"])
else: else:
print("No labels") print("No labels")
except Exception as e: except Exception as e:
print("MongoError:" + str(e)) print("MongoError:" + str(e))
pass pass
g.Max = M self.g.Max = M
g.stepMax = int(M/g.updateInterval)+1 self.g.stepMax = int(M/self.g.updateInterval)+1
# get accuracy and neurons labels (if exist) # get accuracy and neurons labels (if exist)
try: try:
data = next(col, None) data = next(col, None)
g.Accuracy = data["MaxS"] self.g.Accuracy = data["MaxS"]
g.finalLabels = data["NLabel"] self.g.finalLabels = data["NLabel"]
except Exception as e: except Exception as e:
g.Accuracy = "--" self.g.Accuracy = "--"
print("No label or accuracy recorded") print("No label or accuracy recorded")
g.finalLabels = None self.g.finalLabels = None
pass pass
# ------------------------------------------------------------ # ------------------------------------------------------------
print("done ", MODULE_NAME) print("done ", self.MODULE_NAME)
except Exception as e: except Exception as e:
print("Error:" + str(e)) print("Error:" + str(e))
......
...@@ -13,12 +13,13 @@ import dash_html_components as html ...@@ -13,12 +13,13 @@ import dash_html_components as html
from bson.json_util import dumps, loads from bson.json_util import dumps, loads
from dash.exceptions import PreventUpdate from dash.exceptions import PreventUpdate
from dash.dependencies import ALL, MATCH, Input, Output, State from dash.dependencies import ALL, MATCH, Input, Output, State
from src.templates.callbacksOp import callbacksOp
class callbacks(): class callbacks(callbacksOp):
"""Callbacks class """Callbacks class
""" """
def init(self, super, app, g): def __init__(self, super, app, g):
"""Initialize the callback . """Initialize the callback .
Args: Args:
......
...@@ -5,9 +5,9 @@ from collections import deque ...@@ -5,9 +5,9 @@ from collections import deque
import dash_html_components as html import dash_html_components as html
import dash_bootstrap_components as dbc import dash_bootstrap_components as dbc
from .callbacks import callbacks from .callbacks import callbacks
from src.templates.layoutOp import layoutOp
class layout(layoutOp):
class layout():
""" Layout class """ Layout class
""" """
# Needed variables for the graphs -------------------------------- # Needed variables for the graphs --------------------------------
...@@ -21,7 +21,7 @@ class layout(): ...@@ -21,7 +21,7 @@ class layout():
""" """
self.xAxisGraphName.clear() self.xAxisGraphName.clear()
def Vis(self, app, g): def Vis(self):
""" Create layer components """ Create layer components
Args: Args:
...@@ -46,7 +46,7 @@ class layout(): ...@@ -46,7 +46,7 @@ class layout():
) )
) )
# load callbacks # load callbacks
callbacks().init(self, app, g) # callbacks(self.app, self.g)
# Return the Layer # Return the Layer
return layer return layer
except Exception as e: except Exception as e:
......
...@@ -2,36 +2,40 @@ ...@@ -2,36 +2,40 @@
""" """
import traceback import traceback
from src.templates.sparkOp import sparkOp
def process(g): def init(g):
""" Start processing and reduce module count . spark(g)
class spark(sparkOp):
def __init__(self, g):
"""
Args: Args:
g (Global_Var): reference to access global variables g (Global_Var): reference to access global variables
""" """
preProcessing(g) module = ""
g.modulesNbr = g.modulesNbr - 1 document = ""
output_document = ""
super().__init__(g, module, document, output_document)
# Spark operations and pre-processing -------------------------------- # Spark operations and pre-processing --------------------------------
def preProcessing(g): def preProcessing(self):
"""Loads the GDD database into BDD . """ Spark preprocessing .
Args:
g (Global_Var): reference to access global variables
""" """
MODULE_NAME = "" MODULE_NAME = ""
DOCUMENT_NAME = "" DOCUMENT_NAME = ""
OUTPUT_DOCUMENT_NAME = "" OUTPUT_DOCUMENT_NAME = ""
MONGODBURL = g.MONGODBURL MONGODBURL = self.g.MONGODBURL
try: try:
if (not (OUTPUT_DOCUMENT_NAME in g.db.list_collection_names())) and (DOCUMENT_NAME in g.db.list_collection_names()): if (not (OUTPUT_DOCUMENT_NAME in self.g.db.list_collection_names())) and (DOCUMENT_NAME in self.g.db.list_collection_names()):
# Spark setup -------------------------------------------- # Spark setup --------------------------------------------
if g.sparkSession == None: if self.g.sparkSession == None:
g.createSparkSession() self.g.createSparkSession()
# doing the heavy work ----------------------------------- # doing the heavy work -----------------------------------
...@@ -39,15 +43,15 @@ def preProcessing(g): ...@@ -39,15 +43,15 @@ def preProcessing(g):
#df.write.format("com.mongodb.spark.sql.DefaultSource") \ #df.write.format("com.mongodb.spark.sql.DefaultSource") \
# .option("spark.mongodb.output.uri", # .option("spark.mongodb.output.uri",
# MONGODBURL + g.name + "."+OUTPUT_DOCUMENT_NAME+"?authSource=admin&#readPreference=primaryPreferred").mode("overwrite").save() # MONGODBURL + self.g.name + "."+OUTPUT_DOCUMENT_NAME+"?authSource=admin&#readPreference=primaryPreferred").mode("overwrite").save()
#print("Saved to BDD") #print("Saved to BDD")
# ------------------------------------------------------------ # ------------------------------------------------------------
else: else:
if(not DOCUMENT_NAME in g.db.list_collection_names()): if(not DOCUMENT_NAME in self.g.db.list_collection_names()):
print(DOCUMENT_NAME, "not found") print(DOCUMENT_NAME, "not found")
g.modules = [module for module in g.modules if module != MODULE_NAME] self.g.modules = [module for module in self.g.modules if module != MODULE_NAME]
print("done ", MODULE_NAME) print("done ", MODULE_NAME)
except Exception as e: except Exception as e:
......
...@@ -14,12 +14,12 @@ import dash_html_components as html ...@@ -14,12 +14,12 @@ import dash_html_components as html
from bson.json_util import dumps, loads from bson.json_util import dumps, loads
from dash.exceptions import PreventUpdate from dash.exceptions import PreventUpdate
from dash.dependencies import ALL, MATCH, Input, Output, State from dash.dependencies import ALL, MATCH, Input, Output, State
from src.templates.callbacksOp import callbacksOp
class callbacks(): class callbacks(callbacksOp):
""" Callbacks class """ Callbacks class
""" """
def init(self, super, app, g): def __init__(self, super, app, g):
""" Initialize the callback . """ Initialize the callback .
Args: Args:
...@@ -59,10 +59,10 @@ class callbacks(): ...@@ -59,10 +59,10 @@ class callbacks():
if len([item for item in graphsArea if item["props"]["id"] == "spaceHolder"]) == 1: if len([item for item in graphsArea if item["props"]["id"] == "spaceHolder"]) == 1:
updatedBody = [item for item in graphsArea if item["props"]["id"] != "spaceHolder"] updatedBody = [item for item in graphsArea if item["props"]["id"] != "spaceHolder"]
updatedBody.append(self.addNeuronArea(addButtonClick, selectedLayer, selectedNeuron, g)) updatedBody.append(addNeuronArea(addButtonClick, selectedLayer, selectedNeuron, g))
return updatedBody return updatedBody
else: else:
graphsArea.append(self.addNeuronArea(addButtonClick, selectedLayer, selectedNeuron, g)) graphsArea.append(addNeuronArea(addButtonClick, selectedLayer, selectedNeuron, g))
return graphsArea return graphsArea
else: else:
return graphsArea return graphsArea
...@@ -95,12 +95,12 @@ class callbacks(): ...@@ -95,12 +95,12 @@ class callbacks():
["id"] != "VisComponent"+str(itemDeleted)] ["id"] != "VisComponent"+str(itemDeleted)]
if(len(newGraphsArea) == 0): if(len(newGraphsArea) == 0):
newGraphsArea.append(self.addSpaceHolder(g,app)) newGraphsArea.append(addSpaceHolder(app))
return newGraphsArea return newGraphsArea
else: else:
if(len(graphsArea) == 0): if(len(graphsArea) == 0):
graphsArea.append(self.addSpaceHolder(g,app)) graphsArea.append(addSpaceHolder(app))
return graphsArea return graphsArea
except Exception as e: except Exception as e:
print("InsertRemoveNewElement: " + str(e)) print("InsertRemoveNewElement: " + str(e))
...@@ -193,7 +193,7 @@ class callbacks(): ...@@ -193,7 +193,7 @@ class callbacks():
if dash.callback_context.triggered[0]["prop_id"] != '.': if dash.callback_context.triggered[0]["prop_id"] != '.':
# data processing # data processing
if(sliderValue > 0 and sliderValue <= g.stepMax): if(sliderValue > 0 and sliderValue <= g.stepMax):
data = self.getSpikesOfNeuron( data = getSpikesOfNeuron(
int(sliderValue)*float(updateInterval), g, selectedItem) int(sliderValue)*float(updateInterval), g, selectedItem)
if (len(super.xAxisSpikeNbrLabel[selectedItem["index"]]) == 0 or super.xAxisSpikeNbrLabel[selectedItem["index"]][-1] != ("["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]")): if (len(super.xAxisSpikeNbrLabel[selectedItem["index"]]) == 0 or super.xAxisSpikeNbrLabel[selectedItem["index"]][-1] != ("["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]")):
...@@ -208,15 +208,15 @@ class callbacks(): ...@@ -208,15 +208,15 @@ class callbacks():
super.xAxisSpikeNbrLabel[selectedItem["index"]].append( super.xAxisSpikeNbrLabel[selectedItem["index"]].append(
"["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]") "["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]")
output = [self.spikeCountDrawGraph(selectedItem["index"], data, super.xAxisSpikeNbrGraph[selectedItem["index"]], super.xAxisSpikeNbrLabel[selectedItem["index"]], super.yAxisSpikeNbrGraph, isOnSpike), output = [spikeCountDrawGraph(selectedItem["index"], data, super.xAxisSpikeNbrGraph[selectedItem["index"]], super.xAxisSpikeNbrLabel[selectedItem["index"]], super.yAxisSpikeNbrGraph, isOnSpike),
self.classDrawGraph(selectedItem["index"], data, isOnAcc, g.finalLabels)] classDrawGraph(selectedItem["index"], data, isOnAcc, g.finalLabels)]
return output return output
else: else:
raise PreventUpdate raise PreventUpdate
else: else:
if(sliderValue == 0): if(sliderValue == 0):
data = self.getSpikesOfNeuron( data = getSpikesOfNeuron(
int(sliderValue)*float(updateInterval), g, selectedItem) int(sliderValue)*float(updateInterval), g, selectedItem)
super.xAxisSpikeNbrGraph[selectedItem["index"]].append(sliderValue) super.xAxisSpikeNbrGraph[selectedItem["index"]].append(sliderValue)
...@@ -224,8 +224,8 @@ class callbacks(): ...@@ -224,8 +224,8 @@ class callbacks():
super.xAxisSpikeNbrLabel[selectedItem["index"]].append( super.xAxisSpikeNbrLabel[selectedItem["index"]].append(
"["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]") "["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]")
output = [self.spikeCountDrawGraph(selectedItem["index"], data, super.xAxisSpikeNbrGraph[selectedItem["index"]], super.xAxisSpikeNbrLabel[selectedItem["index"]], super.yAxisSpikeNbrGraph, isOnSpike), output = [spikeCountDrawGraph(selectedItem["index"], data, super.xAxisSpikeNbrGraph[selectedItem["index"]], super.xAxisSpikeNbrLabel[selectedItem["index"]], super.yAxisSpikeNbrGraph, isOnSpike),
self.classDrawGraph(selectedItem["index"], data, isOnAcc, g.finalLabels)] classDrawGraph(selectedItem["index"], data, isOnAcc, g.finalLabels)]
return output return output
else: else:
...@@ -233,7 +233,7 @@ class callbacks(): ...@@ -233,7 +233,7 @@ class callbacks():
else: else:
# after adding to the screen # after adding to the screen
if(selectedItem["index"] not in super.xAxisSpikeNbrLabel): if(selectedItem["index"] not in super.xAxisSpikeNbrLabel):
data = self.getSpikesOfNeuron( data = getSpikesOfNeuron(
int(sliderValue)*float(updateInterval), g, selectedItem) int(sliderValue)*float(updateInterval), g, selectedItem)
super.xAxisSpikeNbrGraph[selectedItem["index"]] = deque(maxlen=100) super.xAxisSpikeNbrGraph[selectedItem["index"]] = deque(maxlen=100)
...@@ -243,8 +243,8 @@ class callbacks(): ...@@ -243,8 +243,8 @@ class callbacks():
super.xAxisSpikeNbrLabel[selectedItem["index"]].append("["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]") super.xAxisSpikeNbrLabel[selectedItem["index"]].append("["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]")
output = [self.spikeCountDrawGraph(selectedItem["index"], data, super.xAxisSpikeNbrGraph[selectedItem["index"]], super.xAxisSpikeNbrLabel[selectedItem["index"]], super.yAxisSpikeNbrGraph, isOnSpike), output = [spikeCountDrawGraph(selectedItem["index"], data, super.xAxisSpikeNbrGraph[selectedItem["index"]], super.xAxisSpikeNbrLabel[selectedItem["index"]], super.yAxisSpikeNbrGraph, isOnSpike),
self.classDrawGraph(selectedItem["index"], data, isOnAcc, g.finalLabels)] classDrawGraph(selectedItem["index"], data, isOnAcc, g.finalLabels)]
return output return output
else: else:
...@@ -275,7 +275,7 @@ class callbacks(): ...@@ -275,7 +275,7 @@ class callbacks():
if dash.callback_context.triggered[0]["prop_id"] != '.': if dash.callback_context.triggered[0]["prop_id"] != '.':
# data processing # data processing
if(sliderValue > 0 and sliderValue <= g.stepMax): if(sliderValue > 0 and sliderValue <= g.stepMax):
data = self.getPotentialOfNeuron( data = getPotentialOfNeuron(
int(sliderValue)*float(updateInterval), g, selectedItem) int(sliderValue)*float(updateInterval), g, selectedItem)
if (len(super.xAxisPotentialLabel[selectedItem["index"]]) == 0 or super.xAxisPotentialLabel[selectedItem["index"]][-1] != ("["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]")): if (len(super.xAxisPotentialLabel[selectedItem["index"]]) == 0 or super.xAxisPotentialLabel[selectedItem["index"]][-1] != ("["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]")):
...@@ -290,13 +290,13 @@ class callbacks(): ...@@ -290,13 +290,13 @@ class callbacks():
super.xAxisPotentialLabel[selectedItem["index"]].append( super.xAxisPotentialLabel[selectedItem["index"]].append(
"["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]") "["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]")
output = [self.neuronPotentialDrawGraph(selectedItem["index"], data, super.xAxisPotentialGraph[selectedItem["index"]], super.xAxisPotentialLabel[selectedItem["index"]], super.yAxisPotentialGraph, isOn)] output = [neuronPotentialDrawGraph(selectedItem["index"], data, super.xAxisPotentialGraph[selectedItem["index"]], super.xAxisPotentialLabel[selectedItem["index"]], super.yAxisPotentialGraph, isOn)]
return output return output
else: else:
raise PreventUpdate raise PreventUpdate
else: else:
if(sliderValue == 0): if(sliderValue == 0):
data = self.getPotentialOfNeuron( data = getPotentialOfNeuron(
int(sliderValue)*float(updateInterval), g, selectedItem) int(sliderValue)*float(updateInterval), g, selectedItem)
super.xAxisPotentialGraph[selectedItem["index"]].append(sliderValue) super.xAxisPotentialGraph[selectedItem["index"]].append(sliderValue)
...@@ -304,7 +304,7 @@ class callbacks(): ...@@ -304,7 +304,7 @@ class callbacks():
super.xAxisPotentialLabel[selectedItem["index"]].append( super.xAxisPotentialLabel[selectedItem["index"]].append(
"["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]") "["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]")
output = [self.neuronPotentialDrawGraph(selectedItem["index"], data, super.xAxisPotentialGraph[selectedItem["index"]], super.xAxisPotentialLabel[selectedItem["index"]], super.yAxisPotentialGraph, isOn)] output = [neuronPotentialDrawGraph(selectedItem["index"], data, super.xAxisPotentialGraph[selectedItem["index"]], super.xAxisPotentialLabel[selectedItem["index"]], super.yAxisPotentialGraph, isOn)]
return output return output
else: else:
...@@ -312,7 +312,7 @@ class callbacks(): ...@@ -312,7 +312,7 @@ class callbacks():
else: else:
# after adding to the screen # after adding to the screen
if(selectedItem["index"] not in super.xAxisPotentialLabel): if(selectedItem["index"] not in super.xAxisPotentialLabel):
data = self.getPotentialOfNeuron( data = getPotentialOfNeuron(
int(sliderValue)*float(updateInterval), g, selectedItem) int(sliderValue)*float(updateInterval), g, selectedItem)
super.xAxisPotentialGraph[selectedItem["index"]] = deque(maxlen=100) super.xAxisPotentialGraph[selectedItem["index"]] = deque(maxlen=100)
...@@ -322,7 +322,7 @@ class callbacks(): ...@@ -322,7 +322,7 @@ class callbacks():
super.xAxisPotentialLabel[selectedItem["index"]].append("["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]") super.xAxisPotentialLabel[selectedItem["index"]].append("["+g.getLabelTime(g.updateInterval, sliderValue)+","+g.getLabelTime(g.updateInterval, sliderValue+1)+"]")
output = [self.neuronPotentialDrawGraph(selectedItem["index"], data, super.xAxisPotentialGraph[selectedItem["index"]], super.xAxisPotentialLabel[selectedItem["index"]], super.yAxisPotentialGraph, isOn)] output = [neuronPotentialDrawGraph(selectedItem["index"], data, super.xAxisPotentialGraph[selectedItem["index"]], super.xAxisPotentialLabel[selectedItem["index"]], super.yAxisPotentialGraph, isOn)]
return output return output
else: else:
...@@ -336,7 +336,7 @@ class callbacks(): ...@@ -336,7 +336,7 @@ class callbacks():
# Helper functions # Helper functions
# ------------------------------------------------------------ # ------------------------------------------------------------
def addNeuronArea(self, index, layer, neuron, g): def addNeuronArea(index, layer, neuron, g):
""" Adds a new neuron area that contains graphs. """ Adds a new neuron area that contains graphs.
Args: Args:
...@@ -391,11 +391,10 @@ class callbacks(): ...@@ -391,11 +391,10 @@ class callbacks():
dcc.Graph(id={"index": str(index), "type": "AccuracyGraph", "layer": layer, "neuron": neuron}, style={ dcc.Graph(id={"index": str(index), "type": "AccuracyGraph", "layer": layer, "neuron": neuron}, style={
"width": "100%", "height": "290px"}, className="col-12", animate=False, config={"displaylogo": False})], className="col-lg-4 col-sm-6 col-xs-6" if(g.finalLabels != None) else "", style={} if(g.finalLabels != None) else {'display': 'none'})],className="row")], style={"background": "rgb(242, 248, 255)", "paddingBottom": "10px", "marginBottom": "10px", "margin": "5px", "borderRadius": "10px"}, id="VisComponent"+str(index)) "width": "100%", "height": "290px"}, className="col-12", animate=False, config={"displaylogo": False})], className="col-lg-4 col-sm-6 col-xs-6" if(g.finalLabels != None) else "", style={} if(g.finalLabels != None) else {'display': 'none'})],className="row")], style={"background": "rgb(242, 248, 255)", "paddingBottom": "10px", "marginBottom": "10px", "margin": "5px", "borderRadius": "10px"}, id="VisComponent"+str(index))
def addSpaceHolder(self,globalVar,app): def addSpaceHolder(app):
""" Adds a space holder area when no graphs are selected. """ Adds a space holder area when no graphs are selected.
Args: Args:
globalVar (Global_Var): reference to access global variables
app : Flask app app : Flask app
Returns: Returns:
html component that contains the neuron area graphs html component that contains the neuron area graphs
...@@ -407,7 +406,7 @@ class callbacks(): ...@@ -407,7 +406,7 @@ class callbacks():
id="spaceHolder") id="spaceHolder")
def spikeCountDrawGraph(self, index, data, xAxis, xAxisLabel, yAxisList, isOn): def spikeCountDrawGraph(index, data, xAxis, xAxisLabel, yAxisList, isOn):
""" Create bar chart for selected neuron, that contains """ Create bar chart for selected neuron, that contains
spikes count per interval. spikes count per interval.
...@@ -489,7 +488,7 @@ class callbacks(): ...@@ -489,7 +488,7 @@ class callbacks():
except Exception as e: except Exception as e:
print("spikeCountDrawGraph: "+str(e)) print("spikeCountDrawGraph: "+str(e))
def neuronPotentialDrawGraph(self, index, data, xAxis, xAxisLabel, yAxisList, isOn): def neuronPotentialDrawGraph(index, data, xAxis, xAxisLabel, yAxisList, isOn):
""" Create scatter plot for selected neuron, that contains """ Create scatter plot for selected neuron, that contains
membrane potential per interval. membrane potential per interval.
...@@ -587,7 +586,7 @@ class callbacks(): ...@@ -587,7 +586,7 @@ class callbacks():
except Exception as e: except Exception as e:
print("neuronPotentialDrawGraph: "+str(e)) print("neuronPotentialDrawGraph: "+str(e))
def classDrawGraph(self, index, data, isOn, labels): def classDrawGraph(index, data, isOn, labels):
""" Create scatter polar plot for selected neuron, that represents """ Create scatter polar plot for selected neuron, that represents
class responsible on firing theis neuron. class responsible on firing theis neuron.
...@@ -680,7 +679,7 @@ class callbacks(): ...@@ -680,7 +679,7 @@ class callbacks():
# MongoDB operations # MongoDB operations
# ------------------------------------------------------------ # ------------------------------------------------------------
def getSpikesOfNeuron(self, timestamp, g, neuronInfo): def getSpikesOfNeuron(timestamp, g, neuronInfo):
""" Get spikes per neuron activity in a given interval. """ Get spikes per neuron activity in a given interval.
Args: Args:
...@@ -708,7 +707,7 @@ class callbacks(): ...@@ -708,7 +707,7 @@ class callbacks():
return None return None
return spikeOfNeuron return spikeOfNeuron
def getPotentialOfNeuron(self, timestamp, g, neuronInfo): def getPotentialOfNeuron(timestamp, g, neuronInfo):
""" Get neuron membrane potential activity in a given interval. """ Get neuron membrane potential activity in a given interval.
Args: Args:
......
...@@ -11,9 +11,9 @@ import plotly.graph_objects as go ...@@ -11,9 +11,9 @@ import plotly.graph_objects as go
import dash_core_components as dcc import dash_core_components as dcc
import dash_html_components as html import dash_html_components as html
import dash_bootstrap_components as dbc import dash_bootstrap_components as dbc
from src.templates.layoutOp import layoutOp
class layout(layoutOp):
class layout():
""" Layout class """ Layout class
""" """
# Needed variables for the graphs -------------------------------- # Needed variables for the graphs --------------------------------
...@@ -45,7 +45,7 @@ class layout(): ...@@ -45,7 +45,7 @@ class layout():
self.yAxisSpikeNbrGraph[index] = deque(maxlen=100) self.yAxisSpikeNbrGraph[index] = deque(maxlen=100)
self.yAxisPotentialGraph[index] = deque(maxlen=100) self.yAxisPotentialGraph[index] = deque(maxlen=100)
def Vis(self, app, g): def Vis(self):
""" Create layer components """ Create layer components
Args: Args:
...@@ -58,7 +58,7 @@ class layout(): ...@@ -58,7 +58,7 @@ class layout():
try: try:
self.clearData([]) self.clearData([])
print("neuron-vis") print("neuron-vis")
self.SpikePerNeuron = self.getSpikePerNeuron(g) self.SpikePerNeuron = self.getSpikePerNeuron(self.g)
layer = dbc.Card( layer = dbc.Card(
dbc.CardBody( dbc.CardBody(
[ [
...@@ -68,10 +68,10 @@ class layout(): ...@@ -68,10 +68,10 @@ class layout():
dcc.Dropdown( dcc.Dropdown(
id='LayerFilterNeuron', id='LayerFilterNeuron',
options=[{'label': str(i), 'value': str(i)} for i in ( options=[{'label': str(i), 'value': str(i)} for i in (
i for i in g.Layer_Neuron if i != "Input")], i for i in self.g.Layer_Neuron if i != "Input")],
multi=False, multi=False,
value=[{'label': str(i), 'value': str(i)} for i in ( value=[{'label': str(i), 'value': str(i)} for i in (
i for i in g.Layer_Neuron if i != "Input")][0]["value"], i for i in self.g.Layer_Neuron if i != "Input")][0]["value"],
style={'width': '150px', "marginLeft": "10px", "textAlign": "start"}), style={'width': '150px', "marginLeft": "10px", "textAlign": "start"}),
dcc.Dropdown( dcc.Dropdown(
id='NeuronFilterNeuron', id='NeuronFilterNeuron',
...@@ -81,14 +81,14 @@ class layout(): ...@@ -81,14 +81,14 @@ class layout():
dbc.Button("+", id="AddComponentNeuron", n_clicks=0, style={ dbc.Button("+", id="AddComponentNeuron", n_clicks=0, style={
"fontWeight": "500", "marginLeft": "20px", "height": "36px", "backgroundColor": "rgb(68, 71, 99)", "borderColor": "rgb(68, 71, 99)"}), html.Div(id='clear-Neuron', children="False", style={'display': 'none'}), html.Div(id='display-Neuron', children="False", style={'display': 'none'}) "fontWeight": "500", "marginLeft": "20px", "height": "36px", "backgroundColor": "rgb(68, 71, 99)", "borderColor": "rgb(68, 71, 99)"}), html.Div(id='clear-Neuron', children="False", style={'display': 'none'}), html.Div(id='display-Neuron', children="False", style={'display': 'none'})
], className="row"), ], className="row"),
html.Div(id={'type': "GraphsAreaNeuron"}, children=[html.Div(id={'type': "OutputNeurons"}, children=[dcc.Graph(id="SpikePerNeuronFreq", figure=self.SpikePerNeuron3D(g), config={"displaylogo": False}, className="col-6"), html.Div(id={'type': "GraphsAreaNeuron"}, children=[html.Div(id={'type': "OutputNeurons"}, children=[dcc.Graph(id="SpikePerNeuronFreq", figure=self.SpikePerNeuron3D(self.g), config={"displaylogo": False}, className="col-6"),
dcc.Graph(id="SpikePerNeuronNbr", animate=False, config={"displaylogo": False}, className="col-6")], className="row")], style={"textAlign": "-webkit-center", "paddingTop": "10px"}) if(g.finalLabels != None) else html.Div(id={'type': "GraphsAreaNeuron"}, children=[], style={"textAlign": "-webkit-center", "paddingTop": "10px"}) dcc.Graph(id="SpikePerNeuronNbr", animate=False, config={"displaylogo": False}, className="col-6")], className="row")], style={"textAlign": "-webkit-center", "paddingTop": "10px"}) if(self.g.finalLabels != None) else html.Div(id={'type': "GraphsAreaNeuron"}, children=[], style={"textAlign": "-webkit-center", "paddingTop": "10px"})
]) ])
], style={"textAlign": "center", "padding": "10px"} ], style={"textAlign": "center", "padding": "10px"}
)) ))
# load callbacks # load callbacks
callbacks().init(self, app, g) callbacks(self,self.app, self.g)
# Return the Layer # Return the Layer
return layer return layer
except Exception as e: except Exception as e:
......
...@@ -2,39 +2,38 @@ ...@@ -2,39 +2,38 @@
""" """
import traceback import traceback
from src.templates.sparkOp import sparkOp
def process(g): def init(g):
""" Start processing and reduce module count . spark(g)
class spark(sparkOp):
def __init__(self, g):
"""
Args: Args:
g (Global_Var): reference to access global variables g (Global_Var): reference to access global variables
""" """
preProcessing(g) module = "Neuron"
g.modulesNbr = g.modulesNbr - 1 document = "spikes"
output_document = "SpikePerNeuron"
super().__init__(g, module, document, output_document)
# Spark operations and preprocessing---------------------------------- # Spark operations and preprocessing----------------------------------
def preProcessing(self):
def preProcessing(g):
""" Apache Spark pre-processing. """ Apache Spark pre-processing.
Args:
g (Global_Var): reference to access global variables
""" """
MODULE_NAME = "Neuron"
DOCUMENT_NAME = "spikes"
OUTPUT_DOCUMENT_NAME = "SpikePerNeuron"
MONGODB_URL = g.MONGODBURL
try: try:
if (not (OUTPUT_DOCUMENT_NAME in g.db.list_collection_names())) and (DOCUMENT_NAME in g.db.list_collection_names()): if (not (self.OUTPUT_DOCUMENT_NAME in self.g.db.list_collection_names())) and (self.DOCUMENT_NAME in self.g.db.list_collection_names()):
# Spark setup -------------------------------------------- # Spark setup --------------------------------------------
if g.sparkSession == None: if self.g.sparkSession == None:
g.createSparkSession() self.g.createSparkSession()
# -------------------------------------------------------- # --------------------------------------------------------
# Spike per neuron # Spike per neuron
df = g.sparkSession.read.format("com.mongodb.spark.sql") \ df = self.g.sparkSession.read.format("com.mongodb.spark.sql") \
.option("spark.mongodb.input.uri", MONGODB_URL + g.name + "."+DOCUMENT_NAME+"?authSource=admin&readPreference=primaryPreferred") \ .option("spark.mongodb.input.uri", self.MONGODB_URL + self.g.name + "."+self.DOCUMENT_NAME+"?authSource=admin&readPreference=primaryPreferred") \
.option("pipeline", "[{ $sort : { T : 1} }]") .option("pipeline", "[{ $sort : { T : 1} }]")
df = df.load() df = df.load()
...@@ -49,14 +48,14 @@ def preProcessing(g): ...@@ -49,14 +48,14 @@ def preProcessing(g):
df.write.format("com.mongodb.spark.sql.DefaultSource") \ df.write.format("com.mongodb.spark.sql.DefaultSource") \
.option("spark.mongodb.output.uri", .option("spark.mongodb.output.uri",
MONGODB_URL + g.name + "."+OUTPUT_DOCUMENT_NAME+"?authSource=admin&readPreference=primaryPreferred").mode("overwrite").save() self.MONGODB_URL + self.g.name + "."+self.OUTPUT_DOCUMENT_NAME+"?authSource=admin&readPreference=primaryPreferred").mode("overwrite").save()
# -------------------------------------------------------- # --------------------------------------------------------
else: else:
if(not DOCUMENT_NAME in g.db.list_collection_names()): if(not self.DOCUMENT_NAME in self.g.db.list_collection_names()):
print(DOCUMENT_NAME, "not found") print(self.DOCUMENT_NAME, "not found")
g.modules = [module for module in g.modules if module != MODULE_NAME] self.g.modules = [module for module in self.g.modules if module != self.MODULE_NAME]
print("done ", MODULE_NAME) print("done ", self.MODULE_NAME)
except Exception as e: except Exception as e:
print("Error:" + str(e)) print("Error:" + str(e))
......
...@@ -20,13 +20,13 @@ import statistics ...@@ -20,13 +20,13 @@ import statistics
from itertools import groupby from itertools import groupby
import numpy as np import numpy as np
import pandas as pd import pandas as pd
from src.templates.callbacksOp import callbacksOp
class callbacks(callbacksOp):
class callbacks():
""" Callbacks class """ Callbacks class
""" """
def init(self, super, app, g): def __init__(self, super, app, g):
""" Initialize the callback . """ Initialize the callback .
Args: Args:
......
...@@ -6,9 +6,9 @@ import dash_html_components as html ...@@ -6,9 +6,9 @@ import dash_html_components as html
import dash_bootstrap_components as dbc import dash_bootstrap_components as dbc
from .callbacks import callbacks from .callbacks import callbacks
import dash_core_components as dcc import dash_core_components as dcc
from src.templates.layoutOp import layoutOp
class layout(layoutOp):
class layout():
""" Layout class """ Layout class
""" """
# Needed variables for the graphs -------------------------------- # Needed variables for the graphs --------------------------------
...@@ -38,7 +38,7 @@ class layout(): ...@@ -38,7 +38,7 @@ class layout():
self.globalHeatMap.clear() self.globalHeatMap.clear()
def Vis(self,app,g): def Vis(self):
""" Create layer components """ Create layer components
Args: Args:
...@@ -59,7 +59,7 @@ class layout(): ...@@ -59,7 +59,7 @@ class layout():
dcc.Dropdown( dcc.Dropdown(
id='LayerFilterSynapse', id='LayerFilterSynapse',
options=[{'label': str(i), 'value': str(i)} for i in ( options=[{'label': str(i), 'value': str(i)} for i in (
i for i in g.Layer_Neuron if i != "Input")], multi=False, i for i in self.g.Layer_Neuron if i != "Input")], multi=False,
style={'width': '150px', "marginLeft": "10px", "textAlign": "start"}), style={'width': '150px', "marginLeft": "10px", "textAlign": "start"}),
dcc.Dropdown( dcc.Dropdown(
id='NeuronFilterSynapse', options=[], multi=False, id='NeuronFilterSynapse', options=[], multi=False,
...@@ -91,7 +91,7 @@ class layout(): ...@@ -91,7 +91,7 @@ class layout():
) )
) )
# load callbacks # load callbacks
callbacks().init(self,app,g) callbacks(self,self.app,self.g)
# Return the Layer # Return the Layer
return layer return layer
except Exception as e: except Exception as e:
......
...@@ -4,39 +4,37 @@ ...@@ -4,39 +4,37 @@
import pymongo import pymongo
import traceback import traceback
from pyspark.sql import functions as F from pyspark.sql import functions as F
from src.templates.sparkOp import sparkOp
def init(g):
spark(g)
def process(g): class spark(sparkOp):
""" Start processing and reduce module count .
def __init__(self, g):
"""
Args: Args:
g (Global_Var): reference to access global variables g (Global_Var): reference to access global variables
""" """
preProcessing(g) module = "Synapse"
g.modulesNbr = g.modulesNbr - 1 document = "synapseWeight"
output_document = "synapseWeightFinal"
super().__init__(g, module, document, output_document)
# Spark operations and preprocessing---------------------------------- # Spark operations and preprocessing----------------------------------
def preProcessing(self):
def preProcessing(g):
""" Apache Spark pre-processing. """ Apache Spark pre-processing.
Args:
g (Global_Var): reference to access global variables
""" """
MODULE_NAME = "Synapse"
DOCUMENT_NAME = "synapseWeight"
OUTPUT_DOCUMENT_NAME_FINAL = "synapseWeightFinal"
MONGODB_URL = g.MONGODBURL
try: try:
if (not (OUTPUT_DOCUMENT_NAME_FINAL in g.db.list_collection_names())) and (DOCUMENT_NAME in g.db.list_collection_names()): if (not (self.OUTPUT_DOCUMENT_NAME in self.g.db.list_collection_names())) and (self.DOCUMENT_NAME in self.g.db.list_collection_names()):
# Spark setup--------------------------------------- # Spark setup---------------------------------------
if g.sparkSession == None: if self.g.sparkSession == None:
g.createSparkSession() self.g.createSparkSession()
# -------------------------------------------------- # --------------------------------------------------
df = g.sparkSession.read.format("com.mongodb.spark.sql") \ df = self.g.sparkSession.read.format("com.mongodb.spark.sql") \
.option("spark.mongodb.input.uri", MONGODB_URL + g.name + "."+DOCUMENT_NAME+"?authSource=admin&readPreference=primaryPreferred") \ .option("spark.mongodb.input.uri", self.MONGODBURL + self.g.name + "."+self.DOCUMENT_NAME+"?authSource=admin&readPreference=primaryPreferred") \
.option("pipeline", "[{ $sort: { T: 1 } },{$group : { _id : {To:'$To', C:'$C'}, L : { $last: '$L'}, T : { $last: '$T'},V : { $last: '$V'},index : { $last: '$index'} } }]") .option("pipeline", "[{ $sort: { T: 1 } },{$group : { _id : {To:'$To', C:'$C'}, L : { $last: '$L'}, T : { $last: '$T'},V : { $last: '$V'},index : { $last: '$index'} } }]")
df = df.load() df = df.load()
...@@ -45,13 +43,13 @@ def preProcessing(g): ...@@ -45,13 +43,13 @@ def preProcessing(g):
df.write.format("com.mongodb.spark.sql.DefaultSource") \ df.write.format("com.mongodb.spark.sql.DefaultSource") \
.option("spark.mongodb.output.uri", .option("spark.mongodb.output.uri",
MONGODB_URL + g.name + "."+OUTPUT_DOCUMENT_NAME_FINAL+"?authSource=admin&readPreference=primaryPreferred").mode('append').save() self.MONGODBURL + self.g.name + "."+self.OUTPUT_DOCUMENT_NAME+"?authSource=admin&readPreference=primaryPreferred").mode('append').save()
# Indexes creation --------------------------------------- # Indexes creation ---------------------------------------
print("Indexes creation (please wait...)") print("Indexes creation (please wait...)")
col = pymongo.collection.Collection(g.db, OUTPUT_DOCUMENT_NAME_FINAL) col = pymongo.collection.Collection(self.g.db, self.OUTPUT_DOCUMENT_NAME)
col.create_index([("_id.L", 1)]) col.create_index([("_id.L", 1)])
col.create_index([("_id", 1)]) col.create_index([("_id", 1)])
col.create_index([("_id.To", 1),("_id.C", 1)]) col.create_index([("_id.To", 1),("_id.C", 1)])
...@@ -59,13 +57,12 @@ def preProcessing(g): ...@@ -59,13 +57,12 @@ def preProcessing(g):
col.create_index([("index", 1)]) col.create_index([("index", 1)])
col.create_index([("V", 1)]) col.create_index([("V", 1)])
# -------------------------------------------------------- # --------------------------------------------------------
else: else:
if(not DOCUMENT_NAME in g.db.list_collection_names()): if(not self.DOCUMENT_NAME in self.g.db.list_collection_names()):
print(DOCUMENT_NAME, "not found") print(self.DOCUMENT_NAME, "not found")
g.modules = [module for module in g.modules if module != MODULE_NAME] self.g.modules = [module for module in self.g.modules if module != self.MODULE_NAME]
print("done", MODULE_NAME) print("done", self.MODULE_NAME)
except Exception as e: except Exception as e:
print("Error:" + str(e)) print("Error:" + str(e))
......
\ No newline at end of file
""" This class contains Dash callbacks
Dash callbacks are the responsible on updating graphs each step.
"""
class callbacks():
"""Callbacks class
"""
def init(self, super, app, g):
"""Initialize the callback .
Args:
app : Flask app
g (Global_Var): reference to access global variables
"""
# ------------------------------------------------------------
# Callbacks
# ------------------------------------------------------------
# to prevent creating duplicate callbacks next use
if not g.checkExistance(app, "name of one element from layout"):
# Add callbacks here
# Use Input("v-step", "children"),State("interval",
# "value") to keep track of time slider
try:
# @app.callback([Output()], [Input()])
# def NeuronSelection(input):
# do process here
# return output
print("Add Callbacks here")
try:
@app.callback([Output("display-template", "children"), Output("clear-template", "children")],
[Input("v-step", "children")], [State("interval", "value"), State("clear", "children"),
State("clear-Neuron", "children"), State("display-Neuron", "children")])
def display(value, step, clear, localClear, display):
if(localClear != clear):
super.clearData()
return [display, clear]
except Exception as e:
print("display:"+str(e))
except Exception as e:
print("CallbackName:" + str(e))
# ----------------------------------------------------------------
# Callback related Data retreiving functions
# ----------------------------------------------------------------
try:
print("Add functions here")
# ------------------------------------------------------------
# Data gethering (MongoDB operations)
# ------------------------------------------------------------
print("Add MongoDB functions here")
except Exception as e:
print("Data process loading:"+str(e))
"""Create a dash layout for the module
"""
from collections import deque
import dash_html_components as html
import dash_bootstrap_components as dbc
from .callbacks import callbacks
class layout():
""" Layout class
"""
# Needed variables for the graphs --------------------------------
#
xAxisGraphName = deque(maxlen=100)
#
# ----------------------------------------------------------------
def clearData(self):
""" Clear the data when moved forward or backward for more than one step
"""
self.xAxisGraphName.clear()
def Vis(self, app, g):
""" Create layer components
Args:
app : Flask app
g (Global_Var): reference to access global variables
Returns:
Dash app layer
"""
try:
layer = dbc.Card(
dbc.CardBody(
[
html.Div(id="template-vis", children=[
# Global show based on selected layer
html.H3("Your Layer here"),
html.Div(id='clear-template', children="False", style={'display': 'none'}),
html.Div(id='display-template', children="False", style={'display': 'none'})
])
], style={"textAlign": "center", "padding": "10px"}
)
)
# load callbacks
callbacks().init(self, app, g)
# Return the Layer
return layer
except Exception as e:
print("Layer:" + str(e))
""" Spark pre-processing operations .
"""
import traceback
def process(g):
""" Start processing and reduce module count .
Args:
g (Global_Var): reference to access global variables
"""
preProcessing(g)
g.modulesNbr = g.modulesNbr - 1
# Spark operations and pre-processing --------------------------------
def preProcessing(g):
"""Loads the GDD database into BDD .
Args:
g (Global_Var): reference to access global variables
"""
MODULE_NAME = ""
DOCUMENT_NAME = ""
OUTPUT_DOCUMENT_NAME = ""
MONGODBURL = g.MONGODBURL
try:
if (not (OUTPUT_DOCUMENT_NAME in g.db.list_collection_names())) and (DOCUMENT_NAME in g.db.list_collection_names()):
# Spark setup --------------------------------------------
if g.sparkSession == None:
g.createSparkSession()
# doing the heavy work -----------------------------------
# Save data into Mongo------------------------------------
#df.write.format("com.mongodb.spark.sql.DefaultSource") \
# .option("spark.mongodb.output.uri",
# MONGODBURL + g.name + "."+OUTPUT_DOCUMENT_NAME+"?authSource=admin&readPreference=primaryPreferred").mode("overwrite").save()
#print("Saved to BDD")
# ------------------------------------------------------------
else:
if(not DOCUMENT_NAME in g.db.list_collection_names()):
print(DOCUMENT_NAME, "not found")
g.modules = [module for module in g.modules if module != MODULE_NAME]
print("done ", MODULE_NAME)
except Exception as e:
print("Error:" + str(e))
traceback.print_exc()
pass
""" Dash callbacks template.
"""
from dash.dependencies import Output, Input, State
class callbacksOp():
def __init__(self, app, g):
self.app = app
self.g = g
"""Initialize the callback .
Args:
app : Flask app
g (Global_Var): reference to access global variables
"""
# ------------------------------------------------------------
# Callbacks
# ------------------------------------------------------------
# to prevent creating duplicate callbacks next use (to be copied)
if not g.checkExistance(app, "name of one element from layout"):
try:
@app.callback([Output("display-template", "children"), Output("clear-template", "children")],
[Input("v-step", "children")], [State("interval", "value"), State("clear", "children"),
State("clear-Neuron", "children"), State("display-Neuron", "children")])
def display(value, step, clear, localClear, display):
if(localClear != clear):
super.clearData()
return [display, clear]
except Exception as e:
print("display:"+str(e))
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment