diff --git a/simulators/N2S3_MongoDB.scala b/simulators/N2S3_MongoDB.scala
index 7db04afe12fd6a61bb7f90b51b4b676a1c0a3a08..31c2d577a5721f9fc58782a5d3f76407f52f9875 100755
--- a/simulators/N2S3_MongoDB.scala
+++ b/simulators/N2S3_MongoDB.scala
@@ -34,7 +34,7 @@ class SimMongoLog(n2s3 : N2S3,list : IndexedSeq[IndexedSeq[IndexedSeq[Connection
val USERNAME = ""
val PASSWORD = ""
val HOST = "127.0.0.1:27017"
- val mongoClient: MongoClient = MongoClient("mongodb://"+USERNAME+":"+PASSWORD+"@"+HOST+"/?authSource=admin&readPreference=primaryPreferred")
+ val mongoClient: MongoClient = MongoClient(if (USERNAME != "" && PASSWORD != "") "mongodb://"+USERNAME+":"+PASSWORD+"@"+HOST+"/?authSource=admin&readPreference=primaryPreferred" else "mongodb://"+HOST+"/?authSource=admin&readPreference=primaryPreferred")
val database: MongoDatabase = mongoClient.getDatabase(name+"-"+time.replaceAll("\\s", "-"))
var singleLayersDocument = Document()
var neuroneS,neuroneE = 0
@@ -109,7 +109,7 @@ class SimMongoLog(n2s3 : N2S3,list : IndexedSeq[IndexedSeq[IndexedSeq[Connection
})
InfoCollection.insertOne(Document(
- "MaxS" -> (benchmark.getResult.evaluationByMaxSpiking.score * 100.00D) / benchmark.getResult.inputNumber,
+ "Acc" -> (benchmark.getResult.evaluationByMaxSpiking.score * 100.00D) / benchmark.getResult.inputNumber,
"NLabel" -> NeuronLabel)).subscribe(observer)
/**
"FirstS" -> (benchmark.getResult.evaluationByFirstSpiking.score * 100.00D) / benchmark.getResult.inputNumber,
@@ -227,7 +227,7 @@ class NeuronsFireLog(name : String,mongoCollection: MongoCollection[Document], o
// if (((System.currentTimeMillis() - globalTimestamps)) - time >= 0.250){
time = (System.currentTimeMillis() - globalTimestamps)
documents = documents.::(Document("i" -> Document("L" -> path.actor.path.name.split(":")(0),
- "N" -> BsonInt32.apply(Integer.valueOf(path.actor.path.name.split(":")(1)))),"T" -> BsonDouble.apply(BigDecimal(time/1000.0).setScale(6, BigDecimal.RoundingMode.HALF_UP).toDouble),"Input" -> Integer.valueOf(GlobalLogVars.ActualInput)))
+ "N" -> BsonInt32.apply(Integer.valueOf(path.actor.path.name.split(":")(1)))),"T" -> BsonDouble.apply(BigDecimal(time/1000.0).setScale(6, BigDecimal.RoundingMode.HALF_UP).toDouble), if(GlobalLogVars.ActualInput == "") "Input" -> "" else "Input" -> BsonInt32.apply(Integer.valueOf(GlobalLogVars.ActualInput))))
id = id + 1
i = i + 1
@@ -408,7 +408,7 @@ class SynapticWeight(list : Seq[Seq[Seq[ConnectionPath]]], refreshRate : Int = 1
y = cnx._2
}
})
- documents = documents.::(Document("T" -> BsonDouble.apply(BigDecimal((System.currentTimeMillis() - globalTimestamps)/1000.0).setScale(6, BigDecimal.RoundingMode.HALF_UP).toDouble), "C" -> v._1, "To" -> Integer.valueOf(v._2.split(":")(1)), "V" ->
+ documents = documents.::(Document("T" -> BsonDouble.apply(BigDecimal((System.currentTimeMillis() - globalTimestamps)/1000.0).setScale(6, BigDecimal.RoundingMode.HALF_UP).toDouble), "C" -> v._1, "To" -> BsonInt32.apply(Integer.valueOf(v._2.split(":")(1))), "V" ->
BsonDouble.apply(BigDecimal(v._3).setScale(6, BigDecimal.RoundingMode.HALF_UP).toDouble),"index" -> Document("x" -> y, "y" -> x), "L" -> v._2.split(":")(0)))
})
valuesTmp = values
@@ -477,4 +477,4 @@ class ActivityStoreActor(n2s3 : N2S3) extends NetworkActor {
object GlobalLogVars {
var ActualInput = ""
var globalInput = 0
-}
\ No newline at end of file
+}
diff --git a/simulators/Nengo_MongoDB.py b/simulators/Nengo_MongoDB.py
index 6291fc643848e56dcd77e0c06dcd5b6b3e34a40e..0aea37ca4ad92295bb0063012819cb0674668c68 100755
--- a/simulators/Nengo_MongoDB.py
+++ b/simulators/Nengo_MongoDB.py
@@ -7,6 +7,8 @@ import numpy as np
import nengo
import time
import math
+import pdb
+
class Nengo_MongoDB:
"""
Class to collect data from simulation and store it somewhere
@@ -17,6 +19,7 @@ class Nengo_MongoDB:
self.MongoDB = MongoDB()
self.MongoClient = None
self.fig = None
+ self.toMongo = False
self.globalTime = 0
self.img = None
self.simName = ""
@@ -30,6 +33,7 @@ class Nengo_MongoDB:
self.WeightsData = []
self.WeightsDataTMP = []
self.weightV2 = []
+ self.depth = 0
self.PotentialCollection = None
self.PotentialData = []
self.PotentialDataTMP = []
@@ -52,46 +56,49 @@ class Nengo_MongoDB:
# ----------------------------------------
- def set(self,sim,Args,weightVar = None):
+ def set(self,simulation,simulationData,toMongo = False,weightVar = None):
"""
Set the simulation argument
Parameters
----------
- sim : Simulator
- Simulator argument
- Args : Collection
- Collection of data related to the simulation
+ simulation : Simulator argument
+ simulationData : Collection of data related to the simulation
+ toMongo : store data in mongoDB or not
"""
- self.sim = sim
- self.backend = Args["backend"]
- self.Dataset = Args["Dataset"]
- self.Labels = Args["Labels"]
- self.step_time = Args["step_time"]
- self.input_nbr = Args["input_nbr"]
+ self.sim = simulation
+ self.backend = simulationData["backend"]
+ self.Dataset = simulationData["Dataset"]
+ self.step_time = simulationData["step_time"]
+ self.input_nbr = simulationData["input_nbr"]
self.simName = self.sim.model.toplevel.label.replace(" ","-")+"-"+self.date
self.weightV2 = weightVar
+ self.toMongo = toMongo
- # Read Mongo credentials
- self.MongoClient = self.MongoDB.MongoConnect()
- assert self.MongoClient , "Can't Connect to MongoDB !"
- print("Connected to MongoDB successfully")
-
- if self.MongoDB.CreateDataBase(self.MongoClient,self.simName):
- print("Database created")
- else:
- print("Database already exists")
+ if toMongo:
+ # Read Mongo credentials
+ self.MongoClient = self.MongoDB.MongoConnect()
+ assert self.MongoClient , "Can't Connect to MongoDB !"
+ print("Connected to MongoDB successfully")
- # Store info and network arch
- self.InfoAndArchi()
-
- # Initialize Client for other Collections
- self.LabelsCollection = self.MongoClient[self.simName]["labels"]
- self.SpikesCollection = self.MongoClient[self.simName]["spikes"]
- self.WeightsCollection = self.MongoClient[self.simName]["synapseWeight"]
- self.PotentialCollection = self.MongoClient[self.simName]["potential"]
+ if self.MongoDB.CreateDataBase(self.MongoClient,self.simName):
+ print("Database created")
+ else:
+ print("Database already exists")
+ # Store info and network arch
+ self.InfoAndArchi()
+
+ # Initialize Client for other Collections
+ self.SpikesCollection = self.MongoClient[self.simName]["spikes"]
+ self.WeightsCollection = self.MongoClient[self.simName]["synapseWeight"]
+ self.PotentialCollection = self.MongoClient[self.simName]["potential"]
+
+ if("Labels" in simulationData):
+ self.Labels = simulationData["Labels"]
+ self.LabelsCollection = self.MongoClient[self.simName]["labels"]
+
# Helper functions -----------------------
def DrawRealTime(self,fig,img,data,t):
@@ -106,11 +113,6 @@ class Nengo_MongoDB:
img.set_array(np.random.random((50,50)))
# redraw the figure
fig.canvas.draw()
-
- def storeToMongo(self,label,data):
- # Collect for couple of steps
- # Add to Mongo
- print("Mongo processing data")
def storeLabels(self,t):
# Collect Labels for couple of steps
@@ -118,7 +120,7 @@ class Nengo_MongoDB:
self.LabelsData.append({"L":int(str(self.Labels[int(round(t,2) / self.step_time)-1])),"T":round(round(t,2)-round(self.step_time,2),2),"G":int(round(t,2) / self.step_time)})
# Add to Mongo
- if((len(self.LabelsData) == self.input_nbr * 0.25)):
+ if((len(self.LabelsData) == self.input_nbr * self.step_time)):
self.LabelsCollection.insert_many(self.LabelsData)
self.LabelsData.clear()
@@ -127,51 +129,51 @@ class Nengo_MongoDB:
if(len(self.sim._sim_data[Probe_spikes])!=0):
for i,n in enumerate(self.sim._sim_data[Probe_spikes][0].tolist()):
if(n != 0):
- self.SpikesData.append({"i": {"L": Probe_spikes.target.ensemble.label,"N":i},"T":round(t,3),"Input":int(str(self.Labels[int(round(t,2) / self.step_time)-1])),"S":int(n*self.sim.dt)})
+ self.SpikesData.append({"i": {"L": Probe_spikes.target.ensemble.label,"N":i},"T":round(t,3),"Input":int(str(self.Labels[int(round(t,2) / self.step_time)-1])) if self.Labels is not None else ""})
# Add to Mongo
- if(len(self.SpikesData) > 1000):
+
+ if(len(self.SpikesData) > 10000):
self.SpikesCollection.insert_many(self.SpikesData)
self.SpikesData.clear()
- def storeWeights(self,t,Probe_weights):
+ def storeWeights(self,t,Probe_weights, depth):
# Collect Weights for couple of steps
-
- if(len(self.WeightsDataTMP) == 0):
- for i,f in enumerate(self.sim._sim_data[Probe_weights][0].tolist()):
- for j,v in enumerate(f):
- x,y = self.getXY(j)
- self.WeightsData.append( {"T":round(t,3), "C": j, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": str(Probe_weights.label).split("_")[0]} )
- self.WeightsDataTMP.append( {"T":round(t,3), "C": j, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": str(Probe_weights.label).split("_")[0]} )
- else:
- for i,f in enumerate(self.sim._sim_data[Probe_weights][0].tolist()):
- for j,v in enumerate(f):
- x,y = self.getXY(j)
- if(self.WeightsDataTMP[i*784 + j]["V"] != round(v,9) and self.WeightsDataTMP[i*784 + j]["T"] != round(t,3)):
- self.WeightsData.append( {"T":round(t,3), "C": j, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": str(Probe_weights.label).split("_")[0]} )
- self.WeightsDataTMP[i*784 + j] = {"T":round(t,3), "C": j, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": str(Probe_weights.label).split("_")[0]}
-
+ if self.sim._sim_data[Probe_weights] != []:
+ if(len(self.WeightsDataTMP) == 0):
+ for i,f in enumerate(self.sim._sim_data[Probe_weights][0].tolist()):
+ for j,v in enumerate(f):
+ x,y = self.getXY(j)
+ self.WeightsData.append( {"T":round(t,3), "C": self.depth, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": str(Probe_weights.label).split("_")[0]} )
+ self.WeightsDataTMP.append( {"T":round(t,3), "C": self.depth, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": str(Probe_weights.label).split("_")[0]} )
+ else:
+ for i,f in enumerate(self.sim._sim_data[Probe_weights][0].tolist()):
+ for j,v in enumerate(f):
+ x,y = self.getXY(j)
+ if(self.WeightsDataTMP[i*784 + j]["V"] != round(v,9) and self.WeightsDataTMP[i*784 + j]["T"] != round(t,3)):
+ self.WeightsData.append( {"T":round(t,3), "C": self.depth, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": str(Probe_weights.label).split("_")[0]} )
+ self.WeightsDataTMP[i*784 + j] = {"T":round(t,3), "C": self.depth, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": str(Probe_weights.label).split("_")[0]}
+
# Add to Mongo
if(len(self.WeightsData) > 10000):
self.WeightsCollection.insert_many(self.WeightsData)
self.WeightsData.clear()
- def storeWeightsV2(self,t,weights):
+ def storeWeightsV2(self,t,weights, depth):
# Collect Weights for couple of steps
- # 423 ms ± 7.91 ms per loop (mean ± std. dev. of 10 runs, 1 loop each) normal one
if(len(self.WeightsDataTMP) == 0):
for i,f in enumerate(weights[0].tolist()):
for j,v in enumerate(f):
x,y = self.getXY(j)
- self.WeightsData.append( {"T":round(t,3), "C": j, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": "Layer1"} )
- self.WeightsDataTMP.append( {"T":round(t,3), "C": j, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": "Layer1"})
+ self.WeightsData.append( {"T":round(t,3), "C": self.depth, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": "Layer1"} )
+ self.WeightsDataTMP.append( {"T":round(t,3), "C": self.depth, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": "Layer1"})
else:
for i,f in enumerate(weights[0].tolist()):
for j,v in enumerate(f):
x,y = self.getXY(j)
if(self.WeightsDataTMP[i*784 + j]["V"] != round(v,9) and self.WeightsDataTMP[i*784 + j]["T"] != round(t,3)): #
- self.WeightsData.append( {"T":round(t,3), "C": j, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": "Layer1"} )
- self.WeightsDataTMP[i*784 + j] = {"T":round(t,3), "C": j, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": "Layer1"}
+ self.WeightsData.append( {"T":round(t,3), "C": self.depth, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": "Layer1"} )
+ self.WeightsDataTMP[i*784 + j] = {"T":round(t,3), "C": self.depth, "To":i,"V":round(v,9), "index": {"x":x ,"y":y},"L": "Layer1"}
# Add to Mongo
if(len(self.WeightsData) > 10000):
self.WeightsCollection.insert_many(self.WeightsData)
@@ -201,24 +203,26 @@ class Nengo_MongoDB:
def storeTestInfo(self,Acc,Labels,outputLayer):
InfoCollection = self.MongoClient[self.simName]["info"]
- InfoCollection.insert_one({"MaxS":Acc,"NLabel":[{"L" : outputLayer, "N" : int(K), "Label" : int(Labels[K])} for K in Labels]})
+ InfoCollection.insert_one({"Acc":Acc,"NLabel":[{"L" : outputLayer, "N" : int(K), "Label" : int(Labels[K])} for K in Labels]})
+
def closeLog(self):
"""
Close log file
"""
- # Add any left data
- if(len(self.LabelsData)!=0):
- self.LabelsCollection.insert_many(self.LabelsData)
- if(len(self.SpikesData)!=0):
- self.SpikesCollection.insert_many(self.SpikesData)
- if(len(self.WeightsData)!=0):
- self.WeightsCollection.insert_many(self.WeightsData)
- if(len(self.PotentialData)!=0):
- self.PotentialCollection.insert_many(self.PotentialData)
-
- # Close Mongo client
- print("Closing Client")
- self.MongoClient.close()
+ if self.toMongo:
+ # Add any left data
+ if(len(self.LabelsData)!=0):
+ self.LabelsCollection.insert_many(self.LabelsData)
+ if(len(self.SpikesData)!=0):
+ self.SpikesCollection.insert_many(self.SpikesData)
+ if(len(self.WeightsData)!=0):
+ self.WeightsCollection.insert_many(self.WeightsData)
+ if(len(self.PotentialData)!=0):
+ self.PotentialCollection.insert_many(self.PotentialData)
+
+ # Close Mongo client
+ print("Closing Client")
+ self.MongoClient.close()
# ----------------------------------------
@@ -230,30 +234,32 @@ class Nengo_MongoDB:
assert len(self.sim.model.probes) != 0 , "No Probes to store"
for probe in self.sim.model.probes:
-
if(self.backend == "Nengo"):
if len(self.sim._sim_data[probe]) != 0:
self.sim._sim_data[probe] = [self.sim._sim_data[probe][-1]]
elif(self.backend == "NengoDL"):
- if len(self.sim.model.params[probe]) != 0:
+ if len(self.sim.model.params[probe]) > 100:
self.sim.model.params[probe] = [self.sim.model.params[probe][-1]]
+
elif(self.backend == "NengoOCL"):
if len(self.sim._probe_outputs[probe]) != 0:
self.sim._probe_outputs[probe] = [self.sim._probe_outputs[probe][-1]]
- # Process simulation data
- if (probe.attr == "spikes"):
- self.storeSpikes(t,probe)
- if (probe.attr == "voltage"):
- self.storePotential(t,probe)
-
- if Decimal(str(round(t,3))) % Decimal(str(self.sim.dt)) == 0:
- if (probe.attr == "weights"):
- self.storeWeights(t,probe)
- if (self.weightV2 != None and len(self.weightV2) != None and not isinstance(self.weightV2[0],int)):
- self.storeWeightsV2(t,self.weightV2)
-
- self.storeLabels(t)
+ if self.toMongo:
+ # Process simulation data
+ if (probe.attr == "spikes"):
+ self.storeSpikes(t,probe)
+ if (probe.attr == "voltage"):
+ self.storePotential(t,probe)
+
+ if Decimal(str(round(t,3))) % Decimal(str(self.sim.dt)) == 0:
+ if (probe.attr == "weights"):
+ self.storeWeights(t,probe, self.depth)
+ if (self.weightV2 != None and len(self.weightV2) != None and not isinstance(self.weightV2[0],int)):
+ self.storeWeightsV2(t,self.weightV2, self.depth)
+
+ if self.toMongo and self.Labels is not None:
+ self.storeLabels(t)
# ---------------------------------------------
# MongoDB class
@@ -292,7 +298,7 @@ class MongoDB:
def CreateDataBase(self,client : MongoClient,name):
""" Create a new Database
- Args:
+ simulationData:
client (MongoClient): MongoDB Client instance
name (String): Simulation label