diff --git a/n2s3/src/main/scala/fr/univ_lille/cristal/emeraude/n2s3/features/logging/SimMongoLog.scala b/n2s3/src/main/scala/fr/univ_lille/cristal/emeraude/n2s3/features/logging/SimMongoLog.scala
index ebfb969938cc4ab6797b084b038efec6b045994a..4d3d1dad8748e1d0b59a3664dad9b0000170bd28 100755
--- a/n2s3/src/main/scala/fr/univ_lille/cristal/emeraude/n2s3/features/logging/SimMongoLog.scala
+++ b/n2s3/src/main/scala/fr/univ_lille/cristal/emeraude/n2s3/features/logging/SimMongoLog.scala
@@ -10,13 +10,10 @@ import fr.univ_lille.cristal.emeraude.n2s3.features.builder.{N2S3, NeuronGroupOb
import fr.univ_lille.cristal.emeraude.n2s3.features.io.report.BenchmarkMonitorRef
import fr.univ_lille.cristal.emeraude.n2s3.support.GlobalTypesAlias.Timestamp
import fr.univ_lille.cristal.emeraude.n2s3.support.actors.{LocalActorDeploymentStrategy, Message}
-import javax.management.modelmbean.ModelMBean
import javax.swing.JPanel
import org.mongodb.scala.bson.{BsonDouble, BsonInt32}
-import org.mongodb.scala.model.Filters
-import org.mongodb.scala.result.UpdateResult
import org.mongodb.scala.{Observer, _}
-
+import sys.process._
import scala.collection.immutable
/**
@@ -34,9 +31,9 @@ import scala.collection.immutable
* Setup DB connection & Collection
* *****************************************************************************************************************/
- println("Connecting to database")
+ println("Connecting to database ")
val mongoClient: MongoClient = MongoClient()
- val database: MongoDatabase = mongoClient.getDatabase("Simulation")
+ val database: MongoDatabase = mongoClient.getDatabase(name+"-"+time.replaceAll("\\s", "-"))
var singleLayersDocument = Document()
var neuroneS,neuroneE = 0
var id = 0
@@ -100,9 +97,9 @@ import scala.collection.immutable
}
def storeBenchmarkTestInfo(benchmark : BenchmarkMonitorRef): Unit ={
- var NeuronLabel : List[Document] = List.empty
+ var NeuronLabel : List[Document]= List()
benchmark.getResult.evaluationByMaxSpiking.labelAssoc.foreach(s => {
- NeuronLabel = NeuronLabel.::(Document(s._1 -> s._2))
+ NeuronLabel = NeuronLabel.::(Document("L" -> s._1.split(":")(0),"N" -> s._1.split(":")(1), "Label"-> s._2))
})
InfoCollection.insertOne(Document(
@@ -119,23 +116,24 @@ import scala.collection.immutable
// SynapseWeightCollection.createIndex(Indexes.ascending("T"))
mongoClient.close()
+ val restartMongo = "service mongod restart" !
}
/**
* Collections
*/
- ArchiCollection = database.getCollection(name+"archi")
+ ArchiCollection = database.getCollection("archi")
ArchiCollection.deleteMany(new BasicDBObject())
- InfoCollection = database.getCollection(name+"info")
+ InfoCollection = database.getCollection("info")
InfoCollection.deleteMany(new BasicDBObject())
- PotentialCollection = database.getCollection(name+"potential")
+ PotentialCollection = database.getCollection("potential")
PotentialCollection.deleteMany(new BasicDBObject())
- SpikesCollection = database.getCollection(name+"spikes")
+ SpikesCollection = database.getCollection("spikes")
SpikesCollection.deleteMany(new BasicDBObject())
- LabelCollection = database.getCollection(name+"labels")
+ LabelCollection = database.getCollection("labels")
LabelCollection.deleteMany(new BasicDBObject())
- SynapseWeightCollection = database.getCollection(name+"synapseWeight")
+ SynapseWeightCollection = database.getCollection("synapseWeight")
SynapseWeightCollection.deleteMany(new BasicDBObject())
var neuronsFireLogDocuments : List[Document] = List.empty
@@ -159,11 +157,11 @@ import scala.collection.immutable
//ArchiCollection.insertOne(Document("n" -> name,"i" -> n2s3.layers(0).neurons.size)).subscribe(observer)
- var layerNeron : List[Document] = List.empty
+ var layerNeron : Map[String,Int] = Map()
n2s3.layers.foreach(layer => {
- layerNeron = layerNeron.::(Document(layer.identifier -> layer.neurons.size))
+ layerNeron += (layer.identifier -> layer.neurons.size)
})
- InfoCollection.insertOne(Document("n" -> name,"L:N"->layerNeron,"T"->time,"D"->Data)).subscribe(observer)
+ InfoCollection.insertOne(Document("n" -> name,"L:N"->Document(layerNeron),"T"->time,"D"->Data)).subscribe(observer)
n2s3.layers.foreach(elt => {
Layers = Layers + (elt.getIdentifier -> Lid)
@@ -260,7 +258,7 @@ import scala.collection.immutable
if (((System.currentTimeMillis() - globalTimestamps) * 1000) - time > 250){
time = (System.currentTimeMillis() - globalTimestamps) * 1000
if(i < step){
- documents = documents.::(Document("i" -> Document("L" -> Layers.get(path.actor.path.name.split(":")(0)),
+ documents = documents.::(Document("i" -> Document("L" -> path.actor.path.name.split(":")(0),
"N" -> BsonInt32.apply(Integer.valueOf(path.actor.path.name.split(":")(1)))),"T" -> time,"Input" -> GlobalLogVars.ActualInput))
// ,"tt" -> BsonDouble(step*i)
id = id + 1
@@ -347,7 +345,8 @@ class LabelMonitoring(n2s3 : N2S3, outputNeuron : Seq[NeuronGroupRef],mongoColle
case LabelChangeResponse(start, end, label) =>
time = (System.currentTimeMillis() - globalTimestamps) * 1000
GlobalLogVars.ActualInput = label
- documents = documents.::(Document("L" -> BsonInt32(Integer.valueOf(label)),"T" ->time))
+ GlobalLogVars.globalInput = GlobalLogVars.globalInput + 1
+ documents = documents.::(Document("L" -> BsonInt32(Integer.valueOf(label)),"T" ->time,"G" -> GlobalLogVars.globalInput))
id = id + 1
i = i + 1
@@ -499,5 +498,6 @@ class ActivityStoreActor(n2s3 : N2S3) extends NetworkActor {
object GlobalLogVars {
var ActualInput = ""
+ var globalInput = 0
}
diff --git a/n2s3_examples/src/main/scala/fr/univ_lille/cristal/emeraude/n2s3/apps/ExampleMnistPrune1Epoch.scala b/n2s3_examples/src/main/scala/fr/univ_lille/cristal/emeraude/n2s3/apps/ExampleMnistPrune1Epoch.scala
index 6498bd54f5f52e8e1056bf336fbb4510620b1fb3..357b2703e53be8c4ca8854ee12e2430a4211d858 100755
--- a/n2s3_examples/src/main/scala/fr/univ_lille/cristal/emeraude/n2s3/apps/ExampleMnistPrune1Epoch.scala
+++ b/n2s3_examples/src/main/scala/fr/univ_lille/cristal/emeraude/n2s3/apps/ExampleMnistPrune1Epoch.scala
@@ -56,16 +56,18 @@ object ExampleMnistPrune1Epoch extends App {
var NeuroneLabels = List[(Int, String)]()
var logger: SimMongoLog = null
var log = true
- var prune = true
+ var prune = false
+ var globalIteration = 6
+ var sizeChunk = 10000
- for(x <- 1 to 1){
+ for(x <- 1 to 1) {
implicit val timeout = Config.longTimeout
- alpha = ArrAlpha((x-1) / 5)
- beta = ArrBeta((x-1) % 5)
+ alpha = ArrAlpha((x - 1) / 5)
+ beta = ArrBeta((x - 1) % 5)
alpha = 0.05F
beta = 0.1F
- SimName = "20NCropped"+"-"+x
- println("a:"+alpha+" b:"+beta)
+ SimName = "100N" + "-" + x
+ println("a:" + alpha + " b:" + beta)
QBGParameters.alf_m = 0.005f
QBGParameters.alf_p = 0.01f
QBGParameters.beta_m = 2f
@@ -77,11 +79,13 @@ object ExampleMnistPrune1Epoch extends App {
var simTime = ""
var benchmarkMonitor: BenchmarkMonitorRef = null
n2s3 = new N2S3("N2S3")
+
/** Vars of Threshold **/
CTotal = 0F
CRmaining = 0F
Ratio = 0F
+
/** **/
folderName = ""
@@ -98,7 +102,7 @@ object ExampleMnistPrune1Epoch extends App {
val unsupervisedLayer = n2s3.createNeuronGroup()
.setIdentifier("Layer1")
- .setNumberOfNeurons(20)
+ .setNumberOfNeurons(100)
.setNeuronModel(LIF, Seq(
(MembranePotentialThreshold, 35 millivolts)))
@@ -117,7 +121,7 @@ object ExampleMnistPrune1Epoch extends App {
//showHeatMap(inputLayer,unsupervisedLayer)
SynapsesIndex(inputLayer, unsupervisedLayer)
SimTime = formatter.format(new Date())
- folderName = SimName.split("-")(0) + " [" +x+"] "+ SimTime
+ folderName = SimName.split("-")(0) + " [" + x + "] " + SimTime
new File(folderName).mkdir()
//println("Start loading "+System.currentTimeMillis())
@@ -134,41 +138,42 @@ object ExampleMnistPrune1Epoch extends App {
val inputToClassifierIndex = new ConnectionIndex(inputLayer, unsupervisedLayer)
var listOfConnexions = for (outputIndex <- 0 until unsupervisedLayer.shape.getNumberOfPoints) yield {
- for (inputX <- 0 until InputMnist.shape.dimensions(0)) yield {
- for {
- inputY <- 0 until InputMnist.shape.dimensions(1)
- input = inputLayer.getNeuronPathAt(inputX, inputY)
- output = unsupervisedLayer.getNeuronPathAt(outputIndex)
- if inputToClassifierIndex.getConnectionsBetween(input, output).nonEmpty
- } yield inputToClassifierIndex.getConnectionsBetween(input, output).head
- }
+ for (inputX <- 0 until InputMnist.shape.dimensions(0)) yield {
+ for {
+ inputY <- 0 until InputMnist.shape.dimensions(1)
+ input = inputLayer.getNeuronPathAt(inputX, inputY)
+ output = unsupervisedLayer.getNeuronPathAt(outputIndex)
+ if inputToClassifierIndex.getConnectionsBetween(input, output).nonEmpty
+ } yield inputToClassifierIndex.getConnectionsBetween(input, output).head
+ }
}
val dataTestFile = N2S3ResourceManager.getByName("mnist-test-images").getAbsolutePath
val labelTestFile = N2S3ResourceManager.getByName("mnist-test-labels").getAbsolutePath
- while (iteration <= 6) {
+ while (iteration <= globalIteration) {
simTime = ""
unsupervisedLayer.unfixNeurons()
n2s3.first = false
- stream = InputMnist.DataParts(dataFile, labelFile,0,10000)
+ stream = InputMnist.DataParts(dataFile, labelFile, sizeChunk*(iteration-1), sizeChunk)
inputStream.append(stream)
- //stream.goto()
+ stream.goto()
println("Start Training ... " + "[" + iteration + "]")
- if(iteration == 1 && log) {
- logger = new SimMongoLog(n2s3, listOfConnexions, SimName.split("-")(0), true, true, true, true,"MNIST",SimTime)
- }
+ if (iteration == 1 && log) {
+ logger = new SimMongoLog(n2s3, listOfConnexions, SimName.split("-")(0), true, true, true, true, "MNIST", SimTime)
+ }
duration = System.currentTimeMillis()
- simTime = "Train "+ iteration
+ simTime = "Train " + iteration
n2s3.runAndWait()
println(System.currentTimeMillis() - duration)
simTime = simTime + " | " + (System.currentTimeMillis() - duration) + "\n"
- if(iteration == 6 && log) {
- logger.storeSimInfoAndDestroy()}
+ if (iteration == globalIteration && log) {
+ logger.storeSimInfoAndDestroy()
+ }
println("saving : " + System.currentTimeMillis())
NetworkSaving.save(n2s3, croppedCnx, SynapseIndex, folderName + "/" + SimName + " Arch " + iteration)
@@ -177,7 +182,7 @@ object ExampleMnistPrune1Epoch extends App {
saveCroppedSynapses(unsupervisedLayer, "[" + iteration + "]")
println("end : " + System.currentTimeMillis())
-
+ /*
inputStream.append(InputMnist.DataFrom(dataTestFile, labelTestFile))
println("Start Testing " + "[" + iteration + "]")
@@ -197,17 +202,18 @@ object ExampleMnistPrune1Epoch extends App {
benchmarkMonitor.exportToHtmlView(folderName + "/" + "test " + SimName + " [" + iteration + "]" + ".html")
benchmarkMonitor.saveCrops(folderName + "/" + "Sim " + SimName + " info a:" + alpha + " no b:" + beta, simTime, " Ratio " + Ratio)
+*/
-
- if(prune) {
+ if (prune) {
Ratio = Ratio + (alpha * (CRmaining / CTotal))
println("Ratio " + Ratio)
println("cropping : " + System.currentTimeMillis())
CropInfo = cropNetwork(inputLayer, Ratio)
println("end : " + System.currentTimeMillis())
// cropInput(n2s3.inputLayerRef.get, Ratio)
+ }
-
+ if (iteration == globalIteration) {
inputStream.append(InputMnist.DataFrom(dataTestFile, labelTestFile))
println("Start Testing cropped " + "[" + iteration + "]" + " with " + Ratio)
@@ -230,15 +236,17 @@ object ExampleMnistPrune1Epoch extends App {
benchmarkMonitor.saveCrops(folderName + "/" + "Sim " + SimName + " info a:" + alpha + " b:" + beta, simTime, CropInfo + " Ratio " + Ratio)
}
- if(iteration == 6 && log){
+ if (iteration == globalIteration && log) {
logger.storeBenchmarkTestInfo(benchmarkMonitor)
}
iteration += 1
}
-}
+ }
//logger.destroy()
n2s3.destroy()
+ sys.exit(0)
+
def showHeatMap(inputL: InputNeuronGroupRef, layer : NeuronGroupRef): Unit ={
val inputToClassifierIndex = new ConnectionIndex(inputL, layer)