Merge branch 'V1.2' into 'master'

V1.2

See merge request !2

n2s3/src/main/scala/fr/univ_lille/cristal/emeraude/n2s3/features/logging/SimMongoLog.scala

@@ -10,13 +10,10 @@ import fr.univ_lille.cristal.emeraude.n2s3.features.builder.{N2S3, NeuronGroupOb
 import fr.univ_lille.cristal.emeraude.n2s3.features.io.report.BenchmarkMonitorRef
 import fr.univ_lille.cristal.emeraude.n2s3.support.GlobalTypesAlias.Timestamp
 import fr.univ_lille.cristal.emeraude.n2s3.support.actors.{LocalActorDeploymentStrategy, Message}
-import javax.management.modelmbean.ModelMBean
 import javax.swing.JPanel
 import org.mongodb.scala.bson.{BsonDouble, BsonInt32}
-import org.mongodb.scala.model.Filters
-import org.mongodb.scala.result.UpdateResult
 import org.mongodb.scala.{Observer, _}
-
+import sys.process._
 import scala.collection.immutable
 
 /**
@@ -36,7 +33,7 @@ import scala.collection.immutable
 
   println("Connecting to database ")
   val mongoClient: MongoClient = MongoClient()
-  val database: MongoDatabase = mongoClient.getDatabase("Simulation")
+  val database: MongoDatabase = mongoClient.getDatabase(name+"-"+time.replaceAll("\\s", "-"))
   var singleLayersDocument = Document()
   var neuroneS,neuroneE = 0
   var id = 0
@@ -100,9 +97,9 @@ import scala.collection.immutable
   }
 
   def storeBenchmarkTestInfo(benchmark : BenchmarkMonitorRef): Unit ={
-    var NeuronLabel : List[Document] = List.empty
+    var NeuronLabel : List[Document]= List()
     benchmark.getResult.evaluationByMaxSpiking.labelAssoc.foreach(s => {
-      NeuronLabel = NeuronLabel.::(Document(s._1 -> s._2))
+      NeuronLabel = NeuronLabel.::(Document("L" -> s._1.split(":")(0),"N" -> s._1.split(":")(1), "Label"-> s._2))
     })
 
     InfoCollection.insertOne(Document(
@@ -119,23 +116,24 @@ import scala.collection.immutable
 //    SynapseWeightCollection.createIndex(Indexes.ascending("T"))
 
     mongoClient.close()
+    val restartMongo = "service mongod restart" !
   }
 
   /**
     * Collections
     */
 
-  ArchiCollection = database.getCollection(name+"archi")
+  ArchiCollection = database.getCollection("archi")
   ArchiCollection.deleteMany(new BasicDBObject())
-  InfoCollection = database.getCollection(name+"info")
+  InfoCollection = database.getCollection("info")
   InfoCollection.deleteMany(new BasicDBObject())
-  PotentialCollection = database.getCollection(name+"potential")
+  PotentialCollection = database.getCollection("potential")
   PotentialCollection.deleteMany(new BasicDBObject())
-  SpikesCollection = database.getCollection(name+"spikes")
+  SpikesCollection = database.getCollection("spikes")
   SpikesCollection.deleteMany(new BasicDBObject())
-  LabelCollection = database.getCollection(name+"labels")
+  LabelCollection = database.getCollection("labels")
   LabelCollection.deleteMany(new BasicDBObject())
-  SynapseWeightCollection = database.getCollection(name+"synapseWeight")
+  SynapseWeightCollection = database.getCollection("synapseWeight")
   SynapseWeightCollection.deleteMany(new BasicDBObject())
 
   var neuronsFireLogDocuments : List[Document] = List.empty
@@ -159,11 +157,11 @@ import scala.collection.immutable
 
 
   //ArchiCollection.insertOne(Document("n" -> name,"i" -> n2s3.layers(0).neurons.size)).subscribe(observer)
-  var layerNeron : List[Document] = List.empty
+  var layerNeron : Map[String,Int] = Map()
   n2s3.layers.foreach(layer => {
-    layerNeron = layerNeron.::(Document(layer.identifier -> layer.neurons.size))
+    layerNeron += (layer.identifier -> layer.neurons.size)
   })
-  InfoCollection.insertOne(Document("n" -> name,"L:N"->layerNeron,"T"->time,"D"->Data)).subscribe(observer)
+  InfoCollection.insertOne(Document("n" -> name,"L:N"->Document(layerNeron),"T"->time,"D"->Data)).subscribe(observer)
 
   n2s3.layers.foreach(elt => {
         Layers = Layers + (elt.getIdentifier -> Lid)
@@ -260,7 +258,7 @@ import scala.collection.immutable
         if (((System.currentTimeMillis() - globalTimestamps) * 1000) - time > 250){
             time = (System.currentTimeMillis() - globalTimestamps) * 1000
             if(i < step){
-              documents = documents.::(Document("i" -> Document("L" -> Layers.get(path.actor.path.name.split(":")(0)),
+              documents = documents.::(Document("i" -> Document("L" -> path.actor.path.name.split(":")(0),
               "N" -> BsonInt32.apply(Integer.valueOf(path.actor.path.name.split(":")(1)))),"T" -> time,"Input" -> GlobalLogVars.ActualInput))
               // ,"tt" -> BsonDouble(step*i)
             id = id + 1
@@ -347,7 +345,8 @@ class LabelMonitoring(n2s3 : N2S3, outputNeuron : Seq[NeuronGroupRef],mongoColle
      case LabelChangeResponse(start, end, label) =>
            time = (System.currentTimeMillis() - globalTimestamps) * 1000
            GlobalLogVars.ActualInput = label
-           documents = documents.::(Document("L" -> BsonInt32(Integer.valueOf(label)),"T" ->time))
+           GlobalLogVars.globalInput = GlobalLogVars.globalInput + 1
+           documents = documents.::(Document("L" -> BsonInt32(Integer.valueOf(label)),"T" ->time,"G" -> GlobalLogVars.globalInput))
 
            id = id + 1
            i = i + 1
@@ -499,5 +498,6 @@ class ActivityStoreActor(n2s3 : N2S3) extends NetworkActor {
 
 object GlobalLogVars {
   var ActualInput = ""
+  var globalInput = 0
 }
 

n2s3_examples/src/main/scala/fr/univ_lille/cristal/emeraude/n2s3/apps/ExampleMnistPrune1Epoch.scala

@@ -56,7 +56,9 @@ object  ExampleMnistPrune1Epoch extends App {
   var NeuroneLabels = List[(Int, String)]()
   var logger: SimMongoLog = null
   var log = true
-  var prune = true
+  var prune = false
+  var globalIteration = 6
+  var sizeChunk = 10000
 
   for(x <- 1 to 1) {
     implicit val timeout = Config.longTimeout
@@ -64,7 +66,7 @@ object  ExampleMnistPrune1Epoch extends App {
     beta = ArrBeta((x - 1) % 5)
     alpha = 0.05F
     beta = 0.1F
-    SimName = "20NCropped"+"-"+x
+    SimName = "100N" + "-" + x
     println("a:" + alpha + " b:" + beta)
     QBGParameters.alf_m = 0.005f
     QBGParameters.alf_p = 0.01f
@@ -77,11 +79,13 @@ object  ExampleMnistPrune1Epoch extends App {
     var simTime = ""
     var benchmarkMonitor: BenchmarkMonitorRef = null
     n2s3 = new N2S3("N2S3")
+
     /** Vars of Threshold  **/
 
     CTotal = 0F
     CRmaining = 0F
     Ratio = 0F
+
     /** **/
 
     folderName = ""
@@ -98,7 +102,7 @@ object  ExampleMnistPrune1Epoch extends App {
 
     val unsupervisedLayer = n2s3.createNeuronGroup()
       .setIdentifier("Layer1")
-      .setNumberOfNeurons(20)
+      .setNumberOfNeurons(100)
       .setNeuronModel(LIF, Seq(
         (MembranePotentialThreshold, 35 millivolts)))
 
@@ -148,13 +152,13 @@ object  ExampleMnistPrune1Epoch extends App {
     val labelTestFile = N2S3ResourceManager.getByName("mnist-test-labels").getAbsolutePath
 
 
-    while (iteration <= 6) {
+    while (iteration <= globalIteration) {
       simTime = ""
       unsupervisedLayer.unfixNeurons()
       n2s3.first = false
-      stream = InputMnist.DataParts(dataFile, labelFile,0,10000)
+      stream = InputMnist.DataParts(dataFile, labelFile, sizeChunk*(iteration-1), sizeChunk)
       inputStream.append(stream)
-      //stream.goto()
+      stream.goto()
       println("Start Training ... " + "[" + iteration + "]")
 
       if (iteration == 1 && log) {
@@ -167,8 +171,9 @@ object  ExampleMnistPrune1Epoch extends App {
       println(System.currentTimeMillis() - duration)
       simTime = simTime + " | " + (System.currentTimeMillis() - duration) + "\n"
 
-      if(iteration == 6 && log) {
-      logger.storeSimInfoAndDestroy()}
+      if (iteration == globalIteration && log) {
+        logger.storeSimInfoAndDestroy()
+      }
 
       println("saving : " + System.currentTimeMillis())
       NetworkSaving.save(n2s3, croppedCnx, SynapseIndex, folderName + "/" + SimName + " Arch " + iteration)
@@ -177,7 +182,7 @@ object  ExampleMnistPrune1Epoch extends App {
       saveCroppedSynapses(unsupervisedLayer, "[" + iteration + "]")
       println("end : " + System.currentTimeMillis())
 
-
+      /*
        inputStream.append(InputMnist.DataFrom(dataTestFile, labelTestFile))
 
         println("Start Testing " + "[" + iteration + "]")
@@ -197,7 +202,7 @@ object  ExampleMnistPrune1Epoch extends App {
 
         benchmarkMonitor.exportToHtmlView(folderName + "/" + "test " + SimName + " [" + iteration + "]" + ".html")
         benchmarkMonitor.saveCrops(folderName + "/" + "Sim " + SimName + " info a:" + alpha + " no b:" + beta, simTime, " Ratio " + Ratio)
-
+*/
 
       if (prune) {
         Ratio = Ratio + (alpha * (CRmaining / CTotal))
@@ -206,8 +211,9 @@ object  ExampleMnistPrune1Epoch extends App {
         CropInfo = cropNetwork(inputLayer, Ratio)
         println("end : " + System.currentTimeMillis())
         //     cropInput(n2s3.inputLayerRef.get, Ratio)
+      }
 
-
+      if (iteration == globalIteration) {
         inputStream.append(InputMnist.DataFrom(dataTestFile, labelTestFile))
 
         println("Start Testing cropped " + "[" + iteration + "]" + " with " + Ratio)
@@ -230,7 +236,7 @@ object  ExampleMnistPrune1Epoch extends App {
 
         benchmarkMonitor.saveCrops(folderName + "/" + "Sim " + SimName + " info a:" + alpha + " b:" + beta, simTime, CropInfo + " Ratio " + Ratio)
       }
-      if(iteration == 6 && log){
+      if (iteration == globalIteration && log) {
         logger.storeBenchmarkTestInfo(benchmarkMonitor)
       }
       iteration += 1
@@ -239,6 +245,8 @@ object  ExampleMnistPrune1Epoch extends App {
   //logger.destroy()
   n2s3.destroy()
 
+  sys.exit(0)
+
   def showHeatMap(inputL: InputNeuronGroupRef, layer : NeuronGroupRef): Unit ={
     val inputToClassifierIndex = new ConnectionIndex(inputL, layer)