From 7def968f0dac5b86adb8b4925db5a8c5e9e3738c Mon Sep 17 00:00:00 2001
From: Hammouda Elbez <hammouda.elbez@univ-lille.fr>
Date: Tue, 11 Apr 2023 05:54:38 +0200
Subject: [PATCH] Adding tests
---
Local installation | 4 +
VS2N.py | 572 ++++++++++++------------
__init__.py | 1 +
src/Global_Var.py | 40 +-
src/Modules/General/callbacks.py | 10 +-
src/Modules/Synapse/callbacks.py | 4 +-
src/Modules/Synapse/spark.py | 4 +-
tests/Modules/test_General_callbacks.py | 0
tests/Modules/test_General_layout.py | 0
tests/Modules/test_General_spark.py | 0
tests/Modules/test_Neuron_callbacks.py | 0
tests/Modules/test_Neuron_layout.py | 0
tests/Modules/test_Neuron_spark.py | 0
tests/Modules/test_Synapse_callbacks.py | 0
tests/Modules/test_Synapse_layout.py | 0
tests/Modules/test_Synapse_spark.py | 0
tests/__init__.py | 1 +
tests/conftest.py | 24 +
tests/test_Global_var.py | 4 +
19 files changed, 347 insertions(+), 317 deletions(-)
create mode 100755 __init__.py
create mode 100644 tests/Modules/test_General_callbacks.py
create mode 100644 tests/Modules/test_General_layout.py
create mode 100644 tests/Modules/test_General_spark.py
create mode 100644 tests/Modules/test_Neuron_callbacks.py
create mode 100644 tests/Modules/test_Neuron_layout.py
create mode 100644 tests/Modules/test_Neuron_spark.py
create mode 100644 tests/Modules/test_Synapse_callbacks.py
create mode 100644 tests/Modules/test_Synapse_layout.py
create mode 100644 tests/Modules/test_Synapse_spark.py
create mode 100755 tests/__init__.py
create mode 100644 tests/conftest.py
create mode 100644 tests/test_Global_var.py
diff --git a/Local installation b/Local installation
index 3be5553..9faf06c 100644
--- a/Local installation
+++ b/Local installation
@@ -1,6 +1,7 @@
# Install Spark:
# Install scala
+
sudo apt install scala
# Download Spark from: https://spark.apache.org/downloads.html
tar xvf spark-3.2.1-bin-hadoop3.2.tgz
@@ -16,6 +17,9 @@ sudo apt-get update
sudo apt-get install -y mongodb-org
# Start mongoDB
+
sudo service mongod start
+
# Check mongoDB status
+
sudo service mongod status
diff --git a/VS2N.py b/VS2N.py
index 39a92a2..13671d2 100755
--- a/VS2N.py
+++ b/VS2N.py
@@ -3,319 +3,317 @@ Main VS2N class
"""
+import _thread
+import hashlib
+import importlib
+import logging
+import os
+import sqlite3
+import time
+import traceback
+import webbrowser
+
import dash
import dash_bootstrap_components as dbc
import flask
-import hashlib
-from flask import Flask, request, render_template
import flask_login
-from werkzeug.serving import run_simple
+from flask import Flask, render_template, request
from werkzeug.middleware.dispatcher import DispatcherMiddleware
+from werkzeug.serving import run_simple
+
from src.Global_Var import Global_Var
from src.Modules.General.layout import layout
-import os
-import traceback
-import sqlite3
-import importlib
-import time
-import _thread
-import webbrowser
-import logging
-
-# Print only errors in console ---------------------------------------
-log = logging.getLogger('werkzeug')
-log.setLevel(logging.ERROR)
-# --------------------------------------------------------------------
-modulesNbr = 0
-mongo_exists = False
-app_vis = None
-users = dict()
-g = Global_Var()
-__name__ = "__main__"
-url = g.config.VS2N_HOST
-port = g.config.VS2N_PORT
-
-# Test port availability ---------------------------------------------
-if(not g.testPort(port)):
- port += 1
- print("Loading another instance of VS2N")
-# --------------------------------------------------------------------
-
-# Create flask app instance ------------------------------------------
-flask_app = Flask(__name__, template_folder=os.path.curdir +
- "/src/templates", static_folder=os.path.curdir + "/src/static")
-flask_app.secret_key = g.config.SECRET_KEY
-
-login_manager = flask_login.LoginManager()
-login_manager.init_app(flask_app)
-login_manager.session_protection = "strong"
-
-# Flask user preperation ---------------------------------------------
-
-flask_app.config['LOGIN_DISABLED'] = False
-
-try:
- conn = sqlite3.connect("info.db")
- users = dict([u for u in conn.execute("SELECT * FROM users")])
- conn.close()
-except Exception as e:
- # If users database is not generated, disable login
- flask_app.config['LOGIN_DISABLED'] = True
class User(flask_login.UserMixin):
- pass
-
-# BDD connection -----------------------------------------------------
-mongo_exists = g.mongoConnect()
-# --------------------------------------------------------------------
-
-# Login manager ------------------------------------------------------
-
-
-@login_manager.unauthorized_handler
-def unauthorized_handler():
- """ Redirect any unauthorized access to login screen.
-
- Returns:
- redirection to login
- """
- return flask.redirect(flask.url_for('login'))
-
-
-@flask_app.errorhandler(404)
-def page_not_found(e):
- """ Handle the 404 error.
-
- Returns:
- redirection to login
- """
- return flask.redirect(flask.url_for('login'))
-
-
-@login_manager.user_loader
-def user_loader(email):
- """ Load user from email address.
-
- Args:
- email (String): user email
-
- Returns:
- user instance
- """
- try:
- users[email]
- except Exception as e:
- print("exception : user_loader")
- return None
-
- user = User()
- user.id = email
- return user
-
-
-@login_manager.request_loader
-def request_loader(request):
- """ Process request for authentication.
-
- Args:
- request
-
- Returns:
- user instance
- """
- try:
- email = request.form.get('email')
- email = email if email == None else hashlib.sha512(
- bytes(email, encoding='utf-8')).hexdigest()
- users[email]
- except Exception as e:
- return
- user = User()
- user.id = email
-
- user.is_authenticated = hashlib.sha512(
- bytes(request.form['password'], encoding='utf-8')).hexdigest() == users[email]
- return user
-
-# --------------------------------------------------------------------
-
-# VS2N Rooting -------------------------------------------------------
-
-
-@flask_app.route('/login', methods=['GET', 'POST'])
-def login():
- """ Handle requests to '/login' page.
+ pass
- Returns:
- redirection to login if not connected otherwise to home screen.
- """
- if flask_app.config['LOGIN_DISABLED']:
- return flask.redirect(flask.url_for('home'))
- else:
- if flask.request.method == 'GET':
- return render_template('login.html')
+class VS2N():
+
+ def __init__(self):
+ # Print only errors in console ---------------------------------------
+ log = logging.getLogger('werkzeug')
+ log.setLevel(logging.ERROR)
+ # --------------------------------------------------------------------
+ self.modulesNbr = 0
+ self.mongo_exists = False
+ self.app_vis = None
+ self.users = dict()
+ self.g = Global_Var()
+ self.url = self.g.config.VS2N_HOST
+ self.port = self.g.config.VS2N_PORT
+
+ # Test port availability ---------------------------------------------
+ if(not self.g.testPort(self.port)):
+ self.port += 1
+ print("Loading another instance of VS2N")
+ # --------------------------------------------------------------------
+
+ # Create flask app instance ------------------------------------------
+ self.flask_app = Flask(__name__, template_folder=os.path.curdir +
+ "/src/templates", static_folder=os.path.curdir + "/src/static")
+ self.flask_app.secret_key = self.g.config.SECRET_KEY
+
+ # Flask user preperation ---------------------------------------------
+
+ self.flask_app.config['LOGIN_DISABLED'] = False
- email = hashlib.sha512(
- bytes(flask.request.form['email'], encoding='utf-8')).hexdigest()
try:
- if users[email] == hashlib.sha512(bytes(flask.request.form['password'], encoding='utf-8')).hexdigest():
- user = User()
- user.id = email
- flask_login.login_user(user)
- return flask.redirect(flask.url_for('home'))
+ conn = sqlite3.connect("info.db")
+ self.users = dict([u for u in conn.execute("SELECT * FROM users")])
+ conn.close()
except Exception as e:
- print("Authentication failed !")
- pass
-
- return flask.redirect(flask.url_for('login'))
-
-
-@flask_app.route('/logout')
-def logout():
- """ Logout from VS2N.
-
- Returns:
- redirection to login screen.
- """
- if flask_app.config['LOGIN_DISABLED']:
- return flask.redirect(flask.url_for('home'))
- else:
- flask_login.logout_user()
- return flask.redirect(flask.url_for('login'))
-
-
-@flask_app.route('/')
-def main():
- """ Handle incoming requests to VS2N and check authentication status.
-
- Returns:
- redirection to home if user is authenticated otherwise to login screen.
- """
- if flask_app.config['LOGIN_DISABLED']:
- return flask.redirect(flask.url_for('home'))
- else:
- if not flask_login.current_user.is_authenticated:
- return render_template('login.html')
- else:
- return flask.redirect(flask.url_for('home'))
-
-
-@flask_app.route('/home')
-@flask_login.login_required
-def home():
- """ Get existing modules and check MongoDB availability.
-
- Returns:
- main screen if everything is ok, otherwise show exception.
- """
- global mongo_exists
-
- # init all variables
- g.__init__()
- # if MongoDB exist
- if mongo_exists:
- m = os.listdir(os.path.curdir + "/src/Modules")
- simulations = []
- g.db = g.client.list_database_names()
-
- # Load existing simulations
- for bdd in g.db:
- if bdd != "local" and bdd != "admin" and bdd != "config":
- simulations.append(bdd)
- i = 1
-
- # get modules names
- modules = []
- for module in m:
- if ".pyc" not in module and "__" not in module and "General" not in module:
- modules.append(module.rsplit(".", 1)[0])
- i = i + 1
- sparkVersion = g.createSparkSession()
- # return main screen with list of modules and existing simulations
- return render_template('index.html', modules=modules, Simulations=simulations, mongodb=g.client.server_info()["version"], vs2n=g.config.VS2N_VERSION, spark=sparkVersion)
- else:
- mongo_exists = g.mongoConnect()
- return render_template('exception.html')
-
-
-def Module(n, g):
- """ Helper function to execute each module sparks pre processing (if exist).
-
- Args:
- n (String): module name
- g (Global_Var): reference to access global variables
- """
- importlib.import_module("src.Modules." + n + ".spark").init(g)
-
-
-def LoadingModules():
- """ Run spark operation for the selected modules
- """
- try:
- # Spark & MongoDB tasks launch
- g.CreateIndexes(g.name)
- for module in g.modules:
- _thread.start_new_thread(Module, (module, g,))
-
- except Exception as e:
- print("LoadingModules:" + str(e))
-
-
-@flask_app.route('/processVis', methods=['POST'])
-@flask_login.login_required
-def processing():
- """ Start processing and load modules into dashboard.
- """
- g.name = request.get_json()["name"]
- g.modules = request.get_json()["modules"]
- g.updateInterval = float(request.get_json()["updateInterval"])
-
- g.modules.append("General")
- g.modulesNbr = len(g.modules)
-
- LoadingModules()
+ # If users database is not generated, disable login
+ self.flask_app.config['LOGIN_DISABLED'] = True
+
+ self.login_manager = flask_login.LoginManager()
+ self.login_manager.init_app(self.flask_app)
+ self.login_manager.session_protection = "strong"
+
+ # BDD connection -----------------------------------------------------
+ if __name__ == '__main__':
+ self.mongo_exists = self.g.mongoConnect()
+ # --------------------------------------------------------------------
+
+ # Create dash app instance -------------------------------------------
+ if __name__ == '__main__':
+ self.app_vis = dash.Dash(
+ __name__,
+ requests_pathname_prefix='/vis/',
+ external_stylesheets=[dbc.themes.BOOTSTRAP, dbc.icons.FONT_AWESOME],
+ suppress_callback_exceptions=True,
+ title="VS2N")
+ self.app_vis.enable_dev_tools(debug=self.g.config.DEBUG)
+
+ # Login manager ------------------------------------------------------
+
+ @self.login_manager.unauthorized_handler
+ def unauthorized_handler():
+ """ Redirect any unauthorized access to login screen
+
+ Returns:
+ redirection to login
+ """
+ return flask.redirect(flask.url_for('login'))
+
+
+ @self.flask_app.errorhandler(404)
+ def page_not_found(e):
+ """ Handle the 404 error
+
+ Returns:
+ redirection to login
+ """
+ return flask.redirect(flask.url_for('login'))
+
+
+ @self.login_manager.user_loader
+ def user_loader(email):
+ """ Load user from email address
+
+ Args:
+ email (String): user email
+
+ Returns:
+ user instance
+ """
+ try:
+ self.users[email]
+ except Exception as e:
+ print("exception : user_loader")
+ return None
+
+ user = User()
+ user.id = email
+ return user
+
+
+ @self.login_manager.request_loader
+ def request_loader(request):
+ """ Process request for authentication
+
+ Args:
+ request
+
+ Returns:
+ user instance
+ """
+ try:
+ email = request.form.get('email')
+ email = email if email == None else hashlib.sha512(
+ bytes(email, encoding='utf-8')).hexdigest()
+ self.users[email]
+ except Exception as e:
+ return
+ user = User()
+ user.id = email
+
+ user.is_authenticated = hashlib.sha512(
+ bytes(request.form['password'], encoding='utf-8')).hexdigest() == self.users[email]
+ return user
+
+ # --------------------------------------------------------------------
+
+ # VS2N Rooting -------------------------------------------------------
+
+
+ @self.flask_app.route('/login', methods=['GET', 'POST'])
+ def login():
+ """ Handle requests to '/login' page
+
+ Returns:
+ redirection to login if not connected otherwise to home screen
+ """
+ if self.flask_app.config['LOGIN_DISABLED']:
+ return flask.redirect(flask.url_for('home'))
+ else:
+ if flask.request.method == 'GET':
+ return render_template('login.html')
+
+ email = hashlib.sha512(
+ bytes(flask.request.form['email'], encoding='utf-8')).hexdigest()
+ try:
+ if self.users[email] == hashlib.sha512(bytes(flask.request.form['password'], encoding='utf-8')).hexdigest():
+ user = User()
+ user.id = email
+ flask_login.login_user(user)
+ return flask.redirect(flask.url_for('home'))
+ except Exception as e:
+ print("Authentication failed !")
+ pass
+
+ return flask.redirect(flask.url_for('login'))
+
+
+ @self.flask_app.route('/logout')
+ def logout():
+ """ Logout from VS2N.
+
+ Returns:
+ redirection to login screen.
+ """
+ if self.flask_app.config['LOGIN_DISABLED']:
+ return flask.redirect(flask.url_for('home'))
+ else:
+ flask_login.logout_user()
+ return flask.redirect(flask.url_for('login'))
- while (g.modulesNbr != 0):
- time.sleep(1)
- continue
- # Loading Modules to Dashboard
- layout().load(app_vis, g)
- return "done"
+ @self.flask_app.route('/')
+ def main():
+ """ Handle incoming requests to VS2N and check authentication status
-# Create dash app instance -------------------------------------------
+ Returns:
+ redirection to home if user is authenticated otherwise to login screen
+ """
+ if self.flask_app.config['LOGIN_DISABLED']:
+ return flask.redirect(flask.url_for('home'))
+ else:
+ if not flask_login.current_user.is_authenticated:
+ return render_template('login.html')
+ else:
+ return flask.redirect(flask.url_for('home'))
+
+
+ @self.flask_app.route('/home')
+ @flask_login.login_required
+ def home():
+ """ Get existing modules and check MongoDB availability
+
+ Returns:
+ main screen if everything is ok, otherwise show exception
+ """
+
+ # if MongoDB exist
+ if self.mongo_exists:
+ m = os.listdir(os.path.curdir + "/src/Modules")
+ simulations = []
+ self.g.db = self.g.client.list_database_names()
+
+ # Load existing simulations
+ for bdd in self.g.db:
+ if bdd != "local" and bdd != "admin" and bdd != "config":
+ simulations.append(bdd)
+ i = 1
+
+ # get modules names
+ modules = []
+ for module in m:
+ if ".pyc" not in module and "__" not in module and "General" not in module:
+ modules.append(module.rsplit(".", 1)[0])
+ i = i + 1
+ sparkVersion = self.g.createSparkSession()
+ # return main screen with list of modules and existing simulations
+ return render_template('index.html', modules=modules, Simulations=simulations, mongodb=self.g.client.server_info()["version"], vs2n=self.g.config.VS2N_VERSION, spark=sparkVersion)
+ else:
+ self.mongo_exists = self.g.mongoConnect()
+ return render_template('exception.html')
+
+ @self.flask_app.route('/processVis', methods=['POST'])
+ @flask_login.login_required
+ def processing():
+ """ Start processing and load modules into dashboard
+ """
+ self.g.name = request.get_json()["name"]
+ self.g.modules = request.get_json()["modules"]
+ self.g.updateInterval = float(request.get_json()["updateInterval"])
+
+ self.g.modules.append("General")
+ self.g.modulesNbr = len(self.g.modules)
+
+ self.LoadingModules()
+
+ while (self.g.modulesNbr != 0):
+ time.sleep(1)
+ continue
+
+ # Loading Modules to Dashboard
+ layout().load(self.app_vis, self.g)
+
+ def Module(self, n, g):
+ """ Helper function to execute each module sparks pre processing (if exist)
+
+ Args:
+ n (String): module name
+ g (Global_Var): reference to access global variables
+ """
+ importlib.import_module("src.Modules." + n + ".spark").init(g)
+
+
+ def LoadingModules(self):
+ """ Run spark operation for the selected modules
+ """
+ try:
+ # Spark & MongoDB tasks launch
+ self.g.CreateIndexes(self.g.name)
+ for module in self.g.modules:
+ _thread.start_new_thread(self.Module, (module, self.g,))
-app_vis = dash.Dash(
- __name__,
- requests_pathname_prefix='/vis/',
- external_stylesheets=[dbc.themes.BOOTSTRAP,dbc.icons.FONT_AWESOME],
- suppress_callback_exceptions=True,
- title="VS2N")
+ except Exception as e:
+ print("LoadingModules:" + str(e))
-app_vis.enable_dev_tools(debug=g.config.DEBUG)
-# Start VS2N ---------------------------------------------------------
+ # Start VS2N ---------------------------------------------------------
-def start_VS2N():
- """ Start the virtual server on the web browser .
- Returns:
- [type]: [description]
- """
- print("Starting VS2N")
- app_vis.serve_reload_hash
+ def start_VS2N(self):
+ """ Start the virtual server on the web browser
- webbrowser.open("http://"+url+":"+str(port))
- return app_vis.server
+ Returns:
+ Dash server
+ """
+ print("Starting VS2N")
+ self.app_vis.serve_reload_hash
+ webbrowser.open("http://"+self.url+":"+str(self.port))
+ return self.app_vis.server
-application = DispatcherMiddleware(flask_app, {
- '/vis': start_VS2N()})
+ # Start server -------------------------------------------------------
-# Start server -------------------------------------------------------
if __name__ == '__main__':
try:
- run_simple(url, port, application, use_debugger=g.config.DEBUG)
+ vs2n = VS2N()
+ application = DispatcherMiddleware(vs2n.flask_app, {'/vis': vs2n.start_VS2N()})
+ run_simple(vs2n.url, vs2n.port, application, use_debugger=vs2n.g.config.DEBUG)
except Exception as e:
- traceback.print_exc()
\ No newline at end of file
+ traceback.print_exc()
diff --git a/__init__.py b/__init__.py
new file mode 100755
index 0000000..0519ecb
--- /dev/null
+++ b/__init__.py
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/src/Global_Var.py b/src/Global_Var.py
index 2c0a2d8..747e8cb 100755
--- a/src/Global_Var.py
+++ b/src/Global_Var.py
@@ -1,15 +1,17 @@
""" This class contains global variables and functions.
"""
-from pyspark.sql import SparkSession
-from pymongo import MongoClient
+import socket
+import traceback
from datetime import timedelta
-from pyspark import SparkConf
-from config import config
+
import numpy as np
import pymongo
-import socket
-import traceback
+from pymongo import MongoClient
+from pyspark import SparkConf
+from pyspark.sql import SparkSession
+
+from config import config
class Global_Var():
@@ -32,6 +34,7 @@ class Global_Var():
# General network information ------------------------------------
LayersNeuronsInfo = []
Layer_Neuron = None
+ NeuronsSize = {}
NeuronsNbr = 0
LayersNbr = 0
Dataset = ""
@@ -47,9 +50,8 @@ class Global_Var():
"""
self.stepMax = 0
self.Max = 0
-
self.LayersNeuronsInfo = []
- self.NeuronsSize = {"x":0,"y":0}
+ self.NeuronsSize = {"x": 0, "y": 0}
self.Layer_Neuron = None
self.NeuronsNbr = 0
self.LayersNbr = 0
@@ -82,7 +84,7 @@ class Global_Var():
authMechanism='SCRAM-SHA-1')
self.MONGODBURL = config().MONGODBURL
- self.client.server_info() # will throw an exception if mongodb is not detected
+ self.client.server_info() # will throw an exception if mongodb is not detected
self.db = self.client.list_database_names()
return 1
except Exception:
@@ -128,7 +130,7 @@ class Global_Var():
col.create_index([("i.N", 1)])
if self.config.DEBUG:
print("Spikes index done")
-
+
if ('potential' in self.db.list_collection_names()):
col = pymongo.collection.Collection(self.db, 'potential')
col.create_index([("T", 1)])
@@ -136,7 +138,7 @@ class Global_Var():
col.create_index([("N", 1)])
if self.config.DEBUG:
print("Potential index done")
-
+
if ('synapseWeight' in self.db.list_collection_names()):
col = pymongo.collection.Collection(self.db, 'synapseWeight')
col.create_index([("T", 1)])
@@ -195,27 +197,24 @@ class Global_Var():
heatmap = np.zeros((x, y))
heatmap[:] = -1
data = data.to_numpy()
- if(depth == 0): # single depth
+ if(depth == 0): # single depth
for d in data:
if rotation:
heatmap[int(d[0])][int(d[1])] = d[3]
else:
heatmap[int(d[1])][int(d[0])] = d[3]
- else: # multiple dimensions
+ else: # multiple dimensions
for d in data:
if(d[2] == 0):
- #if(heatmap[int(d[0])][int(d[1])] == -1):
if rotation:
heatmap[int(d[0])][int(d[1])] = d[3]
else:
heatmap[int(d[1])][int(d[0])] = d[3]
- #else:
- # heatmap[int(d[0])][int(d[1])] = np.mean(heatmap[int(d[0])][int(d[1])]+d[3])
return heatmap
-
+
except Exception:
- print("createHeatMap: "+ traceback.format_exc())
+ print("createHeatMap: " + traceback.format_exc())
def createVerticalHeatMap(self, data):
""" Create a vertical heat map from a given data array .
@@ -257,7 +256,7 @@ class Global_Var():
heatMapsZ = [(v * 100)/maxHeatMaps for v in heatMapsZ]
return [heatMapsZ, heatMapsZLabel]
else:
- return [heatMapsZ,heatMapsZ]
+ return [heatMapsZ, heatMapsZ]
def norm(data, Max):
""" Normalize data to be between 0 and 1 .
@@ -286,14 +285,13 @@ class Global_Var():
self.sparkSession = SparkSession.builder.config(conf=conf) \
.config('spark.jars.packages', 'org.mongodb.spark:mongo-spark-connector_2.12:2.4.4') \
.getOrCreate()
-
+
if not self.config.DEBUG:
self.sparkSession.sparkContext.setLogLevel("Error")
return self.sparkSession.version
except Exception:
print("createSparkSession:" + traceback.format_exc())
return ""
-
def testPort(self, port):
"""Test if a given port is currently in use .
diff --git a/src/Modules/General/callbacks.py b/src/Modules/General/callbacks.py
index 61c1ebb..38c1a47 100755
--- a/src/Modules/General/callbacks.py
+++ b/src/Modules/General/callbacks.py
@@ -715,7 +715,7 @@ class callbacks(callbacksOp):
}},
{"$group": {"_id": "$L", "C": {"$sum": 1}, "G": {"$max": "$G"}}},
{"$sort": {"_id": 1}}
- ])
+ ], allowDiskUse = True)
# ToJson ---------------------------------------------
labels = loads(dumps(labels))
@@ -751,12 +751,12 @@ class callbacks(callbacksOp):
spikes = col.aggregate([
{"$match": {"$and": [{"T": {'$gt': timestamp, '$lte': (timestamp+interval)}},{"i.L": {'$in': layer}}]}},
{"$group": {"_id": {"L":"$i.L","N":"$i.N","Input":"$Input"},"spikes": {"$sum":1}}},{"$sort": {"_id": 1}}
- ])
+ ], allowDiskUse = True)
else:
spikes = col.aggregate([
{"$match": {"$and": [{"T": {'$gt': timestamp, '$lte': (timestamp+interval)}},{"i.L": {'$in': layer}}]}},
{"$group": {"_id": "$i.L","spikes": {"$sum":1}}},{"$sort": {"_id": 1}}
- ])
+ ], allowDiskUse = True)
# ToJson----------------------
spikes = loads(dumps(spikes))
@@ -787,7 +787,7 @@ class callbacks(callbacksOp):
synapse = col.aggregate([
{"$match": {"$and": [{"T": {'$gt': timestamp, '$lte': (timestamp+interval)}},{"L": {'$in': layer}}]}},
{"$group": {"_id": "$L","synapseUpdate": {"$sum":1}}},{"$sort": {"_id": 1}}
- ])
+ ], allowDiskUse = True)
# ----------------------------
@@ -816,7 +816,7 @@ class callbacks(callbacksOp):
potential = col.aggregate([
{"$match": {"$and": [{"T": {'$gt': timestamp, '$lte': (timestamp+interval)}},{"L": {'$in': layer}}]}},
{"$group": {"_id": "$L","potential": {"$sum":1}}},{"$sort": {"_id": 1}}
- ])
+ ], allowDiskUse = True)
# ----------------------------
# ToJson----------------------
diff --git a/src/Modules/Synapse/callbacks.py b/src/Modules/Synapse/callbacks.py
index 6eb7026..8e66f73 100755
--- a/src/Modules/Synapse/callbacks.py
+++ b/src/Modules/Synapse/callbacks.py
@@ -679,7 +679,7 @@ class callbacks(callbacksOp):
{"L": {'$in': layer}},
{"To": {'$in': neuron}},
{"T": {'$gt': timestamp, '$lte': (timestamp+g.updateInterval)}} ]}
- }])
+ }], allowDiskUse = True)
# ToDF----------------------
SynapseWeight = pd.DataFrame(list(SynapseWeight))
@@ -703,7 +703,7 @@ class callbacks(callbacksOp):
# MongoDB---------------------
col = pymongo.collection.Collection(g.db, 'synapseWeightFinal')
- globalSynapseWeights = col.aggregate([{"$match": { "_id.L": {"$eq": layer}}}])
+ globalSynapseWeights = col.aggregate([{"$match": { "_id.L": {"$eq": layer}}}], allowDiskUse = True)
# ToDF----------------------
globalSynapseWeights = pd.DataFrame(list(globalSynapseWeights))
diff --git a/src/Modules/Synapse/spark.py b/src/Modules/Synapse/spark.py
index 89cce7a..c6de5e9 100755
--- a/src/Modules/Synapse/spark.py
+++ b/src/Modules/Synapse/spark.py
@@ -34,7 +34,7 @@ class spark(sparkOp):
self.g.createSparkSession()
# --------------------------------------------------
col = pymongo.collection.Collection(self.g.db, self.DOCUMENT_NAME)
- globalSynapseWeights = col.aggregate([{ "$sort": { "T": 1 } },{"$group" : { "_id" : {"To":'$To', "C":'$C', "index":'$index', "L":'$L'}, "T" : { "$last": '$T'},"V" : { "$last": '$V'} } }])
+ globalSynapseWeights = col.aggregate([{ "$sort": { "T": 1 } },{"$group" : { "_id" : {"To":'$To', "C":'$C', "index":'$index', "L":'$L'}, "T" : { "$last": '$T'},"V" : { "$last": '$V'} } }], allowDiskUse = True)
# Data save into MongoDB ---------------------------------
col = pymongo.collection.Collection(self.g.db, self.OUTPUT_DOCUMENT_NAME)
@@ -59,7 +59,7 @@ class spark(sparkOp):
# get dimensions for the heatmap
col = pymongo.collection.Collection(self.g.db, 'synapseWeightFinal')
- globalSynapseWeights = pd.DataFrame(list(col.aggregate([{"$group": {"_id":None, "x":{"$max":"$_id.index.x"}, "y":{"$max":"$_id.index.y"}}}])))
+ globalSynapseWeights = pd.DataFrame(list(col.aggregate([{"$group": {"_id":None, "x":{"$max":"$_id.index.x"}, "y":{"$max":"$_id.index.y"}}}], allowDiskUse = True)))
self.g.NeuronsSize = {"x":globalSynapseWeights["x"].max(),"y":globalSynapseWeights["y"].max()}
if self.g.config.DEBUG:
diff --git a/tests/Modules/test_General_callbacks.py b/tests/Modules/test_General_callbacks.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/Modules/test_General_layout.py b/tests/Modules/test_General_layout.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/Modules/test_General_spark.py b/tests/Modules/test_General_spark.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/Modules/test_Neuron_callbacks.py b/tests/Modules/test_Neuron_callbacks.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/Modules/test_Neuron_layout.py b/tests/Modules/test_Neuron_layout.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/Modules/test_Neuron_spark.py b/tests/Modules/test_Neuron_spark.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/Modules/test_Synapse_callbacks.py b/tests/Modules/test_Synapse_callbacks.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/Modules/test_Synapse_layout.py b/tests/Modules/test_Synapse_layout.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/Modules/test_Synapse_spark.py b/tests/Modules/test_Synapse_spark.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100755
index 0000000..0519ecb
--- /dev/null
+++ b/tests/__init__.py
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..0243da4
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,24 @@
+import pytest
+from VS2N import flask_app
+
+@pytest.fixture
+def flask_app():
+ yield flask_app
+
+#@pytest.fixture
+#def dashApp():
+# # Init dash app
+# app_vis = dash.Dash(
+# __name__,
+# requests_pathname_prefix='/vis/',
+# external_stylesheets=[VS2N.dbc.themes.BOOTSTRAP, VS2N.dbc.icons.FONT_AWESOME],
+# suppress_callback_exceptions=True,
+# title="VS2N")
+#
+# app_vis.enable_dev_tools(debug=VS2N.g.config.DEBUG)
+#
+# # Loading modules
+#
+# request = {'name': 'csnn-23-3-2023-16:32:48', 'modules': ['Neuron', 'Synapse'], 'updateInterval': '1'}
+#
+# yield app_vis
\ No newline at end of file
diff --git a/tests/test_Global_var.py b/tests/test_Global_var.py
new file mode 100644
index 0000000..214fecb
--- /dev/null
+++ b/tests/test_Global_var.py
@@ -0,0 +1,4 @@
+from flask import Flask
+
+def test_test(flask_app: Flask):
+ assert isinstance(flask_app,Flask)
\ No newline at end of file
--
GitLab