SQL optimisation

lib/include/IOptimizer.h

@@ -16,6 +16,7 @@ namespace mdd {
 			virtual bool setEvaluation(std::string func) = 0;
 			virtual double update() = 0;
 			virtual void attachCallback(std::function<json(const json&)> callback) = 0;
+			virtual json getPermutations() = 0;
 	};
 }
 #endif // !IOPTIMIZER_H

lib/include/OptimizerBase.h

@@ -2,6 +2,8 @@
 #include "IOptimizer.h"
 #include "exprtk.hpp"
 #include <thread>
+#include <sqlite3.h> 
+#include <mutex>
 
 namespace mdd {
 	////////////////////////////////////////////////////////////
@@ -13,6 +15,7 @@ namespace mdd {
 		: public IOptimizer 
 	{
 	protected:
+		std::mutex _mutex;
 		json _base_config;
 		std::string key;
 		const std::string type = "optimizer";
@@ -36,53 +39,17 @@ namespace mdd {
 		OptimizerBase(const std::string& base_config = "{}");
 
 		bool configureChild(const json& config) override;
-
 	public:
 		struct Permutation {
 		public:
 			std::vector<std::vector<double>> dna;
 			double fitness = 0;
-			bool operator== (const Permutation& ind) {
-				return (dna == ind.dna);
-			};
-
-			static bool same(Permutation const& a, Permutation const& b) {
-				bool check1 = (a.dna == b.dna);
-				bool check2 = true;
-				if (a.dna.size() != b.dna.size())
-				{
-					check2 = false;
-				}
-				else {
-					for (size_t i = 0; i < a.dna.size(); i++)
-					{
-						if (a.dna[i].size() != b.dna[i].size())
-						{
-							check2 = false;
-						}
-						else
-						{
-							for (size_t j = 0; j < a.dna[i].size(); j++)
-							{
-								if (a.dna[i][j] != b.dna[i][j])
-								{
-									check2 = false;
-								}
-							}
-						}
-						
-					}
-				}
-				if (check1 != check2)
-				{
-					int test = 0;
-				}
-				return check1;
-			}
-
-			static bool smaller(Permutation const& a, Permutation const& b) {
-				return (a.dna < b.dna);
-			}
+			double time = 0;
+			state status = state::UNCHANGED;
+
+			bool operator== (const Permutation& ind);
+			static bool same(Permutation const& a, Permutation const& b);
+			static bool smaller(Permutation const& a, Permutation const& b);
 		};
 
 		bool connect(std::shared_ptr<IModule> module) override;
@@ -99,8 +66,45 @@ namespace mdd {
 		void attachCallback(std::function<json(const json&)> callback) override;
 		//state update() override;
 		void callback_add_step(const std::vector<Permutation>& pers);
-		void callback_evaluate_opt(const opt_state& state, const Permutation& per);
-
+		void callback_evaluate_opt(const Permutation& per);
+		void callback_reset();
 		~OptimizerBase();
+
+	private:
+		bool exec_sql(const std::string& sql);
+		json exec_sql_with_callback( const std::string& sql);
+
+		json _jcall;
+
+	protected:
+		std::string _db_path;
+		json Permutation_to_JSON(const Permutation& per);
+		Permutation JSON_to_Permutation(const json& j);
+
+		bool createDB(const std::string& db_path);
+		bool clearDB();
+		virtual bool loadDB(const json& j);
+		virtual bool loadStepDB(const json& j);
+		virtual bool loadPermutationDB(const json& j);
+
+		json dumpPermutations();
+		json dumpSteps();
+
+		bool addPermutation(const Permutation& per);
+		bool addStep(int step, json dna);
+		bool changePermutation(const Permutation& per);
+
+		virtual void reset();
+		virtual bool reachAbort(const std::vector<Permutation>& pers);
+
+		
+
+	public:
+		std::shared_ptr<Permutation> findPermutation(std::vector<std::vector<double>> dna);
+		std::shared_ptr<Permutation> getPermutation(int step, std::vector<std::vector<double>> dna);
+		json getPermutations() override;
+		std::vector<Permutation> findBest(int step);
+
+		int sql_callback(int argc, char** argv, char** azColName);
 	};
 }

lib/include/OptimizerEvolutionary.h

@@ -13,21 +13,30 @@ namespace mdd {
 		}
 
 		Permutation generatePermutation();
-		Permutation combine(Permutation par1, Permutation par2);
+		Permutation combine(Permutation par1, Permutation par2, int try_counter = 0);
 
 		std::vector<Permutation> _children;
 		std::vector<Permutation> _bests;
+		size_t _same_counter = 0;
+		Permutation _old_best;
+
 		int _min_generations;
 		double _max_fitness;
 		size_t _grow_generation;
 		size_t _converges;
 		double _precision;
 
-		void evolve(std::vector<Permutation> parents);
+		void evolve();
 		void evaluate(size_t ignore_len = 0);
 		std::vector<double> mutateGene(std::shared_ptr<IInput> input, std::vector<double> seed = std::vector<double>());
 		bool configureChild(const json& config) override;
 
+		bool loadStepDB(const json& j) override;
+		bool loadDB(const json& j) override;
+
+		void reset() override;
+		bool reachAbort(const std::vector<Permutation>& pers) override;
+
 	public:
 		OptimizerEvolutionary();
 		double update() override;

lib/include/state.h

@@ -4,8 +4,8 @@
 namespace mdd {
     enum class state : char {
         STATE_ERROR = -1,
-        CHANGED = 0,
-        UNCHANGED = 1
+        UNCHANGED = 0,
+        CHANGED = 1
     };
 }
 #endif

lib/src/Connector.cpp

@@ -441,7 +441,8 @@ namespace mdd {
 	json  Connector::encode()
 	{
 		json ret;
-		ret = _root->dump();
+		ret["processor"] = _root->dump();
+		ret["optimizer"]["permutations"] = _opt->getPermutations();
 		std::cout << "[Connector]: encode: " << ret.dump() << std::endl;
 		return ret;
 	}

lib/src/OptimizerBase.cpp

@@ -1,7 +1,53 @@
 #include "OptimizerBase.h"
 #include <iostream>
+#include <filesystem>
+#include <stdio.h>
+#include <stdlib.h>
 
 namespace mdd{
+	bool OptimizerBase::Permutation::operator== (const Permutation& ind) {
+		return (dna == ind.dna);
+	}
+
+	bool OptimizerBase::Permutation::same(Permutation const& a, Permutation const& b) {
+		bool check1 = (a.dna == b.dna);
+		bool check2 = true;
+		if (a.dna.size() != b.dna.size())
+		{
+			check2 = false;
+		}
+		else {
+			for (size_t i = 0; i < a.dna.size(); i++)
+			{
+				if (a.dna[i].size() != b.dna[i].size())
+				{
+					check2 = false;
+				}
+				else
+				{
+					for (size_t j = 0; j < a.dna[i].size(); j++)
+					{
+						if (a.dna[i][j] != b.dna[i][j])
+						{
+							check2 = false;
+						}
+					}
+				}
+
+			}
+		}
+		if (check1 != check2)
+		{
+			int test = 0;
+		}
+		return check1;
+	}
+
+	bool OptimizerBase::Permutation::smaller(Permutation const& a, Permutation const& b) {
+		return (a.dna < b.dna);
+	}
+
+
 	bool OptimizerBase::connect(std::shared_ptr<IModule> module) {
 		_module = module;
 		return true;
@@ -99,14 +145,20 @@ namespace mdd{
 				std::string key = (*jstate_it)["value"].get<std::string>();
 				if (key == "run")
 				{
+					if (_base_config["optimizer"]["state"]["value"] == "stop")
+					{
+						reset();
+					}
+					
+					_base_config["optimizer"]["state"]["value"] = "run";
 					updateLayout();
 					_thread_ptr = std::make_shared<std::thread>(&OptimizerBase::update,this);
 				}
 				else if (key == "pause") {
-
+					_base_config["optimizer"]["state"]["value"] = "pause";
 				}
 				else {//stop
-
+					_base_config["optimizer"]["state"]["value"] = "stop";
 				}
 			}
 			jstate_it = jit->find("evaluate");
@@ -135,12 +187,27 @@ namespace mdd{
 	//virtual std::string getGeneratorID() = 0;
 	void OptimizerBase::load(const json& j) {
 		configure(j);
+		auto it = j.find("db");
+		if (it != j.end())
+		{
+			createDB(it->get<std::string>());
+		}
+		else {
+			createDB("C:/Users/Willi/Documents/projects/mdd/lib/test/db/optimization.db");
+		}
+		
+		loadDB(j);
 	}
 	json OptimizerBase::dump() {
 		json ret = _base_config;
 		ret["ID"] = getIdentifier();
 		ret["type"] = type;
 		ret["key"] = key;
+
+		ret["db"] = "C:/Users/Willi/Documents/projects/mdd/lib/test/db/optimization.db";
+
+		ret["permutations"] = dumpPermutations();
+		ret["steps"] = dumpSteps();
 		return ret;
 	}
 
@@ -162,13 +229,15 @@ namespace mdd{
 	void OptimizerBase::callback_add_step(const std::vector<Permutation>& pers){
 		json ret;
 		ret["step"] = step;
-		for (auto& ind : pers)
+		_mutex.lock();
+		for (int i = 0; i < pers.size(); ++i)
 		{
-			json jind;
-			jind["fitness"] = ind.fitness;
-			jind["dna"] = ind.dna;
-			ret["permutations"].push_back(jind);
+			auto ind = pers[i];
+			addPermutation(ind);
+			addStep(step, ind.dna);
+			ret["permutations"].push_back(Permutation_to_JSON(ind));
 		}
+		_mutex.unlock();
 		json jcall;
 		jcall["operation"] = "add";
 		jcall["args"]["subject"] = getIdentifier();
@@ -176,20 +245,12 @@ namespace mdd{
 		_callback(jcall);
 	}
 
-	void OptimizerBase::callback_evaluate_opt(const OptimizerBase::opt_state& state, const Permutation& per) {
+	void OptimizerBase::callback_evaluate_opt(const Permutation& per) {
 		json ret;
 		ret["step"] = step;
-		json jind;
-		jind["fitness"] = state.opt_value;
-		jind["dna"] = per.dna;
-		if (state.module_state == state::STATE_ERROR)
-		{
-			jind["status"] = "error";
-		}
-		else {
-			jind["status"] = "ok";
-		}
-		jind["processor"] = _module->dump();
+		changePermutation(per);
+		json jind = Permutation_to_JSON(per);
+		//jind["processor"] = _module->dump();
 		ret["permutation"] = jind;
 		json jcall;
 		jcall["operation"] = "change";
@@ -198,10 +259,356 @@ namespace mdd{
 		_callback(jcall);
 	}
 
+	void OptimizerBase::callback_reset() {
+		json ret;
+		ret["step"] = step;
+		
+		json jcall;
+		jcall["operation"] = "reset";
+		jcall["args"]["subject"] = getIdentifier();
+		jcall["args"]["object"] = ret;
+		_callback(jcall);
+	}
+
+	void OptimizerBase::reset(){
+		step = -1;
+		callback_reset();
+	}
+
+	bool OptimizerBase::reachAbort(const std::vector<Permutation>& pers) {
+		return true;
+	}
+
 	OptimizerBase::~OptimizerBase() {
 		if (_thread_ptr != nullptr)
 		{
 			_thread_ptr->join();
 		}
 	}
+
+	int sql_callback(void* data, int argc, char** argv, char** azColName) {
+		static_cast<OptimizerBase*>(data)->sql_callback(argc, argv, azColName);
+		return 0;
+	}
+
+	bool OptimizerBase::exec_sql(const std::string& sql) {
+		char* zErrMsg = 0;
+		int rc;
+		sqlite3* _db;
+
+		// Open database
+		//if (std::filesystem::exists(_db_path)) {
+		//	sqlite3_open(_db_path.c_str(), &_db);
+		//	return true;
+		//}
+		rc = sqlite3_open(_db_path.c_str(), &_db);
+
+		if (rc) {
+			fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(_db));
+			return false;
+		}
+		else {
+			//fprintf(stdout, "Opened database successfully\n");
+		}
+
+
+		// Execute SQL statement
+		rc = sqlite3_exec(_db, sql.c_str(), mdd::sql_callback, this, &zErrMsg);
+
+		if (rc != SQLITE_OK) {
+			fprintf(stderr, "SQL error: %s\n", zErrMsg);
+			sqlite3_free(zErrMsg);
+			sqlite3_close(_db);
+			return false;
+		}
+
+		sqlite3_close(_db);
+
+		return true;
+	}
+
+	json OptimizerBase::exec_sql_with_callback(const std::string& sql) {
+		char* zErrMsg = 0;
+		int rc;
+		sqlite3* _db;
+		_jcall = json::basic_json();
+
+		// Open database
+		rc = sqlite3_open(_db_path.c_str(), &_db);
+
+		if (rc) {
+			fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(_db));
+		}
+		else {
+			//fprintf(stdout, "Opened database successfully\n");
+		}
+		
+		// Execute SQL statement
+		rc = sqlite3_exec(_db, sql.c_str(), mdd::sql_callback, this, &zErrMsg);
+
+		if (rc != SQLITE_OK) {
+			fprintf(stderr, "SQL error: %s\n", zErrMsg);
+			sqlite3_free(zErrMsg);
+			sqlite3_close(_db);
+			return json::basic_json();
+		}
+		sqlite3_close(_db);
+
+		return _jcall;
+	}
+
+	int OptimizerBase::sql_callback(int argc, char** argv, char** azColName) {
+		json jelem;
+		for (size_t i = 0; i < argc; i++)
+		{
+			jelem[azColName[i]] = std::string((argv[i] ? argv[i] : "NULL"));
+		}
+		_jcall.push_back(jelem);
+		//printf(jelem.dump().c_str());
+		///TODO test
+		return 0;
+	}
+
+	json OptimizerBase::Permutation_to_JSON(const Permutation& per) {
+		json ret;
+		ret["dna"] = per.dna;
+		ret["fitness"] = per.fitness;
+		ret["status"] = per.status;
+		ret["time"] = per.time;
+		return ret;
+	}
+
+	OptimizerBase::Permutation OptimizerBase::JSON_to_Permutation(const json& j) {
+		Permutation per;
+		json jelem;
+		if (j.is_array())
+		{
+			jelem = *j.begin();
+		}
+		else {
+			jelem = j;
+		}
+		auto ittest = jelem.find("dna");
+		if (ittest == jelem.end())
+		{
+			int test = 0;
+		}
+		json jval = json::parse(jelem["dna"].get<std::string>());
+		per.dna = jval.get<std::vector<std::vector<double>>>();
+
+		jval = json::parse(jelem["fitness"].get<std::string>());
+		per.fitness = jval.get<double>();
+		auto it = jelem.find("status");
+		if (it != jelem.end())
+		{
+			jval = json::parse(it->get<std::string>());
+			per.status = jval.get<state>();
+		}
+		it = jelem.find("time");
+		if (it != jelem.end())
+		{
+			jval = json::parse(it->get<std::string>());
+			per.time = jval.get<double>();
+		}
+		
+		return per;
+	}
+
+	bool OptimizerBase::createDB(const std::string& db_path) {
+		_db_path = db_path;
+		if (std::filesystem::exists(_db_path)) {
+			//return clearDB(db_path);
+			return true;
+		}
+		sqlite3* _db;
+		int rc;
+		char* sql;
+
+		bool ret;
+
+		// Create SQL statement
+		sql = "CREATE TABLE PERMUTATIONS("  \
+			"dna		TEXT	NOT NULL	PRIMARY KEY," \
+			"fitness	FLOAT," \
+			"time		FLOAT," \
+			"status		TEXT" \
+			");";
+
+		ret = exec_sql(sql);
+
+		// Create SQL statement
+		sql = "CREATE TABLE STEPS("  \
+			"step		INTEGER		NOT NULL," \
+			"dna		TEXT		NOT NULL," \
+			"CONSTRAINT PK_PERMUTATIONS PRIMARY KEY (step, dna)"
+			");";
+
+		ret = ret && exec_sql(sql);
+		return ret;
+	}
+
+	bool OptimizerBase::clearDB() {
+
+		std::string sql;
+		sql = "DELETE FROM PERMUTATIONS;";
+		exec_sql(sql);
+		sql = "DELETE FROM STEPS;";
+		return exec_sql(sql);
+	}
+
+	bool OptimizerBase::loadDB(const json& j) {
+		bool ret = true;
+		bool found = false;
+		clearDB();
+		auto it = j.find("permutations");
+		if (it != j.end())
+		{
+			found = true;
+			ret = ret && loadPermutationDB(*it);
+		}
+
+		it = j.find("steps");
+		if (it != j.end())
+		{
+			found = found && true;
+			ret = ret && loadStepDB(*it);
+		}
+		step = -1;
+		if (found)
+		{
+			std::vector<Permutation> bests;
+			do
+			{
+				++step;
+				bests = findBest(step);
+				if (bests.empty())
+				{
+					--step;
+					break;
+				}
+			} while (!reachAbort(bests));
+		}
+
+		return ret && found;
+	}
+	bool OptimizerBase::loadStepDB(const json& j) {
+		bool ret = true;
+		for (size_t i = 0; i < j.size(); i++)
+		{
+			json jstep = j.at(i);
+			ret = ret && addStep(json::parse(jstep["step"].get<std::string>()).get<int>(), json::parse(jstep["dna"].get<std::string>()));
+		}
+		return ret;
+	}
+
+	bool OptimizerBase::loadPermutationDB(const json& j){
+		bool ret = true;
+		for (size_t i = 0; i < j.size(); i++)
+		{
+			ret = ret && addPermutation(JSON_to_Permutation(j.at(i)));
+		}
+		return ret;
+	}
+	
+
+	json OptimizerBase::dumpPermutations() {
+		std::string sql = "SELECT * FROM PERMUTATIONS;";
+		json ret = exec_sql_with_callback(sql);
+		return ret;
+	}
+
+	json OptimizerBase::dumpSteps() {
+		std::string sql = "SELECT * FROM STEPS;";
+		json ret = exec_sql_with_callback(sql);
+		return ret;
+	}
+
+	bool OptimizerBase::addPermutation(const Permutation& per) {
+		// Create SQL statement
+		json jdna;
+		jdna = per.dna;
+		std::string sval =  "'" + jdna.dump() + "', " + std::to_string(per.fitness)+ ", " + std::to_string(per.time) + ", " + std::to_string((char)(per.status));
+		std::string sql;
+		sql =	"INSERT INTO PERMUTATIONS (dna, fitness, time, status) "\
+				"VALUES (" + sval + "); ";
+
+		return exec_sql(sql);
+	}
+
+	bool OptimizerBase::addStep(int step, json dna) {
+		std::string  sql = "INSERT INTO STEPS (step, dna) "\
+			"VALUES (" + std::to_string(step) + ", '" + dna.dump() + "');";
+
+		return exec_sql(sql);
+	}
+
+	bool OptimizerBase::changePermutation(const Permutation& per) {
+		json jdna;
+		jdna = per.dna;
+		std::string sql;
+		sql =	"UPDATE PERMUTATIONS "\
+				"SET  fitness = "+ std::to_string(per.fitness)+ ", time = "+ std::to_string(per.time) + ", status = " + std::to_string((char)(per.status)) + " "\
+				"WHERE dna = '" + jdna.dump() + "';";
+
+		return exec_sql(sql);
+	}
+
+	std::shared_ptr<OptimizerBase::Permutation> OptimizerBase::findPermutation(std::vector<std::vector<double>> dna) {
+		json jdna;
+		jdna = dna;
+		std::string sql =	"SELECT * FROM PERMUTATIONS "\
+							"WHERE dna = '" + jdna.dump() + "' AND status = " + std::to_string((char)(state::CHANGED)) + "; ";
+		json ret;
+		ret = exec_sql_with_callback(sql);
+		if (ret != nullptr)
+		{
+			return std::make_shared<Permutation>(JSON_to_Permutation(ret));
+		}
+		return ret;
+	}
+
+	std::shared_ptr<OptimizerBase::Permutation> OptimizerBase::getPermutation(int step, std::vector<std::vector<double>> dna) {
+		json jdna;
+		jdna = dna;
+		std::string sql =	"SELECT p.dna, p.fitness, p.time, p.status FROM PERMUTATIONS AS p "\
+							"LEFT JOIN STEPS AS s ON s.dna = p.dna "\
+							"WHERE s.step = " + std::to_string(step) + " AND s.dna = '" + jdna.dump() + "';";
+		json ret = exec_sql_with_callback(sql);
+		if (ret != nullptr)
+		{
+			return std::make_shared<Permutation>(JSON_to_Permutation(ret));
+		}
+		return nullptr;
+	}
+
+	json OptimizerBase::getPermutations() {
+		std::string sql;
+		sql = "SELECT s.step, p.dna, p.fitness, p.time, p.status FROM PERMUTATIONS AS p "\
+			"LEFT JOIN STEPS AS s ON s.dna = p.dna; ";
+		json ret = exec_sql_with_callback(sql);
+
+		return ret;
+	}
+
+	std::vector<OptimizerBase::Permutation> OptimizerBase::findBest(int step) {
+		std::string sql = "SELECT * FROM PERMUTATIONS "\
+			"WHERE dna = ("\
+			"SELECT p.dna FROM PERMUTATIONS AS p "\
+			"LEFT JOIN STEPS AS s ON s.dna = p.dna "\
+			"WHERE s.step = " + std::to_string(step) +" AND ROUND(p.fitness,3) = ( "\
+			"SELECT MIN(ROUND(p.fitness,3)) FROM PERMUTATIONS AS p "\
+			"LEFT JOIN STEPS AS s ON s.dna = p.dna "\
+			"WHERE s.step = " + std::to_string(step)+"));";
+		json ret = exec_sql_with_callback(sql);
+		std::vector<Permutation> vec;
+		if (ret.is_array())
+		{
+			for (size_t i = 0; i < ret.size(); i++)
+			{
+				vec.push_back(JSON_to_Permutation(ret[i]));
+			}
+			
+		}
+		return vec;
+	}
 }

lib/src/OptimizerEvolutionary.cpp

@@ -3,6 +3,7 @@
 namespace mdd {
 	void OptimizerEvolutionary::evaluate(size_t ignore_len)
 	{
+		_mutex.lock();
 		//clear duplicates
 		//std::sort(_children.begin(), _children.end(), Permutation::smaller);
 		//_children.erase(std::unique(_children.begin(), _children.end(), Permutation::same), _children.end());
@@ -25,30 +26,43 @@ namespace mdd {
 			}
 		}
 
-		//
+		//print Modules which are not evaluated again
 		opt_state def_state;
 		def_state.module_state = state::UNCHANGED;
 		for (int i = 0; i < ignore_len; ++i)
 		{
-			def_state.opt_value = _children[i].fitness;
-			callback_evaluate_opt(def_state, _children[i]);
+			callback_evaluate_opt(_children[i]);
 		}
 
 		//calculate fitness
 		for (auto it = _children.begin() + ignore_len; it != _children.end(); ++it)
 		{
-			for (size_t j = 0; j < _inputs.size(); j++)
+			auto per = findPermutation(it->dna);
+			if (per == nullptr)
 			{
-				//std::cout << "set: " << j << ": " << it->dna[j] << std::endl;
-				_inputs[j]->setValue() = it->dna[j];
-			}
-			opt_state opt = updateOutputs();
-			callback_evaluate_opt(opt, *it);
-			if (opt.module_state == state::STATE_ERROR) {
-				_children.erase(it);
-			}
-			else {
+				for (size_t j = 0; j < _inputs.size(); j++)
+				{
+					//std::cout << "set: " << j << ": " << it->dna[j] << std::endl;
+					_inputs[j]->setValue() = it->dna[j];
+				}
+				// Start timer
+				auto start = std::chrono::steady_clock::now();
+				// Update Module
+				opt_state opt = updateOutputs();
+				// Stop timer
+				auto end = std::chrono::steady_clock::now();
+				std::chrono::duration<double> elapsed_seconds = end - start;
+				it->time = elapsed_seconds.count();
+				it->status = opt.module_state;
 				it->fitness = opt.opt_value;
+				callback_evaluate_opt(*it);
+
+				if (opt.module_state == state::STATE_ERROR) {
+					_children.erase(it);
+				}
+			}
+			else{
+				callback_evaluate_opt(*per);
 			}
 		}
 		//find fitest
@@ -79,21 +93,7 @@ namespace mdd {
 				}
 			}
 		}
-		
-		// json ret;
-		// ret["step"] = step;
-		// for (auto& ind : _bests)
-		// {
-		// 	json jind;
-		// 	jind["fitness"] = ind.fitness;
-		// 	jind["dna"] = ind.dna;
-		// 	ret["bests"].push_back(jind);
-		// }
-		// json jcall;
-		// jcall["operation"] = "add";
-		// jcall["args"]["subject"] = getIdentifier();
-		// jcall["args"]["object"] = ret;
-		// _callback(jcall);
+		_mutex.unlock();
 	}
 
 	OptimizerEvolutionary::OptimizerEvolutionary()
@@ -194,62 +194,92 @@ namespace mdd {
 		return found;
 	}
 
-	double OptimizerEvolutionary::update()
-	{
-		step = -1;
-		bool check;
-		Permutation old_best;
-		size_t same_counter = 0;
-		if (_inputs.size() == 0)
+	bool OptimizerEvolutionary::loadStepDB(const json& j) {
+		bool ret = true;
+		for (size_t i = 0; i < j.size(); i++)
 		{
-			std::cout << "ERROR: No optimizable inputs detected!" << std::endl;
-			return -1.0;
+			json jstep = j.at(i);
+			int counter = json::parse(jstep["step"].get<std::string>()).get<int>();
+			json dna = json::parse(jstep["dna"].get<std::string>());
+			ret = ret && addStep(counter, dna);
+			if (this->step != counter)
+			{
+				++this->step;
+				_children.clear();
+			}
+			std::shared_ptr<Permutation> per_ptr = getPermutation(counter, dna);
+			_children.push_back(*per_ptr);
 		}
+		return ret;
+	}
+
+	bool OptimizerEvolutionary::loadDB(const json& j) {
+		bool ret = OptimizerBase::loadDB(j);
+		if (step == -1)
+		{
+			return ret;
+		}
+
+		_bests = findBest(step -1);
+
+		return ret;
+	}
+
+	void OptimizerEvolutionary::reset() {
+		OptimizerBase::reset();
 		_children.clear();
 		_bests.clear();
-		do
+		_same_counter = 0;
+		_old_best = Permutation();
+	}
+
+	bool OptimizerEvolutionary::reachAbort(const std::vector<Permutation>& pers) {
+		bool check;
+		check = step < _min_generations || pers[0].fitness > _max_fitness;
+		if (!check && _converges > 0)
 		{
-			++step;
-			std::cout << _children.size() << " | " << step << std::endl;
-			if (_children.empty())
+			bool found = false;
+			for (size_t i = 0; i < pers.size(); i++)
 			{
-				for (size_t i = 0; i < _grow_generation*2; i++)
+				Permutation p_container = pers.at(i);
+				if (p_container == _old_best)
 				{
-					_children.push_back(generatePermutation());
+					found = true;
+					break;
 				}
-				callback_add_step(_children);
-				evaluate();
 			}
-			else {
-				evolve(_children);
+			if (found)
+			{
+				++_same_counter;
 			}
-			check = step < _min_generations || _bests[0].fitness > _max_fitness;
-			if (!check && _converges > 0)
+			else
 			{
-				bool found = false;
-				for (size_t i = 0; i < _bests.size(); i++)
-				{
-					if (_bests[i] == old_best)
-					{
-						found = true;
-						break;
-					}
-				}
-				if (found)
-				{
-					++same_counter;
-				}
-				else
-				{
-					old_best = _bests[0];
-					same_counter = 0;
-				}
-				if (same_counter < _converges)
-				{
-					check = true;
-				}
+				_old_best = pers[0];
+				_same_counter = 0;
+			}
+			if (_same_counter < _converges)
+			{
+				check = true;
 			}
-		} while (check);
+		}
+		return !check;
+	}
+
+	double OptimizerEvolutionary::update()
+	{
+
+		if (_inputs.size() == 0)
+		{
+			std::cout << "ERROR: No optimizable inputs detected!" << std::endl;
+			return -1.0;
+		}
+		do
+		{
+			++step;
+			std::cout << _children.size() << " | " << step << std::endl;
+			evolve();
+			
+		} while (!reachAbort(_bests));
 		
 		return _bests[0].fitness;
 	}
@@ -340,65 +370,101 @@ namespace mdd {
 		return input->getValue();//ERROR
 	}
 
-	void OptimizerEvolutionary::evolve(std::vector<Permutation> parents)
+	void OptimizerEvolutionary::evolve()
 	{
-		////fittest should breed more
-		//detect lower border
-		double max = parents[0].fitness;
-		for (size_t i = 1; i < parents.size(); i++)
+		_mutex.lock();
+		if (_children.empty())
 		{
-			if (max < parents[i].fitness) {
-				max = parents[i].fitness;
+			for (size_t i = 0; i < _grow_generation * 2; i++)
+			{
+				bool not_found = true;
+				Permutation per;
+				while (not_found)
+				{
+					per = generatePermutation();
+					if (getPermutation(step, per.dna) == nullptr)
+					{
+						not_found = false;
+					}
+				}
+				if (_children.size() != i)
+				{
+					int test = 0;
+				}
+				_children.push_back(per);
 			}
 		}
-		//detect lower border
-		double sum = 0;
-		for (size_t i = 0; i < parents.size(); i++)
-		{
-			sum += max - parents[i].fitness;
-		}
-		if (sum == 0)
-		{
-			sum = 1.0;
-		}
-		//multiply fitter parents
-		std::vector<Permutation> gen_pool = parents;
-		for (size_t i = 0; i < parents.size(); i++)
-		{
-			int additional = std::round((max - parents[i].fitness) / sum * 20);
-			//std::cout << additional << std::endl;
-			for (size_t j = 1; j < additional; j++)
+		else {
+			////fittest should breed more
+			//detect lower border
+			double max = _children[0].fitness;
+			for (size_t i = 1; i < _children.size(); i++)
 			{
-				gen_pool.push_back(parents[i]);
+				if (max < _children[i].fitness) {
+					max = _children[i].fitness;
+				}
+			}
+			//detect lower border
+			double sum = 0;
+			for (size_t i = 0; i < _children.size(); i++)
+			{
+				sum += max - _children[i].fitness;
+			}
+			if (sum == 0)
+			{
+				sum = 1.0;
+			}
+			//multiply fitter parents
+			std::vector<Permutation> gen_pool = _children;
+			for (size_t i = 0; i < _children.size(); i++)
+			{
+				int additional = std::round((max - _children[i].fitness) / sum * 20);
+				//std::cout << additional << std::endl;
+				for (size_t j = 1; j < additional; j++)
+				{
+					gen_pool.push_back(_children[i]);
+				}
+			}
+			//fittest parents will also be new childrens
+			_children = _bests;
+
+			//breed
+			for (size_t i = 0; i < _grow_generation; i++)
+			{
+				bool not_found = true;
+				Permutation per;
+				int try_counter = 0;
+				while (not_found && try_counter < 20)
+				{
+					int p1 = (int)random_num(0, gen_pool.size() - 1);
+					int p2 = (int)random_num(0, gen_pool.size() - 1);
+					per = combine(gen_pool[p1], gen_pool[p2]);
+					++try_counter;
+					if (getPermutation(step, per.dna) == nullptr)
+					{
+						not_found = false;
+					}
+				}
+				_children.push_back(per);
 			}
 		}
-		//fittest parents will also be new childrens
-		_children = _bests;
-		
-		//breed
-		size_t init_len = _children.size();
-		for (size_t i = 0; i < _grow_generation; i++)
-		{
-			int p1 = (int)random_num(0, gen_pool.size() - 1);
-			int p2 = (int)random_num(0, gen_pool.size() - 1);
-			_children.push_back(combine(gen_pool[p1], gen_pool[p2]));
-		}
+		_mutex.unlock();
 		callback_add_step(_children);
-		evaluate(init_len);
+		evaluate(_bests.size());
 	}
 
-	OptimizerEvolutionary::Permutation OptimizerEvolutionary::combine(Permutation par1, Permutation par2)
+	OptimizerEvolutionary::Permutation OptimizerEvolutionary::combine(Permutation par1, Permutation par2, int try_counter)
 	{
 		size_t len = par1.dna.size();
 		Permutation child;
 		for (size_t i = 0; i < len; i++)
 		{
 			double p = random_num(0.0, 100.0, 1.0);
-			if (p<45.0)
+			if (p<45.0 - try_counter)
 			{
 				child.dna.push_back(par1.dna[i]);
 			}
-			else if(p<90.0){
+			else if(p<90.0 - 2*try_counter){
 				child.dna.push_back(par2.dna[i]);
 			}
 			else