Initial commit

src/WorkflowRuntime.cpp

@@ -0,0 +1,74 @@
+
+#include <wfr/WorkflowRuntime.h>
+#include <wfr/common/wfs/Workflow.h>
+#include <wfr/execution/ExecutionManager.h>
+#include <wfr/prediction/PredictionManager.h>
+#include <wfr/common/resource/ResourceManager.h>
+#include <wfr/common/eventsystem/EventSystem.h>
+#include <wfr/common/eventsystem/Event.h>
+#include <wfr/enactment/EnactmentEngine.h>
+#include <wfr/scheduling/SchedulerManager.h>
+#include <boost/program_options.hpp>
+#include <iostream>
+
+
+namespace wfr {
+
+
+std::map<std::string, WorkflowRuntime::ConfigEntry> WorkflowRuntime::globalConfig;
+
+std::default_random_engine WorkflowRuntime::randomEngine;
+
+std::unordered_set<std::shared_ptr<wfs::Workflow>> WorkflowRuntime::workflowInstances;
+
+
+void WorkflowRuntime::init(int argc, char** argv) {
+	bool isHelp = false;
+
+    boost::program_options::options_description opts("Global Options");
+    opts.add_options()
+    	    ("help", "Print help message.")
+			("sim-mode", "Enable simulation-mode.");
+    boost::program_options::variables_map vm;
+    auto parsed = boost::program_options::command_line_parser(argc, argv).options(opts).allow_unregistered().run();
+    boost::program_options::store(parsed, vm);
+    boost::program_options::notify(vm);
+
+    if (vm.count("help")) {
+    	std::cerr << opts << std::endl;
+    	isHelp = true;
+    }
+    if (vm.count("sim-mode")) {
+    	setGlobalConfig("sim-mode", true);
+    }
+
+    wfr::eventsystem::EventSystem::init(argc, argv);
+    wfr::resource::ResourceManager::init(argc, argv);
+    wfr::enactment::EnactmentEngine::init(argc, argv);
+    wfr::prediction::PredictionManager::init(argc, argv);
+    wfr::execution::ExecutionManager::init(argc, argv);
+    wfr::scheduling::SchedulerManager::init(argc, argv);
+
+	if (isHelp) exit(0);
+}
+
+void WorkflowRuntime::finish() {
+    wfr::scheduling::SchedulerManager::finish();
+	wfr::execution::ExecutionManager::finish();
+	wfr::prediction::PredictionManager::finish();
+	wfr::enactment::EnactmentEngine::finish();
+	wfr::resource::ResourceManager::finish();
+	wfr::eventsystem::EventSystem::finish();
+	if (workflowInstances.size() == 1) {
+		(*workflowInstances.begin())->printStatistics();
+	} else if (workflowInstances.size() > 1) {
+		std::cerr << "ERROR: Print statistics for more than one workflow instance not implemented." << std::endl;
+	}
+}
+
+void WorkflowRuntime::logMessage(std::string message, const wfr_timestamp_sc_t& timestamp, bool noLocking) {
+	wfr::eventsystem::GeneralLoggingEvent::fireEvent(message, timestamp, noLocking);
+}
+
+
+} /* namespace wfr */

src/cmake.inc

@@ -0,0 +1,13 @@
+ 
+set(wfr_cpp_files "${wfr_cpp_files}" 
+	src/WorkflowRuntime.cpp
+)
+
+include(src/common/cmake.inc)
+include(src/eventsystem/cmake.inc)
+include(src/enactment/cmake.inc)
+include(src/wep/cmake.inc)
+include(src/utils/cmake.inc)
+include(src/prediction/cmake.inc)
+include(src/scheduling/cmake.inc)
+include(src/execution/cmake.inc)

src/common/cmake.inc

@@ -0,0 +1,4 @@
+
+include(src/common/wfs/cmake.inc)
+include(src/common/wep/cmake.inc)
+include(src/common/resource/cmake.inc)

src/common/resource/Resource.cpp

@@ -0,0 +1,155 @@
+
+#include <wfr/common/resource/Resource.h>
+#include <wfr/common/wfs/AtomicActivityImplementation.h>
+#include <algorithm>
+
+
+namespace wfr {
+namespace resource {
+
+
+bool AbstractResourceType::hasCapability(const RequiredResourceCapability& type) const noexcept {
+	for (auto& cap : capabilities) {
+		if (cap == type) {
+			return true;
+		}
+	}
+	return false;
+}
+
+
+bool AbstractResourceType::hasCapability(wfr::resource::ResourceCapabilityType type) const noexcept {
+	for (auto& cap : capabilities) {
+		if (cap.type == type) {
+			return true;
+		}
+	}
+	return false;
+}
+
+
+std::vector<std::string>  AbstractResourceType::getPropertyKeys() const noexcept {
+	std::vector<std::string> retval;
+	for (auto i : properties) {
+		retval.push_back(i.first);
+	}
+	return retval;
+}
+
+
+std::string  AbstractResourceType::getProperty(const std::string& key) const noexcept {
+	auto i = properties.find(key);
+	if (i != properties.end()) {
+		return i->second;
+	} else {
+		return std::string();
+	}
+}
+
+
+
+std::map<std::string, ComputingDeviceClass> registeredDeviceTypes = {
+		{"cpu", ComputingDeviceClass::CPU},
+		{"gpu", ComputingDeviceClass::GPU}
+};
+
+
+
+std::map<std::string, ResourceCapabilityType> registeredDeviceCapabilityTypes = {
+		{"genericcpu", ResourceCapabilityType::GenericCPU},
+		{"cuda", ResourceCapabilityType::CUDA},
+		{"opencl", ResourceCapabilityType::OpenCL}
+};
+
+
+
+ComputingDeviceClass parseComputingDeviceType(const std::string& type) {
+	ComputingDeviceClass retval = ComputingDeviceClass::Unknown;
+	std::string tmp(type);
+	std::transform(tmp.begin(), tmp.end(), tmp.begin(), ::tolower);
+	auto item = registeredDeviceTypes.find(tmp);
+	if (item != registeredDeviceTypes.end()) {
+		retval = item->second;
+	}
+	return retval;
+}
+
+
+std::string getComputingDeviceTypeString(const ComputingDeviceClass& type) {
+	std::string retval;
+	for (auto& item : registeredDeviceTypes) {
+		if (item.second == type) {
+			retval = item.first;
+			break;
+		}
+	}
+	return retval;
+}
+
+
+ResourceCapabilityType parseResourceCapabilityType(const std::string& type) {
+	ResourceCapabilityType retval = ResourceCapabilityType::Unknown;
+	std::string tmp(type);
+	std::transform(tmp.begin(), tmp.end(), tmp.begin(), ::tolower);
+	auto item = registeredDeviceCapabilityTypes.find(tmp);
+	if (item != registeredDeviceCapabilityTypes.end()) {
+		retval = item->second;
+	}
+	return retval;
+}
+
+
+std::string getResourceCapabilityTypeString(const ResourceCapabilityType& type) {
+	std::string retval;
+	for (auto& item : registeredDeviceCapabilityTypes) {
+		if (item.second == type) {
+			retval = item.first;
+			break;
+		}
+	}
+	return retval;
+}
+
+
+bool ComputingResource::reserveSlots(const wfr_size_t& slots) noexcept {
+	if (slots <= getFreeSlotCount()) {
+		reservedSlots += slots;
+		return true;
+	} else {
+		return false;
+	}
+}
+
+bool ComputingResource::markReservedSlotsBusy(const wfr_size_t& slots) noexcept {
+	if (slots <= reservedSlots) {
+		reservedSlots -= slots;
+		busySlots += slots;
+		return true;
+	} else {
+		return false;
+	}
+
+}
+
+bool ComputingResource::markFreeSlotsBusy(const wfr_size_t& slots) noexcept {
+	if (slots <= getFreeSlotCount()) {
+		busySlots += slots;
+		return true;
+	} else {
+		return false;
+	}
+}
+
+bool ComputingResource::releaseBusySlots(const wfr_size_t& slots) noexcept {
+	if (slots <= busySlots) {
+		busySlots -= slots;
+		return true;
+	} else {
+		return false;
+	}
+}
+
+
+
+} /* namespace resource */
+} /* namespace wfr */

src/common/resource/ResourceManager.cpp

@@ -0,0 +1,372 @@
+
+#include <wfr/common/resource/ResourceManager.h>
+#include <wfr/common/exceptions/exceptions.h>
+#include <wfr/common/types/ResourceTypes.h>
+#include <rapidjson/document.h>
+#include <rapidjson/error/en.h>
+#include <iostream>
+#include <fstream>
+#include <algorithm>
+#include <thread>
+#include <chrono>
+#include <thread>
+#include <boost/program_options.hpp>
+#include <boost/filesystem.hpp>
+#include <boost/format.hpp>
+#include <wfr/common/resource/Resource.h>
+#include <wfr/common/resource/ResourceReservation.h>
+
+namespace wfr {
+namespace resource {
+
+
+ResourceManager* ResourceManager::singleton = nullptr;
+
+void ResourceManager::init(int argc, char** argv) {
+    std::string resourceConfigFile;
+    bool cpuAutoConfig = true;
+    bool printResourceTable = false;
+
+    boost::program_options::options_description opts("ResourceManager Options");
+    opts.add_options()
+    	    ("help", "Print help message.")
+    	    ("no-cpu-autoconfig", "Disable CPU auto-configuration.")
+    	    ("print-resource-table", "Print resource table.")
+    		("resource-config-file", boost::program_options::value<std::string>(), "Set resource config file.");
+    boost::program_options::variables_map vm;
+    auto parsed = boost::program_options::command_line_parser(argc, argv).options(opts).allow_unregistered().run();
+    boost::program_options::store(parsed, vm);
+    boost::program_options::notify(vm);
+    if (vm.count("help")) {
+    	std::cerr << opts << std::endl;
+    	return;
+    }
+    if (vm.count("no-cpu-autoconfig")) {
+    	cpuAutoConfig = false;
+    }
+    if (vm.count("print-resource-table")) {
+    	printResourceTable = true;
+    }
+    if (vm.count("resource-config-file")) {
+    	resourceConfigFile = vm["resource-config-file"].as<std::string>();
+    	boost::filesystem::path resourceConfigFilePath(resourceConfigFile);
+    	if (boost::filesystem::is_regular_file(resourceConfigFilePath)) {
+    		std::cerr << "Resource config file is set to \""<< resourceConfigFile << "\"." << std::endl;
+    	} else {
+    		throw wfr::exception::wfr_illegal_argument((boost::format("%1% does not exist or is not a file.") % resourceConfigFile).str());
+    	}
+    }
+
+	singleton = new ResourceManager(resourceConfigFile, cpuAutoConfig, printResourceTable);
+}
+
+void ResourceManager::finish() noexcept {
+	if (singleton != nullptr) {
+		delete singleton;
+		singleton = nullptr;
+	}
+}
+
+ResourceManager& ResourceManager::getInstance() noexcept {
+	if (singleton == nullptr) {
+		throw wfr::exception::wfr_missing_instance("ResourceManager instance not found.");
+	}
+	return *singleton;
+}
+
+ResourceManager::ResourceManager(std::string resourceConfigFile, bool cpuAutoConfig, bool printResourceTable) {
+	if (cpuAutoConfig) {
+		unsigned threadCount = std::thread::hardware_concurrency();
+		ComputingResourceType* resType = new ComputingResourceType("CPU", ComputingDeviceClass::CPU, threadCount, {{ResourceCapabilityType::GenericCPU, 0, 0}});
+		std::string tmp(resType->resourceTypeName);
+		computingResourceTypes.insert({tmp, resType});
+		ComputingResource* dev = new ComputingResource("cpu0", *resType);
+		computingResources.insert({dev->resourceIdentifier, dev});
+	}
+	if (!resourceConfigFile.empty()) {
+		readConfigFile(resourceConfigFile);
+	}
+	if (printResourceTable) {
+		std::this_thread::sleep_for(std::chrono::milliseconds(10));
+		this->printResourceTable(std::cout);
+		std::this_thread::sleep_for(std::chrono::milliseconds(10));
+	}
+}
+
+
+void ResourceManager::readConfigFile(std::string resourceConfigFile) {
+	std::ifstream configFile(resourceConfigFile);
+	std::string configContent((std::istreambuf_iterator<char>(configFile)), std::istreambuf_iterator<char>());
+	rapidjson::Document configDocument;
+	if (!configDocument.Parse(configContent.c_str()).HasParseError()) {
+		if (!configDocument.IsObject()) {
+			throw wfr::exception::wfr_fatal_error("Resource config file: toplevel item is not an object.");
+		}
+		if (!configDocument.HasMember("devicetypes")) {
+			throw wfr::exception::wfr_fatal_error("Resource config file: No devicetypes item found.");
+		}
+		auto& types = configDocument["devicetypes"];
+		if (!types.IsObject()) {
+			throw wfr::exception::wfr_fatal_error("Resource config file: devicetypes item is not an object.");
+		}
+		// Iterate over device types
+		for(rapidjson::Value::ConstMemberIterator it=types.MemberBegin(); it != types.MemberEnd(); it++) {
+			auto& type = *it;
+			if (!type.value.IsObject()) {
+				throw wfr::exception::wfr_fatal_error("Resource config file: type item is not an object.");
+			}
+    		std::string resourceName = type.name.GetString();
+    		ComputingDeviceClass computingDeviceClass = ComputingDeviceClass::Unknown;
+    		if (!type.value.HasMember("deviceClass")) {
+    			throw wfr::exception::wfr_fatal_error((boost::format("Resource config file: device type \"%1%\" has no deviceClass property.") % resourceName).str());
+    		}
+    		if (!type.value["deviceClass"].IsString()) {
+    			throw wfr::exception::wfr_fatal_error("Resource config file: device type property \"deviceClass\" has wrong type.");
+    		}
+    		std::string classStr = type.value["deviceClass"].GetString();
+			if (classStr.compare("CPU") == 0) {
+				computingDeviceClass = ComputingDeviceClass::CPU;
+			} else if (classStr.compare("GPU") == 0) {
+				computingDeviceClass = ComputingDeviceClass::GPU;
+			} else {
+				computingDeviceClass = ComputingDeviceClass::Unknown;
+			}
+			if (computingDeviceClass == ComputingDeviceClass::Unknown) {
+				throw wfr::exception::wfr_fatal_error((boost::format("Resource config file: unknown device type class \"%1%\".") % classStr).str());
+			}
+    		wfr_size_t deviceSlots = 1;
+    		if (type.value.HasMember("slots")) {
+				if (!type.value["slots"].IsUint()) {
+					throw wfr::exception::wfr_fatal_error("Resource config file: device type property \"slots\" has wrong type.");
+				}
+				deviceSlots = type.value["slots"].GetUint();
+    		}
+    		std::set<ResourceCapability> capabilities = std::set<ResourceCapability>();
+    		if (type.value.HasMember("capabilities")) {
+				if (!type.value["capabilities"].IsArray()) {
+					throw wfr::exception::wfr_fatal_error("Resource config file: device type property \"capabilities\" has wrong type.");
+				}
+				auto& caps = type.value["capabilities"];
+				for(rapidjson::Value::ConstValueIterator it2=caps.Begin(); it2 != caps.End(); it2++) {
+					auto& cap = *it2;
+					if (!cap.IsArray()) {
+						throw wfr::exception::wfr_fatal_error("Resource config file: capability has wrong type.");
+					}
+					if (!cap[0].IsString()) {
+						throw wfr::exception::wfr_fatal_error("Resource config file: capability param 0 has wrong type.");
+					}
+					if (!cap[1].IsInt()) {
+						throw wfr::exception::wfr_fatal_error("Resource config file: capability param 1 has wrong type.");
+					}
+					if (!cap[2].IsInt()) {
+						throw wfr::exception::wfr_fatal_error("Resource config file: capability param 2 has wrong type.");
+					}
+					std::string capname = cap[0].GetString();
+					ResourceCapabilityType captype = ResourceCapabilityType::Unknown;
+					if (capname.compare("GenericCPU") == 0) {
+						captype = ResourceCapabilityType::GenericCPU;
+					} else if (capname.compare("OpenCL") == 0) {
+						captype = ResourceCapabilityType::OpenCL;
+					} else if (capname.compare("CUDA") == 0) {
+						captype = ResourceCapabilityType::CUDA;
+					} else {
+						throw wfr::exception::wfr_fatal_error((boost::format("Resource config file: unknown capability \"%1%\".") % capname).str());
+					}
+					wfr_size_t minVersion = cap[1].GetInt();
+					wfr_size_t maxVersion = cap[2].GetInt();
+					capabilities.insert({captype, minVersion, maxVersion});
+				}
+    		}
+    		std::map<std::string, std::string> properties = std::map<std::string, std::string>();
+    		if (type.value.HasMember("properties")) {
+				if (!type.value["properties"].IsArray()) {
+					throw wfr::exception::wfr_fatal_error("Resource config file: device type property \"properties\" has wrong type.");
+				}
+				auto& props = type.value["properties"];
+				for(rapidjson::Value::ConstValueIterator it2=props.Begin(); it2 != props.End(); it2++) {
+					auto& prop = *it2;
+					if (!prop.IsArray()) {
+						throw wfr::exception::wfr_fatal_error("Resource config file: property has wrong type.");
+					}
+					if (!prop[0].IsString()) {
+						throw wfr::exception::wfr_fatal_error("Resource config file: property name has wrong type.");
+					}
+					if (!prop[1].IsString()) {
+						throw wfr::exception::wfr_fatal_error("Resource config file: property value has wrong type.");
+					}
+					properties.insert({prop[0].GetString(), prop[1].GetString()});
+				}
+    		}
+			ComputingResourceType* resType = new ComputingResourceType(resourceName, computingDeviceClass, deviceSlots, capabilities, properties);
+			computingResourceTypes.insert({resourceName, resType});
+		}
+		if (!configDocument.HasMember("devices")) {
+			throw wfr::exception::wfr_fatal_error("Resource config file: No devices item found.");
+		}
+		auto& devices = configDocument["devices"];
+		if (!devices.IsObject()) {
+			throw wfr::exception::wfr_fatal_error("Resource config file: devices item is not an object.");
+		}
+		// Iterate over devices
+		for(rapidjson::Value::ConstMemberIterator it2=devices.MemberBegin(); it2 != devices.MemberEnd(); it2++) {
+			auto& dev = *it2;
+			if (!dev.value.IsString()) {
+				throw wfr::exception::wfr_fatal_error("Resource config file: device item is not a string.");
+			}
+			std::string devName = dev.name.GetString();
+			std::string devTypeName = dev.value.GetString();
+			auto devType = computingResourceTypes.find(devTypeName);
+			if (devType == computingResourceTypes.end()) {
+				throw wfr::exception::wfr_fatal_error((boost::format("Resource config file: unknown device type \"%1%\".") % devTypeName).str());
+			}
+
+			ComputingResource* device = new ComputingResource(devName, *(*devType).second);
+			computingResources.insert({device->resourceIdentifier, device});
+		}
+
+	} else {
+		rapidjson::ParseErrorCode e = configDocument.GetParseError();
+		size_t o = configDocument.GetErrorOffset();
+		std::stringstream ss;
+		ss << "Error during parsing resource config file: \"" << rapidjson::GetParseError_En(e) << "\"";
+		ss << " at offset " << o << " near '" << configContent.substr(o, 10) << "...'" << std::endl;
+		throw wfr::exception::wfr_fatal_error(ss.str());
+	}
+}
+
+
+std::vector<ComputingResource*> ResourceManager::getComputingResources() const noexcept {
+	std::vector<ComputingResource*> retval;
+	for (auto dev : computingResources) {
+		retval.push_back(dev.second);
+	}
+	return retval;
+}
+
+
+ComputingResource* ResourceManager::getComputingResource(std::string identifier) const noexcept {
+	std::vector<ComputingResource*> retval;
+	for (auto dev : computingResources) {
+		if (dev.second->resourceIdentifier.compare(identifier) == 0) {
+			return dev.second;
+		}
+	}
+	return nullptr;
+}
+
+
+std::vector<ComputingResource*> ResourceManager::filterComputingResources(std::vector<RequiredResourceCapability> filter) const noexcept {
+	std::vector<ComputingResource*> retval;
+	for (auto dev : computingResources) {
+		for (auto cap : filter) {
+			if (dev.second->hasCapability(cap)) {
+				retval.push_back(dev.second);
+			}
+		}
+	}
+	return retval;
+}
+
+
+ComputingResourceReservation ResourceManager::reserveComputingResource(ComputingResource* device, const wfr_size_t& slots) noexcept {
+	if (device->getFreeSlotCount() >= slots) {
+		if (device->reserveSlots(slots)) {
+			return ComputingResourceReservation(device, slots);
+		}
+	}
+	return ComputingResourceReservation();
+}
+
+
+
+std::vector<ComputingResourceType*> ResourceManager::getComputingResourceTypes() const noexcept {
+	std::vector<ComputingResourceType*> retval;
+	for (auto i : computingResourceTypes) {
+		retval.push_back(i.second);
+	}
+	return retval;
+}
+
+
+ComputingResourceType* ResourceManager::getComputingResourceType(std::string name) const noexcept {
+	return computingResourceTypes.at(name);
+}
+
+
+
+ResourceManager::~ResourceManager() {
+	for (auto dev : computingResources) {
+		delete dev.second;
+	}
+	computingResources.clear();
+	for (auto type : computingResourceTypes) {
+		delete type.second;
+	}
+	computingResourceTypes.clear();
+}
+
+
+
+void ResourceManager::printResourceTable(std::ostream& out) const noexcept {
+	out << "======= Computing Resource Type Table ========" << std::endl;
+	for (auto& res : computingResourceTypes) {
+		out << "ResourceType: \"" << res.first << "\":" << std::endl;
+		std::string deviceClassStr;
+		switch (res.second->computingDeviceClass) {
+			case ComputingDeviceClass::CPU:
+				deviceClassStr = "CPU";
+				break;
+			case ComputingDeviceClass::GPU:
+				deviceClassStr = "GPU";
+				break;
+			default:
+				deviceClassStr = "Unknown";
+				break;
+		}
+		out << "\tDeviceClass: " << deviceClassStr << std::endl;
+		out << "\tSlots: " << res.second->maxSlots << std::endl;
+		out << "\tCapabilities: " << std::endl;
+		auto& caps = res.second->getCapabilities();
+		if (caps.size() > 0) {
+			for (auto& cap : caps) {
+				std::string capName;
+				switch (cap.type) {
+					case ResourceCapabilityType::GenericCPU:
+						capName = "GenericCPU";
+						break;
+					case ResourceCapabilityType::CUDA:
+						capName = "CUDA";
+						break;
+					case ResourceCapabilityType::OpenCL:
+						capName = "OpenCL";
+						break;
+					default:
+						capName = "Unknown";
+						break;
+				}
+				out << "\t\t" << capName << " (Min: " << cap.minorVersion << ", Max: " << cap.majorVersion << ")" << std::endl;
+			}
+		} else {
+			out << "\t\tNone" << std::endl;
+		}
+		out << "\tProperties: " << std::endl;
+		auto& props = res.second->getProperties();
+		if (props.size() > 0) {
+			for (auto& prop : props) {
+				out << "\t\t" << prop.first << " = " << prop.second << std::endl;
+			}
+		} else {
+			out << "\t\tNone" << std::endl;
+		}
+	}
+	out << "===== Computing Resource Type Table End ======" << std::endl;
+	out << "======= Computing Resource Table ========" << std::endl;
+	for (auto& res : computingResources) {
+		out << "Resource: \"" << res.second->resourceIdentifier << "\":" << std::endl;
+		out << "\tResourceType: \"" << res.second->resourceType.resourceTypeName << "\"" << std::endl;
+	}
+	out << "===== Computing Resource Table End ======" << std::endl;
+}
+
+} /* namespace resource */
+} /* namespace wfr */

src/common/resource/cmake.inc

@@ -0,0 +1,5 @@
+
+set(wfr_cpp_files "${wfr_cpp_files}" 
+	src/common/resource/Resource.cpp
+	src/common/resource/ResourceManager.cpp
+)

src/common/wep/AbstractWepNode.cpp

@@ -0,0 +1,15 @@
+
+#include <wfr/common/wep/AbstractWepNode.h>
+
+namespace wfr {
+namespace wep {
+
+std::atomic<wfr_id_t> AbstractWepNode::nextWepNodeID = ATOMIC_VAR_INIT(1);
+
+} /* namespace wep */
+} /* namespace wfr */
+
+
+
+
+

src/common/wep/ActivityWepNode.cpp

@@ -0,0 +1,26 @@
+
+#include <wfr/common/wep/ActivityWepNode.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+
+namespace wfr {
+namespace wep {
+
+ActivityWepNode::ActivityWepNode(WepNodeType nodeType, wfs::ActivityInstance& activity) noexcept
+		: AbstractWepNode(nodeType, true), lastKnownActivityStatus(activity.getInstanceStatus()), activity(activity) {}
+
+void ActivityWepNode::updateInstanceStatus() {
+	lastKnownActivityStatus = activity.getInstanceStatus();
+}
+
+void ActivityWepNode::addToVisitationCount(uint64_t val) noexcept {
+	visitationCount += val;
+	activity.addToVisitationCount(val);
+}
+
+} /* namespace wep */
+} /* namespace wfr */
+
+
+
+
+

src/common/wep/WorkflowExecutionPlan.cpp

@@ -0,0 +1,36 @@
+
+#include <wfr/common/wep/WorkflowExecutionPlan.h>
+#include <wfr/common/wep/AbstractWepNode.h>
+#include <set>
+#include <queue>
+
+namespace wfr {
+namespace wep {
+
+std::atomic<wfr_id_t> WorkflowExecutionPlan::nextWepID = ATOMIC_VAR_INIT(1);
+
+
+WorkflowExecutionPlan::~WorkflowExecutionPlan() noexcept {
+	if (startNode) {
+		std::set<AbstractWepNode*> visitedNodes;
+		std::queue<AbstractWepNode*> nodeQueue;
+		nodeQueue.push(startNode);
+		visitedNodes.insert(startNode);
+		do {
+			auto node = nodeQueue.front();
+			nodeQueue.pop();
+			for (auto s : node->getSuccessors()) {
+				if (visitedNodes.find(s) == visitedNodes.end()) {
+					nodeQueue.push(s);
+					visitedNodes.insert(s);
+				}
+			}
+			delete node;
+		} while (!nodeQueue.empty());
+	}
+}
+
+
+} /* namespace wep */
+} /* namespace wfr */
+

src/common/wep/activities/IterationEndWepNode.cpp

@@ -0,0 +1,22 @@
+
+#include <wfr/common/wep/activities/IterationEndWepNode.h>
+#include <wfr/common/wep/activities/IterationStartWepNode.h>
+
+
+namespace wfr {
+namespace wep {
+
+
+void IterationEndWepNode::updateInstanceStatus() {
+	startNode->updateInstanceStatus();
+}
+
+
+wfs::ActivityInstanceStatus IterationEndWepNode::getLastKnownInstanceStatus() const noexcept {
+	return startNode->getLastKnownInstanceStatus();
+}
+
+
+} /* namespace wep */
+} /* namespace wfr */
+

src/common/wep/activities/IterationStartWepNode.cpp

@@ -0,0 +1,16 @@
+
+#include <wfr/common/wep/activities/IterationStartWepNode.h>
+
+
+namespace wfr {
+namespace wep {
+
+
+void IterationStartWepNode::updateInstanceStatus() {
+
+}
+
+
+} /* namespace wep */
+} /* namespace wfr */
+

src/common/wep/cmake.inc

@@ -0,0 +1,8 @@
+
+set(wfr_cpp_files "${wfr_cpp_files}" 
+	src/common/wep/AbstractWepNode.cpp
+	src/common/wep/ActivityWepNode.cpp
+	src/common/wep/WorkflowExecutionPlan.cpp
+	src/common/wep/activities/IterationStartWepNode.cpp
+	src/common/wep/activities/IterationEndWepNode.cpp
+)

src/common/wfs/Activity.cpp

@@ -0,0 +1,183 @@
+
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/ActivityTemplate.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/common/wfs/Workflow.h>
+#include <wfr/common/wfs/instances/AtomicActivityInstance.h>
+#include <wfr/common/wfs/instances/AssignActivityInstance.h>
+#include <wfr/common/wfs/instances/ContainerActivityInstance.h>
+#include <wfr/common/wfs/instances/ConditionalActivityInstance.h>
+#include <wfr/common/wfs/instances/ForLoopActivityInstance.h>
+#include <wfr/common/wfs/templates/ForLoopActivityTemplate.h>
+#include <wfr/common/wfs/instances/WhileLoopActivityInstance.h>
+#include <wfr/common/eventsystem/Event.h>
+#include <boost/format.hpp>
+
+namespace wfr {
+namespace wfs {
+
+
+std::atomic<wfr_id_t> Activity::nextActivityID = ATOMIC_VAR_INIT(1);
+
+
+Activity::Activity( Workflow& workflow, ActivityTemplate& baseTemplate )
+		: status(ActivityStatus::Unknown),
+		  activityID(getNextActivityID()),
+		  workflow(workflow),
+		  baseTemplate(baseTemplate) {
+	setActivityStatus(ActivityStatus::Created);
+}
+
+
+Activity::~Activity() {
+	for (auto i : instanceList) {
+		if (i) delete i;
+	}
+	instanceList.clear();
+}
+
+
+ActivityType Activity::activityType() const noexcept {
+	return baseTemplate.activityType;
+}
+
+bool Activity::isCompositeActivity() const noexcept {
+	return baseTemplate.isCompositeActivity;
+}
+
+bool Activity::isLoop() const noexcept {
+	return baseTemplate.isLoop;
+}
+
+bool Activity::isConditional() const noexcept {
+	return baseTemplate.isConditional;
+}
+
+
+wfr_size_t Activity::getChildrenCount(wfr_id_t branchID) const noexcept {
+	wfr_size_t retval = 0;
+	for (auto i : workflow.getActivities()) {
+		if (i->parent && i->parent->activityID == activityID && i->baseTemplate.branchID == branchID) {
+			retval++;
+		}
+	}
+	return retval;
+}
+
+std::vector<Activity*> Activity::getChildren(wfr_id_t branchID) const noexcept {
+	std::vector<Activity*> retval;
+	for (auto i : workflow.getActivities()) {
+		if (i->parent && i->parent->activityID == activityID && i->baseTemplate.branchID == branchID) {
+			retval.push_back(i);
+		}
+	}
+	return retval;
+}
+
+bool Activity::isSuccessor(wfr_id_t aid) const noexcept {
+	for (auto i : successors) {
+		if (i->activityID == aid) {
+			return true;
+		}
+	}
+	return false;
+}
+
+bool Activity::isSuccessor(Activity& act) const noexcept {
+	for (auto i : successors) {
+		if (i == &act) {
+			return true;
+		}
+	}
+	return false;
+}
+
+bool Activity::isPredecessor(wfr_id_t aid) const noexcept {
+	for (auto i : predecessors) {
+		if (i->activityID == aid) {
+			return true;
+		}
+	}
+	return false;
+}
+
+bool Activity::isPredecessor(Activity& act) const noexcept {
+	for (auto i : predecessors) {
+		if (i == &act) {
+			return true;
+		}
+	}
+	return false;
+}
+
+
+void Activity::setActivityStatus(ActivityStatus status) {
+	if (status != this->status) {
+		if (this->status > status) {
+			throw exception::wfr_exception((boost::format("Illegal state change in activity with id %1%.") % activityID).str());
+		}
+		ActivityStatus oldStatus = this->status;
+		this->status = status;
+		wfr::eventsystem::ActivityStatusEvent::fireEvent(*this, oldStatus, status);
+	}
+}
+
+
+void Activity::initActivity() {
+	// Retrieve parent activity
+	if (baseTemplate.parentActivityID > 0) {
+		parent = workflow.getActivityByTemplateID(baseTemplate.parentActivityID);
+		assert(parent);
+	}
+	// Retrieve successors
+	for (auto i : baseTemplate.succRawList) {
+		auto tmp = workflow.getActivityByTemplateID(i);
+		assert(tmp);
+		successors.push_back(tmp);
+	}
+	// Retrieve predecessors
+	for (auto i : baseTemplate.predRawList) {
+		auto tmp = workflow.getActivityByTemplateID(i);
+		assert(tmp);
+		predecessors.push_back(tmp);
+	}
+	setActivityStatus(ActivityStatus::Initialized);
+}
+
+
+
+ActivityInstance* Activity::createNewInstance(wfr_id_t iterationID) {
+	ActivityInstance* retval = nullptr;
+	switch (activityType()) {
+		case ActivityType::Atomic:
+			retval = new AtomicActivityInstance(*this, iterationID);
+			break;
+		case ActivityType::Assign:
+			retval = new AssignActivityInstance(*this, iterationID);
+			break;
+		case ActivityType::Conditional:
+			retval = new ConditionalActivityInstance(*this, iterationID);
+			break;
+		case ActivityType::Container:
+			retval = new ContainerActivityInstance(*this, iterationID);
+			break;
+		case ActivityType::ForLoop:
+			retval = new ForLoopActivityInstance(*this, baseTemplate.as<ForLoopActivityTemplate>().isParallel, iterationID);
+			break;
+		case ActivityType::WhileLoop:
+			retval = new WhileLoopActivityInstance(*this, iterationID);
+			break;
+		default:
+			throw wfr::exception::wfr_fatal_error("Unknown instance type encountered.");
+	}
+	if (retval != nullptr) {
+		instanceList.push_back(retval);
+	}
+	return retval;
+}
+
+
+} /* namespace wfs */
+} /* namespace wfr */
+
+

src/common/wfs/ActivityInstance.cpp

@@ -0,0 +1,176 @@
+/*
+ * ActivityInstance.cpp
+ *
+ *  Created on: 06.02.2014
+ *      Author: matthias
+ */
+
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/common/wfs/ActivityTemplate.h>
+#include <wfr/common/wfs/instances/CompositeActivityInstance.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/Workflow.h>
+#include <wfr/common/eventsystem/Event.h>
+#include <wfr/common/eventsystem/EventSystem.h>
+#include <wfr/common/exceptions/exceptions.h>
+#include <wfr/common/scheduling/SchedulingDecision.h>
+#include <wfr/WorkflowRuntime.h>
+#include <boost/format.hpp>
+#include <chrono>
+
+namespace wfr {
+namespace wfs {
+
+
+
+std::atomic<wfr_id_t>ActivityInstance::nextInstanceID = ATOMIC_VAR_INIT(1);
+
+wfr_id_t ActivityInstance::getNextInstanceID() {
+	return nextInstanceID.fetch_add(1);
+}
+
+
+ActivityInstance::ActivityInstance(Activity& baseActivity, const wfr_id_t& iterationID) noexcept
+	: status(ActivityInstanceStatus::Unknown),
+	  instanceID(getNextInstanceID()),
+	  baseActivity(baseActivity),
+	  iterationID(iterationID) {
+	setInstanceStatus(ActivityInstanceStatus::Created);
+}
+
+ActivityType ActivityInstance::activityType() const noexcept {
+	return baseActivity.baseTemplate.activityType;
+}
+
+bool ActivityInstance::isCompositeActivity() const noexcept {
+	return baseActivity.baseTemplate.isCompositeActivity;
+}
+
+bool ActivityInstance::isLoop() const noexcept {
+	return baseActivity.baseTemplate.isLoop;
+}
+
+bool ActivityInstance::isConditional() const noexcept {
+	return baseActivity.baseTemplate.isConditional;
+}
+
+void ActivityInstance::setInstanceStatus(const ActivityInstanceStatus& status) {
+	if (status != this->status) {
+		if (this->status > status) {
+			throw wfr::exception::wfr_fatal_error((boost::format("Illegal activity instance state change in activity instance with id %1%.") % instanceID).str());
+		}
+
+		ActivityInstanceStatus oldStatus = this->status;
+		this->status = status;
+
+		switch (status) {
+			case wfr::wfs::ActivityInstanceStatus::ReadyForExecution:
+				readyTime = std::chrono::system_clock::now();
+				break;
+			case wfr::wfs::ActivityInstanceStatus::Running:
+				startTime = std::chrono::system_clock::now();
+				break;
+			case wfr::wfs::ActivityInstanceStatus::Finished:
+				endTime = std::chrono::system_clock::now();
+				/*if (parentActivity) {
+					if (parentActivity->isCompositeActivity()) {
+						wfr::wfs::CompositeActivityInstance& composite = parentActivity->as<wfs::CompositeActivityInstance>();
+						composite.notifyChildHasFinished(*this);
+					} else {
+						throw wfr::exception::wfr_fatal_error("Non-Composite activity cannot be a parent.");
+					}
+				}*/
+				break;
+			case wfr::wfs::ActivityInstanceStatus::Error:
+				endTime = std::chrono::system_clock::now();
+				break;
+			default:
+				break;
+		}
+
+		eventsystem::ActivityInstanceStatusEvent::fireEvent(*this, oldStatus, status);
+
+		#ifdef WFR_ENABLE_STATISTICS
+			switch (activityType()) {
+				case ActivityType::Atomic:
+					switch (status) {
+						case wfr::wfs::ActivityInstanceStatus::Created:
+							baseActivity.workflow.addToCreatedAtomicInstanceCount();
+							break;
+						case wfr::wfs::ActivityInstanceStatus::Running:
+							baseActivity.workflow.addToCumulaticeAtomicReadyWaitingDuration(startTime-readyTime);
+							break;
+						case wfr::wfs::ActivityInstanceStatus::Finished:
+							baseActivity.workflow.addToFinishedAtomicInstanceCount();
+							baseActivity.workflow.addToCumulaticeAtomicRunningDuration(endTime-startTime);
+							break;
+						case wfr::wfs::ActivityInstanceStatus::Error:
+							baseActivity.workflow.addToErrorAtomicInstanceCount();
+							baseActivity.workflow.addToCumulaticeAtomicRunningDuration(endTime-startTime);
+							break;
+						default:
+							break;
+					}
+					break;
+				default:
+					switch (status) {
+						case wfr::wfs::ActivityInstanceStatus::Created:
+							baseActivity.workflow.addToCreatedCompositeInstanceCount();
+							break;
+						case wfr::wfs::ActivityInstanceStatus::Finished:
+							baseActivity.workflow.addToFinishedCompositeInstanceCount();
+							break;
+						case wfr::wfs::ActivityInstanceStatus::Error:
+							baseActivity.workflow.addToErrorCompositeInstanceCount();
+							break;
+						default:
+							break;
+					}
+					break;
+			}
+		#endif
+	}
+}
+
+
+void ActivityInstance::setErrorState(std::exception_ptr exception) noexcept {
+	errorCause = exception;
+	setInstanceStatus(ActivityInstanceStatus::Error);
+}
+
+
+void ActivityInstance::executeSimMode() noexcept {
+	if (WorkflowRuntime::getGlobalConfig<bool>("sim-mode")) {
+		if (schedulingDecision) {
+			std::cout << "Simulating execution ... " << std::endl;
+			auto prediction = schedulingDecision->getPrediction();
+			if (prediction) {
+				auto& predTime = prediction.getExecutionTime();
+				double time = WorkflowRuntime::getRandomNumber(predTime.getValue() - predTime.getDeviation(), predTime.getValue() + predTime.getDeviation());
+				std::cout << "Sleeping for " << time << " seconds." << std::endl;
+				std::this_thread::sleep_for(std::chrono::milliseconds((long int)(time * 1000)));
+				/*std::cout << "Doing stuff for " << time << " seconds." << std::endl;
+				std::chrono::system_clock::time_point endtime = std::chrono::system_clock::now() + std::chrono::milliseconds((long int)(time * 1000));
+				while (true) {
+					if (std::chrono::system_clock::now() >= endtime) {
+						break;
+					} else {
+						volatile int i = 0;
+						for (int t = 0; t < 1000; t++) {
+							i++;
+						}
+					}
+				}*/
+			} else {
+				std::cout << "Warning: No valid prediction found" << std::endl;
+			}
+		} else {
+			throw exception::wfr_fatal_error("No scheduling decision found.");
+		}
+	}
+}
+
+
+} /* namespace wfs */
+} /* namespace wfr */
+

src/common/wfs/ActivityTemplate.cpp

@@ -0,0 +1,181 @@
+
+
+#include <wfr/common/wfs/ActivityTemplate.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/WorkflowTemplate.h>
+#include <wfr/common/exceptions/exceptions.h>
+#include <wfr/common/wfs/templates/AtomicActivityTemplate.h>
+#include <wfr/common/wfs/templates/AssignActivityTemplate.h>
+#include <wfr/common/wfs/templates/ContainerActivityTemplate.h>
+#include <wfr/common/wfs/templates/ForLoopActivityTemplate.h>
+#include <wfr/common/wfs/templates/WhileLoopActivityTemplate.h>
+#include <wfr/common/wfs/templates/ConditionalActivityTemplate.h>
+#include <boost/format.hpp>
+
+
+#include <iostream>
+namespace wfr {
+namespace wfs {
+
+
+
+ActivityTemplate::ActivityTemplate( wfr_id_t id,
+									ActivityType type,
+									const std::string& name,
+									wfr_id_t parentID,
+									wfr_id_t branchID,
+									const std::initializer_list<wfr_id_t>& succList,
+									const std::initializer_list<wfr_id_t>& predList,
+									bool isCompositeActivity,
+									bool isLoop,
+									bool isConditional ) noexcept
+		: succRawList(succList), predRawList(predList), templateID(id), activityName(name), activityType(type), parentActivityID(parentID),
+		  branchID(branchID), isCompositeActivity(isCompositeActivity), isLoop(isLoop), isConditional(isConditional) {
+	if (id == 0) {
+		throw exception::wfr_fatal_error((boost::format("ID of Activity template \"%1%\" must be greater than 0.") % activityName).str());
+	}
+}
+
+
+void ActivityTemplate::initActivityTemplate(WorkflowTemplate& parentWorkflow) {
+	this->parentWorkflow = &parentWorkflow;
+	if (parentActivityID > 0) {
+		this->parentActivity = parentWorkflow.getActivityByID(parentActivityID);
+		if (!this->parentActivity) {
+			throw exception::wfr_fatal_error((boost::format("Activity template \"%1%\" has invalid parent activity id.") % activityName).str());
+		}
+	}
+	for (auto i : succRawList) {
+		auto act = parentWorkflow.getActivityByID(i);
+		if (!act) {
+			throw exception::wfr_fatal_error((boost::format("Activity template \"%1%\" has invalid successor id %2%.") % activityName % i).str());
+		} else {
+			successors.push_back(act);
+		}
+	}
+	for (auto i : predRawList) {
+		auto act = parentWorkflow.getActivityByID(i);
+		if (!act) {
+			throw exception::wfr_fatal_error((boost::format("Activity template \"%1%\" has invalid predecessor id %2%.") % activityName % i).str());
+		} else {
+			predecessors.push_back(act);
+		}
+	}
+}
+
+
+
+void ActivityTemplate::checkStructure() {
+	// Check parent activity
+	if (parentActivity) {
+		if (!parentActivity->isCompositeActivity) {
+			throw exception::wfr_fatal_error((boost::format("Parent \"%1%\" of activity template \"%2%\" is is not a composite activity.") % parentActivity->activityName % activityName).str());
+		}
+	}
+	// Check that every successor has this activity as predecessor
+	for (auto i : successors) {
+		if (!i->isPredecessor(*this)) {
+			throw exception::wfr_fatal_error((boost::format("Activity template \"%2%\" is a successor of \"%1%\", but \"%1%\" is not a predecessor of \"%2%\".") % activityName % i->activityName).str());
+		}
+	}
+	// Check that every predecessor has this activity as successor
+	for (auto i : predecessors) {
+		if (!i->isSuccessor(*this)) {
+			throw exception::wfr_fatal_error((boost::format("Activity template \"%2%\" is a predecessor of \"%1%\", but \"%1%\" is not a successor of \"%2%\".") % activityName % i->activityName).str());
+		}
+	}
+}
+
+
+
+Activity* ActivityTemplate::createActivityFromTemplate(wfr::wfs::Workflow& workflow) {
+	return new Activity(workflow, *this);
+}
+
+
+
+ActivityTemplate* ActivityTemplate::atomic( wfr_id_t id,
+											 const std::string& name,
+											 wfr_id_t parentID,
+											 wfr_id_t branchID,
+											 std::initializer_list<wfr_id_t> succList,
+											 std::initializer_list<wfr_id_t> predList,
+											 InputHookFunc inputHook,
+											 std::shared_ptr<AtomicActivityType> atomicActivityType ) {
+	return new AtomicActivityTemplate(id, name, parentID, branchID, succList, predList, inputHook, atomicActivityType);
+}
+
+
+ActivityTemplate* ActivityTemplate::assign( wfr_id_t id,
+											 const std::string& name,
+											 wfr_id_t parentID,
+											 wfr_id_t branchID,
+											 std::initializer_list<wfr_id_t> succList,
+											 std::initializer_list<wfr_id_t> predList,
+											 InputHookFunc inputHook,
+											 ExecutionHookFunc execHook ) {
+	return new AssignActivityTemplate(id, name, parentID, branchID, succList, predList, inputHook, execHook);
+}
+
+
+ActivityTemplate* ActivityTemplate::container( wfr_id_t id,
+												 const std::string& name,
+												 wfr_id_t parentID,
+												 wfr_id_t branchID,
+												 std::initializer_list<wfr_id_t> succList,
+												 std::initializer_list<wfr_id_t> predList,
+												 InputHookFunc inputHook,
+												 OutputHookFunc outputHook ) {
+	return new ContainerActivityTemplate(id, name, parentID, branchID, succList, predList, inputHook, outputHook);
+}
+
+
+ActivityTemplate* ActivityTemplate::forLoop( wfr_id_t id,
+											const std::string& name,
+											wfr_id_t parentID,
+											wfr_id_t branchID,
+											const std::initializer_list<wfr_id_t>& succList,
+											const std::initializer_list<wfr_id_t>& predList,
+											InputHookFunc inputFuncHook,
+											OutputHookFunc outputFuncHook,
+											ForCounterHookFunc counterFuncHook,
+											LoopIterationHookFunc iterationFuncHook,
+											bool isParallel ) {
+	return new ForLoopActivityTemplate(id, name, parentID, branchID, succList, predList, inputFuncHook, outputFuncHook, counterFuncHook, iterationFuncHook, isParallel);
+}
+
+
+
+ActivityTemplate* ActivityTemplate::whileLoop( wfr_id_t id,
+												const std::string& name,
+												wfr_id_t parentID,
+												wfr_id_t branchID,
+												const std::initializer_list<wfr_id_t>& succList,
+												const std::initializer_list<wfr_id_t>& predList,
+												InputHookFunc inputFuncHook,
+												OutputHookFunc outputFuncHook,
+												WhileConditionHookFunc conditionFuncHook,
+												LoopIterationHookFunc iterationFuncHook ) {
+	return new WhileLoopActivityTemplate(id, name, parentID, branchID, succList, predList, inputFuncHook, outputFuncHook, conditionFuncHook, iterationFuncHook);
+}
+
+
+
+ActivityTemplate* ActivityTemplate::conditional( wfr_id_t id,
+												const std::string& name,
+												wfr_id_t parentID,
+												wfr_id_t branchID,
+												const std::initializer_list<wfr_id_t>& succList,
+												const std::initializer_list<wfr_id_t>& predList,
+												InputHookFunc inputFuncHook,
+												OutputHookFunc outputFuncHook,
+												BranchConditionHookFunc branchFuncHook ) {
+	return new ConditionalActivityTemplate(id, name, parentID, branchID, succList, predList, inputFuncHook, outputFuncHook, branchFuncHook);
+}
+
+} /* namespace wfs */
+} /* namespace wfr */
+
+
+
+

src/common/wfs/AtomicActivityImplementation.cpp

@@ -0,0 +1,100 @@
+
+#include <wfr/common/wfs/AtomicActivityImplementation.h>
+#include <wfr/common/wfs/implementations/PThreadActivityImplementation.h>
+#include <wfr/common/wfs/implementations/OpenCLActivityImplementation.h>
+#include <wfr/common/wfs/implementations/CUDAActivityImplementation.h>
+#include <sstream>
+#include <iostream>
+
+namespace wfr {
+namespace wfs {
+
+
+std::atomic<wfr_id_t> AtomicActivityImplementation::nextImplementationID = ATOMIC_VAR_INIT(1);
+
+
+AtomicActivityImplementation*
+AtomicActivityImplementation::PThread( const ExecutionHookFunc& executionFuncHook,
+									  const std::string& name,
+									  utils::PropertyList properties,
+									  resource::RequiredCapabilityList capabilities ) noexcept {
+	std::vector<resource::RequiredResourceCapability> caps(capabilities);
+	caps.push_back(wfr::resource::RequiredResourceCapability::GenericCPU());
+	return new PThreadActivityImplementation(executionFuncHook, caps, properties, name);
+}
+
+
+AtomicActivityImplementation*
+AtomicActivityImplementation::OpenCL( const ExecutionHookFunc& executionFuncHook,
+									const std::string& name,
+									std::initializer_list<wfr_size_t> minVersion,
+									std::initializer_list<wfr_size_t> maxVersion,
+									utils::PropertyList properties,
+									resource::RequiredCapabilityList capabilities ) noexcept {
+	std::vector<resource::RequiredResourceCapability> caps(capabilities);
+	wfr_size_t minVersion1 = 0, minVersion2 = 0, maxVersion1 = 0, maxVersion2 = 0;
+	wfr_size_t versionCount = 0;
+	for (auto i : minVersion) {
+		if (versionCount == 0) {
+			minVersion1 = i;
+		} else if (versionCount == 1) {
+			minVersion2 = i;
+		} else {
+			break;
+		}
+		versionCount++;
+	}
+	versionCount = 0;
+	for (auto i : maxVersion) {
+		if (versionCount == 0) {
+			maxVersion1 = i;
+		} else if (versionCount == 1) {
+			maxVersion2 = i;
+		} else {
+			break;
+		}
+		versionCount++;
+	}
+	caps.push_back(wfr::resource::RequiredResourceCapability::OpenCL(minVersion1, minVersion2, maxVersion1, maxVersion2));
+	return new OpenCLActivityImplementation(executionFuncHook, capabilities, properties, name);
+}
+
+
+AtomicActivityImplementation*
+AtomicActivityImplementation::CUDA( const ExecutionHookFunc& executionFuncHook,
+		const std::string& name,
+		std::initializer_list<wfr_size_t> minVersion,
+		std::initializer_list<wfr_size_t> maxVersion,
+		utils::PropertyList properties,
+		resource::RequiredCapabilityList capabilities ) noexcept {
+	std::vector<resource::RequiredResourceCapability> caps(capabilities);
+	wfr_size_t minVersion1 = 0, minVersion2 = 0, maxVersion1 = 0, maxVersion2 = 0;
+	wfr_size_t versionCount = 0;
+	for (auto i : minVersion) {
+		if (versionCount == 0) {
+			minVersion1 = i;
+		} else if (versionCount == 1) {
+			minVersion2 = i;
+		} else {
+			break;
+		}
+		versionCount++;
+	}
+	versionCount = 0;
+	for (auto i : maxVersion) {
+		if (versionCount == 0) {
+			maxVersion1 = i;
+		} else if (versionCount == 1) {
+			maxVersion2 = i;
+		} else {
+			break;
+		}
+		versionCount++;
+	}
+	caps.push_back(wfr::resource::RequiredResourceCapability::CUDA(minVersion1, minVersion2, maxVersion1, maxVersion2));
+	return new CUDAActivityImplementation(executionFuncHook, capabilities, properties, name);
+}
+
+
+} /* namespace execution */
+} /* namespace wfr */

src/common/wfs/AtomicActivityType.cpp

@@ -0,0 +1,21 @@
+
+#include <wfr/common/wfs/AtomicActivityType.h>
+#include <wfr/common/wfs/AtomicActivityImplementation.h>
+
+
+namespace wfr {
+namespace wfs {
+
+AtomicActivityType::~AtomicActivityType() noexcept {
+		for (auto i : implementations) {
+			if (i) delete i;
+		}
+		implementations.clear();
+	}
+
+
+} /* namespace wfs */
+} /* namespace wfr */
+
+
+

src/common/wfs/Workflow.cpp

@@ -0,0 +1,156 @@
+
+#include <wfr/common/wfs/Workflow.h>
+#include <wfr/common/wfs/WorkflowTemplate.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/ActivityTemplate.h>
+#include <wfr/common/utils/print.h>
+#include <wfr/common/eventsystem/Event.h>
+#include <wfr/enactment/EnactmentEngine.h>
+#include <wfr/scheduling/SchedulerManager.h>
+#include <wfr/scheduling/AbstractScheduler.h>
+#include <boost/format.hpp>
+#include <cassert>
+#include <iostream>
+
+namespace wfr {
+
+namespace scheduling { class AbstractScheduler; }
+
+namespace wfs {
+
+
+
+std::atomic<wfr_id_t> Workflow::nextWorkflowID = ATOMIC_VAR_INIT(1);
+
+
+Workflow::Workflow(WorkflowTemplate& baseTemplate)
+		: status(WorkflowStatus::Unknown),
+		workflowID(getNextWorkflowID()),
+		baseTemplate(baseTemplate) {
+	setWorkflowStatus(WorkflowStatus::Created);
+	// Create activity instances
+	for (auto i : baseTemplate.getActivities()) {
+		auto tmp = i->createActivityFromTemplate(*this);
+		assert(tmp);
+		activities.push_back(tmp);
+	}
+	// Init activity instances
+	for (auto i : activities) {
+		i->initActivity();
+	}
+	// Retrieve top-level activity
+	topLevelActivity = getActivityByID(baseTemplate.getTopLevelActivityID());
+	assert(topLevelActivity);
+	setWorkflowStatus(WorkflowStatus::Initialized);
+}
+
+Workflow::~Workflow() {
+	for (auto i : activities) {
+		if (i) delete i;
+	}
+	activities.clear();
+}
+
+
+
+void Workflow::setWorkflowStatus (WorkflowStatus status) {
+	if (status != this->status) {
+		if (this->status > status) {
+			throw exception::wfr_exception((boost::format("Illegal state change in workflow with id %1%.") % workflowID).str());
+		}
+		WorkflowStatus oldStatus = this->status;
+		this->status = status;
+		switch (status) {
+			case WorkflowStatus::Created:
+				createTime = std::chrono::system_clock::now();
+				break;
+			case WorkflowStatus::Running:
+				startTime = std::chrono::system_clock::now();
+				break;
+			case WorkflowStatus::Finished:
+			case WorkflowStatus::Error:
+				endTime = std::chrono::system_clock::now();
+				break;
+			default:
+				break;
+		}
+		wfr::eventsystem::WorkflowStatusEvent::fireEvent(*this, oldStatus, status);
+	}
+}
+
+Activity* Workflow::getActivityByID(wfr_id_t id) const noexcept {
+	for (auto i : activities) {
+		if (i->activityID == id) return i;
+	}
+	return nullptr;
+}
+
+
+Activity* Workflow::getActivityByTemplateID(wfr_id_t id) const noexcept {
+	for (auto i : activities) {
+		if (i->baseTemplate.templateID == id) return i;
+	}
+	return nullptr;
+}
+
+
+void Workflow::setErrorState(std::exception_ptr cause) {
+	errorCause = cause;
+	setWorkflowStatus(WorkflowStatus::Error);
+}
+
+
+WorkflowStatus Workflow::executeSync( lwd::_wfr_lwd* input, lwd::_wfr_lwd* output ) {
+	scheduling::AbstractScheduler* scheduler = scheduling::SchedulerManager::getInstance().getDefaultScheduler();
+	{ // We want to have the enactment engine in its own scope so that it does not exist anymore when we delete the scheduler.
+		enactment::EnactmentEngine engine(*this, *scheduler);
+		setWorkflowStatus(WorkflowStatus::Running);
+		engine.workflowExecutionLoop(input, output);
+	}
+	cummulativePredictionTime = scheduler->getCummulativePredictionTime();
+	delete scheduler;
+	return status;
+}
+
+
+#ifdef WFR_ENABLE_STATISTICS
+
+	void Workflow::printStatistics(std::ostream& sout) {
+		sout << "WFR-Statistics: Created Instances: " << (createdCompositeInstanceCount+createdAtomicInstanceCount) << std::endl;
+		sout << "WFR-Statistics: Finished Instances: " << (finishedCompositeInstanceCount+finishedAtomicInstanceCount) << std::endl;
+		sout << "WFR-Statistics: Error Instances: " << (errorCompositeInstanceCount+errorAtomicInstanceCount) << std::endl;
+
+		sout << "WFR-Statistics: Created Composite Instances: " << createdCompositeInstanceCount << std::endl;
+		sout << "WFR-Statistics: Finished Composite Instances: " << finishedCompositeInstanceCount << std::endl;
+		sout << "WFR-Statistics: Error Composite Instances: " << errorCompositeInstanceCount << std::endl;
+
+		sout << "WFR-Statistics: Created Atomic Instances: " << createdAtomicInstanceCount << std::endl;
+		sout << "WFR-Statistics: Finished Atomic Instances: " << finishedAtomicInstanceCount << std::endl;
+		sout << "WFR-Statistics: Error Atomic Instances: " << errorAtomicInstanceCount << std::endl;
+
+		sout << "WFR-Statistics: WEP Node Count: " << wepNodeCount << std::endl;
+		sout << "WFR-Statistics: WEP Visitation Count: " << wepVisitationCount << std::endl;
+		sout << "WFR-Statistics: WEP Update Count: " << wepUpdateCount << std::endl;
+		sout << "WFR-Statistics: WEP Update Notify Count: " << wepUpdateNotifyCount << std::endl;
+
+		sout << "WFR-Statistics: Cumulative Atomic ReadyWait Duration: "
+			 << ((double)std::chrono::duration_cast<std::chrono::milliseconds>(cumulativeAtomicReadyWaitingDuration).count())/1000.0 << " sec" << std::endl;
+		sout << "WFR-Statistics: Cumulative Atomic Running Duration: "
+			 << ((double)std::chrono::duration_cast<std::chrono::milliseconds>(cumulativeAtomicRunningDuration).count())/1000.0 << " sec" << std::endl;
+		sout << "WFR-Statistics: Cumulative Prediction Time: " << cummulativePredictionTime << " sec" << std::endl;
+
+		sout << "WFR-Statistics: Enactment Time: "
+			 << ((double)std::chrono::duration_cast<std::chrono::milliseconds>(enactmentTime).count())/1000.0 << " sec" << std::endl;
+		sout << "WFR-Statistics: Workflow Duration: "
+			 << ((double)std::chrono::duration_cast<std::chrono::milliseconds>(getWorkflowDuration()).count())/1000.0 << " sec" << std::endl;
+		sout << "WFR-Statistics: Makespan: "
+			 << ((double)std::chrono::duration_cast<std::chrono::milliseconds>(getMakespan()).count())/1000.0 << " sec" << std::endl;
+	}
+
+#endif
+
+} /* namespace wfs */
+} /* namespace wfr */
+
+
+

src/common/wfs/WorkflowTemplate.cpp

@@ -0,0 +1,72 @@
+
+#include <wfr/common/wfs/WorkflowTemplate.h>
+#include <wfr/common/wfs/Workflow.h>
+#include <wfr/common/wfs/ActivityTemplate.h>
+#include <wfr/WorkflowRuntime.h>
+#include <boost/format.hpp>
+
+namespace wfr {
+namespace wfs {
+
+
+WorkflowTemplate::WorkflowTemplate( const std::string& name,
+								    wfr_id_t topLevelActivityID,
+								    std::initializer_list<ActivityTemplate*> activities)
+		: activities(activities), workflowName(name), topLevelActivityID(topLevelActivityID) {
+	if (activities.size() == 0) {
+		throw exception::wfr_fatal_error((boost::format("Workflow \"%1%\" has no activities.") % workflowName).str());
+	}
+	for (auto i : activities) {
+		i->initActivityTemplate(*this);
+	}
+	toplevelActivity = getActivityByID(topLevelActivityID);
+	if (!toplevelActivity) {
+		throw exception::wfr_fatal_error((boost::format("Workflow \"%1%\" has invalid top-level activity id %2%.") % workflowName % topLevelActivityID).str());
+	}
+	checkStructure();
+}
+
+
+WorkflowTemplate::~WorkflowTemplate() noexcept {
+	for (auto i : activities) {
+		if (i) delete i;
+	}
+	activities.clear();
+}
+
+
+ActivityTemplate* WorkflowTemplate::getActivityByID(wfr_id_t id) const noexcept {
+	ActivityTemplate* retval = nullptr;
+	for (auto i : activities) {
+		if (i->templateID == id) {
+			retval = i;
+			break;
+		}
+	}
+	return retval;
+}
+
+
+void WorkflowTemplate::checkStructure() {
+	for (auto i : activities) {
+		i->checkStructure();
+	}
+}
+
+
+std::shared_ptr<Workflow> WorkflowTemplate::execute(lwd::_wfr_lwd* input, lwd::_wfr_lwd* output) {
+	std::shared_ptr<Workflow> wf = std::shared_ptr<Workflow>(new Workflow(*this));
+	WorkflowRuntime::addWorkflowInstance(wf);
+	try {
+		wf->executeSync(input, output);
+	} catch (std::exception& e) {
+		std::cerr << "Error while executing workflow: Caught exception: " << e.what() << std::endl;
+	} catch (...) {
+		std::cerr << "Error while executing workflow: Caught unknown exception." << std::endl;
+	}
+	return wf;
+}
+
+
+} /* namespace wfs */
+} /* namespace wfr */

src/common/wfs/cmake.inc

@@ -0,0 +1,18 @@
+
+set(wfr_cpp_files "${wfr_cpp_files}" 
+	src/common/wfs/WorkflowTemplate.cpp
+	src/common/wfs/Workflow.cpp
+	src/common/wfs/ActivityTemplate.cpp
+	src/common/wfs/Activity.cpp
+	src/common/wfs/ActivityInstance.cpp
+	src/common/wfs/AtomicActivityImplementation.cpp
+	src/common/wfs/AtomicActivityType.cpp
+	src/common/wfs/instances/AssignActivityInstance.cpp
+	src/common/wfs/instances/AtomicActivityInstance.cpp
+	src/common/wfs/instances/CompositeActivityInstance.cpp
+	src/common/wfs/instances/ConditionalActivityInstance.cpp
+	src/common/wfs/instances/ContainerActivityInstance.cpp
+	src/common/wfs/instances/LoopActivityInstance.cpp
+	src/common/wfs/instances/ForLoopActivityInstance.cpp
+	src/common/wfs/instances/WhileLoopActivityInstance.cpp
+)

src/common/wfs/instances/AssignActivityInstance.cpp

@@ -0,0 +1,55 @@
+
+#include <wfr/common/wfs/instances/AssignActivityInstance.h>
+#include <wfr/common/wfs/instances/CompositeActivityInstance.h>
+#include <wfr/common/wfs/templates/AssignActivityTemplate.h>
+#include <wfr/common/wfs/Workflow.h>
+#include <wfr/common/wfs/Activity.h>
+
+
+namespace wfr {
+namespace wfs {
+
+
+bool AssignActivityInstance::executePreExecutionHooks(bool earlyEvaluation) {
+	bool retval = false;
+	if (getInstanceStatus() == ActivityInstanceStatus::Initialized) {
+		setInstanceStatus(ActivityInstanceStatus::WaitingForData);
+	}
+
+	auto& templ = baseActivity.baseTemplate.as<AssignActivityTemplate>();
+	if (templ.inputFuncHook != nullptr) {
+		retval = templ.inputFuncHook(*this);
+	} else {
+		allInputsAvailable = true;
+		setInstanceStatus(ActivityInstanceStatus::ReadyForExecution);
+	}
+	return retval;
+}
+
+bool AssignActivityInstance::executeExecutionHook(bool earlyEvaluation) {
+	bool retval = false;
+	setInstanceStatus(ActivityInstanceStatus::Running);
+	auto& templ = baseActivity.baseTemplate.as<AssignActivityTemplate>();
+	if (templ.execFuncHook != nullptr) {
+		retval = templ.execFuncHook(*this);
+	}
+	setInstanceStatus(ActivityInstanceStatus::Finished);
+	return retval;
+}
+
+
+void AssignActivityInstance::notifyParent() noexcept {
+	if (parentActivity != nullptr) { // propagate to parent
+		parentActivity->as<wfs::CompositeActivityInstance>().notifyChildHasFinished(*this);
+	} else { // if there is no parent then it's the toplevel activity
+		//baseActivity.workflow.setWorkflowStatus(WorkflowStatus::Finished);
+		baseActivity.workflow.setFinishFlag(true);
+	}
+}
+
+} /* namespace wfs */
+} /* namespace wfr */
+
+
+
+

src/common/wfs/instances/AtomicActivityInstance.cpp

@@ -0,0 +1,30 @@
+
+#include <wfr/common/wfs/instances/AtomicActivityInstance.h>
+#include <wfr/common/wfs/templates/AtomicActivityTemplate.h>
+#include <wfr/common/wfs/Activity.h>
+
+
+namespace wfr {
+namespace wfs {
+
+bool AtomicActivityInstance::executePreExecutionHooks(bool earlyEvaluation) {
+	bool retval = false;
+	if (getInstanceStatus() == ActivityInstanceStatus::Initialized) {
+		setInstanceStatus(ActivityInstanceStatus::WaitingForData);
+	}
+	auto& templ = baseActivity.baseTemplate.as<AtomicActivityTemplate>();
+	if (templ.inputFuncHook != nullptr) {
+		retval = templ.inputFuncHook(*this);
+	} else {
+		allInputsAvailable = true;
+		setInstanceStatus(ActivityInstanceStatus::ReadyForExecution);
+	}
+	return retval;
+}
+
+} /* namespace wfs */
+} /* namespace wfr */
+
+
+
+

src/common/wfs/instances/CompositeActivityInstance.cpp

@@ -0,0 +1,62 @@
+
+#include <wfr/common/wfs/instances/CompositeActivityInstance.h>
+#include <wfr/common/wfs/Workflow.h>
+#include <wfr/common/wfs/Activity.h>
+
+
+namespace wfr {
+namespace wfs {
+
+ActivityInstance* CompositeActivityInstance::getChild(wfr_id_t instanceID) const noexcept {
+	auto i = childrenByInstanceID.find(instanceID);
+	if (i != childrenByInstanceID.end()) {
+		return i->second;
+	} else {
+		return nullptr;
+	}
+	/*for (auto i : children) {
+		if (i->instanceID == instanceID) {
+			return i;
+		}
+	}
+	return nullptr;*/
+}
+
+ActivityInstance* CompositeActivityInstance::getChild(wfr_id_t activityID, wfr_id_t iterationID) const noexcept {
+	/*auto i = childrenByActivityID.find(activityID);
+	if (i != childrenByActivityID.end()) {
+		for (auto a : i->second) {
+			if (a->iterationID == iterationID) {
+				return a;
+			}
+		}
+	}
+	return nullptr;*/
+
+	for (auto i : children) {
+		if (i->baseActivity.activityID == activityID && i->iterationID == iterationID) {
+			return i;
+		}
+	}
+	return nullptr;
+}
+
+void CompositeActivityInstance::notifyChildHasFinished(ActivityInstance& ai) noexcept {
+	finishedChildrenCount++;
+	//std::cout << "notifyChildHasFinished(" << finishedChildrenCount << ", " << baseActivity.getChildrenCount() << ")" << std::endl;
+	if (finishedChildrenCount >= baseActivity.getChildrenCount()) { // see if all children are finished
+		//std::cout << "notifyChildHasFinished: parent now finished" << std::endl;
+		setInstanceStatus(ActivityInstanceStatus::Finished);
+		if (parentActivity != nullptr) { // propagate to parent
+			parentActivity->as<CompositeActivityInstance>().notifyChildHasFinished(*this);
+		} else { // if there is no parent then it's the toplevel activity
+			//baseActivity.workflow.setWorkflowStatus(WorkflowStatus::Finished);
+			baseActivity.workflow.setFinishFlag(true);
+		}
+	}
+}
+
+} /* namespace wfs */
+} /* namespace wfr */
+
+

src/common/wfs/instances/ConditionalActivityInstance.cpp

@@ -0,0 +1,46 @@
+
+#include <wfr/common/wfs/instances/ConditionalActivityInstance.h>
+#include <wfr/common/wfs/templates/ConditionalActivityTemplate.h>
+#include <wfr/common/wfs/Activity.h>
+
+
+namespace wfr {
+namespace wfs {
+
+
+bool ConditionalActivityInstance::executePreExecutionHooks(bool earlyEvaluation) {
+	bool retval = false;
+	if (getInstanceStatus() == ActivityInstanceStatus::Initialized) {
+		setInstanceStatus(ActivityInstanceStatus::WaitingForData);
+	}
+	auto& templ = baseActivity.baseTemplate.as<ConditionalActivityTemplate>();
+	if (templ.inputFuncHook != nullptr) {
+		retval = templ.inputFuncHook(*this);
+	} else {
+		allInputsAvailable = true;
+	}
+	if (templ.branchFuncHook != nullptr) {
+		retval = templ.branchFuncHook(*this) || retval;
+	} else {
+		setErrorState(std::make_exception_ptr(wfr::exception::wfr_fatal_error("BranchFuncHook is null.")));
+	}
+	return retval;
+}
+
+
+bool ConditionalActivityInstance::executePostExecutionHooks(bool earlyEvaluation) {
+	bool retval = false;
+	if (status >= ActivityInstanceStatus::ReadyForExecution) {
+		auto& templ = baseActivity.baseTemplate.as<ConditionalActivityTemplate>();
+		if (templ.outputFuncHook != nullptr) {
+			retval = templ.outputFuncHook(*this);
+		} else {
+			setErrorState(std::make_exception_ptr(wfr::exception::wfr_fatal_error("OutputFuncHook is null.")));
+		}
+	}
+	return retval;
+}
+
+
+} /* namespace wfs */
+} /* namespace wfr */

src/common/wfs/instances/ContainerActivityInstance.cpp

@@ -0,0 +1,42 @@
+
+#include <wfr/common/wfs/instances/ContainerActivityInstance.h>
+#include <wfr/common/wfs/templates/ContainerActivityTemplate.h>
+#include <wfr/common/wfs/Activity.h>
+
+
+namespace wfr {
+namespace wfs {
+
+
+bool ContainerActivityInstance::executePreExecutionHooks(bool earlyEvaluation) {
+	bool retval = false;
+	if (getInstanceStatus() == ActivityInstanceStatus::Initialized) {
+		setInstanceStatus(ActivityInstanceStatus::WaitingForData);
+	}
+	auto& templ = baseActivity.baseTemplate.as<ContainerActivityTemplate>();
+	if (templ.inputFuncHook != nullptr) {
+		retval = templ.inputFuncHook(*this);
+	} else {
+		allInputsAvailable = true;
+		setInstanceStatus(ActivityInstanceStatus::ReadyForExecution);
+	}
+	return retval;
+}
+
+
+bool ContainerActivityInstance::executePostExecutionHooks(bool earlyEvaluation) {
+	bool retval = false;
+	if (status >= ActivityInstanceStatus::ReadyForExecution) {
+		auto& templ = baseActivity.baseTemplate.as<ContainerActivityTemplate>();
+		if (templ.outputFuncHook != nullptr) {
+			retval = templ.outputFuncHook(*this);
+		} else {
+			setErrorState(std::make_exception_ptr(wfr::exception::wfr_fatal_error("OutputFuncHook is null.")));
+		}
+	}
+	return retval;
+}
+
+
+} /* namespace wfs */
+} /* namespace wfr */

src/common/wfs/instances/ForLoopActivityInstance.cpp

@@ -0,0 +1,66 @@
+
+#include <wfr/common/wfs/instances/ForLoopActivityInstance.h>
+#include <wfr/common/wfs/templates/ForLoopActivityTemplate.h>
+#include <wfr/common/wfs/Activity.h>
+
+
+namespace wfr {
+namespace wfs {
+
+
+void ForLoopActivityInstance::notifyChildHasFinished(ActivityInstance& ai) noexcept {
+	LoopActivityInstance::notifyChildHasFinished(ai);
+	auto& templ = baseActivity.baseTemplate.as<ForLoopActivityTemplate>();
+	if (templ.iterationFuncHook != nullptr) {
+		templ.iterationFuncHook(*this);
+	}
+}
+
+bool ForLoopActivityInstance::executePreExecutionHooks(bool earlyEvaluation) {
+	bool retval = false;
+	if (getInstanceStatus() == ActivityInstanceStatus::Initialized) {
+		setInstanceStatus(ActivityInstanceStatus::WaitingForData);
+	}
+	auto& templ = baseActivity.baseTemplate.as<ForLoopActivityTemplate>();
+	if (templ.inputFuncHook != nullptr) {
+		retval = templ.inputFuncHook(*this);
+	} else {
+		allInputsAvailable = true;
+	}
+	if (templ.counterFuncHook != nullptr) {
+		retval = templ.counterFuncHook(*this) || retval;
+	} else {
+		setErrorState(std::make_exception_ptr(wfr::exception::wfr_fatal_error("IteratorFuncHook is null.")));
+	}
+	return retval;
+}
+
+bool ForLoopActivityInstance::executePostExecutionHooks(bool earlyEvaluation) {
+	bool retval = false;
+	if (status >= ActivityInstanceStatus::ReadyForExecution) {
+		if (earlyEvaluation || (finalisedFlag && finishedChildrenCount >= children.size())) {
+			auto& templ = baseActivity.baseTemplate.as<ForLoopActivityTemplate>();
+			if (templ.outputFuncHook != nullptr) {
+				retval = templ.outputFuncHook(*this);
+			} else {
+				setErrorState(std::make_exception_ptr(wfr::exception::wfr_fatal_error("OutputFuncHook is null.")));
+			}
+		}
+	}
+	return retval;
+}
+
+void ForLoopActivityInstance::setLoopCounter(wfr_size_t start, wfr_size_t step, wfr_size_t max) {
+	this->loopCounterStart = start;
+	this->loopCounterStep = step;
+	this->loopCounterMax = max;
+	wfr_size_t iters = ((max + 1) - start) / step;
+	if (start + iters*step < max) {
+		iters++;
+	}
+	setIterationCount(iters);
+	loopCounterValueSet = true;
+}
+
+} /* namespace wfs */
+} /* namespace wfr */

src/common/wfs/instances/LoopActivityInstance.cpp

@@ -0,0 +1,57 @@
+
+#include <wfr/common/wfs/instances/LoopActivityInstance.h>
+#include <wfr/common/wfs/Workflow.h>
+#include <wfr/common/wfs/Activity.h>
+
+
+namespace wfr {
+namespace wfs {
+
+
+void LoopActivityInstance::setIterationCount(wfr_size_t count) noexcept {
+	iterationCount = count;
+	finishedChildrenPerIteration.resize(count, 0);
+	finishedIterations.resize(count, false);
+	runningIterations.resize(count, false);
+}
+
+void LoopActivityInstance::notifyChildHasFinished(ActivityInstance& ai) noexcept {
+	finishedChildrenCount++;
+	auto iid = ai.iterationID;
+	finishedChildrenPerIteration[iid]++;
+	if (finishedChildrenPerIteration[iid] >= baseActivity.getChildrenCount()) {
+		finishedIterations[iid] = true;
+	}
+	if (finishedChildrenCount >= baseActivity.getChildrenCount() * iterationCount) { // see if all children are finished
+		setInstanceStatus(ActivityInstanceStatus::Finished);
+		if (parentActivity != nullptr) { // propagate to parent
+			parentActivity->as<CompositeActivityInstance>().notifyChildHasFinished(*this);
+		} else { // if there is no parent then it's the toplevel activity
+			//baseActivity.workflow.setWorkflowStatus(WorkflowStatus::Finished);
+			baseActivity.workflow.setFinishFlag(true);
+		}
+	}
+}
+
+
+ActivityInstanceStatus LoopActivityInstance::getIterationInstanceStatus(wfr_id_t iterationID) const noexcept {
+	if (iterationID >= getIterationCount()) {
+		return wfs::ActivityInstanceStatus::Unknown;
+	}
+	if (status == wfs::ActivityInstanceStatus::ReadyForExecution) {
+		if (hasIterationFinished(iterationID)) {
+			return wfs::ActivityInstanceStatus::Finished;
+		} else if (isIterationRunning(iterationID)) {
+			return wfs::ActivityInstanceStatus::Running;
+		} else {
+			if (!isParallel && iterationID > 0 && !hasIterationFinished(iterationID-1)) {
+				return wfs::ActivityInstanceStatus::WaitingForDependency;
+			}
+		}
+	}
+	return status;
+}
+
+
+} /* namespace wfs */
+} /* namespace wfr */

src/common/wfs/instances/WhileLoopActivityInstance.cpp

@@ -0,0 +1,79 @@
+
+#include <wfr/common/wfs/instances/WhileLoopActivityInstance.h>
+#include <wfr/common/wfs/templates/WhileLoopActivityTemplate.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/Workflow.h>
+
+
+namespace wfr {
+namespace wfs {
+
+bool WhileLoopActivityInstance::executePreExecutionHooks(bool earlyEvaluation) {
+	bool retval = false;
+	if (getInstanceStatus() == ActivityInstanceStatus::Initialized) {
+		setInstanceStatus(ActivityInstanceStatus::WaitingForData);
+	}
+	auto& templ = baseActivity.baseTemplate.as<WhileLoopActivityTemplate>();
+	if (templ.inputFuncHook != nullptr) {
+		retval = templ.inputFuncHook(*this);
+	} else {
+		retval = true;
+		allInputsAvailable = true;
+	}
+	if (templ.conditionFuncHook != nullptr) {
+		if (iterationCount <= 0 || finishedIterations[iterationCount-1]) {
+			retval = templ.conditionFuncHook(*this) || retval;
+		}
+	} else {
+		setErrorState(std::make_exception_ptr(wfr::exception::wfr_fatal_error("ConditionFuncHook is null.")));
+	}
+	return retval;
+}
+
+bool WhileLoopActivityInstance::executePostExecutionHooks(bool earlyEvaluation) {
+	bool retval = false;
+	if (status >= ActivityInstanceStatus::ReadyForExecution) {
+		if (earlyEvaluation || (finalisedFlag && finishedChildrenCount >= children.size())) {
+			auto& templ = baseActivity.baseTemplate.as<WhileLoopActivityTemplate>();
+			if (templ.outputFuncHook != nullptr) {
+				retval = templ.outputFuncHook(*this);
+			} else {
+				setErrorState(std::make_exception_ptr(wfr::exception::wfr_fatal_error("OutputFuncHook is null.")));
+			}
+		}
+	}
+	return retval;
+}
+
+void WhileLoopActivityInstance::notifyChildHasFinished(ActivityInstance& ai) noexcept {
+	auto& templ = baseActivity.baseTemplate.as<WhileLoopActivityTemplate>();
+	finishedChildrenCount++;
+	auto iid = ai.iterationID;
+	finishedChildrenPerIteration[iid]++;
+	if (finishedChildrenPerIteration[iid] >= baseActivity.getChildrenCount()) {
+		finishedIterations[iid] = true;
+	}
+	if (templ.iterationFuncHook != nullptr && finishedIterations[iid]) {
+		templ.iterationFuncHook(*this);
+	}
+	if (!finalisedFlag && finishedIterations[iid]) {
+		if (templ.conditionFuncHook != nullptr) {
+			templ.conditionFuncHook(*this);
+		} else {
+			setErrorState(std::make_exception_ptr(wfr::exception::wfr_fatal_error("ConditionFuncHook is null.")));
+		}
+	}
+	if (finishedChildrenCount >= baseActivity.getChildrenCount() * iterationCount) { // see if all children are finished
+		setInstanceStatus(ActivityInstanceStatus::Finished);
+		if (parentActivity != nullptr) { // propagate to parent
+			parentActivity->as<CompositeActivityInstance>().notifyChildHasFinished(*this);
+		} else { // if there is no parent then it's the toplevel activity
+			//baseActivity.workflow.setWorkflowStatus(WorkflowStatus::Finished);
+			baseActivity.workflow.setFinishFlag(true);
+		}
+	}
+}
+
+
+} /* namespace wfs */
+} /* namespace wfr */

src/enactment/EnactmentEngine.cpp

@@ -0,0 +1,173 @@
+
+#include <wfr/enactment/EnactmentEngine.h>
+#include <wfr/enactment/WepBuilder.h>
+#include <wfr/common/wep/WorkflowExecutionPlan.h>
+#include <wfr/common/wep/WorkflowExecutionPlan.h>
+#include <wfr/scheduling/AbstractScheduler.h>
+#include <wfr/common/wfs/Workflow.h>
+#include <wfr/wep/utils/WepDotFileWriter.h>
+#include <wfr/common/eventsystem/Event.h>
+#include <wfr/common/eventsystem/EventSystem.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/ActivityTemplate.h>
+#include <wfr/WorkflowRuntime.h>
+#include <wfr/config.h>
+#include <boost/program_options.hpp>
+#include <iostream>
+#include <cstdlib>
+#include <sys/resource.h>
+#include <unistd.h>
+
+
+namespace wfr {
+namespace enactment {
+
+
+void EnactmentEngine::init(int argc, char** argv) {
+    boost::program_options::options_description opts("EnactmentEngine Options");
+    opts.add_options()
+			("help", "Print help message.")
+    		("disable-wepdotwriter", "Disable WEP to .dot writer.")
+    		("early-evaluation", "Enables early evaluation.")
+    		("late-evaluation", "Enables late evaluation [Default].");
+    boost::program_options::variables_map vm;
+    auto parsed = boost::program_options::command_line_parser(argc, argv).options(opts).allow_unregistered().run();
+    boost::program_options::store(parsed, vm);
+    boost::program_options::notify(vm);
+    if (vm.count("help")) {
+        boost::program_options::options_description show_opts("EnactmentEngine Options");
+        show_opts.add_options()
+        		("disable-wepdotwriter", "Disable WEP to .dot writer.")
+	    		("early-evaluation", "Enables early evaluation [Default].")
+	    		("late-evaluation", "Enables late evaluation.");
+    	std::cerr << show_opts << std::endl;
+    	return;
+    }
+
+    if (vm.count("disable-wepdotwriter")) {
+    	WorkflowRuntime::setGlobalConfig("dotWriterEnabled", false);
+    	std::cerr << "WEP-to-.dot writer is disabled." << std::endl;
+    } else {
+    	WorkflowRuntime::setGlobalConfig("dotWriterEnabled", true);
+    	std::cerr << "WEP-to-.dot writer is enabled." << std::endl;
+    }
+
+    if (vm.count("early-evaluation") && vm.count("late-evaluation")) {
+    	std::cerr << "Cannot have early and late evaluation enabled at the same time." << std::endl;
+    	std::abort();
+    } else if (vm.count("late-evaluation")) {
+    	WorkflowRuntime::setGlobalConfig("earlyEvaluationEnabled", false);
+    	std::cerr << "Using late evaluation mode." << std::endl;
+    } else {
+    	WorkflowRuntime::setGlobalConfig("earlyEvaluationEnabled", true);
+    	std::cerr << "Using early evaluation mode." << std::endl;
+    }
+}
+
+void EnactmentEngine::finish() {
+	// NOP
+}
+
+
+EnactmentEngine::EnactmentEngine(wfs::Workflow& workflow, scheduling::AbstractScheduler& scheduler) noexcept
+		: enableWepDotFileWriter(WorkflowRuntime::getGlobalConfig<bool>("dotWriterEnabled")),
+		  enableEarlyEvaluation(WorkflowRuntime::getGlobalConfig<bool>("earlyEvaluationEnabled")),
+		  workflow(workflow), scheduler(scheduler) {}
+
+
+void EnactmentEngine::workflowExecutionLoop(lwd::_wfr_lwd* input, lwd::_wfr_lwd* output) {
+
+	// Give thread a higher priority
+	pid_t myPID = getpid();
+	int oldPriority = getpriority(PRIO_PROCESS, myPID);
+	setpriority(PRIO_PROCESS, myPID, -20);
+
+	auto eventBusy = eventsystem::ComponentStatusEvent::fireEvent(wfs::ComponentType::Enactment,
+			wfs::ComponentStatus::Unknown, wfs::ComponentStatus::Busy, &workflow);
+
+	WepBuilder wepBuilder(enableEarlyEvaluation);
+	wep = wepBuilder.createWEP(workflow, input, output);
+
+	#ifdef WFR_ENABLE_WEPDOTWRITER
+		std::shared_ptr<wep::utils::WepDotFileWriter> dotWriter;
+		if (enableWepDotFileWriter) {
+			dotWriter = std::make_shared<wep::utils::WepDotFileWriter>(wep);
+			dotWriter->printToFile();
+		}
+	#endif
+
+	scheduler.registerWEP(*wep, *this);
+	auto ren = wepBuilder.getReadyToExecuteAtomicNodes();
+	scheduler.notifyWEPUpdate(*wep, std::set<wep::AtomicWepNode*>(ren.begin(), ren.end()));
+
+	while (!breakExecutionLoop) {
+		bool dirtyFlag = false;
+		std::set<wep::AtomicWepNode*> readyToExecuteAtomicNodes;
+		{
+			std::unique_lock<std::mutex> lk(updateMutex);
+			bool eventWasSend = false;
+			while (updateQueue.empty()) {
+				if (!eventWasSend) {
+						auto event = eventsystem::ComponentStatusEvent::fireEvent(wfs::ComponentType::Enactment,
+										wfs::ComponentStatus::Busy, wfs::ComponentStatus::Idle, &workflow);
+						eventWasSend = true;
+						workflow.addToEnactmentTime(event->timestamp - eventBusy->timestamp);
+						eventBusy.reset();
+				}
+				updateCondition.wait(lk);
+			}
+			if (eventWasSend) {
+				eventBusy = eventsystem::ComponentStatusEvent::fireEvent(wfs::ComponentType::Enactment, wfs::ComponentStatus::Idle,
+						wfs::ComponentStatus::Busy, &workflow);
+			}
+
+			//std::cout << "Starting Execution Plan Update: " << updateQueue.size() << " Nodes in Queue." << std::endl;
+			workflow.addToWepUpdateCount();
+
+			dirtyFlag = wepBuilder.updateExecutionPlan(updateQueue);
+			auto ren = wepBuilder.getReadyToExecuteAtomicNodes();
+			readyToExecuteAtomicNodes.insert(ren.begin(), ren.end());
+		}
+		// update wep here
+		if (dirtyFlag) {
+			#ifdef WFR_ENABLE_WEPDOTWRITER
+				if (enableWepDotFileWriter) {
+					dotWriter->printToFile();
+				}
+			#endif
+			if (wep->workflow.getWorkflowStatus() == wfr::wfs::WorkflowStatus::Error
+					|| wep->workflow.getWorkflowStatus() == wfr::wfs::WorkflowStatus::Finished) {
+				breakExecutionLoop = true;
+			} else if (wep->workflow.getFinishFlag()) {
+				breakExecutionLoop = true;
+				wep->workflow.setWorkflowStatus(wfs::WorkflowStatus::Finished);
+			} else {
+				scheduler.notifyWEPUpdate(*wep, readyToExecuteAtomicNodes);
+				workflow.addToWepUpdateNotifyCount();
+			}
+		}
+	}
+	auto event = eventsystem::ComponentStatusEvent::fireEvent(wfs::ComponentType::Enactment,
+					wfs::ComponentStatus::Busy, wfs::ComponentStatus::Finished, &workflow);
+	workflow.addToEnactmentTime(event->timestamp - eventBusy->timestamp);
+	eventBusy.reset();
+	scheduler.unregisterWEP(*wep.get());
+
+	// Reset thread priority
+	setpriority(PRIO_PROCESS, myPID, oldPriority);
+}
+
+
+void wfr::enactment::EnactmentEngine::notifyActivityInstanceStateUpdate(wfs::ActivityInstance& ai, scheduling::AbstractScheduler& scheduler) {
+    std::lock_guard<std::mutex> lk(updateMutex);
+    //std::cout << "notifyActivityInstanceStateUpdate: " << ai.baseActivity.baseTemplate.activityName << " - " << ai.baseActivity.activityID  << "-" << ai.instanceID << std::endl;
+    updateQueue.push(&ai);
+    updateCondition.notify_one();
+}
+
+} /* namespace wep */
+} /* namespace wfr */
+
+
+

src/enactment/WepBuilder.cpp

@@ -0,0 +1,595 @@
+
+#include <wfr/enactment/WepBuilder.h>
+#include <wfr/common/wfs/Workflow.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/common/wfs/ActivityTemplate.h>
+#include <wfr/common/wfs/instances/CompositeActivityInstance.h>
+#include <wfr/common/wfs/instances/ConditionalActivityInstance.h>
+#include <wfr/common/wfs/instances/LoopActivityInstance.h>
+#include <wfr/common/wfs/instances/AssignActivityInstance.h>
+#include <wfr/common/wfs/instances/WhileLoopActivityInstance.h>
+#include <wfr/common/wfs/instances/AtomicActivityInstance.h>
+#include <wfr/common/wep/activities/AtomicWepNode.h>
+#include <wfr/common/wep/activities/AssignWepNode.h>
+#include <wfr/common/wep/activities/CompositeStartWepNode.h>
+#include <wfr/common/wep/activities/CompositeEndWepNode.h>
+#include <wfr/common/wep/activities/IterationStartWepNode.h>
+#include <wfr/common/wep/activities/IterationEndWepNode.h>
+#include <wfr/common/wep/WorkflowExecutionPlan.h>
+#include <wfr/common/utils/print.h>
+#include <wfr/wep/utils/WepSearch.h>
+
+
+namespace wfr {
+namespace enactment {
+
+WepBuilder::~WepBuilder() {
+	if (wep) {
+		wep->workflow.setWepNodeCount(wepNodeCount);
+		wep->workflow.addToWepVisitationCount(wepVisitationCount);
+	}
+}
+
+std::shared_ptr<wep::WorkflowExecutionPlan> WepBuilder::createWEP(wfr::wfs::Workflow& wf, lwd::_wfr_lwd* input, lwd::_wfr_lwd* output) {
+	readyToExecuteAtomicNodes.clear();
+	readyToExecuteAssignNodes.clear();
+
+	// Create top-level Activity Instance
+	auto ta = wf.getTopLevelActivity();
+	auto ti = ta->createNewInstance(0);
+
+	// Set Workflow input/output buffer
+	ti->setInputData(input, false);
+	ti->setOutputData(output, false);
+
+	// Execute pre-execution hooks
+	ti->executePreExecutionHooks(earlyEvaluationMode);
+
+	// Create WEPnode
+	auto wepNode = createWEPnode(*ta, *ti, wepNodeMap);
+
+	// Check activity status (top-level threads needs to be ReadyForExecution, everything else is an error)
+	if (ti->getInstanceStatus() == wfs::ActivityInstanceStatus::ReadyForExecution) {
+
+		// Composite activity branch
+		if (ta->isCompositeActivity()) {
+
+			// Fetch child activities
+			std::vector<wfr::wfs::Activity*> children;
+			if (ti->isConditional()) {
+				children = ta->getChildren(ti->as<wfs::ConditionalActivityInstance>().getExecutionBranchID());
+			} else {
+				children = ta->getChildren();
+			}
+
+			// Push parent on stacks
+			parentStack.push(ti);
+			parentWepStack.push(wepNode);
+
+			// Loop Branch
+			if (ti->isLoop()) {
+				auto loop = ti->as<wfs::LoopActivityInstance>();
+				auto maxIterCount = loop.getIterationCount();
+				wfr_id_t i;
+				for ( i = loop.getHandledIterationCount(); i < maxIterCount; i++ ) {
+					// Create and push iteration wep-node
+					auto iterationWepNode = createIterationWepNodes(*ti, wepNode, i);
+					iterationIdStack.push(i);
+					parentWepStack.push(iterationWepNode);
+					activityInstanceMap_t instanceMap;
+					// Visit children
+					for (auto& child : children) {
+						visitActivity(*child, instanceMap, wepNodeMap);
+					}
+					iterationIdStack.pop();
+					parentWepStack.pop();
+				}
+				loop.setHandledIterationCount(i);
+
+			// Non-Loop Branch
+			} else {
+				iterationIdStack.push(0);
+				activityInstanceMap_t instanceMap;
+				// Visit children
+				for (auto& child : children) {
+					visitActivity(*child, instanceMap, wepNodeMap);
+				}
+				iterationIdStack.pop();
+			}
+
+			// Pop parent from stack
+			parentStack.pop();
+			parentWepStack.pop();
+
+		// Assign Activity branch
+		} else if (wepNode.first->nodeType == wep::WepNodeType::Assign) {
+			wepNode.first->as<wep::AssignWepNode>().activity.executeExecutionHook(earlyEvaluationMode);
+			readyToExecuteAssignNodes.push_back(&wepNode.first->as<wep::AssignWepNode>());
+
+		// Atomic Activity branch
+		} else {
+			readyToExecuteAtomicNodes.push_back(&wepNode.first->as<wep::AtomicWepNode>());
+		}
+
+	}  else if (ti->getInstanceStatus() == wfs::ActivityInstanceStatus::WaitingForData) {
+		if (wepNode.first->getSuccessors().empty()) {
+			wepNode.first->successors.push_back(wepNode.second);
+			wepNode.second->predecessors.push_back(wepNode.first);
+		}
+
+	} else {
+		throw wfr::exception::wfr_fatal_error("toplevel node is not ready for execution");
+	}
+	for (auto n : readyToExecuteAssignNodes) {
+		n->activity.as<wfs::AssignActivityInstance>().notifyParent();
+	}
+	readyToExecuteAssignNodes.clear();
+	// Create and return execution plan
+	wep = std::make_shared<wep::WorkflowExecutionPlan>(wf, wepNode.first, wepNode.second);
+	return wep;
+}
+
+
+WepBuilder::wepNodeMap_t::mapped_type WepBuilder::createIterationWepNodes(wfr::wfs::ActivityInstance& ai, wepNodeMap_t::mapped_type& parent, wfr_id_t iterationID) {
+	wep::IterationStartWepNode* isn = new wep::IterationStartWepNode(ai, iterationID);
+	wep::IterationEndWepNode* ien = new wep::IterationEndWepNode(ai, iterationID);
+	isn->endNode = ien;
+	ien->startNode = isn;
+	bool isParallel = true;
+	if (ai.isLoop()) {
+		isParallel = ai.as<wfs::LoopActivityInstance>().isParallel;
+	}
+	if (isParallel) {
+		isn->predecessors.push_back(parent.first);
+		parent.first->successors.push_back(isn);
+		ien->successors.push_back(parent.second);
+		parent.second->predecessors.push_back(ien);
+	} else {
+		wfr_size_t pPredCount = parent.second->getPredecessors().size();
+		if (pPredCount > 0) {
+			auto lastIter = parent.second->getPredecessors()[pPredCount-1];
+			lastIter->successors.clear();
+			lastIter->successors.push_back(isn);
+			isn->predecessors.push_back(lastIter);
+			parent.second->predecessors.clear();
+			parent.second->predecessors.push_back(ien);
+			ien->successors.push_back(parent.second);
+		} else {
+			parent.first->successors.push_back(isn);
+			isn->predecessors.push_back(parent.first);
+			parent.second->predecessors.push_back(ien);
+			ien->successors.push_back(parent.second);
+
+		}
+	}
+	wepNodeMap_t::mapped_type wepNodeMapEntry;
+	wepNodeMapEntry.first = isn;
+	wepNodeMapEntry.second= ien;
+	wepNodeCount++;
+	if (wepNodeMapEntry.first != wepNodeMapEntry.second) {
+		wepNodeCount++;
+	}
+	return wepNodeMapEntry;
+}
+
+
+WepBuilder::wepNodeMap_t::mapped_type WepBuilder::createWEPnode(wfs::Activity& a, wfs::ActivityInstance& ai, wepNodeMap_t& wepNodeMap) {
+	wepNodeMap_t::mapped_type wepNodeMapEntry;
+	if (a.isCompositeActivity()) {
+		wep::CompositeStartWepNode* cs = new wep::CompositeStartWepNode(ai);
+		wep::CompositeEndWepNode* ce = new wep::CompositeEndWepNode(ai);
+		cs->endNode = ce;
+		ce->startNode = cs;
+		wepNodeMapEntry.first = cs;
+		wepNodeMapEntry.second= ce;
+	} else if (a.activityType() == wfr::wfs::ActivityType::Atomic) {
+		wep::AtomicWepNode* a = new wep::AtomicWepNode(ai);
+		wepNodeMapEntry.first = a;
+		wepNodeMapEntry.second= a;
+	} else if (a.activityType() == wfr::wfs::ActivityType::Assign) {
+		wep::AssignWepNode* a = new wep::AssignWepNode(ai);
+		wepNodeMapEntry.first = a;
+		wepNodeMapEntry.second= a;
+	} else {
+		throw wfr::exception::wfr_fatal_error("This branch should never be executed.");
+	}
+	wepNodeCount++;
+	if (wepNodeMapEntry.first != wepNodeMapEntry.second) {
+		wepNodeCount++;
+	}
+	wepNodeMap.insert(wepNodeMap_t::value_type(&ai, wepNodeMapEntry));
+	return wepNodeMapEntry;
+}
+
+
+
+void WepBuilder::visitActivity(wfr::wfs::Activity& element, activityInstanceMap_t& instanceMap, wepNodeMap_t& wepNodeMap) {
+
+	//wep->workflow.addToWepVisitationCount();
+	wepVisitationCount++;
+
+	// Create activity instance
+	auto ti = element.createNewInstance(iterationIdStack.top());
+	auto parent = parentStack.top();
+	ti->setParentActivity(parent);
+
+	// Add to parents children
+	if (parent->isCompositeActivity()) {
+		parent->as<wfs::CompositeActivityInstance>().children.push_back(ti);
+		parent->as<wfs::CompositeActivityInstance>().childrenByInstanceID.insert({ti->instanceID, ti});
+
+		auto tmpi = parent->as<wfs::CompositeActivityInstance>().childrenByActivityID.find(ti->baseActivity.activityID);
+		if (tmpi != parent->as<wfs::CompositeActivityInstance>().childrenByActivityID.end()) {
+			tmpi->second.push_back(ti);
+		} else {
+			std::vector<wfs::ActivityInstance*> tmp;
+			tmp.push_back(ti);
+			parent->as<wfs::CompositeActivityInstance>().childrenByActivityID.insert({ti->baseActivity.activityID, tmp});
+		}
+	} else {
+		throw wfr::exception::wfr_fatal_error("Cannot add a child to a non-composite activity");
+	}
+
+	// Save into instance-map
+	instanceMap[&element] = ti;
+
+	// Set predecessors/successors
+	auto& preds = element.getPredecessors();
+	auto predCount = element.getPredecessorCount();
+	for (wfr_id_t i = 0; i < predCount; i++) {
+		auto pred = preds[i];
+		auto predRef = instanceMap.find(pred);
+		if (predRef == instanceMap.end()) {
+			throw wfr::exception::wfr_fatal_error("Could not find predRef");
+		}
+		ti->predecessors.push_back(predRef->second);
+		predRef->second->successors.push_back(ti);
+	}
+	ti->setInstanceStatus(wfs::ActivityInstanceStatus::Initialized);
+	// Execute pre-execution hooks
+	try {
+		ti->executePreExecutionHooks(earlyEvaluationMode);
+	} catch (...) {
+		ti->setErrorState(std::current_exception());
+	}
+	// Create WEPnode
+	auto wepNode = createWEPnode(element, *ti, wepNodeMap);
+	wepNode.first->addToVisitationCount();
+	// Set wep predecessors/successors
+	for (wfr_id_t i = 0; i < predCount; i++) {
+		auto pred = preds[i];
+		auto predRef = instanceMap.find(pred);
+		if (predRef == instanceMap.end()) {
+			throw wfr::exception::wfr_fatal_error("Could not find predRef");
+		}
+		auto predWepNode = wepNodeMap.find((*predRef).second);
+		if (predWepNode == wepNodeMap.end()) {
+			throw wfr::exception::wfr_fatal_error("Could not find predWepNode");
+		}
+		wepNode.first->predecessors.push_back(predWepNode->second.second);
+		predWepNode->second.second->successors.push_back(wepNode.first);
+	}
+	// set parent as wep-predecessors if there are no other predecessors
+	if (ti->getParentActivity() != nullptr && element.getPredecessorCount() == 0) {
+		auto pWepNode = parentWepStack.top();
+		wepNode.first->predecessors.push_back(pWepNode.first);
+		pWepNode.first->successors.push_back(wepNode.first);
+	}
+	// set parent as wep-successors if there are no other successors
+	if (ti->getParentActivity() != nullptr && element.getSuccessorCount() == 0) {
+		auto pWepNode = parentWepStack.top();
+		wepNode.second->successors.push_back(pWepNode.second);
+		pWepNode.second->predecessors.push_back(wepNode.second);
+	}
+	// Check activity status
+	if (ti->getInstanceStatus() == wfr::wfs::ActivityInstanceStatus::ReadyForExecution
+			|| ti->getInstanceStatus() == wfr::wfs::ActivityInstanceStatus::WaitingForDependency) {
+		if (element.isCompositeActivity()) { // Composite activity branch
+			auto& composite = ti->as<wfs::CompositeActivityInstance>();
+			// Push parent on stacks
+			parentStack.push(ti);
+			parentWepStack.push(wepNode);
+			// Handle Children
+			handleChildren(composite, wepNode);
+			// Execute post-execution hooks (can we skip this?)
+			ti->executePostExecutionHooks(earlyEvaluationMode);
+			// Pop parent from stack
+			parentStack.pop();
+			parentWepStack.pop();
+		} else if (ti->activityType() == wfs::ActivityType::Assign) {
+			if (ti->getInstanceStatus() == wfr::wfs::ActivityInstanceStatus::ReadyForExecution) {
+				wepNode.first->as<wep::AssignWepNode>().activity.executeExecutionHook(earlyEvaluationMode);
+				readyToExecuteAssignNodes.push_back(&wepNode.first->as<wep::AssignWepNode>());
+			}
+		} else { // Atomic Activity branch
+			if (ti->getInstanceStatus() == wfr::wfs::ActivityInstanceStatus::ReadyForExecution) {
+				readyToExecuteAtomicNodes.push_back(&wepNode.first->as<wep::AtomicWepNode>());
+			}
+		}
+	} else /*if (ti->getInstanceStatus() == wfr::wfs::ActivityInstanceStatus::WaitingForData)*/ {
+		if (wepNode.first->getSuccessors().empty() && wepNode.first != wepNode.second) {
+			wepNode.first->successors.push_back(wepNode.second);
+			wepNode.second->predecessors.push_back(wepNode.first);
+		}
+	}
+}
+
+
+void WepBuilder::handleChildren(wfs::CompositeActivityInstance& composite, wepNodeMap_t::mapped_type& parent, wepNodeMap_t::mapped_type* prevIter) {
+	// Fetch child activities
+	std::vector<wfr::wfs::Activity*> children;
+	if (composite.isConditional()) {
+		children = composite.baseActivity.getChildren(composite.as<wfs::ConditionalActivityInstance>().getExecutionBranchID());
+	} else {
+		children = composite.baseActivity.getChildren();
+	}
+	if (composite.isLoop()) { // Loop Branch
+		auto& loop = composite.as<wfs::LoopActivityInstance>();
+		auto maxIterCount = loop.getIterationCount();
+		wfr_id_t i;
+		for ( i = loop.getHandledIterationCount(); i < maxIterCount; i++ ) {
+			// Create and push iteration wep-node
+			auto iterationWepNode = createIterationWepNodes(composite, parent, i);
+			iterationIdStack.push(i);
+			parentWepStack.push(iterationWepNode);
+			activityInstanceMap_t instanceMap;
+			// Visit children
+			for (auto& child : children) {
+				visitActivity(*child, instanceMap, wepNodeMap);
+			}
+			iterationIdStack.pop();
+			parentWepStack.pop();
+		}
+		loop.setHandledIterationCount(i);
+	} else { // Non-Loop Branch
+		iterationIdStack.push(0);
+		activityInstanceMap_t instanceMap;
+		// Visit children
+		for (auto& child : children) {
+			visitActivity(*child, instanceMap, wepNodeMap);
+		}
+		iterationIdStack.pop();
+	}
+	if (parent.first->successors.empty()) {
+		parent.first->successors.push_back(parent.second);
+		parent.second->predecessors.push_back(parent.first);
+	}
+}
+
+
+bool WepBuilder::updateExecutionPlan(std::queue<wfs::ActivityInstance*>& updateInstances) {
+	bool dirtyFlag = false;
+	bool isError = false;
+	readyToExecuteAtomicNodes.clear();
+	readyToExecuteAssignNodes.clear();
+
+	auto visitFunc = [&](wep::AbstractWepNode* n) -> bool {
+		bool localDirtyFlag = false;
+		bool stateChange = false;
+		auto& node = n->as<wep::ActivityWepNode>();
+
+		wepVisitationCount++;
+		n->addToVisitationCount();
+
+		/*std::cout << "Visiting node "
+				<< node.activity.baseActivity.baseTemplate.activityName
+				<< " (" << node.nodeID << ")"
+				<< " [" << print::toString(node.activity.activityType()) << "]: "
+				<< node.activity.baseActivity.baseTemplate.activityName << " - " << node.activity.baseActivity.activityID  << "-" << node.activity.instanceID
+				<< std::endl;*/
+
+		// This activity is currently running. Do nothing since this activity will be handled when it has finished
+		if (node.nodeType == wep::WepNodeType::Atomic && node.activity.as<wfs::AtomicActivityInstance>().ownedByExecutionLayer()) {
+			//std::cout << "Activity is currently running, abort" << std::endl;
+			return false;
+		}
+
+		// Execute pre-execution hooks
+		if (!node.activity.areAllInputsAvailable() && node.activity.getInstanceStatus() < wfr::wfs::ActivityInstanceStatus::Finished) {
+			node.activity.executePreExecutionHooks(earlyEvaluationMode);
+		}
+
+		//std::cout << "ActivityState: " << print::toString(node.activity.getInstanceStatus()) << " - " << print::toString(node.getLastKnownInstanceStatus()) << std::endl;
+		// Activity instance is dirty
+		if (node.getLastKnownInstanceStatus() != node.activity.getInstanceStatus() || node.activity.isDirtyState()) {
+			//std::cout << "State is dirty, old state: " << print::toString(node.getLastKnownInstanceStatus()) << std::endl;
+			// handle state-change
+			switch (node.nodeType) {
+				case wep::WepNodeType::Atomic:
+					//Check error condition
+					if (node.activity.getInstanceStatus() == wfs::ActivityInstanceStatus::Error) {
+						// Propagate Error to Parents
+						wfr::wfs::ActivityInstance* ai = node.activity.parentActivity;
+						while (ai != nullptr) {
+							ai->setErrorState(std::make_exception_ptr(wfr::exception::wfr_fatal_error("A child activity is in error state.")));
+							ai = ai->parentActivity;
+						}
+						wep->workflow.setErrorState(std::make_exception_ptr(wfr::exception::wfr_fatal_error("A child activity is in error state.")));
+						isError = true;
+					// Check finish condition
+					} else if (!isError && node.activity.getInstanceStatus() == wfs::ActivityInstanceStatus::Finished) {
+						// Propagate Finish state to Parents
+						wfr::wfs::ActivityInstance* ai = node.activity.parentActivity;
+						if (ai) {
+							//std::cout << "Propagate finish state to parent" << std::endl;
+							if (ai->isCompositeActivity()) {
+								wfr::wfs::CompositeActivityInstance& composite = ai->as<wfs::CompositeActivityInstance>();
+								composite.notifyChildHasFinished(node.activity);
+							} else {
+								throw wfr::exception::wfr_fatal_error("Non-Composite activity cannot be a parent.");
+							}
+						} else { // if there is no parent then it's the toplevel activity
+							//node.activity.baseActivity.workflow.setWorkflowStatus(wfs::WorkflowStatus::Finished);
+							node.activity.baseActivity.workflow.setFinishFlag(true);
+						}
+					} else if (node.activity.getInstanceStatus() == wfs::ActivityInstanceStatus::ReadyForExecution) {
+						readyToExecuteAtomicNodes.push_back(&node.as<wep::AtomicWepNode>());
+					}
+					break;
+				case wep::WepNodeType::Assign:
+					if (node.activity.getInstanceStatus() == wfs::ActivityInstanceStatus::ReadyForExecution) {
+						readyToExecuteAssignNodes.push_back(&node.as<wep::AssignWepNode>());
+						node.activity.as<wfs::AssignActivityInstance>().executeExecutionHook(earlyEvaluationMode);
+					}
+					break;
+				case wep::WepNodeType::CompositeStart:
+					//Check if we need to unroll
+					if (node.getLastKnownInstanceStatus() == wfs::ActivityInstanceStatus::WaitingForData) {
+						activityInstanceMap_t instanceMap;
+						parentStack.push(&node.activity);
+						auto wnmi = wepNodeMap.find(&node.activity);
+						if (wnmi != wepNodeMap.end()) {
+							wnmi->second.first->getSuccessors().clear();
+							wnmi->second.second->getPredecessors().clear();
+							parentWepStack.push((*wnmi).second);
+							// Handle Children
+							handleChildren(node.activity.as<wfs::CompositeActivityInstance>(), wnmi->second);
+							parentWepStack.pop();
+						} else {
+							throw wfr::exception::wfr_fatal_error("Could not find wepNodeMap entry.");
+						}
+
+						parentStack.pop();
+					}
+					break;
+				case wep::WepNodeType::IterationStart:
+					break;
+				case wep::WepNodeType::Hole:
+					break;
+				case wep::WepNodeType::CompositeEnd:
+					break;
+				case wep::WepNodeType::IterationEnd:
+					break;
+				default:
+					break;
+			}
+			// Further unroll while loop
+			if (!isError && node.activity.getInstanceStatus() == wfs::ActivityInstanceStatus::Finished) {
+				wfr::wfs::ActivityInstance* ai = node.activity.parentActivity;
+				if (ai != nullptr && ai->isCompositeActivity()) {
+					auto& composite = ai->as<wfs::CompositeActivityInstance>();
+					if (composite.baseActivity.activityType() == wfs::ActivityType::WhileLoop) {
+						auto& loop = composite.as<wfs::WhileLoopActivityInstance>();
+						if (loop.getHandledIterationCount() < loop.getIterationCount()) {
+							this->parentStack.push(&composite);
+							auto wnmi = wepNodeMap.find(&composite);
+							handleChildren(composite, wnmi->second);
+							this->parentStack.pop();
+						}
+					}
+				} else if (ai != nullptr) {
+					throw wfr::exception::wfr_fatal_error("Non-Composite activity cannot be a parent.");
+				}
+			}
+			node.updateInstanceStatus();
+			dirtyFlag = true;
+			localDirtyFlag = true;
+			stateChange = true;
+			node.activity.resetDirtyState();
+		} else {
+			switch (node.nodeType) {
+				case wep::WepNodeType::Atomic:
+					break;
+				case wep::WepNodeType::CompositeStart:
+				case wep::WepNodeType::IterationStart:
+				case wep::WepNodeType::Hole:
+				case wep::WepNodeType::CompositeEnd:
+				case wep::WepNodeType::IterationEnd:
+					localDirtyFlag = true;
+					break;
+				default:
+					break;
+			}
+		}
+
+		// Execute post-execution hooks
+		if (!node.activity.areAllOutputsAvailable()) {
+			try {
+				node.activity.executePostExecutionHooks(earlyEvaluationMode);
+			} catch (wfr::exception::wfr_exception& e) {
+				node.activity.setErrorState(std::current_exception());
+			}
+		}
+
+		if (earlyEvaluationMode) {
+			return localDirtyFlag;
+		} else if (localDirtyFlag) {
+
+			// if node is composite:
+			//   - if node is CompositeStart or IterationStart => true
+			//   - statechange => true
+
+			// if node is atomic:
+			//   - if parent is finished => true
+			//   - statechange => true
+			if (node.activity.isCompositeActivity()) {
+				if (node.nodeType == wep::WepNodeType::CompositeStart || node.nodeType == wep::WepNodeType::IterationStart) {
+					return true;
+				} else if (stateChange) {
+					return true;
+				} else {
+					if (node.activity.isLoop()) {
+						auto& loop = node.activity.as<wfs::LoopActivityInstance>();
+						if (!loop.isParallel && node.nodeType == wep::WepNodeType::IterationEnd) {
+							return true;
+						}
+					}
+					return false;
+				}
+			} else {
+				auto& parent = node.activity.parentActivity->as<wfs::CompositeActivityInstance>();
+				if (stateChange) {
+					return true;
+				} else if (parent.getInstanceStatus() >= wfs::ActivityInstanceStatus::Finished) {
+					return true;
+				} else {
+					return false;
+				}
+			}
+
+		} else {
+			return false;
+		}
+	};
+
+	wfr_size_t ec = updateInstances.size();
+	for (wfr_size_t t = 0; t < ec; t++) {
+		auto updateInstance = updateInstances.front();
+		updateInstances.pop();
+		auto startNode = wepNodeMap.find(updateInstance)->second.first;
+
+	    //std::cout << "handleActivityUpdate: " << updateInstance->baseActivity.baseTemplate.activityName << " - " << updateInstance->baseActivity.activityID  << "-" << updateInstance->instanceID << std::endl;
+
+
+		if (updateInstance->activityType() == wfs::ActivityType::Atomic) {
+			updateInstance->as<wfs::AtomicActivityInstance>().setOwnedByExecutionLayer(false);
+		}
+
+		/*if (startNode->withActivityInstance) {
+			auto& inode = startNode->as<wep::ActivityWepNode>();
+			std::cout << "Updating from Instance "
+						<< inode.activity.baseActivity.baseTemplate.activityName
+						<< " (" << inode.activity.instanceID << ")"
+						<< " [" << print::toString(inode.activity.baseActivity.activityType()) << "]"
+						<< std::endl;
+		} else {
+			std::cout << "Updating from unknown WEP node" << std::endl;
+		}*/
+
+		wfr::utils::wep::wepBreadthFirstSearch(startNode, visitFunc);
+	}
+	for (auto n : readyToExecuteAssignNodes) {
+		n->activity.as<wfs::AssignActivityInstance>().notifyParent();
+	}
+	readyToExecuteAssignNodes.clear();
+	if (dirtyFlag) {
+		wep->increaseVersion();
+	}
+	return dirtyFlag;
+}
+
+} /* namespace enactment */
+} /* namespace wfr */
+
+

src/enactment/cmake.inc

@@ -0,0 +1,5 @@
+
+set(wfr_cpp_files "${wfr_cpp_files}"
+	src/enactment/EnactmentEngine.cpp
+	src/enactment/WepBuilder.cpp
+)

src/eventsystem/Event.cpp

@@ -0,0 +1,202 @@
+
+#include <wfr/common/eventsystem/Event.h>
+#include <wfr/common/eventsystem/EventSystem.h>
+#include <wfr/common/wfs/Workflow.h>
+#include <wfr/common/wfs/WorkflowTemplate.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/common/wfs/ActivityTemplate.h>
+#include <wfr/common/wfs/AtomicActivityImplementation.h>
+#include <wfr/common/utils/print.h>
+#include <ctime>
+#include <sstream>
+#include <iomanip>
+
+
+namespace wfr {
+namespace eventsystem {
+
+
+std::string GeneralLoggingEvent::toString() const noexcept {
+	std::stringstream ss;
+	ss << print::toString(timestamp);
+	ss << ": General Message: ";
+	ss << logMessage;
+	return ss.str();
+}
+
+std::shared_ptr<GeneralLoggingEvent> GeneralLoggingEvent::fireEvent(const std::string& message, const wfr_timestamp_sc_t& timestamp, bool noLocking) noexcept {
+	auto event = std::make_shared<GeneralLoggingEvent>(message, timestamp);
+	EventSystem::getInstance().fireEvent(event, noLocking);
+	return event;
+}
+
+
+std::string WorkflowStatusEvent::toString() const noexcept {
+	std::stringstream ss;
+	ss << print::toString(timestamp);
+	ss << ": WorkflowStatusEvent: \"";
+	ss << workflow.baseTemplate.workflowName;
+	ss << "\"[";
+	ss << workflow.workflowID;
+	ss << "] changed from '";
+	ss << print::toString(oldStatus);
+	ss << "' to '";
+	ss << print::toString(newStatus);
+	ss << "'.";
+	return ss.str();
+}
+
+std::shared_ptr<WorkflowStatusEvent> WorkflowStatusEvent::fireEvent(wfr::wfs::Workflow& workflow,
+									wfr::wfs::WorkflowStatus& oldStatus,
+									wfr::wfs::WorkflowStatus& newStatus,
+									const wfr_timestamp_sc_t& timestamp,
+									bool noLocking ) noexcept {
+	auto event = std::make_shared<WorkflowStatusEvent>(workflow, oldStatus, newStatus, timestamp);
+	EventSystem::getInstance().fireEvent(event, noLocking);
+	return event;
+}
+
+
+
+std::string ActivityStatusEvent::toString() const noexcept {
+	std::stringstream ss;
+	ss << print::toString(timestamp);
+	ss << ": ActivityStatusEvent: \"";
+	ss << activity.baseTemplate.activityName;
+	ss << "\"[";
+	ss << activity.workflow.workflowID;
+	ss << "-";
+	ss << activity.activityID;
+	ss << "] changed from '";
+	ss << print::toString(oldStatus);
+	ss << "' to '";
+	ss << print::toString(newStatus);
+	ss << "'.";
+	return ss.str();
+}
+
+std::shared_ptr<ActivityStatusEvent> ActivityStatusEvent::fireEvent(wfr::wfs::Activity& activity,
+									wfr::wfs::ActivityStatus oldStatus,
+									wfr::wfs::ActivityStatus newStatus,
+									const wfr_timestamp_sc_t& timestamp,
+									bool noLocking ) noexcept {
+	auto event = std::make_shared<ActivityStatusEvent>(activity, oldStatus, newStatus, timestamp);
+	EventSystem::getInstance().fireEvent(event, noLocking);
+	return event;
+}
+
+
+
+
+std::string ActivityInstanceStatusEvent::toString() const noexcept {
+	std::stringstream ss;
+	ss << print::toString(timestamp);
+	ss << ": ActivityInstanceStatusEvent: \"";
+	ss << activityInstance.baseActivity.baseTemplate.activityName;
+	ss << "\"[";
+	ss << activityInstance.baseActivity.workflow.workflowID;
+	ss << "-";
+	ss << activityInstance.baseActivity.activityID;
+	ss << "-";
+	ss << activityInstance.instanceID;
+	ss << "] changed from '";
+	ss << print::toString(oldStatus);
+	ss << "' to '";
+	ss << print::toString(newStatus);
+	ss << "'.";
+	if (newStatus == wfs::ActivityInstanceStatus::Error) {
+		// Really no better way than to rethrow?
+		try {
+			auto eptr = activityInstance.getErrorCause();
+			if (eptr) {
+				std::rethrow_exception(eptr);
+			}
+		} catch(const std::exception& e) {
+			ss << " Error Cause: " << e.what();
+		}
+	}
+	return ss.str();
+}
+
+std::shared_ptr<ActivityInstanceStatusEvent> ActivityInstanceStatusEvent::fireEvent(wfr::wfs::ActivityInstance& activity,
+											wfr::wfs::ActivityInstanceStatus oldStatus,
+											wfr::wfs::ActivityInstanceStatus newStatus,
+											const wfr_timestamp_sc_t& timestamp,
+											bool noLocking ) noexcept {
+	auto event = std::make_shared<ActivityInstanceStatusEvent>(activity, oldStatus, newStatus, timestamp);
+	EventSystem::getInstance().fireEvent(event, noLocking);
+	return event;
+}
+
+std::string ComponentStatusEvent::toString() const noexcept {
+	std::stringstream ss;
+	ss << print::toString(timestamp);
+	ss << ": ComponentStatusEvent: ";
+	ss << print::toString(componentType);
+	ss << " changed from '";
+	ss << print::toString(oldStatus);
+	ss << "' to '";
+	ss << print::toString(newStatus);
+	ss << "'.";
+	return ss.str();
+}
+
+std::shared_ptr<ComponentStatusEvent> ComponentStatusEvent::fireEvent(	wfs::ComponentType componentType,
+										wfs::ComponentStatus oldStatus,
+										wfs::ComponentStatus newStatus,
+										wfr::wfs::Workflow* wf,
+										const wfr_timestamp_sc_t& timestamp,
+										bool noLocking ) noexcept {
+	auto event = std::make_shared<ComponentStatusEvent>(componentType, oldStatus, newStatus, wf, timestamp);
+	EventSystem::getInstance().fireEvent(event, noLocking);
+	return event;
+}
+
+
+
+std::string ComputingResourceStatusEvent::toString() const noexcept {
+	std::stringstream ss;
+	ss << print::toString(timestamp);
+	ss << ": DeviceInstanceStatusEvent: ";
+	ss << print::toString(deviceType);
+	ss << "[";
+	ss << deviceID;
+	ss << "] changed from '";
+	ss << print::toString(oldStatus);
+	ss << "' to '";
+	ss << print::toString(newStatus);
+	if (executedTask != nullptr) {
+		ss << "' - Task: ";
+		ss << executedTask->baseActivity.baseTemplate.activityName;
+		ss << " [InstID=";
+		ss << executedTask->instanceID;
+		ss << ", ImplID=";
+		ss << executedImpl->implementationID;
+		ss << ", slots=";
+		ss << deviceSlotCount;
+		ss << "].";
+	} else {
+		ss << "'.";
+	}
+	return ss.str();
+}
+
+std::shared_ptr<ComputingResourceStatusEvent> ComputingResourceStatusEvent::fireEvent(	wfr::wfs::Workflow* wf,
+												wfr::wfs::ActivityImplementationType deviceType,
+												wfr_id_t deviceID,
+												wfr::wfs::ComputingResourceStatus oldStatus,
+												wfr::wfs::ComputingResourceStatus newStatus,
+												wfr::wfs::ActivityInstance* executedTask,
+												wfr::wfs::AtomicActivityImplementation* executedImpl,
+												wfr_size_t deviceSlotCount,
+												wfr_timestamp_sc_t timestamp,
+												bool noLocking ) noexcept {
+	auto event = std::make_shared<ComputingResourceStatusEvent>(wf, deviceType, deviceID, oldStatus, newStatus, executedTask, executedImpl, deviceSlotCount, timestamp);
+	EventSystem::getInstance().fireEvent(event, noLocking);
+	return event;
+}
+
+} /* namespace eventsystem */
+} /* namespace wfr */
+

src/eventsystem/EventSystem.cpp

@@ -0,0 +1,203 @@
+/*
+ * EventSystem.cpp
+ *
+ *  Created on: 17.02.2014
+ *      Author: matthias
+ */
+
+#include <wfr/common/eventsystem/Event.h>
+#include <wfr/common/eventsystem/EventSystem.h>
+#include <wfr/common/eventsystem/EventListener.h>
+#include <wfr/common/exceptions/exceptions.h>
+#include <wfr/eventsystem/OStreamLogger.h>
+#include <wfr/eventsystem/SQLiteLogger.h>
+#include <boost/program_options.hpp>
+#include <boost/format.hpp>
+#include <iostream>
+
+namespace wfr {
+namespace eventsystem {
+
+
+EventSystem EventSystem::singleton;
+
+#ifdef WFR_ENABLE_STDLOGGER
+	std::unique_ptr<OStreamLogger> EventSystem::oStreamLogger;
+#endif
+
+#ifdef WFR_ENABLE_PERFDATABASE
+	std::unique_ptr<SQLiteLogger> EventSystem::sqliteLogger;
+#endif
+
+
+void EventSystem::init(int argc, char** argv) {
+    std::string sqliteFile;
+    bool disableStdLogger = false;
+    bool disableSqlite = false;
+
+    boost::program_options::options_description opts("EventSystem Options");
+    opts.add_options()
+			("help", "Print help message.")
+    		("disable-stdlogger", "Disable logging to standard out.")
+    		("disable-sqlite", "Disable logging to sqlite.")
+    		("sqlite-file", boost::program_options::value<std::string>(), "Set sqlite database file.");
+    boost::program_options::variables_map vm;
+    auto parsed = boost::program_options::command_line_parser(argc, argv).options(opts).allow_unregistered().run();
+    boost::program_options::store(parsed, vm);
+    boost::program_options::notify(vm);
+    if (vm.count("help")) {
+        boost::program_options::options_description show_opts("EventSystem Options");
+        show_opts.add_options()
+        		("disable-stdlogger", "Disable logging to standard out.")
+        		("disable-sqlite", "Disable logging to sqlite.")
+        		("sqlite-file", boost::program_options::value<std::string>(), "Set sqlite database file.");
+    	std::cerr << show_opts << std::endl;
+    	return;
+    }
+    if (vm.count("disable-stdlogger")) {
+    	disableStdLogger = true;
+    	std::cerr << "Logging to standard out is disabled." << std::endl;
+    }
+    if (vm.count("disable-sqlite")) {
+    	disableSqlite = true;
+    	std::cerr << "Logging to sqlite is disabled." << std::endl;
+    }
+    if (vm.count("sqlite-file")) {
+    	sqliteFile = vm["sqlite-file"].as<std::string>();
+    	std::cerr << "Sqlite database file is set to \""<< sqliteFile << "\"." << std::endl;
+    }
+
+	#ifdef WFR_ENABLE_STDLOGGER
+		if (!disableStdLogger) {
+			oStreamLogger = std::unique_ptr<OStreamLogger>(new wfr::eventsystem::OStreamLogger(std::cout));
+			singleton.addListener(*oStreamLogger.get());
+		}
+	#endif
+
+	#ifdef WFR_ENABLE_PERFDATABASE
+		if (!disableSqlite) {
+			if (sqliteFile.empty()) {
+				sqliteFile = "wfr_events.sqlite";
+			}
+			if (!sqliteFile.empty()) {
+				sqliteLogger = std::unique_ptr<SQLiteLogger>(new wfr::eventsystem::SQLiteLogger(sqliteFile));
+				if (sqliteLogger->isReady()) {
+					singleton.addListener(*sqliteLogger.get());
+				} else {
+					sqliteLogger.reset();
+				}
+			}
+		}
+	#endif
+}
+
+
+
+EventSystem::EventSystem() noexcept
+		: threadStarted(false) {
+	try {
+		eventThread = std::unique_ptr<std::thread>(new std::thread(eventThreadFunc, this));
+		while (!threadStarted.load()) {
+			std::this_thread::yield();
+		}
+	} catch (const std::system_error& e) {
+		std::string message = (boost::format("Caught exception while trying to start event thread: %1% (%2%)") % e.what() % e.code()).str();
+		throw exception::wfr_fatal_error(message, std::shared_ptr<std::exception>(new std::runtime_error(e)));
+	}
+}
+
+
+void EventSystem::finish() noexcept {
+	if (!singleton.isFinished) {
+		singleton.finishInstance();
+		#ifdef WFR_ENABLE_PERFDATABASE
+			if (sqliteLogger != nullptr) {
+				singleton.removeListener(*sqliteLogger.get());
+				sqliteLogger.reset();
+			}
+		#endif
+		#ifdef WFR_ENABLE_STDLOGGER
+			if (oStreamLogger != nullptr) {
+				singleton.removeListener(*oStreamLogger.get());
+				oStreamLogger.reset();
+			}
+		#endif
+	}
+}
+
+void EventSystem::finishInstance() noexcept {
+	{ // mutex region
+		std::lock_guard<std::mutex> threadLock(eventThreadMutex);
+		stopThread = true;
+		eventThreadCondVar.notify_one();
+	} // mutex region end
+	eventThread->join();
+	isFinished = true;
+}
+
+EventSystem::~EventSystem() noexcept {
+	if (!isFinished) {
+		finishInstance();
+	}
+}
+
+
+
+void EventSystem::eventThreadFunc(EventSystem* eventSystem) noexcept {
+	std::unique_lock<std::mutex> threadLock(eventSystem->eventThreadMutex);
+	eventSystem->threadStarted.store(true);
+	while (!eventSystem->stopThread || !eventSystem->eventQueue.empty()) {
+		// Go through event queue and notify listeners
+		while (!eventSystem->eventQueue.empty()) {
+			auto event = eventSystem->eventQueue.front();
+			eventSystem->eventQueue.pop();
+			// iterate through listeners
+			for (auto l : eventSystem->eventListeners) {
+				l->notifyEvent(event);
+			}
+		} // end while
+		// Go to sleep and wait for new events
+		if (!eventSystem->stopThread) {
+			eventSystem->eventThreadCondVar.wait(threadLock);
+		}
+	}
+	eventSystem->threadStarted = true;
+}
+
+
+void EventSystem::fireEvent(std::shared_ptr<Event> event, bool noLocking) noexcept {
+	if (event != nullptr) {
+		if (noLocking) {
+			eventQueue.push(event);
+			eventThreadCondVar.notify_one();
+		} else {
+			std::lock_guard<std::mutex> lock(eventThreadMutex);
+			eventQueue.push(event);
+			eventThreadCondVar.notify_one();
+		}
+	}
+}
+
+
+bool EventSystem::addListener(EventListener& listener) noexcept {
+	std::unique_lock<std::mutex> threadLock(eventThreadMutex);
+	if (eventListeners.find(&listener) == eventListeners.end()) {
+		eventListeners.insert(&listener);
+		return true;
+	} else {
+		return false;
+	}
+}
+
+bool EventSystem::removeListener(EventListener& listener) noexcept {
+	std::unique_lock<std::mutex> threadLock(eventThreadMutex);
+	if (eventListeners.find(&listener) != eventListeners.end()) {
+		eventListeners.erase(&listener);
+		return true;
+	} else {
+		return false;
+	}
+}
+
+} /* namespace eventsystem */
+} /* namespace wfr */

src/eventsystem/OStreamLogger.cpp

@@ -0,0 +1,15 @@
+
+#include <wfr/eventsystem/OStreamLogger.h>
+#include <wfr/common/eventsystem/Event.h>
+#include <iostream>
+
+namespace wfr {
+namespace eventsystem {
+
+void OStreamLogger::notifyEvent(const std::shared_ptr<Event>& event) noexcept {
+	oStream << event->toString() << std::endl;
+	oStream.flush();
+}
+
+} /* namespace eventsystem */
+} /* namespace wfr */

src/eventsystem/SQLiteLogger.cpp

@@ -0,0 +1,483 @@
+/*
+ * SQLiteLogger.cpp
+ *
+ *  Created on: 05.05.2014
+ *      Author: matthias
+ */
+
+#include <wfr/eventsystem/SQLiteLogger.h>
+#include <wfr/common/eventsystem/Event.h>
+#include <wfr/common/wfs/Workflow.h>
+#include <wfr/common/wfs/WorkflowTemplate.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/common/wfs/ActivityTemplate.h>
+#include <wfr/WorkflowRuntime.h>
+#include <ctime>
+#include <sstream>
+#include <thread>
+#include <iostream>
+#include <chrono>
+
+#include <sys/resource.h>
+#include <unistd.h>
+
+#include <boost/uuid/uuid.hpp>
+#include <boost/uuid/uuid_generators.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/uuid/uuid_io.hpp>
+
+#include <odb/sqlite/exceptions.hxx>
+#include <odb/schema-catalog.hxx>
+#include <SimpleDBModel-odb.hxx>
+#include <wfr/common/wfs/AtomicActivityImplementation.h>
+
+
+namespace wfr {
+namespace eventsystem {
+
+
+SQLiteLogger::SQLiteLogger(std::string filename) : filename(filename), statProcessedEvents(0), statWrittenEvents(0) {
+	std::stringstream ss;
+	ss << "Writting event data to sqlite database \"" << filename << "\"";
+	WorkflowRuntime::logMessage(ss.str());
+	try {
+		dbHandle = std::unique_ptr<odb::sqlite::database>(new odb::sqlite::database(filename, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE));
+	} catch (odb::exception& e) {
+		std::stringstream ss;
+		ss << "Could not open sqlite database \"" << filename << "\": " << e.what();
+		WorkflowRuntime::logMessage(ss.str());
+		dbHandle = nullptr;
+	}
+	try {
+		if (odb::schema_catalog::exists(*dbHandle, "")) {
+			odb::transaction t(dbHandle->begin());
+			odb::schema_catalog::create_schema(*dbHandle, "", false);
+			t.commit();
+		} else {
+			WorkflowRuntime::logMessage("No database schema detected ...");
+		}
+	} catch (odb::exception& e) {
+		std::stringstream ss;
+		ss << "Could not create sqlite database schema: " << e.what();
+		WorkflowRuntime::logMessage(ss.str());
+	}
+	loggerThread = std::unique_ptr<std::thread>(new std::thread(dbwriterfunc, this));
+	readyFlag = true;
+}
+
+
+SQLiteLogger::~SQLiteLogger() {
+	if (loggerThread) {
+		threadBreakFlag = true;
+		{
+			std::lock_guard<std::mutex> lock(threadMutex);
+			threadCondition.notify_one();
+		}
+		wfr_timestamp_sc_t timestamp;
+		bool queueNotEmpty = false;
+		if (!eventQueue.empty()) {
+			queueNotEmpty = true;
+			timestamp = std::chrono::system_clock::now();
+			WorkflowRuntime::logMessage("SQLiteLogger is still writing events ...");
+		}
+		loggerThread->join();
+		if (queueNotEmpty) {
+			wfr_duration_sc_t diff = std::chrono::system_clock::now() - timestamp;
+			std::chrono::milliseconds ms = std::chrono::duration_cast<std::chrono::milliseconds>(diff);
+			std::stringstream ss;
+			ss << "SQLiteLogger finished writing events (needed " << ms.count() << " ms).";
+			WorkflowRuntime::logMessage(ss.str());
+		}
+		loggerThread.reset();
+	}
+	if (dbHandle) {
+		dbHandle.reset();
+	}
+}
+
+
+void SQLiteLogger::dbwriterfunc(SQLiteLogger* logger) {
+
+	// Give thread a lower priority
+	pid_t myPID = getpid();
+	setpriority(PRIO_PROCESS, myPID, 19);
+
+	odb::session mysession;
+
+	logger->threadRunning = true;
+	odb::transaction* trans = nullptr;
+	uint32_t statsWrittenSnapshot = 0;
+	while (!logger->threadBreakFlag || !logger->eventQueue.empty()) {
+		std::shared_ptr<Event> event;
+		{
+			std::unique_lock<std::mutex> threadLock(logger->threadMutex);
+			while (logger->eventQueue.empty() && !logger->threadBreakFlag) {
+				logger->threadCondition.wait(threadLock);
+			}
+			if (!logger->eventQueue.empty()) {
+				event = logger->eventQueue.front();
+				logger->eventQueue.pop();
+			}
+		}
+		if (event) {
+			logger->statProcessedEvents++;
+
+			if (trans == nullptr) {
+				trans = new odb::transaction(logger->dbHandle->begin());
+				statsWrittenSnapshot = logger->statWrittenEvents;
+			}
+			/*switch (event->eventType) {
+				case EventType::WorkflowStatusEvent:
+					logger->handleWorkflowStatusEvent((WorkflowStatusEvent&)*event);
+					logger->statWrittenEvents++;
+					break;
+				case EventType::ActivityStatusEvent:
+					logger->handleActicityStatusEvent((ActivityStatusEvent&)*event);
+					logger->statWrittenEvents++;
+					break;
+				case EventType::ActivityInstanceStatusEvent:
+					logger->handleActicityInstanceStatusEvent((ActivityInstanceStatusEvent&)*event);
+					logger->statWrittenEvents++;
+					break;
+				case EventType::ComputingResourceStatusEvent:
+					logger->handleComputingResourceStatusEvent((ComputingResourceStatusEvent&)*event);
+					logger->statWrittenEvents++;
+					break;
+				case EventType::ComponentStatusEvent:
+					logger->handleComponentStatusEvent((ComponentStatusEvent&)*event);
+					logger->statWrittenEvents++;
+					break;
+				default:
+					break;
+			}*/
+			if (trans != nullptr && logger->statWrittenEvents - statsWrittenSnapshot > 100) {
+				trans->commit();
+				delete trans;
+				trans = nullptr;
+			}
+		}
+	}
+	if (trans != nullptr) {
+		trans->commit();
+		delete trans;
+		trans = nullptr;
+	}
+	{
+		std::stringstream ss;
+		ss << "SQLiteLogger: processed " << logger->statProcessedEvents << " Events.";
+		WorkflowRuntime::logMessage(ss.str());
+	}
+	{
+		std::stringstream ss;
+		ss << "SQLiteLogger: wrote " << logger->statWrittenEvents << " Events.";
+		WorkflowRuntime::logMessage(ss.str());
+	}
+	logger->threadRunning = false;
+}
+
+
+void SQLiteLogger::notifyEvent(const std::shared_ptr<Event>& event) noexcept {
+	std::lock_guard<std::mutex> lock(threadMutex);
+	eventQueue.push(event);
+	threadCondition.notify_one();
+}
+
+
+/*
+std::shared_ptr<odbmodel::Workflow> SQLiteLogger::getWorkflowFromWfsMap(wfs::Workflow& wf) {
+	std::shared_ptr<odbmodel::Workflow> ret;
+	auto iter = workflowMap.find(wf.getWorkflowID());
+	if (iter != workflowMap.end()) {
+		ret = (*iter).second;
+	}
+	return ret;
+}
+
+void SQLiteLogger::persistWorkflow(std::shared_ptr<odbmodel::Workflow>& pWf, wfs::Workflow& wf) {
+	dbHandle->persist<odbmodel::Workflow>(pWf);
+	workflowMap.insert(workflowMap_t::value_type(wf.getWorkflowID(), pWf));
+}
+
+std::shared_ptr<odbmodel::Activity> SQLiteLogger::getActivityFromWfsMap(wfs::Activity& act) {
+	std::shared_ptr<odbmodel::Activity> ret;
+	auto iter = activityMap.find(act.getActivityID());
+	if (iter != activityMap.end()) {
+		ret = (*iter).second;
+	}
+	return ret;
+}
+
+void SQLiteLogger::persistActivity(std::shared_ptr<odbmodel::Activity>& ap, wfs::Activity& a) {
+	dbHandle->persist<odbmodel::Activity>(ap);
+	activityMap.insert(activityMap_t::value_type(a.getActivityID(), ap));
+}
+
+std::shared_ptr<odbmodel::ActivityInstance> SQLiteLogger::getActivityInstanceFromWfsMap(wfs::ActivityInstance& act) {
+	std::shared_ptr<odbmodel::ActivityInstance> ret;
+	auto iter = activityInstanceMap.find(act.getInstanceID());
+	if (iter != activityInstanceMap.end()) {
+		ret = (*iter).second;
+	}
+	return ret;
+}
+
+void SQLiteLogger::persistActivityInstance(std::shared_ptr<odbmodel::ActivityInstance>& ap, wfs::ActivityInstance& a) {
+	dbHandle->persist<odbmodel::ActivityInstance>(ap);
+	activityInstanceMap.insert(activityInstanceMap_t::value_type(a.getInstanceID(), ap));
+}
+
+
+void SQLiteLogger::handleWorkflowStatusEvent(WorkflowStatusEvent &event) {
+	auto& wf = event.getWorkflow();
+	switch (event.getNewStatus()) {
+		case wfs::WorkflowStatus::Created: {
+
+				boost::uuids::uuid uuid = boost::uuids::random_generator()();
+				const std::string tmp = boost::lexical_cast<std::string>(uuid);
+
+				std::shared_ptr<odbmodel::Workflow> pWf(new odbmodel::Workflow(wf.getWorkflowID(), tmp));
+				pWf->setName(wf.getBaseTemplate().getWorkflowName());
+				persistWorkflow(pWf, wf);
+				writeWorkflowStatus(pWf, event);
+			}
+			break;
+		case wfs::WorkflowStatus::Initialized: {
+				auto wfp = getWorkflowFromWfsMap(wf);
+				auto ap = getActivityFromWfsMap(*wf.getTopLevelActivity());
+				if (wfp != nullptr && ap != nullptr) {
+					wfp->setToplevelActivity(ap);
+					dbHandle->update(wfp);
+					writeWorkflowStatus(wfp, event);
+				} else {
+					WorkflowRuntime::logMessage("Workflow or Activity not found in wfsMaps", true);
+				}
+			}
+			break;
+		default:
+			auto wfp = getWorkflowFromWfsMap(wf);
+			if (wfp != nullptr) {
+				writeWorkflowStatus(wfp, event);
+			}
+			break;
+	}
+}
+
+void SQLiteLogger::writeWorkflowStatus(std::shared_ptr<odbmodel::Workflow>& pWf, WorkflowStatusEvent &event) {
+
+	std::shared_ptr<odbmodel::WorkflowStatusEvent> pWSE(new odbmodel::WorkflowStatusEvent());
+	pWSE->setParentWorkflow(pWf);
+	pWSE->setOldStatus(EventMemberUtils::toString(event.getOldStatus()));
+	pWSE->setNewStatus(EventMemberUtils::toString(event.getNewStatus()));
+
+	typedef std::chrono::time_point<std::chrono::system_clock, std::chrono::nanoseconds> ns_t;
+	ns_t ns = std::chrono::time_point_cast<std::chrono::nanoseconds>(event.getTimestamp());
+	pWSE->setTimestamp(ns.time_since_epoch().count());
+
+	dbHandle->persist<odbmodel::WorkflowStatusEvent>(pWSE);
+}
+
+
+void SQLiteLogger::handleActicityStatusEvent(ActivityStatusEvent &event) {
+	auto& a = event.getActivity();
+	auto& wf = a.getWorkflow();
+	switch (event.getNewStatus()) {
+		case wfs::ActivityStatus::Created: {
+				auto wfp = getWorkflowFromWfsMap(wf);
+				if (wfp != nullptr) {
+					std::shared_ptr<odbmodel::Activity> pA(new odbmodel::Activity(a.getActivityID(), a.getBaseTemplate().getBranchID()));
+					pA->setName(a.getBaseTemplate().getActivityName());
+					pA->setType(EventMemberUtils::toString(a.getActivityType()));
+					pA->setParentWorkflow(wfp);
+					persistActivity(pA, a);
+					writeActivityStatus(pA, event);
+				} else {
+					WorkflowRuntime::logMessage("Workflow not found in wfsMaps", true);
+				}
+			}
+			break;
+		case wfs::ActivityStatus::Initialized: {
+				auto wfp = getWorkflowFromWfsMap(wf);
+				auto ap = getActivityFromWfsMap(a);
+				if (wfp != nullptr && ap != nullptr) {
+					for (wfr_size_t t = 0; t < a.getSuccessorCount(); t++) {
+						auto sp = getActivityFromWfsMap(*a.getSuccessorList()[t]);
+						if (sp != nullptr) {
+							ap->getSuccessors().push_back(sp);
+						}
+					}
+					for (wfr_size_t t = 0; t < a.getPredecessorCount(); t++) {
+						auto pp = getActivityFromWfsMap(*a.getPredecessorList()[t]);
+						if (pp != nullptr) {
+							ap->getPredecessors().push_back(pp);
+						}
+					}
+					wfs::Activity* parent = a.getParent();
+					if (parent != nullptr) {
+						auto pa = getActivityFromWfsMap(*parent);
+						ap->setParent(pa);
+					}
+					dbHandle->update(ap);
+					writeActivityStatus(ap, event);
+				} else {
+					WorkflowRuntime::logMessage("Workflow or Activity not found in wfsMaps", true);
+				}
+			}
+			break;
+		default:
+			auto ap = getActivityFromWfsMap(a);
+			if (ap != nullptr) {
+				writeActivityStatus(ap, event);
+			}
+			break;
+	}
+}
+
+void SQLiteLogger::writeActivityStatus(std::shared_ptr<odbmodel::Activity>& pA, ActivityStatusEvent &event) {
+
+	std::shared_ptr<odbmodel::ActivityStatusEvent> pASE(new odbmodel::ActivityStatusEvent());
+	pASE->setParentActivity(pA);
+	pASE->setOldStatus(EventMemberUtils::toString(event.getOldStatus()));
+	pASE->setNewStatus(EventMemberUtils::toString(event.getNewStatus()));
+
+	typedef std::chrono::time_point<std::chrono::system_clock, std::chrono::nanoseconds> ns_t;
+	ns_t ns = std::chrono::time_point_cast<std::chrono::nanoseconds>(event.getTimestamp());
+	pASE->setTimestamp(ns.time_since_epoch().count());
+
+	dbHandle->persist<odbmodel::ActivityStatusEvent>(pASE);
+}
+
+
+void SQLiteLogger::handleActicityInstanceStatusEvent(ActivityInstanceStatusEvent &event) {
+	auto& ai = event.getActivityInstance();
+	auto pai = ai.getParent();
+	auto& a = ai.getBaseActivity();
+	auto& wf = a.getWorkflow();
+	switch (event.getNewStatus()) {
+		case wfs::ActivityInstanceStatus::Created: {
+				auto wfp = getWorkflowFromWfsMap(wf);
+				auto ap = getActivityFromWfsMap(a);
+				if (wfp != nullptr && ap != nullptr) {
+					std::shared_ptr<odbmodel::ActivityInstance> pAI(new odbmodel::ActivityInstance(ai.getInstanceID(), ai.getIterationID()));
+					pAI->setBaseActivity(ap);
+					pAI->setParentWorkflow(wfp);
+					persistActivityInstance(pAI, ai);
+					writeActivityInstanceStatus(pAI, event);
+				} else {
+					WorkflowRuntime::logMessage("Workflow or Activity not found in wfsMaps", true);
+				}
+			}
+			break;
+		case wfs::ActivityInstanceStatus::Initialized: {
+				auto pAi = getActivityInstanceFromWfsMap(ai);
+				auto ppAi = getActivityInstanceFromWfsMap(*pai);
+				if (pAi != nullptr && ppAi != nullptr) {
+					pAi->setParentInstance(ppAi);
+					for (wfr_size_t t = 0; t < ai.getPredecessorCount(); t++) {
+						auto ppai = getActivityInstanceFromWfsMap(*ai.getPredecessor(t));
+						if (ppai != nullptr) {
+							pAi->getPredecessors().push_back(ppai);
+						}
+					}
+					dbHandle->update(pAi);
+					writeActivityInstanceStatus(pAi, event);
+				} else {
+					WorkflowRuntime::logMessage("Workflow or Activity not found in wfsMaps", true);
+				}
+			}
+			break;
+		default:
+			auto aip = getActivityInstanceFromWfsMap(ai);
+			if (aip != nullptr) {
+				writeActivityInstanceStatus(aip, event);
+			}
+			break;
+	}
+}
+
+
+void SQLiteLogger::writeActivityInstanceStatus(std::shared_ptr<odbmodel::ActivityInstance>& pA, ActivityInstanceStatusEvent &event) {
+
+	std::shared_ptr<odbmodel::ActivityInstanceStatusEvent> pASE(new odbmodel::ActivityInstanceStatusEvent());
+	pASE->setParentInstance(pA);
+	pASE->setOldStatus(EventMemberUtils::toString(event.getOldStatus()));
+	pASE->setNewStatus(EventMemberUtils::toString(event.getNewStatus()));
+
+	typedef std::chrono::time_point<std::chrono::system_clock, std::chrono::nanoseconds> ns_t;
+	ns_t ns = std::chrono::time_point_cast<std::chrono::nanoseconds>(event.getTimestamp());
+	pASE->setTimestamp(ns.time_since_epoch().count());
+
+	dbHandle->persist<odbmodel::ActivityInstanceStatusEvent>(pASE);
+}
+
+
+void SQLiteLogger::handleDeviceInstanceStatusEvent(DeviceInstanceStatusEvent &event) {
+	auto ai = event.getExecutedTask();
+	auto wf = event.getWorkflow();
+	std::shared_ptr<odbmodel::ActivityInstance> pai;
+	std::shared_ptr<odbmodel::Workflow> pwf;
+	if (ai != nullptr) {
+		pai = getActivityInstanceFromWfsMap(*ai);
+	}
+	if (wf != nullptr) {
+		pwf = getWorkflowFromWfsMap(*wf);
+	}
+	writeDeviceInstanceStatus(pwf, pai, event);
+}
+
+
+void SQLiteLogger::writeDeviceInstanceStatus(std::shared_ptr<odbmodel::Workflow>& pwf, std::shared_ptr<odbmodel::ActivityInstance>& pai, DeviceInstanceStatusEvent &event) {
+
+	std::shared_ptr<odbmodel::DeviceInstanceStatusEvent> pDSE(new odbmodel::DeviceInstanceStatusEvent());
+	pDSE->setParentWorkflow(pwf);
+	pDSE->setDeviceSlotCount(event.getDeviceSlotCount());
+	pDSE->setDeviceId(event.getDeviceID());
+	pDSE->setDeviceType(EventMemberUtils::toString(event.getDeviceType()));
+	if (pai != nullptr) {
+		pDSE->setExecutedInstance(pai);
+	}
+	if (event.getExecutedImpl() != nullptr) {
+		pDSE->setExecutedImplId(event.getExecutedImpl()->getImplementationId());
+	}
+
+	pDSE->setOldStatus(EventMemberUtils::toString(event.getOldStatus()));
+	pDSE->setNewStatus(EventMemberUtils::toString(event.getNewStatus()));
+
+	typedef std::chrono::time_point<std::chrono::system_clock, std::chrono::nanoseconds> ns_t;
+	ns_t ns = std::chrono::time_point_cast<std::chrono::nanoseconds>(event.getTimestamp());
+	pDSE->setTimestamp(ns.time_since_epoch().count());
+
+	dbHandle->persist<odbmodel::DeviceInstanceStatusEvent>(pDSE);
+}
+
+
+void SQLiteLogger::handleComponentStatusEvent(ComponentStatusEvent &event) {
+	auto wf = event.getWorkflow();
+	std::shared_ptr<odbmodel::ActivityInstance> pai;
+	std::shared_ptr<odbmodel::Workflow> pwf;
+	if (wf != nullptr) {
+		pwf = getWorkflowFromWfsMap(*wf);
+	}
+	writeComponentStatus(pwf, event);
+}
+
+
+void SQLiteLogger::writeComponentStatus(std::shared_ptr<odbmodel::Workflow>& pwf, ComponentStatusEvent &event) {
+
+	std::shared_ptr<odbmodel::ComponentStatusEvent> pCSE(new odbmodel::ComponentStatusEvent());
+	pCSE->setWorkflow(pwf);
+	pCSE->setComponentType(EventMemberUtils::toString(event.getComponentType()));
+
+	pCSE->setOldStatus(EventMemberUtils::toString(event.getOldStatus()));
+	pCSE->setNewStatus(EventMemberUtils::toString(event.getNewStatus()));
+
+	typedef std::chrono::time_point<std::chrono::system_clock, std::chrono::nanoseconds> ns_t;
+	ns_t ns = std::chrono::time_point_cast<std::chrono::nanoseconds>(event.getTimestamp());
+	pCSE->setTimestamp(ns.time_since_epoch().count());
+
+	dbHandle->persist<odbmodel::ComponentStatusEvent>(pCSE);
+}
+*/
+
+} /* namespace eventsystem */
+} /* namespace wfr */

src/eventsystem/cmake.inc

@@ -0,0 +1,32 @@
+ 
+set(wfr_cpp_files "${wfr_cpp_files}"
+	src/eventsystem/Event.cpp
+	src/eventsystem/EventSystem.cpp
+	src/eventsystem/OStreamLogger.cpp
+	src/eventsystem/SQLiteLogger.cpp
+)
+
+set(wfr_eventsystem_odbmodel_dir "include/wfr/eventsystem/odb-model")
+set(wfr_eventsystem_odbmodel_files "SimpleDBModel.h")
+set(wfr_eventsystem_odbmodel_outdir "${CMAKE_CURRENT_BINARY_DIR}/odb-model")
+list(APPEND wfr_add_include_dirs "${WFR_INCLUDE_DIR}/wfr/eventsystem/odb-model" "${CMAKE_CURRENT_BINARY_DIR}/odb-model")
+
+set(odb_args --database sqlite --std c++11 --schema-format embedded --schema-format sql --generate-schema --generate-query)
+
+set(wfr_eventsystem_odbmodel_incfiles "")
+foreach(_file ${wfr_eventsystem_odbmodel_files})
+   string(REPLACE ".h" "-odb.cxx" gencppfile ${_file})
+   string(REPLACE ".h" "-odb.hxx" genhxxfile ${_file})
+   add_custom_command(
+     OUTPUT ${wfr_eventsystem_odbmodel_outdir}/${gencppfile}
+     COMMAND mkdir -p ${wfr_eventsystem_odbmodel_outdir} 
+		&& ${Odb_Bin} ${odb_args} -o ${wfr_eventsystem_odbmodel_outdir} ${CMAKE_CURRENT_SOURCE_DIR}/${wfr_eventsystem_odbmodel_dir}/${_file} -I${WFR_INCLUDE_DIR}
+     DEPENDS libodb ${wfr_eventsystem_odbmodel_dir}/${_file}
+     COMMENT "Building odb-model for ${_file}"
+     )
+   list(APPEND wfr_cpp_files ${wfr_eventsystem_odbmodel_outdir}/${gencppfile})
+   list(APPEND wfr_eventsystem_odbmodel_incfiles ${wfr_eventsystem_odbmodel_outdir}/${genhxxfile})
+endforeach()
+
+install(FILES ${wfr_eventsystem_odbmodel_incfiles} DESTINATION ${CMAKE_INSTALL_PREFIX}/include)
+

src/execution/ExecutionManager.cpp

@@ -0,0 +1,122 @@
+
+#include <wfr/execution/ExecutionManager.h>
+#include <wfr/execution/ExecutionHandle.h>
+#include <wfr/execution/backends/pthreads/PThreadBackend.h>
+//#include <wfr/execution/backends/cuda/CudaBackend.h>
+#include <wfr/common/exceptions/exceptions.h>
+#include <wfr/common/scheduling/SchedulingDecision.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/common/wfs/AtomicActivityImplementation.h>
+#include <wfr/scheduling/AbstractScheduler.h>
+#include <wfr/WorkflowRuntime.h>
+#include <iostream>
+#include <sstream>
+#include <thread>
+#include <boost/program_options.hpp>
+#include <wfr/common/resource/Resource.h>
+#include <wfr/common/resource/ResourceManager.h>
+
+namespace wfr {
+namespace execution {
+
+
+ExecutionManager ExecutionManager::singleton;
+
+void ExecutionManager::init(int argc, char** argv) {
+    boost::program_options::options_description opts("Execution Options");
+    opts.add_options()
+    	    ("help", "Print help message.");
+    boost::program_options::variables_map vm;
+    auto parsed = boost::program_options::command_line_parser(argc, argv).options(opts).allow_unregistered().run();
+    boost::program_options::store(parsed, vm);
+    boost::program_options::notify(vm);
+    if (vm.count("help")) {
+    	std::cerr << opts << std::endl;
+    	return;
+    }
+
+    auto& resourceManager = resource::ResourceManager::getInstance();
+
+    std::set<resource::ComputingResource*> pthreadResources;
+	for (auto dev : resourceManager.getComputingResources()) {
+		if (dev->hasCapability(resource::ResourceCapabilityType::GenericCPU)) {
+			pthreadResources.insert(dev);
+		}
+	}
+	singleton.pThreadBackend = new PThreadBackend(singleton, pthreadResources);
+
+    std::set<resource::ComputingResource*> cudaResources;
+	for (auto dev : resourceManager.getComputingResources()) {
+		if (dev->hasCapability(resource::ResourceCapabilityType::CUDA)) {
+			cudaResources.insert(dev);
+		}
+	}
+	//singleton.cudaBackend = new CudaBackend(singleton, cudaResources);
+
+	{
+		std::stringstream ss;
+		ss << "ExecutionManager: PthreadBackend: using " << singleton.pThreadBackend->getTotalSlotCount() << " pthread threads.";
+		wfr::WorkflowRuntime::logMessage(ss.str());
+	}
+}
+
+void ExecutionManager::finish() {
+	delete singleton.pThreadBackend;
+}
+
+ExecutionManager::ExecutionManager() : pThreadBackend(nullptr) {
+}
+
+ExecutionManager::~ExecutionManager() {
+}
+
+
+bool ExecutionManager::executeSchedulingDecision( std::shared_ptr<scheduling::SchedulingDecision> sd, scheduling::AbstractScheduler& scheduler) {
+	std::lock_guard<std::mutex> lock(managerMutex);
+	bool retval = false;
+	auto type = sd->activityImplementation->implementationType;
+	switch (type) {
+		case wfs::ActivityImplementationType::PThread:
+			{
+				ExecutionHandle* handle = new ExecutionHandle(sd);
+				if (pThreadBackend->queueTaskExecution(*handle)) {
+					schedulerHandleMap.insert({handle, &scheduler});
+					retval = true;
+				} else {
+					throw exception::wfr_fatal_error("ToDo: Not implemented");
+				}
+			}
+			break;
+		case wfs::ActivityImplementationType::CUDA:
+			/*{
+				ExecutionHandle* handle = new ExecutionHandle(&sd);
+				if (cudaBackend->queueTaskExecution(*handle)) {
+					schedulerHandleMap.insert({handle, &scheduler});
+					retval = true;
+				} else {
+					throw exception::wfr_fatal_error("ToDo: Not implemented");
+				}
+			}*/
+			break;
+		case wfs::ActivityImplementationType::OpenCL:
+		case wfs::ActivityImplementationType::OpenMP:
+		default:
+			sd->activityInstance->setErrorState(std::make_exception_ptr(wfr::exception::wfr_exception("Unknown Device Type")));
+			retval = false;
+			break;
+	}
+	return retval;
+}
+
+void ExecutionManager::notifyExecutionHandleUpdate(ExecutionHandle& handle) {
+	std::lock_guard<std::mutex> lock(managerMutex);
+	auto iter = schedulerHandleMap.find(&handle);
+	if (iter != schedulerHandleMap.end()) {
+		(*iter).second->notifySchedulingDecisionStatusUpdate(handle.schedulingDecision);
+		schedulerHandleMap.erase(iter);
+	}
+	delete &handle;
+}
+
+} /* namespace execution */
+} /* namespace wfr */

src/execution/backends/pthreads/PThreadBackend.cpp

@@ -0,0 +1,124 @@
+
+#include <wfr/common/resource/Resource.h>
+#include <wfr/execution/backends/pthreads/PThreadBackend.h>
+#include <wfr/execution/backends/pthreads/PThreadWorker.h>
+#include <wfr/execution/ExecutionHandle.h>
+#include <wfr/execution/ExecutionManager.h>
+#include <wfr/common/scheduling/SchedulingDecision.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/WorkflowRuntime.h>
+#include <wfr/common/eventsystem/Event.h>
+#include <wfr/common/eventsystem/EventSystem.h>
+#include <sstream>
+
+namespace wfr {
+namespace execution {
+
+PThreadBackend::PThreadBackend(ExecutionManager& manager, const std::set<resource::ComputingResource*>& resources)
+		: manager(manager), resources(resources), maxWorkerCount(0) {
+	for (auto dev : resources) {
+		maxWorkerCount += dev->getMaxSlotCount();
+	}
+}
+
+PThreadBackend::~PThreadBackend() {
+	std::unique_lock<std::mutex> objLock(objMutex);
+	while (currentBusyWorkerCount > 0 || !taskQueue.empty()) {
+		objLock.unlock();
+		std::this_thread::yield();
+		objLock.lock();
+	}
+	for (auto w : idleWorkers) {
+		w->finishThread();
+	}
+	isShutdown = true;
+	workerCondVar.notify_all();
+	for (auto w : idleWorkers) {
+		objLock.unlock();
+		w->join();
+		objLock.lock();
+		delete w;
+	}
+	idleWorkers.clear();
+}
+
+void PThreadBackend::notifyExecutionHandleUpdate(ExecutionHandle& handle, PThreadWorker& worker) {
+	manager.notifyExecutionHandleUpdate(handle);
+}
+
+
+ExecutionHandle* PThreadBackend::popTaskExecution(PThreadWorker& worker) {
+	std::unique_lock<std::mutex> objLock(objMutex);
+	if (worker.isBusy()) {
+		occupiedSlotCount-=worker.getUsedSlotCount();
+		worker.setUsedSlotCount(0);
+		currentBusyWorkerCount--;
+		worker.setBusy(false);
+		worker.setStatus(wfr::wfs::ComputingResourceStatus::Idle);
+	}
+	bool wasPutIntoIdlePool = false;
+	ExecutionHandle* handle = nullptr;
+	while (!isShutdown && handle == nullptr) {
+		while (!isShutdown && taskQueue.empty()) {
+			if (!wasPutIntoIdlePool) {
+				idleWorkers.insert(&worker);
+				wasPutIntoIdlePool = true;
+			}
+			worker.setStatus(wfr::wfs::ComputingResourceStatus::Sleep);
+			workerCondVar.wait(objLock);
+			worker.setStatus(wfr::wfs::ComputingResourceStatus::WokenUp);
+		}
+		if (!isShutdown) {
+			handle = taskQueue.front();
+			taskQueue.pop();
+			slotsInQueue -= handle->schedulingDecision->slotCount;
+		}
+	}
+	if (!isShutdown) {
+		if (wasPutIntoIdlePool) {
+			idleWorkers.erase(idleWorkers.find(&worker));
+		}
+		currentBusyWorkerCount++;
+		worker.setBusy(true);
+		worker.setUsedSlotCount(handle->schedulingDecision->slotCount);
+		auto sd = handle->schedulingDecision;
+		auto reservation = sd->reservationHandle;
+		if (reservation.isValid()) {
+			reservation.resource->markReservedSlotsBusy(reservation.slotCount);
+		} else if (sd->resource) {
+			sd->resource->markFreeSlotsBusy(sd->slotCount);
+		}
+		occupiedSlotCount+=worker.getUsedSlotCount();
+		//#ifdef WFR_ENABLE_STATISTICS
+		//	statistics::WfrStatistics::setMaxObservedBusyPthreadCount(currentBusyWorkerCount);
+		//#endif
+		worker.setStatus(wfr::wfs::ComputingResourceStatus::Busy, handle->schedulingDecision);
+		return handle;
+	} else {
+		return nullptr;
+	}
+}
+
+bool PThreadBackend::queueTaskExecution(ExecutionHandle& handle) {
+	bool retval = true;
+	std::lock_guard<std::mutex> objLock(objMutex);
+	taskQueue.push(&handle);
+	slotsInQueue += handle.schedulingDecision->slotCount;
+	handle.schedulingDecision->activityInstance->setInstanceStatus(wfs::ActivityInstanceStatus::Queued);
+	if (slotsInQueue > idleWorkers.size()) {
+		if (maxWorkerCount == 0 || currentTotalWorkerCount < maxWorkerCount) {
+			PThreadWorker::createNewThread(*this);
+			currentTotalWorkerCount++;
+			//#ifdef WFR_ENABLE_STATISTICS
+			//	statistics::WfrStatistics::setMaxObservedCurrentPthreadCount(currentTotalWorkerCount);
+			//#endif
+		} else {
+			handle.schedulingDecision->activityInstance->setInstanceStatus(wfs::ActivityInstanceStatus::WaitingForFreeResources);
+		}
+	}
+	workerCondVar.notify_all();
+	return retval;
+}
+
+} /* namespace execution */
+} /* namespace wfr */

src/execution/backends/pthreads/PThreadWorker.cpp

@@ -0,0 +1,107 @@
+
+#include <wfr/execution/backends/pthreads/PThreadWorker.h>
+#include <wfr/common/exceptions/exceptions.h>
+#include <wfr/execution/ExecutionHandle.h>
+#include <wfr/execution/backends/pthreads/PThreadBackend.h>
+#include <wfr/common/scheduling/SchedulingDecision.h>
+#include <wfr/common/wfs/AtomicActivityImplementation.h>
+#include <wfr/common/wfs/implementations/PThreadActivityImplementation.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/common/resource/Resource.h>
+#include <wfr/WorkflowRuntime.h>
+#include <wfr/common/eventsystem/Event.h>
+#include <wfr/common/eventsystem/EventSystem.h>
+#include <sstream>
+
+namespace wfr {
+namespace execution {
+
+std::atomic<wfr_id_t> PThreadWorker::nextThreadID = ATOMIC_VAR_INIT(1);
+
+wfr_id_t PThreadWorker::getNextThreadID() {
+	return nextThreadID.fetch_add(1);
+}
+
+PThreadWorker::PThreadWorker(PThreadBackend& backend)
+	: threadID(getNextThreadID()), stopThread(false), threadRunning(false), threadBusy(false), backend(backend), thread(nullptr),
+	  usedSlotCount(0), status(wfs::ComputingResourceStatus::Created) {
+	thread = new std::thread(workerFunc, this);
+
+	wfr::eventsystem::ComputingResourceStatusEvent::fireEvent(nullptr, wfr::wfs::ActivityImplementationType::PThread, threadID,
+			wfr::wfs::ComputingResourceStatus::Unknown, wfr::wfs::ComputingResourceStatus::Created);
+}
+
+PThreadWorker::~PThreadWorker() {
+	if (thread != nullptr) {
+		while (threadRunning) {
+			std::this_thread::yield();
+		}
+		delete thread;
+		thread = nullptr;
+	}
+}
+
+
+void PThreadWorker::workerFunc(PThreadWorker* threadObj) {
+	threadObj->threadRunning = true;
+	while (!threadObj->stopThread) {
+		ExecutionHandle* handle;
+		while ((handle = threadObj->backend.popTaskExecution(*threadObj)) != nullptr) {
+			auto sd = handle->schedulingDecision;
+			auto execFunc = sd->activityImplementation->as<wfs::PThreadActivityImplementation>().executionFuncHook;
+			sd->activityInstance->setInstanceStatus(wfr::wfs::ActivityInstanceStatus::Running);
+			threadObj->setStatus(wfs::ComputingResourceStatus::ExecutingTask, handle->schedulingDecision);
+			handle->setState(ExecutionHandleState::Running);
+			try {
+				execFunc(*sd->activityInstance);
+				auto reservation = sd->reservationHandle;
+				if (reservation.isValid()) {
+					reservation.resource->releaseBusySlots(reservation.slotCount);
+					reservation.invalidateReservation();
+				} else if (sd->resource) {
+					sd->resource->releaseBusySlots(sd->slotCount);
+				}
+				handle->setState(ExecutionHandleState::Finished);
+				threadObj->setStatus(wfr::wfs::ComputingResourceStatus::FinishedTask, handle->schedulingDecision);
+				sd->activityInstance->setInstanceStatus(wfr::wfs::ActivityInstanceStatus::Finished);
+			} catch (exception::wfr_exception& e) {
+				// Need to implement better error handling, but for now just rethrow
+				throw e;
+				/*
+				handle->setErrorCause(std::current_exception());
+				threadObj->setStatus(wfr::wfs::DeviceInstanceStatus::Error, &handle->getSchedulingDecision());
+				sd.getActivityInstance().setInstanceStatus(wfr::wfs::ActivityInstanceStatus::Error);
+				*/
+			}
+			threadObj->backend.notifyExecutionHandleUpdate(*handle, *threadObj);
+		}
+	}
+	threadObj->threadRunning = false;
+}
+
+
+void PThreadWorker::finishThread() {
+	stopThread = true;
+}
+
+void PThreadWorker::join() {
+	if (thread != nullptr) {
+		thread->join();
+	}
+}
+
+void PThreadWorker::setStatus(wfs::ComputingResourceStatus status, std::shared_ptr<wfr::scheduling::SchedulingDecision> sd) {
+	wfr::eventsystem::ComputingResourceStatusEvent::fireEvent( (sd.get() == nullptr) ? nullptr : &sd->activityInstance->baseActivity.workflow,
+			wfr::wfs::ActivityImplementationType::PThread, threadID, this->status, status, (sd.get() == nullptr) ? nullptr : sd->activityInstance,
+			(sd.get() == nullptr) ? nullptr : sd->activityImplementation,  (sd.get() == nullptr) ? 0 : sd->slotCount);
+	this->status = status;
+}
+
+PThreadWorker* PThreadWorker::createNewThread(PThreadBackend& backend) {
+	auto thread = new PThreadWorker(backend);
+	return thread;
+}
+
+} /* namespace execution */
+} /* namespace wfr */

src/execution/cmake.inc

@@ -0,0 +1,6 @@
+
+set(wfr_cpp_files "${wfr_cpp_files}"
+	src/execution/backends/pthreads/PThreadWorker.cpp
+	src/execution/backends/pthreads/PThreadBackend.cpp
+	src/execution/ExecutionManager.cpp
+)

src/prediction/PredictionManager.cpp

@@ -0,0 +1,71 @@
+
+#include <boost/program_options.hpp>
+#include <boost/format.hpp>
+#include <wfr/prediction/services/oracle/OraclePredictionService.h>
+#include <wfr/prediction/PredictionManager.h>
+#include <wfr/prediction/PredictionService.h>
+#include <iostream>
+
+
+
+namespace wfr {
+namespace prediction {
+
+
+PredictionManager PredictionManager::singleton;
+
+void PredictionManager::init(int argc, char** argv) {
+	std::string predictionConfig;
+    boost::program_options::options_description opts("Prediction Options");
+    opts.add_options()
+    	    ("predictionconfig", boost::program_options::value<std::string>(), "Set prediction service config.")
+    	    ("help", "Print help message.");
+    boost::program_options::variables_map vm;
+    auto parsed = boost::program_options::command_line_parser(argc, argv).options(opts).allow_unregistered().run();
+    boost::program_options::store(parsed, vm);
+    boost::program_options::notify(vm);
+    if (vm.count("help")) {
+    	std::cerr << opts << std::endl;
+    	return;
+    }
+    if (vm.count("predictionconfig")) {
+    	predictionConfig = vm["predictionconfig"].as<std::string>();
+    	std::cerr << "Prediction Service Config is set to \""<< predictionConfig << "\"." << std::endl;
+    }
+    singleton.service = new OraclePredictionService(predictionConfig);
+}
+
+
+void PredictionManager::finish() noexcept {
+}
+
+
+PredictionManager::~PredictionManager() noexcept {
+	if (service) {
+		delete service;
+		service = nullptr;
+	}
+}
+
+
+ExecutionTimeResult PredictionManager::predictExecutionTime(wfs::ActivityInstance& instance, wfs::AtomicActivityImplementation& implementation, resource::ComputingResourceType& resourceType, const wfr_size_t& slots) noexcept {
+	if (service) {
+		return service->predictExecutionTime(instance, implementation, resourceType, slots);
+	} else {
+		std::cerr << "No Prediction Service configured." << std::endl;
+		return ExecutionTimeResult();
+	}
+}
+
+CommunicationTimeResult PredictionManager::predictCommunicationTime(wfs::ActivityInstance& srcInstance, resource::ComputingResource& srcResource,
+		wfs::ActivityInstance& destInstance, resource::ComputingResource& destResource) noexcept {
+	if (service) {
+		return service->predictCommunicationTime(srcInstance, srcResource, destInstance, destResource);
+	} else {
+		std::cerr << "No Prediction Service configured." << std::endl;
+		return CommunicationTimeResult();
+	}
+}
+
+} /* namespace prediction */
+} /* namespace wfr */

src/prediction/cmake.inc

@@ -0,0 +1,6 @@
+ 
+set(wfr_cpp_files "${wfr_cpp_files}"
+	src/prediction/PredictionManager.cpp
+	src/prediction/services/oracle/OraclePredictionService.cpp
+)
+

src/prediction/services/oracle/OraclePredictionService.cpp

@@ -0,0 +1,49 @@
+
+#include <wfr/prediction/services/oracle/OraclePredictionService.h>
+#include <wfr/common/resource/Resource.h>
+#include <wfr/common/wfs/ActivityTemplate.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/common/wfs/AtomicActivityImplementation.h>
+#include <boost/algorithm/string/predicate.hpp>
+#include <boost/lexical_cast.hpp>
+#include <iostream>
+
+
+namespace wfr {
+namespace prediction {
+
+
+OraclePredictionService::OraclePredictionService(const std::string& config) {
+
+}
+
+
+ExecutionTimeResult OraclePredictionService::predictExecutionTime(wfs::ActivityInstance& instance, wfs::AtomicActivityImplementation& implementation,
+		resource::ComputingResourceType& resourceType, const wfr_size_t& slots) {
+	auto& props = implementation.getProperties();
+	for (auto& prop : props.getProperties()) {
+		if (boost::algorithm::starts_with(prop.first, "exectime.")) {
+			auto devname = prop.first.substr(9);
+			if (devname.compare(resourceType.resourceTypeName) == 0) {
+				double exectime = boost::lexical_cast<double>(prop.second);
+				std::cout << "Exectime: " << devname << " = " << exectime << std::endl;
+				return ExecutionTimeResult(&instance, &implementation, &resourceType, NumericalMeasurement(exectime, 0));
+			}
+		}
+	}
+	return ExecutionTimeResult(&instance, &implementation, &resourceType, NumericalMeasurement(1, 0));
+}
+
+
+CommunicationTimeResult OraclePredictionService::predictCommunicationTime(wfs::ActivityInstance& srcInstance, resource::ComputingResource& srcResource,
+		wfs::ActivityInstance& destInstance, resource::ComputingResource& destResource) {
+	return CommunicationTimeResult(&srcInstance, &srcResource, &destInstance, &destResource, NumericalMeasurement(0, 0, true));
+}
+
+
+} /* namespace prediction */
+} /* namespace wfr */
+
+
+

src/scheduling/AbstractScheduler.cpp

@@ -0,0 +1,108 @@
+
+#include "wfr/scheduling/AbstractScheduler.h"
+#include <wfr/prediction/PredictionManager.h>
+#include <wfr/common/resource/ResourceManager.h>
+#include <wfr/WorkflowRuntime.h>
+#include <wfr/common/wep/WorkflowExecutionPlan.h>
+#include <wfr/common/wep/AbstractWepNode.h>
+#include <wfr/common/wep/activities/AtomicWepNode.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/common/wfs/AtomicActivityImplementation.h>
+#include <wfr/common/wfs/instances/AtomicActivityInstance.h>
+#include <wfr/enactment/EnactmentEngine.h>
+#include <wfr/common/scheduling/SchedulingDecision.h>
+#include <wfr/execution/ExecutionManager.h>
+#include <wfr/wep/utils/WepSearch.h>
+#include <iostream>
+#include <set>
+#include <queue>
+
+
+namespace wfr {
+namespace scheduling {
+
+SchedulerBase::SchedulerBase() : simulationMode(WorkflowRuntime::getGlobalConfig<bool>("sim-mode")) {}
+
+
+SchedulerBase::~SchedulerBase() noexcept {
+	wepMap.clear();
+}
+
+
+void SchedulerBase::registerWEP(wep::WorkflowExecutionPlan& wep, enactment::EnactmentEngine& ee) noexcept {
+	{
+		std::unique_lock<std::mutex> objLock(schedulerMutex);
+		wepMap.emplace(std::make_pair(&wep, WepMapEntry(&ee, &wep)));
+		_registerWEP(wepMap.find(&wep)->second, objLock);
+	}
+}
+
+
+void SchedulerBase::unregisterWEP(wep::WorkflowExecutionPlan& wep) noexcept {
+	std::unique_lock<std::mutex> objLock(schedulerMutex);
+	auto wepe = wepMap.find(&wep);
+	if (wepe != wepMap.end()) {
+		// Wait till all pending scheduling decisions have been handled
+		while (!wepe->second.decisions.empty()) {
+			objLock.unlock();
+			std::this_thread::yield();
+			objLock.lock();
+		}
+		_unregisterWEP(wepe->second, objLock);
+		wepMap.erase(wepe);
+	}
+}
+
+
+void SchedulerBase::notifyWEPUpdate(wep::WorkflowExecutionPlan& wep, std::set<wep::AtomicWepNode*> readyToExecute) noexcept {
+	std::unique_lock<std::mutex> objLock(schedulerMutex);
+	auto wepe = wepMap.find(&wep);
+	if (wepe != wepMap.end()) {
+		_notifyWEPUpdate(wepe->second, readyToExecute, objLock);
+	}
+}
+
+
+bool SchedulerBase::executeSchedulingDecision(WepMapEntry& wepe, std::shared_ptr<SchedulingDecision> sd, std::unique_lock<std::mutex>& schedulerLock) noexcept {
+	wepe.decisions.insert(sd);
+	schedulerLock.unlock();
+	sd->activityInstance->setSchedulingDecision(sd);
+	if (sd->activityInstance->activityType() == wfs::ActivityType::Atomic) {
+		sd->activityInstance->as<wfs::AtomicActivityInstance>().setOwnedByExecutionLayer(true);
+	}
+	if (sd->getPrediction().isValid()) {
+		cummulativePredictionTime += sd->getPrediction().executionTime.getValue();
+	}
+	bool retval = execution::ExecutionManager::getInstance().executeSchedulingDecision(sd, *this);
+	if (!retval) {
+		// Some error happened (Notify enactment engine)
+		notifySchedulingDecisionStatusUpdate(sd);
+		sd->activityInstance->setSchedulingDecision();
+		wepe.decisions.erase(sd);
+	}
+	schedulerLock.lock();
+	return retval;
+}
+
+
+void SchedulerBase::notifySchedulingDecisionStatusUpdate(std::shared_ptr<SchedulingDecision> sd) noexcept {
+	{
+		std::unique_lock<std::mutex> objLock(schedulerMutex);
+		auto wepe = wepMap.find(sd->wep);
+		if (wepe != wepMap.end()) {
+			_notifySchedulingDecisionStatusUpdate(wepe->second, sd, objLock);
+			// Notify enactment engine
+			auto ee = wepe->second.engine;
+			ee->notifyActivityInstanceStateUpdate(*sd->activityInstance, *this);
+			wepe->second.decisions.erase(sd);
+			sd->activityInstance->setSchedulingDecision();
+		}
+	}
+}
+
+
+} /* namespace scheduling */
+} /* namespace wfr */
+
+
+

src/scheduling/SchedulerManager.cpp

@@ -0,0 +1,107 @@
+
+#include <wfr/scheduling/SchedulerManager.h>
+#include <wfr/scheduling/AbstractScheduler.h>
+#include <wfr/common/exceptions/exceptions.h>
+#include <iostream>
+#include <boost/program_options.hpp>
+#include <boost/format.hpp>
+#include <wfr/prediction/PredictionManager.h>
+#include <wfr/common/resource/ResourceManager.h>
+#include <wfr/scheduling/scheduler/dummy/DummyScheduler.h>
+#include <wfr/scheduling/scheduler/heft/HEFTScheduler.h>
+#include <wfr/scheduling/scheduler/random/RandomScheduler.h>
+#include <wfr/scheduling/scheduler/justintime/JustInTimeScheduler.h>
+
+namespace wfr {
+namespace scheduling {
+
+
+SchedulerManager* SchedulerManager::singleton = nullptr;
+
+
+void SchedulerManager::init(int argc, char** argv) {
+    std::string defaultScheduler = "dummy";
+
+    boost::program_options::options_description opts("SchedulerFactory Options");
+    opts.add_options()
+    	    ("help", "Print help message.")
+    	    ("list-schedulers", "List available schedulers.")
+    		("default-scheduler", boost::program_options::value<std::string>(), "Set the default scheduler.");
+    boost::program_options::variables_map vm;
+    auto parsed = boost::program_options::command_line_parser(argc, argv).options(opts).allow_unregistered().run();
+    boost::program_options::store(parsed, vm);
+    boost::program_options::notify(vm);
+    if (vm.count("help")) {
+    	std::cerr << opts << std::endl;
+    	return;
+    }
+    if (vm.count("default-scheduler")) {
+    	defaultScheduler = vm["default-scheduler"].as<std::string>();
+    	std::cerr << "Default scheduler is set to \""<< defaultScheduler << "\"." << std::endl;
+    }
+
+	singleton = new SchedulerManager(defaultScheduler);
+
+	if (vm.count("list-schedulers")) {
+		std::cerr << "Available schedulers:" << std::endl;
+		for (auto item : singleton->schedulerFactories) {
+			std::cerr << "\t" << item.first << std::endl;
+		}
+		exit(0);
+	}
+}
+
+void SchedulerManager::finish() {
+	if (singleton != nullptr) {
+		delete singleton;
+		singleton = nullptr;
+	}
+}
+
+SchedulerManager& SchedulerManager::getInstance() {
+	if (singleton == nullptr) {
+		throw wfr::exception::wfr_missing_instance("ResourceManager instance not found.");
+	}
+	return *singleton;
+}
+
+
+SchedulerManager::SchedulerManager(std::string defaultScheduler) : defaultScheduler(defaultScheduler) {
+    schedulerFactories.insert(factoryMap_t::value_type("dummy", []()->AbstractScheduler* {
+    	return new DummyScheduler(resource::ResourceManager::getInstance(), prediction::PredictionManager::getInstance());
+    }));
+    schedulerFactories.insert(factoryMap_t::value_type("heft", []()->AbstractScheduler* {
+    	return new HEFTScheduler(resource::ResourceManager::getInstance(), prediction::PredictionManager::getInstance());
+    }));
+    schedulerFactories.insert(factoryMap_t::value_type("random", []()->AbstractScheduler* {
+    	return new RandomScheduler(resource::ResourceManager::getInstance(), prediction::PredictionManager::getInstance());
+    }));
+    schedulerFactories.insert(factoryMap_t::value_type("jit", []()->AbstractScheduler* {
+    	return new JustInTimeScheduler(resource::ResourceManager::getInstance(), prediction::PredictionManager::getInstance());
+    }));
+    if (schedulerFactories.find(defaultScheduler) == schedulerFactories.end()) {
+    	throw wfr::exception::wfr_fatal_error((boost::format("Default scheduler \"%1%\" does not exist.") % defaultScheduler).str());
+    }
+}
+
+AbstractScheduler* SchedulerManager::getDefaultScheduler() {
+	return getSchedulerByName(defaultScheduler);
+}
+
+AbstractScheduler* SchedulerManager::getSchedulerByName(std::string& name) {
+	auto scheduler = schedulerFactories.find(name);
+	if (scheduler != schedulerFactories.end()) {
+		return scheduler->second();
+	} else {
+		throw wfr::exception::wfr_illegal_argument((boost::format("Could not find scheduler \"%1%\".") % name).str());
+	}
+}
+
+void SchedulerManager::markSchedulerForDeletion(AbstractScheduler* scheduler) {
+	delete scheduler;
+}
+
+} /* namespace scheduling */
+} /* namespace wfr */
+
+

src/scheduling/cmake.inc

@@ -0,0 +1,9 @@
+
+set(wfr_cpp_files "${wfr_cpp_files}"
+	src/scheduling/AbstractScheduler.cpp
+	src/scheduling/SchedulerManager.cpp
+	src/scheduling/scheduler/dummy/DummyScheduler.cpp
+	src/scheduling/scheduler/heft/HEFTScheduler.cpp
+	src/scheduling/scheduler/random/RandomScheduler.cpp
+	src/scheduling/scheduler/justintime/JustInTimeScheduler.cpp
+)

src/scheduling/scheduler/dummy/DummyScheduler.cpp

@@ -0,0 +1,51 @@
+
+#include <wfr/scheduling/scheduler/dummy/DummyScheduler.h>
+#include <wfr/common/wep/WorkflowExecutionPlan.h>
+#include <wfr/common/wep/AbstractWepNode.h>
+#include <wfr/common/wep/activities/AtomicWepNode.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/common/wfs/AtomicActivityImplementation.h>
+#include <wfr/common/wfs/instances/AtomicActivityInstance.h>
+#include <wfr/common/wfs/templates/AtomicActivityTemplate.h>
+#include <wfr/prediction/PredictionManager.h>
+#include <wfr/common/resource/ResourceManager.h>
+#include <wfr/enactment/EnactmentEngine.h>
+#include <wfr/common/scheduling/SchedulingDecision.h>
+#include <wfr/execution/ExecutionManager.h>
+#include <wfr/WorkflowRuntime.h>
+#include <wfr/wep/utils/WepSearch.h>
+#include <iostream>
+#include <set>
+#include <queue>
+
+
+namespace wfr {
+namespace scheduling {
+
+
+void DummyScheduler::_notifyWEPUpdate(SchedulerBase::WepMapEntry& wepe, std::set<wep::AtomicWepNode*> readyToExecute, std::unique_lock<std::mutex>& schedulerLock) {
+	auto resourceTypes = resourceManager.getComputingResourceTypes();
+	for (auto n : readyToExecute) {
+		auto& ai = n->activity.as<wfs::AtomicActivityInstance>();
+		ai.setInstanceStatus(wfs::ActivityInstanceStatus::Scheduled);
+		wfs::AtomicActivityImplementation* impl = nullptr;
+		auto& tmp = *ai.baseActivity.baseTemplate.as<wfs::AtomicActivityTemplate>().atomicActivityType;
+		for (auto i : tmp.getImplementations()) {
+			if (i->implementationType == wfs::ActivityImplementationType::PThread) {
+				impl = i;
+				break;
+			}
+		}
+		if (impl) {
+			std::shared_ptr<SchedulingDecision> sd = std::make_shared<SchedulingDecision>(&ai, impl, wepe.wep);
+			executeSchedulingDecision(wepe, sd, schedulerLock);
+		} else {
+			throw exception::wfr_fatal_error("No suitable activity implementation found.");
+		}
+	}
+}
+
+
+} /* namespace scheduling */
+} /* namespace wfr */

src/scheduling/scheduler/heft/HEFTScheduler.cpp

@@ -0,0 +1,212 @@
+
+
+#include <wfr/prediction/PredictionManager.h>
+#include <wfr/common/resource/ResourceManager.h>
+#include <wfr/scheduling/scheduler/heft/HEFTScheduler.h>
+#include <wfr/common/wep/WorkflowExecutionPlan.h>
+#include <wfr/common/wep/AbstractWepNode.h>
+#include <wfr/common/wep/activities/AtomicWepNode.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/common/wfs/instances/AtomicActivityInstance.h>
+#include <wfr/common/wfs/templates/AtomicActivityTemplate.h>
+#include <wfr/enactment/EnactmentEngine.h>
+#include <wfr/common/scheduling/SchedulingDecision.h>
+#include <wfr/execution/ExecutionManager.h>
+#include <wfr/wep/utils/WepSearch.h>
+#include <iostream>
+#include <set>
+#include <boost/format.hpp>
+#include <queue>
+#include <memory>
+
+
+namespace wfr {
+namespace scheduling {
+
+
+
+NumericalMeasurement HEFTScheduler::getAverageExecutionCost(wfs::AtomicActivityInstance& ai, InstanceUserData* instanceData, const std::vector<resource::ComputingResourceType*>& resources) {
+	// Iterate over resources and calculate average execution time
+	NumericalMeasurement average;
+	wfr_size_t _count = 0;
+	for (auto dev : resources) {
+		NumericalMeasurement resTime;
+		// Get fastest implementation (better said its execution time) for given resource
+		auto& aat = ai.baseActivity.baseTemplate.as<wfs::AtomicActivityTemplate>();
+		if (aat.atomicActivityType->getImplementationCount() > 0) {
+			for (auto i : aat.atomicActivityType->getImplementations()) {
+				auto execPred = predictionManager.predictExecutionTime(ai, *i, *dev);
+				if (execPred.isValid() && (!resTime.isValid() || execPred.getExecutionTime() < resTime)) {
+					resTime = execPred.getExecutionTime();
+					instanceData->predictions[dev] = std::make_tuple(i, execPred);
+				}
+			}
+			if (resTime.isValid()) {
+				_count++;
+				average += resTime;
+			}
+		}
+	}
+	if (_count > 0) {
+		average /= _count;
+	}
+	return average;
+}
+
+
+NumericalMeasurement HEFTScheduler::getMaxSuccValue(wfs::AtomicActivityInstance& ai, InstanceUserData* instanceData, const std::vector<resource::ComputingResourceType*>& resourceTypes, const std::vector<resource::ComputingResource*>& resources) {
+	NumericalMeasurement retval(0, 0, true);
+
+	// iterate over successors
+	for(wfr_size_t i = 0; i < ai.getSuccessorCount(); i++) {
+		auto succ = ai.getSuccessor(i);
+
+		auto succDataPtr = succ->getUserData();
+		InstanceUserData* succData;
+		if (succDataPtr) {
+			succData = (InstanceUserData*)succDataPtr.get();
+		} else {
+			succ->setUserData(std::make_shared<InstanceUserData>());
+			succData = (InstanceUserData*)succ->getUserData().get();
+		}
+		auto succRank = getRank(succ->as<wfs::AtomicActivityInstance>(), instanceData, resourceTypes, resources);
+		if (succRank.isValid()) {
+
+			// Calculate average communication costs
+			NumericalMeasurement average(0, 0, true);
+			wfr_size_t _count = 0;
+			for (auto& dev1 : resources) {
+				for (auto& dev2 : resources) {
+					if (dev1 != dev2) {
+						auto commTime = predictionManager.predictCommunicationTime(ai, *dev1, *succ, *dev2);
+						if (commTime.isValid) {
+							_count++;
+							average += commTime.communicationTime;
+						}
+					}
+				}
+			}
+			if (_count > 0) {
+				average /= _count;
+			}
+			NumericalMeasurement tmp = succData->rank + average;
+			if (tmp.isValid() && (!retval.isValid() || retval < tmp)) {
+				retval = tmp;
+			}
+		} else {
+			throw exception::wfr_fatal_error("Invalid successor rank!");
+		}
+	}
+	return retval;
+}
+
+
+NumericalMeasurement HEFTScheduler::getRank(wfs::AtomicActivityInstance& ai, InstanceUserData* instanceData, const std::vector<resource::ComputingResourceType*>& resourceTypes, const std::vector<resource::ComputingResource*>& resources) {
+
+	if (!instanceData->isComplete) {
+		if (!instanceData->averageComputingCost.isValid()) {
+			instanceData->averageComputingCost = getAverageExecutionCost(ai, instanceData, resourceTypes);
+		}
+		NumericalMeasurement maxSuccRank = getMaxSuccValue(ai, instanceData, resourceTypes, resources);
+		if (instanceData->averageComputingCost.isValid() && maxSuccRank.isValid()) {
+			instanceData->rank = instanceData->averageComputingCost + maxSuccRank;
+			instanceData->isComplete = true;
+		}
+	}
+
+	return instanceData->rank;
+}
+
+
+void HEFTScheduler::_notifyWEPUpdate(SchedulerBase::WepMapEntry& wepe, std::set<wep::AtomicWepNode*> readyToExecute, std::unique_lock<std::mutex>& schedulerLock) {
+
+	auto resourceTypes = resourceManager.getComputingResourceTypes();
+	auto resources = resourceManager.getComputingResources();
+
+	// Visitor function for breadth first search.
+	auto visitFunc = [&](wep::AbstractWepNode* node) -> bool {
+		if (node->nodeType == wep::WepNodeType::Atomic) {
+			auto& ai = node->as<wep::AtomicWepNode>().activity.as<wfs::AtomicActivityInstance>();
+			auto instanceDataPtr = ai.getUserData();
+			InstanceUserData* instanceData;
+			if (instanceDataPtr) {
+				instanceData = (InstanceUserData*)instanceDataPtr.get();
+			} else {
+				ai.setUserData(std::make_shared<InstanceUserData>());
+				instanceData = (InstanceUserData*)ai.getUserData().get();
+			}
+
+			getRank(ai, instanceData, resourceTypes, resources);
+			if (instanceData->isComplete && !instanceData->isScheduled && ai.getInstanceStatus() == wfs::ActivityInstanceStatus::ReadyForExecution) {
+				SchedulingQueueEntry tmp;
+				tmp.instance = &ai;
+				tmp.instanceData = instanceData;
+				tmp.node = node;
+				schedulingPriorityQueue.push(tmp);
+				instanceData->isScheduled = true;
+			}
+		}
+		return true;
+	};
+	// Visit all nodes of the WEP and calculate the rank for each atomic activity.
+	wfr::utils::wep::wepBreadthFirstReverseSearch(wepe.wep->getEndNode(), visitFunc);
+
+
+	// Go through scheduling queue and schedule activities;
+	/*while (!schedulingQueue.empty()) {
+		auto e = schedulingQueue.front();*/
+	while (!schedulingPriorityQueue.empty()) {
+		auto e = schedulingPriorityQueue.top();
+		wfs::AtomicActivityImplementation* fastestImpl = nullptr;
+		resource::ComputingResource* fastestResource = nullptr;
+		prediction::ExecutionTimeResult prediction;
+		wfr_size_t busyResourceCount = 0;
+		for (auto res : resources) {
+			if (res->getFreeSlotCount() > 0) {
+				auto& pe = e.instanceData->predictions[&res->resourceType];
+				wfs::AtomicActivityImplementation* impl = std::get<0>(pe);
+				if (impl) {
+					auto& pred = std::get<1>(pe);
+					if (pred.isValid() && (!prediction.isValid() || pred.getExecutionTime() < prediction.getExecutionTime())) {
+						fastestImpl = std::get<0>(pe);
+						fastestResource = res;
+						prediction = pred;
+					}
+				}
+			} else {
+				busyResourceCount++;
+			}
+		}
+		if (fastestImpl) {
+			auto reservation = resourceManager.reserveComputingResource(fastestResource, 1);
+			if (reservation.isValid()) {
+				std::shared_ptr<SchedulingDecision> sd = std::make_shared<SchedulingDecision>(e.instance, fastestImpl, wepe.wep, reservation);
+				sd->setPrediction(prediction);
+				e.instance->setInstanceStatus(wfs::ActivityInstanceStatus::Scheduled);
+				executeSchedulingDecision(wepe, sd, schedulerLock);
+				//schedulingQueue.pop();
+				schedulingPriorityQueue.pop();
+
+				/*std::string d1 = (boost::format("%4%: %1%: Rank %2%, Prediction %3%")
+					% e.instance->baseActivity.baseTemplate.activityName
+					% e.instanceData->rank.getValue()
+					% prediction.executionTime.getValue()
+					% debugIteration).str();
+				debugLines.push_back(d1);*/
+
+			}
+		} else if (busyResourceCount == resources.size()) {
+			// If all resources are busy we can break the loop
+			break;
+		}
+	}
+	debugIteration++;
+}
+
+
+
+} /* namespace scheduler */
+} /* namespace wfr */
+
+

src/scheduling/scheduler/justintime/JustInTimeScheduler.cpp

@@ -0,0 +1,78 @@
+
+#include <wfr/scheduling/scheduler/justintime/JustInTimeScheduler.h>
+#include <wfr/common/wep/WorkflowExecutionPlan.h>
+#include <wfr/common/wep/AbstractWepNode.h>
+#include <wfr/common/wep/activities/AtomicWepNode.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/common/wfs/AtomicActivityImplementation.h>
+#include <wfr/common/wfs/instances/AtomicActivityInstance.h>
+#include <wfr/common/wfs/templates/AtomicActivityTemplate.h>
+#include <wfr/prediction/PredictionManager.h>
+#include <wfr/common/resource/ResourceManager.h>
+#include <wfr/enactment/EnactmentEngine.h>
+#include <wfr/common/scheduling/SchedulingDecision.h>
+#include <wfr/execution/ExecutionManager.h>
+#include <wfr/WorkflowRuntime.h>
+#include <wfr/wep/utils/WepSearch.h>
+#include <iostream>
+#include <set>
+#include <queue>
+
+
+namespace wfr {
+namespace scheduling {
+
+
+void JustInTimeScheduler::_notifyWEPUpdate(SchedulerBase::WepMapEntry& wepe, std::set<wep::AtomicWepNode*> readyToExecute, std::unique_lock<std::mutex>& schedulerLock) {
+	for (auto n : readyToExecute) {
+		nodeQueue.insert(n);
+	}
+	auto resourceTypes = resourceManager.getComputingResourceTypes();
+	std::vector<wep::AtomicWepNode*> scheduledQueue;
+	for (auto n : nodeQueue) {
+		auto& ai = n->activity.as<wfs::AtomicActivityInstance>();
+		wfs::AtomicActivityImplementation* fastestImpl = nullptr;
+		resource::ComputingResource* fastestRes = nullptr;
+		prediction::ExecutionTimeResult fastestPred;
+		for (auto restype : resourceTypes) {
+			resource::ComputingResource* freeRes = nullptr;
+			for (auto res : restype->getResources()) {
+				if (res->getFreeSlotCount() > 0) {
+					freeRes = res;
+					break;
+				}
+			}
+			if (freeRes) {
+				auto& aat = ai.baseActivity.baseTemplate.as<wfs::AtomicActivityTemplate>();
+				for (auto impl : aat.atomicActivityType->getImplementations()) {
+					auto execPred = predictionManager.predictExecutionTime(ai, *impl, *restype);
+					if (execPred.isValid()) {
+						if (!fastestImpl || !fastestPred.isValid() || execPred.executionTime.getValue() < fastestPred.executionTime.getValue()) {
+							fastestImpl = impl;
+							fastestRes = freeRes;
+							fastestPred = execPred;
+						}
+					}
+				}
+			}
+		}
+		if (fastestImpl) {
+			auto reservation = resourceManager.reserveComputingResource(fastestRes, 1);
+			auto sd = std::make_shared<SchedulingDecision>(&ai, fastestImpl, wepe.wep, reservation);
+			sd->setPrediction(fastestPred);
+			ai.setInstanceStatus(wfs::ActivityInstanceStatus::Scheduled);
+			scheduledQueue.push_back(n);
+			executeSchedulingDecision(wepe, sd, schedulerLock);
+		} else {
+			//throw exception::wfr_fatal_error("JustInTimeScheduler could not find a suitable implementation.");
+		}
+	}
+	for (auto n : scheduledQueue) {
+		nodeQueue.erase(n);
+	}
+}
+
+
+} /* namespace scheduling */
+} /* namespace wfr */

src/scheduling/scheduler/random/RandomScheduler.cpp

@@ -0,0 +1,80 @@
+
+#include <wfr/scheduling/scheduler/random/RandomScheduler.h>
+#include <wfr/common/wep/WorkflowExecutionPlan.h>
+#include <wfr/common/wep/AbstractWepNode.h>
+#include <wfr/common/wep/activities/AtomicWepNode.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/common/wfs/AtomicActivityImplementation.h>
+#include <wfr/common/wfs/instances/AtomicActivityInstance.h>
+#include <wfr/common/wfs/templates/AtomicActivityTemplate.h>
+#include <wfr/prediction/PredictionManager.h>
+#include <wfr/common/resource/ResourceManager.h>
+#include <wfr/enactment/EnactmentEngine.h>
+#include <wfr/common/scheduling/SchedulingDecision.h>
+#include <wfr/execution/ExecutionManager.h>
+#include <wfr/WorkflowRuntime.h>
+#include <wfr/wep/utils/WepSearch.h>
+#include <iostream>
+#include <algorithm>
+#include <set>
+#include <queue>
+
+
+namespace wfr {
+namespace scheduling {
+
+
+void RandomScheduler::_notifyWEPUpdate(SchedulerBase::WepMapEntry& wepe, std::set<wep::AtomicWepNode*> readyToExecute, std::unique_lock<std::mutex>& schedulerLock) {
+	for (auto n : readyToExecute) {
+		nodeQueue.insert(n);
+	}
+	auto resourceTypes = resourceManager.getComputingResourceTypes();
+	std::vector<wep::AtomicWepNode*> scheduledQueue;
+	for (auto n : nodeQueue) {
+		auto& ai = n->activity.as<wfs::AtomicActivityInstance>();
+		std::vector<std::tuple<wfs::AtomicActivityInstance*, wfs::AtomicActivityImplementation*, resource::ComputingResource*, prediction::ExecutionTimeResult>> predResults;
+		for (auto restype : resourceTypes) {
+			resource::ComputingResource* freeRes = nullptr;
+			for (auto res : restype->getResources()) {
+				if (res->getFreeSlotCount() > 0) {
+					freeRes = res;
+					break;
+				}
+			}
+			if (freeRes) {
+				auto& aat = ai.baseActivity.baseTemplate.as<wfs::AtomicActivityTemplate>();
+				for (auto impl : aat.atomicActivityType->getImplementations()) {
+					auto execPred = predictionManager.predictExecutionTime(ai, *impl, *restype);
+					if (execPred.isValid()) {
+						auto tuple = std::make_tuple(&ai, impl, freeRes, execPred);
+						predResults.push_back(tuple);
+					}
+				}
+			}
+		}
+		if (predResults.size() > 0) {
+			auto dice = WorkflowRuntime::getRandomNumber<wfr_size_t>(0, predResults.size()-1);
+			auto& selected = predResults[dice];
+			auto instance = std::get<0>(selected);
+			auto impl = std::get<1>(selected);
+			auto res = std::get<2>(selected);
+			auto pred = std::get<3>(selected);
+			auto reservation = resourceManager.reserveComputingResource(res, 1);
+			auto sd = std::make_shared<SchedulingDecision>(instance, impl, wepe.wep, reservation);
+			sd->setPrediction(pred);
+			instance->setInstanceStatus(wfs::ActivityInstanceStatus::Scheduled);
+			scheduledQueue.push_back(n);
+			executeSchedulingDecision(wepe, sd, schedulerLock);
+		} else {
+			//throw exception::wfr_fatal_error("RandomScheduler could not find a suitable implementation.");
+		}
+	}
+	for (auto n : scheduledQueue) {
+		nodeQueue.erase(n);
+	}
+}
+
+
+} /* namespace scheduling */
+} /* namespace wfr */

src/utils/cmake.inc

@@ -0,0 +1,4 @@
+
+set(wfr_cpp_files "${wfr_cpp_files}"
+	src/utils/filesystem.cpp
+)

src/utils/filesystem.cpp

@@ -0,0 +1,80 @@
+#include <wfr/common/utils/filesystem.h>
+#include <wfr/common/eventsystem/Event.h>
+#include <wfr/common/eventsystem/EventSystem.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/ActivityTemplate.h>
+#include <boost/filesystem.hpp>
+#include <boost/format.hpp>
+#include <boost/uuid/uuid.hpp>
+#include <boost/uuid/uuid_generators.hpp>
+#include <boost/lexical_cast.hpp>
+#include <boost/uuid/uuid_io.hpp>
+
+
+namespace wfr {
+
+namespace filesystem {
+
+
+boost::filesystem::path createWorkingFolder(wfr::wfs::ActivityInstance& instance) {
+	boost::uuids::uuid uuid = boost::uuids::random_generator()();
+	std::string tmp = "wfr_wdir_";
+	tmp += instance.baseActivity.baseTemplate.activityName + std::string("_");
+	tmp = tmp + boost::lexical_cast<std::string>(uuid);
+
+	boost::filesystem::path wdir(tmp);
+	boost::system::error_code ec;
+	if (boost::filesystem::create_directories(wdir, ec)) {
+		eventsystem::GeneralLoggingEvent::fireEvent(boost::str(boost::format("Created working directory \"%1%\" for activity instance ID %2%.") % tmp % instance.instanceID));
+	} else {
+		std::string errmsg = boost::str(boost::format("Failed to create working directory \"%1%\" for activity instance ID %2%: %3%") % tmp % instance.instanceID % ec.message());
+		throw exception::wfr_fatal_error(errmsg);
+	}
+	return wdir;
+}
+
+boost::filesystem::path createWorkingFolder(std::string activityName) {
+
+	boost::uuids::uuid uuid = boost::uuids::random_generator()();
+	std::string tmp = "wfr_wdir_" + activityName + std::string("_");
+	tmp = tmp + boost::lexical_cast<std::string>(uuid);
+
+	boost::filesystem::path wdir(tmp);
+	boost::system::error_code ec;
+	if (boost::filesystem::create_directories(wdir, ec)) {
+		eventsystem::GeneralLoggingEvent::fireEvent(boost::str(boost::format("Created working directory \"%1%\" for activity \"%2%\".") % tmp % activityName));
+	} else {
+		std::string errmsg = boost::str(boost::format("Failed to create working directory \"%1%\" for activity \"%2%\": %3%") % tmp % activityName % ec.message());
+		throw exception::wfr_fatal_error(errmsg);
+	}
+	return wdir;
+}
+
+boost::filesystem::path createFolder(wfr::wfs::ActivityInstance& instance, boost::filesystem::path pdir, std::string fname) {
+
+	boost::filesystem::path folder = pdir/boost::filesystem::path(fname);
+	boost::system::error_code ec;
+	if (boost::filesystem::create_directories(folder, ec)) {
+		eventsystem::GeneralLoggingEvent::fireEvent(boost::str(boost::format("Created directory \"%1%\" for activity instance ID %2%.") % folder.string() % instance.instanceID));
+	} else {
+		std::string errmsg = boost::str(boost::format("Failed to create directory \"%1%\" for activity instance ID %2%: %3%") % folder.string() % instance.instanceID % ec.message());
+		throw exception::wfr_fatal_error(errmsg);
+	}
+	return folder;
+}
+
+
+void copy_file(const boost::filesystem::path& from, const boost::filesystem::path& to, boost::filesystem::copy_option::enum_type option) {
+	boost::system::error_code ec;
+	boost::filesystem::copy_file(from, to, option, ec);
+	if (ec) {
+		std::string errmsg = boost::str(boost::format("Failed to copy file \"%1%\" to \"%2%\": %3%") % from.string() % to.string() % ec.message());
+		throw exception::wfr_fatal_error(errmsg);
+	}
+}
+
+
+} // end namespace filesystem
+
+} // end namespace wfr
+

src/wep/cmake.inc

@@ -0,0 +1,4 @@
+
+set(wfr_cpp_files "${wfr_cpp_files}"
+	src/wep/utils/WepDotFileWriter.cpp
+)

src/wep/utils/WepDotFileWriter.cpp

@@ -0,0 +1,211 @@
+
+#include <wfr/wep/utils/WepDotFileWriter.h>
+#include <wfr/common/wep/AbstractWepNode.h>
+#include <wfr/common/wep/ActivityWepNode.h>
+#include <wfr/common/wep/activities/IterationStartWepNode.h>
+#include <wfr/common/wep/activities/IterationEndWepNode.h>
+#include <wfr/common/wep/WorkflowExecutionPlan.h>
+#include <wfr/common/wfs/Workflow.h>
+#include <wfr/common/wfs/WorkflowTemplate.h>
+#include <wfr/common/wfs/Activity.h>
+#include <wfr/common/wfs/ActivityTemplate.h>
+#include <wfr/common/wfs/ActivityInstance.h>
+#include <wfr/WorkflowRuntime.h>
+#include <boost/format.hpp>
+#include <cassert>
+#include <sstream>
+
+
+namespace wfr {
+namespace wep {
+namespace utils {
+
+
+WepDotFileWriter::WepDotFileWriter(const std::shared_ptr<WorkflowExecutionPlan>& wep) noexcept
+		: wep(wep), workflowName(wep->workflow.baseTemplate.workflowName) {
+	assert(wep);
+}
+
+
+void WepDotFileWriter::printToFile() {
+	printToFile((boost::format("%1%.%2%.dot") % workflowName % wep->getVersion()).str());
+}
+
+
+void WepDotFileWriter::printToFile(const std::string fileName) {
+	std::ofstream outfile(fileName);
+
+	// Write Prolog
+	outfile << "digraph \"" << workflowName << "_" << wep->getVersion() << "\" {" << std::endl;
+
+	// Write Nodes and Edges
+	visitedNodes_t visitedNodes;
+	writeWepNodeAndEdges(outfile, wep->getStartNode(), visitedNodes);
+
+	// Write Epilog
+	outfile << "}" << std::endl;
+
+	// Flush and Close
+	outfile.flush();
+	outfile.close();
+
+	// Log
+	wfr::WorkflowRuntime::logMessage((boost::format("WEP to DOT-File Writer: Finished writing WEP version %1% to file \"%2%\"") % wep->getVersion() % fileName).str());
+}
+
+
+void WepDotFileWriter::writeWepNodeAndEdges(std::ofstream& outfile, AbstractWepNode* node, visitedNodes_t& visitedNodes) {
+	if (visitedNodes.find(node) == visitedNodes.end()) {
+		visitedNodes.insert(node);
+
+		// Write node info
+		outfile << "\t" << getNodeID(node) << " ["
+				<< "label=\"" << getNodeLabel(node) << "\""
+				<< ",shape=" << getNodeShape(node)
+				<< "]" << std::endl;
+
+		// Call this function recursively for successors and then write edges
+		std::string nodeID = getNodeID(node);
+		for (auto e : node->getSuccessors()) {
+			writeWepNodeAndEdges(outfile, e, visitedNodes);
+			outfile << "\t" << nodeID << " -> " << getNodeID(e) << std::endl;
+		}
+	}
+}
+
+
+std::string WepDotFileWriter::getNodeType(AbstractWepNode* node) {
+	if (node->withActivityInstance) {
+		switch (node->as<ActivityWepNode>().activity.activityType()) {
+			case wfs::ActivityType::Conditional:
+				return "Conditional";
+			case wfs::ActivityType::Container:
+				return "Container";
+			case wfs::ActivityType::ForLoop:
+				return "ForLoop";
+			case wfs::ActivityType::WhileLoop:
+				return "WhileLoop";
+			case wfs::ActivityType::Atomic:
+				return "Atomic";
+			case wfs::ActivityType::Assign:
+				return "Assign";
+			default:
+				return "<<UnknownInstance>>";
+		}
+	} else {
+		return "<<NoInstance>>";
+	}
+}
+
+
+std::string WepDotFileWriter::getNodeLabel(AbstractWepNode* node) {
+	std::stringstream ss;
+	ss << "#" << node->nodeID << "\\n";
+	switch (node->nodeType) {
+		case wep::WepNodeType::Atomic:
+			ss << node->as<ActivityWepNode>().activity.baseActivity.baseTemplate.activityName
+				<< " (IID: " << node->as<ActivityWepNode>().activity.instanceID << ")"
+				<< "\\n<" << getNodeStatus(node) << ">";
+			break;
+		case wep::WepNodeType::Assign:
+			ss << node->as<ActivityWepNode>().activity.baseActivity.baseTemplate.activityName
+				<< " (IID: " << node->as<ActivityWepNode>().activity.instanceID << ")"
+				<< "\\n<" << getNodeStatus(node) << ">";
+			break;
+		case wep::WepNodeType::CompositeStart:
+			ss << getNodeType(node)
+				<< "\\n<" << getNodeStatus(node) << ">";
+			break;
+		case wep::WepNodeType::CompositeEnd:
+			ss << getNodeType(node);
+			break;
+		case wep::WepNodeType::IterationStart:
+
+			ss << "Iteration " << node->as<IterationStartWepNode>().iterationID
+				<< "\\n<" << getNodeStatus(node) << ">";
+			break;
+		case wep::WepNodeType::IterationEnd:
+			ss << "Iteration " << node->as<IterationEndWepNode>().iterationID;
+			break;
+		case wep::WepNodeType::Hole:
+			ss << getNodeType(node)
+				<< "\\n<" << getNodeStatus(node) << ">";
+			break;
+		default:
+			ss << "<<ERROR>>";
+			break;
+	}
+	ss << "\\nVisitations(" << getNodeVisitationCount(node) << ")";
+	return ss.str();
+}
+
+
+std::string WepDotFileWriter::getNodeStatus(AbstractWepNode* node) {
+	if (node->withActivityInstance) {
+		auto& inode = node->as<ActivityWepNode>();
+		switch (inode.getLastKnownInstanceStatus()) {
+			case wfr::wfs::ActivityInstanceStatus::Created:
+				return "Created";
+			case wfr::wfs::ActivityInstanceStatus::Initialized:
+				return "Initialized";
+			case wfr::wfs::ActivityInstanceStatus::WaitingForData:
+				return "WaitingForData";
+			case wfr::wfs::ActivityInstanceStatus::WaitingForDependency:
+				return "WaitingForDependency";
+			case wfr::wfs::ActivityInstanceStatus::ReadyForExecution:
+				return "ReadyForExecution";
+			case wfs::ActivityInstanceStatus::Scheduled:
+				return "Scheduled";
+			case wfs::ActivityInstanceStatus::Queued:
+				return "Queued";
+			case wfs::ActivityInstanceStatus::WaitingForFreeResources:
+				return "WaitingForFreeResources";
+			case wfs::ActivityInstanceStatus::Running:
+				return "Running";
+			case wfr::wfs::ActivityInstanceStatus::Finished:
+				return "Finished";
+			case wfr::wfs::ActivityInstanceStatus::Error:
+				return "Error";
+			default:
+				return "<<UnknownStatus>>";
+		}
+	} else {
+		return "<<NoInstance>>";
+	}
+}
+
+std::string WepDotFileWriter::getNodeID(AbstractWepNode* node) {
+	return (boost::format("node_%1%") % node->nodeID).str();
+}
+
+std::string WepDotFileWriter::getNodeShape(AbstractWepNode* node) {
+	switch (node->nodeType) {
+		case wep::WepNodeType::Atomic:
+			return "box";
+		case wep::WepNodeType::Assign:
+			return "box";
+		case wep::WepNodeType::CompositeStart:
+			return "triangle";
+		case wep::WepNodeType::CompositeEnd:
+			return "invtriangle";
+		case wep::WepNodeType::IterationStart:
+			return "octagon";
+		case wep::WepNodeType::IterationEnd:
+			return "doubleoctagon";
+		case wep::WepNodeType::Hole:
+			return "egg";
+		default:
+			return "<<UnknownType>>";
+	}
+}
+
+
+std::string WepDotFileWriter::getNodeVisitationCount(AbstractWepNode* node) {
+	return std::to_string(node->getVisitationCount());
+}
+
+} /* namespace utils */
+} /* namespace wep */
+} /* namespace wfr */
+
+