Sort config types by their load dependencies once

lib/base/initialize.hpp

@@ -23,6 +23,7 @@ enum class InitializePriority {
 	RegisterBuiltinTypes,
 	RegisterFunctions,
 	RegisterTypes,
+	SortTypes,
 	EvaluateConfigFragments,
 	Default,
 	FreezeNamespaces,

lib/base/type.cpp

@@ -1,9 +1,13 @@
 /* Icinga 2 | (c) 2012 Icinga GmbH | GPLv2+ */
 
 #include "base/type.hpp"
+#include "base/atomic.hpp"
+#include "base/configobject.hpp"
+#include "base/debug.hpp"
 #include "base/scriptglobal.hpp"
 #include "base/namespace.hpp"
 #include "base/objectlock.hpp"
+#include <functional>
 
 using namespace icinga;
 
@@ -32,6 +36,43 @@ INITIALIZE_ONCE_WITH_PRIORITY([]() {
 	Type::Register(type);
 }, InitializePriority::RegisterTypeType);
 
+static std::vector<Type::Ptr> l_SortedByLoadDependencies;
+static Atomic l_SortingByLoadDependenciesDone (false);
+
+INITIALIZE_ONCE_WITH_PRIORITY([] {
+	std::unordered_set<Type*> visited;
+
+	std::function<void(Type*)> visit;
+	// Please note that this callback does not detect any cyclic load dependencies,
+	// instead, it relies on the "sort_by_load_after" unit test to fail.
+	visit = ([&visit, &visited](Type* type) {
+		if (visited.find(type) != visited.end()) {
+			return;
+		}
+		visited.emplace(type);
+
+		for (auto dependency : type->GetLoadDependencies()) {
+			visit(dependency);
+		}
+
+		// We have managed to reach the final/top node in this dependency graph,
+		// so let's place them in reverse order to their final place.
+		l_SortedByLoadDependencies.emplace_back(type);
+	});
+
+	// Sort the types by their load_after dependencies in a Depth-First search manner.
+	for (const Type::Ptr& type : Type::GetAllTypes()) {
+		// Note that only those types that are assignable to the dynamic ConfigObject type can have "load_after"
+		// dependencies, otherwise they are just some Icinga 2 primitive types such as Number, String, etc. and
+		// we need to ignore them.
+		if (ConfigObject::TypeInstance->IsAssignableFrom(type)) {
+			visit(type.get());
+		}
+	}
+
+	l_SortingByLoadDependenciesDone.store(true);
+}, InitializePriority::SortTypes);
+
 String Type::ToString() const
 {
 	return "type '" + GetName() + "'";
@@ -72,6 +113,12 @@ std::vector<Type::Ptr> Type::GetAllTypes()
 	return types;
 }
 
+const std::vector<Type::Ptr>& Type::GetConfigTypesSortedByLoadDependencies()
+{
+	VERIFY(l_SortingByLoadDependenciesDone.load());
+	return l_SortedByLoadDependencies;
+}
+
 String Type::GetPluralName() const
 {
 	String name = GetName();

lib/base/type.hpp

@@ -83,6 +83,21 @@ class Type : public Object
 	static Type::Ptr GetByName(const String& name);
 	static std::vector<Type::Ptr> GetAllTypes();
 
+	/**
+	 * Returns a list of config types sorted by their "load_after" dependencies.
+	 *
+	 * All dependencies of a given type are listed at a lower index than that of the type itself. In other words,
+	 * if a `Service` type load depends on the `Host` and `ApiListener` types, the Host and ApiListener types are
+	 * guaranteed to appear first on the list. Nevertheless, the order of the Host and ApiListener types themselves
+	 * is arbitrary if the two types are not dependent.
+	 *
+	 * It should be noted that this method will fail fatally when used prior to the completion
+	 * of namespace initialization.
+	 *
+	 * @return std::vector<Type::Ptr>
+	 */
+	static const std::vector<Ptr>& GetConfigTypesSortedByLoadDependencies();
+
 	void SetField(int id, const Value& value, bool suppress_events = false, const Value& cookie = Empty) override;
 	Value GetField(int id) const override;
 

lib/config/configitem.cpp

@@ -444,180 +444,131 @@ bool ConfigItem::CommitNewItems(const ActivationContext::Ptr& context, WorkQueue
 		<< "Committing " << total << " new items.";
 #endif /* I2_DEBUG */
 
-	std::set<Type::Ptr> types;
-	std::set<Type::Ptr> completed_types;
 	int itemsCount {0};
 
-	for (const Type::Ptr& type : Type::GetAllTypes()) {
-		if (ConfigObject::TypeInstance->IsAssignableFrom(type))
-			types.insert(type);
-	}
-
-	while (types.size() != completed_types.size()) {
-		for (const Type::Ptr& type : types) {
-			if (completed_types.find(type) != completed_types.end())
-				continue;
+	for (auto& type : Type::GetConfigTypesSortedByLoadDependencies()) {
+		std::atomic<int> committed_items(0);
 
-			bool unresolved_dep = false;
+		{
+			auto items (itemsByType.find(type.get()));
 
-			/* skip this type (for now) if there are unresolved load dependencies */
-			for (auto pLoadDep : type->GetLoadDependencies()) {
-				if (types.find(pLoadDep) != types.end() && completed_types.find(pLoadDep) == completed_types.end()) {
-					unresolved_dep = true;
-					break;
+			if (items != itemsByType.end()) {
+				for (const ItemPair& pair: items->second) {
+					newItems.emplace_back(pair.first);
 				}
-			}
-
-			if (unresolved_dep)
-				continue;
 
-			std::atomic<int> committed_items(0);
+				upq.ParallelFor(items->second, [&committed_items](const ItemPair& ip) {
+					const ConfigItem::Ptr& item = ip.first;
 
-			{
-				auto items (itemsByType.find(type.get()));
+					if (!item->Commit(ip.second)) {
+						if (item->IsIgnoreOnError()) {
+							item->Unregister();
+						}
 
-				if (items != itemsByType.end()) {
-					for (const ItemPair& pair: items->second) {
-						newItems.emplace_back(pair.first);
+						return;
 					}
 
-					upq.ParallelFor(items->second, [&committed_items](const ItemPair& ip) {
-						const ConfigItem::Ptr& item = ip.first;
+					committed_items++;
+				});
 
-						if (!item->Commit(ip.second)) {
-							if (item->IsIgnoreOnError()) {
-								item->Unregister();
-							}
-
-							return;
-						}
-
-						committed_items++;
-					});
-
-					upq.Join();
-				}
+				upq.Join();
 			}
+		}
 
-			itemsCount += committed_items;
-
-			completed_types.insert(type);
+		itemsCount += committed_items;
 
 #ifdef I2_DEBUG
-			if (committed_items > 0)
-				Log(LogDebug, "configitem")
-					<< "Committed " << committed_items << " items of type '" << type->GetName() << "'.";
+		if (committed_items > 0)
+			Log(LogDebug, "configitem")
+				<< "Committed " << committed_items << " items of type '" << type->GetName() << "'.";
 #endif /* I2_DEBUG */
 
-			if (upq.HasExceptions())
-				return false;
-		}
+		if (upq.HasExceptions())
+			return false;
 	}
 
 #ifdef I2_DEBUG
 	Log(LogDebug, "configitem")
 		<< "Committed " << itemsCount << " items.";
 #endif /* I2_DEBUG */
 
-	completed_types.clear();
+	for (auto& type : Type::GetConfigTypesSortedByLoadDependencies()) {
+		std::atomic<int> notified_items(0);
 
-	while (types.size() != completed_types.size()) {
-		for (const Type::Ptr& type : types) {
-			if (completed_types.find(type) != completed_types.end())
-				continue;
+		{
+			auto items (itemsByType.find(type.get()));
 
-			bool unresolved_dep = false;
+			if (items != itemsByType.end()) {
+				upq.ParallelFor(items->second, [&notified_items](const ItemPair& ip) {
+					const ConfigItem::Ptr& item = ip.first;
 
-			/* skip this type (for now) if there are unresolved load dependencies */
-			for (auto pLoadDep : type->GetLoadDependencies()) {
-				if (types.find(pLoadDep) != types.end() && completed_types.find(pLoadDep) == completed_types.end()) {
-					unresolved_dep = true;
-					break;
-				}
-			}
+					if (!item->m_Object)
+						return;
 
-			if (unresolved_dep)
-				continue;
-
-			std::atomic<int> notified_items(0);
-
-			{
-				auto items (itemsByType.find(type.get()));
-
-				if (items != itemsByType.end()) {
-					upq.ParallelFor(items->second, [&notified_items](const ItemPair& ip) {
-						const ConfigItem::Ptr& item = ip.first;
-
-						if (!item->m_Object)
-							return;
+					try {
+						item->m_Object->OnAllConfigLoaded();
+						notified_items++;
+					} catch (const std::exception& ex) {
+						if (!item->m_IgnoreOnError)
+							throw;
 
-						try {
-							item->m_Object->OnAllConfigLoaded();
-							notified_items++;
-						} catch (const std::exception& ex) {
-							if (!item->m_IgnoreOnError)
-								throw;
+						Log(LogNotice, "ConfigObject")
+							<< "Ignoring config object '" << item->m_Name << "' of type '" << item->m_Type->GetName() << "' due to errors: " << DiagnosticInformation(ex);
 
-							Log(LogNotice, "ConfigObject")
-								<< "Ignoring config object '" << item->m_Name << "' of type '" << item->m_Type->GetName() << "' due to errors: " << DiagnosticInformation(ex);
+						item->Unregister();
 
-							item->Unregister();
-
-							{
-								std::unique_lock<std::mutex> lock(item->m_Mutex);
-								item->m_IgnoredItems.push_back(item->m_DebugInfo.Path);
-							}
+						{
+							std::unique_lock<std::mutex> lock(item->m_Mutex);
+							item->m_IgnoredItems.push_back(item->m_DebugInfo.Path);
 						}
-					});
+					}
+				});
 
-					upq.Join();
-				}
+				upq.Join();
 			}
-
-			completed_types.insert(type);
+		}
 
 #ifdef I2_DEBUG
-			if (notified_items > 0)
-				Log(LogDebug, "configitem")
-					<< "Sent OnAllConfigLoaded to " << notified_items << " items of type '" << type->GetName() << "'.";
+		if (notified_items > 0)
+			Log(LogDebug, "configitem")
+				<< "Sent OnAllConfigLoaded to " << notified_items << " items of type '" << type->GetName() << "'.";
 #endif /* I2_DEBUG */
 
-			if (upq.HasExceptions())
-				return false;
+		if (upq.HasExceptions())
+			return false;
 
-			notified_items = 0;
-			for (auto loadDep : type->GetLoadDependencies()) {
-				auto items (itemsByType.find(loadDep));
+		notified_items = 0;
+		for (auto loadDep : type->GetLoadDependencies()) {
+			auto items (itemsByType.find(loadDep));
 
-				if (items != itemsByType.end()) {
-					upq.ParallelFor(items->second, [&type, &notified_items](const ItemPair& ip) {
-						const ConfigItem::Ptr& item = ip.first;
+			if (items != itemsByType.end()) {
+				upq.ParallelFor(items->second, [&type, &notified_items](const ItemPair& ip) {
+					const ConfigItem::Ptr& item = ip.first;
 
-						if (!item->m_Object)
-							return;
+					if (!item->m_Object)
+						return;
 
-						ActivationScope ascope(item->m_ActivationContext);
-						item->m_Object->CreateChildObjects(type);
-						notified_items++;
-					});
-				}
+					ActivationScope ascope(item->m_ActivationContext);
+					item->m_Object->CreateChildObjects(type);
+					notified_items++;
+				});
 			}
+		}
 
-			upq.Join();
+		upq.Join();
 
 #ifdef I2_DEBUG
-			if (notified_items > 0)
-				Log(LogDebug, "configitem")
-					<< "Sent CreateChildObjects to " << notified_items << " items of type '" << type->GetName() << "'.";
+		if (notified_items > 0)
+			Log(LogDebug, "configitem")
+				<< "Sent CreateChildObjects to " << notified_items << " items of type '" << type->GetName() << "'.";
 #endif /* I2_DEBUG */
 
-			if (upq.HasExceptions())
-				return false;
+		if (upq.HasExceptions())
+			return false;
 
-			// Make sure to activate any additionally generated items
-			if (!CommitNewItems(context, upq, newItems))
-				return false;
-		}
+		// Make sure to activate any additionally generated items
+		if (!CommitNewItems(context, upq, newItems))
+			return false;
 	}
 
 	return true;