RFC: more verbose, more readable TT construction

cmake/modules/ExternalDependenciesVersions.cmake

@@ -6,5 +6,5 @@ set(TTG_TRACKED_BOOST_VERSION 1.66)
 set(TTG_TRACKED_CATCH2_VERSION 2.13.1)
 set(TTG_TRACKED_CEREAL_VERSION 1.3.0)
 set(TTG_TRACKED_MADNESS_TAG 31b2470ca722a6a2d84d4de08d32fb72ae8fdeda)
-set(TTG_TRACKED_PARSEC_TAG 9fc74b6f165605a133125d8a5b62cf55642c1907) 
+set(TTG_TRACKED_PARSEC_TAG 8690b1eb6080500af1ad32ffcd4b95aa38f65c22) 
 set(TTG_TRACKED_BTAS_TAG d73153ad9bc41a177e441ef04eceff7fab0c766d)

cmake/modules/FindOrFetchPARSEC.cmake

@@ -16,7 +16,7 @@ if (NOT TARGET PaRSEC::parsec)
 
   FetchContent_Declare(
           PARSEC
-          GIT_REPOSITORY      https://github.com/ICLDisco/parsec.git
+          GIT_REPOSITORY      https://github.com/devreal/parsec-1.git
           GIT_TAG             ${TTG_TRACKED_PARSEC_TAG}
   )
   FetchContent_MakeAvailable(PARSEC)

tests/unit/tt.cc

@@ -437,4 +437,35 @@ TEST_CASE("TemplateTask", "[core]") {
       static_assert(!ttg::meta::is_generic_callable_v<decltype(&args_pmf::X::g<int>)>);
     }
   }
+
+#if 0
+  SECTION("split_construction") {
+    ttg::Edge<int, float> a2b, b2a;
+    /* construct without edges */
+    auto tta = ttg::make_tt([](int key, float value){});
+    tta->set_input<0>(b2a)
+       .set_output<0>(a2b)
+       .set_name("A");
+
+    auto ttb = ttg::make_tt([](int key, float value){});
+    ttb->set_input<0>(a2b)
+       .set_output<0>(b2a)
+       .set_name("B");
+
+    bool is_executable = ttg::make_graph_executable(tta);
+    CHECK(is_executable);
+  }
+#endif // 0
+
+
+  SECTION("make_connect") {
+    ttg::Edge<int, float> a2b, b2a;
+    /* construct without edges */
+    auto tta = ttg::make_tt([](int key, float value){});
+    auto ttb = ttg::make_tt([](int key, float value){});
+    ttg::connect<0, 0>(ttb, tta); // B -> A
+    ttg::connect<0, 0>(tta, ttb); // A -> B
+    bool is_executable = ttg::make_graph_executable(tta);
+    CHECK(is_executable);
+  }
 }

ttg/ttg/base/terminal.h

@@ -88,6 +88,11 @@ namespace ttg {
       return name;
     }
 
+    TerminalBase& set_name(const std::string& name) {
+      this->name = name;
+      return *this;
+    }
+
     /// Returns string representation of key type
     const std::string &get_key_type_str() const {
       if (!tt) throw "ttg::TerminalBase:get_key_type_str() but tt is null";

ttg/ttg/base/tt.h

@@ -60,6 +60,13 @@ namespace ttg {
       outputs[i] = t;
     }
 
+    void set_output_dynamic(size_t i, TerminalBase *t) {
+      if (i >= outputs.size()) {
+        outputs.resize(i+1);
+      }
+      outputs[i] = t;
+    }
+
     template <bool out, typename terminalT, std::size_t i, typename setfuncT>
     void register_terminal(terminalT &term, const std::string &name, const setfuncT setfunc) {
       term.set(this, i, name, detail::demangled_type_name<typename terminalT::key_type>(),
@@ -211,7 +218,10 @@ namespace ttg {
     }
 
     /// Sets the name of this operation
-    void set_name(const std::string &name) { this->name = name; }
+    TTBase& set_name(const std::string &name) {
+      this->name = name;
+      return *this;
+    }
 
     /// Gets the name of this operation
     const std::string &get_name() const { return name; }

ttg/ttg/func.h

@@ -98,11 +98,25 @@ namespace ttg {
   inline void connect(ttg::TerminalBase *out, ttg::TerminalBase *in) { out->connect(in); }
 
   /// \brief Connect producer output terminal outindex to consumer input terminal inindex (via unique or otherwise
-  /// wrapped pointers to TTs) \tparam outindex The index of the output terminal on the producer. \tparam inindex  The
-  /// index of the input terminal on the consumer. \param p The producer TT \param c The consumer TT
+  /// wrapped pointers to TTs)
+  /// \tparam outindex The index of the output terminal on the producer.
+  /// \tparam inindex  The index of the input terminal on the consumer.
+  /// \param p The producer TT
+  /// \param c The consumer TT
   template <std::size_t outindex, std::size_t inindex, typename producer_tt_ptr, typename successor_tt_ptr>
   inline void connect(producer_tt_ptr &p, successor_tt_ptr &s) {
-    connect(p->template out<outindex>(), s->template in<inindex>());
+    // check whether the output terminal is available at compile time
+    using producer_type  = typename std::pointer_traits<producer_tt_ptr>::element_type;
+    if constexpr (std::tuple_size_v<typename producer_type::output_terminals_type> == 0) {
+      // need to make sure the output terminal is available
+      using successor_type = typename std::pointer_traits<successor_tt_ptr>::element_type;
+      using input_terminal_type = std::tuple_element_t<inindex, typename successor_type::input_terminals_type>;
+      using key_type   = typename input_terminal_type::key_type;
+      using value_type = typename input_terminal_type::value_type;
+      connect(p->template out<outindex, key_type, value_type>(), s->template in<inindex>());
+    } else {
+      connect(p->template out<outindex>(), s->template in<inindex>());
+    }
   }
 
   /// \brief Connect producer output terminal outindex to consumer input terminal inindex (via bare pointers to TTs)

ttg/ttg/make_tt.h

@@ -374,6 +374,21 @@ auto make_tt_tpl(funcT &&func, const std::tuple<ttg::Edge<keyT, input_edge_value
   return std::make_unique<wrapT>(std::forward<funcT>(func), inedges, outedges, name, innames, outnames);
 }
 
+namespace detail {
+  template<typename funcT>
+  struct key_type_from_func
+  {
+    static_assert(!ttg::meta::is_generic_callable_v<funcT>);
+    using func_args_t = decltype(ttg::meta::compute_arg_binding_types_r<void>(std::declval<funcT>(), ttg::typelist<>{}));
+    /* select the first argument as key type or Void if there are none */
+    using type = std::conditional_t<0 < std::tuple_size_v<func_args_t>, std::tuple_element_t<0, func_args_t>, ttg::Void>;
+  };
+
+  template<typename funcT>
+  using key_type_from_func_t = typename key_type_from_func<funcT>::type;
+
+} // namespace detail
+
 // clang-format off
 /// @brief Factory function to assist in wrapping a callable with signature
 ///
@@ -421,9 +436,70 @@ auto make_tt_tpl(funcT &&func, const std::tuple<ttg::Edge<keyT, input_edge_value
 /// @warning Although generic arguments annotated by `const auto&` are also permitted, their use is discouraged to avoid confusion;
 ///          namely, `const auto&` denotes a _consumable_ argument, NOT read-only, despite the `const`.
 // clang-format on
+template <typename keyT = ttg::Void, typename funcT>
+auto make_tt(funcT &&func) {
+
+  using key_type = std::conditional_t<ttg::meta::is_Void_v<keyT>, detail::key_type_from_func_t<funcT>, keyT>;
+  static_assert(!ttg::meta::is_Void_v<key_type>,
+                "Failed to determine key type (no arguments given and no key type provided?)");
+
+  constexpr auto void_key = ttg::meta::is_void_v<key_type>;
+
+  // gross argument typelist for invoking func, can include void for optional args
+  constexpr static auto func_is_generic = ttg::meta::is_generic_callable_v<funcT>;
+
+  static_assert(!func_is_generic, "Generic functions not supported without edges specified!");
+
+  using gross_func_args_t = decltype(ttg::meta::compute_arg_binding_types_r<void>(func, ttg::typelist<>{}));
+
+  // ensure input types do not contain Void
+  static_assert(ttg::meta::is_none_Void_v<gross_func_args_t>, "ttg::Void is for internal use only, do not use it");
+
+  // net argument typelist
+  using func_args_t = gross_func_args_t;
+  constexpr auto num_args = std::tuple_size_v<func_args_t>;
+
+  // if given task id, make sure it's passed via const lvalue ref
+  constexpr bool TASK_ID_PASSED_AS_CONST_LVALUE_REF =
+      !void_key ? ttg::meta::probe_first_v<ttg::meta::is_const_lvalue_reference, true, func_args_t> : true;
+  constexpr bool TASK_ID_PASSED_AS_NONREF =
+      !void_key ? !ttg::meta::probe_first_v<std::is_reference, true, func_args_t> : true;
+  static_assert(
+      TASK_ID_PASSED_AS_CONST_LVALUE_REF || TASK_ID_PASSED_AS_NONREF,
+      "ttg::make_tt(func, ...): if given to func, the task id must be passed by const lvalue ref or by value");
+
+  // if given out-terminal tuple, make sure it's passed via nonconst lvalue ref
+  constexpr bool have_outterm_tuple =
+      func_is_generic ? !ttg::meta::is_last_void_v<gross_func_args_t>
+                      : ttg::meta::probe_last_v<ttg::meta::decays_to_output_terminal_tuple, false, gross_func_args_t>;
+  constexpr bool OUTTERM_TUPLE_PASSED_AS_NONCONST_LVALUE_REF =
+      have_outterm_tuple ? ttg::meta::probe_last_v<ttg::meta::is_nonconst_lvalue_reference, false, func_args_t> : true;
+  static_assert(
+      OUTTERM_TUPLE_PASSED_AS_NONCONST_LVALUE_REF,
+      "ttg::make_tt(func, ...): if given to func, the output terminal tuple must be passed by nonconst lvalue ref");
+
+  // input_args_t = {input_valuesT&&...}
+  using input_args_t = typename ttg::meta::take_first_n<
+      typename ttg::meta::drop_first_n<func_args_t, std::size_t(void_key ? 0 : 1)>::type,
+      std::tuple_size_v<func_args_t> - (void_key ? 0 : 1) - (have_outterm_tuple ? 1 : 0)>::type;
+  constexpr auto NO_ARGUMENTS_PASSED_AS_NONCONST_LVALUE_REF =
+      !ttg::meta::is_any_nonconst_lvalue_reference_v<input_args_t>;
+  static_assert(
+      NO_ARGUMENTS_PASSED_AS_NONCONST_LVALUE_REF,
+      "ttg::make_tt(func, inedges, outedges): one or more arguments to func can only be passed by nonconst lvalue "
+      "ref; this is illegal, should only pass arguments as const lvalue ref or (nonconst) rvalue ref");
+  /* using empty output terminals */
+  using output_terminals_type = std::tuple<>;
+  using wrapT = typename CallableWrapTTArgsAsTypelist<funcT, have_outterm_tuple, key_type, output_terminals_type,
+                                                      input_args_t>::type;
+  std::size_t num_value_args = have_outterm_tuple ? num_args - 1 : num_args;
+  const std::vector<std::string> &innames = std::vector<std::string>(num_value_args, "input");
+  return std::make_unique<wrapT>(std::forward<funcT>(func), "DynamicTT", innames, std::vector<std::string>{});
+}
+
 template <typename keyT = void, typename funcT, typename... input_edge_valuesT, typename... output_edgesT>
-auto make_tt(funcT &&func, const std::tuple<ttg::Edge<keyT, input_edge_valuesT>...> &inedges = std::tuple<>{},
-             const std::tuple<output_edgesT...> &outedges = std::tuple<>{}, const std::string &name = "wrapper",
+auto make_tt(funcT &&func, const std::tuple<ttg::Edge<keyT, input_edge_valuesT>...> &inedges,
+             const std::tuple<output_edgesT...> &outedges, const std::string &name = "wrapper",
              const std::vector<std::string> &innames = std::vector<std::string>(sizeof...(input_edge_valuesT), "input"),
              const std::vector<std::string> &outnames = std::vector<std::string>(sizeof...(output_edgesT), "output")) {
   // ensure input types do not contain Void
@@ -447,6 +523,7 @@ auto make_tt(funcT &&func, const std::tuple<ttg::Edge<keyT, input_edge_valuesT>.
 
   // gross argument typelist for invoking func, can include void for optional args
   constexpr static auto func_is_generic = ttg::meta::is_generic_callable_v<funcT>;
+
   using gross_func_args_t = decltype(ttg::meta::compute_arg_binding_types_r<void>(func, candidate_func_args_t{}));
   constexpr auto DETECTED_HOW_TO_INVOKE_GENERIC_FUNC =
       func_is_generic ? !std::is_same_v<gross_func_args_t, ttg::typelist<>> : true;
@@ -499,6 +576,7 @@ auto make_tt(funcT &&func, const std::tuple<ttg::Edge<keyT, input_edge_valuesT>.
   return std::make_unique<wrapT>(std::forward<funcT>(func), inedges, outedges, name, innames, outnames);
 }
 
+
 template <typename keyT, typename funcT, typename... input_valuesT, typename... output_edgesT>
 [[deprecated("use make_tt_tpl instead")]] inline auto wrapt(
     funcT &&func, const std::tuple<ttg::Edge<keyT, input_valuesT>...> &inedges,

ttg/ttg/parsec/ttg.h

@@ -1156,6 +1156,8 @@ namespace ttg_parsec {
 
     input_terminals_type input_terminals;
     output_terminalsT output_terminals;
+    std::vector<std::unique_ptr<ttg::TerminalBase>> dynamic_outterms; // used for split construction TTs
+
 
    protected:
     const auto &get_output_terminals() const { return output_terminals; }
@@ -2789,7 +2791,7 @@ namespace ttg_parsec {
         , priomap(decltype(keymap)(std::forward<priomapT>(priomap_)))
         , static_stream_goal() {
       // Cannot call these in base constructor since terminals not yet constructed
-      if (innames.size() != numinedges) throw std::logic_error("ttg_parsec::TT: #input names != #input terminals");
+      //if (innames.size() != numinedges) throw std::logic_error("ttg_parsec::TT: #input names != #input terminals");
       if (outnames.size() != numouts) throw std::logic_error("ttg_parsec::TT: #output names != #output terminals");
 
       auto &world_impl = world.impl();
@@ -3016,12 +3018,51 @@ namespace ttg_parsec {
     }
 
     // Returns reference to output terminal for purpose of connection --- terminal
-    // cannot be copied, moved or assigned
+    // cannot be copied, moved or assigned.
+    // Only valid if terminals are provided during construction.
     template <std::size_t i>
-    std::tuple_element_t<i, output_terminalsT> *out() {
+    auto *out() {
+      static_assert(std::tuple_size_v<output_terminalsT> > 0,
+                    "TT::out<i>() only available for statically known output terminals. "
+                    "Use TT::out<i, K, V>() instead!");
       return &std::get<i>(output_terminals);
     }
 
+    template <std::size_t i, typename KeyT, typename ValueT>
+    auto *out(const std::string name = {}) {
+      using terminal_type = typename ttg::Out<KeyT, ValueT>;
+      if constexpr (std::tuple_size_v<output_terminalsT> == 0) {
+        auto& outputs = this->get_outputs();
+        terminal_type *terminal;
+        if (outputs.size() <= i || nullptr == outputs[i]) {
+          // create a new terminal
+          std::string auto_name;
+          if (name.empty()) {
+            auto_name = "DynamicOut" + std::to_string(i);
+          }
+          terminal = new terminal_type();
+          dynamic_outterms.push_back(std::unique_ptr<ttg::TerminalBase>(terminal));
+          register_terminal<true, terminal_type, i>(*terminal, name.empty() ? auto_name : name, &ttT::set_output_dynamic);
+        } else {
+#ifndef NDEBUG
+          // use dyanmic cast in debug builds
+          terminal = dynamic_cast<terminal_type*>(outputs[i]);
+          if (nullptr == terminal) {
+            throw std::runtime_error(std::string("TT::out<i, K, V>(): failed to cast existing output terminal ") + std::to_string(i));
+          }
+#else
+          terminal = static_cast<terminal_type*>(outputs[i]);
+#endif // NDEBUG
+        }
+        return terminal;
+      } else {
+        static_assert(std::is_same_v<terminal_type, std::tuple_element_t<i, output_terminalsT>>,
+                      "TT::out<i, K, V>() key/value types do not match compile-time output terminals.");
+        return &std::get<i>(output_terminals);
+      }
+    }
+
+
     // Manual injection of a task with all input arguments specified as a tuple
     template <typename Key = keyT>
     std::enable_if_t<!ttg::meta::is_void_v<Key> && !ttg::meta::is_empty_tuple_v<input_values_tuple_type>, void> invoke(
@@ -3105,8 +3146,9 @@ namespace ttg_parsec {
 
     /// keymap setter
     template <typename Keymap>
-    void set_keymap(Keymap &&km) {
+    auto& set_keymap(Keymap &&km) {
       keymap = km;
+      return *this;
     }
 
     /// priority map accessor
@@ -3116,8 +3158,43 @@ namespace ttg_parsec {
     /// priomap setter
     /// @arg pm a function that maps a key to an integral priority value.
     template <typename Priomap>
-    void set_priomap(Priomap &&pm) {
+    auto& set_priomap(Priomap &&pm) {
       priomap = std::forward<Priomap>(pm);
+      return *this;
+    }
+
+    template<std::size_t i>
+    auto& set_input(ttg::Edge<keyT, std::tuple_element_t<i, input_values_tuple_type>>& edge,
+                    const std::string name = {}) {
+
+      edge.set_out(&std::get<i>(input_terminals));
+      if (!name.empty()) {
+        std::get<i>(input_terminals).set_name(name);
+      }
+      return *this;
+    }
+
+    template<std::size_t i, typename Key, typename Value>
+    auto& set_output(ttg::Edge<Key, Value>& edge,
+                     const std::string name = {}) {
+      {
+        /* check whether the terminal is already set */
+        auto& outputs = this->get_outputs();
+        if (outputs.size() > i && nullptr != outputs[i]) {
+          throw(this->get_name() + ": output terminal " + std::to_string(i) + " already set");
+        }
+      }
+      std::string auto_name;
+      if (name.empty()) {
+        auto_name = "DynamicOut" + std::to_string(i);
+      }
+      //std::get<i>(input_terminals).set_name(name);
+      using term_t = typename ttg::Edge<Key, Value>::output_terminal_type;
+      term_t *term = new term_t();
+      dynamic_outterms.push_back(std::unique_ptr<ttg::TerminalBase>(term));
+      register_terminal<true, term_t, i>(*term, name.empty() ? auto_name : name, &ttT::set_output_dynamic);
+      edge.set_in(term);
+      return *this;
     }
 
     // Register the static_op function to associate it to instance_id

ttg/ttg/util/meta/callable.h

@@ -58,7 +58,7 @@ namespace ttg::meta {
   }
 
   template <std::size_t Ordinal, typename Func, typename... Typelists, std::size_t... ArgIdx>
-  auto compute_arg_binding_types_impl(Func& func, typelist<Typelists...> argument_type_lists,
+  auto compute_arg_binding_types_impl(Func&& func, typelist<Typelists...> argument_type_lists,
                                       std::index_sequence<ArgIdx...> arg_idx = {}) {
     using arg_typelists_t = typelist<Typelists...>;
     constexpr auto Order = sizeof...(Typelists);
@@ -80,7 +80,7 @@ namespace ttg::meta {
   }
 
   template <std::size_t Ordinal, typename ReturnType, typename Func, typename... Typelists, std::size_t... ArgIdx>
-  auto compute_arg_binding_types_r_impl(Func& func, typelist<Typelists...> argument_type_lists,
+  auto compute_arg_binding_types_r_impl(Func&& func, typelist<Typelists...> argument_type_lists,
                                         std::index_sequence<ArgIdx...> arg_idx = {}) {
     using arg_typelists_t = typelist<Typelists...>;
     constexpr auto Order = sizeof...(Typelists);
@@ -110,7 +110,7 @@ namespace ttg::meta {
   ///         - the first invocable combination of argument types discovered by row-major iteration, if @p func is a
   ///         generic callable
   template <typename Func, typename... Typelists>
-  auto compute_arg_binding_types(Func& func, typelist<Typelists...> argument_type_lists) {
+  auto compute_arg_binding_types(Func&& func, typelist<Typelists...> argument_type_lists) {
     constexpr auto is_generic__args = callable_args<Func&>;
     constexpr bool is_generic = is_generic__args.first;
     if constexpr (is_generic) {
@@ -131,7 +131,7 @@ namespace ttg::meta {
   ///         - the first invocable combination of argument types discovered by row-major iteration, if @p func is a
   ///         generic callable
   template <typename ReturnType, typename Func, typename... Typelists>
-  auto compute_arg_binding_types_r(Func& func, typelist<Typelists...> argument_type_lists) {
+  auto compute_arg_binding_types_r(Func&& func, typelist<Typelists...> argument_type_lists) {
     constexpr auto is_generic__args = callable_args<Func&>;
     constexpr bool is_generic = is_generic__args.first;
     if constexpr (is_generic) {