[WIP] Even more registers

include/tc/core/polyhedral/memory_promotion.h

@@ -39,11 +39,20 @@ enum class AccessType : short { Read, Write };
 // constant size.
 struct ScopedFootprintDim {
  public:
-  ScopedFootprintDim(isl::aff lb, isl::val s) : lowerBound(lb), size(s) {}
+  ScopedFootprintDim(isl::aff lb, isl::val s) : lowerBound(lb), size(s), stride(isl::val::zero(s.get_ctx())), shift(isl::aff()) {}
+  ScopedFootprintDim(isl::aff lowerBound_, isl::val size_, isl::val stride_, isl::aff shift_)
+    : lowerBound(lowerBound_), size(size_), stride(stride_), shift(shift_) {}
+
+  bool hasStride() const {
+    return stride != 0;
+  }
 
  public:
   isl::aff lowerBound;
   isl::val size;
+
+  isl::val stride;
+  isl::aff shift;
 };
 
 // Rectangular overapproximation of a tensor elements accessed through a single
@@ -54,8 +63,12 @@ struct ScopedFootprintDim {
 struct ScopedFootprint : std::vector<ScopedFootprintDim> {
   isl::set footprint(isl::set domain) const;
   isl::multi_aff lowerBounds() const;
+  isl::multi_aff shifts() const;
+  isl::multi_val strides() const;
 };
 
+ScopedFootprint outputRanges(isl::map access);
+
 // Descriptor of tensor reference in a Scop.
 // May be scoped to a specific position in a schedule tree, the user is
 // responsible for maintaining the correspondance between schedule tree
@@ -78,6 +91,11 @@ class TensorReference {
   // reference group is introduced in the tree.
   isl::map scopedAccess;
 
+  // Access relation in terms of full schedule.
+  // FIXME: scopedAccess can always be obtained by projecting out from tis if
+  // we know the scoping depth.
+  isl::map scheduledAccess;
+
   // Access direction (read or write).
   AccessType type;
 
@@ -106,6 +124,10 @@ class TensorReferenceGroup {
   static TensorGroups accessedBySubtree(
       const detail::ScheduleTree* tree,
       const Scop& scop);
+  static TensorGroups accessedByThreadsInSubtree(
+    const detail::ScheduleTree* tree,
+    const detail::ScheduleTree* threadMappedTree,
+    const Scop& scop);
 
   bool isReadOnly() const;
 
@@ -208,7 +230,19 @@ detail::ScheduleTree* insertCopiesUnder(
     Scop& scop,
     detail::ScheduleTree* tree,
     const TensorReferenceGroup& group,
+    bool useExactReads,
     isl::id tensorId,
     isl::id groupId = isl::id());
+
+detail::ScheduleTree* insertIntraCopiesUnder(
+    Scop& scop,
+    detail::ScheduleTree* tree,
+    const TensorReferenceGroup& group,
+    const TensorReferenceGroup& outerScopeGroup,
+    bool useExactReads,
+    isl::id tensorId,
+    isl::id groupId,
+    isl::id outerScopeGroupId);
+
 } // namespace polyhedral
 } // namespace tc

include/tc/core/polyhedral/scop.h

@@ -329,6 +329,12 @@ struct Scop {
     return activePromotions_;
   }
 
+  std::vector<std::pair<isl::union_set, Scop::PromotionInfo>> activePromotions(
+      isl::union_set activePoints,
+      isl::id tensorId) const {
+    return promotionsAtIndexes(activePromotionsIndexes(activePoints, tensorId));
+  }
+
   detail::ScheduleTree* scheduleRoot() {
     return scheduleTreeUPtr.get();
   }
@@ -379,6 +385,8 @@ struct Scop {
       isl::union_map schedule,
       bool forceLastExtentOdd = false);
 
+  void demoteGroup(isl::id groupId);
+
   // Given a tree node under which the promotion copy statements were
   // introduced, insert syncthread statements before and after the copies.
   // The tree should match the structure:
@@ -408,6 +416,22 @@ struct Scop {
       isl::schedule_constraints constraints,
       const SchedulerOptionsView& schedulerOptions);
 
+  // Get the indexes of active promotions in the activePromotions_.
+  std::vector<size_t> activePromotionsIndexes(
+      isl::union_set domain,
+      isl::id tensorId) const;
+  std::vector<std::pair<isl::union_set, Scop::PromotionInfo>>
+  promotionsAtIndexes(const std::vector<size_t>& indexes) const;
+
+  void promoteWithCopyFromGlobal(
+      isl::union_set activePoints,
+      PromotedDecl::Kind kind,
+      isl::id tensorId,
+      std::unique_ptr<TensorReferenceGroup>&& gr,
+      detail::ScheduleTree* tree,
+      isl::union_map schedule,
+      bool forceLastExtentOdd = false);
+
  public:
   // Halide stuff
   struct {

include/tc/external/detail/islpp-inl.h

@@ -44,6 +44,10 @@ inline isl::aff operator/(isl::aff A, int i) {
   return A.div(T);
 }
 
+inline isl::aff operator/(isl::aff A, isl::val v) {
+  return A.scale_down(v);
+}
+
 inline isl::aff operator+(int i, isl::aff A) {
   isl::ctx ctx = A.get_ctx();
   return A + isl::val(ctx, i);

include/tc/external/detail/islpp.h

@@ -121,6 +121,7 @@ isl::aff operator*(isl::aff A, isl::val V);
 isl::aff operator*(isl::val V, isl::aff A);
 
 isl::aff operator/(isl::aff A, int i);
+isl::aff operator/(isl::aff A, isl::val v);
 
 isl::aff operator+(int i, isl::aff A);
 isl::aff operator+(isl::aff A, isl::aff B);

src/core/polyhedral/codegen_cuda.cc

@@ -803,7 +803,10 @@ string emitCudaKernel(
   astBuild = isl::manage(isl_ast_build_set_at_each_domain(
       astBuild.release(), collect, &iteratorMaps));
   astBuild = astBuild.set_iterators(Codegen::makeLoopIterators(ctx, maxDepth));
+  isl_ctx_reset_operations(astBuild.get_ctx().get());
+  isl_ctx_set_max_operations(astBuild.get_ctx().get(), 10000000);
   auto astNode = astBuild.node_from(schedule);
+  isl_ctx_set_max_operations(astBuild.get_ctx().get(), 0);
   AstPrinter(CodegenContext(ss, mscop, iteratorMaps)).emit(astNode);
   ss << "}" << endl;