InvariantAnnotationsForLocals.patch

Index: lib/CodeGen/CGDecl.cpp
===================================================================
--- lib/CodeGen/CGDecl.cpp	(revision 222615)
+++ lib/CodeGen/CGDecl.cpp	(working copy)
@@ -518,7 +518,24 @@
         ->setDoesNotThrow();
     }
   };
+
+  /// A cleanup to call @llvm.invariant.end.
+  class CallInvariantEnd : public EHScopeStack::Cleanup {
+    llvm::Value *Start;
+    llvm::Value *Addr;
+    llvm::Value *Size;
+  public:
+    CallInvariantEnd(llvm::Value* start, llvm::Value *addr, llvm::Value *size)
+      : Start(start), Addr(addr), Size(size) {}
+
+    void Emit(CodeGenFunction &CGF, Flags flags) override {
+      llvm::Value *castAddr = CGF.Builder.CreateBitCast(Addr, CGF.Int8PtrTy);
+      CGF.Builder.CreateCall3(CGF.CGM.getLLVMInvariantEndFn(),
+                              Start, Size, castAddr)
+        ->setDoesNotThrow();
+    }
+  };
}

 /// EmitAutoVarWithLifetime - Does the setup required for an automatic
 /// variable with lifetime.
@@ -954,17 +971,30 @@
       // Emit a lifetime intrinsic if meaningful.  There's no point
       // in doing this if we don't have a valid insertion point (?).
       uint64_t size = CGM.getDataLayout().getTypeAllocSize(LTy);
-      if (HaveInsertPoint() && shouldUseLifetimeMarkers(*this, D, size)) {
-        llvm::Value *sizeV = llvm::ConstantInt::get(Int64Ty, size);
+      if (HaveInsertPoint()) {
+        if (shouldUseLifetimeMarkers(*this, D, size)) {
+          emission.IsLifetimeAnnotated = true;

-        emission.SizeForLifetimeMarkers = sizeV;
-        llvm::Value *castAddr = Builder.CreateBitCast(Alloc, Int8PtrTy);
+          llvm::Value *castAddr = Builder.CreateBitCast(Alloc, Int8PtrTy);
-        Builder.CreateCall2(CGM.getLLVMLifetimeStartFn(), sizeV, castAddr)
+          Builder.CreateCall2(CGM.getLLVMLifetimeStartFn(),
+                              emission.storeSizeForMarkers(Int64Ty, size), castAddr)
-          ->setDoesNotThrow();
+            ->setDoesNotThrow();
-      } else {
+        } else {
-        assert(!emission.useLifetimeMarkers());
+          assert(!emission.useLifetimeMarkers());
-      }
+        }
+
+        // If the object is constant, emit an invariant intrinsic as well
+        if (Ty.isConstQualified()) {
+          llvm::Value *castAddr = Builder.CreateBitCast(Alloc, Int8PtrTy);
+          llvm::CallInst* start
+              = Builder.CreateCall2(CGM.getLLVMInvariantStartFn(),
+                                    emission.storeSizeForMarkers(Int64Ty, size),
+                                    castAddr);
+          start->setDoesNotThrow();
+          emission.InvariantStartValue = start;
+        }
+      }
     }
   } else {
     EnsureInsertPoint();

@@ -1306,9 +1336,18 @@
   if (emission.useLifetimeMarkers()) {
     EHStack.pushCleanup<CallLifetimeEnd>(NormalCleanup,
                                          emission.getAllocatedAddress(),
-                                         emission.getSizeForLifetimeMarkers());
+                                         emission.getSizeForMarkers());
   }

+  // Just before the lifetime ends, call @llvm.invariant.end if the
+  // we previously called @llvm.invariant.start.
+  if (llvm::CallInst *ISV = emission.getInvariantStartValue()) {
+    EHStack.pushCleanup<CallInvariantEnd>(NormalCleanup,
+                                          ISV,
+                                          emission.getAllocatedAddress(),
+                                          emission.getSizeForMarkers());
+  }
+
   // Check the type for a cleanup.
   if (QualType::DestructionKind dtorKind = D.getType().isDestructedType())
     emitAutoVarTypeCleanup(emission, dtorKind);
@@ -1619,6 +1658,22 @@
   return LifetimeEndFn;
 }

+/// Lazily declare the @llvm.invariant.start intrinsic.
+llvm::Constant *CodeGenModule::getLLVMInvariantStartFn() {
+  if (InvariantStartFn) return InvariantStartFn;
+  InvariantStartFn = llvm::Intrinsic::getDeclaration(&getModule(),
+                                            llvm::Intrinsic::invariant_start);
+  return InvariantStartFn;
+}
+
+/// Lazily declare the @llvm.invariant.end intrinsic.
+llvm::Constant *CodeGenModule::getLLVMInvariantEndFn() {
+  if (InvariantEndFn) return InvariantEndFn;
+  InvariantEndFn = llvm::Intrinsic::getDeclaration(&getModule(),
+                                              llvm::Intrinsic::invariant_end);
+  return InvariantEndFn;
+}
+
 namespace {
   /// A cleanup to perform a release of an object at the end of a
   /// function.  This is used to balance out the incoming +1 of a
Index: lib/CodeGen/CodeGenFunction.h
===================================================================
--- lib/CodeGen/CodeGenFunction.h	(revision 222615)
+++ lib/CodeGen/CodeGenFunction.h	(working copy)
@@ -1867,9 +1867,17 @@
     /// initializer.
     bool IsConstantAggregate;

-    /// Non-null if we should use lifetime annotations.
-    llvm::Value *SizeForLifetimeMarkers;
+    // True if the variable should be annotated with @llvm.lifetime.start
+    // and @llvm.lifetime.end
+    bool IsLifetimeAnnotated;

+    /// Non-null if there is a lifetime or invariant annotation.
+    llvm::Value *SizeForMarkers;
+
+    /// Represents value returned by @llvm.invariant.start, to be used
+    /// for the matching @llvm.invariant.end.
+    llvm::CallInst *InvariantStartValue;
+
     struct Invalid {};
     AutoVarEmission(Invalid) : Variable(nullptr) {}

@@ -1876,21 +1884,34 @@
     AutoVarEmission(const VarDecl &variable)
       : Variable(&variable), Address(nullptr), NRVOFlag(nullptr),
         IsByRef(false), IsConstantAggregate(false),
-        SizeForLifetimeMarkers(nullptr) {}
+        IsLifetimeAnnotated(false),
+        SizeForMarkers(nullptr), InvariantStartValue(nullptr) {}

     bool wasEmittedAsGlobal() const { return Address == nullptr; }

+    llvm::Value *storeSizeForMarkers(llvm::IntegerType* IntTy, uint64_t size) {
+      if (!SizeForMarkers) {
+        SizeForMarkers = llvm::ConstantInt::get(IntTy, size);
+      }
+      return SizeForMarkers;
+    }
+
   public:
     static AutoVarEmission invalid() { return AutoVarEmission(Invalid()); }

     bool useLifetimeMarkers() const {
-      return SizeForLifetimeMarkers != nullptr;
+      return IsLifetimeAnnotated;
     }
-    llvm::Value *getSizeForLifetimeMarkers() const {
-      assert(useLifetimeMarkers());
-      return SizeForLifetimeMarkers;
+
+    bool useInvariantMarkers() const {
+        return InvariantStartValue != nullptr;
     }

+    llvm::Value *getSizeForMarkers() const {
+      assert(useLifetimeMarkers() || useInvariantMarkers());
+      return SizeForMarkers;
+    }
+
     /// Returns the raw, allocated address, which is not necessarily
     /// the address of the object itself.
     llvm::Value *getAllocatedAddress() const {
@@ -1897,6 +1918,10 @@
       return Address;
     }

+    llvm::CallInst* getInvariantStartValue() const {
+        return InvariantStartValue;
+    }
+
     /// Returns the address of the object within this declaration.
     /// Note that this does not chase the forwarding pointer for
     /// __block decls.
Index: lib/CodeGen/CodeGenModule.h
===================================================================
--- lib/CodeGen/CodeGenModule.h	(revision 222615)
+++ lib/CodeGen/CodeGenModule.h	(working copy)
@@ -473,6 +473,12 @@
   /// void @llvm.lifetime.end(i64 %size, i8* nocapture <ptr>)
   llvm::Constant *LifetimeEndFn;

+  /// declare {}* @llvm.invariant.start(i64 <size>, i8* nocapture <ptr>)
+  llvm::Constant *InvariantStartFn;
+
+  /// void @llvm.invariant.end({}* <start>, i64 <size>, i8* nocapture <ptr>)
+  llvm::Constant *InvariantEndFn;
+
   GlobalDecl initializedGlobalDecl;

   std::unique_ptr<SanitizerMetadata> SanitizerMD;
@@ -891,6 +897,9 @@
   llvm::Constant *getLLVMLifetimeStartFn();
   llvm::Constant *getLLVMLifetimeEndFn();

+  llvm::Constant *getLLVMInvariantStartFn();
+  llvm::Constant *getLLVMInvariantEndFn();
+
   // Make sure that this type is translate