Add memory invariant assumptions for virtual calls to improve devirtualization (#4596)

Assumptions:
- class methods do not change the object's vtable pointer
- loads through vtable pointer are invariant (vtables are immutable throughout program life)

Author: JohanEngelen

gen/classes.cpp

@@ -422,7 +422,8 @@ DValue *DtoDynamicCastInterface(const Loc &loc, DValue *val, Type *_to) {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-LLValue *DtoVirtualFunctionPointer(DValue *inst, FuncDeclaration *fdecl) {
+std::pair<llvm::Value *, llvm::Value *>
+DtoVirtualFunctionPointer(DValue *inst, FuncDeclaration *fdecl) {
   // sanity checks
   assert(fdecl->isVirtual());
   assert(!fdecl->isFinalFunc());
@@ -440,17 +441,22 @@ LLValue *DtoVirtualFunctionPointer(DValue *inst, FuncDeclaration *fdecl) {
   const auto irtc = getIrType(tc->sym->type, true)->isClass();
   const auto vtblType = irtc->getVtblType();
 
-  LLValue *funcval = vthis;
+  LLValue *vtable = vthis;
   // get the vtbl for objects
-  funcval = DtoGEP(irtc->getMemoryLLType(), funcval, 0u, 0);
+  vtable = DtoGEP(irtc->getMemoryLLType(), vthis, 0u, 0);
   // load vtbl ptr
-  funcval = DtoLoad(vtblType->getPointerTo(), funcval);
+  vtable = DtoLoad(vtblType->getPointerTo(), vtable);
   // index vtbl
   const std::string name = fdecl->toChars();
   const auto vtblname = name + "@vtbl";
-  funcval = DtoGEP(vtblType, funcval, 0, fdecl->vtblIndex, vtblname.c_str());
-  // load opaque pointer
+  LLValue *funcval =
+      DtoGEP(vtblType, vtable, 0, fdecl->vtblIndex, vtblname.c_str());
+  // load opaque pointer.
   funcval = DtoAlignedLoad(vtblType->getElementType(), funcval);
+  // Because vtables are immutable, LLVM's !invariant.load
+  // can be applied (helps with devirtualization).
+  llvm::cast<llvm::LoadInst>(funcval)->setMetadata(
+      "invariant.load", llvm::MDNode::get(gIR->context(), {}));
 
   IF_LOG Logger::cout() << "funcval: " << *funcval << '\n';
 
@@ -462,5 +468,5 @@ LLValue *DtoVirtualFunctionPointer(DValue *inst, FuncDeclaration *fdecl) {
 
   IF_LOG Logger::cout() << "funcval casted: " << *funcval << '\n';
 
-  return funcval;
+  return std::make_pair(funcval, vtable);
 }

gen/classes.h

@@ -14,6 +14,8 @@
 
 #pragma once
 
+#include <utility>
+
 #include "gen/structs.h"
 
 class ClassDeclaration;
@@ -37,4 +39,6 @@ DValue *DtoDynamicCastObject(const Loc &loc, DValue *val, Type *to);
 
 DValue *DtoDynamicCastInterface(const Loc &loc, DValue *val, Type *to);
 
-llvm::Value *DtoVirtualFunctionPointer(DValue *inst, FuncDeclaration *fdecl);
+/// Returns pair of function pointer and vtable pointer.
+std::pair<llvm::Value *, llvm::Value *>
+DtoVirtualFunctionPointer(DValue *inst, FuncDeclaration *fdecl);

gen/dvalue.cpp

@@ -105,11 +105,13 @@ LLValue *DSliceValue::getPtr() {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-DFuncValue::DFuncValue(Type *t, FuncDeclaration *fd, LLValue *v, LLValue *vt)
-    : DRValue(t, v), func(fd), vthis(vt) {}
+DFuncValue::DFuncValue(Type *t, FuncDeclaration *fd, LLValue *v, LLValue *vt,
+                       LLValue *vtable)
+    : DRValue(t, v), func(fd), vthis(vt), vtable(vtable) {}
 
-DFuncValue::DFuncValue(FuncDeclaration *fd, LLValue *v, LLValue *vt)
-    : DFuncValue(fd->type, fd, v, vt) {}
+DFuncValue::DFuncValue(FuncDeclaration *fd, LLValue *v, LLValue *vt,
+                       LLValue *vtable)
+    : DFuncValue(fd->type, fd, v, vt, vtable) {}
 
 bool DFuncValue::definedInFuncEntryBB() {
   return isDefinedInFuncEntryBB(val) &&
@@ -235,7 +237,7 @@ DRValue *DDcomputeLValue::getRVal() {
     llvm_unreachable("getRVal() for memory-only type");
     return nullptr;
   }
-  
+
   LLValue *rval = DtoLoad(lltype, val);
 
   const auto ty = type->toBasetype()->ty;

gen/dvalue.h

@@ -137,15 +137,18 @@ class DSliceValue : public DRValue {
   llvm::Value *getPtr();
 };
 
-/// Represents a D function value with optional this/context pointer.
+/// Represents a D function value with optional this/context pointer, and
+/// optional vtable pointer.
 class DFuncValue : public DRValue {
 public:
   FuncDeclaration *func;
   llvm::Value *vthis;
+  llvm::Value *vtable;
 
   DFuncValue(Type *t, FuncDeclaration *fd, llvm::Value *v,
-             llvm::Value *vt = nullptr);
-  DFuncValue(FuncDeclaration *fd, llvm::Value *v, llvm::Value *vt = nullptr);
+             llvm::Value *vt = nullptr, llvm::Value *vtable = nullptr);
+  DFuncValue(FuncDeclaration *fd, llvm::Value *v, llvm::Value *vt = nullptr,
+             llvm::Value *vtable = nullptr);
 
   bool definedInFuncEntryBB() override;
 

gen/ms-cxx-helper.cpp

@@ -97,8 +97,7 @@ void cloneBlocks(const std::vector<llvm::BasicBlock *> &srcblocks,
     for (auto &II : *bb) {
       llvm::Instruction *Inst = &II;
       llvm::Instruction *newInst = nullptr;
-      if (funclet &&
-          !llvm::isa<llvm::DbgInfoIntrinsic>(Inst)) { // IntrinsicInst?
+      if (funclet && !llvm::isa<llvm::IntrinsicInst>(Inst)) {
         if (auto IInst = llvm::dyn_cast<llvm::InvokeInst>(Inst)) {
           auto invoke = llvm::InvokeInst::Create(
               IInst, llvm::OperandBundleDef("funclet", funclet));

gen/toir.cpp

@@ -771,13 +771,54 @@ class ToElemVisitor : public Visitor {
 
     // get func value if any
     DFuncValue *dfnval = fnval->isFunc();
+
+    // If this is a virtual function call, the object is passed by reference
+    // through the `this` parameter, and therefore the optimizer has to assume
+    // that the vtable field might be overwritten. This prevents optimization of
+    // subsequent virtual calls on the same object. We help the optimizer by
+    // allowing it to assume that the vtable field contents is the same after
+    // the call. Equivalent D code:
+    // ```
+    //  auto saved_vtable = a.__vptr;     // emitted as part of `a.foo()`,
+    //                                    // except when e->directcall==true for
+    //                                    // final method calls.
+    //  a.foo();
+    //  assume(a.__vptr == saved_vtable); // <-- added assumption
+    // ```
+    // Only emit this extra code from -O2.
+    // This optimization is only valid for D class method calls (not C++).
+    bool canEmitVTableUnchangedAssumption =
+        dfnval && dfnval->func && (dfnval->func->_linkage == LINK::d) &&
+        (optLevel() >= 2);
+
     if (dfnval && dfnval->func) {
       assert(!DtoIsMagicIntrinsic(dfnval->func));
+
+      // If loading the vtable was not needed for function call, we have to load
+      // it here to do the "assume" optimization below.
+      if (canEmitVTableUnchangedAssumption && !dfnval->vtable &&
+          dfnval->vthis && dfnval->func->isVirtual()) {
+        dfnval->vtable =
+            DtoLoad(getVoidPtrType(),
+                    DtoBitCast(dfnval->vthis, getVoidPtrType()->getPointerTo()),
+                    "saved_vtable");
+      }
     }
 
     DValue *result =
         DtoCallFunction(e->loc, e->type, fnval, e->arguments, sretPointer);
 
+    if (canEmitVTableUnchangedAssumption && dfnval->vtable) {
+      // Reload vtable ptr. It's the first element so instead of GEP+load we can
+      // do a void* load+bitcast (at this point in the code we don't have easy
+      // access to the type of the class to do a GEP).
+      auto vtable = DtoLoad(
+          dfnval->vtable->getType(),
+          DtoBitCast(dfnval->vthis, dfnval->vtable->getType()->getPointerTo()));
+      auto cmp = p->ir->CreateICmpEQ(vtable, dfnval->vtable);
+      p->ir->CreateCall(GET_INTRINSIC_DECL(assume), {cmp});
+    }
+
     if (delayedDtorVar) {
       delayedDtorVar->edtor = delayedDtorExp;
       pushVarDtorCleanup(p, delayedDtorVar);
@@ -1044,16 +1085,17 @@ class ToElemVisitor : public Visitor {
 
       // Get the actual function value to call.
       LLValue *funcval = nullptr;
+      LLValue *vtable = nullptr;
       if (nonFinal) {
         DtoResolveFunction(fdecl);
-        funcval = DtoVirtualFunctionPointer(l, fdecl);
+        std::tie(funcval, vtable) = DtoVirtualFunctionPointer(l, fdecl);
       } else {
         funcval = DtoCallee(fdecl);
       }
       assert(funcval);
 
       LLValue *vthis = (DtoIsInMemoryOnly(l->type) ? DtoLVal(l) : DtoRVal(l));
-      result = new DFuncValue(fdecl, funcval, vthis);
+      result = new DFuncValue(fdecl, funcval, vthis, vtable);
     } else {
       llvm_unreachable("Unknown target for VarDeclaration.");
     }
@@ -1950,7 +1992,7 @@ class ToElemVisitor : public Visitor {
 
     if (e->e1->op != EXP::super_ && e->e1->op != EXP::dotType &&
         e->func->isVirtual() && !e->func->isFinalFunc()) {
-      castfptr = DtoVirtualFunctionPointer(u, e->func);
+      castfptr = DtoVirtualFunctionPointer(u, e->func).first;
     } else if (e->func->isAbstract()) {
       llvm_unreachable("Delegate to abstract method not implemented.");
     } else if (e->func->toParent()->isInterfaceDeclaration()) {

tests/codegen/devirtualization_assumevtable.d

@@ -0,0 +1,86 @@
+// Tests that class member function calls do not prevent devirtualization (vtable cannot change in class member call).
+
+// RUN: %ldc -output-ll -of=%t.ll %s -O3 && FileCheck %s < %t.ll
+
+class A {
+    void foo();
+    final void oof();
+}
+class B : A {
+    override void foo();
+}
+
+// CHECK-LABEL: define{{.*}}ggg
+void ggg()
+{
+    A a = new A();
+    // CHECK: call {{.*}}_D29devirtualization_assumevtable1A3foo
+    a.foo();
+    // CHECK: call {{.*}}_D29devirtualization_assumevtable1A3foo
+    a.foo();
+}
+
+// CHECK-LABEL: define{{.*}}hhh
+void hhh()
+{
+    A a = new A();
+    // CHECK: call {{.*}}_D29devirtualization_assumevtable1A3oof
+    a.oof();
+    // CHECK: call {{.*}}_D29devirtualization_assumevtable1A3foo
+    a.foo();
+}
+
+// CHECK-LABEL: define{{.*}}directcall
+void directcall()
+{
+    A a = new A();
+    // CHECK: call {{.*}}_D29devirtualization_assumevtable1A3foo
+    a.A.foo();
+    // CHECK: call {{.*}}_D29devirtualization_assumevtable1A3foo
+    a.foo();
+}
+// CHECK-LABEL: define{{.*}}exacttypeunknown
+void exacttypeunknown(A a, A b)
+{
+    // CHECK: %[[FOO:[0-9a-z]+]] = load {{.*}}!invariant
+    // CHECK: call{{.*}} void %[[FOO]](
+    a.foo();
+    // CHECK: call{{.*}} void %[[FOO]](
+    a.foo();
+
+    a = b;
+    // CHECK: %[[FOO2:[0-9a-z]+]] = load {{.*}}!invariant
+    // CHECK: call{{.*}} void %[[FOO2]](
+    a.foo();
+}
+
+// No vtable loading and assume calls for struct method calls.
+struct S {
+    void foo();
+}
+// CHECK-LABEL: define{{.*}}structS
+void structS(S s)
+{
+    // CHECK-NOT: llvm.assume
+    // CHECK-NOT: load
+    s.foo();
+    // CHECK: ret void
+}
+
+// The devirtualization is not valid for C++ methods.
+extern(C++)
+class CPPClass {
+    void foo();
+    void oof();
+}
+
+// CHECK-LABEL: define{{.*}}exactCPPtypeunknown
+void exactCPPtypeunknown(CPPClass a)
+{
+    // CHECK: %[[FOO:[0-9a-z]+]] = load {{.*}}!invariant
+    // CHECK: call{{.*}} void %[[FOO]](
+    a.foo();
+    // CHECK: %[[FOO2:[0-9a-z]+]] = load {{.*}}!invariant
+    // CHECK: call{{.*}} void %[[FOO2]](
+    a.foo();
+}