Windows: modules other than gc share the process heap; attributes named once; Debug DEBUG_TYPE fix

tslang/include/TypeScript/Defines.h

@@ -152,6 +152,15 @@
 #define SHARED_LIB_OWN_FACTS "__tsown_"
 // a module attribute, so it has to carry the dialect's prefix
 #define SHARED_LIB_OWN_NO_DROPS_ATTR_NAME "ts.own_imported_no_drops"
+
+// The process-heap allocator of a Windows module (TypeScript/ProcessHeap.h), which ProcessHeapPass
+// calls instead of the C runtime's. Defined by the async runtime library a program links,
+// TypeScriptRuntime.dll and the JIT.
+#define PROCESS_HEAP_MALLOC "__tslang_heap_malloc"
+#define PROCESS_HEAP_CALLOC "__tslang_heap_calloc"
+#define PROCESS_HEAP_REALLOC "__tslang_heap_realloc"
+#define PROCESS_HEAP_FREE "__tslang_heap_free"
+#define PROCESS_HEAP_ALIGNED_ALLOC "__tslang_heap_aligned_alloc"
 #define DLL_EXPORT "dllexport"
 #define DLL_IMPORT "dllimport"
 #define DLL_NAME "dllname"

tslang/include/TypeScript/Passes.h

@@ -47,6 +47,7 @@ std::unique_ptr<mlir::Pass> createOwnedReturnConsumptionPass(CompileOptions&);
 
 /// GC Pass to replace malloc, realloc, free with GC_malloc, GC_realloc, GC_free
 std::unique_ptr<mlir::Pass> createGCPass(CompileOptions&);
+std::unique_ptr<mlir::Pass> createProcessHeapPass();
 /// MemAlloc Pass to replace ts_malloc, ts_realloc, ts_free
 std::unique_ptr<mlir::Pass> createMemAllocPass(CompileOptions&);
 

tslang/include/TypeScript/ProcessHeap.h

@@ -0,0 +1,95 @@
+#ifndef TYPESCRIPT_PROCESS_HEAP_H_
+#define TYPESCRIPT_PROCESS_HEAP_H_
+
+// The allocator every module tslang builds on Windows uses, under every memory model but gc
+// (ProcessHeapPass renames the module's malloc, calloc, realloc, free and aligned_alloc to these).
+//
+// A block made by one module is freed by another - an object of a library's class under -mm=rc or
+// -mm=own is made in the library and destroyed by the program, a coroutine frame comes from the
+// runtime and goes back through the program - so they have to share one allocator, and the C
+// runtime's is not one: each module links its own copy of the static CRT, and a debug CRT keeps a
+// list of its blocks per copy, while tslang.exe's own `malloc` is rpmalloc when LLVM is the
+// prebuilt package. The process heap is the same heap in every module and every build.
+//
+// Header-only, so each place that has to provide them - the async runtime library a program links,
+// TypeScriptRuntime.dll and the JIT - compiles the same code.
+
+#ifdef _WIN32
+
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
+#include <windows.h>
+
+#include <cstddef>
+#include <cstdint>
+
+namespace typescript
+{
+namespace process_heap
+{
+
+// What HeapAlloc guarantees: 16 bytes on x64, 8 on x86.
+constexpr size_t alignment = MEMORY_ALLOCATION_ALIGNMENT;
+
+inline void *allocate(size_t size)
+{
+    // malloc(0) hands back a block of its own too
+    return HeapAlloc(GetProcessHeap(), 0, size ? size : 1);
+}
+
+inline void *allocateZeroed(size_t count, size_t size)
+{
+    if (size != 0 && count > SIZE_MAX / size)
+    {
+        return nullptr;
+    }
+
+    auto bytes = count * size;
+    return HeapAlloc(GetProcessHeap(), HEAP_ZERO_MEMORY, bytes ? bytes : 1);
+}
+
+inline void release(void *ptr)
+{
+    if (ptr)
+    {
+        HeapFree(GetProcessHeap(), 0, ptr);
+    }
+}
+
+inline void *reallocate(void *ptr, size_t size)
+{
+    if (!ptr)
+    {
+        return allocate(size);
+    }
+
+    // as the C runtime's realloc: a block resized to nothing is freed, and there is none
+    if (size == 0)
+    {
+        release(ptr);
+        return nullptr;
+    }
+
+    return HeapReAlloc(GetProcessHeap(), 0, ptr, size);
+}
+
+// Every request anything makes fits what HeapAlloc aligns to (the coroutine frame asks for 8). A
+// stricter one cannot be served and still go back through release(); handing back under-aligned
+// memory silently would be the worse failure, so it fails.
+inline void *allocateAligned(size_t align, size_t size)
+{
+    if (align > alignment)
+    {
+        return nullptr;
+    }
+
+    return allocate(size);
+}
+
+} // namespace process_heap
+} // namespace typescript
+
+#endif // _WIN32
+
+#endif // TYPESCRIPT_PROCESS_HEAP_H_

tslang/lib/ProcessHeapExports.inc

@@ -0,0 +1,34 @@
+// The process-heap allocator under the names ProcessHeapPass calls (PROCESS_HEAP_* in
+// TypeScript/Defines.h). Included by the async runtime library, which a program and a library link,
+// and by TypeScriptRuntime.dll, which the JIT loads; see TypeScript/ProcessHeap.h for why.
+
+#ifdef _WIN32
+
+#include "TypeScript/ProcessHeap.h"
+
+extern "C" void *__tslang_heap_malloc(size_t size)
+{
+    return typescript::process_heap::allocate(size);
+}
+
+extern "C" void *__tslang_heap_calloc(size_t count, size_t size)
+{
+    return typescript::process_heap::allocateZeroed(count, size);
+}
+
+extern "C" void *__tslang_heap_realloc(void *ptr, size_t size)
+{
+    return typescript::process_heap::reallocate(ptr, size);
+}
+
+extern "C" void __tslang_heap_free(void *ptr)
+{
+    typescript::process_heap::release(ptr);
+}
+
+extern "C" void *__tslang_heap_aligned_alloc(size_t alignment, size_t size)
+{
+    return typescript::process_heap::allocateAligned(alignment, size);
+}
+
+#endif // _WIN32

tslang/lib/TypeScript/CMakeLists.txt

@@ -36,6 +36,7 @@ add_mlir_dialect_library(MLIRTypeScript
     OwnershipSignaturePass.cpp
     OwnedReturnConsumptionPass.cpp
     GCPass.cpp
+    ProcessHeapPass.cpp
     AsyncTargetWidthPass.cpp
     ObjDumper.cpp
     DeclarationPrinter.cpp

tslang/lib/TypeScript/MLIRGenCast.cpp

@@ -3,6 +3,10 @@
 #include "MLIRGenImpl.h"
 #include "TypeScript/MLIRLogic/MLIRRTTIHelperVC.h"
 
+// the RTTI helper undefines it on the way out
+#undef DEBUG_TYPE
+#define DEBUG_TYPE "mlir"
+
 namespace typescript
 {
 namespace mlirgen

tslang/lib/TypeScript/MLIRGenImpl.h

@@ -2676,15 +2676,26 @@ class MLIRGenImpl
 #endif
         // add decorations, "noinline, optnone"
 
+        // An attribute list may name each attribute once (a DictionaryAttr asserts it). A
+        // declaration can say the same thing twice - `@dllname("f") @linkname("f")` both become
+        // DLL_NAME, and checkLinkNameDecorators has made sure they agree - so the first stands.
+        auto addAttr = [&](StringRef name, mlir::Attribute value) {
+            auto present = llvm::any_of(attrs, [&](mlir::NamedAttribute &attr) { return attr.getName() == name; });
+            if (!present)
+            {
+                attrs.push_back({mlir::StringAttr::get(builder.getContext(), name), value});
+            }
+        };
+
         iterateDecorators(functionLikeDeclarationBaseAST, genContext, [&](StringRef name, SmallVector<StringRef> args) {
             if (isFuncAttr(name))
             {
-                attrs.push_back({mlir::StringAttr::get(builder.getContext(), name), mlir::UnitAttr::get(builder.getContext())});
+                addAttr(name, mlir::UnitAttr::get(builder.getContext()));
             }
 
             if (name == "varargs") 
             {
-                attrs.push_back({mlir::StringAttr::get(builder.getContext(), "func.varargs"), mlir::BoolAttr::get(builder.getContext(), true)});
+                addAttr("func.varargs", mlir::BoolAttr::get(builder.getContext(), true));
             }
 
             if (name == "used") {
@@ -2693,7 +2704,7 @@ class MLIRGenImpl
 
             if (name == DLL_NAME && args.size() > 0)
             {
-                attrs.push_back({mlir::StringAttr::get(builder.getContext(), DLL_NAME), mlir::StringAttr::get(builder.getContext(), args.front())});
+                addAttr(DLL_NAME, mlir::StringAttr::get(builder.getContext(), args.front()));
             }
         });
 

tslang/lib/TypeScript/ProcessHeapPass.cpp

@@ -0,0 +1,186 @@
+#include "mlir/Pass/Pass.h"
+
+#include "TypeScript/TypeScriptDialect.h"
+#include "TypeScript/TypeScriptOps.h"
+#include "TypeScript/Passes.h"
+#include "TypeScript/Pass/ModulePass.h"
+#include "TypeScript/Defines.h"
+
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "pass"
+
+using namespace ::typescript;
+namespace mlir_ts = mlir::typescript;
+
+namespace
+{
+
+// On Windows, under every memory model but gc: the module's malloc, calloc, realloc, free and
+// aligned_alloc become the process-heap allocator of TypeScript/ProcessHeap.h. The C runtime's are
+// not one allocator across modules - each links its own copy of the static CRT, a debug CRT keeps
+// its blocks' list per copy, and with the prebuilt LLVM tslang.exe's `malloc` is rpmalloc - while a
+// block made in one module is routinely freed in another (a library's object destroyed by the
+// program, a coroutine frame from the runtime). gc has GCPass, which renames the same set to the
+// collector's.
+//
+// Renamed, not rewritten: the helpers take what the C functions take. The names are not ones LLVM
+// knows, so the allocating ones are marked as allocators, as GCPass marks GC_malloc - without that,
+// two calls with equal arguments and nothing between them merge into one block.
+class ProcessHeapPass : public mlir::PassWrapper<ProcessHeapPass, ModulePass>
+{
+  public:
+    MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(ProcessHeapPass)
+
+    void runOnModule() override
+    {
+        auto m = getModule();
+
+        llvm::SmallVector<LLVM::LLVMFuncOp> declarations;
+        m.walk([&](mlir::Operation *op) {
+            if (auto funcOp = dyn_cast<LLVM::LLVMFuncOp>(op))
+            {
+                if (funcOp.getBody().empty() && !mapName(funcOp.getSymName()).empty())
+                {
+                    declarations.push_back(funcOp);
+                }
+
+                return;
+            }
+
+            if (auto callOp = dyn_cast<LLVM::CallOp>(op))
+            {
+                if (auto callee = callOp.getCallee())
+                {
+                    if (auto newName = mapName(*callee); !newName.empty())
+                    {
+                        callOp.setCalleeAttr(mlir::FlatSymbolRefAttr::get(op->getContext(), newName));
+                    }
+                }
+
+                return;
+            }
+
+            // a function's address taken - a destructor slot, say
+            if (auto addressOfOp = dyn_cast<LLVM::AddressOfOp>(op))
+            {
+                if (auto newName = mapName(addressOfOp.getGlobalName()); !newName.empty())
+                {
+                    addressOfOp.setGlobalNameAttr(mlir::FlatSymbolRefAttr::get(op->getContext(), newName));
+                }
+            }
+        });
+
+        for (auto funcOp : declarations)
+        {
+            auto newName = mapName(funcOp.getSymName());
+
+            // `free` and `aligned_free` both become the one release; the first declaration stays
+            if (auto existing = m.lookupSymbol<LLVM::LLVMFuncOp>(newName); existing && existing != funcOp)
+            {
+                funcOp.erase();
+                continue;
+            }
+
+            funcOp.setSymName(newName);
+            markAsAllocator(newName, funcOp);
+        }
+    }
+
+  private:
+    static llvm::StringRef mapName(llvm::StringRef name)
+    {
+        return llvm::StringSwitch<llvm::StringRef>(name)
+            .Case("malloc", PROCESS_HEAP_MALLOC)
+            .Case("calloc", PROCESS_HEAP_CALLOC)
+            .Case("realloc", PROCESS_HEAP_REALLOC)
+            .Case("free", PROCESS_HEAP_FREE)
+            .Case("aligned_alloc", PROCESS_HEAP_ALIGNED_ALLOC)
+            .Case("aligned_free", PROCESS_HEAP_FREE)
+            .Default("");
+    }
+
+    // What LLVM knows of malloc, calloc, realloc and free by name, said of the helpers: they are
+    // one allocator family, each allocation is a block of its own, the release takes the block
+    // back, and the allocator's own state is memory nobody else reads. It is not only that two equal
+    // allocations must not merge: optimization has to keep treating the program as it treated the C
+    // functions - an allocation that is only released, say, is deleted together with its releases.
+    static void markAsAllocator(llvm::StringRef name, LLVM::LLVMFuncOp funcOp)
+    {
+        // AllocFnKind: Alloc = 1 << 0, Realloc = 1 << 1, Free = 1 << 2, Uninitialized = 1 << 3,
+        // Zeroed = 1 << 4
+        uint64_t allocKind;
+        auto takesBlock = false;
+        if (name == PROCESS_HEAP_MALLOC || name == PROCESS_HEAP_ALIGNED_ALLOC)
+        {
+            allocKind = 1 | (1 << 3);
+        }
+        else if (name == PROCESS_HEAP_CALLOC)
+        {
+            allocKind = 1 | (1 << 4);
+        }
+        else if (name == PROCESS_HEAP_REALLOC)
+        {
+            allocKind = (1 << 1) | (1 << 3);
+            takesBlock = true;
+        }
+        else if (name == PROCESS_HEAP_FREE)
+        {
+            allocKind = 1 << 2;
+            takesBlock = true;
+        }
+        else
+        {
+            return;
+        }
+
+        auto *context = funcOp->getContext();
+
+        // an allocation writes only the block it makes (other memory, as GCPass says it of
+        // GC_malloc) and the allocator's state; a release or a resize reads and writes the block it
+        // is given as well
+        auto memoryEffects =
+            takesBlock ? LLVM::MemoryEffectsAttr::get(context, LLVM::ModRefInfo::NoModRef, LLVM::ModRefInfo::ModRef,
+                                                      LLVM::ModRefInfo::ModRef, LLVM::ModRefInfo::NoModRef,
+                                                      LLVM::ModRefInfo::NoModRef, LLVM::ModRefInfo::NoModRef)
+                       : LLVM::MemoryEffectsAttr::get(context, LLVM::ModRefInfo::Mod, LLVM::ModRefInfo::NoModRef,
+                                                      LLVM::ModRefInfo::ModRef, LLVM::ModRefInfo::NoModRef,
+                                                      LLVM::ModRefInfo::NoModRef, LLVM::ModRefInfo::NoModRef);
+        funcOp.setMemoryEffectsAttr(memoryEffects);
+
+        if (takesBlock && funcOp.getNumArguments() > 0)
+        {
+            funcOp.setArgAttr(0, LLVM::LLVMDialect::getAllocatedPointerAttrName(), mlir::UnitAttr::get(context));
+        }
+
+        auto entry = [&](llvm::StringRef key, llvm::StringRef value) {
+            return mlir::ArrayAttr::get(context, {mlir::StringAttr::get(context, key), mlir::StringAttr::get(context, value)});
+        };
+
+        llvm::SmallVector<mlir::Attribute> passthrough;
+        if (auto existing = funcOp.getPassthroughAttr())
+        {
+            passthrough.append(existing.begin(), existing.end());
+        }
+
+        for (auto added : {entry("allockind", std::to_string(allocKind)), entry("alloc-family", "malloc")})
+        {
+            if (!llvm::is_contained(passthrough, added))
+            {
+                passthrough.push_back(added);
+            }
+        }
+
+        funcOp.setPassthroughAttr(mlir::ArrayAttr::get(context, passthrough));
+    }
+};
+} // end anonymous namespace
+
+#undef DEBUG_TYPE
+
+std::unique_ptr<mlir::Pass> mlir_ts::createProcessHeapPass()
+{
+    return std::make_unique<ProcessHeapPass>();
+}