A type alias that contains itself is an error, not a stack overflow

tslang/lib/TypeScript/MLIRGenCast.cpp

@@ -3,6 +3,10 @@
 #include "MLIRGenImpl.h"
 #include "TypeScript/MLIRLogic/MLIRRTTIHelperVC.h"
 
+// the RTTI helper undefines it on the way out
+#undef DEBUG_TYPE
+#define DEBUG_TYPE "mlir"
+
 namespace typescript
 {
 namespace mlirgen

tslang/lib/TypeScript/MLIRGenExpressions.cpp

@@ -1720,7 +1720,7 @@ namespace mlirgen
 
         // TODO: error, when we use  function_name(index: index) and index value is not provided in call function_name(index), index will be mistakenly tearted
         // as embeded type "index"
-        if (!isEmbededType(name))
+        if (!isEmbededType(name) && !circularTypeAliases.contains(name))
             emitError(location, "can't resolve name: ") << name;
 
         return mlir::failure();

tslang/lib/TypeScript/MLIRGenImpl.h

@@ -9666,6 +9666,8 @@ class MLIRGenImpl
 
     mlir::Value resolveFunctionNameInNamespace(mlir::Location location, StringRef name, const GenContext &genContext);
 
+    mlir::Type expandTypeAlias(mlir::Location location, StringRef name, llvm::function_ref<mlir::Type()> expand);
+
     mlir::Type resolveTypeByNameInNamespace(mlir::Location location, StringRef name, const GenContext &genContext);
 
     mlir::Type resolveTypeByName(mlir::Location location, StringRef name, const GenContext &genContext);
@@ -9941,11 +9943,13 @@ class MLIRGenImpl
             {
                 if (hasExportModifier)
                 {
-                    GenContext typeAliasGenContext(genContext);
-                    auto type = getType(typeAliasDeclarationAST->type, typeAliasGenContext);
+                    // registered unresolved first, so a mention of itself inside is found - and
+                    // found to be circular - as for an alias that is not exported
+                    getTypeAliasMap()[namePtr] = { mlir::Type(), typeAliasDeclarationAST->type };
+                    auto type = resolveTypeByNameInNamespace(loc(typeAliasDeclarationAST), namePtr, genContext);
                     if (type)
                     {
-                        getTypeAliasMap().insert({ namePtr, { type, undefined } });
+                        getTypeAliasMap()[namePtr] = { type, undefined };
                         addTypeDeclarationToExport(namePtr, currentNamespace, type);
                     }
                 }
@@ -12917,6 +12921,17 @@ class MLIRGenImpl
     // imports a) stops instead of recursing until the stack overflows.
     llvm::StringSet<> filesInProgress;
 
+    // Type aliases whose declaration is being turned into a type, by full name: one that meets
+    // itself on the way is circular (resolveTypeByNameInNamespace).
+    llvm::StringSet<> typeAliasesInProgress;
+    // Aliases reported circular, by name: that one error says it, and "can't find type" for each
+    // mention after it would only repeat it.
+    llvm::StringSet<> circularTypeAliases;
+    // The alias whose circularity was just reported, until the outermost expansion of it has
+    // unwound: every error on the way out follows from the one reported.
+    std::string circularTypeAliasUnwinding;
+    bool reportingCircularTypeAlias = false;
+
     // the declaration (no body) each declared function's full name came from, and the module it was
     // generated into - discovery's is thrown away, and a library's __decls is parsed again for the
     // next pass; see mlirGen(FunctionDeclaration)

tslang/lib/TypeScript/MLIRGenTypes.cpp

@@ -644,7 +644,7 @@ namespace mlirgen
                 return mlir::Type();
             }
 
-            return getType(typeNode, genericTypeGenContext);
+            return expandTypeAlias(location, name, [&]() { return getType(typeNode, genericTypeGenContext); });
         }  
 
         return mlir::Type();      
@@ -675,7 +675,7 @@ namespace mlirgen
                 return createTypeReferenceType(typeReferenceAST, genericTypeGenContext);
             }
 
-            return getType(typeNode, genericTypeGenContext);
+            return expandTypeAlias(location, name, [&]() { return getType(typeNode, genericTypeGenContext); });
         }
 
         if (auto genericClassTypeInfo = lookupGenericClassesMap(name))
@@ -789,7 +789,22 @@ namespace mlirgen
                     return embedType;
                 }
 
-                emitError(location, "generic type ") << name << " can't be found";
+                // `Reference<Node>` with `Node` already reported circular, or `List<T>` itself:
+                // not a missing generic
+                auto argumentIsCircular = circularTypeAliases.contains(name) || llvm::any_of(typeReferenceAST->typeArguments, [&](TypeNode typeArg) {
+                    if (typeArg != SyntaxKind::TypeReference)
+                    {
+                        return false;
+                    }
+
+                    auto argName = MLIRHelper::getName(typeArg.as<TypeReferenceNode>()->typeName);
+                    return circularTypeAliases.contains(argName);
+                });
+                if (!argumentIsCircular)
+                {
+                    emitError(location, "generic type ") << name << " can't be found";
+                }
+
                 return mlir::Type();
             }
         }

tslang/lib/TypeScript/MLIRGenVariables.cpp

@@ -1138,6 +1138,46 @@ namespace mlirgen
         return mlir::Value();
     }
 
+    // `type Node = { value: number, next: Node }`: the declaration names itself, and each mention
+    // expanded it again until the stack ran out. A type here has a fixed shape - a tuple holds its
+    // fields in place - so no alias can contain itself, not even behind `Reference<T>`, whose tuple
+    // type would have to name itself; an interface or a class is the recursive type. A generic
+    // alias is the same: `List<T>` inside `List<T>` expands without end, whatever its arguments.
+    //
+    // One error says it. Every error on the way back out of the expansion follows from it, and so
+    // does "can't find type" at each later mention of the alias; those are left out.
+    mlir::Type MLIRGenImpl::expandTypeAlias(mlir::Location location, StringRef name, llvm::function_ref<mlir::Type()> expand)
+    {
+        auto fullName = getFullNamespaceName(name).str();
+        if (!typeAliasesInProgress.insert(fullName).second)
+        {
+            reportingCircularTypeAlias = true;
+            emitError(location) << "type alias '" << name
+                                << "' circularly references itself; use an interface or a class for a recursive type";
+            reportingCircularTypeAlias = false;
+            circularTypeAliases.insert(name);
+            circularTypeAliasUnwinding = fullName;
+            return mlir::Type();
+        }
+
+        mlir::Type type;
+        {
+            mlir::ScopedDiagnosticHandler followOn(builder.getContext(), [&](mlir::Diagnostic &) {
+                return mlir::success(!circularTypeAliasUnwinding.empty() && !reportingCircularTypeAlias);
+            });
+
+            type = expand();
+        }
+
+        typeAliasesInProgress.erase(fullName);
+        if (circularTypeAliasUnwinding == fullName)
+        {
+            circularTypeAliasUnwinding.clear();
+        }
+
+        return type;
+    }
+
     mlir::Type MLIRGenImpl::resolveTypeByNameInNamespace(mlir::Location location, StringRef name, const GenContext &genContext)
     {
         // support generic types
@@ -1166,14 +1206,10 @@ namespace mlirgen
             }
 
             assert(typeAliasInfo.second);
-            GenContext typeAliasGenContext(genContext);
-            auto type = getType(typeAliasInfo.second, typeAliasGenContext);
-            if (!type)
-            {
-                typeAliasInfo.first = type;
-            }
-
-            return type;
+            return expandTypeAlias(location, name, [&]() {
+                GenContext typeAliasGenContext(genContext);
+                return getType(typeAliasInfo.second, typeAliasGenContext);
+            });
         }
 
         if (getClassesMap().count(name))
@@ -1280,7 +1316,7 @@ namespace mlirgen
             }
         }    
 
-        if (!isEmbededType(name))
+        if (!isEmbededType(name) && !circularTypeAliases.contains(name))
             emitError(location, "can't find type by name: ") << name;
 
         return mlir::Type();    

tslang/test/tester/CMakeLists.txt

@@ -2354,6 +2354,22 @@ foreach(call_arity_case ${call_arity_cases})
                                     FAIL_REGULAR_EXPRESSION "Stack dump|Assertion failed")
 endforeach()
 
+# A type alias that contains itself is one error, not a stack overflow (issue #370). The follow-on
+# messages ("can't find type", "generic type ... can't be found") would only repeat it. A stack
+# overflow prints nothing, so it fails the PASS expression.
+foreach(type_alias_circular_name tuple_reference object object_optional declared_param mutual exported
+                                 generic union_array namespaced)
+    add_test(NAME test-compile-type-alias-circular-${type_alias_circular_name}
+             COMMAND $<TARGET_FILE:tslang> --emit=obj --no-default-lib -mm=none
+                     "${PROJECT_SOURCE_DIR}/test/tester/type-alias-circular/${type_alias_circular_name}.ts"
+                     -o "${CMAKE_CURRENT_BINARY_DIR}/type-alias-circular-${type_alias_circular_name}.obj")
+    set_tests_properties(test-compile-type-alias-circular-${type_alias_circular_name}
+                         PROPERTIES PASS_REGULAR_EXPRESSION "type alias '[A-Za-z]+' circularly references itself"
+                                    FAIL_REGULAR_EXPRESSION "can't find type|can't be found|can't resolve name|Stack dump|Assertion failed")
+endforeach()
+tslang_add_test(NAME test-jit-type-alias-not-circular COMMAND test-runner -jit "${PROJECT_SOURCE_DIR}/test/tester/type-alias-circular/positive.ts")
+tslang_add_test(NAME test-compile-type-alias-not-circular COMMAND test-runner "${PROJECT_SOURCE_DIR}/test/tester/type-alias-circular/positive.ts")
+
 # An error reported by a pass that still returns success - a lowering pattern that cannot build a
 # value, says so and carries on - fails the compile. It used to be printed and then ignored: the object
 # file was written, and under --emit=jit the program ran and crashed. Two tests because one

tslang/test/tester/type-alias-circular/declared_param.ts

@@ -0,0 +1,8 @@
+// A type alias that contains itself is an error, not a stack overflow (issue #370).
+type Node = [value: s32, next: Reference<Node>];
+
+declare function take(n: Reference<Node>): void;
+
+function main() {
+    print("ok");
+}

tslang/test/tester/type-alias-circular/exported.ts

@@ -0,0 +1,7 @@
+// A type alias that contains itself is an error, not a stack overflow (issue #370).
+export type Node = { value: number, next: Node };
+
+function main() {
+    let n: Node;
+    print("ok");
+}

tslang/test/tester/type-alias-circular/generic.ts

@@ -0,0 +1,7 @@
+// A type alias that contains itself is an error, not a stack overflow (issue #370).
+type List<T> = { v: T, next: List<T> };
+
+function main() {
+    let n: List<number>;
+    print("ok");
+}

tslang/test/tester/type-alias-circular/mutual.ts

@@ -0,0 +1,8 @@
+// A type alias that contains itself is an error, not a stack overflow (issue #370).
+type A = { v: number, b: B };
+type B = { w: number, a: A };
+
+function main() {
+    let n: A;
+    print("ok");
+}

tslang/test/tester/type-alias-circular/namespaced.ts

@@ -0,0 +1,9 @@
+// A type alias that contains itself is an error, not a stack overflow (issue #370).
+namespace N {
+    export type Node = { v: number, next: Node };
+}
+
+function main() {
+    let n: N.Node;
+    print("ok");
+}

tslang/test/tester/type-alias-circular/object.ts

@@ -0,0 +1,7 @@
+// A type alias that contains itself is an error, not a stack overflow (issue #370).
+type Node = { value: number, next: Node };
+
+function main() {
+    let n: Node;
+    print("ok");
+}

tslang/test/tester/type-alias-circular/object_optional.ts

@@ -0,0 +1,7 @@
+// A type alias that contains itself is an error, not a stack overflow (issue #370).
+type Node = { value: number, next?: Node };
+
+function main() {
+    const n: Node = { value: 1 };
+    print(n.value);
+}

tslang/test/tester/type-alias-circular/positive.ts

@@ -0,0 +1,26 @@
+// Aliases that mention another alias - or the same one, one after another rather than inside
+// itself - are not circular (issue #370 made a type alias meeting itself an error).
+type P = [x: number, y: number];
+type Seg = [from: P, to: P];
+type Pair<T> = [first: T, second: T];
+type Box<T> = { v: T };
+type Nested = Pair<Pair<number>>;
+type Boxes<T> = Box<Box<T>>;
+
+interface Node {
+    value: number;
+    next?: Node;
+}
+
+function main() {
+    const s: Seg = [[1, 2], [3, 4]];
+    const n: Nested = [[1, 2], [3, 4]];
+    const b: Boxes<number> = { v: { v: 7 } };
+    const tail: Node = { value: 2 };
+    const head: Node = { value: 1, next: tail };
+    assert(s[1][0] == 3);
+    assert(n[1][1] == 4);
+    assert(b.v.v == 7);
+    assert(head.next.value == 2);
+    print("done.");
+}

tslang/test/tester/type-alias-circular/tuple_reference.ts

@@ -0,0 +1,7 @@
+// A type alias that contains itself is an error, not a stack overflow (issue #370).
+type Node = [value: s32, next: Reference<Node>];
+
+function main() {
+    let n: Node;
+    print("ok");
+}

tslang/test/tester/type-alias-circular/union_array.ts

@@ -0,0 +1,7 @@
+// A type alias that contains itself is an error, not a stack overflow (issue #370).
+type Json = number | Json[];
+
+function main() {
+    const j: Json = 1;
+    print("ok");
+}