Add recursion limit to type resolution.

This is part of #1867 - there is a fair bit more work to do, but this
recursion limit gives a simple way to prevent overflows without a pretty
unecessary cycle detector.

Types within containers (like vectors) or references will continue to
segfault - that happens at type unification. Then, in order to allow
references, more will be necessary.

hilti/toolchain/include/ast/types/name.h

@@ -2,8 +2,6 @@
 
 #pragma once
 
-#include <memory>
-#include <string>
 #include <utility>
 
 #include <hilti/ast/ast-context.h>
@@ -19,13 +17,16 @@ class Name : public UnqualifiedType {
     bool isBuiltIn() const { return _builtin; }
 
     // resolves recursively
-    UnqualifiedType* resolvedType() const {
+    UnqualifiedType* resolvedType(size_t recursion_depth = 0) const {
         if ( ! _resolved_type_index )
             return nullptr;
 
+        if ( recursion_depth > 1000 )
+            return nullptr;
+
         auto t = context()->lookup(_resolved_type_index);
         if ( auto n = t->tryAs<type::Name>() )
-            return n->resolvedType();
+            return n->resolvedType(recursion_depth + 1);
         else
             return t;
     }

hilti/toolchain/src/compiler/resolver.cc

@@ -66,7 +66,10 @@ struct VisitorPass1 : visitor::MutatingPostOrder {
         }
 
         if ( n->resolvedTypeIndex() ) {
-            if ( auto resolved = n->resolvedType(); resolved->isOnHeap() ) {
+            auto resolved = n->resolvedType();
+            if ( ! resolved )
+                n->addError(util::fmt("type %s cannot be resolved by its name", n->id()));
+            else if ( resolved->isOnHeap() ) {
                 if ( auto qtype = n->parent()->tryAs<QualifiedType>() ) {
                     auto replace = false;
 

tests/Baseline/hilti.types.id.recursive/output

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+[error] <...>/recursive.hlt:6:15-6:20: type Direct cannot be resolved by its name
+[error] hiltic: aborting after errors

tests/hilti/types/id/recursive.hlt

@@ -0,0 +1,8 @@
+# @TEST-EXEC-FAIL: ${HILTIC} -p %INPUT > output 2>&1
+# @TEST-EXEC: btest-diff output
+
+module Test {
+
+type Direct = Direct;
+
+}