Enable productions to provide the number of bytes they parse, if known.

For productions that parse a static amount of data, that size can now
be retrieved. This includes cases where the number is determined
through some expression evaluated at runtime, as long as parsing the
production is guaranteed to consume always exactly as many bytes as
the expression yields.

hilti/toolchain/include/ast/builder/builder.h

@@ -430,6 +430,9 @@ class Builder : public builder::NodeFactory {
     auto division(Expression* op1, Expression* op2, const Meta& m = Meta()) {
         return expressionUnresolvedOperator(operator_::Kind::Division, {op1, op2}, m);
     }
+    auto multiple(Expression* op1, Expression* op2, const Meta& m = Meta()) {
+        return expressionUnresolvedOperator(operator_::Kind::Multiple, {op1, op2}, m);
+    }
 
     // Other expressions
 

spicy/toolchain/CMakeLists.txt

@@ -34,6 +34,7 @@ set(SOURCES_COMPILER
     src/compiler/codegen/parsers/literals.cc
     src/compiler/codegen/parsers/types.cc
     src/compiler/codegen/production.cc
+    src/compiler/codegen/productions/ctor.cc
     src/compiler/codegen/productions/look-ahead.cc
     src/compiler/codegen/productions/switch.cc
     src/compiler/codegen/productions/while.cc

spicy/toolchain/include/ast/types/unit-items/field.h

@@ -78,12 +78,6 @@ class Field : public unit::Item {
     /** Get the `&convert` expression, if any. */
     std::optional<std::pair<Expression*, QualifiedType*>> convertExpression() const;
 
-    /**
-     * Returns an expression representing the number of bytes the fields
-     * consumes, if known.
-     */
-    Expression* size(ASTContext* ctx) const;
-
     void setForwarding(bool is_forwarding) { _is_forwarding = is_forwarding; }
     void setTransient(bool is_transient) { _is_transient = is_transient; }
     void setDDType(ASTContext* ctx, QualifiedType* t);
@@ -94,6 +88,8 @@ class Field : public unit::Item {
 
     QualifiedType* itemType() const final { return child<QualifiedType>(3); }
 
+    Expression* parseSize(Builder* builder) const;
+
     bool isResolved(hilti::node::CycleDetector* cd) const final {
         return type() || item() || itemType()->isResolved(cd);
     }

spicy/toolchain/include/compiler/detail/codegen/production.h

@@ -179,6 +179,20 @@ class Production {
     /** Returns any type associated with this production. */
     virtual QualifiedType* type() const { return nullptr; };
 
+    /**
+     * Returns an expression yielding the raw number of bytes that parsing this
+     * production will consume. The expression must evaluate to a constant
+     * value of type `uint64`. Returns null if the size isn't constant, or
+     * cannot be computed.
+     *
+     * This does not take any field parsing attributes into account (e.g.,
+     * `&size`). It's up to the caller to apply those if needed.
+     *
+     * @param builder builder to use for creating the expression
+     * @return expression yielding the size, or null if not available
+     */
+    virtual Expression* parseSize(Builder* builder) const = 0;
+
     /**
      * Returns a ID for this literal that's guaranteed to be globally unique
      * for the literal's value, including across grammars. Returns a negative

spicy/toolchain/include/compiler/detail/codegen/productions/block.h

@@ -2,14 +2,14 @@
 
 #pragma once
 
-#include <algorithm>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
 #include <hilti/base/util.h>
 
+#include <spicy/ast/builder/builder.h>
 #include <spicy/ast/types/unit.h>
 #include <spicy/compiler/detail/codegen/production.h>
 #include <spicy/compiler/detail/codegen/productions/visitor.h>
@@ -54,6 +54,27 @@ class Block : public Production {
         return rhss;
     }
 
+    Expression* parseSize(Builder* builder) const final {
+        if ( _condition || ! _else_prods.empty() )
+            return nullptr;
+
+        // TODO: What about attributes()?
+
+        Expression* size = nullptr;
+        for ( const auto& p : _prods ) {
+            auto psize = p->parseSize(builder);
+            if ( ! psize )
+                return nullptr;
+
+            if ( ! size )
+                size = psize;
+            else
+                size = builder->sum(size, psize);
+        }
+
+        return size;
+    }
+
     std::string dump() const final {
         auto true_ = hilti::util::join(hilti::util::transform(_prods, [](const auto& p) { return p->symbol(); }), " ");
         auto false_ =

spicy/toolchain/include/compiler/detail/codegen/productions/counter.h

@@ -7,6 +7,7 @@
 #include <utility>
 #include <vector>
 
+#include <spicy/ast/builder/builder.h>
 #include <spicy/ast/types/unit.h>
 #include <spicy/compiler/detail/codegen/production.h>
 #include <spicy/compiler/detail/codegen/productions/visitor.h>
@@ -33,6 +34,17 @@ class Counter : public Production {
     Expression* expression() const final { return _expression; }
 
     std::vector<std::vector<Production*>> rhss() const final { return {{_body.get()}}; };
+    Expression* parseSize(Builder* builder) const final {
+        if ( ! _expression->isConstant() )
+            return nullptr;
+
+        auto size = _body->parseSize(builder);
+        if ( ! size )
+            return nullptr;
+
+        return builder->multiple(_expression, size);
+    }
+
     std::string dump() const override { return hilti::util::fmt("counter(%s): %s", *_expression, _body->symbol()); }
 
     SPICY_PRODUCTION

spicy/toolchain/include/compiler/detail/codegen/productions/ctor.h

@@ -2,9 +2,7 @@
 
 #pragma once
 
-#include <memory>
 #include <string>
-#include <utility>
 
 #include <hilti/ast/expressions/ctor.h>
 
@@ -29,9 +27,9 @@ class Ctor : public Production {
     bool isNullable() const final { return false; };
     bool isTerminal() const final { return true; };
 
-    // std::vector<std::vector<Production*>> rhss() const final { return {}; };
     Expression* expression() const final { return _ctor; }
     QualifiedType* type() const final { return _ctor->type(); };
+    Expression* parseSize(Builder* builder) const final;
 
     int64_t tokenID() const final {
         return static_cast<int64_t>(Production::tokenID(hilti::util::fmt("%s|%s", *_ctor, *_ctor->type())));

spicy/toolchain/include/compiler/detail/codegen/productions/deferred.h

@@ -2,7 +2,6 @@
 
 #pragma once
 
-#include <memory>
 #include <string>
 
 #include <spicy/compiler/detail/codegen/production.h>
@@ -35,6 +34,7 @@ class Deferred : public Production {
     bool isLiteral() const final { return false; }
     bool isNullable() const final { return false; }
     bool isTerminal() const final { return false; }
+    Expression* parseSize(Builder* builder) const final { return nullptr; }
     int64_t tokenID() const final { return _resolved ? _resolved->tokenID() : -1; };
 
     std::string dump() const final {

spicy/toolchain/include/compiler/detail/codegen/productions/enclosure.h

@@ -9,6 +9,7 @@
 
 #include <spicy/ast/types/unit.h>
 #include <spicy/compiler/detail/codegen/production.h>
+#include <spicy/compiler/detail/codegen/productions/visitor.h>
 
 namespace spicy::detail::codegen::production {
 
@@ -33,6 +34,7 @@ class Enclosure : public Production {
 
     std::vector<std::vector<Production*>> rhss() const final { return {{_child.get()}}; };
     QualifiedType* type() const final { return _child->type(); };
+    Expression* parseSize(Builder* builder) const final { return _child->parseSize(builder); }
 
     std::string dump() const override { return _child->symbol(); }
 

spicy/toolchain/include/compiler/detail/codegen/productions/epsilon.h

@@ -2,10 +2,13 @@
 
 #pragma once
 
-#include <memory>
 #include <string>
 #include <utility>
 
+#include <hilti/ast/ctors/integer.h>
+#include <hilti/ast/expressions/ctor.h>
+
+#include <spicy/ast/builder/builder.h>
 #include <spicy/compiler/detail/codegen/production.h>
 #include <spicy/compiler/detail/codegen/productions/visitor.h>
 
@@ -22,6 +25,8 @@ class Epsilon : public Production {
     bool isNullable() const final { return true; };
     bool isTerminal() const final { return true; };
 
+    Expression* parseSize(Builder* builder) const final { return builder->integer(0U); }
+
     std::string dump() const final { return "()"; }
 
     SPICY_PRODUCTION

spicy/toolchain/include/compiler/detail/codegen/productions/for-each.h

@@ -30,6 +30,8 @@ class ForEach : public Production {
 
     std::vector<std::vector<Production*>> rhss() const final { return {{_body.get()}}; };
 
+    Expression* parseSize(Builder* builder) const final { return nullptr; }
+
     std::string dump() const override { return hilti::util::fmt("foreach: %s", _body->symbol()); }
 
     SPICY_PRODUCTION

spicy/toolchain/include/compiler/detail/codegen/productions/look-ahead.h

@@ -55,6 +55,8 @@ class LookAhead : public Production {
         return {{_alternatives.first.get()}, {_alternatives.second.get()}};
     }
 
+    Expression* parseSize(Builder* builder) const final { return nullptr; }
+
     std::string dump() const final;
 
     /**

spicy/toolchain/include/compiler/detail/codegen/productions/reference.h

@@ -2,9 +2,7 @@
 
 #pragma once
 
-#include <memory>
 #include <string>
-#include <utility>
 #include <vector>
 
 #include <spicy/compiler/detail/codegen/production.h>
@@ -38,6 +36,7 @@ class Reference : public Production {
     std::vector<std::vector<Production*>> rhss() const final { return _production->rhss(); }
     Expression* expression() const final { return _production->expression(); }
     QualifiedType* type() const final { return _production->type(); }
+    Expression* parseSize(Builder* builder) const final { return _production->parseSize(builder); }
     int64_t tokenID() const final { return _production->tokenID(); };
 
     std::string dump() const final { return hilti::util::fmt("ref(%s)", _production->dump()); }

spicy/toolchain/include/compiler/detail/codegen/productions/sequence.h

@@ -2,14 +2,14 @@
 
 #pragma once
 
-#include <algorithm>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
 #include <hilti/base/util.h>
 
+#include <spicy/ast/builder/builder.h>
 #include <spicy/ast/types/unit.h>
 #include <spicy/compiler/detail/codegen/production.h>
 #include <spicy/compiler/detail/codegen/productions/visitor.h>
@@ -36,6 +36,22 @@ class Sequence : public Production {
         return {hilti::util::transform(_prods, [](const auto& p) { return p.get(); })};
     }
 
+    Expression* parseSize(Builder* builder) const final {
+        Expression* size = nullptr;
+        for ( const auto& p : _prods ) {
+            auto psize = p->parseSize(builder);
+            if ( ! psize )
+                return nullptr;
+
+            if ( ! size )
+                size = psize;
+            else
+                size = builder->sum(size, psize);
+        }
+
+        return size;
+    }
+
     std::string dump() const final {
         return hilti::util::join(hilti::util::transform(_prods, [](const auto& p) { return p->symbol(); }), " ");
     }

spicy/toolchain/include/compiler/detail/codegen/productions/skip.h

@@ -52,6 +52,15 @@ class Skip : public Production {
 
     QualifiedType* type() const final { return _void; };
 
+    Expression* parseSize(Builder* builder) const final {
+        if ( _ctor )
+            return _ctor->parseSize(builder);
+        else {
+            assert(_field);
+            return _field->parseSize(builder);
+        }
+    }
+
     std::string dump() const override {
         return hilti::util::fmt("skip: %s", _ctor ? to_string(*_ctor) : _field->print());
     }

spicy/toolchain/include/compiler/detail/codegen/productions/switch.h

@@ -47,6 +47,7 @@ class Switch : public Production {
 
     Expression* expression() const final { return _expression; }
     std::vector<std::vector<Production*>> rhss() const final;
+    Expression* parseSize(Builder* builder) const final { return nullptr; }
 
     std::string dump() const final;
 

spicy/toolchain/include/compiler/detail/codegen/productions/type-literal.h

@@ -2,9 +2,7 @@
 
 #pragma once
 
-#include <memory>
 #include <string>
-#include <utility>
 
 #include <hilti/ast/expressions/type.h>
 
@@ -31,6 +29,8 @@ class TypeLiteral : public Production {
 
     Expression* expression() const final { return _expr; }
     QualifiedType* type() const final { return _type; };
+    Expression* parseSize(Builder* builder) const final { return nullptr; }
+
     int64_t tokenID() const final { return static_cast<int64_t>(Production::tokenID(_type->print())); }
 
     std::string dump() const final { return _type->print(); }

spicy/toolchain/include/compiler/detail/codegen/productions/unit.h

@@ -2,12 +2,12 @@
 
 #pragma once
 
-#include <algorithm>
 #include <memory>
 #include <string>
 #include <utility>
 #include <vector>
 
+#include <spicy/ast/builder/builder.h>
 #include <spicy/ast/types/unit.h>
 #include <spicy/compiler/detail/codegen/production.h>
 #include <spicy/compiler/detail/codegen/productions/visitor.h>
@@ -46,6 +46,22 @@ class Unit : public Production {
 
     QualifiedType* type() const final { return _type; };
 
+    Expression* parseSize(Builder* builder) const final {
+        Expression* size = nullptr;
+        for ( const auto& f : _fields ) {
+            auto psize = f->parseSize(builder);
+            if ( ! psize )
+                return nullptr;
+
+            if ( ! size )
+                size = psize;
+            else
+                size = builder->sum(size, psize);
+        }
+
+        return size;
+    }
+
     std::string dump() const final {
         return hilti::util::join(hilti::util::transform(_fields, [](const auto& p) { return p->symbol(); }), " ");
     }

spicy/toolchain/include/compiler/detail/codegen/productions/variable.h

@@ -2,9 +2,10 @@
 
 #pragma once
 
-#include <memory>
 #include <string>
-#include <utility>
+
+#include <hilti/ast/ctors/integer.h>
+#include <hilti/ast/expressions/ctor.h>
 
 #include <spicy/compiler/detail/codegen/production.h>
 #include <spicy/compiler/detail/codegen/productions/visitor.h>
@@ -30,6 +31,13 @@ class Variable : public Production {
 
     QualifiedType* type() const final { return _type; };
 
+    Expression* parseSize(Builder* builder) const final {
+        if ( auto* field = meta().field() )
+            return field->parseSize(builder);
+        else
+            return nullptr;
+    }
+
     std::string dump() const final { return hilti::util::fmt("%s", *_type); }
 
     SPICY_PRODUCTION

spicy/toolchain/include/compiler/detail/codegen/productions/while.h

@@ -72,6 +72,8 @@ class While : public Production {
     /** Returns the loop expression if passed into the corresponding constructor. */
     Expression* expression() const final { return _expression; }
 
+    Expression* parseSize(Builder* builder) const final { return nullptr; }
+
     std::string dump() const final;
 
     SPICY_PRODUCTION