filterx: added support for "declared" variables

Signed-off-by: Balazs Scheidler <[email protected]>
bazsi · Apr 10, 2024 · 0edcde5 · 0edcde5
1 parent 966a022
commit 0edcde5
Show file tree

Hide file tree

Showing 8 changed files with 87 additions and 37 deletions.
diff --git a/lib/filterx/expr-variable.c b/lib/filterx/expr-variable.c
@@ -28,6 +28,7 @@
 typedef struct _FilterXVariableExpr
 {
   FilterXExpr super;
+  FilterXVariableType type;
   NVHandle handle;
 } FilterXVariableExpr;
 
@@ -41,15 +42,15 @@ _pull_variable_from_message(FilterXVariableExpr *self, FilterXEvalContext *conte
     return NULL;
 
   FilterXObject *msg_ref = filterx_message_value_new_borrowed(value, value_len, t);
-  filterx_scope_register_variable(context->scope, self->handle, FALSE, msg_ref);
+  filterx_scope_register_variable(context->scope, self->handle, msg_ref);
   return msg_ref;
 }
 
 /* NOTE: unset on a variable that only exists in the LogMessage, without making the message writable */
 static void
 _whiteout_variable(FilterXVariableExpr *self, FilterXEvalContext *context)
 {
-  filterx_scope_register_variable(context->scope, self->handle, FALSE, NULL);
+  filterx_scope_register_variable(context->scope, self->handle, NULL);
 }
 
 static FilterXObject *
@@ -63,7 +64,9 @@ _eval(FilterXExpr *s)
   if (variable)
     return filterx_variable_get_value(variable);
 
-  return _pull_variable_from_message(self, context, context->msgs[0]);
+  if (self->type == FX_VAR_MESSAGE)
+    return _pull_variable_from_message(self, context, context->msgs[0]);
+  return NULL;
 }
 
 static void
@@ -88,15 +91,15 @@ _assign(FilterXExpr *s, FilterXObject *new_value)
     {
       /* NOTE: we pass NULL as initial_value to make sure the new variable
        * is considered changed due to the assignment */
-      variable = filterx_scope_register_variable(scope, self->handle, FALSE, NULL);
+
+      variable = filterx_scope_register_variable(scope, self->handle, NULL);
+      if (self->type == FX_VAR_DECLARED)
+        filterx_variable_mark_declared(variable);
     }
 
   /* this only clones mutable objects */
   new_value = filterx_object_clone(new_value);
   filterx_variable_set_value(variable, new_value);
-
-
-
   filterx_object_unref(new_value);
   return TRUE;
 }
@@ -137,7 +140,7 @@ _unset(FilterXExpr *s)
 }
 
 FilterXExpr *
-filterx_variable_expr_new(const gchar *name)
+filterx_variable_expr_new(const gchar *name, FilterXVariableType type)
 {
   FilterXVariableExpr *self = g_new0(FilterXVariableExpr, 1);
 
@@ -147,6 +150,7 @@ filterx_variable_expr_new(const gchar *name)
   self->super.assign = _assign;
   self->super.isset = _isset;
   self->super.unset = _unset;
-  self->handle = log_msg_get_value_handle(name);
+  self->handle = filterx_scope_map_variable_to_handle(name, type);
+  self->type = type;
   return &self->super;
 }
diff --git a/lib/filterx/expr-variable.h b/lib/filterx/expr-variable.h
@@ -25,6 +25,6 @@
 
 #include "filterx/filterx-expr.h"
 
-FilterXExpr *filterx_variable_expr_new(const gchar *name);
+FilterXExpr *filterx_variable_expr_new(const gchar *name, FilterXVariableType type);
 
 #endif
diff --git a/lib/filterx/filterx-eval.c b/lib/filterx/filterx-eval.c
@@ -65,9 +65,8 @@ _evaluate_statement(FilterXExpr *expr)
   if (!success || trace_flag)
     {
       GString *buf = scratch_buffers_alloc();
-      LogMessageValueType t;
 
-      if (!filterx_object_marshal(res, buf, &t))
+      if (res && !filterx_object_repr(res, buf))
         {
           g_assert_not_reached();
         }
@@ -78,16 +77,14 @@ _evaluate_statement(FilterXExpr *expr)
                                  expr->lloc.name, expr->lloc.first_line, expr->lloc.first_column,
                                  expr->expr_text ? : "n/a"),
                   evt_tag_str("status", res == NULL ? "error" : "falsy"),
-                  evt_tag_str("value", buf->str),
-                  evt_tag_str("type", log_msg_value_type_to_str(t)));
+                  evt_tag_mem("value", buf->str, buf->len));
       else
         msg_trace("FILTERX",
                   evt_tag_printf("expr", "%s:%d:%d| %s",
                                  expr->lloc.name, expr->lloc.first_line, expr->lloc.first_column,
                                  expr->expr_text ? : "n/a"),
                   evt_tag_str("status", res == NULL ? "error" : (success ? "truthy" : "falsy")),
-                  evt_tag_str("value", buf->str),
-                  evt_tag_str("type", log_msg_value_type_to_str(t)),
+                  evt_tag_mem("value", buf->str, buf->len),
                   evt_tag_printf("result", "%p", res));
     }
 

diff --git a/lib/filterx/filterx-expr.h b/lib/filterx/filterx-expr.h
@@ -48,6 +48,8 @@ struct _FilterXExpr
   gboolean (*isset)(FilterXExpr *self);
   /* unset the expression */
   gboolean (*unset)(FilterXExpr *self);
+  /* declare a new value be this expr */
+  gboolean (*declare)(FilterXExpr *self, FilterXObject *new_value);
 
   void (*free_fn)(FilterXExpr *self);
   CFG_LTYPE lloc;
@@ -108,7 +110,6 @@ filterx_expr_eval_typed(FilterXExpr *self)
   return unmarshalled;
 }
 
-
 static inline gboolean
 filterx_expr_assign(FilterXExpr *self, FilterXObject *new_value)
 {
@@ -130,6 +131,14 @@ filterx_expr_unset(FilterXExpr *self)
 {
   if (self->unset)
     return self->unset(self);
+  return TRUE;
+}
+
+static inline gboolean
+filterx_expr_declare(FilterXExpr *self, FilterXObject *new_value)
+{
+  if (self->declare)
+    return self->declare(self, new_value);
   return FALSE;
 }
 

diff --git a/lib/filterx/filterx-grammar.ym b/lib/filterx/filterx-grammar.ym
@@ -67,7 +67,8 @@ construct_template_expr(LogTemplate *template)
 			filterx_string_new(log_template_get_literal_value(template, NULL), -1)));
   else if (log_template_is_trivial(template))
     result = filterx_variable_expr_new(
-                log_msg_get_value_name(log_template_get_trivial_value_handle(template)));
+                log_msg_get_value_name(log_template_get_trivial_value_handle(template), NULL),
+                FX_VAR_MESSAGE);
   else
     result = filterx_template_new(log_template_ref(template));
   log_template_unref(template);
@@ -89,6 +90,7 @@ construct_template_expr(LogTemplate *template)
 %token KW_ENUM
 %token KW_ISSET
 %token KW_UNSET
+%token KW_DECLARE
 
 %type <ptr> stmts
 %type <node> stmt
@@ -132,6 +134,7 @@ stmt
 	| conditional ';'			{ $$ = $1; }
 	;
 
+
 expr
 	: expr_value				{ $$ = $1; }
 	| function_call				{ $$ = $1; }
@@ -211,11 +214,12 @@ literal_object
 	;
 
 lvalue
-	: '$' LL_IDENTIFIER			{ $$ = filterx_variable_expr_new(log_msg_get_value_handle($2)); free($2); }
-	| LL_MESSAGE_REF			{ $$ = filterx_variable_expr_new(log_msg_get_value_handle($1)); free($1); }
+	: '$' LL_IDENTIFIER			{ $$ = filterx_variable_expr_new($2, FALSE); free($2); }
+	| LL_MESSAGE_REF			{ $$ = filterx_variable_expr_new($1, FALSE); free($1); }
+	| LL_IDENTIFIER				{ $$ = filterx_variable_expr_new($1, TRUE); free($1); }
+	| KW_DECLARE LL_IDENTIFIER              { $$ = filterx_variable_expr_new($2, TRUE); free($2); }
 	;
 
-
 boolean
 	: KW_TRUE				{ $$ = 1; }
 	| KW_FALSE				{ $$ = 0; }

diff --git a/lib/filterx/filterx-parser.c b/lib/filterx/filterx-parser.c
@@ -50,6 +50,7 @@ static CfgLexerKeyword filterx_keywords[] =
 
   { "isset",              KW_ISSET },
   { "unset",              KW_UNSET },
+  { "declare",            KW_DECLARE },
 
   { CFG_KEYWORD_STOP },
 };

diff --git a/lib/filterx/filterx-scope.c b/lib/filterx/filterx-scope.c
@@ -23,16 +23,21 @@
 #include "filterx/filterx-scope.h"
 #include "scratch-buffers.h"
 
+#define FILTERX_HANDLE_FLOATING_BIT (1UL << 31)
+
 struct _FilterXVariable
 {
   NVHandle handle;
   /*
    * floating -- Indicates that this variable is not tied to the log
    *             message, it is a floating variable
+   *
    * assigned -- Indicates that the variable was assigned to a new value
+   *
+   * pipeline -- this variable is retained for the entire input pipeline
    */
-  guint32 floating:1,
-          assigned:1;
+  guint32 assigned:1,
+          declared:1;
   FilterXObject *value;
 };
 
@@ -62,6 +67,12 @@ filterx_variable_is_set(FilterXVariable *v)
   return v->value != NULL;
 }
 
+void
+filterx_variable_mark_declared(FilterXVariable *v)
+{
+  v->declared = TRUE;
+}
+
 static void
 _variable_free(FilterXVariable *v)
 {
@@ -77,7 +88,7 @@ struct _FilterXScope
 };
 
 static gboolean
-_lookup_variable(FilterXScope *self, NVHandle handle, FilterXVariable **v_slot)
+_lookup_variable(FilterXScope *self, FilterXVariableHandle handle, FilterXVariable **v_slot)
 {
   gint l, h, m;
 
@@ -90,7 +101,7 @@ _lookup_variable(FilterXScope *self, NVHandle handle, FilterXVariable **v_slot)
 
       FilterXVariable *m_elem = &g_array_index(self->variables, FilterXVariable, m);
 
-      NVHandle mv = m_elem->handle;
+      FilterXVariableHandle mv = m_elem->handle;
       if (mv == handle)
         {
           *v_slot = m_elem;
@@ -109,8 +120,18 @@ _lookup_variable(FilterXScope *self, NVHandle handle, FilterXVariable **v_slot)
   return FALSE;
 }
 
+FilterXVariableHandle
+filterx_scope_map_variable_to_handle(const gchar *name, FilterXVariableType type)
+{
+  NVHandle nv_handle = log_msg_get_value_handle(name);
+
+  if (type == FX_VAR_MESSAGE)
+    return (FilterXVariableHandle) nv_handle;
+  return (FilterXVariableHandle) nv_handle | FILTERX_HANDLE_FLOATING_BIT;
+}
+
 FilterXVariable *
-filterx_scope_lookup_variable(FilterXScope *self, NVHandle handle)
+filterx_scope_lookup_variable(FilterXScope *self, FilterXVariableHandle handle)
 {
   FilterXVariable *v;
 
@@ -121,7 +142,7 @@ filterx_scope_lookup_variable(FilterXScope *self, NVHandle handle)
 
 FilterXVariable *
 filterx_scope_register_variable(FilterXScope *self,
-                                NVHandle handle, gboolean floating,
+                                FilterXVariableHandle handle,
                                 FilterXObject *initial_value)
 {
   FilterXVariable v, *v_slot;
@@ -138,7 +159,6 @@ filterx_scope_register_variable(FilterXScope *self,
 
   v.handle = handle;
   v.assigned = FALSE;
-  v.floating = floating;
   v.value = filterx_object_ref(initial_value);
   g_array_insert_val(self->variables, v_index, v);
 
@@ -172,7 +192,7 @@ filterx_scope_sync_to_message(FilterXScope *self, LogMessage *msg)
        *     place (for mutable objects), and was not assigned to.
        *
        */
-      if (v->floating)
+      if (v->handle & FILTERX_HANDLE_FLOATING_BIT)
         continue;
 
       if (v->value == NULL)
@@ -211,13 +231,20 @@ filterx_scope_clone(FilterXScope *other)
 {
   FilterXScope *self = filterx_scope_new();
 
-  for (gint i = 0; i < other->variables->len; i++)
+  for (gint src_index = 0, dst_index = 0; src_index < other->variables->len; src_index++)
     {
-      FilterXVariable *v = &g_array_index(other->variables, FilterXVariable, i);
-      g_array_append_val(self->variables, *v);
-      FilterXVariable *v_clone = &g_array_index(self->variables, FilterXVariable, i);
+      FilterXVariable *v = &g_array_index(other->variables, FilterXVariable, src_index);
 
-      v_clone->value = filterx_object_clone(v->value);
+      if (v->declared)
+        {
+          g_array_append_val(self->variables, *v);
+          FilterXVariable *v_clone = &g_array_index(self->variables, FilterXVariable, dst_index);
+
+          v_clone->value = filterx_object_clone(v->value);
+          dst_index++;
+          msg_trace("Filterx scope, cloning scope variable",
+                    evt_tag_str("variable", log_msg_get_value_name((v->handle & ~FILTERX_HANDLE_FLOATING_BIT), NULL)));
+        }
     }
 
   /* NOTE: we don't clone weak references, those only relate to mutable

diff --git a/lib/filterx/filterx-scope.h b/lib/filterx/filterx-scope.h
@@ -27,12 +27,19 @@
 #include "logmsg/logmsg.h"
 
 typedef struct _FilterXVariable FilterXVariable;
+typedef guint32 FilterXVariableHandle;
+typedef enum
+{
+  FX_VAR_MESSAGE,
+  FX_VAR_FLOATING,
+  FX_VAR_DECLARED,
+} FilterXVariableType;
 
 FilterXObject *filterx_variable_get_value(FilterXVariable *v);
 void filterx_variable_set_value(FilterXVariable *v, FilterXObject *new_value);
 void filterx_variable_unset_value(FilterXVariable *v);
 gboolean filterx_variable_is_set(FilterXVariable *v);
-
+void filterx_variable_mark_declared(FilterXVariable *v);
 
 /*
  * FilterXScope represents variables in a filterx scope.
@@ -49,9 +56,10 @@ typedef struct _FilterXScope FilterXScope;
 
 void filterx_scope_sync_to_message(FilterXScope *self, LogMessage *msg);
 
-FilterXVariable *filterx_scope_lookup_variable(FilterXScope *self, NVHandle handle);
+FilterXVariableHandle filterx_scope_map_variable_to_handle(const gchar *name, FilterXVariableType type);
+FilterXVariable *filterx_scope_lookup_variable(FilterXScope *self, FilterXVariableHandle handle);
 FilterXVariable *filterx_scope_register_variable(FilterXScope *self,
-                                                 NVHandle handle, gboolean floating,
+                                                 FilterXVariableHandle handle,
                                                  FilterXObject *initial_value);
 
 void filterx_scope_store_weak_ref(FilterXScope *self, FilterXObject *object);