Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Hpcc 30401 j trace optional traceid support #17934

Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions esp/scm/ws_logaccess.ecm
Original file line number Diff line number Diff line change
@@ -76,13 +76,14 @@ ESPenum LogColumnValueType : string
enum("enum")
};

ESPStruct LogColumn
ESPStruct [nil_remove] LogColumn
{
string Name;
ESPenum LogColumnType LogType;
ESParray<string> EnumeratedValues;
LogSelectColumnMode ColumnMode;
ESPenum LogColumnValueType ColumnType;
[min_ver("1.06")] string Alias;
};

ESPResponse GetLogAccessInfoResponse
@@ -219,7 +220,7 @@ ESPResponse GetLogsResponse
[min_ver("1.02")] unsigned int TotalLogLinesAvailable;
};

ESPservice [auth_feature("WsLogAccess:READ"), version("1.05"), default_client_version("1.05"), exceptions_inline("xslt/exceptions.xslt")] ws_logaccess
ESPservice [auth_feature("WsLogAccess:READ"), version("1.06"), default_client_version("1.06"), exceptions_inline("xslt/exceptions.xslt")] ws_logaccess
{
ESPmethod GetLogAccessInfo(GetLogAccessInfoRequest, GetLogAccessInfoResponse);
ESPmethod GetLogs(GetLogsRequest, GetLogsResponse);
20 changes: 20 additions & 0 deletions esp/services/ws_logaccess/WsLogAccessService.cpp
Original file line number Diff line number Diff line change
@@ -71,6 +71,26 @@ bool Cws_logaccessEx::onGetLogAccessInfo(IEspContext &context, IEspGetLogAccessI
WARNLOG("Invalid col type found in logaccess logmap config");
}
}
if (clientVer >= 1.06)
{
if (column.hasProp("@alias"))
{
try
{
espLogColumn->setAlias(column.queryProp("@alias"));
}
catch (IException *e)
{
VStringBuffer msg("Invalid alias found in logaccess logmap config for '%s'", csearchColumn);
EXCLOG(e, msg.str());
e->Release();
}
catch(...)
{
WARNLOG("Invalid alias found in logaccess logmap config");
}
}
}
logColumns.append(*espLogColumn.getClear());
}
else
Original file line number Diff line number Diff line change
@@ -59,7 +59,7 @@ global:
- type: "instance"
storeName: "ContainerInventory"
searchColumn: "Name"
projectName: "hpcc_log_component"
alias: "hpcc_log_component"
columnMode: "ALL"
columnType: "string"
- type: "node"
15 changes: 14 additions & 1 deletion helm/hpcc/templates/_helpers.tpl
Original file line number Diff line number Diff line change
@@ -288,6 +288,18 @@ logging:
{{- end -}}
{{- end -}}

{{/*
Generate local tracing info, merged with global
Pass in dict with root and me
*/}}
{{- define "hpcc.generateTracingConfig" -}}
{{- $tracing := deepCopy (.me.tracing | default dict) | mergeOverwrite dict (.root.Values.global.tracing | default dict) -}}
{{- if not (empty $tracing) }}
tracing:
{{ toYaml $tracing | indent 2 }}
{{- end -}}
{{- end -}}

{{/*
Generate local metrics configuration, merged with global
Pass in dict with root and me
@@ -1286,8 +1298,9 @@ data:
{{ $configMapName }}.yaml:
version: 1.0
sasha:
{{ toYaml (omit .me "logging") | indent 6 }}
{{ toYaml (omit .me "logging" "tracing") | indent 6 }}
{{- include "hpcc.generateLoggingConfig" . | indent 6 }}
{{- include "hpcc.generateTracingConfig" . | indent 6 }}
{{ include "hpcc.generateVaultConfig" . | indent 6 }}
{{- if hasKey .me "plane" }}
{{- $sashaStoragePlane := .me.plane | default (include "hpcc.getFirstPlaneForCategory" (dict "root" .root "category" "sasha")) }}
3 changes: 2 additions & 1 deletion helm/hpcc/templates/dafilesrv.yaml
Original file line number Diff line number Diff line change
@@ -10,8 +10,9 @@ data:
{{ .me.name }}.yaml:
version: 1.0
dafilesrv:
{{ toYaml (omit .me "logging" "env") | indent 6 }}
{{ toYaml (omit .me "logging" "tracing" "env") | indent 6 }}
{{- include "hpcc.generateLoggingConfig" . | indent 6 }}
{{- include "hpcc.generateTracingConfig" . | indent 6 }}
global:
{{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }}
{{- end -}}
3 changes: 2 additions & 1 deletion helm/hpcc/templates/dali.yaml
Original file line number Diff line number Diff line change
@@ -171,9 +171,10 @@ data:
{{ $dali.name }}.yaml: |
version: 1.0
dali:
{{ toYaml (omit $dali "logging" "services" "env") | indent 6 }}
{{ toYaml (omit $dali "logging" "tracing" "services" "env") | indent 6 }}
dataPath: {{ include "hpcc.getPlanePrefix" (dict "root" $ "planeName" $daliStoragePlane) }}
{{- include "hpcc.generateLoggingConfig" $commonCtx | indent 6 }}
{{- include "hpcc.generateTracingConfig" $commonCtx | indent 6 }}
{{- include "hpcc.generateMetricsConfig" $commonCtx | indent 6 }}
{{ include "hpcc.generateVaultConfig" (dict "root" $ "secretsCategories" $daliSecretsCategories ) | indent 6 }}
foreignAccess: {{ include "hpcc.isForeignAccessConfigured" (dict "root" $ "me" .) | default "false" }}
3 changes: 2 additions & 1 deletion helm/hpcc/templates/dfuserver.yaml
Original file line number Diff line number Diff line change
@@ -29,8 +29,9 @@ data:
{{ .me.name }}.yaml:
version: 1.0
dfuserver:
{{ toYaml (omit .me "logging" "env") | indent 6 }}
{{ toYaml (omit .me "logging" "tracing" "env") | indent 6 }}
{{- include "hpcc.generateLoggingConfig" . | indent 6 }}
{{- include "hpcc.generateTracingConfig" . | indent 6 }}
global:
{{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }}
{{- end -}}
6 changes: 4 additions & 2 deletions helm/hpcc/templates/eclagent.yaml
Original file line number Diff line number Diff line change
@@ -35,11 +35,13 @@ data:
{{ .me.name }}.yaml:
version: 1.0
eclagent:
{{ toYaml (omit .me "logging" "env") | indent 6 }}
{{ toYaml (omit .me "logging" "tracing" "env") | indent 6 }}
{{- include "hpcc.generateLoggingConfig" . | indent 6 }}
{{- include "hpcc.generateTracingConfig" . | indent 6 }}
{{ $apptype }}:
{{ toYaml (omit .me "logging" "env") | indent 6 }}
{{ toYaml (omit .me "logging" "tracing" "env") | indent 6 }}
{{- include "hpcc.generateLoggingConfig" . | indent 6 }}
{{- include "hpcc.generateTracingConfig" . | indent 6 }}
{{ include "hpcc.generateVaultConfig" . | indent 6 }}
global:
{{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }}
3 changes: 2 additions & 1 deletion helm/hpcc/templates/eclccserver.yaml
Original file line number Diff line number Diff line change
@@ -34,8 +34,9 @@ data:
{{ .me.name }}.yaml:
version: 1.0
eclccserver:
{{ toYaml (omit .me "logging") | indent 6 }}
{{ toYaml (omit .me "logging" "tracing") | indent 6 }}
{{- include "hpcc.generateLoggingConfig" . | indent 6 }}
{{- include "hpcc.generateTracingConfig" . | indent 6 }}
queues:
{{ include "hpcc.generateConfigMapQueues" .root | indent 6 }}
{{ include "hpcc.generateVaultConfig" . | indent 6 }}
3 changes: 2 additions & 1 deletion helm/hpcc/templates/eclscheduler.yaml
Original file line number Diff line number Diff line change
@@ -33,8 +33,9 @@ data:
{{ .me.name }}.yaml:
version: 1.0
eclscheduler:
{{ toYaml (omit .me "logging" "env") | indent 6 }}
{{ toYaml (omit .me "logging" "tracing" "env") | indent 6 }}
{{- include "hpcc.generateLoggingConfig" . | indent 6 }}
{{- include "hpcc.generateTracingConfig" . | indent 6 }}
queues:
{{ include "hpcc.generateConfigMapQueues" .root | indent 6 }}
{{ include "hpcc.generateVaultConfig" . | indent 6 }}
3 changes: 2 additions & 1 deletion helm/hpcc/templates/esp.yaml
Original file line number Diff line number Diff line change
@@ -33,8 +33,9 @@ data:
{{ .me.name }}.yaml:
version: 1.0
esp:
{{ toYaml (omit .me "logging" "metrics" "env") | indent 6 }}
{{ toYaml (omit .me "logging" "tracing" "metrics" "env") | indent 6 }}
{{- include "hpcc.generateLoggingConfig" . | indent 6 }}
{{- include "hpcc.generateTracingConfig" . | indent 6 }}
{{- include "hpcc.generateMetricsConfig" . | indent 6 }}
{{- if and .root.Values.certificates .root.Values.certificates.enabled }}
{{- $externalCert := (ne (include "hpcc.isVisibilityPublic" (dict "root" .root "visibility" .me.service.visibility)) "") -}}
3 changes: 2 additions & 1 deletion helm/hpcc/templates/localroxie.yaml
Original file line number Diff line number Diff line change
@@ -33,8 +33,9 @@ data:
{{ .me.name }}.yaml:
version: 1.0
roxie:
{{ toYaml (omit .me "logging" "env") | indent 6 }}
{{ toYaml (omit .me "logging" "tracing" "env") | indent 6 }}
{{- include "hpcc.generateLoggingConfig" . | indent 6 }}
{{- include "hpcc.generateTracingConfig" . | indent 6 }}
{{ include "hpcc.generateVaultConfig" . | indent 6 }}
global:
{{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }}
6 changes: 4 additions & 2 deletions helm/hpcc/templates/roxie.yaml
Original file line number Diff line number Diff line change
@@ -44,7 +44,7 @@ data:
{{- include "hpcc.addTLSServiceEntries" (dict "root" $root "service" $service "component" $component "visibility" $service.visibility "remoteClients" $service.remoteClients "trustClients" $service.trustClients "includeTrustedPeers" true "incluedRoxieAndEspServices" true) | indent 6 }}
{{- end }}
{{- end }}
{{ toYaml ( omit .me "logging" "topoServer" "encryptInTransit" "env" "services") | indent 6 }}
{{ toYaml ( omit .me "logging" "tracing" "topoServer" "encryptInTransit" "env" "services") | indent 6 }}
numChannels: {{ .numChannels }}
topologyServers: "{{ .toponame }}:{{ .topoport }}"
heartbeatInterval: {{ .heartbeatInterval }}
@@ -60,6 +60,7 @@ data:
encryptInTransit: {{ $mtlsEnabled }}
{{ end -}}
{{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" .me) | indent 6 }}
{{- include "hpcc.generateTracingConfig" (dict "root" .root "me" .me) | indent 6 }}
{{ include "hpcc.generateVaultConfig" . | indent 6 }}
global:
{{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }}
@@ -73,8 +74,9 @@ data:
{{ .toponame }}.yaml:
version: 1.0
toposerver:
{{ toYaml ( omit .toposerver "logging" "env") | indent 6 }}
{{ toYaml ( omit .toposerver "logging" "tracing" "env") | indent 6 }}
{{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" .toposerver) | indent 6 }}
{{- include "hpcc.generateTracingConfig" (dict "root" .root "me" .toposerver) | indent 6 }}
global:
{{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }}
{{- end -}}
6 changes: 5 additions & 1 deletion helm/hpcc/templates/thor.yaml
Original file line number Diff line number Diff line change
@@ -32,7 +32,7 @@ Pass in dict with root and me
{{- $thorAgentScope := dict "name" .thorAgentName "replicas" .thorAgentReplicas "maxActive" .me.maxGraphs | merge (pick .me "keepJobs") }}
{{- $eclAgentResources := .me.eclAgentResources | default dict -}}
{{- $hthorScope := dict "name" $hthorName "jobMemorySectionName" "eclAgentMemory" | merge (pick .me "multiJobLinger" "maxGraphStartupTime") | merge (dict "resources" (deepCopy $eclAgentResources)) }}
{{- $thorScope := omit .me "eclagent" "thoragent" "hthor" "logging" "env" "eclAgentResources" "eclAgentUseChildProcesses" "eclAgentReplicas" "thorAgentReplicas" "eclAgentType" }}
{{- $thorScope := omit .me "eclagent" "thoragent" "hthor" "logging" "Tracing" "env" "eclAgentResources" "eclAgentUseChildProcesses" "eclAgentReplicas" "thorAgentReplicas" "eclAgentType" }}
{{- $misc := .root.Values.global.misc | default dict }}
{{- $postJobCommand := $misc.postJobCommand | default "" }}
{{- $eclAgentJobName := printf "%s-job-_HPCC_JOBNAME_" $eclAgentType }}
@@ -51,20 +51,24 @@ data:
thor:
{{ toYaml $thorScope | indent 6 }}
{{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" $thorScope) | indent 6 }}
{{- include "hpcc.generateTracingConfig" (dict "root" .root "me" $thorScope) | indent 6 }}
{{ include "hpcc.generateVaultConfig" . | indent 6 }}
{{ $eclAgentType }}: # hthor or roxie
{{ toYaml $hthorScope | indent 6 }}
platform:
type: "thor"
width: {{ mul ($thorScope.numWorkers | default 1) ( $thorScope.channelsPerWorker | default 1) }}
{{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" $hthorScope ) | indent 6 }}
{{- include "hpcc.generateTracingConfig" (dict "root" .root "me" $hthorScope ) | indent 6 }}
{{ include "hpcc.generateVaultConfig" . | indent 6 }}
eclagent: # main agent Q handler
{{ toYaml $eclAgentScope | indent 6 }}
{{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" $eclAgentScope) | indent 6 }}
{{- include "hpcc.generateTracingConfig" (dict "root" .root "me" $eclAgentScope) | indent 6 }}
thoragent: # Thor graph handler
{{ toYaml $thorAgentScope | indent 6 }}
{{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" $thorAgentScope) | indent 6 }}
{{- include "hpcc.generateTracingConfig" (dict "root" .root "me" $thorAgentScope) | indent 6 }}
type: thor
global:
{{ include "hpcc.generateGlobalConfigMap" .root| indent 6 }}
6 changes: 5 additions & 1 deletion helm/hpcc/values.schema.json
Original file line number Diff line number Diff line change
@@ -1102,6 +1102,10 @@
"alwaysCreateGlobalIds": {
"type": "boolean",
"description": "If true, allocate global ids to any requests that do not supply one"
},
"alwaysCreateTraceIds": {
"type": "boolean",
"description": "If true, components generate trace/span ids to aid in unit of worktracing"
}
},
"additionalProperties": { "type": ["integer", "string", "boolean"] }
@@ -3003,7 +3007,7 @@
"type": {
"type": "string",
"description" : "The searchable HPCC log column to be mapped - 'global' applies to all known fields",
"enum": [ "global", "workunits", "components", "audience", "class", "instance", "host", "node", "message", "logid", "processid", "threadid", "timestamp"]
"enum": [ "global", "workunits", "components", "audience", "class", "instance", "host", "node", "message", "logid", "processid", "threadid", "timestamp", "alias"]
},
"timeStampColumn": {
"description" : "Name of timestamp column related to mapped field (only requried for 'global' mapping)",
3 changes: 2 additions & 1 deletion helm/hpcc/values.yaml
Original file line number Diff line number Diff line change
@@ -24,7 +24,8 @@ global:
# logging sets the default logging information for all components. Can be overridden locally
logging:
detail: 80

tracing:
alwaysCreateTraceIds: true
## resource settings for stub components
#stubInstanceResources:
# memory: "200Mi"
44 changes: 38 additions & 6 deletions system/jlib/jtrace.cpp
Original file line number Diff line number Diff line change
@@ -16,7 +16,7 @@
############################################################################## */



#include "opentelemetry/sdk/trace/random_id_generator_factory.h"
#include "opentelemetry/trace/semantic_conventions.h" //known span defines
#include "opentelemetry/context/propagation/global_propagator.h" // context::propagation::GlobalTextMapPropagator::GetGlobalPropagator
#include "opentelemetry/sdk/trace/tracer_provider_factory.h" //opentelemetry::sdk::trace::TracerProviderFactory::Create(context)
@@ -602,11 +602,6 @@ class CServerSpan : public CSpan
{
if (httpHeaders)
{
// perform any key mapping needed...
//Instrumented http client/server Capitalizes the first letter of the header name
//if (key == opentel_trace::propagation::kTraceParent || key == opentel_trace::propagation::kTraceState )
// theKey[0] = toupper(theKey[0]);

if (httpHeaders->hasProp(kGlobalIdHttpHeaderName))
hpccGlobalId.set(httpHeaders->queryProp(kGlobalIdHttpHeaderName));
else if (httpHeaders->hasProp(kLegacyGlobalIdHttpHeaderName))
@@ -637,6 +632,35 @@ class CServerSpan : public CSpan
opts.parent = remoteParentSpanCtx;
}
}

if ((!httpHeaders || !httpHeaders->hasProp("traceparent")) && queryTraceManager().alwaysCreateTraceIds())
{
//Generate random trace and span IDs

//Don't get the generator every time
auto randomIDGenerator = opentelemetry::sdk::trace::RandomIdGeneratorFactory::Create();

auto randomTraceID = randomIDGenerator->GenerateTraceId();
auto randomSpanID = randomIDGenerator->GenerateSpanId();

remoteParentSpanCtx = opentelemetry::trace::SpanContext(randomTraceID, randomSpanID, opentelemetry::trace::TraceFlags(), true);
opts.parent = remoteParentSpanCtx;


//this is only for debugging:
char trace_id[32] = {0};

randomTraceID.ToLowerBase16(trace_id);
StringAttr traceID;
traceID.set(trace_id, 32);

char span_id[16] = {0};
randomSpanID.ToLowerBase16(span_id);
StringAttr spanID;
spanID.set(span_id, 16);

DBGLOG("!!No remoteParent received, generating remoteParent IDs: Trace '%s' span '%s' ", traceID.get(), spanID.get());
}
}

bool getSpanContext(IProperties * ctxProps, bool otelFormatted) const override
@@ -710,6 +734,7 @@ class CTraceManager : implements ITraceManager, public CInterface
private:
bool enabled = true;
bool optAlwaysCreateGlobalIds = false;
bool optAlwaysCreateTraceIds = true;
StringAttr moduleName;

//Initializes the global trace provider which is required for all Otel based tracing operations.
@@ -816,6 +841,7 @@ class CTraceManager : implements ITraceManager, public CInterface
tracing: #optional - tracing enabled by default
disable: true #optional - disable OTel tracing
alwaysCreateGlobalIds : false #optional - should global ids always be created?
alwaysCreateTraceIds #optional - should trace ids always be created?
exporter: #optional - Controls how trace data is exported/reported
type: OTLP #OS|OTLP|Prometheus|HPCC (default: no export, jlog entry)
endpoint: "localhost:4317" #exporter specific key/value pairs
@@ -870,6 +896,7 @@ class CTraceManager : implements ITraceManager, public CInterface
if (traceConfig)
{
optAlwaysCreateGlobalIds = traceConfig->getPropBool("@alwaysCreateGlobalIds", optAlwaysCreateGlobalIds);
optAlwaysCreateTraceIds = traceConfig->getPropBool("@alwaysCreateTraceIds", optAlwaysCreateTraceIds);
}

// The global propagator should be set regardless of whether tracing is enabled or not.
@@ -942,6 +969,11 @@ class CTraceManager : implements ITraceManager, public CInterface
{
return optAlwaysCreateGlobalIds;
}

virtual bool alwaysCreateTraceIds() const
{
return optAlwaysCreateTraceIds;
}
};

static Singleton<CTraceManager> theTraceManager;
1 change: 1 addition & 0 deletions system/jlib/jtrace.hpp
Original file line number Diff line number Diff line change
@@ -62,6 +62,7 @@ interface ITraceManager : extends IInterface
virtual ISpan * createServerSpan(const char * name, const IProperties * httpHeaders, SpanFlags flags = SpanFlags::None) = 0;
virtual bool isTracingEnabled() const = 0;
virtual bool alwaysCreateGlobalIds() const = 0;
virtual bool alwaysCreateTraceIds() const = 0;
virtual const char * getTracedComponentName() const = 0;
};