Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HPCC-29674 LogAccess support for pod column type #17913

Merged
merged 1 commit into from
Oct 26, 2023
Merged

HPCC-29674 LogAccess support for pod column type #17913

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions esp/scm/ws_logaccess.ecm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ ESPenum LogAccessType : int
ByTargetAudience(4, "ByTargetAudience"),
BySourceInstance(5, "BySourceInstance"),
BySourceNode(6, "BySourceNode"),
ByFieldName(7, "ByFieldName")
ByFieldName(7, "ByFieldName"),
ByPod(8, "ByPod")
};

ESPenum LogAccessLogFormat : int
Expand Down Expand Up @@ -65,7 +66,8 @@ ESPenum LogColumnType : string
logid("logid"),
processid("processid"),
threadid("threadid"),
timestamp("timestamp")
timestamp("timestamp"),
pod("pod")
};

ESPenum LogColumnValueType : string
Expand Down Expand Up @@ -188,7 +190,8 @@ ESPenum SortColumType : int
ByTargetAudience(4, "ByTargetAudience"),
BySourceInstance(5, "BySourceInstance"),
BySourceNode(6, "BySourceNode"),
ByFieldName(7, "ByFieldName")
ByFieldName(7, "ByFieldName"),
ByPod(8, "ByPod")
};

ESPStruct SortCondition
Expand Down
2 changes: 2 additions & 0 deletions esp/services/ws_logaccess/WsLogAccessService.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,8 @@ ILogAccessFilter * buildLogFilterByFields(CLogAccessType searchByCategory, const
return getJobIDLogAccessFilter(searchByValue);
case CLogAccessType_ByComponent:
return getComponentLogAccessFilter(searchByValue);
case CLogAccessType_ByPod:
return getPodLogAccessFilter(searchByValue);
case CLogAccessType_ByLogType:
{
LogMsgClass logType = LogMsgClassFromAbbrev(searchByValue);
Expand Down
4 changes: 4 additions & 0 deletions helm/examples/azure/log-analytics/loganalytics-hpcc-logaccessV2.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,10 @@ global:
searchColumn: "hpcc_log_timestamp"
columnMode: "MIN"
columnType: "datetime"
- type: "pod"
searchColumn: "PodName"
columnMode: "DEFAULT"
columnType: "string"
secrets:
esp:
azure-logaccess: "azure-logaccess"
Expand Down
2 changes: 1 addition & 1 deletion helm/hpcc/values.schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3003,7 +3003,7 @@
"type": {
"type": "string",
"description" : "The searchable HPCC log column to be mapped - 'global' applies to all known fields",
"enum": [ "global", "workunits", "components", "audience", "class", "instance", "host", "node", "message", "logid", "processid", "threadid", "timestamp"]
"enum": [ "global", "workunits", "components", "audience", "class", "instance", "host", "node", "message", "logid", "processid", "threadid", "timestamp", "pod"]
},
"timeStampColumn": {
"description" : "Name of timestamp column related to mapped field (only requried for 'global' mapping)",
Expand Down
11 changes: 11 additions & 0 deletions system/jlib/jlog.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3191,6 +3191,11 @@ ILogAccessFilter * getComponentLogAccessFilter(const char * component)
return new FieldLogAccessFilter(component, LOGACCESS_FILTER_component);
}

ILogAccessFilter * getPodLogAccessFilter(const char * podName)
{
return new FieldLogAccessFilter(podName, LOGACCESS_FILTER_pod);
}

ILogAccessFilter * getAudienceLogAccessFilter(MessageAudience audience)
{
return new FieldLogAccessFilter(LogMsgAudienceToFixString(audience), LOGACCESS_FILTER_audience);
Expand Down Expand Up @@ -3240,6 +3245,12 @@ bool fetchComponentLog(LogQueryResultDetails & resultDetails, StringBuffer & ret
return fetchLog(resultDetails, returnbuf, logAccess, getComponentLogAccessFilter(component), timeRange, cols, format);
}

// Fetches log entries based on provided pod name, via provided IRemoteLogAccess instance
bool fetchPodLog(LogQueryResultDetails & resultDetails, StringBuffer & returnbuf, IRemoteLogAccess & logAccess, const char * podName, LogAccessTimeRange timeRange, StringArray & cols, LogAccessLogFormat format = LOGACCESS_LOGFORMAT_json)
{
return fetchLog(resultDetails, returnbuf, logAccess, getPodLogAccessFilter(podName), timeRange, cols, format);
}

// Fetches log entries based on provided audience, via provided IRemoteLogAccess instance
bool fetchLogByAudience(LogQueryResultDetails & resultDetails, StringBuffer & returnbuf, IRemoteLogAccess & logAccess, MessageAudience audience, LogAccessTimeRange timeRange, StringArray & cols, LogAccessLogFormat format = LOGACCESS_LOGFORMAT_json)
{
Expand Down
7 changes: 7 additions & 0 deletions system/jlib/jlog.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1373,6 +1373,7 @@ typedef enum
LOGACCESS_FILTER_instance,
LOGACCESS_FILTER_host,
LOGACCESS_FILTER_column,
LOGACCESS_FILTER_pod,
LOGACCESS_FILTER_unknown
} LogAccessFilterType;

Expand All @@ -1388,6 +1389,8 @@ inline const char * logAccessFilterTypeToString(LogAccessFilterType field)
return "audience";
case LOGACCESS_FILTER_component:
return "component";
case LOGACCESS_FILTER_pod:
return "pod";
case LOGACCESS_FILTER_instance:
return "instance";
case LOGACCESS_FILTER_host:
Expand Down Expand Up @@ -1416,6 +1419,8 @@ inline unsigned logAccessFilterTypeFromName(char const * name)
return LOGACCESS_FILTER_audience;
if(strieq(name, "component"))
return LOGACCESS_FILTER_component;
if(strieq(name, "pod"))
return LOGACCESS_FILTER_pod;
if(strieq(name, "instance"))
return LOGACCESS_FILTER_instance;
if(strieq(name, "host"))
Expand Down Expand Up @@ -1466,6 +1471,7 @@ enum LogAccessMappedField
LOGACCESS_MAPPEDFIELD_class,
LOGACCESS_MAPPEDFIELD_audience,
LOGACCESS_MAPPEDFIELD_instance,
LOGACCESS_MAPPEDFIELD_pod,
LOGACCESS_MAPPEDFIELD_host,
LOGACCESS_MAPPEDFIELD_unmapped
};
Expand Down Expand Up @@ -1665,6 +1671,7 @@ extern jlib_decl ILogAccessFilter * getInstanceLogAccessFilter(const char * inst
extern jlib_decl ILogAccessFilter * getHostLogAccessFilter(const char * host);
extern jlib_decl ILogAccessFilter * getJobIDLogAccessFilter(const char * jobId);
extern jlib_decl ILogAccessFilter * getComponentLogAccessFilter(const char * component);
extern jlib_decl ILogAccessFilter * getPodLogAccessFilter(const char * podName);
extern jlib_decl ILogAccessFilter * getAudienceLogAccessFilter(MessageAudience audience);
extern jlib_decl ILogAccessFilter * getClassLogAccessFilter(LogMsgClass logclass);
extern jlib_decl ILogAccessFilter * getBinaryLogAccessFilter(ILogAccessFilter * arg1, ILogAccessFilter * arg2, LogAccessFilterType type);
Expand Down
24 changes: 24 additions & 0 deletions system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ static constexpr const char * defaultHPCCLogComponentCol = "hpcc_log_component
static constexpr const char * defaultHPCCLogTypeCol = "hpcc_log_class";
static constexpr const char * defaultHPCCLogAudCol = "hpcc_log_audience";
static constexpr const char * defaultHPCCLogComponentTSCol = "TimeGenerated";
static constexpr const char * defaultPodHPCCLogCol = "PodName";

static constexpr const char * logMapIndexPatternAtt = "@storeName";
static constexpr const char * logMapSearchColAtt = "@searchColumn";
Expand Down Expand Up @@ -426,6 +427,13 @@ AzureLogAnalyticsCurlClient::AzureLogAnalyticsCurlClient(IPropertyTree & logAcce
else
OERRLOG("%s: Possible LogMap collision detected, 'host' and 'node' refer to same log column!", COMPONENT_NAME);
}
else if (streq(logMapType, "pod"))
{
if (logMap.hasProp(logMapIndexPatternAtt))
m_podIndexSearchPattern = logMap.queryProp(logMapIndexPatternAtt);
if (logMap.hasProp(logMapSearchColAtt))
m_podSearchColName = logMap.queryProp(logMapSearchColAtt);
}
else
{
ERRLOG("Encountered invalid LogAccess field map type: '%s'", logMapType);
Expand Down Expand Up @@ -479,6 +487,9 @@ void AzureLogAnalyticsCurlClient::getDefaultReturnColumns(StringBuffer & columns
columns.append(", ");
}

if (!isEmptyString(m_podSearchColName))
columns.appendf("%s, ", m_podSearchColName.str());

columns.appendf("%s, %s, %s, %s, %s, %s, %s",
m_globalIndexTimestampField.str(), defaultHPCCLogMessageCol, m_classSearchColName.str(),
m_audienceSearchColName.str(), m_workunitSearchColName.str(), defaultHPCCLogSeqCol, defaultHPCCLogThreadIDCol);
Expand Down Expand Up @@ -725,6 +736,19 @@ void AzureLogAnalyticsCurlClient::populateKQLQueryString(StringBuffer & queryStr
queryString.append(" ) ");
return; // queryString populated, need to break out
}
case LOGACCESS_FILTER_pod:
{
queryField = m_podSearchColName.str();

if (!m_podIndexSearchPattern.isEmpty())
{
throwIfMultiIndexDetected(queryIndex.str(), m_podIndexSearchPattern.str());
queryIndex = m_podIndexSearchPattern.str();
}

DBGLOG("%s: Searching log entries by Pod: '%s'", COMPONENT_NAME, queryValue.str() );
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not specific to this new column, but we should probably avoid these log lines by default, i.e. set them at a higher logging level.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well... I agree these types of messages are helpful for debugging ergo the DBGLOG seems appropriate, the problem from my perspective is the default log level for our deployments is 80 which is equivalent to debug, I've been advocating for a lower default log level for a long time.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I do agree - but would require a more comprehensive review of all DBGLOG's I suspect before we commit to turning them off by default.

Is there a JIRA this topic?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yep, in fact we have too many jiras related to this topic:

HPCC-22646 Revisit logging levels
HPCC-18399 Use jlog logging more consistently
HPCC-24433 Log levels above max threshold should be reevaluated
HPCC-30629 Clean up roxie logs
HPCC-26886 Consider adding detail level to the log output
HPCC-22650 Switch to using the tracing levels in logging

break;
}
case LOGACCESS_FILTER_column:
if (filter->getFieldName() == nullptr)
throw makeStringExceptionV(-1, "%s: empty field name detected in filter by column!", COMPONENT_NAME);
Expand Down
3 changes: 3 additions & 0 deletions system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ class AzureLogAnalyticsCurlClient : public CInterfaceOf<IRemoteLogAccess>
StringBuffer m_hostSearchColName;
StringBuffer m_hostIndexSearchPattern;

StringBuffer m_podIndexSearchPattern;
StringBuffer m_podSearchColName;

StringBuffer m_logAnalyticsWorkspaceID;
StringBuffer m_aadTenantID;
StringBuffer m_aadClientID;
Expand Down