Skip to content

Commit

Permalink
HPCC-32874
Browse files Browse the repository at this point in the history 
Add WsLogAcces Health Report Method

- Adds healthreport method to logaccess interface
- Implements healthreport method for all logaccess plugins

Signed-off-by: Rodrigo Pastrana <[email protected]>
  • Loading branch information
@rpastrana
rpastrana committed Nov 20, 2024
1 parent 6823b35 commit 9224ed6
Show file tree
Hide file tree
Showing 10 changed files with 534 additions and 32 deletions.
12 changes: 11 additions & 1 deletion esp/scm/ws_logaccess.ecm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,10 +229,20 @@ ESPResponse GetLogsResponse
[min_ver("1.02")] unsigned int TotalLogLinesAvailable;
};

ESPservice [auth_feature("WsLogAccess:READ"), version("1.06"), default_client_version("1.06"), exceptions_inline("xslt/exceptions.xslt")] ws_logaccess
ESPRequest GetHealthReportRequest
{
};

ESPResponse GetHealthReportResponse
{
ESParray<string> messages;
};

ESPservice [auth_feature("WsLogAccess:READ"), version("1.07"), default_client_version("1.07"), exceptions_inline("xslt/exceptions.xslt")] ws_logaccess
{
ESPmethod GetLogAccessInfo(GetLogAccessInfoRequest, GetLogAccessInfoResponse);
ESPmethod GetLogs(GetLogsRequest, GetLogsResponse);
ESPmethod [min_ver("1.07")] GetHealthReport(GetHealthReportRequest, GetHealthReportResponse);
};

SCMexportdef(ws_logaccess);
Expand Down
26 changes: 26 additions & 0 deletions esp/services/ws_logaccess/WsLogAccessService.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -385,3 +385,29 @@ bool Cws_logaccessEx::onGetLogs(IEspContext &context, IEspGetLogsRequest &req, I

return true;
}

bool Cws_logaccessEx::onGetHealthReport(IEspContext &context, IEspGetHealthReportRequest &req, IEspGetHealthReportResponse &resp)
{
StringArray testMessages;
bool success = true;
if (!queryRemoteLogAccessor())
{
testMessages.append("LogAccess plugin not available, review logAccess configuration!");
success = false;
}
else
{
try
{
queryRemoteLogAccessor()->healthReport(testMessages);
}
catch(...)
{
testMessages.append("Encountered unknown exception while performing connectivity test");
}
}

resp.setMessages(testMessages);

return success;
}
1 change: 1 addition & 0 deletions esp/services/ws_logaccess/WsLogAccessService.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ class Cws_logaccessEx : public Cws_logaccess
virtual ~Cws_logaccessEx();
virtual bool onGetLogAccessInfo(IEspContext &context, IEspGetLogAccessInfoRequest &req, IEspGetLogAccessInfoResponse &resp);
virtual bool onGetLogs(IEspContext &context, IEspGetLogsRequest &req, IEspGetLogsResponse & resp);
virtual bool onGetHealthReport(IEspContext &context, IEspGetHealthReportRequest &req, IEspGetHealthReportResponse &resp);
};

#endif
3 changes: 2 additions & 1 deletion system/jlib/jlog.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1690,6 +1690,7 @@ interface IRemoteLogAccess : extends IInterface
virtual IPropertyTree * queryLogMap() const = 0;
virtual const char * fetchConnectionStr() const = 0;
virtual bool supportsResultPaging() const = 0;
virtual bool healthReport(StringArray & messages) = 0;
};

// Helper functions to construct log access filters
Expand All @@ -1714,7 +1715,7 @@ extern jlib_decl bool fetchLog(LogQueryResultDetails & resultDetails, StringBuff
extern jlib_decl bool fetchJobIDLog(LogQueryResultDetails & resultDetails, StringBuffer & returnbuf, IRemoteLogAccess & logAccess, const char *jobid, LogAccessTimeRange timeRange, StringArray & cols, LogAccessLogFormat format);
extern jlib_decl bool fetchComponentLog(LogQueryResultDetails & resultDetails, StringBuffer & returnbuf, IRemoteLogAccess & logAccess, const char * component, LogAccessTimeRange timeRange, StringArray & cols, LogAccessLogFormat format);
extern jlib_decl bool fetchLogByAudience(LogQueryResultDetails & resultDetails, StringBuffer & returnbuf, IRemoteLogAccess & logAccess, MessageAudience audience, LogAccessTimeRange timeRange, StringArray & cols, LogAccessLogFormat format);
extern jlib_decl bool fetchLogByClass(LogQueryResultDetails & resultDetails, StringBuffer & returnbuf, IRemoteLogAccess & logAccess, LogMsgClass logclass, LogAccessTimeRange timeRange, StringArray & cols, LogAccessLogFormat format);
extern jlib_decl bool fetchLogByClass(LogQueryResultDetails & resultDetails, StringBuffer & returnbuf, IRemoteLogAccess & logAccess, LogMsgClass logclass, LogAccessTimeRange timeRange, StringArray & cols, LogAccessLogFormat format);
extern jlib_decl IRemoteLogAccess * queryRemoteLogAccessor();

#endif
102 changes: 101 additions & 1 deletion system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -556,7 +556,6 @@ bool generateHPCCLogColumnstAllColumns(StringBuffer & kql, const char * colName,
else
kql.append("LogEntrySource, TimeOfCommand, SourceSystem");


return true;
}

Expand Down Expand Up @@ -1058,6 +1057,107 @@ bool AzureLogAnalyticsCurlClient::processSearchJsonResp(LogQueryResultDetails &
return true;
}

bool AzureLogAnalyticsCurlClient::healthReport(StringArray & messages)
{
try
{
StringBuffer scratch;
scratch.setf("Target Azure Log Analytics workspace ID: '%s'", m_logAnalyticsWorkspaceID.str());
messages.append(scratch.str());

scratch.setf("Target Azure Log Analytics tenant ID: '%s'", m_aadTenantID.str());
messages.append(scratch.str());

scratch.setf("Target Azure Log Analytics client ID: '%s'", m_aadClientID.str());
messages.append(scratch.str());

scratch.setf("Target Azure Log Analytics secret is%s empty", m_aadClientSecret.length()==0 ? "" : " not");
messages.append(scratch.str());

scratch.setf("Targets ContainerLogV%c", targetIsContainerLogV2 ? '2' : '1');
messages.append(scratch.str());

scratch.setf("Components query joins %senabled", m_disableComponentNameJoins ? "not " : "");
messages.append(scratch.str());

if (m_pluginCfg)
{
StringBuffer configXML;
toXML(m_pluginCfg, configXML);

scratch.setf("Configuration tree: '%s'", configXML.str());
messages.append(scratch.str());
}
else
{
messages.append("Configuration tree is empty!!!");
}

scratch.set("LogMaps:");
scratch.appendf("\n\tGlobal column: '%s', index pattern: '%s', timestamp column: '%s'", m_globalSearchColName.str(), m_globalIndexSearchPattern.str(), m_globalIndexTimestampField.str());
scratch.appendf("\n\tWorkunits column: '%s', index pattern: '%s'", m_workunitSearchColName.str(), m_workunitIndexSearchPattern.str());
scratch.appendf("\n\tComponents column: '%s', index pattern: '%s'", m_componentsSearchColName.str(), m_componentsIndexSearchPattern.str());
scratch.appendf("\n\tAudience column: '%s', index pattern: '%s'", m_audienceSearchColName.str(), m_audienceIndexSearchPattern.str());
scratch.appendf("\n\tLog Class column: '%s', index pattern: '%s'", m_classSearchColName.str(), m_classIndexSearchPattern.str());
scratch.appendf("\n\tInstance column: '%s', index pattern: '%s'", m_instanceSearchColName.str(), m_instanceIndexSearchPattern.str());
scratch.appendf("\n\tPod column: '%s', index pattern: '%s'", m_podSearchColName.str(), m_podIndexSearchPattern.str());
scratch.appendf("\n\tTraceID column: '%s', index pattern: '%s'", m_traceSearchColName.str(), m_traceIndexSearchPattern.str());
scratch.appendf("\n\tSpanID column: '%s', index pattern: '%s'", m_spanSearchColName.str(), m_spanIndexSearchPattern.str());
scratch.appendf("\n\tHost column: '%s', index pattern: '%s'", m_hostSearchColName.str(), m_hostIndexSearchPattern.str());
messages.append(scratch.str());

try
{
LogAccessLogFormat outputFormat = LOGACCESS_LOGFORMAT_xml;
LogAccessConditions queryOptions;

queryOptions.setFilter(getComponentLogAccessFilter(""));

struct LogAccessTimeRange range;
CDateTime endtt;
endtt.setNow();
range.setEnd(endtt);

CDateTime startt;
startt.setNow();
startt.adjustTimeSecs(-60); //an hour ago
range.setStart(startt);

StringBuffer startstr;
startt.getString(startstr);

queryOptions.setTimeRange(range);
queryOptions.setLimit(100);

StringBuffer logs;
LogQueryResultDetails resultDetails;
fetchLog(resultDetails, queryOptions, logs, outputFormat);
scratch.setf("Sample ALA query resulted in %d log records.", resultDetails.totalReceived);
messages.append(scratch.str());
messages.append(logs.str());
}
catch(IException * e)
{
StringBuffer description;
e->errorMessage(description);
scratch.setf("Exception while executing sample Grafana/Loki query (%d) - %s", e->errorCode(), description.str());
messages.append(scratch.str());
e->Release();
}
catch(...)
{
messages.append("Unknown exception while executing sample Grafana/Loki query");
}
}
catch(...)
{
messages.append("Encountered unexpected exception during health report");
return false;
}

return true;
}

bool AzureLogAnalyticsCurlClient::fetchLog(LogQueryResultDetails & resultDetails, const LogAccessConditions & options, StringBuffer & returnbuf, LogAccessLogFormat format)
{
StringBuffer token;
Expand Down
1 change: 1 addition & 0 deletions system/logaccess/Azure/LogAnalytics/CurlClient/AzureLogAnalyticsCurlClient.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,4 +100,5 @@ class AzureLogAnalyticsCurlClient : public CInterfaceOf<IRemoteLogAccess>
virtual IRemoteLogAccessStream * getLogReader(const LogAccessConditions & options, LogAccessLogFormat format) override;
virtual IRemoteLogAccessStream * getLogReader(const LogAccessConditions & options, LogAccessLogFormat format, unsigned int pageSize) override;
virtual bool supportsResultPaging() const override { return false;}
virtual bool healthReport(StringArray & messages) override;
};
158 changes: 158 additions & 0 deletions system/logaccess/ElasticStack/ElasticStackLogAccess.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,164 @@ const IPropertyTree * ElasticStackLogAccess::getESStatus()
return performAndLogESRequest(Client::HTTPMethod::GET, "_cluster/health", "", "Target cluster health");
}

bool ElasticStackLogAccess::healthReport(StringArray & messages)
{
try
{
StringBuffer scratch;
scratch.setf("Connection string: '%s'", m_esConnectionStr.str());
messages.append(scratch.str());

if (m_pluginCfg)
{
StringBuffer configXML;
toXML(m_pluginCfg, configXML);

scratch.setf("Configuration tree: '%s'", configXML.str());
messages.append(scratch.str());
}
else
{
messages.append("Configuration tree is empty!!!");
}

scratch.set("LogMaps:");
scratch.appendf("\n\tGlobal column: '%s', index pattern: '%s', timestamp column: '%s'", m_globalSearchColName.str(), m_globalIndexSearchPattern.str(), m_globalIndexTimestampField.str());
scratch.appendf("\n\tWorkunits column: '%s', index pattern: '%s'", m_workunitSearchColName.str(), m_workunitIndexSearchPattern.str());
scratch.appendf("\n\tComponents column: '%s', index pattern: '%s'", m_componentsSearchColName.str(), m_componentsIndexSearchPattern.str());
scratch.appendf("\n\tAudience column: '%s', index pattern: '%s'", m_audienceSearchColName.str(), m_audienceIndexSearchPattern.str());
scratch.appendf("\n\tLog Class column: '%s', index pattern: '%s'", m_classSearchColName.str(), m_classIndexSearchPattern.str());
scratch.appendf("\n\tInstance column: '%s', index pattern: '%s'", m_instanceSearchColName.str(), m_instanceIndexSearchPattern.str());
scratch.appendf("\n\tPod column: '%s', index pattern: '%s'", m_podSearchColName.str(), m_podIndexSearchPattern.str());
scratch.appendf("\n\tTraceID column: '%s', index pattern: '%s'", m_traceSearchColName.str(), m_traceIndexSearchPattern.str());
scratch.appendf("\n\tSpanID column: '%s', index pattern: '%s'", m_spanSearchColName.str(), m_spanIndexSearchPattern.str());
scratch.appendf("\n\tHost column: '%s', index pattern: '%s'", m_hostSearchColName.str(), m_hostIndexSearchPattern.str());
messages.append(scratch.str());

try
{
StringBuffer out;
const IPropertyTree * status = getESStatus();
if (status)
{
toXML(status, out);
scratch.setf("\n\tES Status: '%s'", out.str());
messages.append(scratch.str());
}
}
catch(IException * e)
{
StringBuffer description;
e->errorMessage(description);
scratch.setf("\n\tException fetching ES Status (%d) - %s", e->errorCode(), description.str());
messages.append(description.str());
e->Release();
}
catch(...)
{
messages.append("\n\tUnknown exception while fetching ES Status");
}

try
{
StringBuffer out;
const IPropertyTree * is = getIndexSearchStatus(m_globalIndexSearchPattern);
if (is)
{
toXML(is, out.clear());
scratch.setf("ES available indices: '%s'", out.str());
messages.append(scratch.str());
}
}
catch(IException * e)
{
StringBuffer description;
e->errorMessage(description);
scratch.setf("\n\tException fetching available ES indices (%d) - %s", e->errorCode(), description.str());
messages.append(description.str());
e->Release();
}
catch(...)
{
messages.append("\n\tUnknown exception while fetching available ES indices");
}

try
{
StringBuffer out;
const IPropertyTree * ts = getTimestampTypeFormat(m_globalIndexSearchPattern, m_globalIndexTimestampField);
if (ts)
{
toXML(ts, out.clear());
scratch.setf("ES timestamp field '%s' format for index '%s': '%s'", m_globalIndexTimestampField.str(), m_globalIndexSearchPattern.str(), out.str());
messages.append(scratch.str());
}
}
catch(IException * e)
{
StringBuffer description;
e->errorMessage(description);
scratch.setf("\n\tException fetching target ES timestamp format (%d) - %s", e->errorCode(), description.str());
messages.append(description.str());
e->Release();
}
catch(...)
{
messages.append("\n\tUnknown exception while fetching target ES timestamp format");
}

try
{
LogAccessLogFormat outputFormat = LOGACCESS_LOGFORMAT_xml;
LogAccessConditions queryOptions;

queryOptions.setFilter(getComponentLogAccessFilter("eclwatch"));

struct LogAccessTimeRange range;
CDateTime endtt;
endtt.setNow();
range.setEnd(endtt);

CDateTime startt;
startt.setNow();
startt.adjustTimeSecs(-60); //an hour ago
range.setStart(startt);

StringBuffer startstr;
startt.getString(startstr);

queryOptions.setTimeRange(range);
queryOptions.setLimit(100);

StringBuffer logs;
LogQueryResultDetails resultDetails;
fetchLog(resultDetails, queryOptions, logs, outputFormat);
scratch.setf("Sample ALA query resulted in %d log records.", resultDetails.totalReceived);
messages.append(scratch.str());
messages.append(logs.str());
}
catch(IException * e)
{
StringBuffer description;
e->errorMessage(description);
scratch.setf("Exception while executing sample Grafana/Loki query (%d) - %s", e->errorCode(), description.str());
messages.append(scratch.str());
e->Release();
}
catch(...)
{
messages.append("Unknown exception while executing sample Grafana/Loki query");
}
}
catch(...)
{
messages.append("Encountered unexpected exception during health report");
return false;
}

return true;
}

/*
* Transform iterator of hits/fields to back-end agnostic response
*
Expand Down
1 change: 1 addition & 0 deletions system/logaccess/ElasticStack/ElasticStackLogAccess.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,4 +103,5 @@ class ElasticStackLogAccess : public CInterfaceOf<IRemoteLogAccess>
virtual IRemoteLogAccessStream * getLogReader(const LogAccessConditions & options, LogAccessLogFormat format) override;
virtual IRemoteLogAccessStream * getLogReader(const LogAccessConditions & options, LogAccessLogFormat format, unsigned int pageSize) override;
virtual bool supportsResultPaging() const override { return true;}
virtual bool healthReport(StringArray & messages) override;
};
Loading

0 comments on commit 9224ed6

Please sign in to comment.