diff --git a/ozburst-all/mimeter-engine/mimeter-agent-manager/README.md b/ozburst-all/mimeter-engine/mimeter-agent-manager/README.md new file mode 100644 index 000000000..5e383f300 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-manager/README.md @@ -0,0 +1,6 @@ ++ 用来分配和调度压测任务 ++ new_mibench_manager ++ env ++ mione.faas.func.env.id=1;mione.faas.func.id=1 (agent-manager) ++ jvm ++ --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.math=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED \ No newline at end of file diff --git a/ozburst-all/mimeter-engine/mimeter-agent-manager/dependency-reduced-pom.xml b/ozburst-all/mimeter-engine/mimeter-agent-manager/dependency-reduced-pom.xml new file mode 100644 index 000000000..39ed65bed --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-manager/dependency-reduced-pom.xml @@ -0,0 +1,85 @@ + + + + mimeter-engine + run.mone + 1.0-SNAPSHOT + + 4.0.0 + mimeter-agent-manager + + + + maven-shade-plugin + 3.1.1 + + + package + + shade + + + + + + + + + + com.xiaomi.mone + sautumn-serverless-api + 1.0.0-SNAPSHOT + provided + + + youpin-infra-rpc + com.xiaomi.youpin + + + docean-plugin-dubbo-serverless + run.mone + + + sautumn-filter-api + com.xiaomi.mone + + + sidecar-docs-annotations + run.mone + + + + + javax.annotation + javax.annotation-api + 1.3.2 + provided + + + run.mone + docean-plugin-rpc + 1.4-SNAPSHOT + provided + + + rpc + run.mone + + + docean-plugin-config + run.mone + + + + + org.projectlombok + lombok + 1.18.24 + provided + + + + 20 + 20 + + diff --git a/ozburst-all/mimeter-engine/mimeter-agent-manager/pom.xml b/ozburst-all/mimeter-engine/mimeter-agent-manager/pom.xml new file mode 100644 index 000000000..1220fa0ca --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-manager/pom.xml @@ -0,0 +1,127 @@ + + + + mimeter-engine + run.mone + 1.0-SNAPSHOT + + 4.0.0 + + mimeter-agent-manager + + + 20 + 20 + + + + + + + + com.xiaomi.mone + sautumn-serverless-api + 1.0.0-SNAPSHOT + provided + + + youpin-infra-rpc + com.xiaomi.youpin + + + + + + run.mone + docean + 1.4-SNAPSHOT + + + + + com.xiaomi.faas + hera-api-dubboapi + 1.0.6 + + + + javax.annotation + javax.annotation-api + 1.3.2 + provided + + + + run.mone + docean-plugin-rpc + 1.4-SNAPSHOT + provided + + + + run.mone + mimeter-dashboard-api + 1.0.0-SNAPSHOT + + + com.xiaomi.youpin + youpin-infra-rpc + + + + + + com.xiaomi.mone + mimonitor-api + 1.0-SNAPSHOT + + + + com.xiaomi.youpin + tesla-k8s-proxy-api + 1.0.0-SNAPSHOT + + + + run.mone + struct + 1.4-SNAPSHOT + + + + run.mone + mimeter-api + 1.0-SNAPSHOT + + + + run.mone + mimeter-dashboard-api + 1.0.0-SNAPSHOT + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.1 + + + + + package + + shade + + + + + + + + \ No newline at end of file diff --git a/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/DataStatService.java b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/DataStatService.java new file mode 100644 index 000000000..a9fcfd4f5 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/DataStatService.java @@ -0,0 +1,556 @@ +package run.mone.mimeter.agent.manager; + +import com.xiaomi.faas.func.api.PrometheusService; +import com.xiaomi.faas.func.domain.MimeterApiDetailRes; +import com.xiaomi.youpin.docean.anno.Service; +import com.xiaomi.youpin.infra.rpc.Result; +import lombok.extern.slf4j.Slf4j; +import org.apache.dubbo.annotation.DubboReference; +import run.mone.mimeter.dashboard.bo.common.EmitterTypeEnum; +import run.mone.mimeter.dashboard.bo.statistics.ApiStatistics; +import run.mone.mimeter.dashboard.bo.statistics.ErrorTypeAnalysis; +import run.mone.mimeter.dashboard.bo.statistics.TotalStatAnalysisEvent; +import run.mone.mimeter.dashboard.service.BenchBroadcastService; +import run.mone.mimeter.engine.agent.bo.data.AgentReq; +import run.mone.mimeter.engine.agent.bo.stat.SceneTotalCountContextDTO; +import run.mone.mimeter.engine.agent.bo.stat.TotalCounterStatistic; +import run.mone.mimeter.engine.agent.bo.task.DagTaskRps; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.LongAdder; + +import static common.Const.*; +import static common.Const.API_REQ_FAIL; + +@Service +@Slf4j +public class DataStatService { + + @DubboReference(check = false, group = "${mimeter.dashboard.dubbo.group}", version = "${mimeter.dashboard.dubbo.version}", interfaceClass = BenchBroadcastService.class, timeout = 20000) + private BenchBroadcastService benchBroadcastService; + + /** + * 监控打点数据服务 + */ + @DubboReference(check = false, interfaceClass = PrometheusService.class, timeout = 5000) + private PrometheusService prometheusService; + + /** + * <"reportId",{}> + */ + private final ConcurrentMap totalCountStatistic = new ConcurrentHashMap<>(); + + public void processTotalCountCtxEvent(AgentReq req) throws InterruptedException { + //处理回调过来的返回数据 + SceneTotalCountContextDTO totalCountContextDTO = req.getTotalCountContextDTO(); + + //get p95 p99 data from remote + Map p95Map = new HashMap<>(); + Map p99Map = new HashMap<>(); + try { + Result pRtData = prometheusService.getMimeterApiDetail(String.valueOf(totalCountContextDTO.getSceneId()), totalCountContextDTO.getSceneType()); + log.debug("get remote p99 p95,scene id:{},type:{}, data :{}", totalCountContextDTO.getSceneId(), totalCountContextDTO.getSceneType(), pRtData.getData()); + if (pRtData.getCode() == 0 || pRtData.getData() != null) { + MimeterApiDetailRes res = (MimeterApiDetailRes) pRtData.getData(); + if (res.getP95() != null) { + res.getP95().forEach((k, v) -> { + if (k.contains(":")) { + Integer apiId = Integer.valueOf(k.substring(k.indexOf(":") + 1)); + double value = Double.parseDouble(v); + p95Map.put(apiId, (int) value); + } + }); + } + if (res.getP99() != null) { + res.getP99().forEach((k, v) -> { + if (k.contains(":")) { + Integer apiId = Integer.valueOf(k.substring(k.indexOf(":") + 1)); + double value = Double.parseDouble(v); + p99Map.put(apiId, (int) value); + } + }); + } + } + } catch (Exception e) { + log.error("get p99 and p95 data from remote failed,cause by:{}", e.getMessage()); + } + log.debug("get p99 p95 data p95:{},p99:{}", p95Map, p99Map); + + if (totalCountStatistic.containsKey(totalCountContextDTO.getReportId())) { + + //总记录中存在本压测任务 + //合并数据 错误数据、rt、tps数据 + //通知 + mergeTotalStatistic(totalCountContextDTO); + notifyTotalStatistic(totalCountContextDTO.getReportId(), p95Map, p99Map, false); + if (totalCountContextDTO.isLastTime()) { + //该压测机结束 + TotalCounterStatistic statistic = totalCountStatistic.get(totalCountContextDTO.getReportId()); + int finishNum = statistic.getFinishAgentNum().addAndGet(1); + if (finishNum >= totalCountContextDTO.getConnectTaskNum() * totalCountContextDTO.getAgentNum()) { + //所有机器执行完成 + notifyTotalStatistic(totalCountContextDTO.getReportId(), p95Map, p99Map, true); + //清除该任务记录 + totalCountStatistic.remove(totalCountContextDTO.getReportId()); + } + } + } else { + totalCountStatistic.put(totalCountContextDTO.getReportId(), new TotalCounterStatistic(new AtomicInteger(), new LongAdder(), new LongAdder(), new LongAdder(), + new LongAdder(), new LongAdder(), new ConcurrentHashMap<>(), new ConcurrentHashMap<>(), new ConcurrentHashMap<>(), new ConcurrentHashMap<>())); + //总记录中存在本压测任务 + //合并数据 + //通知 + mergeTotalStatistic(totalCountContextDTO); + notifyTotalStatistic(totalCountContextDTO.getReportId(), p95Map, p99Map, false); + } + } + + /** + * 合并总体统计数据 + */ + private void mergeTotalStatistic(SceneTotalCountContextDTO countContextDTO) { + String reportId = countContextDTO.getReportId(); + TotalCounterStatistic statistic = this.totalCountStatistic.get(reportId); + + //场景发压比例 + statistic.setRpsRate(countContextDTO.getRpsRate()); + + //合并总请求数 + statistic.getTotalReq().add(countContextDTO.getTotalReq()); + + //合并总处理数 + statistic.getTotalTCount().add(countContextDTO.getTotalTCount()); + + //丢失连接数 + statistic.getLossConnNum().add(countContextDTO.getLossConnNum()); + + //更新链路rps + updateLinkRps(countContextDTO, statistic); + + //合并更新错误统计数据 + mergeErrData(countContextDTO, statistic); + + //合并更新rt、tps相关数据 + mergeRtAndTpsData(countContextDTO, statistic); + + //合并更新各接口统计数据 + mergeApiCountData(countContextDTO, statistic); + + //合并完后替换原先数据 + totalCountStatistic.put(reportId, statistic); + } + + private void updateLinkRps(SceneTotalCountContextDTO countContextDTO, TotalCounterStatistic statistic) { + if (statistic.getReportLinkRps() != null) { + DagTaskRps dagTaskRps = countContextDTO.getDagTaskRps(); + if (statistic.getReportLinkRps().get(dagTaskRps.getLinkId()) != null) { + statistic.getReportLinkRps().get(dagTaskRps.getLinkId()).setRps(dagTaskRps.getRps()); + } else { + statistic.getReportLinkRps().put(dagTaskRps.getLinkId(), dagTaskRps); + } + } + } + + /** + * 合并统计错误数据 + */ + private void mergeErrData(SceneTotalCountContextDTO countContextDTO, TotalCounterStatistic statistic) { + //合并总错误次数 + statistic.getTotalErrReq().add(countContextDTO.getTotalErrReq()); + //合并总错误次数 + statistic.getTotalSuccReq().add(countContextDTO.getTotalSuccReq()); + + //合并错误统计 + //errKey:s_code_errorCode countMap: + //例:<"s_code_404",<2342,10>> or <"cp_id_141242",<2342,10>> + countContextDTO.getCounterMap().forEach((errKey, countMap) -> { + if (statistic.getCounterMap().containsKey(errKey)) { + //已存在该错误类型 + countMap.forEach((apiId, count) -> { + if (statistic.getCounterMap().get(errKey).containsKey(apiId)) { + //已记录该api + statistic.getCounterMap().get(errKey).get(apiId).add(count); + } else { + //尚未记录该api + LongAdder longAdder = new LongAdder(); + longAdder.add(count); + statistic.getCounterMap().get(errKey).put(apiId, longAdder); + } + }); + } else { + //尚不存在该类错误 + ConcurrentHashMap tmpMap = new ConcurrentHashMap<>(); + countMap.forEach((k, v) -> { + LongAdder tmpAdder = new LongAdder(); + tmpAdder.add(v); + tmpMap.put(k, tmpAdder); + }); + statistic.getCounterMap().put(errKey, tmpMap); + } + }); + } + + /** + * 合并接口中rt、tps、rps 数据 + */ + private void mergeRtAndTpsData(SceneTotalCountContextDTO countContextDTO, TotalCounterStatistic statistic) { + //合并接口中rt、tps数据 + //所有接口本次的rt汇总 + CopyOnWriteArrayList tmpAllRt = new CopyOnWriteArrayList<>(); + CopyOnWriteArrayList tmpAllTps = new CopyOnWriteArrayList<>(); + CopyOnWriteArrayList tmpAllRps = new CopyOnWriteArrayList<>(); + + countContextDTO.getApiRtMap().forEach((apiId, typeMap) -> { + if (statistic.getApiRtAndTpsMap().containsKey(apiId)) { + //已存在该api的记录 + AtomicInteger apiAvgRt = new AtomicInteger(); + final int[] apiMaxRt = {0}; + ConcurrentHashMap apiMap = statistic.getApiRtAndTpsMap().get(apiId); + typeMap.forEach((type, list) -> { + if (type.equals(RT_LIST)) { + //rt 列表 + tmpAllRt.addAll(list); + //之前存储的平均值 + int oriAvgRt = apiMap.get(AVG_RT).get(); + AtomicInteger tmpApiTotalRt = new AtomicInteger(); + list.forEach(apiRt -> { + tmpApiTotalRt.addAndGet(apiRt); + if (apiRt >= apiMaxRt[0]) { + apiMaxRt[0] = apiRt; + } + }); + //接口平均rt + if (list.size() != 0) { + if (oriAvgRt != 0) { + apiAvgRt.set((tmpApiTotalRt.get() + oriAvgRt) / (list.size() + 1)); + } else { + apiAvgRt.set(tmpApiTotalRt.get() / list.size()); + } + //更新接口平均rt + apiMap.get(AVG_RT).set(apiAvgRt.get()); + } + //更新接口最大rt + if (apiMaxRt[0] > apiMap.get(MAX_RT).get()) { + apiMap.get(MAX_RT).set(apiMaxRt[0]); + } + } + }); + //平均rps + if (!countContextDTO.isLastTime()) { + int apiAvgRps; + int tmpCount = (countContextDTO.getApiRpsMap().get(apiId)) * countContextDTO.getAgentNum(); + if (apiMap.get(AVG_RPS).get() != 0) { + apiAvgRps = (tmpCount + apiMap.get(AVG_RPS).get()) / 2; + } else { + apiAvgRps = tmpCount; + } + apiMap.get(AVG_RPS).set(apiAvgRps); + + if (tmpCount != 0) { + tmpAllRps.add(tmpCount); + } + //最大rps + if (apiAvgRps > apiMap.get(MAX_RPS).get()) { + apiMap.get(MAX_RPS).set(apiAvgRps); + } + //平均tps + int apiAvgTps; + int tmpTCount = (countContextDTO.getApiTpsMap().get(apiId)) * countContextDTO.getAgentNum(); + if (apiMap.get(AVG_TPS).get() != 0) { + apiAvgTps = (tmpTCount + apiMap.get(AVG_TPS).get()) / 2; + } else { + apiAvgTps = tmpTCount; + } + apiMap.get(AVG_TPS).set(apiAvgTps); + + if (tmpTCount != 0) { + tmpAllTps.add(tmpTCount); + } + //最大tps + if (apiAvgTps > apiMap.get(MAX_TPS).get()) { + apiMap.get(MAX_TPS).set(apiAvgTps); + } + } + } else { + //初始化 + ConcurrentHashMap apiMap = new ConcurrentHashMap<>(); + apiMap.put(AVG_RT, new AtomicInteger(0)); + apiMap.put(MAX_RT, new AtomicInteger(0)); + apiMap.put(AVG_TPS, new AtomicInteger(0)); + apiMap.put(MAX_TPS, new AtomicInteger(0)); + apiMap.put(MAX_RPS, new AtomicInteger(0)); + apiMap.put(AVG_RPS, new AtomicInteger(0)); + statistic.getApiRtAndTpsMap().put(apiId, apiMap); + } + }); + //场景平均rt + int avgRt; + //最大rt + AtomicInteger maxRt = new AtomicInteger(); + AtomicInteger totalRt = new AtomicInteger(0); + tmpAllRt.forEach(rt -> { + totalRt.addAndGet(rt); + if (rt >= maxRt.get()) { + maxRt.set(rt); + } + }); + if (tmpAllRt.size() != 0) { + if (statistic.getAvgRt() != 0) { + avgRt = (totalRt.get() + statistic.getAvgRt()) / (tmpAllRt.size() + 1); + } else { + avgRt = totalRt.get() / tmpAllRt.size(); + } + //更新平均rt + statistic.setAvgRt(avgRt); + } + //更新最大rt + if (maxRt.get() >= statistic.getMaxRt()) { + statistic.setMaxRt(maxRt.get()); + } + + if (!countContextDTO.isLastTime()) { + //场景平均rps + int avgRps; + AtomicInteger maxRps = new AtomicInteger(); + AtomicInteger totalRps = new AtomicInteger(); + if (tmpAllRps.size() != 0) { + tmpAllRps.forEach(rps -> { + totalRps.addAndGet(rps); + if (rps > maxRps.get()) { + maxRps.set(rps); + } + }); + + int tmpAvgRps = totalRps.get() / tmpAllRps.size(); + + if (statistic.getAvgRps() != 0) { + avgRps = (statistic.getAvgRps() + tmpAvgRps) / 2; + } else { + avgRps = tmpAvgRps; + } + statistic.setAvgRps(avgRps); + } + //更新最大rps + if (maxRps.get() > statistic.getMaxRps()) { + statistic.setMaxRps(maxRps.get()); + } + + //场景平均tps + int avgTps; + AtomicInteger maxTps = new AtomicInteger(); + AtomicInteger totalTps = new AtomicInteger(); + if (tmpAllTps.size() != 0) { + tmpAllTps.forEach(tps -> { + totalTps.addAndGet(tps); + if (tps > maxTps.get()) { + maxTps.set(tps); + } + }); + + int tmpAvgTps = totalTps.get() / tmpAllTps.size(); + + if (statistic.getAvgTps() != 0) { + avgTps = (statistic.getAvgTps() + tmpAvgTps) / 2; + } else { + avgTps = tmpAvgTps; + } + statistic.setAvgTps(avgTps); + } + + //更新最大tps + if (maxTps.get() > statistic.getMaxTps()) { + statistic.setMaxTps(maxTps.get()); + } + } + } + + /** + * 合并各接口统计数据 + */ + private void mergeApiCountData(SceneTotalCountContextDTO countContextDTO, TotalCounterStatistic statistic) { + ConcurrentHashMap> apiCountMap = countContextDTO.getApiCountMap(); + apiCountMap.forEach((apiId, countMap) -> { + if (statistic.getApiCountMap().containsKey(apiId)) { + //已记录该接口数据 + countMap.forEach((type, count) -> statistic.getApiCountMap().get(apiId).get(type).add(count)); + } else { + //未记录,初始化第一次 + ConcurrentHashMap apiAdderMap = new ConcurrentHashMap<>(); + countMap.forEach((type, count) -> { + LongAdder longAdder = new LongAdder(); + longAdder.add(count); + apiAdderMap.put(type, longAdder); + }); + statistic.getApiCountMap().put(apiId, apiAdderMap); + } + }); + } + + /** + * 推送错误统计数据 + */ + private void notifyTotalStatistic(String reportId, Map p95Map, Map p99Map, boolean finish) throws InterruptedException { + if (!totalCountStatistic.containsKey(reportId)) { + return; + } + if (finish) { + //等下同时间推送的事件merge完成 + Thread.sleep(100); + } + TotalCounterStatistic totalCounterStatistic = totalCountStatistic.get(reportId); + + TotalStatAnalysisEvent analysisEvent = new TotalStatAnalysisEvent(); + analysisEvent.setFinish(finish); + int totalCount = totalCounterStatistic.getTotalReq().intValue(); + int totalSuccCount = totalCounterStatistic.getTotalSuccReq().intValue(); + int totalErrCount = totalCounterStatistic.getTotalErrReq().intValue(); + + //链路rps + ConcurrentHashMap innerRpsMap = totalCounterStatistic.getReportLinkRps(); + Map dagTaskRpsMap = new HashMap<>(); + innerRpsMap.forEach((linkId,rpsObj) -> dagTaskRpsMap.putIfAbsent(linkId,new run.mone.mimeter.dashboard.bo.task.DagTaskRps(linkId,rpsObj.getTaskId(), rpsObj.getRps()))); + analysisEvent.setLinkToDagTaskRpsMap(dagTaskRpsMap); + + //当前发压比例 + analysisEvent.setRpsRate(totalCounterStatistic.getRpsRate()); + //总请求数 + analysisEvent.setTotalReq(totalCount); + + analysisEvent.setLossConnNum(totalCounterStatistic.getLossConnNum().intValue()); + //业务总处理次数 + analysisEvent.setTotalTCount(totalCounterStatistic.getTotalTCount().intValue()); + //总成功数 + analysisEvent.setTotalSuccReq(totalSuccCount); + //总错误数 + analysisEvent.setTotalErrReq(totalErrCount); + //总平均rt + analysisEvent.setAvgRt(totalCounterStatistic.getAvgRt()); + //总最大rt + analysisEvent.setMaxRt(totalCounterStatistic.getMaxRt()); + //总平均tps + analysisEvent.setAvgTps(totalCounterStatistic.getAvgTps()); + //总最大tps + analysisEvent.setMaxTps(totalCounterStatistic.getMaxTps()); + //总最大rps + analysisEvent.setMaxRps(totalCounterStatistic.getMaxRps()); + //总平均rps + analysisEvent.setAvgRps(totalCounterStatistic.getAvgRps()); + + double totalErrRate = 0; + if (totalCount != 0) { + totalErrRate = (1.0 * totalErrCount / totalCount) * 100d; + } + + analysisEvent.setTotalErrRate(format2(totalErrRate)); + + double totalSuccRate = 0; + if (totalCount != 0) { + totalSuccRate = (1.0 * totalSuccCount / totalCount) * 100d; + } + analysisEvent.setTotalSuccRate(format2(totalSuccRate)); + + List errorTypeAnalyses = new ArrayList<>(); + totalCounterStatistic.getCounterMap().forEach((errFlag, apiMap) -> { + ErrorTypeAnalysis analysis = new ErrorTypeAnalysis(); + + String[] flagAndCode = errFlag.split("_", 2); + + if (errFlag.startsWith(ERR_STATUS_CODE_PREFIX)) { + //错误状态码 + analysis.setErrorType(ERR_STATUS_CODE_TYPE); + analysis.setErrorCode(Integer.parseInt(flagAndCode[1])); + + } else if (errFlag.startsWith(ERR_CHECKPOINT_PREFIX)) { + //检查点规则 + analysis.setErrorType(ERR_CHECKPOINT_TYPE); + //检查点id + analysis.setCheckPointId(Integer.valueOf(flagAndCode[1])); + } else if (errFlag.equals(ERR_STATUS_DUBBO_PREFIX)) { + //检查点规则 + analysis.setErrorType(ERR_DUBBO_CALL_TYPE); + //检查点id + analysis.setCheckPointId(500); + } + //本错误总数 + AtomicInteger thisErrTotalCount = new AtomicInteger(); + AtomicInteger mostErrApiId = new AtomicInteger(); + final int[] tmpMostErrApiCount = {0}; + Map errInApis = new HashMap<>(); + apiMap.forEach((apiId, count) -> { + errInApis.put(apiId, count.intValue()); + thisErrTotalCount.addAndGet(count.intValue()); + + if (count.intValue() >= tmpMostErrApiCount[0]) { + tmpMostErrApiCount[0] = count.intValue(); + mostErrApiId.set(apiId); + } + }); + //本错误占比 + double thisErrRate = 0; + if (totalErrCount != 0) { + thisErrRate = (1.0 * thisErrTotalCount.get() / totalErrCount) * 100d; + } + + analysis.setErrRate(format2(thisErrRate)); + analysis.setMostErrApi(mostErrApiId.get()); + analysis.setErrInApis(errInApis); + + errorTypeAnalyses.add(analysis); + }); + //错误分析数据 + analysisEvent.setErrorTypeAnalyses(errorTypeAnalyses); + + List apiStatisticsList = new ArrayList<>(); + ConcurrentHashMap> apiCountMaps = totalCounterStatistic.getApiCountMap(); + + totalCounterStatistic.getApiRtAndTpsMap().forEach((apiId, apiMap) -> { + ApiStatistics apiStatistics = new ApiStatistics(); + apiStatistics.setApiId(apiId); + apiStatistics.setAvgRt(apiMap.getOrDefault(AVG_RT, new AtomicInteger(0)).get()); + apiStatistics.setMaxRt(apiMap.getOrDefault(MAX_RT, new AtomicInteger(0)).get()); + //remote 的两个数据p95 p99 + apiStatistics.setP95Rt(p95Map.getOrDefault(apiId, 0)); + apiStatistics.setP99Rt(p99Map.getOrDefault(apiId, 0)); + + apiStatistics.setAvgTps(apiMap.getOrDefault(AVG_TPS, new AtomicInteger(0)).get()); + apiStatistics.setMaxTps(apiMap.getOrDefault(MAX_TPS, new AtomicInteger(0)).get()); + apiStatistics.setMaxRps(apiMap.getOrDefault(MAX_RPS, new AtomicInteger(0)).get()); + apiStatistics.setAvgRps(apiMap.getOrDefault(AVG_RPS, new AtomicInteger(0)).get()); + + ConcurrentHashMap apiCountMap = apiCountMaps.get(apiId); + if (apiCountMap != null) { + int apiTotal = apiCountMap.getOrDefault(API_REQ_TOTAL_R, new LongAdder()).intValue(); + int apiTransTotal = apiCountMap.getOrDefault(API_REQ_TOTAL_T, new LongAdder()).intValue(); + int apiSucc = apiCountMap.getOrDefault(API_REQ_SUCC, new LongAdder()).intValue(); + int apiFail = apiCountMap.getOrDefault(API_REQ_FAIL, new LongAdder()).intValue(); + apiStatistics.setReqTotal(apiTotal); + apiStatistics.setTansTotal(apiTransTotal); + apiStatistics.setReqSucc(apiSucc); + apiStatistics.setReqFail(apiFail); + + double apiSuccRate = 0; + if (apiTotal != 0) { + apiSuccRate = (1.0 * apiSucc / apiTotal) * 100d; + } + apiStatistics.setSuccRate(format2(apiSuccRate)); + } + apiStatisticsList.add(apiStatistics); + }); + analysisEvent.setApiStatisticsList(apiStatisticsList); + //推送错误统计数据 + this.benchBroadcastService.notifyEvent(EmitterTypeEnum.TOTAL_STAT_ANALYSIS, reportId, analysisEvent); + } + + public static String format2(double value) { + /* + * %.2f % 表示 小数点前任意位数 2 表示两位小数 格式后的结果为 f 表示浮点型 + */ + return new Formatter().format("%.2f", value).toString(); + } + + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/ManagerHandler.java b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/ManagerHandler.java new file mode 100644 index 000000000..1e9e4b9c8 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/ManagerHandler.java @@ -0,0 +1,179 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.agent.manager; + +import com.xiaomi.youpin.docean.anno.Controller; +import com.xiaomi.youpin.docean.anno.RequestMapping; +import lombok.extern.slf4j.Slf4j; +import org.nutz.dao.impl.NutDao; +import run.mone.mimeter.agent.manager.bo.MibenchTask; +import run.mone.mimeter.dashboard.bo.agent.*; +import run.mone.mimeter.engine.agent.bo.data.AgentInfoDTO; +import run.mone.mimeter.engine.agent.bo.data.HttpResult; +import run.mone.mimeter.engine.agent.bo.task.CancelType; +import run.mone.mimeter.engine.agent.bo.task.ChangeQpsReq; +import run.mone.mimeter.engine.agent.bo.task.Task; + +import javax.annotation.Resource; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; + +/** + * @author goodjava@qq.com + * @date 2022/5/19 + */ +@Controller +@Slf4j +public class ManagerHandler { + + @Resource + private ManagerService managerService; + + @Resource(name = "$daoName:mibench_st_db", description = "mysql") + private NutDao dao; + + @RequestMapping(path = "/version") + public String version() { + return "2022-10-1 1.0.0"; + } + + @RequestMapping(path = "/system") + public String system() { + return System.getProperty("os.name"); + } + + + @RequestMapping(path = "/test") + public int test() { + MibenchTask dagTask = dao.fetch(MibenchTask.class, 61381); + log.info("test log info:{}",dagTask); + return dagTask.getIncreasePercent(); + } + + /** + * 获取agent 列表 + * @return + */ + @RequestMapping(path = "/agent/list") + public List agentList() { + return managerService.agents().stream().map(it -> { + AgentInfoDTO info = new AgentInfoDTO(); + info.setIp(it.remoteAddress().toString()); + return info; + }).collect(Collectors.toList()); + } + + @RequestMapping(path = "/agent/addr/list") + public List>> agentAddrList() { + Map> result = new HashMap<>(); + List ipList = new ArrayList<>(); + + managerService.agents().forEach(it -> { + try { + String addrStr = it.remoteAddress().toString(); + String[] addrArr = addrStr.split("/"); + String ipAndPort = addrArr[addrArr.length - 1]; + String[] ipAndPortArr = ipAndPort.split(":"); + ipList.add(ipAndPortArr[0] + ":8080"); + } catch (Exception ex) { + log.error("[ManagerHandler]agentAddrList parse agent's ip and port error; " + ex.getMessage()); + } + }); + result.put("targets", ipList); + return Collections.singletonList(result); + } + + /** + * 手动直接修改单台发压机hosts文件 + */ + @RequestMapping(path = "/manual/edit/hosts",timeout = 10000) + public HttpResult manualEditHosts(HostForAgentReq req) throws Exception { + return managerService.manualEditHosts(req); + } + + /** + * 修改发压机hosts文件 + */ + @RequestMapping(path = "/edit/hosts",timeout = 10000) + public HttpResult editHosts(DomainApplyReq domainApplyReq) throws Exception { + return managerService.editHosts(domainApplyReq); + } + + /** + * 同步发压机hosts域名绑定 + */ + @RequestMapping(path = "/sync/hosts",timeout = 10000) + public HttpResult syncHosts(SyncHostsReq syncHostsReq) throws Exception { + return managerService.syncHosts(syncHostsReq); + } + + /** + * 删除发压机hosts文件某项域名绑定 + */ + @RequestMapping(path = "/del/hosts",timeout = 10000) + public HttpResult delHosts(DelHostForAgentsReq req) throws Exception { + return managerService.delHosts(req); + } + + /** + * 删除发压机hosts文件某项域名绑定 + */ + @RequestMapping(path = "/load/hosts",timeout = 10000) + public HttpResult loadHosts(LoadHostsFileReq req) throws Exception { + return managerService.loadHostsFile(req); + } + + /** + * 基于场景创建执行任务 + */ + @RequestMapping(path = "/submit/task",timeout = 10000) + public HttpResult submitTask(Task task) throws Exception { + return managerService.submitTask(task); + } + + /** + * 手动停止某次压测任务 + * + */ + @RequestMapping(path = "/cancel/task") + public HttpResult cancelTask(Task task) { + task.setCancelType(CancelType.Manual.code); + return managerService.cancelTask(task); + } + + /** + * 调整指定场景某次压测任务的qps + * + */ + @RequestMapping(path = "/task/manualUpdateQps") + public HttpResult manualUpdateQps(ChangeQpsReq req) { + return managerService.manualUpdateQps(req); + } + + @RequestMapping(path = "/view") + public String view() { + try { + return new String(Files.readAllBytes(Paths.get("/Users/dongzhenxing/Documents/Mi/Projects/mibench-engine/mibench-agent-manager/src/main/resources/upload.html"))); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/ManagerService.java b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/ManagerService.java new file mode 100644 index 000000000..078233a7c --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/ManagerService.java @@ -0,0 +1,1770 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.agent.manager; + +import com.google.common.collect.Lists; +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; +import com.google.gson.reflect.TypeToken; +import com.xiaomi.data.push.context.AgentContext; +import com.xiaomi.data.push.rpc.RpcServer; +import com.xiaomi.data.push.rpc.netty.AgentChannel; +import com.xiaomi.data.push.rpc.protocol.RemotingCommand; +import com.xiaomi.data.push.schedule.task.graph.GraphTaskContext; +import com.xiaomi.data.push.schedule.task.graph.TaskEdgeData; +import com.xiaomi.data.push.schedule.task.graph.TaskVertexData; +import common.Const; +import common.Util; +import org.apache.dubbo.annotation.DubboReference; +import org.nutz.trans.Trans; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; +import com.xiaomi.youpin.docean.anno.Service; +import org.nutz.dao.impl.NutDao; +import run.mone.mimeter.agent.manager.bo.MibenchTask; +import run.mone.mimeter.dashboard.bo.agent.*; +import run.mone.mimeter.dashboard.bo.common.Constants; +import run.mone.mimeter.dashboard.bo.dataset.DatasetLineNum; +import run.mone.mimeter.dashboard.bo.scene.*; +import run.mone.mimeter.dashboard.bo.sceneapi.ApiTrafficInfo; +import run.mone.mimeter.dashboard.bo.sceneapi.ApiX5Info; +import run.mone.mimeter.dashboard.bo.sceneapi.FormParamValue; +import run.mone.mimeter.dashboard.bo.task.BenchIncreaseModeEnum; +import run.mone.mimeter.dashboard.bo.task.BenchModeEnum; +import run.mone.mimeter.dashboard.service.DatasetInfoSercice; +import run.mone.mimeter.dashboard.service.SceneInfoService; +import run.mone.mimeter.engine.agent.bo.MibenchCmd; +import run.mone.mimeter.engine.agent.bo.data.*; +import run.mone.mimeter.engine.agent.bo.task.*; + +import javax.annotation.Resource; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static common.Const.*; +import static run.mone.mimeter.dashboard.bo.common.Constants.CONTENT_TYPE_APP_FORM2; + +/** + * @author goodjava@qq.com + * @author dongzhenxing + * @date 2022/5/19 + */ +@Service +public class ManagerService { + + @Resource(name = "rpcServer") + private RpcServer rpcServer; + + @Resource(name = "$daoName:mibench_st_db", description = "mysql") + private NutDao dao; + + @Resource + private SlaService slaService; + + private final ExecutorService pool = Executors.newVirtualThreadPerTaskExecutor(); + + private static final Logger log = LoggerFactory.getLogger(ManagerService.class); + + private static final int DEFAULT_SCENE_MAX_QPS = 500; + + private static final Gson gson = Util.getGson(); + + private static final Pattern EL_PATTERN_MUTI = Pattern.compile("\\$\\{([^}]*)\\}"); + + private final Random random = new Random(System.currentTimeMillis()); + + private final ConcurrentHashMap rateUpWorkingTaskCache = new ConcurrentHashMap<>(); + + @DubboReference(check = false, group = "${mimeter.dashboard.dubbo.group}", version = "${mimeter.dashboard.dubbo.version}", interfaceClass = SceneInfoService.class, timeout = 3000) + private SceneInfoService sceneInfoService; + + @DubboReference(check = false, group = "${mimeter.dashboard.dubbo.group}", version = "${mimeter.dashboard.dubbo.version}", interfaceClass = DatasetInfoSercice.class, timeout = 10000) + private DatasetInfoSercice datasetInfoSercice; + + public List agents() { + return new ArrayList<>(AgentContext.ins().map.values()); + } + + public HttpResult submitTask(Task task) throws Exception { + int num = AgentContext.ins().list().size(); + if (num <= 0) { + return HttpResult.fail(500, "agent num <= 0", "暂无可用的发压机"); + } + List agentList = new ArrayList<>(AgentContext.ins().map.values()); + //构建插入任务 + MibenchTask mibenchTask = buildMibenchTask(task); + + //本场景同一次任务标记 + if (task.getReportId() == null) { + task.setReportId(""); + } + //默认不开启录制流量使用 + task.setEnableTraffic(false); + mibenchTask.setReportId(task.getReportId()); + + switch (task.getSubmitTaskType()) { + case Const.SINGLE_API_DEBUG -> { + return singleApiDebug(task, mibenchTask, agentList); + } + case Const.SCENE_DEBUG -> { + return sceneDebug(task, mibenchTask, agentList); + } + case Const.SINGLE_BENCH -> { + return sceneBench(task, mibenchTask, agentList); + } + default -> { + return HttpResult.fail(500, "error param task type", "错误的任务类型"); + } + } + } + + /** + * 单接口调试 + */ + private HttpResult singleApiDebug(Task task, MibenchTask mibenchTask, List agentList) throws Exception { + //单接口调试 + task.setDebug(true); + + DebugSceneApiInfoReq apiInfo = task.getApiInfo(); + if (apiInfo.getApiType() == Const.API_TYPE_HTTP) { + //处理http调试参数 + try { + httpDebug(apiInfo, task, mibenchTask); + } catch (Exception e) { + return HttpResult.fail(500, "fail", "invalid http param"); + } + } else if (apiInfo.getApiType() == Const.API_TYPE_DUBBO) { + //处理dubbo调试参数 + try { + dubboDebug(apiInfo, task, mibenchTask); + } catch (Exception e) { + log.error("singleApiDebug dubbo error:{}",e); + return HttpResult.fail(500, "fail", "invalid dubbo param"); + } + } + int index = random.nextInt(agentList.size()); + AgentChannel c = agentList.get(index); + agentList.clear(); + agentList.add(c); + + //记录单接口测试任务 + mibenchTask.setAgentNum(agentList.size()); + mibenchTask.setQps(1); + SubmitTaskRes res = null; + try { + Trans.begin(); + dao.insert(mibenchTask); + task.setTime(1); + task.setQps(1); + task.setId(mibenchTask.getId()); + task.setAgentNum(agentList.size()); + + AgentReq agentReq = new AgentReq(); + agentReq.setCmd(AgentReq.SUBMIT_TASK_CMD); + agentReq.setTask(task); + + //发送单接口测试任务 + List agentIps = new ArrayList<>(agentList.stream().map(AgentChannel::getRemoteAddr).toList()); + TaskResult tr = syncCallAgent(agentList.get(0), agentReq); + res = new SubmitTaskRes(task.getReportId(), tr, new HashMap<>(), agentIps); + Trans.commit(); + } catch (Exception e) { + Trans.rollback(); + log.error("single debug error,e:{}", e.getMessage()); + } finally { + Trans.close(); + } + return HttpResult.success(gson.toJson(res)); + } + + + /** + * 场景调试 + */ + private HttpResult sceneDebug(Task task, MibenchTask mibenchTask, List agentList) throws Exception { + //场景调试及场景压测,都需要先基于场景数据构建任务图 + //场景数据 + SceneDTO sceneInfo = sceneInfoService.getSceneByID(task.getSceneId()).getData(); + if (sceneInfo == null) { + log.error("can not get scene info,scene id:{}", task.getSceneId()); + return HttpResult.fail(500, "faild", "can not get scene info"); + } + List agentDTOList = sceneInfo.getAgentDTOList(); + if (agentDTOList == null) { + agentDTOList = new ArrayList<>(); + } + //过滤不可用机器 + agentDTOList = agentDTOList.stream().filter(AgentDTO::getEnable).collect(Collectors.toList()); + + //绑定的机器ip + List bindIps = agentDTOList.stream().filter(AgentDTO::getEnable).map(AgentDTO::getIp).toList(); + + if (bindIps.size() == 0) { + return HttpResult.fail(500, "agent num <= 0", "暂无可用的发压机"); + } + //过滤未启用链路 + filterLink(sceneInfo); + + //链路id与生成的dag 任务id映射 + Map linkTaskIdMap = new HashMap<>(sceneInfo.getSerialLinkDTOs().size()); + + //场景调试 + task.setDebug(true); + task.setType(TaskType.dag); + //施压时间 + task.setTime(sceneInfo.getBenchTime()); + //场景任务整体超时时间 + task.setTimeout(sceneInfo.getRequestTimeout()); + task.setQps(1); + + //自定义成功状态码 + task.setSuccessCode(sceneInfo.getSuccessCode()); + task.setConnectTaskNum(sceneInfo.getSerialLinkDTOs().size()); + List tmpAgentList; + + //有绑定机器 + tmpAgentList = agentList.stream().filter(channel -> { + //过滤在绑定机器内的ip + return bindIps.contains(channel.getRemoteAddr().substring(0, channel.getRemoteAddr().indexOf(":"))); + }).toList(); + + if (tmpAgentList.size() == 0) { + return HttpResult.fail(500, "agent num <= 0", "failed"); + } + int index = random.nextInt(tmpAgentList.size()); + + AgentChannel c = tmpAgentList.get(index); + agentList.clear(); + agentList.add(c); + + //图任务 + mibenchTask.setTaskType(TaskType.dag.code); + mibenchTask.setAgentNum(agentList.size()); + mibenchTask.setQps(1); + mibenchTask.setTime(sceneInfo.getBenchTime()); + mibenchTask.setConnectTaskNum(sceneInfo.getSerialLinkDTOs().size()); + try { + //记录场景调试任务 + Trans.begin(); + //build && 发送场景调试请求 + buildAgentReqs(sceneInfo, task, agentList.size(), mibenchTask, linkTaskIdMap).forEach(agentReq -> { + try { + callTaskAgents(agentList, agentReq); + } catch (Exception e) { + log.error("call agent error:{}", e.getMessage()); + } + }); + Trans.commit(); + } catch (Exception e) { + Trans.rollback(); + log.error("scene debug error,e:{}", e.getMessage()); + } finally { + Trans.close(); + } + List agentIps = new ArrayList<>(agentList.stream().map(AgentChannel::getRemoteAddr).toList()); + return HttpResult.success(gson.toJson(new SubmitTaskRes(task.getReportId(), new TaskResult(), new HashMap<>(), agentIps))); + } + + /** + * 场景压测 + */ + private HttpResult sceneBench(Task task, MibenchTask mibenchTask, List agentList) throws Exception { + //场景调试及场景压测,都需要先基于场景数据构建任务图 + //场景数据 + task.setType(TaskType.dag); + SceneDTO sceneInfo = sceneInfoService.getSceneByID(task.getSceneId()).getData(); + if (sceneInfo == null) { + log.error("can not get scene info,scene id:{}", task.getSceneId()); + return HttpResult.fail(500, "faild", "can not get scene info"); + } + if (sceneInfo.getSceneType() == SCENE_TYPE_HTTP) { + task.setSceneType(TaskType.http); + } else if (sceneInfo.getSceneType() == SCENE_TYPE_DUBBO) { + task.setSceneType(TaskType.dubbo); + } + //过滤链路 + filterLink(sceneInfo); + + //链路id与生成的dag 任务id映射 + Map linkTaskIdMap = new HashMap<>(sceneInfo.getSerialLinkDTOs().size()); + + task.setSceneId(sceneInfo.getId()); + List agentDTOList = sceneInfo.getAgentDTOList(); + if (agentDTOList == null) { + agentDTOList = new ArrayList<>(); + } + //过滤不可用机器 + agentDTOList = agentDTOList.stream().filter(AgentDTO::getEnable).collect(Collectors.toList()); + + //绑定的机器ip + List bindIps = agentDTOList.stream().filter(AgentDTO::getEnable).map(AgentDTO::getIp).toList(); + + if (bindIps.size() == 0) { + return HttpResult.fail(500, "agent num <= 0", "暂无可用的发压机"); + } + + List tmpAgentList; + + //有绑定机器 + tmpAgentList = agentList.stream().filter(channel -> { + //过滤在绑定机器内的ip + return bindIps.contains(channel.getRemoteAddr().substring(0, channel.getRemoteAddr().indexOf(":"))); + }).toList(); + + if (tmpAgentList.size() == 0) { + return HttpResult.fail(500, "agent num <= 0", "请绑定至少一台压测机"); + } + + if (sceneInfo.getMaxBenchQps() < 50) { + int index = random.nextInt(tmpAgentList.size()); + + AgentChannel c = tmpAgentList.get(index); + agentList.clear(); + agentList.add(c); + } else { + agentList = tmpAgentList; + } + int benchAgentNum = agentList.size(); + //校验本场景所使用的数据源行数是否大于压测机数量 + AtomicBoolean ok = new AtomicBoolean(true); + List datasetLineNums = datasetInfoSercice.getLineNumBySceneId(task.getSceneId()).getData(); + datasetLineNums.forEach(datasetLineNum -> { + if (datasetLineNum.getFileRaw() < benchAgentNum){ + ok.set(false); + } + }); + if (!ok.get()){ + return HttpResult.fail(500, "dataset line num < agent num", "数据源文件行数必须大于等于所使用的发压机数量"); + } + + //为图任务 + mibenchTask.setTaskType(TaskType.dag.code); + mibenchTask.setAgentNum(agentList.size()); + mibenchTask.setConnectTaskNum(sceneInfo.getSerialLinkDTOs().size()); + + //总qps,即 + int sceneMaxQps; + if (sceneInfo.getMaxBenchQps() == null) { + sceneMaxQps = DEFAULT_SCENE_MAX_QPS; + } else { + sceneMaxQps = sceneInfo.getMaxBenchQps(); + } + mibenchTask.setQps(sceneMaxQps); + + task.setTime(sceneInfo.getBenchTime()); + //场景任务整体超时时间 + task.setTimeout(sceneInfo.getRequestTimeout()); + task.setLogRate(sceneInfo.getLogRate()); + task.setConnectTaskNum(sceneInfo.getSerialLinkDTOs().size()); + task.setAgentNum(agentList.size()); + //自定义成功状态码 + task.setSuccessCode(sceneInfo.getSuccessCode()); + + try { + //记录场景压测任务 + Trans.begin(); + //构建任务图请求 + List reqList = buildAgentReqs(sceneInfo, task, agentList.size(), mibenchTask, linkTaskIdMap); + + //分发链路任务 + List finalTmpAgentList = agentList; + log.info("use agent num:{},agent list:{}", finalTmpAgentList.size(), finalTmpAgentList); + reqList.forEach(agentReq -> callTaskAgents(finalTmpAgentList, agentReq)); + Trans.commit(); + + //更新场景状态为运行中 + sceneInfoService.updateSceneStatus(task.getSceneId(), TaskStatus.Running.code); + + //开启SLA实时通知 + try { + slaService.processSlaNotifyTask(sceneInfo, task.getReportId()); + } catch (Exception e) { + log.warn("slaService start failed,cause by:{}", e.getMessage()); + } + } catch (Exception e) { + Trans.rollback(); + log.error("scene bench error,e:{}", e.getMessage()); + } finally { + Trans.close(); + } + List agentIps = new ArrayList<>(agentList.stream().map(AgentChannel::getRemoteAddr).toList()); + return HttpResult.success(gson.toJson(new SubmitTaskRes(task.getReportId(), new TaskResult(), linkTaskIdMap, agentIps))); + } + + /** + * 取消任务 + */ + public HttpResult cancelTask(Task task) { + AgentReq ar = new AgentReq(); + ar.setCmd(AgentReq.CANCEL_TASK_CMD); + ar.setTask(task); + RemotingCommand req = RemotingCommand.createGsonRequestCommand(MibenchCmd.TASK, ar); + if (AgentContext.ins().list().size() != 0) { + AgentContext.ins().list().forEach(c -> rpcServer.tell(c.getChannel(), req)); + } else { + //没有agent机器,直接更新场景及任务状态 + directStopTask(task.getIds()); + } + return HttpResult.success("ok"); + } + + private void directStopTask(List taskIds) { + taskIds.forEach(taskId -> { + MibenchTask task = dao.fetch(MibenchTask.class, taskId); + if (task != null) { + task.setState(TaskStatus.Stopped.code); + task.setUtime(System.currentTimeMillis()); + dao.update(task); + } + }); + } + + /** + * 图压测任务开始执行后的回调通知 + * 百分比递增模式下,周期性执行rps递增 + */ + public void taskRunningNotify(MibenchTask dagTask) { + if (rateUpWorkingTaskCache.containsKey(dagTask.getId())) { + return; + } + //同一个链路任务只接收一次通知,仅用一个协程进行调度 + rateUpWorkingTaskCache.put(dagTask.getId(), true); + + log.info("manualUpdateQps in bench mode:{},increase mode :{},dagTask increasePercent:{}", dagTask.getBenchMode(), dagTask.getIncreaseMode(), dagTask.getIncreasePercent()); + if (dagTask.getBenchMode() == BenchModeEnum.RPS.code && dagTask.getIncreaseMode() == BenchIncreaseModeEnum.PERCENT_INCREASE.code) { + //百分比递增默认比例 + List increasePercentList = Lists.newArrayList(5, 10, 20, 25, 50); + // RPS 百分比递增模式 + pool.submit(() -> { + long start = System.currentTimeMillis(); + int benchTime = dagTask.getTime(); + int percentNum = dagTask.getIncreasePercent(); + if (benchTime >= 60 && increasePercentList.contains(percentNum)) { + int curRps = dagTask.getOriginQps(); + double percent = percentNum / 100d; + //调度周期 ms + long schedule = (long) (benchTime * percent * 1000); + //最大、最小RPS差值 + int distance = dagTask.getMaxQps() - curRps; + //差值100以内不处理 + if (distance >= 100) { + //步长 + int step = (int) Math.ceil(distance * percent); + log.info("manualUpdateQps step:{},schedule:{}", step, schedule); + try { + //起始rps需要先跑一个周期 + Thread.sleep(schedule); + } catch (InterruptedException e) { + log.error("taskRunningNotify thread sleep failed,cause by:{}", e.getMessage()); + } + while (System.currentTimeMillis() - start < benchTime * 1000L) { + //在压测时间内 + curRps += step; + if (curRps > dagTask.getMaxQps()) { + curRps = dagTask.getMaxQps(); + } + if (curRps <= dagTask.getMaxQps()) { + DagTaskRps dagTaskRps = new DagTaskRps(dagTask.getReportId(), dagTask.getSerialLinkId(), dagTask.getId(), curRps); + List dagTaskRpsList = Lists.newArrayList(dagTaskRps); + ChangeQpsReq req = new ChangeQpsReq(dagTaskRpsList); + //发送调速命令 + log.info("manualUpdateQps cur rps:{}", curRps); + this.manualUpdateQps(req); + try { + Thread.sleep(schedule); + } catch (InterruptedException e) { + log.error("taskRunningNotify thread sleep failed,cause by:{}", e.getMessage()); + } + if (curRps == dagTask.getMaxQps()) { + break; + } + } else { + break; + } + } + //最多维护到压测时间截止 + rateUpWorkingTaskCache.remove(dagTask.getId()); + } + } + }); + } + } + + public HttpResult manualUpdateQps(ChangeQpsReq qpsReq) { + AgentReq ar = new AgentReq(); + ar.setCmd(AgentReq.CHANGE_TASK_QPS); + ar.setChangeQpsReq(qpsReq); + RemotingCommand req = RemotingCommand.createGsonRequestCommand(MibenchCmd.TASK, ar); + AgentContext.ins().list().forEach(c -> rpcServer.tell(c.getChannel(), req)); + return HttpResult.success("ok"); + } + + private void httpDebug(DebugSceneApiInfoReq apiInfo, Task task, MibenchTask mibenchTask) throws Exception { + //http单接口测试任务 + HttpData httpData = new HttpData(); + httpData.setUrl(apiInfo.getApiUrl()); + + httpData.setTspAuthInfoDTO(apiInfo.getApiTspAuth()); + + if (apiInfo.getOutputParamInfosStr() != null) { + List outputParams = new ArrayList<>(); + List outputParamList = gson.fromJson(apiInfo.getOutputParamInfosStr(), new TypeToken>() { + }.getType()); + // key 表达式,需要取上游接口的出参定义合成 + outputParamList.forEach(oParam -> outputParams.add(new OutputParam(oParam.getOrigin(), oParam.getParamName(), oParam.getParseExpr()))); + + httpData.setOutputParams(new CopyOnWriteArrayList<>(outputParams)); + } + if (apiInfo.getCheckPointInfoListStr() != null) { + List checkPointInfos = gson.fromJson(apiInfo.getCheckPointInfoListStr(), new TypeToken>() { + }.getType()); + httpData.setCheckPointInfoList(new CopyOnWriteArrayList<>(checkPointInfos)); + } + List types = new ArrayList<>(); + ConcurrentHashMap headerMap; + //请求头 + try { + headerMap = gson.fromJson(apiInfo.getApiHeader(), new TypeToken>() { + }.getType()); + + if (headerMap == null) { + headerMap = new ConcurrentHashMap<>(); + } + headerMap.put("User-Agent", Const.MIMETER_UA_KEY); + if (apiInfo.getRequestMethod() == Const.HTTP_REQ_GET || apiInfo.getContentType().equals(Constants.CONTENT_TYPE_APP_FORM) || apiInfo.getContentType().equals(CONTENT_TYPE_APP_FORM2)) { + List paramValues; + //get + if (apiInfo.getRequestMethod() == Const.HTTP_REQ_GET) { + httpData.setMethod(Const.HTTP_GET); + } else { + if (apiInfo.getContentType().equals(CONTENT_TYPE_APP_FORM2)) { + headerMap.put("Content-type", CONTENT_TYPE_APP_FORM); + } else { + headerMap.put("Content-type", apiInfo.getContentType()); + } + httpData.setMethod(Const.HTTP_POST); + } + //表单/get 类型参数 + //例:[{"paramKey": "v1", "paramValue": "v2"}] + CopyOnWriteArrayList tmpValues = new CopyOnWriteArrayList<>(); + if (null != apiInfo.getRequestBody()) { + paramValues = gson.fromJson(apiInfo.getRequestBody(), new TypeToken>() { + }.getType()); + paramValues.forEach(paramValue -> { + ParamType paramType = new ParamType(ParamTypeEnum.primary, paramValue.getParamKey()); + types.add(paramType); + tmpValues.add(paramValue.getParamValue()); + }); + } + httpData.initTypeList(types); + //["a","b"...] + httpData.getParams().addAll(new CopyOnWriteArrayList(tmpValues)); + } else { + headerMap.put("Content-type", apiInfo.getContentType()); + httpData.setMethod(Const.HTTP_POST); + List objectList; + // 处理参数 + if (null != apiInfo.getRequestBody()) { + objectList = gson.fromJson(apiInfo.getRequestBody(), new TypeToken>() { + }.getType()); + if (objectList.size() == 0) { + httpData.setPostParamJson(""); + return; + } + if (isStringType(objectList.get(0).getClass())) { + //字符串 + httpData.getJsonParam().set(objectList.get(0).toString()); + httpData.setPostParamJson(objectList.get(0).toString()); + } else { + String jsonParam = gson.toJson(objectList.get(0)); + httpData.getJsonParam().set(jsonParam); + httpData.setPostParamJson(jsonParam); + } + } + } + } catch (JsonSyntaxException e) { + log.error("parse http param error,url:" + apiInfo.getApiUrl() + ",error:{}", e); + throw new Exception(e); + } + httpData.setHeaders(headerMap); + httpData.setContentType(apiInfo.getContentType()); + httpData.setTimeout(apiInfo.getRequestTimeout()); + + task.setHttpData(httpData); + task.setType(TaskType.http); + + //http任务 + mibenchTask.setTaskType(TaskType.http.code); + } + + private void dubboDebug(DebugSceneApiInfoReq apiInfo, Task task, MibenchTask mibenchTask) { + //dubbo单接口测试任务 + DubboData dubboData = new DubboData(); + dubboData.setServiceName(apiInfo.getServiceName()); + dubboData.setMethodName(apiInfo.getMethodName()); + dubboData.setGroup(apiInfo.getDubboGroup()); + dubboData.setVersion(apiInfo.getDubboVersion()); + dubboData.setMavenVersion(apiInfo.getDubboMavenVersion()); + + if (apiInfo.getOutputParamInfosStr() != null) { + + List outputParams = new ArrayList<>(); + List outputParamList = gson.fromJson(apiInfo.getOutputParamInfosStr(), new TypeToken>() { + }.getType()); + // key 表达式,需要取上游接口的出参定义合成 + outputParamList.forEach(oParam -> outputParams.add(new OutputParam(oParam.getOrigin(), oParam.getParamName(), oParam.getParseExpr()))); + + dubboData.setOutputParams(new CopyOnWriteArrayList<>(outputParams)); + } + if (apiInfo.getAttachments() != null && !apiInfo.getAttachments().isEmpty()) { + List attachmentList = gson.fromJson(apiInfo.getAttachments(), new TypeToken>() { + }.getType()); + ConcurrentHashMap attachmentMap = new ConcurrentHashMap<>(attachmentList.size()); + + //可以覆盖 + attachmentList.forEach(headerInfo -> attachmentMap.put(headerInfo.getParamKey(), headerInfo.getParamValue())); + //mimeter标记 + attachmentMap.put("User-Agent", Const.MIMETER_UA_KEY); + dubboData.setAttachments(attachmentMap); + } + //检查点 + if (apiInfo.getCheckPointInfoListStr() != null) { + List checkPointInfos = gson.fromJson(apiInfo.getCheckPointInfoListStr(), new TypeToken>() { + }.getType()); + dubboData.setCheckPointInfoList(new CopyOnWriteArrayList<>(checkPointInfos)); + } + //参数类型列表 + List paramTypeList = gson.fromJson(apiInfo.getParamTypeList(), new TypeToken>() { + }.getType()); + dubboData.setRequestParamTypeList(paramTypeList); + dubboData.setRequestBody(apiInfo.getDubboParamJson()); + dubboData.setRequestTimeout(apiInfo.getRequestTimeout()); + task.setDubboData(dubboData); + task.setType(TaskType.dubbo); + + //dubbo任务 + mibenchTask.setTaskType(TaskType.dubbo.code); + } + + private List buildAgentReqs(SceneDTO sceneInfo, Task task, int agentNum, MibenchTask dagTask, Map linkTaskIdMap) { + List reqList = new ArrayList<>(); + if (sceneInfo.getSceneType() == SCENE_TYPE_HTTP) { + + //对链路中接口排序 + for (SerialLinkDTO serialLink : sceneInfo.getSerialLinkDTOs()) { + serialLink.setHttpApiInfoDTOList(serialLink.getHttpApiInfoDTOList().stream().sorted().collect(Collectors.toList())); + } + //构建 http接口 任务图 + if (sceneInfo.getSerialLinkDTOs().size() == 1) { + //构建taskContext前的初始化工作 + SerialLinkDTO serialLink = sceneInfo.getSerialLinkDTOs().get(0); + buildDagBefore(sceneInfo, serialLink, dagTask, task, agentNum, linkTaskIdMap); + //http 单链路场景 + GraphTaskContext taskContext = buildHttpDagContext(sceneInfo.getId(), serialLink, task, dagTask, sceneInfo.getApiBenchInfos(), sceneInfo.getGlobalHeaderList(), sceneInfo.getGlobalTspAuth()); + task.setDagInfo(taskContext); + + AgentReq agentReq = new AgentReq(); + agentReq.setCmd(AgentReq.SUBMIT_TASK_CMD); + agentReq.setTask(task); + reqList.add(agentReq); + } else { + //多链路并发场景 + for (SerialLinkDTO serialLink : + sceneInfo.getSerialLinkDTOs()) { + processMutiLinkTask(SCENE_TYPE_HTTP, sceneInfo, serialLink, dagTask, task, agentNum, reqList, linkTaskIdMap); + } + } + } else if (sceneInfo.getSceneType() == Const.SCENE_TYPE_DUBBO) { + //对链路中接口排序 + for (SerialLinkDTO serialLink : sceneInfo.getSerialLinkDTOs()) { + serialLink.setDubboApiInfoDTOList(serialLink.getDubboApiInfoDTOList().stream().sorted().collect(Collectors.toList())); + } + if (sceneInfo.getSerialLinkDTOs().size() == 1) { + //单链路场景 + SerialLinkDTO serialLink = sceneInfo.getSerialLinkDTOs().get(0); + buildDagBefore(sceneInfo, serialLink, dagTask, task, agentNum, linkTaskIdMap); + //构建 dubbo接口 任务图 + GraphTaskContext taskContext = buildDubboDagContext(sceneInfo.getId(), serialLink, task, dagTask, sceneInfo.getApiBenchInfos(), sceneInfo.getGlobalHeaderList()); + task.setDagInfo(taskContext); + + AgentReq agentReq = new AgentReq(); + agentReq.setCmd(AgentReq.SUBMIT_TASK_CMD); + agentReq.setTask(task); + reqList.add(agentReq); + + } else { + //多链路并发场景 + for (SerialLinkDTO serialLink : + sceneInfo.getSerialLinkDTOs()) { + processMutiLinkTask(Const.SCENE_TYPE_DUBBO, sceneInfo, serialLink, dagTask, task, agentNum, reqList, linkTaskIdMap); + } + } + } else { + log.error("un support scene type"); + } + return reqList; + } + + private void processMutiLinkTask(int sceneType, SceneDTO sceneInfo, SerialLinkDTO serialLink, MibenchTask dagTask, Task task, int agentNum, List reqList, Map linkTaskIdMap) { + //构建taskContext前的初始化工作 + buildDagBefore(sceneInfo, serialLink, dagTask, task, agentNum, linkTaskIdMap); + + //多次构建单链路任务 + AgentReq agentReq = new AgentReq(); + agentReq.setCmd(AgentReq.SUBMIT_TASK_CMD); + + Task tempTask = new Task(); + BeanUtils.copyProperties(task, tempTask); + + //该链路下的接口发压信息 + List apiBenchInfos = sceneInfo.getApiBenchInfos().stream().filter(apiBenchInfo -> apiBenchInfo.getSerialName().equals(serialLink.getSerialLinkName())).collect(Collectors.toList()); + //排序 + apiBenchInfos = apiBenchInfos.stream().sorted().collect(Collectors.toList()); + GraphTaskContext taskContext = null; + if (sceneType == SCENE_TYPE_HTTP) { + taskContext = buildHttpDagContext(sceneInfo.getId(), serialLink, task, dagTask, apiBenchInfos, sceneInfo.getGlobalHeaderList(), sceneInfo.getGlobalTspAuth()); + } else if (sceneType == Const.SCENE_TYPE_DUBBO) { + taskContext = buildDubboDagContext(sceneInfo.getId(), serialLink, task, dagTask, apiBenchInfos, sceneInfo.getGlobalHeaderList()); + } + tempTask.setDagInfo(taskContext); + + agentReq.setTask(tempTask); + reqList.add(agentReq); + } + + private void buildDagBefore(SceneDTO sceneInfo, SerialLinkDTO serialLink, MibenchTask dagTask, Task task, int agentNum, Map linkTaskIdMap) { + dagTask.setSerialLinkId(serialLink.getSerialLinkID()); + dagTask.setState(TaskStatus.Init.code); + //链路任务插库 + List apiBenchInfos = sceneInfo.getApiBenchInfos().stream().filter(apiBenchInfo -> apiBenchInfo.getSerialName().equals(serialLink.getSerialLinkName())).toList(); + //固定rps + int finalRps = 0; + int linkBenchTime = 60; + if (sceneInfo.getBenchMode() == BenchModeEnum.RPS.code) { + if (apiBenchInfos.size() != 0) { + //手动和百分比递增都以其实为基准 + if (sceneInfo.getIncrementMode() != BenchIncreaseModeEnum.STABLE.code) { + finalRps = apiBenchInfos.get(0).getOriginRps(); + } else { + finalRps = apiBenchInfos.get(0).getLinkTps(); + } + dagTask.setQps(finalRps); + if (apiBenchInfos.get(0).getLinkBenchTime() != null) { + linkBenchTime = apiBenchInfos.get(0).getLinkBenchTime(); + } + dagTask.setTime(linkBenchTime); + + dagTask.setOriginQps(apiBenchInfos.get(0).getOriginRps()); + dagTask.setMaxQps(apiBenchInfos.get(0).getMaxRps()); + } + } + + task.setQps(Util.calculateAgentRps(finalRps, agentNum)); + + //发压比例 + task.setRpsRate(sceneInfo.getRpsRate()); + //链路施压时间 + task.setTime(linkBenchTime); + //压力模式 + dagTask.setBenchMode(sceneInfo.getBenchMode()); + //rps 递增模式 + if (sceneInfo.getIncrementMode() != null) { + dagTask.setIncreaseMode(sceneInfo.getIncrementMode()); + } + //发压比例 + if (sceneInfo.getIncreasePercent() != null) { + dagTask.setIncreasePercent(sceneInfo.getIncreasePercent()); + } + dao.insert(dagTask); + task.setId(dagTask.getId()); + task.setSerialLinkID(dagTask.getSerialLinkId()); + + //记录dag 任务id与链路的映射关系 + DagTaskRps dagTaskRps = new DagTaskRps(task.getReportId(), dagTask.getSerialLinkId(), dagTask.getId(), finalRps); + task.setDagTaskRps(dagTaskRps); + linkTaskIdMap.putIfAbsent(serialLink.getSerialLinkID(), dagTaskRps); + + HeraContextInfo heraContextInfo = new HeraContextInfo(sceneInfo.getId(), serialLink.getSerialLinkID(), 0, dagTask.getReportId()); + + task.setHeraContextInfo(heraContextInfo); + //单链路,直接对接口对压测qps信息排序 + sceneInfo.setApiBenchInfos(sceneInfo.getApiBenchInfos().stream().sorted().collect(Collectors.toList())); + + task.setAgentNum(agentNum); + } + + /** + * 基于单个链路接口信息构建http任务图 + */ + public GraphTaskContext buildHttpDagContext(int sceneId, SerialLinkDTO serialLinkDTO, Task dagTask, MibenchTask mibenchTask, List apiBenchInfos, List globalHeaders, TspAuthInfo globalTspAuthInfo) { + GraphTaskContext taskContext = new GraphTaskContext<>(); + + //任务图顶点集 + List> taskList = new ArrayList<>(); + //任务图 边集 + List dependList = new ArrayList<>(); + + int benchInfoIndex = 0; + + //用来临时记录每个接口的出参定义,用于依赖连接 + Map outputParamsMap = new HashMap<>(); + + //该链路拉取流量参数的配置信息 + PullTrafficReqBase pullTrafficReqBase = new PullTrafficReqBase(); + dagTask.setTrafficToPullConfList(pullTrafficReqBase); + List apiTrafficReqList = new ArrayList<>(); + pullTrafficReqBase.setApiTrafficReqList(apiTrafficReqList); + //接口在链路中的顺序 + int index = 0; + for (HttpApiInfoDTO apiInfo : serialLinkDTO.getHttpApiInfoDTOList()) { + ApiTrafficInfo apiTrafficInfo = apiInfo.getApiTrafficInfo(); + if (apiTrafficInfo != null && apiTrafficInfo.isEnableTraffic()) { + //顺带校验该链路中是否有启用录制流量参数的api,有的话再开启开关 + dagTask.setEnableTraffic(true); + apiTrafficReqList.add(new PullApiTrafficReq(apiTrafficInfo.getRecordingConfigId(), apiTrafficInfo.getUrl(), apiTrafficInfo.getFromTime(), apiTrafficInfo.getToTime())); + } + + TaskVertexData httpCalVertex = buildVertexForTask(sceneId, serialLinkDTO, mibenchTask, dagTask, index, TaskType.http.code, benchInfoIndex, apiBenchInfos, apiInfo, null, outputParamsMap, globalHeaders, globalTspAuthInfo); + taskList.add(httpCalVertex); + index++; + benchInfoIndex++; + } + //多接口,按顺序构建顶点依赖关系 + if (serialLinkDTO.getHttpApiInfoDTOList().size() > 1) { + List vertexIdList = taskList.stream().map(TaskVertexData::getIndex).toList(); + for (int i = 0; i < vertexIdList.size() - 1; i++) { + //连结当前顶点与下一个顶点 + TaskEdgeData edge = new TaskEdgeData(vertexIdList.get(i), vertexIdList.get(i + 1)); + dependList.add(edge); + } + } + //连接具有结果依赖的结点 + for (TaskVertexData taskVertexData : taskList) { + HttpData httpData = taskVertexData.getData().getTask().getHttpData(); + if (httpData.getMethod().equals("get")) { + //处理get请求的 http任务,更新keys表达式列表及连接依赖的边 + parseHttpGetTaskDep(taskVertexData, httpData.getUrl(), outputParamsMap, taskList, dependList); + } else { + //处理post请求的 http任务 + parseHttpPostTaskDep(taskVertexData, outputParamsMap, taskList, dependList); + } + } + + //解析结点条件过滤关系 + for (TaskVertexData taskVertexData : taskList) { + parseTaskFilterConditionDep(taskVertexData, taskVertexData.getData().getTask().getHttpData().getFilterCondition(), outputParamsMap, taskList, dependList, TaskType.http.code); + } + taskContext.setTaskList(taskList); + //任务是有依赖关系的 + taskContext.setDependList(dependList); + return taskContext; + } + + /** + * 构建dubbo任务图 + * + * @return + */ + public GraphTaskContext buildDubboDagContext(int sceneId, SerialLinkDTO serialLinkDTO, Task dagTask, MibenchTask mibenchTask, List apiBenchInfos, List globalAttachments) { + GraphTaskContext taskContext = new GraphTaskContext<>(); + + //任务图顶点集 + List> taskList = new ArrayList<>(); + //任务图 边集 + List dependList = new ArrayList<>(); + + int benchInfoIndex = 0; + + //用来临时记录每个接口的出参定义,用于依赖连接 + Map outputParamsMap = new HashMap<>(); + int index = 0; + for (DubboApiInfoDTO apiInfo : serialLinkDTO.getDubboApiInfoDTOList()) { + TaskVertexData dubboCalVertex = buildVertexForTask(sceneId, serialLinkDTO, mibenchTask, + dagTask, index, TaskType.dubbo.code, benchInfoIndex, apiBenchInfos, null, apiInfo, outputParamsMap, globalAttachments, null); + taskList.add(dubboCalVertex); + index++; + benchInfoIndex++; + } + + //多接口,按顺序构建顶点依赖关系 + if (serialLinkDTO.getDubboApiInfoDTOList().size() > 1) { + List vertexIdList = taskList.stream().map(TaskVertexData::getIndex).toList(); + for (int i = 0; i < vertexIdList.size() - 1; i++) { + //连结当前顶点与下一个顶点 + TaskEdgeData edge = new TaskEdgeData(vertexIdList.get(i), vertexIdList.get(i + 1)); + dependList.add(edge); + } + } + + //连接具有结果依赖的结点 + for (TaskVertexData taskVertexData : taskList) { + parseDubboTaskDep(taskVertexData, outputParamsMap, taskList, dependList); + } + + //连接具有结果条件依赖的结点 + for (TaskVertexData taskVertexData : taskList) { + parseTaskFilterConditionDep(taskVertexData, taskVertexData.getData().getTask().getDubboData().getFilterCondition(), outputParamsMap, taskList, dependList, TaskType.dubbo.code); + } + taskContext.setTaskList(taskList); + //任务是有依赖关系的 + taskContext.setDependList(dependList); + return taskContext; + } + + private TaskVertexData buildVertexForTask(int sceneId, SerialLinkDTO serialLinkDTO, MibenchTask mibenchTask, + Task dagTask, int apiIndex, int taskType, int benchInfoIndex, + List apiBenchInfos, HttpApiInfoDTO httpApiInfo, DubboApiInfoDTO dubboApiInfo, Map outputParamsMap, + List globalAttachments, TspAuthInfo globalTspAuthInfo) { + MibenchTask apiTask = buildMibenchTask(dagTask); + apiTask.setParentTaskId(mibenchTask.getId()); + if (taskType == TaskType.http.code) { + apiTask.setSceneApiId(httpApiInfo.getApiID()); + } else if (taskType == TaskType.dubbo.code) { + apiTask.setSceneApiId(dubboApiInfo.getApiID()); + } + apiTask.setTaskType(taskType); + apiTask.setAgentNum(mibenchTask.getAgentNum()); + //每个接口需要使用各自的qps + ApiBenchInfo apiBenchInfo = apiBenchInfos.get(benchInfoIndex); + int agentNum = mibenchTask.getAgentNum(); + int apiOneAgentOriTps = apiBenchInfo.getOriginRps() / agentNum; + int apiOneAgentMaxTps = apiBenchInfo.getMaxRps() / agentNum; + + apiTask.setQps(mibenchTask.getQps()); + //起始qps + apiTask.setOriginQps(apiOneAgentOriTps); + //最大qps + apiTask.setMaxQps(apiOneAgentMaxTps); + apiTask.setReportId(mibenchTask.getReportId()); + apiTask.setState(TaskStatus.Init.code); + //每个接口独立任务插库 + dao.insert(apiTask); + + //用于hera传参的信息,任务id使用dagTask的 + HeraContextInfo heraContextInfo; + TaskVertexData calVertex = new TaskVertexData<>(); + //分类型创建任务顶点 + if (taskType == TaskType.http.code) { + heraContextInfo = new HeraContextInfo(sceneId, serialLinkDTO.getSerialLinkID(), httpApiInfo.getApiID(), mibenchTask.getReportId()); + calVertex = createHttpCalVertex(heraContextInfo, apiIndex, apiTask.getId(), httpApiInfo, globalAttachments, globalTspAuthInfo, apiOneAgentOriTps, dagTask, outputParamsMap); + } else if (taskType == TaskType.dubbo.code) { + heraContextInfo = new HeraContextInfo(sceneId, serialLinkDTO.getSerialLinkID(), dubboApiInfo.getApiID(), mibenchTask.getReportId()); + calVertex = createDubboCalVertex(heraContextInfo, apiIndex, apiTask.getId(), dubboApiInfo, globalAttachments, apiOneAgentOriTps, dagTask, outputParamsMap); + } + return calVertex; + } + + /** + * 解析构建具有条件依赖的顶点关系 + */ + private void parseTaskFilterConditionDep(TaskVertexData currentNode, + CopyOnWriteArrayList filterConditions, + Map outputParamsMap, + List> taskList, + List dependList, int taskType) { + if (filterConditions != null && filterConditions.size() != 0) { + List keys = new ArrayList<>(); + + for (CheckPointInfo filterCondition : filterConditions) { + //条件 key 在出参中,即具有依赖关系 + String paramValue = filterCondition.getCheckObj(); + if (outputParamsMap.containsKey(paramValue)) { + int parentTaskVertexId = outputParamsMap.get(paramValue); + + OutputParam outputParam; + Optional> optional = taskList.stream().filter(vertexData -> vertexData.getIndex() == parentTaskVertexId).findFirst(); + + TaskVertexData parentVertexData = optional.get(); + if (taskType == TaskType.http.code) { + outputParam = parentVertexData.getData().getTask().getHttpData().getOutputParams().stream().filter(tmpOutputParam -> tmpOutputParam.getParamName().equals(paramValue)).findFirst().get(); + } else if (taskType == TaskType.dubbo.code) { + outputParam = parentVertexData.getData().getTask().getDubboData().getOutputParams().stream().filter(tmpOutputParam -> tmpOutputParam.getParamName().equals(paramValue)).findFirst().get(); + } else { + return; + } + ExprKey key = new ExprKey(parentTaskVertexId, outputParam.getOrigin(), outputParam.getParamName(), outputParam.getParseExpr(), new ArrayList<>()); + keys.add(key); + //当前顶点依赖该上游顶点 + TaskEdgeData edge = new TaskEdgeData(parentTaskVertexId, currentNode.getIndex()); + if (!dependList.contains(edge)) { + dependList.add(edge); + } + } + } + keys.forEach(key -> currentNode.getData().getExprMap().put(key, "")); + } + } + + /** + * 解析构建 dubbo 请求类型的顶点的依赖关系 + */ + private void parseDubboTaskDep(TaskVertexData currentNode, Map outputParamsMap, List> taskList, List dependList) { + List keys = new ArrayList<>(); + + DubboData dubboData = currentNode.getData().getTask().getDubboData(); + + // json串 + String jsonBody = dubboData.getRequestBody(); + + Matcher m = EL_PATTERN_MUTI.matcher(jsonBody); + + while (m.find()) { + String paramValue = m.group(1); + //该 ${} 参数为上游链路参数,更新结点 key表达式列表 + if (outputParamsMap.containsKey(paramValue)) { + int parentTaskVertexId = outputParamsMap.get(paramValue); + Optional> optional = taskList.stream().filter(vertexData -> vertexData.getIndex() == parentTaskVertexId).findFirst(); + if (optional.isPresent()) { + TaskVertexData parentVertexData = optional.get(); + + OutputParam outputParam = parentVertexData.getData().getTask().getDubboData().getOutputParams().stream() + .filter(tmpOutputParam -> tmpOutputParam.getParamName().equals(paramValue)).findFirst().get(); + + List putValueExpr = new ArrayList<>(); + //用于指定更新http的数据 + putValueExpr.add("dubboData"); + //入参的位置index + putValueExpr.add(String.valueOf(0)); + // 要替换的参数名,大json直接用 例:${name} 内容替代 + putValueExpr.add(m.group(0)); + + ExprKey key = new ExprKey(parentTaskVertexId, outputParam.getOrigin(), outputParam.getParamName(), outputParam.getParseExpr(), putValueExpr); + keys.add(key); + //当前顶点依赖该上游顶点 + TaskEdgeData edge = new TaskEdgeData(parentTaskVertexId, currentNode.getIndex()); + if (!dependList.contains(edge)) { + dependList.add(edge); + } + } + keys.forEach(key -> currentNode.getData().getExprMap().put(key, "")); + } + } + } + + /** + * 解析构建http post请求类型的顶点的依赖关系 + */ + private void parseHttpPostTaskDep(TaskVertexData currentNode, Map outputParamsMap, List> taskList, List dependList) { + //post 参数 + //例:http://www.baidu.com" + + List keys = new ArrayList<>(); + + HttpData httpData = currentNode.getData().getTask().getHttpData(); + + List postParam = httpData.getParams(); + if (httpData.getContentType().equals(Constants.CONTENT_TYPE_APP_FORM) || httpData.getContentType().equals(CONTENT_TYPE_APP_FORM2)) { + //表单 + for (int index = 0; index < postParam.size(); index++) { + if (postParam.get(index).toString().startsWith("${")) { + String paramValue = Util.getElKey(postParam.get(index).toString()).getKey(); + //该 ${} 参数为上游链路参数,更新结点 key表达式列表 + if (outputParamsMap.containsKey(paramValue)) { + int parentTaskVertexId = outputParamsMap.get(paramValue); + Optional> optional = taskList.stream().filter(vertexData -> vertexData.getIndex() == parentTaskVertexId).findFirst(); + if (optional.isPresent()) { + TaskVertexData parentVertexData = optional.get(); + + OutputParam outputParam = parentVertexData.getData().getTask().getHttpData().getOutputParams().stream().filter(tmpOutputParam -> tmpOutputParam.getParamName().equals(paramValue)).findFirst().get(); + + List putValueExpr = new ArrayList<>(); + //用于指定更新http的数据 + putValueExpr.add("httpData"); + //入参的位置index + putValueExpr.add(String.valueOf(index)); + //要替换的参数名 + putValueExpr.add(httpData.getTypes().get(index).getName()); + + ExprKey key = new ExprKey(parentTaskVertexId, outputParam.getOrigin(), outputParam.getParamName(), outputParam.getParseExpr(), putValueExpr); + keys.add(key); + //当前顶点依赖该上游顶点 + TaskEdgeData edge = new TaskEdgeData(parentTaskVertexId, currentNode.getIndex()); + if (!dependList.contains(edge)) { + dependList.add(edge); + } + } + keys.forEach(key -> currentNode.getData().getExprMap().put(key, "")); + } + } + } + } else if (httpData.getContentType().equals(Constants.CONTENT_TYPE_APP_JSON)) { + // json串 + String jsonBody = httpData.getJsonParam().get(); + Matcher m = EL_PATTERN_MUTI.matcher(jsonBody); + + while (m.find()) { + String paramValue = m.group(1); + //该 ${} 参数为上游链路参数,更新结点 key表达式列表 + if (outputParamsMap.containsKey(paramValue)) { + int parentTaskVertexId = outputParamsMap.get(paramValue); + Optional> optional = taskList.stream().filter(vertexData -> vertexData.getIndex() == parentTaskVertexId).findFirst(); + if (optional.isPresent()) { + TaskVertexData parentVertexData = optional.get(); + + OutputParam outputParam = parentVertexData.getData().getTask().getHttpData().getOutputParams().stream().filter(tmpOutputParam -> tmpOutputParam.getParamName().equals(paramValue)).findFirst().get(); + + List putValueExpr = new ArrayList<>(); + //用于指定更新http的数据 + putValueExpr.add("httpData"); + //入参的位置index + putValueExpr.add(String.valueOf(0)); + // 要替换的参数名,大json直接用 例:${name} 内容替代 + putValueExpr.add(m.group(0)); + + ExprKey key = new ExprKey(parentTaskVertexId, outputParam.getOrigin(), outputParam.getParamName(), outputParam.getParseExpr(), putValueExpr); + keys.add(key); + //当前顶点依赖该上游顶点 + TaskEdgeData edge = new TaskEdgeData(parentTaskVertexId, currentNode.getIndex()); + if (!dependList.contains(edge)) { + dependList.add(edge); + } + } + keys.forEach(key -> currentNode.getData().getExprMap().put(key, "")); + } + } + } + } + + private void parseHttpGetTaskDep(TaskVertexData currentNode, String url, Map outputParamsMap, List> taskList, List dependList) { + if (url.contains("?")) { + //带参数 + //例:http://www.baidu.com?name=${name}&age=${rAge}&k=v" + //paramStr: name=${name}&age=${rAge}&k=v + String paramStr = url.split("\\?", 2)[1]; + String[] paramPairStrs = paramStr.split("&"); + + List keys = new ArrayList<>(); + + for (int index = 0; index < paramPairStrs.length; index++) { + String[] kV = paramPairStrs[index].split("=", 2); + if (kV[1].startsWith("${")) { + String paramValue = Util.getElKey(kV[1]).getKey(); + //该 ${} 参数为上游链路参数,更新结点 key表达式列表 + if (outputParamsMap.containsKey(paramValue)) { + int parentTaskVertexId = outputParamsMap.get(paramValue); + Optional> optional = taskList.stream().filter(vertexData -> vertexData.getIndex() == parentTaskVertexId).findFirst(); + if (optional.isPresent()) { + TaskVertexData parentVertexData = optional.get(); + + OutputParam outputParam = parentVertexData.getData().getTask().getHttpData().getOutputParams().stream().filter(tmpOutputParam -> tmpOutputParam.getParamName().equals(paramValue)).findFirst().get(); + + List putValueExpr = new ArrayList<>(); + //用于指定更新http的数据 + putValueExpr.add("httpData"); + //入参的位置index + putValueExpr.add(String.valueOf(index)); + //参数名 + putValueExpr.add(kV[0]); + + ExprKey key = new ExprKey(parentTaskVertexId, outputParam.getOrigin(), outputParam.getParamName(), outputParam.getParseExpr(), putValueExpr); + keys.add(key); + //当前顶点依赖该上游顶点 + TaskEdgeData edge = new TaskEdgeData(parentTaskVertexId, currentNode.getIndex()); + if (!dependList.contains(edge)) { + dependList.add(edge); + } + } + keys.forEach(key -> currentNode.getData().getExprMap().put(key, "")); + } + } + } + currentNode.getData().getTask().getHttpData().setUrl(url.split("\\?")[0]); + } + } + + /** + * 构建 http 任务顶点 + */ + private TaskVertexData createHttpCalVertex(HeraContextInfo heraContextInfo, int apiIndex, int taskId, HttpApiInfoDTO apiInfo, List globalHeaders, TspAuthInfo globalTspAuthInfo, int oriTps, Task dagTask, Map outputParamsMap) { + //任务顶点 + TaskVertexData data = new TaskVertexData<>(); + //结点信息 + NodeInfo nodeInfo = new NodeInfo(); + //任务id + nodeInfo.setId(taskId); + //该结点返回结果名,非必须 + nodeInfo.setResultName("result"); + + //发送给agent的 http 任务体 + Task task = new Task(); + task.setHeraContextInfo(heraContextInfo); + + task.setId(taskId); + task.setDebug(dagTask.isDebug()); + task.setType(TaskType.http); + task.setTimeout(apiInfo.getRequestTimeout()); + task.setQps(oriTps); + + task.setSuccessCode(dagTask.getSuccessCode()); + + HttpData httpData = new HttpData(); + //启用流量数据 + ApiTrafficInfo apiTrafficInfo = apiInfo.getApiTrafficInfo(); + if (apiTrafficInfo != null && apiTrafficInfo.isEnableTraffic()) { + httpData.setEnableTraffic(true); + httpData.setTrafficConfId(apiTrafficInfo.getRecordingConfigId()); + } + //启用x5 + ApiX5Info apiX5Info = apiInfo.getApiX5Info(); + if (apiX5Info != null && apiX5Info.isEnableX5()) { + ApiX5InfoDTO x5InfoDTO = new ApiX5InfoDTO(); + BeanUtils.copyProperties(apiX5Info, x5InfoDTO); + httpData.setApiX5InfoDTO(x5InfoDTO); + } + + httpData.setUrl(apiInfo.getApiUrl()); + httpData.setTimeout(apiInfo.getRequestTimeout()); + + //检查点 + if (apiInfo.getCheckPointInfoList() != null) { + List checkPointInfoList = new ArrayList<>(); + apiInfo.getCheckPointInfoList().forEach(checkPointInfoDTO -> { + CheckPointInfo checkPointInfo = new CheckPointInfo(); + BeanUtils.copyProperties(checkPointInfoDTO, checkPointInfo); + checkPointInfoList.add(checkPointInfo); + }); + httpData.setCheckPointInfoList(new CopyOnWriteArrayList<>(checkPointInfoList)); + } + + //过滤条件 + if (apiInfo.getFilterCondition() != null) { + List filterConditionList = new ArrayList<>(); + apiInfo.getFilterCondition().forEach(filterCondition -> { + CheckPointInfo checkPointInfo = new CheckPointInfo(); + BeanUtils.copyProperties(filterCondition, checkPointInfo); + filterConditionList.add(checkPointInfo); + }); + httpData.setFilterCondition(new CopyOnWriteArrayList<>(filterConditionList)); + } + + String headerStr = apiInfo.getHeaderInfo(); + List headerInfoList = gson.fromJson(headerStr, new TypeToken>() { + }.getType()); + ConcurrentHashMap headerMap = new ConcurrentHashMap<>(headerInfoList.size()); + + //全局请求头 + if (globalHeaders != null && globalHeaders.size() != 0) { + globalHeaders.forEach(globalHeader -> headerMap.putIfAbsent(globalHeader.getHeaderName(), globalHeader.getHeaderValue())); + } + //可以覆盖 + headerInfoList.forEach(headerInfo -> headerMap.put(headerInfo.getHeaderName(), headerInfo.getHeaderValue())); + //mimeter标记 + headerMap.put("User-Agent", Const.MIMETER_UA_KEY); + if (apiInfo.getApiRequestType() == Const.HTTP_REQ_GET) { + httpData.setMethod("get"); + } else { + httpData.setMethod("post"); + if (apiInfo.getContentType().equals(CONTENT_TYPE_APP_FORM2)) { + headerMap.put("Content-type", CONTENT_TYPE_APP_FORM); + httpData.setContentType(CONTENT_TYPE_APP_FORM); + } else { + httpData.setContentType(apiInfo.getContentType()); + headerMap.putIfAbsent("Content-Type", apiInfo.getContentType()); + } + } + httpData.setHeaders(headerMap); + + TspAuthInfoDTO tspAuthInfoDTO = new TspAuthInfoDTO(false, "", ""); + //全局 tsp auth + if (globalTspAuthInfo != null && globalTspAuthInfo.isEnableAuth()) { + //全局启用 + tspAuthInfoDTO.setEnableAuth(true); + tspAuthInfoDTO.setAccessKey(globalTspAuthInfo.getAccessKey()); + tspAuthInfoDTO.setSecretKey(globalTspAuthInfo.getSecretKey()); + } + //接口层 tsp auth + TspAuthInfo apiTspAuth = apiInfo.getApiTspAuth(); + if (apiTspAuth != null && apiTspAuth.isEnableAuth() && !apiTspAuth.getAccessKey().isEmpty() && !apiTspAuth.getSecretKey().isEmpty()) { + //接口层级开启,并有自身配置数据 + tspAuthInfoDTO.setEnableAuth(true); + tspAuthInfoDTO.setAccessKey(apiTspAuth.getAccessKey()); + tspAuthInfoDTO.setSecretKey(apiTspAuth.getSecretKey()); + } else if (apiTspAuth != null && !apiTspAuth.isEnableAuth()) { + tspAuthInfoDTO.setEnableAuth(false); + } + //最终 tsp auth info + httpData.setTspAuthInfoDTO(tspAuthInfoDTO); + + List types = new ArrayList<>(); + List paramValues; + try { + if (apiInfo.getApiRequestType() == Const.HTTP_REQ_GET || (apiInfo.getContentType() != null && (apiInfo.getContentType().equals(Constants.CONTENT_TYPE_APP_FORM) || apiInfo.getContentType().equals(CONTENT_TYPE_APP_FORM2)))) { + //表单类型参数 + //例:[{"paramKey": "v1", "paramValue": "v2"}] + List tmpValues = new ArrayList<>(); + if (null != apiInfo.getRequestInfo()) { + paramValues = gson.fromJson(apiInfo.getRequestInfo(), new TypeToken>() { + }.getType()); + paramValues.forEach(paramValue -> { + ParamType paramType = new ParamType(ParamTypeEnum.primary, paramValue.getParamKey()); + types.add(paramType); + tmpValues.add(paramValue.getParamValue()); + }); + } + httpData.initTypeList(types); + + //["a","b"...] + httpData.getParams().addAll(new CopyOnWriteArrayList<>(tmpValues)); + + } else if (apiInfo.getContentType().equals(Constants.CONTENT_TYPE_APP_JSON)) { + //对象参数 例:"[{\"name\": \"dzx\", \"age\": 19},{\"sex\": \"man\"}]" + + List objectList; + + + if (null != apiInfo.getRequestInfoRaw()) { + objectList = gson.fromJson(apiInfo.getRequestInfoRaw(), new TypeToken>() { + }.getType()); + if (objectList.size() == 0) { + httpData.setOriginJsonParam(""); + httpData.getJsonParam().set(""); + httpData.setPostParamJson(""); + } + String val; + if (isStringType(objectList.get(0).getClass())) { + //字符串 + val = objectList.get(0).toString(); + httpData.setOriginJsonParam(val); + httpData.getJsonParam().set(val); + httpData.setPostParamJson(val); + } else { + val = gson.toJson(objectList.get(0)); + httpData.setOriginJsonParam(val); + httpData.getJsonParam().set(val); + } + } + } + } catch (JsonSyntaxException e) { + log.error("error param type,api id:{},param:{},{}", apiInfo.getApiID(), apiInfo.getRequestInfo(), apiInfo.getRequestInfoRaw()); + throw new RuntimeException(e); + } + + List outputParams = new ArrayList<>(); + + // key 表达式,需要取上游接口的出参定义合成 + apiInfo.getOutputParamInfos().forEach(oParam -> { + outputParamsMap.putIfAbsent(oParam.getParamName(), apiIndex); + outputParams.add(new OutputParam(oParam.getOrigin(), oParam.getParamName(), oParam.getParseExpr())); + }); + + httpData.setOutputParams(new CopyOnWriteArrayList<>(outputParams)); + + task.setHttpData(httpData); + nodeInfo.setTask(task); + + data.setData(nodeInfo); + data.setIndex(apiIndex); + return data; + } + + /** + * 构建dubbo任务顶点 + * + * @return + */ + private TaskVertexData createDubboCalVertex(HeraContextInfo heraContextInfo, int apiIndex, int taskId, DubboApiInfoDTO apiInfo, List globalHeaders, int qps, Task dagTask, Map outputParamsMap) { + //任务顶点 + TaskVertexData data = new TaskVertexData<>(); + //结点信息 + NodeInfo nodeInfo = new NodeInfo(); + //任务id + nodeInfo.setId(taskId); + //该结点返回结果名,非必须 + nodeInfo.setResultName("result"); + + //发送给agent的 dubbo 任务体 + Task task = new Task(); + task.setHeraContextInfo(heraContextInfo); + + task.setId(taskId); + task.setDebug(dagTask.isDebug()); + task.setType(TaskType.dubbo); + task.setTimeout(apiInfo.getRequestTimeout()); + task.setQps(qps); + + DubboData dubboData = new DubboData(); + dubboData.setServiceName(apiInfo.getServiceName()); + dubboData.setMethodName(apiInfo.getMethodName()); + if (apiInfo.getGroup() != null) { + dubboData.setGroup(apiInfo.getGroup()); + } + if (apiInfo.getVersion() != null) { + dubboData.setVersion(apiInfo.getVersion()); + } + if (apiInfo.getDubboMavenVersion() != null) { + dubboData.setMavenVersion(apiInfo.getDubboMavenVersion()); + } + + //检查点 + if (apiInfo.getCheckPointInfoList() != null) { + List checkPointInfoList = new ArrayList<>(); + apiInfo.getCheckPointInfoList().forEach(checkPointInfoDTO -> { + CheckPointInfo checkPointInfo = new CheckPointInfo(); + BeanUtils.copyProperties(checkPointInfoDTO, checkPointInfo); + checkPointInfoList.add(checkPointInfo); + }); + dubboData.setCheckPointInfoList(new CopyOnWriteArrayList<>(checkPointInfoList)); + } + //过滤条件 + if (apiInfo.getFilterCondition() != null) { + List filterConditionList = new ArrayList<>(); + apiInfo.getFilterCondition().forEach(filterCondition -> { + CheckPointInfo checkPointInfo = new CheckPointInfo(); + BeanUtils.copyProperties(filterCondition, checkPointInfo); + filterConditionList.add(checkPointInfo); + }); + dubboData.setFilterCondition(new CopyOnWriteArrayList<>(filterConditionList)); + } + + //注册中心环境 + dubboData.setDubboEnv(apiInfo.getDubboEnv()); + dubboData.setRequestTimeout(apiInfo.getRequestTimeout()); + + String attachmentsStr = apiInfo.getAttachments(); + ConcurrentHashMap attachmentMap = new ConcurrentHashMap<>(8); + + //全局请求头 + if (globalHeaders != null && globalHeaders.size() != 0) { + globalHeaders.forEach(globalHeader -> attachmentMap.putIfAbsent(globalHeader.getHeaderName(), globalHeader.getHeaderValue())); + } + + if (attachmentsStr != null && !attachmentsStr.isEmpty()) { + List attachmentList = gson.fromJson(attachmentsStr, new TypeToken>() { + }.getType()); + //可以覆盖 + attachmentList.forEach(headerInfo -> attachmentMap.put(headerInfo.getParamKey(), headerInfo.getParamValue())); + } + + //mimeter标记 + attachmentMap.put("User-Agent", Const.MIMETER_UA_KEY); + dubboData.setAttachments(attachmentMap); + + dubboData.setRequestParamTypeList(apiInfo.getRequestParamTypeList()); + if (apiInfo.getRequestBody() == null) { + dubboData.setRequestBody("[]"); + dubboData.setOriginJsonParam("[]"); + dubboData.getJsonParam().set("[]"); + } else { + dubboData.setRequestBody(apiInfo.getRequestBody()); + dubboData.setOriginJsonParam(apiInfo.getRequestBody()); + dubboData.getJsonParam().set(apiInfo.getRequestBody()); + } + + List outputParams = new ArrayList<>(); + + // key 表达式,需要取上游接口的出参定义合成 + apiInfo.getOutputParamInfos().forEach(oParam -> { + outputParamsMap.putIfAbsent(oParam.getParamName(), apiIndex); + outputParams.add(new OutputParam(oParam.getOrigin(), oParam.getParamName(), oParam.getParseExpr())); + }); + + dubboData.setOutputParams(new CopyOnWriteArrayList<>(outputParams)); + task.setDubboData(dubboData); + nodeInfo.setTask(task); + data.setData(nodeInfo); + data.setIndex(apiIndex); + return data; + } + + /** + * 过滤链路配置 + */ + private void filterLink(SceneDTO sceneInfo) { + //过滤未启用的链路 + sceneInfo.setSerialLinkDTOs(sceneInfo.getSerialLinkDTOs().stream().filter(SerialLinkDTO::getEnable).collect(Collectors.toList())); + List enableLinkNames = sceneInfo.getSerialLinkDTOs().stream().map(SerialLinkDTO::getSerialLinkName).toList(); + sceneInfo.setApiBenchInfos(sceneInfo.getApiBenchInfos().stream().filter(apiBenchInfo -> enableLinkNames.contains(apiBenchInfo.getSerialName())).collect(Collectors.toList())); + + //默认100%发压比例 + int rpsRate; + if (sceneInfo.getRpsRate() == null) { + rpsRate = 100; + } else { + rpsRate = sceneInfo.getRpsRate(); + } + //配比 + sceneInfo.getApiBenchInfos().forEach(apiBenchInfo -> { + apiBenchInfo.setLinkTps((int) Math.ceil(apiBenchInfo.getLinkTps() * (rpsRate / 100d))); + apiBenchInfo.setOriginRps((int) Math.ceil(apiBenchInfo.getOriginRps() * (rpsRate / 100d))); + apiBenchInfo.setMaxRps((int) Math.ceil(apiBenchInfo.getMaxRps() * (rpsRate / 100d))); + }); + } + + public static boolean isStringType(Class clazz) { + return clazz.getTypeName().startsWith("java.lang.String"); + } + + private MibenchTask buildMibenchTask(Task task) { + MibenchTask mibenchTask = new MibenchTask(); + mibenchTask.setSceneId(task.getSceneId()); + mibenchTask.setCtime(System.currentTimeMillis()); + mibenchTask.setUtime(System.currentTimeMillis()); + if (task.getTime() != 0) { + mibenchTask.setTime(task.getTime()); + } + mibenchTask.setState(TaskStatus.Init.code); + return mibenchTask; + } + + /** + * 获取某台机器host文件内容 + */ + public HttpResult loadHostsFile(LoadHostsFileReq loadHostsFileReq) { + int num = AgentContext.ins().list().size(); + if (num <= 0) { + return HttpResult.fail(500, "agent num <= 0", "暂无可用的发压机"); + } + List agentList = new ArrayList<>(AgentContext.ins().map.values()); + if (loadHostsFileReq.getAgentIp() == null) { + return HttpResult.success("ok"); + } + + AgentReq req = new AgentReq(); + req.setCmd(AgentReq.LOAD_HOST_CMD); + + int i = 0; + boolean find = false; + for (int index = 0; index < agentList.size(); index++) { + String[] ipAndPort = agentList.get(index).getRemoteAddr().split(":", 2); + if (ipAndPort[0].equals(loadHostsFileReq.getAgentIp())) { + i = index; + find = true; + break; + } + } + if (!find) { + return HttpResult.fail(500, "call agent error", "发压机不存在"); + } + HostsFileResult tr; + try { + tr = syncLoadHostsFromAgent(agentList.get(i), req); + return HttpResult.success(tr.getHostsFile()); + } catch (Exception e) { + log.error("call load host interface error"); + return HttpResult.fail(500, "call agent error", e.getMessage()); + } + } + + /** + * 手动修改单台机器host文件 + */ + public HttpResult manualEditHosts(HostForAgentReq hostForAgentReq) { + int num = AgentContext.ins().list().size(); + if (num <= 0) { + return HttpResult.fail(500, "agent num <= 0", "暂无可用的发压机"); + } + List agentList = new ArrayList<>(AgentContext.ins().map.values()); + if (hostForAgentReq.getAgentIp() == null) { + return HttpResult.success("ok"); + } + + AgentReq req = new AgentReq(); + req.setCmd(AgentReq.EDIT_HOST_CMD); + + AgentHostReq hostReq = new AgentHostReq(); + hostReq.setDomain(hostForAgentReq.getDomain()); + hostReq.setIp(hostForAgentReq.getIp()); + req.setAgentHostReqList(Collections.singletonList(hostReq)); + + int i = 0; + boolean find = false; + for (int index = 0; index < agentList.size(); index++) { + String[] ipAndPort = agentList.get(index).getRemoteAddr().split(":", 2); + if (ipAndPort[0].equals(hostForAgentReq.getAgentIp())) { + i = index; + find = true; + break; + } + } + if (!find) { + return HttpResult.fail(500, "call agent error", "发压机不存在"); + } + try { + syncCallAgent(agentList.get(i), req); + } catch (Exception e) { + log.error("call edit host interface error"); + return HttpResult.fail(500, "call agent error", e.getMessage()); + } + return HttpResult.success("ok"); + } + + /** + * 修改host文件 + */ + public HttpResult editHosts(DomainApplyReq domainApplyReq) { + int num = AgentContext.ins().list().size(); + if (num <= 0) { + return HttpResult.fail(500, "agent num <= 0", "暂无可用的发压机"); + } + List agentList = new ArrayList<>(AgentContext.ins().map.values()); + List tmpAgentList = new ArrayList<>(); + if (domainApplyReq.getAgentIPs() == null || domainApplyReq.getAgentIPs().size() == 0) { + return HttpResult.success("ok"); + } + AgentReq req = new AgentReq(); + req.setCmd(AgentReq.EDIT_HOST_CMD); + + AgentHostReq hostReq = new AgentHostReq(); + hostReq.setDomain(domainApplyReq.getDomain()); + hostReq.setIp(domainApplyReq.getIp()); + req.setAgentHostReqList(Collections.singletonList(hostReq)); + + agentList.forEach(channel -> { + String[] ipAndPort = channel.getRemoteAddr().split(":", 2); + if (domainApplyReq.getAgentIPs().contains(ipAndPort[0])) { + tmpAgentList.add(channel); + } + }); + try { + callHostsAgents(tmpAgentList, req); + } catch (Exception e) { + log.error("call edit host interface error"); + return HttpResult.fail(500, "call agent error", e.getMessage()); + } + return HttpResult.success("ok"); + } + + /** + * 同步发压机hosts配置 + */ + public HttpResult syncHosts(SyncHostsReq syncHostsReq) { + int num = AgentContext.ins().list().size(); + if (num <= 0) { + return HttpResult.fail(500, "agent num <= 0", "暂无可用的发压机"); + } + List agentList = new ArrayList<>(AgentContext.ins().map.values()); + if (syncHostsReq.getAgentHostsConfList() == null || syncHostsReq.getAgentHostsConfList().size() == 0) { + return HttpResult.success("ok"); + } + + AgentReq req = new AgentReq(); + req.setCmd(AgentReq.EDIT_HOST_CMD); + + syncHostsReq.getAgentHostsConfList().forEach(agentHostsConf -> { + List hostReqList = new ArrayList<>(); + + agentHostsConf.getDomainConfs().forEach(hostConf -> { + AgentHostReq hostReq = new AgentHostReq(); + hostReq.setDomain(hostConf.getDomain()); + hostReq.setIp(hostConf.getIp()); + hostReqList.add(hostReq); + }); + req.setAgentHostReqList(hostReqList); + + int i = 0; + boolean find = false; + for (int index = 0; index < agentList.size(); index++) { + String[] ipAndPort = agentList.get(index).getRemoteAddr().split(":", 2); + if (ipAndPort[0].equals(agentHostsConf.getAgentIp())) { + i = index; + find = true; + break; + } + } + if (find) { + try { + syncCallAgent(agentList.get(i), req); + } catch (Exception e) { + log.error("call edit host interface error"); + } + } + }); + return HttpResult.success("ok"); + } + + /** + * 删除某项域名绑定配置 + */ + public HttpResult delHosts(DelHostForAgentsReq req) { + int num = AgentContext.ins().list().size(); + if (num <= 0) { + return HttpResult.fail(500, "agent num <= 0", "暂无可用的发压机"); + } + List agentList = new ArrayList<>(AgentContext.ins().map.values()); + List tmpGgentList = new ArrayList<>(); + if (req.getAgentIps() == null || req.getAgentIps().size() == 0) { + return HttpResult.success("ok"); + } + AgentReq agentReq = new AgentReq(); + agentReq.setCmd(AgentReq.DEL_HOST_CMD); + + List hostReqList = new ArrayList<>(); + AgentHostReq hostReq = new AgentHostReq(); + hostReq.setDomain(req.getDomain()); + hostReqList.add(hostReq); + agentReq.setAgentHostReqList(hostReqList); + + agentList.forEach(channel -> { + String[] ipAndPort = channel.getRemoteAddr().split(":", 2); + if (req.getAgentIps().contains(ipAndPort[0])) { + tmpGgentList.add(channel); + } + }); + try { + callHostsAgents(tmpGgentList, agentReq); + } catch (Exception e) { + log.error("call edit host interface error"); + return HttpResult.fail(500, "call agent error", e.getMessage()); + } + return HttpResult.success("ok"); + + } + + private void callTaskAgents(List agentList, AgentReq agentReq) { + for (int agentIndex = 0; agentIndex < agentList.size(); agentIndex++) { + //该机器在所使用发压机集群中的序号索引 + agentReq.getTask().setAgentIndex(agentIndex); + agentReq.setAddr(agentList.get(agentIndex).getRemoteAddr()); + RemotingCommand req = RemotingCommand.createGsonRequestCommand(MibenchCmd.TASK, agentReq, gson); + rpcServer.tell(agentList.get(agentIndex).getChannel(), req); + } + } + + /** + * 修改host文件 + */ + private void callHostsAgents(List agentList, AgentReq agentReq) { + agentList.forEach(ch -> { + agentReq.setAddr(ch.getRemoteAddr()); + RemotingCommand req = RemotingCommand.createGsonRequestCommand(MibenchCmd.TASK, agentReq, gson); + rpcServer.tell(ch.getChannel(), req); + }); + } + + private TaskResult syncCallAgent(AgentChannel channel, AgentReq agentReq) { + agentReq.setAddr(channel.getRemoteAddr()); + RemotingCommand req = RemotingCommand.createGsonRequestCommand(MibenchCmd.TASK, agentReq, gson); + RemotingCommand response = rpcServer.sendMessage(channel, req, 10000); + return gson.fromJson(new String(response.getBody()), TaskResult.class); + } + + private HostsFileResult syncLoadHostsFromAgent(AgentChannel channel, AgentReq agentReq) { + agentReq.setAddr(channel.getRemoteAddr()); + RemotingCommand req = RemotingCommand.createGsonRequestCommand(MibenchCmd.TASK, agentReq, gson); + RemotingCommand response = rpcServer.sendMessage(channel, req, 10000); + return gson.fromJson(new String(response.getBody()), HostsFileResult.class); + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/SlaService.java b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/SlaService.java new file mode 100644 index 000000000..ea9eb8e75 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/SlaService.java @@ -0,0 +1,486 @@ +package run.mone.mimeter.agent.manager; + +import com.xiaomi.faas.func.api.PrometheusService; +import com.xiaomi.faas.func.domain.MimeterApiRes; +import com.xiaomi.mone.monitor.service.MiFeiShuService; +import com.xiaomi.mone.monitor.service.SendSmsService; +import com.xiaomi.youpin.docean.anno.Service; +import com.xiaomi.youpin.infra.rpc.Result; +import lombok.extern.slf4j.Slf4j; +import org.apache.dubbo.annotation.DubboReference; +import org.nutz.dao.Cnd; +import org.nutz.dao.impl.NutDao; +import run.mone.mimeter.agent.manager.bo.MibenchTask; +import run.mone.mimeter.agent.manager.bo.SlaContext; +import run.mone.mimeter.dashboard.bo.common.EmitterTypeEnum; +import run.mone.mimeter.dashboard.bo.scene.SceneDTO; +import run.mone.mimeter.dashboard.bo.sla.*; +import run.mone.mimeter.dashboard.service.BenchBroadcastService; +import run.mone.mimeter.dashboard.service.MonitorInfoService; +import run.mone.mimeter.engine.agent.bo.stat.SysMonitorType; +import run.mone.mimeter.engine.agent.bo.task.CancelType; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskType; + +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import static common.Const.*; + +@Service +@Slf4j +public class SlaService { + + private final ExecutorService executorService = Executors.newVirtualThreadPerTaskExecutor(); + + /** + * 当前sla任务集 + */ + private final ConcurrentHashMap slaContextMap = new ConcurrentHashMap<>(); + + @Resource + private ManagerService managerService; + + @DubboReference(check = false, group = "${mimeter.monitor.dubbo.group}", interfaceClass = MiFeiShuService.class, timeout = 3000) + private MiFeiShuService miFeiShuService; + + @DubboReference(check = false, group = "${mimeter.monitor.dubbo.group}", interfaceClass = SendSmsService.class, timeout = 3000) + private SendSmsService sendSmsService; + @DubboReference(check = false, interfaceClass = BenchBroadcastService.class, group = "${mimeter.dashboard.dubbo.group}", version = "${mimeter.dashboard.dubbo.version}", timeout = 20000) + private BenchBroadcastService benchBroadcastService; + + /** + * 监控打点数据服务 + */ + @DubboReference(check = false, interfaceClass = PrometheusService.class,timeout = 5000) + private PrometheusService prometheusService; + + /** + * 获取trace应用列表服务 + */ + @DubboReference(check = false, group = "${mimeter.dashboard.dubbo.group}", version = "${mimeter.dashboard.dubbo.version}", interfaceClass = MonitorInfoService.class, timeout = 3000) + private MonitorInfoService monitorInfoService; + + @Resource(name = "$daoName:mibench_st_db", description = "mysql") + private NutDao dao; + + /** + * 周期检查时间 10s + * 即每 10s 核验一次 SLA + */ + private static final int SCHEDULE = 10; + + public void processSlaNotifyTask(SceneDTO sceneDTO, String reportId) { + SlaContext slaContext = new SlaContext(reportId, sceneDTO.getId(), false, false); + slaContextMap.put(reportId, slaContext); + executorService.submit(() -> { + try { + processSlaNotify(sceneDTO, slaContext); + } catch (InterruptedException e) { + log.error("processSlaNotify error,reportID:{},error:{}", reportId, e.getMessage()); + } + }); + } + + /** + * 校验规则,推送报警通知 + * + * @param sceneDTO + */ + private void processSlaNotify(SceneDTO sceneDTO, SlaContext slaContext) throws InterruptedException { + + if (sceneDTO.getSlaDto() == null) { + return; + } + if (sceneDTO.getSceneType() == SCENE_TYPE_HTTP) { + slaContext.setSceneType(TaskType.http.name()); + } else if (sceneDTO.getSceneType() == SCENE_TYPE_DUBBO) { + slaContext.setSceneType(TaskType.dubbo.name()); + } + long begin = System.currentTimeMillis(); + SlaDto slaDto = sceneDTO.getSlaDto(); + List alarmDtos = slaDto.getAlarmDtos(); + while (((System.currentTimeMillis() - begin) / 1000) <= (sceneDTO.getBenchTime() + SCHEDULE)) { + //校验、通知逻辑 + List rules = slaDto.getSlaRuleDtos(); + + rules.forEach(slaRule -> { + SlaEvent slaEvent = new SlaEvent(); + overRule(slaRule, slaContext, slaEvent, alarmDtos); + }); + + //结束事件通知 + if (slaContext.isCancel() || slaContext.isFinish()) { + //任务被用户手动停止,退出循环,退出线程 或 任务自然结束,退出循环,退出线程 + log.info("task is cancelled: reportId:{}", slaContext.getReportId()); + clearSlaContext(slaContext.getReportId()); + break; + } + + TimeUnit.SECONDS.sleep(SCHEDULE); + } + } + + private void doAlarm(AlarmDto alarmDto, String content) { + if (alarmDto != null) { + alarmDto.getAlarmMethods().forEach(method -> { + if (method == FEI_SHU_ALARM) { + //飞书 + List users = new ArrayList<>(); + alarmDto.getUsernames().forEach(userDTO -> { + String[] tmpArr = userDTO.getEmail().split("@",2); + users.add(tmpArr[0]); + }); + miFeiShuService.batchSendMsg(users, content); + } else if (method == 1) { + //todo 短信 + sendSmsService.batchSendSms("", "", ""); + } + }); + } + } + + /** + * 是否触发规则 + */ + private void overRule(SlaRuleDto slaRule, SlaContext slaContext, SlaEvent slaEvent, List alarmDtos) { + String ruleType = slaRule.getRuleItemType(); + ConcurrentHashMap labelCount = slaContext.getLabelCount(); + if (ruleType.equals(SlaRuleItemTypeEnum.BusinessMetrics.ruleItemTypeName)) { + Result result = null; + try { + result = prometheusService.getMimeterApiInfo(String.valueOf(slaContext.getSceneId()), slaContext.getSceneType()); + } catch (Exception e) { + log.error("prometheusService call error:{}", e.getMessage()); + } + if (result == null || result.getCode() != 0 || result.getData() == null) { + return; + } + MimeterApiRes apiMonitorMap = (MimeterApiRes) result.getData(); + log.info("prometheusService call result:{}", result); + //业务指标 + switch (slaRule.getRuleItem()) { + case SuccessRate -> { + //成功率 + Map successRateMap = apiMonitorMap.getSuccessRate(); + if (successRateMap != null) { + if (checkBusApiMonitorData(successRateMap, slaRule, slaEvent)) { + //超过接口成功率规则限制 + //校验敏感度 + checkDegree(slaRule, labelCount, slaContext, slaEvent, alarmDtos, SuccessRate); + } + } + } + case P99ResponseTime -> { + //p99 rt + //成功率 + Map p99Rt = apiMonitorMap.getP99Rt(); + if (p99Rt != null) { + if (checkBusApiMonitorData(p99Rt, slaRule, slaEvent)) { + //超过接口成功率规则限制 + //校验敏感度 + checkDegree(slaRule, labelCount, slaContext, slaEvent, alarmDtos, P99ResponseTime); + } + } + } + case AvgResponseTime -> { + //平均 rt + //成功率 + Map avgRt = apiMonitorMap.getAvgRt(); + if (avgRt != null) { + if (checkBusApiMonitorData(avgRt, slaRule, slaEvent)) { + //超过接口成功率规则限制 + //校验敏感度 + checkDegree(slaRule, labelCount, slaContext, slaEvent, alarmDtos, AvgResponseTime); + } + } + } + case RequestPerSecond -> { + //平均 qps + //成功率 + Map tps = apiMonitorMap.getTps(); + if (tps != null) { + if (checkBusApiMonitorData(tps, slaRule, slaEvent)) { + //超过接口成功率规则限制 + //校验敏感度 + checkDegree(slaRule, labelCount, slaContext, slaEvent, alarmDtos, RequestPerSecond); + } + } + } + } + } else if (ruleType.equals(SlaRuleItemTypeEnum.MonitorMetrics.ruleItemTypeName)) { + List appList = null; + try { + appList = monitorInfoService.getAppListByReportID(slaContext.getSceneId(), slaContext.getReportId()); + } catch (Exception e) { + log.error("monitorInfoService call error:{}", e.getMessage()); + } + if (appList == null || appList.size() == 0) { + //还未串联到应用 + return; + } + //监控指标 + switch (slaRule.getRuleItem()) { + case CpuUtilization -> { + //cpu指标 + if (checkSysMonitor(appList, slaRule, SysMonitorType.Cpu_Usage.name, slaEvent)) { + //超过cpu规则限制值 + checkDegree(slaRule, labelCount, slaContext, slaEvent, alarmDtos, CpuUtilization); + } + } + case MemoryUtilization -> { + //内存指标 + if (checkSysMonitor(appList, slaRule, SysMonitorType.Mem_Used.name, slaEvent)) { + //超过 mem 规则限制值 + checkDegree(slaRule, labelCount, slaContext, slaEvent, alarmDtos, MemoryUtilization); + } + } + case Load5Average -> { + //load5平均值指标 + if (checkSysMonitor(appList, slaRule, SysMonitorType.Load_Avg.name, slaEvent)) { + //超过平均load规则限制 + checkDegree(slaRule, labelCount, slaContext, slaEvent, alarmDtos, Load5Average); + } + } + case Load5Max -> { + //load5最大值指标 + if (checkSysMonitor(appList, slaRule, SysMonitorType.Load_Max.name, slaEvent)) { + //超过load5最大值指标 + checkDegree(slaRule, labelCount, slaContext, slaEvent, alarmDtos, Load5Max); + } + } + } + } + } + + /** + * 是否超过容忍度 + */ + private void checkDegree(SlaRuleDto slaRule, ConcurrentHashMap labelCount, SlaContext slaContext, + SlaEvent slaEvent, List alarmDtos, String labelName) { + log.info("checkDegree slaRule:{}", slaEvent); + //超过规则限制值 + String key = labelName + "_" + slaRule.getAction(); + if (labelCount.containsKey(key)) { + int currCount = labelCount.get(key).addAndGet(1); + if (currCount >= slaRule.getDegree()) { + //超过容忍度,需要处理 + processOverRule(slaRule, slaContext, slaEvent, alarmDtos); + //处理通知完后清除归零 + labelCount.remove(key); + } + } else { + labelCount.put(key, new AtomicInteger(1)); + if (1 >= slaRule.getDegree()) { + //超过容忍度,需要处理 + processOverRule(slaRule, slaContext, slaEvent, alarmDtos); + //处理通知完后清除归零 + labelCount.remove(key); + } + } + } + + /** + * 基于action,决定发送的事件,以及是否停止压测 + */ + private void processOverRule(SlaRuleDto slaRule, SlaContext slaContext, SlaEvent slaEvent, List alarmDtos) { + log.info("benchBroadcastService.processOverRule alarmDtos:{}", alarmDtos); + + if (alarmDtos == null || alarmDtos.size() == 0) { + return; + } + AlarmDto warnAlarm = null; + AlarmDto errorAlarm = null; + for (AlarmDto alarm : + alarmDtos) { + if (alarm.getAlarmType().equals(ActionEnum.WARNING.name())) { + warnAlarm = alarm; + } else if (alarm.getAlarmType().equals(ActionEnum.ERROR.name())) { + errorAlarm = alarm; + } + } + switch (slaRule.getAction()) { + case "WARNING" -> { + //报警 通知 + slaEvent.setAlarmLevel(slaRule.getAction()); + slaEvent.setCondition(slaRule.getCondition()); + slaEvent.setRuleItem(slaRule.getRuleItem()); + slaEvent.setSlaRuleName(slaRule.getName()); + slaEvent.setDegree(slaRule.getDegree()); + slaEvent.setValue(Double.valueOf(slaRule.getValue())); + slaEvent.setRuleItemType(slaRule.getRuleItemType()); + + String msg = "场景id:" + slaContext.getSceneId() + "压测触发警告规则:" + slaRule.getName(); + slaEvent.setMsg(msg); + + //推送事件流 + try { + log.info("benchBroadcastService.notifyEvent slaEvent:{}", slaEvent); + benchBroadcastService.notifyEvent(EmitterTypeEnum.SLA_WARN, slaContext.getReportId(), slaEvent); + + //通知报警接收人 + doAlarm(warnAlarm, msg); + } catch (Exception e) { + log.error("notify error :{}", e.getMessage()); + } + } + case "ERROR" -> { + //错误 停止 + slaEvent.setAlarmLevel(slaRule.getAction()); + slaEvent.setCondition(slaRule.getCondition()); + slaEvent.setRuleItem(slaRule.getRuleItem()); + slaEvent.setSlaRuleName(slaRule.getName()); + slaEvent.setDegree(slaRule.getDegree()); + slaEvent.setValue(Double.valueOf(slaRule.getValue())); + slaEvent.setRuleItemType(slaRule.getRuleItemType()); + + String msg = "场景id:" + slaContext.getSceneId() + "压测触发错误规则:" + slaRule.getName() + ",即将停止压测"; + + slaEvent.setMsg(msg); + try { + //推送事件流 + benchBroadcastService.notifyEvent(EmitterTypeEnum.SLA_ERROR, slaContext.getReportId(), slaEvent); + + //通知报警接收人 + doAlarm(errorAlarm, msg); + } catch (Exception e) { + log.error("notify error :{}", e.getMessage()); + } + + //停止压测 + taskQuitBySlaRule(slaContext.getReportId(), msg); + } + } + } + + private boolean checkBusApiMonitorData(Map apiDataMap, SlaRuleDto slaRule, SlaEvent slaEvent) { + //检验成功率 + log.info("checkBusApiMonitorData apiDataMap:{}", apiDataMap); + + AtomicBoolean ok = new AtomicBoolean(false); + if (apiDataMap != null) { + for (Map.Entry entry : apiDataMap.entrySet()) { + if (checkNumValueAndRule(Double.parseDouble(entry.getValue()), slaRule)) { + log.info("checkBusApiMonitorData checkNumValueAndRule,value:{}", entry.getValue()); + //触发的接口id + slaEvent.setTriggerApiId(Integer.parseInt(entry.getKey())); + slaEvent.setTriggerItem("Api_Id:" + entry.getKey()); + //该接口的监控值 + slaEvent.setRuleTargetValue(Double.parseDouble(entry.getValue())); + ok.set(true); + break; + } + } + return ok.get(); + } + return true; + } + + private boolean checkSysMonitor(List appList, SlaRuleDto slaRule, String sysMonitorType, SlaEvent slaEvent) { + StringBuffer appNameStr = new StringBuffer(); + appList.forEach(appName -> appNameStr.append(",").append(appName)); + Result result = prometheusService.getAppSystemInfo(appNameStr.toString(), sysMonitorType); + if (result.getCode() != 0 || result.getData() == null) { + return false; + } + Map resultMap = (Map) result.getData(); + AtomicBoolean match = new AtomicBoolean(false); + for (Map.Entry entry : + resultMap.entrySet()) { + if (checkNumValueAndRule((Double.parseDouble(entry.getValue())), slaRule)) { + match.set(true); + //触发的接口id + slaEvent.setTriggerItem("App_Name:" + entry.getKey()); + //该接口的监控值 + slaEvent.setRuleTargetValue(Double.parseDouble(entry.getValue())); + break; + } + } + return match.get(); + } + + private boolean checkNumValueAndRule(double value, SlaRuleDto slaRule) { + log.info("checkBusApiMonitorData checkNumValueAndRule,slaRule:{}", slaRule); + switch (slaRule.getCondition()) { + case ">" -> { + if (value > slaRule.getValue()) { + return true; + } + } + case "≥" -> { + if (value >= slaRule.getValue()) { + return true; + } + } + case "=" -> { + if (value == slaRule.getValue()) { + return true; + } + } + case "≤" -> { + if (value <= slaRule.getValue()) { + return true; + } + } + default -> { + return false; + } + } + return false; + } + + /** + * 压测任务手动结束需要停止sla任务,移出map + */ + public void stopSlaTaskByManual(String reportId) { + SlaContext slaContext = slaContextMap.get(reportId); + if (slaContext != null) { + slaContext.setCancel(true); + } + } + + /** + * 压测任务自然结束,需要停止sla任务,移出map + */ + public void slaTaskFinish(String reportId) { + SlaContext slaContext = slaContextMap.get(reportId); + if (slaContext != null) { + slaContext.setCancel(true); + } + } + + /** + * 基于规则停止压测任务 + */ + private void taskQuitBySlaRule(String reportId, String ruleContent) { + Task task = new Task(); + + List mibenchTasks = dao.query(MibenchTask.class, Cnd.where("report_id", "=", reportId)); + + List taskIds = mibenchTasks.stream().map(MibenchTask::getId).collect(Collectors.toList()); + task.setIds(taskIds); + task.setCancelType(CancelType.BySla.code); + task.setCancelBySlaRule(ruleContent); + managerService.cancelTask(task); + + //清除 内存中的数据 + clearSlaContext(reportId); + } + + + /** + * 压测任务自然结束、手动结束、sla触发结束,都需要停止sla任务,移出map + */ + public void clearSlaContext(String reportId) { + slaContextMap.remove(reportId); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/bo/AgentInfo.java b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/bo/AgentInfo.java new file mode 100644 index 000000000..0590f494a --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/bo/AgentInfo.java @@ -0,0 +1,56 @@ +package run.mone.mimeter.agent.manager.bo; + +import lombok.Data; +import org.nutz.dao.entity.annotation.Column; +import org.nutz.dao.entity.annotation.Id; + +import java.io.Serializable; + +/** + * 入库的agent信息 + */ +@Data +public class AgentInfo implements Serializable { + + @Id + private Integer id; + + @Column(value = "server_name") + private String serverName; + @Column + private String ip; + + @Column + private String hostname; + + @Column + private int port; + + @Column + private int cpu; + + @Column + private long mem; + + @Column(value = "use_cpu") + private int useCpu; + + @Column(value = "use_mem") + private long useMem; + + @Column + private Long utime; + + @Column + private Long ctime; + + @Column(value = "client_desc") + private String clientDesc; + + @Column(value = "node_ip") + private String nodeIp; + + @Column + private boolean enable; + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/bo/MibenchTask.java b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/bo/MibenchTask.java new file mode 100644 index 000000000..922a50730 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/bo/MibenchTask.java @@ -0,0 +1,139 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.agent.manager.bo; + +import lombok.Data; +import org.nutz.dao.entity.annotation.Column; +import org.nutz.dao.entity.annotation.Id; +import org.nutz.dao.entity.annotation.Table; + +/** + * @author goodjava@qq.com + * @date 2022/5/23 + * 实际执行的压测任务 + */ +@Table("mibench_task") +@Data +public class MibenchTask { + + @Id + private int id; + + @Column(value = "scene_id") + private int sceneId; + + @Column(value = "serial_link_id") + private int serialLinkId; + @Column(value = "scene_api_id") + private int sceneApiId; + + /** + * 1 http 接口任务 + * 2 dubbo 接口任务 + * 3 dag 图任务 + */ + @Column(value = "task_type") + private int taskType; + + @Column + private int qps; + + @Column(value = "origin_qps") + private int originQps; + + @Column(value = "max_qps") + private int maxQps; + + @Column + private int time; + + @Column(value = "agent_num") + private int agentNum; + + @Column(value = "finish_agent_num") + private int finishAgentNum; + + @Column + private long ctime; + + @Column + private long utime; + + @Column + private int state; + + @Column(value = "version", version = true) + private int version; + + @Column(value = "success_num") + private long successNum; + + @Column(value = "failure_num") + private long failureNum; + + @Column(value = "parent_task_id") + private int parentTaskId; + + @Column(value = "debug_result") + private String debugResult; + + @Column(value = "debug_url") + private String debugUrl; + + @Column(value = "debug_result_header") + private String debugResultHeader; + + @Column(value = "debug_req_headers") + private String debugReqHeaders; + + @Column(value = "debug_trigger_cp") + private String debugTriggerCp; + + @Column(value = "debug_trigger_filter_condition") + private String debugTriggerFilterCondition; + + @Column(value = "report_id") + private String reportId; + + @Column(value = "request_params") + private String requestParams; + + @Column(value = "req_param_type") + private int reqParamType; + + @Column(value = "ok") + private boolean ok; + + @Column(value = "connect_task_num") + private int connectTaskNum; + + @Column(value = "debug_rt") + private int debugRt; + + @Column(value = "debug_size") + private int debugSize; + + @Column(value = "bench_mode") + private int benchMode; + + @Column(value = "increase_mode") + private int increaseMode; + + @Column(value = "increase_percent") + private int increasePercent; + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/bo/SlaContext.java b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/bo/SlaContext.java new file mode 100644 index 000000000..fd515fa8d --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/bo/SlaContext.java @@ -0,0 +1,48 @@ +package run.mone.mimeter.agent.manager.bo; + +import lombok.Data; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +@Data +public class SlaContext { + + /** + * 所属任务标识 + */ + private volatile String reportId; + + /** + * 场景id + */ + private volatile int sceneId; + + /** + * 场景类型 + */ + private volatile String sceneType; + + /** + * 完成 + */ + private volatile boolean finish; + + /** + * 被取消掉 + */ + private volatile boolean cancel; + + /** + * 指标名,对应次数触发规则 or + */ + private final ConcurrentHashMap labelCount = new ConcurrentHashMap<>(); + + public SlaContext(String reportId, int sceneId, boolean finish, boolean cancel) { + this.reportId = reportId; + this.sceneId = sceneId; + this.finish = finish; + this.cancel = cancel; + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/processor/ManagerProcessor.java b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/processor/ManagerProcessor.java new file mode 100644 index 000000000..c662dbb4c --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/processor/ManagerProcessor.java @@ -0,0 +1,284 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.agent.manager.processor; + +import com.google.gson.Gson; +import com.xiaomi.data.push.rpc.netty.NettyRequestProcessor; +import com.xiaomi.data.push.rpc.protocol.RemotingCommand; +import com.xiaomi.youpin.docean.anno.Component; +import common.Util; +import io.netty.channel.ChannelHandlerContext; +import org.apache.dubbo.annotation.DubboReference; +import org.nutz.dao.impl.NutDao; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import run.mone.mimeter.agent.manager.DataStatService; +import run.mone.mimeter.agent.manager.ManagerService; +import run.mone.mimeter.agent.manager.SlaService; +import run.mone.mimeter.agent.manager.bo.MibenchTask; +import run.mone.mimeter.dashboard.bo.common.EmitterTypeEnum; +import run.mone.mimeter.dashboard.bo.report.ReportInfoBo; +import run.mone.mimeter.dashboard.service.BenchBroadcastService; +import run.mone.mimeter.dashboard.service.SceneInfoService; +import run.mone.mimeter.engine.agent.bo.MibenchCmd; +import run.mone.mimeter.engine.agent.bo.data.AgentReq; +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.task.CancelType; +import run.mone.mimeter.engine.agent.bo.task.TaskResult; +import run.mone.mimeter.engine.agent.bo.task.TaskStatus; +import run.mone.mimeter.engine.agent.bo.task.TaskStatusBo; + +import javax.annotation.Resource; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author goodjava@qq.com + * @author dongzhenxing + * @date 2022/5/11 + * 用来处理agent回推的结果 + */ +@Component +public class ManagerProcessor implements NettyRequestProcessor { + + private static final Logger log = LoggerFactory.getLogger(ManagerProcessor.class); + + private static final String logPrefix = "[ManagerProcessor]"; + + private final Gson gson = Util.getGson(); + + @Resource(name = "$daoName:mibench_st_db", description = "mysql") + private NutDao dao; + + @DubboReference(check = false, group = "${mimeter.dashboard.dubbo.group}", version = "${mimeter.dashboard.dubbo.version}", interfaceClass = SceneInfoService.class, timeout = 3000) + private SceneInfoService sceneInfoService; + + @DubboReference(check = false, group = "${mimeter.dashboard.dubbo.group}", version = "${mimeter.dashboard.dubbo.version}", interfaceClass = BenchBroadcastService.class, timeout = 20000) + private BenchBroadcastService benchBroadcastService; + + /** + * 数据统计服务 + */ + @Resource + private DataStatService dataStatService; + + /** + * 记录场景单次压测完成的任务数 + * <"reportId",4> + */ + private final ConcurrentHashMap reportFinishNum = new ConcurrentHashMap<>(); + + /** + * 记录每个任务执行完成的发压机数量 + * <"taskId",4> + */ + private final ConcurrentHashMap taskFinishAgentNum = new ConcurrentHashMap<>(); + + + @Resource + private SlaService slaService; + + @Resource + private ManagerService managerService; + + @Override + public RemotingCommand processRequest(ChannelHandlerContext channelHandlerContext, RemotingCommand remotingCommand) throws InterruptedException { + AgentReq req = remotingCommand.getReq(AgentReq.class); + String logMsg = logPrefix + "processRequest "; + switch (req.getCmd()) { + case AgentReq.TASK_RESULT_CMD ->//处理完成返回结果事件 + processTREvent(req, logMsg); + case AgentReq.TOTAL_DATA_COUNT_CMD ->//处理数据统计事件 + dataStatService.processTotalCountCtxEvent(req); + case AgentReq.UP_TASK_STATUS ->//处理任务状态变更事件 + processUpStatusEvent(req, logMsg); + } + return null; + } + + + private synchronized void processTREvent(AgentReq req, String logMsg) { + //处理回调过来的返回数据 + TaskResult tr = req.getTaskResult(); + try { + MibenchTask task = dao.fetch(MibenchTask.class, tr.getId()); + //场景调试的情况 + if (tr.isDebug()) { + // 更新场景接口结果数据 调用 dashboard 接口,更新scene api info debug result + if (tr.getSuccess().get() == 1) { + task.setState(TaskStatus.Success.code); + } else if (tr.getFailure().get() == 1) { + task.setState(TaskStatus.Failure.code); + } + task.setOk(tr.isOk()); + CommonReqInfo commonReqInfo = tr.getCommonReqInfo(); + if (commonReqInfo != null) { + task.setReqParamType(commonReqInfo.getParamsType()); + task.setRequestParams(commonReqInfo.getParamJson()); + if (commonReqInfo.getHeaders() != null) { + task.setDebugReqHeaders(gson.toJson(commonReqInfo.getHeaders())); + } + if (commonReqInfo.getDebugUrl() != null){ + task.setDebugUrl(commonReqInfo.getDebugUrl()); + } + } + task.setDebugResult(tr.getResult()); + if (tr.getRespHeaders() != null) { + task.setDebugResultHeader(gson.toJson(tr.getRespHeaders())); + } + if (tr.getTriggerCpInfo() != null) { + task.setDebugTriggerCp(tr.getTriggerCpInfo()); + } + if (tr.getTriggerFilterCondition() != null){ + task.setDebugTriggerFilterCondition(tr.getTriggerFilterCondition()); + } + task.setDebugRt((int) tr.getRt()); + task.setDebugSize((int) tr.getSize()); + dao.update(task); + } else { + //执行压测的情况下 + String quitMsg = ""; + int finNum = countTaskFinAgent(task.getId()); + if (checkBenchTaskFinish(finNum, task.getAgentNum())) { + //即该链路任务全部执行结束 + task.setState(TaskStatus.Success.code); + //结束链路任务状态 + dao.update(task); + //手动结束 需要通知结束报告 + if (tr.isQuitByManual()) { + //手动的需要通知sla任务停止 + if (tr.getCancelType() == CancelType.Manual.code) { + slaService.stopSlaTaskByManual(tr.getReportId()); + quitMsg = "当前压测任务已手动停止,操作人: " + tr.getOpUser() + ",停止时间: " + getDateTime(System.currentTimeMillis()); + } + if (tr.getCancelType() == CancelType.BySla.code) { + slaService.slaTaskFinish(tr.getReportId()); + quitMsg = "当前压测任务触发SLA警报已停止,触发规则: " + tr.getCancelBySlaRule() + ",停止时间: " + getDateTime(System.currentTimeMillis()); + } + } else { + //自然结束 + //更新场景状态 + sceneInfoService.updateSceneStatus(tr.getSceneId(), TaskStatus.Success.code); + + slaService.slaTaskFinish(tr.getReportId()); + quitMsg = "当前压测任务已完成,结束时间: " + getDateTime(System.currentTimeMillis()); + } + //检查整个场景全部任务是否完成 + int finishTaskNum; + if (reportFinishNum.containsKey(task.getReportId())) { + finishTaskNum = reportFinishNum.get(task.getReportId()).incrementAndGet(); + } else { + reportFinishNum.put(task.getReportId(), new AtomicInteger(1)); + finishTaskNum = reportFinishNum.get(task.getReportId()).get(); + } + if (finishTaskNum >= task.getConnectTaskNum()) { + //本次压测所有任务完成再通知dashboard + try { + this.notifyFinish(task, quitMsg); + } catch (Exception e) { + log.error(logMsg + "======FAILED TO NOTIFY FINISH!", e); + } + } + } + } + } catch (Exception ex) { + log.error(logMsg + " throws error cmd " + req.getCmd() + "; " + ex.getMessage()); + } + } + + private void processUpStatusEvent(AgentReq req, String logMsg) { + //更新状态 + try { + TaskStatusBo statusBo = req.getStatusBo(); + log.info("processUpStatusEvent task id :{}", statusBo.getTaskId()); + MibenchTask dagTask = dao.fetch(MibenchTask.class, statusBo.getTaskId()); + log.info("processUpStatusEvent dag task :{}", dagTask); + if (dagTask == null){ + Thread.sleep(500); + dagTask = dao.fetch(MibenchTask.class, statusBo.getTaskId()); + } + if (dagTask == null){ + log.error("processUpStatusEvent dag is null"); + return; + } + dagTask.setState(statusBo.getTaskStatus().code); + dagTask.setUtime(System.currentTimeMillis()); + int v = dao.update(dagTask); + if (v > 0) { + if (statusBo.getTaskStatus() == TaskStatus.Stopped) { + //已经停止,更新场景状态为 停止 + sceneInfoService.updateSceneStatus(statusBo.getSceneId(), TaskStatus.Stopped.code); + }else if (statusBo.getTaskStatus() == TaskStatus.Running){ + //任务启动 + log.info("taskRunningNotify in bench mode:{},increase mode :{},dagTask increasePercent:{}", dagTask.getBenchMode(),dagTask.getIncreaseMode(),dagTask.getIncreasePercent()); + managerService.taskRunningNotify(dagTask); + } + log.info(logMsg + "update task cmd " + req.getCmd()); + } + } catch (Exception ex) { + log.error(logMsg + " throws error cmd " + req.getCmd() + "; " + ex.getMessage()); + } + } + + /** + * 累计该任务执行完成的压测机数量 + */ + private int countTaskFinAgent(int taskId) { +// synchronized (taskFinishAgentNum) { + if (!taskFinishAgentNum.containsKey(taskId)) { + taskFinishAgentNum.put(taskId, new AtomicInteger(1)); + return 1; + } else { + return taskFinishAgentNum.get(taskId).incrementAndGet(); + } +// } + } + + private static boolean checkBenchTaskFinish(int finNum, int useAgentNum) { + return finNum >= useAgentNum; + } + + private void notifyFinish(MibenchTask task, String quitMsg) { + //扣除前期大概的初始化时间 + long now = System.currentTimeMillis() - 500; + this.benchBroadcastService.notifyEvent(EmitterTypeEnum.FINISH, task.getReportId(), ReportInfoBo.builder() + .sceneId((long) task.getSceneId()) + .reportId(task.getReportId()) + .duration((int) ((now - task.getCtime()) / 1000)) + .finishTime(now) + .extra(quitMsg) + .build()); + } + + private String getDateTime(long now) { + Date millisecondDate = new Date(now); + SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + return formatter.format(millisecondDate); + } + + @Override + public boolean rejectRequest() { + return false; + } + + @Override + public int cmdId() { + return MibenchCmd.MANAGER; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/processor/PingProcessor.java b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/processor/PingProcessor.java new file mode 100644 index 000000000..028ac02db --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/java/run/mone/mimeter/agent/manager/processor/PingProcessor.java @@ -0,0 +1,157 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.agent.manager.processor; + +import com.google.common.collect.Lists; +import com.xiami.mione.tesla.k8s.service.K8sProxyService; +import com.xiaomi.data.push.context.AgentContext; +import com.xiaomi.data.push.rpc.common.RemotingHelper; +import com.xiaomi.data.push.rpc.netty.AgentChannel; +import com.xiaomi.data.push.rpc.netty.NettyRequestProcessor; +import com.xiaomi.data.push.rpc.protocol.RemotingCommand; +import com.xiaomi.mione.tesla.k8s.bo.PodNode; +import com.xiaomi.youpin.docean.anno.Component; +import com.xiaomi.youpin.docean.plugin.dubbo.anno.Reference; +import io.netty.channel.ChannelHandlerContext; +import org.nutz.dao.Cnd; +import org.nutz.dao.impl.NutDao; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeanUtils; +import org.springframework.util.StringUtils; +import run.mone.mimeter.agent.manager.bo.AgentInfo; +import run.mone.mimeter.engine.agent.bo.data.AgentInfoDTO; +import run.mone.mimeter.engine.agent.bo.data.AgentReq; +import run.mone.mimeter.engine.agent.bo.MibenchCmd; + +import javax.annotation.Resource; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Optional; + +/** + * @author goodjava@qq.com + * @date 2022/5/11 + *

+ * 处理agent 发过来的ping信息 + */ +@Component +public class PingProcessor implements NettyRequestProcessor { + + private static final Logger log = LoggerFactory.getLogger(PingProcessor.class); + + + @Reference(interfaceClass = K8sProxyService.class, group = "${k8s.proxy.dubbo.group}", timeout = 10000, check = false) + private K8sProxyService k8sProxyService; + + @Resource(name = "$daoName:mibench_st_db", description = "mysql") + private NutDao dao; + + @Override + public RemotingCommand processRequest(ChannelHandlerContext ctx, RemotingCommand remotingCommand) throws Exception { + AgentReq req = remotingCommand.getReq(AgentReq.class); + Optional.ofNullable(req.getUser()).ifPresent(it -> { + log.info("client:{} ping", req.getUser().getName()); + final String remoteAddress = RemotingHelper.parseChannelRemoteAddr(ctx.channel()); + AgentChannel ch = AgentContext.ins().map.get(remoteAddress); + if (null == ch) { + ch = new AgentChannel(); + ch.setChannel(ctx.channel()); + AgentContext.ins().map.putIfAbsent(remoteAddress, ch); + } + // 机器信息入库 + insertAgentInfoToDB(req); + }); + + RemotingCommand res = RemotingCommand.createResponseCommand(100, "ok"); + res.setBody("ok".getBytes(StandardCharsets.UTF_8)); + return res; + } + + @Override + public boolean rejectRequest() { + return false; + } + + @Override + public int cmdId() { + return MibenchCmd.PING; + } + + /** + * agent入库 + * + * @param req + */ + private void insertAgentInfoToDB(AgentReq req) { + try { + AgentInfoDTO agentInfoDTO = req.getAgentInfoDTO(); + long now = System.currentTimeMillis(); + if (agentInfoDTO.getIp() == null || agentInfoDTO.getIp().equals("")) { + return; + } + + String nodeIp = ""; + List agents = k8sProxyService.getNodeIP(Lists.newArrayList(agentInfoDTO.getIp())).getData(); + if (agents != null && agents.size() != 0) { + log.info("[PingProcessor.insertAgentInfoToDB], getNodeIP agents: {}, podIp: {}", agents, agentInfoDTO.getIp()); + nodeIp = agents.get(0).getNodeIP(); + } else { + //如果最终获取不到nodeId + nodeIp = agentInfoDTO.getIp() + "_" + agentInfoDTO.getDesc(); + } + AgentInfo agentInfo = StringUtils.isEmpty(nodeIp) + ? null + : dao.fetch(AgentInfo.class, Cnd.where("node_ip", "=", nodeIp)); + + if (agentInfo != null) { + //更新 + agentInfo.setUseCpu(agentInfoDTO.getUseCpu()); + agentInfo.setUseMem(agentInfoDTO.getUseMem()); + agentInfo.setIp(agentInfoDTO.getIp()); + agentInfo.setEnable(true); + agentInfo.setUtime(now); + agentInfo.setClientDesc(agentInfoDTO.getDesc()); + log.info("old !!! update agent info podIp:{}, nodeIp:{}, agentInfo: {}", agentInfoDTO.getIp(), nodeIp, agentInfo); + dao.update(agentInfo); + } else { + AgentInfo ipAgentInfo = dao.fetch(AgentInfo.class, Cnd.where("ip", "=", agentInfoDTO.getIp())); + if (ipAgentInfo != null) { + ipAgentInfo.setNodeIp(nodeIp); + ipAgentInfo.setUtime(now); + ipAgentInfo.setEnable(true); + log.info("new !!! update agent info podIp:{},nodeIp:{}", agentInfoDTO.getIp(), nodeIp); + dao.update(ipAgentInfo); + } else { + agentInfo = new AgentInfo(); + BeanUtils.copyProperties(agentInfoDTO, agentInfo); + agentInfo.setCtime(now); + agentInfo.setUtime(now); + agentInfo.setEnable(true); + agentInfo.setClientDesc(agentInfoDTO.getDesc()); + agentInfo.setNodeIp(nodeIp); + log.info("new!!! insert agent info podIp:{},nodeIp:{}", agentInfoDTO.getIp(), nodeIp); + dao.insert(agentInfo); + } + } + } catch (Exception e) { + log.error("update agent info error:{}", e.getMessage()); + } + } + + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/resources/upload.html b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/resources/upload.html new file mode 100644 index 000000000..05ed87388 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-manager/src/main/resources/upload.html @@ -0,0 +1,17 @@ + + + + +

+
+ + +
+
+ +
+
+ + + + \ No newline at end of file diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/pom.xml b/ozburst-all/mimeter-engine/mimeter-agent-service/pom.xml new file mode 100644 index 000000000..588b5030c --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/pom.xml @@ -0,0 +1,156 @@ + + + + mimeter-engine + run.mone + 1.0-SNAPSHOT + + 4.0.0 + + mimeter-agent-service + + + + + + + + + + run.mone + mimeter-api + 1.0-SNAPSHOT + + + + com.xiaomi.youpin + tesla-filter-api + 1.0.1-SNAPSHOT + + + run.mone + mimeter-dashboard-api + 1.0.0-SNAPSHOT + + + com.xiaomi.youpin + youpin-infra-rpc + + + + + + javax.annotation + javax.annotation-api + 1.3.2 + provided + + + + ch.qos.logback + logback-classic + 1.1.2 + provided + + + + ch.qos.logback + logback-core + 1.1.2 + provided + + + + junit + junit + 4.12 + test + + + + run.mone + log + 1.6.1-SNAPSHOT + + + + run.mone + event + 1.4-SNAPSHOT + + + + com.xiaomi.mone + sautumn-serverless-api + 1.0.0-SNAPSHOT + provided + + + + run.mone + docean + 1.4-SNAPSHOT + + + + + run.mone + http + 1.4-SNAPSHOT + + + fastjson + com.alibaba + + + + + + + run.mone + docean-plugin-rpc + 1.4-SNAPSHOT + provided + + + + com.xiaomi.youpin + prometheus-client + 0.0.5-SNAPSHOT + + + + com.google.guava + guava + 25.1-jre + provided + + + + run.mone + antlr + 1.4-SNAPSHOT + + + + com.xiaomi.youpin + hera-trace + 1.4-SNAPSHOT + + + + com.xiaomi.youpin + tesla-traffic-recording-api + 1.0.0-SNAPSHOT + + + org.apache.dubbo + dubbo + + + + + + \ No newline at end of file diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/bo/DataMapCache.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/bo/DataMapCache.java new file mode 100644 index 000000000..f47cc97d1 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/bo/DataMapCache.java @@ -0,0 +1,18 @@ +package run.mone.mimeter.engine.bo; + +import lombok.Data; + +import java.util.List; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; + +@Data +public class DataMapCache { + private AtomicInteger countFinLinkNum; + private TreeMap> dataMap; + + public DataMapCache(AtomicInteger countFinLinkNum, TreeMap> dataMap) { + this.countFinLinkNum = countFinLinkNum; + this.dataMap = dataMap; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/bo/TaskStatus.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/bo/TaskStatus.java new file mode 100644 index 000000000..addfec207 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/bo/TaskStatus.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.bo; + +/** + * + * @author zhangzhiyong + * @date 29/05/2018 + */ +public enum TaskStatus { + Init(0), + Success(1), + Failure(2), + Retry(3), + Running(4), + ; + + public int code; + + private TaskStatus(int code) { + this.code = code; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/base/BaseClient.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/base/BaseClient.java new file mode 100644 index 000000000..05c08b250 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/base/BaseClient.java @@ -0,0 +1,214 @@ +package run.mone.mimeter.engine.client.base; + +import com.google.common.collect.Lists; +import com.xiaomi.youpin.docean.Ioc; +import com.xiaomi.youpin.docean.anno.Component; +import com.xiaomi.youpin.docean.listener.event.EventType; +import lombok.extern.slf4j.Slf4j; +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.data.Result; +import run.mone.mimeter.engine.agent.bo.stat.SceneTotalCountContext; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskContext; +import run.mone.mimeter.engine.filter.common.BasePostFilter; +import run.mone.mimeter.engine.filter.common.BasePreFilter; +import run.mone.mimeter.engine.filter.common.FilterOrder; +import run.mone.mimeter.engine.filter.common.MimeterFilterInfo; +import run.mone.mimeter.engine.filter.postFilter.IPostFilterChain; +import run.mone.mimeter.engine.filter.postFilter.PostFilterAnno; +import run.mone.mimeter.engine.filter.postFilter.filters.MimeterPostFilter; +import run.mone.mimeter.engine.filter.preFilter.filters.MimeterPreFilter; +import run.mone.mimeter.engine.filter.postFilter.PostFilter; +import run.mone.mimeter.engine.filter.preFilter.IPreFilterChain; +import run.mone.mimeter.engine.filter.preFilter.PreFilter; +import run.mone.mimeter.engine.filter.preFilter.PreFilterAnno; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; + +/** + * @author dongzhenxing + */ +@Slf4j +@Component +public class BaseClient implements IClient, IPreFilterChain, IPostFilterChain { + + private final CopyOnWriteArrayList preFilterList = new CopyOnWriteArrayList<>(); + + private final CopyOnWriteArrayList postFilterList = new CopyOnWriteArrayList<>(); + + + private volatile BasePreFilter lastFilter; + + private volatile BasePostFilter firstFilter; + + /** + * 前置过滤器链是否初始化完成 + */ + private final AtomicBoolean preFilterInit = new AtomicBoolean(false); + + /** + * 后置过滤器链是否初始化完成 + */ + private final AtomicBoolean postFilterInit = new AtomicBoolean(false); + + + @Override + public Result call(Task task, TaskContext context, CommonReqInfo commonReqInfo, SceneTotalCountContext totalCountContext) { + return Result.success("ok"); + } + + @Override + public CommonReqInfo doPreFilter(Task task,TaskContext context, CommonReqInfo reqInfo) { + if (!preFilterInit.get()) { + //filter 没加载完成,使用原参数 + return reqInfo; + } + try { + return this.lastFilter.doFilter(task, context,reqInfo); + } catch (Exception e) { + log.error("do filter failed:{}", e.getMessage()); + //filter 执行错误,使用原参数 + return reqInfo; + } + } + + @Override + public Object doPostFilter(Task task, Object resInfo) { + if (!postFilterInit.get()) { + //filter 没加载完成,返回结果不做处理 + return resInfo; + } + try { + return this.firstFilter.doFilter(task, resInfo); + } catch (Exception e) { + log.error("do filter failed:{}", e.getMessage()); + //filter 执行错误,使用原参数 + return resInfo; + } + } + + public void init() { + //加载前置过滤器 + this.loadPreFilter(); + + //加载后置过滤器 + this.loadPostFilter(); + } + + private void loadPreFilter() { + Ioc bizIoc = Ioc.ins().getBean("bizIoc"); + bizIoc.regListener(event -> { + if (event.getEventType().equals(EventType.initFinish)){ + Map preFilterMaps = bizIoc.getBeansWithAnnotation(PreFilterAnno.class); + List preList = new ArrayList<>(preFilterMaps.values()); + log.info("pre filter size:{}", preList.size()); + List preFilters = preList.stream().map(filter -> (PreFilter) filter).toList(); + preFilters = sortPreFilterList(preFilters); + preFilterList.addAll(preFilters); + + try { + BasePreFilter last = new MimeterPreFilter(); + int size = preFilters.size(); + for (int i = 0; i < size; i++) { + log.info("init pre filter index:{}", i); + PreFilter filter = preFilters.get(i); + //最后一个为默认filter + BasePreFilter next = last; + last = (task,reqInfo,context) -> filter.doFilter(task, reqInfo,context, next); + } + log.info("init pre filter finish"); + this.lastFilter = last; + } finally { + log.info("load pre filter end"); + } + preFilterInit.set(true); + } + }); + } + + private void loadPostFilter() { + Ioc bizIoc = Ioc.ins().getBean("bizIoc"); + bizIoc.regListener(event -> { + if (event.getEventType().equals(EventType.initFinish)){ + Map postFilterMaps = bizIoc.getBeansWithAnnotation(PostFilterAnno.class); + List postList = new ArrayList<>(postFilterMaps.values()); + log.info("post filter size:{}", postList.size()); + List postFilters = postList.stream().map(filter -> (PostFilter) filter).toList(); + postFilters = sortPostFilterList(postFilters); + postFilterList.addAll(postFilters); + + try { + BasePostFilter last = new MimeterPostFilter(); + int size = postFilters.size(); + for (int i = 0; i < size; i++) { + log.info("init post filter index:{}", i); + PostFilter filter = postFilters.get(i); + //最后一个为默认filter + BasePostFilter next = last; + last = (task, resInfo) -> filter.doFilter(task, resInfo, next); + } + log.info("init post filter finish"); + this.firstFilter = last; + } finally { + log.info("load post filter end"); + } + postFilterInit.set(true); + } + }); + } + + /** + * 前置filter 按order顺序倒序 如:5,4,3,2,1 + */ + private List sortPreFilterList(List list) { + return list.stream().sorted((a, b) -> { + Integer x = a.getClass().getAnnotation(FilterOrder.class).value(); + Integer y = b.getClass().getAnnotation(FilterOrder.class).value(); + return y.compareTo(x); + }).collect(Collectors.toList()); + } + + /** + * 后置filter 按order顺序正序 如:1,2,3,4,5 + */ + private List sortPostFilterList(List list) { + return list.stream().sorted((a, b) -> { + Integer x = a.getClass().getAnnotation(FilterOrder.class).value(); + Integer y = b.getClass().getAnnotation(FilterOrder.class).value(); + return x.compareTo(y); + }).collect(Collectors.toList()); + } + + /** + * 获取前置过滤器列表 + */ + public List getPreFilterInfoList() { + if (this.preFilterList.size() > 0) { + return this.preFilterList.stream().map(it -> { + MimeterFilterInfo info = new MimeterFilterInfo(); + info.setFilterName(it.getClass().getCanonicalName()); + return info; + }).collect(Collectors.toList()); + } + return Lists.newArrayList(); + } + + /** + * 获取后置过滤器列表 + */ + public List getPostFilterInfoList() { + if (this.postFilterList.size() > 0) { + return this.postFilterList.stream().map(it -> { + MimeterFilterInfo info = new MimeterFilterInfo(); + info.setFilterName(it.getClass().getCanonicalName()); + return info; + }).collect(Collectors.toList()); + } + return Lists.newArrayList(); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/base/DagClient.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/base/DagClient.java new file mode 100644 index 000000000..538ff0fd0 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/base/DagClient.java @@ -0,0 +1,373 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.client.base; + +import com.google.common.base.Stopwatch; +import com.google.gson.JsonArray; +import com.xiaomi.data.push.antlr.expr.Expr; +import com.xiaomi.data.push.graph.Graph; +import com.xiaomi.data.push.graph.Vertex; +import com.xiaomi.data.push.schedule.task.graph.GraphTaskContext; +import com.xiaomi.data.push.schedule.task.graph.TaskVertexData; +import com.xiaomi.youpin.docean.Ioc; +import com.xiaomi.youpin.docean.anno.Component; +import com.xiaomi.youpin.docean.common.MutableObject; +import com.xiaomi.youpin.docean.common.StringUtils; +import common.Replacer; +import common.Util; +import lombok.extern.slf4j.Slf4j; +import run.mone.event.Event; +import run.mone.mimeter.dashboard.bo.sceneapi.OutputOriginEnum; +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.data.NodeInfo; +import run.mone.mimeter.engine.agent.bo.data.Result; +import run.mone.mimeter.engine.agent.bo.stat.SceneTotalCountContext; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskContext; +import run.mone.mimeter.engine.agent.bo.task.TaskResult; +import run.mone.mimeter.engine.agent.bo.task.TaskType; +import run.mone.mimeter.engine.bo.TaskStatus; +import run.mone.mimeter.engine.common.ErrorCode; +import run.mone.mimeter.engine.common.Safe; +import run.mone.mimeter.engine.service.DatasetService; + +import javax.annotation.Resource; +import java.util.*; +import java.util.concurrent.*; +import java.util.stream.Collectors; + +import static common.Const.TASK_CTX_LINE_FLAG; + +/** + * @author goodjava@qq.com + * @author dongzhenxing + * @date 2022/5/30 + */ +@Component(name = "dagMClient") +@Slf4j +public class DagClient implements IClient { + + /** + * 使用jdk19的协程池 + */ + private final ExecutorService pool = Executors.newVirtualThreadPerTaskExecutor(); + + private static final String UP_PARAM_METHOD = "updateParam"; + + @Resource + private Ioc ioc; + + @Resource + private DatasetService datasetService; + + private static final ConcurrentHashMap latchMap = new ConcurrentHashMap<>(); + + + @Override + public Result call(Task task, TaskContext context, CommonReqInfo commonReqInfo, SceneTotalCountContext totalCountContext) { + Stopwatch sw = Stopwatch.createStarted(); + + GraphTaskContext taskContext = task.getDagInfo(); + log.debug("taskContext info :{}", taskContext); + //构建任务图 + Graph> graph = new Graph<>(taskContext.getTaskList().size()); + //添加顶点 + taskContext.getTaskList().forEach(it -> { + it.getData().getTask().setDataMap(task.getDataMap()); + graph.addVertex(new Vertex(it.getIndex(), it)); + }); + //基于接口任务依赖关系添加边 + taskContext.getDependList().forEach(it -> graph.addEdge(it.getFrom(), it.getTo())); + //主线程等待n个任务顶点执行完成 + final int taskSize = taskContext.getTaskList().size(); + CountDownLatch mainThreadLatch = new CountDownLatch(taskSize); + List latchKeyList = new ArrayList<>(16); + try { + graph.getVertexMap().forEach((key, value) -> { + List list = graph.dependList(key); + log.debug("task id:{} {}", value.getData().getTask().getId(), list.size()); + if (list.size() > 0) { + String latchKey = getLatchKey(context.getNum(), value.getData().getTask().getId()); + latchKeyList.add(latchKey); + latchMap.put(latchKey, new CountDownLatch(list.size())); + } + }); + //遍历任务图 + graph.bfsAll((v, d) -> { + try { + //顶点结点数据 + final NodeInfo nodeInfo = d.getData(); + d.setTaskId(nodeInfo.getId()); + + //当前顶点的边,依赖的顶点 + List list = graph.dependList(d.getIndex()); + d.setDependList(list); + //获取 set 边/子任务顶点 + List childList = graph.getAdj()[v]; + d.setChildList(childList.stream().map(graph::getVertexData).collect(Collectors.toList())); + + //初始化状态 + d.setStatus(TaskStatus.Init.code); + + //提交异步发压任务到协程池 + CompletableFuture.runAsync(() -> doWork(d, context, mainThreadLatch, totalCountContext), pool); + return true; + } catch (Throwable e) { + log.error("bfsAll error:" + e.getMessage(), e); + if (task.isDebug()) { + //当前顶点内的任务 + Task currTask = d.getData().getTask(); + TaskResult tr = new TaskResult(); + tr.setDebug(currTask.isDebug()); + tr.setId(currTask.getId()); + } + return false; + } + }); + } finally { + try { + //主线程等待直到tag 任务超时 + mainThreadLatch.await(task.getTimeout(), TimeUnit.MILLISECONDS); + + //这里等整个dag执行完成后,再清理本次dag调用产生的latch + clearLatch(latchKeyList); + } catch (InterruptedException e) { + log.error("[DagClient] call interrupted exception countDownLatch wait exception task id:" + task.getId() + ", report id:" + task.getReportId() + ", scene id:" + + task.getSceneId() + ", serial id:" + task.getSerialLinkID() + "; " + e.getMessage()); + } + + long time = sw.elapsed(TimeUnit.MILLISECONDS); + if (time > 10000) { + log.info("call dag client:{} use time:{}", task.getId() + ":" + context.getNum(), time); + } + + //有没完成的任务 + if (context.getFinishTaskNum().get() != taskSize) { + context.setError(true); + return Result.fail(ErrorCode.ERROR_505.code, ErrorCode.ERROR_505.message, false); + } + } + return Result.success(true); + } + + private void doWork(TaskVertexData d, TaskContext context, CountDownLatch mainThreadLatch, SceneTotalCountContext errorContext) { + try { + //当前任务结点阻塞,等待依赖任务完成,参数拿全 (finally 的safe.run中会更新本结点参数) + log.debug("doWork begin:{},latch num:{}", d.getData().getTask().getId(), latchMap.get(getLatchKey(context.getNum(), d.getData().getTask().getId()))); + await(latchMap.get(getLatchKey(context.getNum(), d.getData().getTask().getId()))); + log.debug("doWork await finish:{},latch num:{}", d.getData().getTask().getId(), latchMap.get(getLatchKey(context.getNum(), d.getData().getTask().getId()))); + + //返回数据 + MutableObject resData = new MutableObject(); + //请求是否成功 + MutableObject success = new MutableObject(false); + + try { + //当前顶点内的任务 + Task currTask = d.getData().getTask(); + TaskResult tr = new TaskResult(); + tr.setDebug(currTask.isDebug()); + tr.setId(currTask.getId()); + + IClient client = getClient(currTask.getType()); + Result res; + try { + //replace with the param dataset + CommonReqInfo commonReqInfo = null; + try { + commonReqInfo = datasetService.processTaskParam(currTask, (Integer) context.getAttachments().getOrDefault(TASK_CTX_LINE_FLAG, 1), currTask.isDebug()); + } catch (Exception e) { + log.error("processTaskParam error,e:{}", e.getMessage()); + } + res = client.call(currTask, context, commonReqInfo, errorContext); + + if (currTask.isDebug()) { + tr.setCode(res.getCode()); + tr.setOk(res.isOk()); + tr.setCommonReqInfo(res.getCommonReqInfo()); + tr.setResult(res.getData().toString()); + tr.setRespHeaders(res.getRespHeaders()); + if (!res.isOk() && res.getTriggerCp() != null) { + tr.setTriggerCpInfo(res.getTriggerCp()); + } + if (!res.isOk() && res.getTriggerFilterCondition() != null) { + tr.setTriggerFilterCondition(res.getTriggerFilterCondition()); + } + tr.setRt(res.getRt()); + tr.setSize(res.getSize()); + tr.getSuccess().incrementAndGet(); + } + success.setObj(res.isSuccess()); + resData.setObj(res.getData()); + } catch (Throwable e) { + log.error("call http error:" + e.getMessage(), e); + if (currTask.isDebug()) { + tr.setCode(ErrorCode.ERROR_500.code); + tr.setResult(e.getMessage()); + tr.getFailure().incrementAndGet(); + } + context.setError(true); + success.setObj(false); + resData.setObj(e.getMessage()); + } + //任务执行成功 + d.setStatus(TaskStatus.Success.code); + + if (currTask.isDebug()) { + //异步推送更新调试结果 + tr.setDebug(currTask.isDebug()); + tr.setId(currTask.getId()); + Event.ins().post(tr); + } + } catch (Throwable ex) { + log.error(ex.getMessage()); + context.setError(true); + throw new RuntimeException(ex); + } finally { + //依赖该任务的顶点列表 + log.debug("cur task id:{},child list:{}", d.getData().getTask().getId(), d.getChildList()); + d.getChildList().forEach(it -> { + //需要先将参数塞入线程本地变量 + if (success.getObj()) { + //当前顶点任务执行成功,需要回填返回数据 + //子节点信息 + final NodeInfo childNI = it.getData(); + //遍历处理参数表达式,将当前顶点取到的结果 更新替换到下游依赖结点的入参 + Safe.run(() -> { + Map> replacerMap = new HashMap<>(8); + childNI.getExprMap().forEach((key, v) -> { + //强制使用为字符串 + boolean forceStr = false; + //需要从这里取值(哪些子节点依赖我的结果,在这里update进去) + if (key.getIndex() == d.getIndex()) { + log.debug("antlr version:{}", Expr.version()); + Object value; + if (key.getOrigin() == OutputOriginEnum.BODY_TXT.code) { + //文本类型,直接赋值 + value = resData.getObj().toString(); + } else { + //json类型,解析 + if (key.getExpr().contains("|")) { + //存在该标识符说明为list中取值 + //表达式 例如:params.json().get(data).get(goodIds)|0.string + try { + String[] exprArr = key.getExpr().split("\\|", 2); + value = Expr.params(resData.getObj().toString(), exprArr[0]); + JsonArray valList = (JsonArray) value; + if (exprArr[1].contains(".")) { + String[] indexAndType = exprArr[1].split("\\.", 2); + value = Util.getListValByType(valList, Integer.parseInt(indexAndType[0]), indexAndType[1]); + } else { + value = valList.get(Integer.parseInt(exprArr[1])); + } + } catch (Exception e) { + value = ""; + log.error("parse get list val error,expr:{},cause by:{}", key.getExpr(), e.getMessage()); + } + } else { + try { + value = Expr.params(resData.getObj().toString(), key.getExpr()); + } catch (Exception e) { + log.error("parse params error,obj:{},expr:{},cause by:{}", resData.getObj(), key.getExpr(), e.getMessage()); + value = ""; + } +// log.debug("debug parse param,expr:{},res:{},value:{}", key, resData.getObj().toString(), value); + //强制指定为字符串 + if (key.getExpr().endsWith("getAsString()")) { + forceStr = true; + } + } + } + log.debug("context.getAttachments().put before key :{}", key); + if (StringUtils.isNotEmpty(key.getName())) { + log.debug("context.getAttachments().put key :{}", key); + context.getAttachments().put(key.getName(), value); + } + List putValueExpr = key.getPutValueExpr(); + if (putValueExpr.size() > 0) { + //httpData(dubboData)(数据协议类型)->0(index)->name(参数名) + int paramIndex = Integer.parseInt(putValueExpr.get(1)); + String paramName = putValueExpr.get(2); + List replacer = new ArrayList<>(Arrays.asList(new Replacer(childNI.getTask().getId(), paramIndex, paramName, value, forceStr))); + List replacers = replacerMap.putIfAbsent(paramName, replacer); + if (replacers != null) { + replacers.addAll(replacer); + } + } + } + }); + if (replacerMap.size() != 0) { + if (context.getAttachments().get("replacerMap") != null) { + replacerMap.entrySet().stream().forEach(entry -> { + List replacerList = ((Map>)context.getAttachments().get("replacerMap")).putIfAbsent(entry.getKey(), entry.getValue()); + if (replacerList != null) { + replacerList.addAll(entry.getValue()); + } + }); + } else { + context.getAttachments().put("replacerMap", replacerMap); + } + } + }); + } + //处理更新完每个依赖本顶点的 顶点任务后,释放其 latch + log.debug("cur child task id:{},context:{},latch num:{}", it.getData().getTask().getId(), context.getNum(), getLatchKey(context.getNum(), it.getData().getTask().getId())); + + //在单个顶点拥有多个父顶点的情况下,直接remove会导致下游同样以该顶点为子顶点的子任务无法再对完成顶点完成 countDown 动作,导致该latch泄漏,对应顶点的awit操作将持续阻塞直到超时。 + //CountDownLatch latch = latchMap.remove(getLatchKey(context.getNum(), it.getData().getTask().getId())); + CountDownLatch latch = latchMap.get(getLatchKey(context.getNum(), it.getData().getTask().getId())); + if (null != latch) { + latch.countDown(); + } + }); + } + } catch (Throwable ex) { + context.setError(true); + log.error(ex.getMessage()); + throw new RuntimeException(ex); + } finally { + //当前顶点执行完成后需要释放一个latch + context.getFinishTaskNum().incrementAndGet(); + mainThreadLatch.countDown(); + } + } + + private void await(CountDownLatch latch) { + if (null == latch) { + return; + } + try { + //容错处理.不至于线程不能归还 + latch.await(30, TimeUnit.SECONDS); + } catch (Throwable e) { + log.error("await error:{}", e.getMessage()); + throw new RuntimeException(e); + } + } + + private void clearLatch(List latchKeys) { + latchKeys.forEach(latchMap::remove); + } + + private static String getLatchKey(int num, int taskId) { + return num + "_" + taskId; + } + + private IClient getClient(TaskType type) { + return ioc.getBean(type.name() + "MClient"); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/base/IClient.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/base/IClient.java new file mode 100644 index 000000000..9167e1348 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/base/IClient.java @@ -0,0 +1,17 @@ +package run.mone.mimeter.engine.client.base; + +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.data.Result; +import run.mone.mimeter.engine.agent.bo.stat.SceneTotalCountContext; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskContext; + +/** + * @author goodjava@qq.com + * @author dongzhenxing + * @date 2022/5/23 + */ +public interface IClient { + Result call(Task task, TaskContext context, CommonReqInfo commonReqInfo, SceneTotalCountContext totalCountContext); + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/impl/DemoClient.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/impl/DemoClient.java new file mode 100644 index 000000000..177d75676 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/impl/DemoClient.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.client.impl; + +import com.xiaomi.youpin.docean.anno.Component; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import run.mone.mimeter.engine.agent.bo.data.DemoData; +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.data.Result; +import run.mone.mimeter.engine.agent.bo.stat.SceneTotalCountContext; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskContext; +import run.mone.mimeter.engine.client.base.IClient; + +import java.util.concurrent.TimeUnit; + +/** + * @author goodjava@qq.com + * @date 2022/5/23 + *

+ * 用来测试代码逻辑 + */ +@Component(name = "demoClient") +public class DemoClient implements IClient { + + private static final Logger log = LoggerFactory.getLogger(DemoClient.class); + + public Result call(Task task, TaskContext context, CommonReqInfo commonReqInfo, SceneTotalCountContext totalCountContext) { + log.info("call task id:{} begin", task.getId()); + DemoData demoData = task.getDemoData(); + int i = 0; + + Result result = new Result<>(); + result.setCode(0); + + if (task.isDebug()) { + result.setData(String.valueOf(i + 1)); + } + result.setMessage("msg:" + task.getId()); + if (task.getAttachments().containsKey("sleep")) { + int time = Integer.valueOf(task.getAttachments().get("sleep")); + if (time > 0) { + try { + TimeUnit.SECONDS.sleep(time); + } catch (InterruptedException e) { + log.error("[DemoClient] call time sleep exception", e); + } + } + } + log.info("call demo task:{} context:{} result:{} success", task.getId(), context, result); + return result; + } + + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/impl/DubboClient.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/impl/DubboClient.java new file mode 100644 index 000000000..c89410004 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/impl/DubboClient.java @@ -0,0 +1,244 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.client.impl; + +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; +import com.google.gson.reflect.TypeToken; +import com.xiaomi.youpin.docean.anno.Component; +import com.xiaomi.youpin.gateway.dubbo.Dubbo; +import com.xiaomi.youpin.gateway.dubbo.MethodInfo; +import common.Util; +import org.apache.dubbo.rpc.RpcContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import run.mone.mimeter.engine.agent.bo.data.CheckFilterConditionRes; +import run.mone.mimeter.engine.agent.bo.data.DubboData; +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.data.Result; +import run.mone.mimeter.engine.agent.bo.stat.DubboResultCheckInfo; +import run.mone.mimeter.engine.agent.bo.stat.SceneTotalCountContext; +import run.mone.mimeter.engine.agent.bo.task.HeraContextInfo; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskContext; +import run.mone.mimeter.engine.client.base.BaseClient; +import run.mone.mimeter.engine.common.CustomLogBuilder; +import run.mone.mimeter.engine.common.ResultCheck; +import run.mone.mimeter.engine.common.TraceUtil; +import run.mone.mimeter.engine.service.LoggerBuilder; + +import javax.annotation.Resource; + + +import static common.Const.*; +import static run.mone.mimeter.dashboard.bo.sceneapi.DubboMavenVersionEnum.ORIGIN_MAVEN; +import static run.mone.mimeter.engine.service.MetricsService.recordApiRpsMetrics; +import static run.mone.mimeter.engine.service.MetricsService.recordTpsAndRtMetrics; + +/** + * @author goodjava@qq.com + * @author dongzhenxing + * @date 2022/5/19 + */ +@Component(name = "dubboMClient") +public class DubboClient extends BaseClient { + @Resource + private Dubbo dubbo; + + private final Gson gson = new Gson(); + + private static final Logger log = LoggerFactory.getLogger(DubboClient.class); + + @Resource + private LoggerBuilder loggerBuilder; + + @Override + public Result call(Task task, TaskContext context, CommonReqInfo commonReqInfo, SceneTotalCountContext totalCountContext) { + //目前的逻辑是有一个发生错误了,后边就不在调用了 + if (context.isError()) { + return Result.cancel(); + } + DubboData dubboData = task.getDubboData(); + //检查是否满足忽略的条件 + Result result = new Result(); + //检查是否满足忽略的条件 + CheckFilterConditionRes checkFilterConditionRes = ResultCheck.checkFilterConditionSatisfy(context.getAttachments(), dubboData.getFilterCondition()); + if (checkFilterConditionRes.isMatch()) { + result.setOk(false); + result.setData("passed"); + result.setCode(0); + result.setTriggerFilterCondition(checkFilterConditionRes.getTriggerFilterCondition()); + return result; + } + HeraContextInfo heraContextInfo = task.getHeraContextInfo(); + + Integer sceneId = 0; + Integer serialId = 0; + String reportId = ""; + Integer apiId = 0; + + if (!task.isDebug()) { + sceneId = heraContextInfo.getSceneId(); + serialId = heraContextInfo.getSerialLinkId(); + reportId = heraContextInfo.getTaskFlag(); + apiId = heraContextInfo.getSceneApiId(); + } + + //前置过滤器,用于根据业务需求自定义处理转换入参 + commonReqInfo = this.doPreFilter(task,context, commonReqInfo); + + MethodInfo mi = new MethodInfo(); + if (dubboData.getServiceName().startsWith("providers:")) { + mi.setServiceName(dubboData.getServiceName().substring("provider:".length() + 1)); + } else { + mi.setServiceName(dubboData.getServiceName()); + } + mi.setMethodName(dubboData.getMethodName()); + if (dubboData.getGroup() != null) { + mi.setGroup(dubboData.getGroup()); + } + if (dubboData.getVersion() != null) { + mi.setVersion(dubboData.getVersion()); + } + if (!ORIGIN_MAVEN.mavenVersion.equals(dubboData.getMavenVersion())) { + mi.setProto("json"); + } + + mi.setArgsProto("gson"); + RpcContext.getContext().setAttachment("gson_generic_args", "true"); + mi.setProtoVersion("v1"); + mi.setTimeout(task.getTimeout()); + mi.setParameterTypes(dubboData.getRequestParamTypeList().toArray(new String[0])); + Object[] params; + try { + params = gson.fromJson(commonReqInfo.getParamJson(), new TypeToken() { + }.getType()); + } catch (JsonSyntaxException e) { + return Result.cancel(); + } + mi.setArgs(params); + + //携带Attachments + setAttachment(dubboData); + + //生成traceid + String trace = trace(); + + //带入trace标记 + injectHeraContext(task.isDebug(), task.getHeraContextInfo()); + + Object res; + String errorInfo = null; + + int code = 500; + long start = 0; + try { + start = System.currentTimeMillis(); + res = dubbo.call(mi); + code = 0; + } catch (Exception exception) { + errorInfo = exception.getMessage(); + log.error("[DubboClient] call invoke exception, task id: " + task.getId() + ", " + "scene id: " + sceneId + + ", report id: " + reportId + ", " + "serial id: " + serialId + + ", submit type: " + task.getSubmitTaskType() + ", task type: " + task.getType().code + ", " + errorInfo); + res = errorInfo; + } + //后置过滤器,用于根据业务需求自定义处理转换出参 + this.doPostFilter(task,res); + + // check point, error analysis + DubboResultCheckInfo checkInfo = ResultCheck.checkDubboResult(dubboData, res, code, task.isDebug()); + + if (!checkInfo.isOk()) { + log.error("dubbo call error:{}", errorInfo); + context.setError(true); + } + long elapsed = System.currentTimeMillis() - start; + + if (!task.isDebug()) { + //打点rps + recordApiRpsMetrics(new String[]{String.valueOf(sceneId), reportId, String.valueOf(serialId), + task.getType().name(), dubboData.getServiceName(), dubboData.getMethodName(), String.valueOf(apiId), String.valueOf(checkInfo.isOk())}); + ResultCheck.recordRpsCount(apiId, totalCountContext); + + // check point, error analysis + String[] labelVals = new String[]{String.valueOf(sceneId), reportId, String.valueOf(serialId), + task.getType().name(), String.valueOf(code), dubboData.getServiceName(), dubboData.getMethodName(), + String.valueOf(apiId), String.valueOf(checkInfo.isOk())}; + + recordTpsAndRtMetrics(elapsed, labelVals, true); + + //rt统计 + ResultCheck.recordApiRtAndCount(apiId, elapsed, checkInfo.isOk(), totalCountContext, true); + + //错误统计 + ResultCheck.dubboApiErrorAnalysis(apiId, checkInfo, totalCountContext); + + //采样日志 + if (sendLog && ((boolean) context.getAttachments().getOrDefault(TASK_CTX_RECORD_LOG, false) || !checkInfo.isOk())) { + loggerBuilder.getLogger().info(CustomLogBuilder.buildApiLog(!checkInfo.isOk(), task.getId(), API_TYPE_DUBBO, dubboData.getServiceName(), dubboData.getMethodName(), + elapsed, code, sceneId, serialId, reportId, apiId, gson.toJson(res), commonReqInfo.getParamJson(), "", "", trace, errorInfo)); + } + } + result.setCode(code); + result.setOk(checkInfo.isOk()); + result.setData(gson.toJson(res)); + result.setCommonReqInfo(commonReqInfo); + result.setRt(elapsed); + result.setSize(res.toString().getBytes().length); + return result; + } + + + /** + * 设置attachments + */ + private void setAttachment(DubboData dubboData) { + RpcContext.getContext().setAttachments(dubboData.getAttachments()); + } + + /** + * 插入trace id + */ + private String trace() { + String traceId = TraceUtil.traceId(); + RpcContext.getContext().getAttachments().put(DUBBO_TRACE_HEADER_KEY, "00-" + traceId + "-" + TraceUtil.spanId() + "-01"); + return traceId; + } + + /** + * 注入tracing信息 + */ + private void injectHeraContext(boolean isDebug, HeraContextInfo heraContextInfo) { + if (!isDebug && checkHeraInfoParam(heraContextInfo)) { + StringBuilder sb = new StringBuilder(); + String sceneTask = heraContextInfo.getSceneId() + "_" + heraContextInfo.getTaskFlag(); + + sb.append(HERA_SCENE_TASK).append(":").append(sceneTask).append(";"); + sb.append(HERA_SERIAL_LINK).append(":").append(heraContextInfo.getSerialLinkId()).append(";"); + sb.append(HERA_API_ID).append(":").append(heraContextInfo.getSceneApiId()).append(";"); + + RpcContext.getContext().getAttachments().put(HEAR_HEADER_KEY, sb.toString()); + } + } + + private boolean checkHeraInfoParam(HeraContextInfo heraContextInfo) { + return heraContextInfo.getTaskFlag() != null && heraContextInfo.getSceneId() != null + && heraContextInfo.getSerialLinkId() != null; + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/impl/HttpClient.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/impl/HttpClient.java new file mode 100644 index 000000000..76ab5aede --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/client/impl/HttpClient.java @@ -0,0 +1,291 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.client.impl; + +import com.google.common.collect.Maps; +import com.google.gson.Gson; +import com.xiaomi.hera.trace.annotation.Trace; +import com.xiaomi.youpin.docean.anno.Component; +import common.*; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import run.mone.mimeter.engine.agent.bo.data.CheckFilterConditionRes; +import run.mone.mimeter.engine.agent.bo.data.HttpData; +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.data.Result; +import run.mone.mimeter.engine.agent.bo.stat.HttpResultCheckInfo; +import run.mone.mimeter.engine.agent.bo.stat.SceneTotalCountContext; +import run.mone.mimeter.engine.agent.bo.task.HeraContextInfo; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskContext; +import run.mone.mimeter.engine.client.base.BaseClient; +import run.mone.mimeter.engine.common.CustomLogBuilder; +import run.mone.mimeter.engine.common.ErrorCode; +import run.mone.mimeter.engine.common.ResultCheck; +import run.mone.mimeter.engine.common.TraceUtil; +import run.mone.mimeter.engine.service.LoggerBuilder; + +import javax.annotation.Resource; +import java.nio.charset.StandardCharsets; +import java.util.*; + +import static common.Const.*; +import static run.mone.mimeter.engine.service.MetricsService.recordApiRpsMetrics; +import static run.mone.mimeter.engine.service.MetricsService.recordTpsAndRtMetrics; + +/** + * @author goodjava@qq.com + * @author dongzhenxing + * @date 2023/3/23 + */ +@Component(name = "httpMClient") +public class HttpClient extends BaseClient { + private static final Logger log = LoggerFactory.getLogger(HttpClient.class); + + private final Gson gson = Util.getGson(); + @Resource + private LoggerBuilder loggerBuilder; + + @Override + @Trace + public Result call(Task task, TaskContext context, CommonReqInfo commonReqInfo, SceneTotalCountContext totalCountContext) { + try { + //目前的逻辑是有一个发生错误了,后边就不在调用了 + if (context.isError()) { + log.info("http call cancel:{}", task.getId()); + return Result.cancel(); + } + + HttpData httpData = task.getHttpData(); + + Result result = new Result(); + //检查是否满足忽略的条件 + CheckFilterConditionRes checkFilterConditionRes = ResultCheck.checkFilterConditionSatisfy(context.getAttachments(), httpData.getFilterCondition()); + if (checkFilterConditionRes.isMatch()) { + result.setOk(false); + result.setData("passed"); + result.setCode(0); + result.setTriggerFilterCondition(checkFilterConditionRes.getTriggerFilterCondition()); + return result; + } + HeraContextInfo heraContextInfo = task.getHeraContextInfo(); + + Integer sceneId = 0; + Integer serialId = 0; + String reportId = ""; + Integer apiId = 0; + + if (!task.isDebug()) { + sceneId = heraContextInfo.getSceneId(); + serialId = heraContextInfo.getSerialLinkId(); + reportId = heraContextInfo.getTaskFlag(); + apiId = heraContextInfo.getSceneApiId(); + } + + String url = httpData.getUrl(); + String method = httpData.getMethod(); + int timeout = Math.max(httpData.getTimeout(), 10); + + HttpResult res = null; + + //前置过滤器,用于根据业务需求自定义处理转换入参 + log.debug("do pre filter before,common req:{}",commonReqInfo); + + commonReqInfo = this.doPreFilter(task,context, commonReqInfo); + + log.debug("do pre filter after,common req:{}",commonReqInfo); + if(StringUtils.isNotBlank(commonReqInfo.getDebugUrl())){ + url = commonReqInfo.getDebugUrl(); + } + String paramBody = ""; + String encoding = "utf-8"; + if (commonReqInfo.getHeaders().getOrDefault(Const.DISABLE_URL_ENCODE, "false").equals("true")) { + encoding = ""; + } + String errorInfo = ""; + boolean needLog = (boolean) context.getAttachments().getOrDefault(TASK_CTX_RECORD_LOG, false); + boolean logIgnore = false; + TpsRecord needRecordTps = new TpsRecord(true); + String traceId = ""; + long start = 0; + try { + //注入trace信息 + traceId = this.trace(commonReqInfo.getHeaders()); + //注入探针信息 + injectHeraContext(task.isDebug(), heraContextInfo, commonReqInfo.getHeaders()); + List headers = new ArrayList<>(); + commonReqInfo.getHeaders().forEach((key, value) -> { + headers.add(key); + headers.add(value); + }); + if (method.equalsIgnoreCase(HTTP_GET)) { + if (needLog) { + paramBody = gson.toJson(commonReqInfo.getQueryParamMap()); + } + start = System.currentTimeMillis(); + log.debug("http get api url:{}.headers:{}", url, headers); + + res = HttpClientV6.httpGet(needRecordTps, url, headers, commonReqInfo.getQueryParamMap(), encoding, timeout); + } else if (method.equalsIgnoreCase(HTTP_POST)) { + //json格式为 对象 {} 或基本类型 "d"/1/true... + //表单参数格式 + if (httpData.getContentType().equals(CONTENT_TYPE_APP_FORM) || httpData.getContentType().equals(CONTENT_TYPE_APP_FORM2)) { + if (needLog) { + paramBody = gson.toJson(commonReqInfo.getQueryParamMap()); + } + start = System.currentTimeMillis(); + res = HttpClientV6.httpPost(needRecordTps, url, commonReqInfo.getHeaders(), commonReqInfo.getQueryParamMap(), encoding, timeout); + } else { + paramBody = commonReqInfo.getParamJson(); + //application/json格式 + start = System.currentTimeMillis(); + log.debug("do call before,common req:{}",commonReqInfo); + res = HttpClientV6.postRt(needRecordTps, url, commonReqInfo.getParamJson().getBytes(StandardCharsets.UTF_8), commonReqInfo.getHeaders(), timeout); + } + } + } catch (Throwable ex) { + needRecordTps.setNeedRecordTps(false); + if (ex.getMessage() != null) { + errorInfo = ex.getMessage(); + } else { + errorInfo = ex.getCause().getMessage(); + } + log.error("[HttpClient] call invoke exception, task id: " + task.getId() + ", " + "scene id: " + sceneId + + ", report id: " + reportId + ", " + "serial id: " + serialId + + ", submit type: " + task.getSubmitTaskType() + ", task type: " + task.getType().code + " " + errorInfo); + + res = new HttpResult(ErrorCode.ERROR_500.code, errorInfo, Maps.newHashMap()); + } finally { + long elapsed = (System.currentTimeMillis() - start); + int respCode = res == null ? ErrorCode.ERROR_500.code : res.code; + + //后置过滤器,用于根据业务需求自定义处理转换出参 + this.doPostFilter(task, res); + + // check point, error analysis + HttpResultCheckInfo checkInfo = ResultCheck.checkHttpResult(httpData, res, task.getSuccessCode(), task.isDebug()); + + if (!checkInfo.isOk()) { + if (method.equalsIgnoreCase(HTTP_GET) || httpData.getContentType().equals(CONTENT_TYPE_APP_FORM) || httpData.getContentType().equals(CONTENT_TYPE_APP_FORM2)) { + paramBody = gson.toJson(commonReqInfo.getQueryParamMap()); + } + log.error("http call error:{}", res.code); + context.setError(true); + } + if (!task.isDebug()) { + if (!needRecordTps.isNeedRecordTps()) { + //httpclient 请求报错 + //检查是否为连接超时 + assert res != null; + if (judgeIfConnectTimeout(res)) { + //统计丢失连接数 + ResultCheck.recordLossConnCount(totalCountContext); + logIgnore = true; + } else { + collectData(task, sceneId, reportId, serialId, url, method, apiId, checkInfo, totalCountContext, elapsed, respCode, needRecordTps); + } + } else { + collectData(task, sceneId, reportId, serialId, url, method, apiId, checkInfo, totalCountContext, elapsed, respCode, needRecordTps); + } + } + result.setCode(res.code); + result.setOk(checkInfo.isOk()); + result.setData(res.content); + result.setRespHeaders(res.getHeaders()); + result.setCommonReqInfo(commonReqInfo); + if (!checkInfo.isOk() && checkInfo.getTriggerCpInfo() != null) { + //触发的检查点信息 + result.setTriggerCp(checkInfo.getTriggerCpInfo()); + } + result.setRt(elapsed); + result.setSize(res.content.getBytes().length); + //采样日志 + if (sendLog && (needLog || (!checkInfo.isOk()) && !logIgnore)) { + loggerBuilder.getLogger().info(CustomLogBuilder.buildApiLog(!checkInfo.isOk(), task.getId(), API_TYPE_HTTP, url, method, elapsed, result.getCode(), sceneId, serialId, + reportId, apiId, Objects.equals("Read timed out",res.content)?(res.content + " ( start = "+start ):res.content, paramBody, gson.toJson(commonReqInfo.getHeaders()), gson.toJson(res.getHeaders()), traceId, errorInfo)); + } + } + return result; + } finally { + log.debug("finish:{}:{}", task.getId(), task.getHttpData().getUrl()); + } + } + + /** + * 统计打点数据 + */ + private void collectData(Task task, Integer sceneId, String reportId, Integer serialId + , String url, String method, Integer apiId, HttpResultCheckInfo checkInfo, + SceneTotalCountContext totalCountContext, long elapsed, Integer respCode, + TpsRecord needRecordTps) { + //打点rps + recordApiRpsMetrics(new String[]{String.valueOf(sceneId), reportId, String.valueOf(serialId), + task.getType().name(), url, method, String.valueOf(apiId), String.valueOf(checkInfo.isOk())}); + ResultCheck.recordRpsCount(apiId, totalCountContext); + + //打点 tps + recordTpsAndRtMetrics(elapsed, new String[]{String.valueOf(sceneId), reportId, String.valueOf(serialId), + task.getType().name(), String.valueOf(respCode), url, method, String.valueOf(apiId), + String.valueOf(checkInfo.isOk())}, needRecordTps.isNeedRecordTps()); + //rt 次数统计 + ResultCheck.recordApiRtAndCount(apiId, elapsed, checkInfo.isOk(), totalCountContext, needRecordTps.isNeedRecordTps()); + //错误统计 + ResultCheck.httpApiErrorAnalysis(apiId, checkInfo, totalCountContext, task.getSuccessCode()); + } + + /** + * 插入trace id + */ + private String trace(Map headers) { + String traceId = TraceUtil.traceId(); + headers.put(DUBBO_TRACE_HEADER_KEY, "00-" + traceId + "-" + TraceUtil.spanId() + "-01"); + return traceId; + } + + /** + * 注入tracing信息 + */ + private void injectHeraContext(boolean isDebug, HeraContextInfo heraContextInfo, Map headers) { + if (!isDebug && checkHeraInfoParam(heraContextInfo)) { + StringBuilder sb = new StringBuilder(); + String sceneTask = heraContextInfo.getSceneId() + "_" + heraContextInfo.getTaskFlag(); + + sb.append(HERA_SCENE_TASK).append(":").append(sceneTask).append(";"); + sb.append(HERA_SERIAL_LINK).append(":").append(heraContextInfo.getSerialLinkId()).append(";"); + sb.append(HERA_API_ID).append(":").append(heraContextInfo.getSceneApiId()).append(";"); + + headers.put(Const.HEAR_HEADER_KEY, sb.toString()); + } + } + + private boolean checkHeraInfoParam(HeraContextInfo heraContextInfo) { + return heraContextInfo.getTaskFlag() != null && heraContextInfo.getSceneId() != null + && heraContextInfo.getSerialLinkId() != null; + } + + /** + * 是否连接超时,是则作为丢失连接 + */ + private boolean judgeIfConnectTimeout(HttpResult res) { + if (res.code != 500) { + return false; + } + return res.content.equals("Connect timed out") || res.content.equals("Connection timed out"); + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/common/CustomLogBuilder.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/common/CustomLogBuilder.java new file mode 100644 index 000000000..d3ae4b959 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/common/CustomLogBuilder.java @@ -0,0 +1,38 @@ +package run.mone.mimeter.engine.common; + + +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; + +import static common.Const.API_TYPE_HTTP; + +public class CustomLogBuilder { + + public static DateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + public static String buildApiLog(boolean failed, int taskId, int apiType,String url, String method, + long elapsed, int code, Integer sceneId, Integer serialId, String reportId, + Integer apiId, String resContent, String paramBody, String reqHeader, + String resHeader, String traceId, String errorInfo) { + return sdf.format(new Date()) + "∩" + + failed + "∩" + + taskId + "∩" + + apiType + "∩" + + url + "∩" + + method + "∩" + + elapsed + "∩" + + code + "∩" + + sceneId + "∩" + + serialId + "∩" + + reportId + "∩" + + apiId + "∩" + + resContent + "∩" + + paramBody + "∩" + + reqHeader + "∩" + + resHeader + "∩" + + traceId + "∩" + + errorInfo; + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/common/ErrorCode.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/common/ErrorCode.java new file mode 100644 index 000000000..bd7c1fa34 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/common/ErrorCode.java @@ -0,0 +1,23 @@ +package run.mone.mimeter.engine.common; + +/** + * @author goodjava@qq.com + * @date 2022/9/20 22:35 + */ +public enum ErrorCode { + + ERROR_505(505, "main_thread_latch_timeout"), + + ERROR_500(500, "500 error"); + + public final int code; + + public final String message; + + ErrorCode(int code, String message) { + this.code = code; + this.message = message; + } + + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/common/ResultCheck.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/common/ResultCheck.java new file mode 100644 index 000000000..c4ebb4c77 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/common/ResultCheck.java @@ -0,0 +1,718 @@ +package run.mone.mimeter.engine.common; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.xiaomi.data.push.antlr.expr.Expr; +import common.HttpResult; +import common.Util; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import run.mone.mimeter.dashboard.bo.sceneapi.OutputOriginEnum; +import run.mone.mimeter.engine.agent.bo.data.BaseData; +import run.mone.mimeter.engine.agent.bo.data.CheckPointInfo; +import run.mone.mimeter.engine.agent.bo.data.CheckFilterConditionRes; +import run.mone.mimeter.engine.agent.bo.stat.DubboResultCheckInfo; +import run.mone.mimeter.engine.agent.bo.stat.HttpResultCheckInfo; +import run.mone.mimeter.engine.agent.bo.data.OutputParam; +import run.mone.mimeter.engine.agent.bo.stat.SceneTotalCountContext; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.LongAdder; + +import static common.Const.*; + +@Slf4j +public class ResultCheck { + + public static final Gson gson = Util.getGson(); + + /** + * 错误分析,检查点 + */ + public static HttpResultCheckInfo checkHttpResult(BaseData baseData, HttpResult res, String successCode, boolean debug) { + HttpResultCheckInfo checkInfo = new HttpResultCheckInfo(); + if (baseData.getCheckPointInfoList() == null || baseData.getCheckPointInfoList().size() == 0) { + //没有配置检查点,以状态码为主 + List succCodes = new ArrayList<>(); + if (successCode != null) { + succCodes.addAll(Arrays.stream(successCode.split(",")).toList()); + } else { + succCodes.add("200"); + succCodes.add("0"); + } + if (!succCodes.contains(String.valueOf(res.code))) { + checkInfo.setOk(false); + checkInfo.setCheckPointId(0); + checkInfo.setHttpStatusCode(String.valueOf(res.code)); + return checkInfo; + } else { + checkInfo.setOk(true); + } + } else { + //以检查点为主 + CheckPointInfo triggerCp = ResultCheck.checkPointSatisfyHttp(baseData.getOutputParams(), baseData.getCheckPointInfoList(), res); + if (triggerCp != null && triggerCp.getId() != null && triggerCp.getId() != 0) { + //触发检查点 + checkInfo.setOk(false); + checkInfo.setCheckPointId(triggerCp.getId()); + if (debug) { + checkInfo.setTriggerCpInfo(gson.toJson(triggerCp)); + } + return checkInfo; + } else { + if (debug && triggerCp != null) { + checkInfo.setOk(false); + checkInfo.setTriggerCpInfo(gson.toJson(triggerCp)); + } + if (triggerCp == null) { + checkInfo.setOk(true); + } + } + checkInfo.setHttpStatusCode(String.valueOf(res.code)); + } + return checkInfo; + } + + /** + * dubbo 错误判断,检查点 + */ + public static DubboResultCheckInfo checkDubboResult(BaseData baseData, Object res, int code, boolean debug) { + DubboResultCheckInfo checkInfo = new DubboResultCheckInfo(); + + if (baseData.getCheckPointInfoList() == null || baseData.getCheckPointInfoList().size() == 0) { + //未配置检查点,默认已code为准,即调用没有报错就算成功 + + if (code != 0) { + checkInfo.setOk(false); + checkInfo.setCheckPointId(0); + return checkInfo; + } else { + checkInfo.setOk(true); + } + } else { + CheckPointInfo triggerCp = ResultCheck.checkPointSatisfyDubbo(baseData.getOutputParams(), baseData.getCheckPointInfoList(), res); + if (triggerCp != null && triggerCp.getId() != null && triggerCp.getId() != 0) { + //触发检查点 + checkInfo.setOk(false); + checkInfo.setCheckPointId(triggerCp.getId()); + if (debug) { + checkInfo.setTriggerCpInfo(gson.toJson(triggerCp)); + } + } else { + checkInfo.setOk(true); + } + } + return checkInfo; + } + + /** + * dubbo请求错误分析统计 + */ + public static void dubboApiErrorAnalysis(int apiId, DubboResultCheckInfo checkInfo, SceneTotalCountContext errorContext) { + if (checkInfo.isOk()) { + errorContext.getTotalSuccReq().increment(); + return; + } + //总错误次数+1 + errorContext.getTotalErrReq().increment(); + String checkPointId = String.valueOf(checkInfo.getCheckPointId()); + if (!checkPointId.equals("0")) { + // 检查点触发 + String checkPointKey = ERR_CHECKPOINT_PREFIX + checkPointId; + if (errorContext.getErrCounterMap().containsKey(checkPointKey)) { + //已存在该检查点错误 + if (errorContext.getErrCounterMap().get(checkPointKey).containsKey(apiId)) { + //已记录该接口,+1 + errorContext.getErrCounterMap().get(checkPointKey).get(apiId).increment(); + } else { + //初始化1次 + LongAdder newAdder = new LongAdder(); + newAdder.increment(); + errorContext.getErrCounterMap().get(checkPointKey).put(apiId, newAdder); + } + } else { + //初始化1次 + ConcurrentHashMap tmpMap = new ConcurrentHashMap<>(); + + LongAdder newAdder = new LongAdder(); + newAdder.increment(); + tmpMap.put(apiId, newAdder); + //该检查点错误尚未记录 + errorContext.getErrCounterMap().put(checkPointKey, tmpMap); + } + } else { + //调用异常触发 + String statusCodeKey = "dubbo_call_fail"; + if (errorContext.getErrCounterMap().containsKey(statusCodeKey)) { + //已存在该错误码 + if (errorContext.getErrCounterMap().get(statusCodeKey).containsKey(apiId)) { + //已记录该接口,+1 + errorContext.getErrCounterMap().get(statusCodeKey).get(apiId).increment(); + } else { + //初始化1次 + LongAdder newAdder = new LongAdder(); + newAdder.increment(); + errorContext.getErrCounterMap().get(statusCodeKey).put(apiId, newAdder); + } + } else { + //该错误码尚未记录 + //初始化1次 + ConcurrentHashMap tmpMap = new ConcurrentHashMap<>(); + LongAdder newAdder = new LongAdder(); + newAdder.increment(); + tmpMap.put(apiId, newAdder); + errorContext.getErrCounterMap().put(statusCodeKey, tmpMap); + } + } + } + + /** + * http请求错误分析统计 + */ + public static void httpApiErrorAnalysis(int apiId, HttpResultCheckInfo checkInfo, SceneTotalCountContext errorContext, String successCode) { + if (checkInfo.isOk()) { + errorContext.getTotalSuccReq().increment(); + return; + } + List succCodes = new ArrayList<>(); + if (successCode != null) { + succCodes.addAll(Arrays.stream(successCode.split(",")).toList()); + } else { + succCodes.add("200"); + succCodes.add("0"); + } + //总错误次数+1 + errorContext.getTotalErrReq().increment(); + String statusCode = checkInfo.getHttpStatusCode(); + String checkPointId = String.valueOf(checkInfo.getCheckPointId()); + if (!checkPointId.equals("0")) { + // 检查点触发 + String checkPointKey = ERR_CHECKPOINT_PREFIX + checkPointId; + if (errorContext.getErrCounterMap().containsKey(checkPointKey)) { + //已存在该检查点错误 + if (errorContext.getErrCounterMap().get(checkPointKey).containsKey(apiId)) { + //已记录该接口,+1 + errorContext.getErrCounterMap().get(checkPointKey).get(apiId).increment(); + } else { + //初始化1次 + LongAdder newAdder = new LongAdder(); + newAdder.increment(); + errorContext.getErrCounterMap().get(checkPointKey).put(apiId, newAdder); + } + } else { + //初始化1次 + ConcurrentHashMap tmpMap = new ConcurrentHashMap<>(); + + LongAdder newAdder = new LongAdder(); + newAdder.increment(); + tmpMap.put(apiId, newAdder); + //该检查点错误尚未记录 + errorContext.getErrCounterMap().put(checkPointKey, tmpMap); + } + } else if (!succCodes.contains(statusCode)) { + //http 状态码触发 + String statusCodeKey = ERR_STATUS_CODE_PREFIX + statusCode; + if (errorContext.getErrCounterMap().containsKey(statusCodeKey)) { + //已存在该错误码 + if (errorContext.getErrCounterMap().get(statusCodeKey).containsKey(apiId)) { + //已记录该接口,+1 + errorContext.getErrCounterMap().get(statusCodeKey).get(apiId).increment(); + } else { + //初始化1次 + LongAdder newAdder = new LongAdder(); + newAdder.increment(); + errorContext.getErrCounterMap().get(statusCodeKey).put(apiId, newAdder); + } + } else { + //该错误码尚未记录 + //初始化1次 + ConcurrentHashMap tmpMap = new ConcurrentHashMap<>(); + LongAdder newAdder = new LongAdder(); + newAdder.increment(); + tmpMap.put(apiId, newAdder); + errorContext.getErrCounterMap().put(statusCodeKey, tmpMap); + } + } + } + + /** + * 是否满足检查点规则 dubbo + */ + public static CheckPointInfo checkPointSatisfyDubbo(List outputParams, List checkPointInfos, Object res) { + if (checkPointInfos == null || checkPointInfos.size() == 0) { + return null; + } + AtomicBoolean ok = new AtomicBoolean(true); + for (CheckPointInfo checkPointInfo : + checkPointInfos) { + //dubbo接口的检查点只有出参校验类型 + if (checkPointInfo.getCheckType() == OUTPUT_CODE) { + String outputKey = checkPointInfo.getCheckObj(); + OutputParam opParam = outputParams.stream().filter(outputParam -> outputParam.getParamName().equals(outputKey)).findFirst().get(); + try { + Object value; + if (opParam.getOrigin() == OutputOriginEnum.BODY_TXT.code) { + value = res.toString(); + } else { + if (opParam.getParseExpr().contains("|")) { + //存在该标识符说明为list中取值 + //表达式 例如:params.json().get(data).get(goodIds)|0.string + try { + String[] exprArr = opParam.getParseExpr().split("\\|", 2); + value = Expr.params(gson.toJson(res), exprArr[0]); + JsonArray valList = (JsonArray) value; + if (exprArr[1].contains(".")) { + String[] indexAndType = exprArr[1].split("\\.", 2); + value = Util.getListValByType(valList, Integer.parseInt(indexAndType[0]), indexAndType[1]); + } else { + value = valList.get(Integer.parseInt(exprArr[1])); + } + } catch (Exception e) { + value = ""; + log.error("dubbo check point parse get list val error,expr:{},cause by:{}", opParam.getParseExpr(), e.getMessage()); + } + } else { + try { + value = Expr.params(gson.toJson(res), opParam.getParseExpr()); + } catch (Exception e) { + value = ""; + } + } + } + ok.set(checkOutput(value, checkPointInfo.getCheckCondition(), checkPointInfo.getCheckContent())); + } catch (Exception e) { + ok.set(false); + } + } + if (!ok.get()) { + //一个不满足即错误 + //一个不满足即错误 + CheckPointInfo triggerCp = new CheckPointInfo(); + BeanUtils.copyProperties(checkPointInfo, triggerCp); + return triggerCp; + } + } + + return null; + } + + /** + * 是否满足过滤规则 + * 条件list中关系为 条件与 and,即满足所有条件则过滤 + * 单条中以 | 间隔为 条件或 or + */ + public static CheckFilterConditionRes checkFilterConditionSatisfy(Map attachments, List filterConditionList) { + CheckFilterConditionRes checkFilterConditionRes = new CheckFilterConditionRes(); + if (filterConditionList == null || filterConditionList.size() == 0) { + checkFilterConditionRes.setMatch(false); + return checkFilterConditionRes; + } + log.debug("checkFilterConditionSatisfy attachments:{},filterCondition :{}", attachments, filterConditionList.get(0)); + //若 + //起始置为匹配,条件list中有不满足的则置为false + AtomicBoolean ok = new AtomicBoolean(true); + for (CheckPointInfo checkPointInfo : filterConditionList) { + if (checkPointInfo.getCheckType() == OUTPUT_CODE) { + String outputKey = checkPointInfo.getCheckObj(); + Object value = attachments.get(outputKey); + //取不到上游值,直接抛出不匹配 + if (value == null) { + checkFilterConditionRes.setMatch(false); + return checkFilterConditionRes; + } + try { + //若取值比较不满足,直接抛出不匹配 + if (!checkOutput(value, checkPointInfo.getCheckCondition(), checkPointInfo.getCheckContent())){ + ok.set(false); + break; + } + } catch (Exception e) { + ok.set(false); + break; + } + } + } + //满足所有条件,需要过滤该接口 + if (ok.get()) { + checkFilterConditionRes.setMatch(ok.get()); + checkFilterConditionRes.setTriggerFilterCondition(gson.toJson(filterConditionList)); + } + return checkFilterConditionRes; + } + + /** + * 是否满足检查点规则 http + */ + public static CheckPointInfo checkPointSatisfyHttp(List outputParams, List checkPointInfos, HttpResult res) { + if (checkPointInfos == null || checkPointInfos.size() == 0) { + return null; + } + AtomicBoolean ok = new AtomicBoolean(true); + for (CheckPointInfo checkPointInfo : + checkPointInfos) { + switch (checkPointInfo.getCheckType()) { + case STATUS_CODE -> + ok.set(checkStatusCode(res.code, checkPointInfo.getCheckCondition(), checkPointInfo.getCheckContent())); + case HEADER_CODE -> { + String headerValue = res.getHeader(checkPointInfo.getCheckObj()); + ok.set(checkHeader(checkPointInfo.getCheckObj(), checkPointInfo.getCheckCondition(), headerValue)); + } + case OUTPUT_CODE -> { + String outputKey = checkPointInfo.getCheckObj(); + OutputParam opParam = outputParams.stream().filter(outputParam -> outputParam.getParamName().equals(outputKey)).findFirst().get(); + Object value; + try { + if (opParam.getOrigin() == OutputOriginEnum.BODY_TXT.code) { + value = res.content; + } else { + if (opParam.getParseExpr().contains("|")) { + //存在该标识符说明为list中取值 + //表达式 例如:params.json().get(data).get(goodIds)|0.string + try { + String[] exprArr = opParam.getParseExpr().split("\\|", 2); + value = Expr.params(res.content, exprArr[0]); + JsonArray valList = (JsonArray) value; + if (exprArr[1].contains(".")) { + String[] indexAndType = exprArr[1].split("\\.", 2); + value = Util.getListValByType(valList, Integer.parseInt(indexAndType[0]), indexAndType[1]); + } else { + value = valList.get(Integer.parseInt(exprArr[1])); + } + } catch (Exception e) { + value = ""; + log.error("http check point parse get list val error,expr:{},cause by:{}", opParam.getParseExpr(), e.getMessage()); + } + } else { + value = Expr.params(res.content, opParam.getParseExpr()); + } + } + ok.set(checkOutput(value, checkPointInfo.getCheckCondition(), checkPointInfo.getCheckContent())); + } catch (Exception e) { + ok.set(false); + } + } + } + if (!ok.get()) { + //一个不满足即错误 + CheckPointInfo triggerCp = new CheckPointInfo(); + BeanUtils.copyProperties(checkPointInfo, triggerCp); + return triggerCp; + } + } + return null; + } + + /** + * 取值比较 + * @param opValue 上游实际值 + * @param condition 比较条件 + * @param targetStr 目标值 + */ + public static boolean checkOutput(Object opValue, int condition, String targetStr) { + String v = opValue.toString(); + String[] targetStrArr; + if (targetStr.contains("|")) { + targetStrArr = targetStr.split("\\|"); + } else { + targetStrArr = new String[]{targetStr}; + } + boolean match = false; + + switch (condition) { + case EQ -> { + //= + for (String s : targetStrArr) { + if (v.equals(s)) { + match = true; + break; + } + } + } + case NOT_EQ -> { + //!= + for (String s : targetStrArr) { + if (!v.equals(s)) { + match = true; + break; + } + } + } + case BIGGER -> { + // > + for (String s : targetStrArr) { + if ((Integer.parseInt(v) > Integer.parseInt(s))) { + match = true; + break; + } + } + } + case BIGGER_AND_EQ -> { + // >= + for (String s : targetStrArr) { + if ((Integer.parseInt(v) >= Integer.parseInt(s))) { + match = true; + break; + } + } + } + case SMLLER -> { + // < + for (String s : targetStrArr) { + if ((Integer.parseInt(v) < Integer.parseInt(s))) { + match = true; + break; + } + } + } + case SMALLER_AND_EQ -> { + // <= + for (String s : targetStrArr) { + if ((Integer.parseInt(v) <= Integer.parseInt(s))) { + match = true; + break; + } + } + } + case CONTAIN -> { + //包含 + for (String s : targetStrArr) { + if (v.contains(s)) { + match = true; + break; + } + } + } + case NOT_CONTAIN -> { + //不包含 + for (String s : targetStrArr) { + if (!v.contains(s)) { + match = true; + break; + } + } + } + } + return match; + } + + public static boolean checkStatusCode(int httpStatus, int condition, String targetStr) { + + String[] targetStrArr; + if (targetStr.contains("|")) { + targetStrArr = targetStr.split("\\|"); + } else { + targetStrArr = new String[]{targetStr}; + } + boolean match = false; + + switch (condition) { + case EQ -> { + //= + for (String target : targetStrArr) { + if (httpStatus == Integer.parseInt(target)) { + match = true; + break; + } + } + } + case NOT_EQ -> { + //= + for (String target : targetStrArr) { + if (httpStatus != Integer.parseInt(target)) { + match = true; + break; + } + } + } + case BIGGER -> { + // > + for (String target : targetStrArr) { + if ((httpStatus > Integer.parseInt(target))) { + match = true; + break; + } + } + } + case BIGGER_AND_EQ -> { + // >= + for (String target : targetStrArr) { + if ((httpStatus >= Integer.parseInt(target))) { + match = true; + break; + } + } + } + case SMLLER -> { + // < + for (String target : targetStrArr) { + if ((httpStatus < Integer.parseInt(target))) { + match = true; + break; + } + } + } + case SMALLER_AND_EQ -> { + // <= + for (String target : targetStrArr) { + if ((httpStatus <= Integer.parseInt(target))) { + match = true; + break; + } + } + } + case CONTAIN -> { + //包含 + String statusCodeStr = String.valueOf(httpStatus); + for (String target : targetStrArr) { + if (statusCodeStr.contains(target)) { + match = true; + break; + } + } + } + case NOT_CONTAIN -> { + //不包含 + String statusCodeStr = String.valueOf(httpStatus); + for (String target : targetStrArr) { + if (!statusCodeStr.contains(target)) { + match = true; + break; + } + } + } + } + return match; + } + + public static boolean checkHeader(String headerVal, int condition, String targetStr) { + + String[] targetStrArr; + if (targetStr.contains("|")) { + targetStrArr = targetStr.split("\\|"); + } else { + targetStrArr = new String[]{targetStr}; + } + boolean match = false; + + switch (condition) { + case EQ -> { + //= + for (String target : targetStrArr) { + if (headerVal.equals(target)) { + match = true; + break; + } + } + } + case NOT_EQ -> { + //= + for (String target : targetStrArr) { + if (!headerVal.equals(target)) { + match = true; + break; + } + } + } + case CONTAIN -> { + //包含 + for (String target : targetStrArr) { + if (headerVal.contains(target)) { + match = true; + break; + } + } + } + case NOT_CONTAIN -> { + //不包含 + for (String target : targetStrArr) { + if (!headerVal.contains(target)) { + match = true; + break; + } + } + } + } + return match; + } + + public static void recordLossConnCount(SceneTotalCountContext totalCountContext) { +// 总请求次数+1 +// totalCountContext.getTotalReq().increment(); + //总丢失连接次数+1 + totalCountContext.getLossConnNum().increment(); + } + + /** + * 统计记录接口rt数据 + */ + public static void recordApiRtAndCount(int apiId, long rt, boolean success, SceneTotalCountContext totalCountContext, boolean recordTps) { + + //tps 请求数+1 + if (recordTps) { + totalCountContext.getTmpTpsCounter().increment(); + totalCountContext.getTotalTCount().increment(); + } + + ConcurrentHashMap>> apiRtAndTpsMap = totalCountContext.getApiRtMap(); + if (!apiRtAndTpsMap.containsKey(apiId)) { + //首次执行该接口 + ConcurrentHashMap> map = new ConcurrentHashMap<>(); + CopyOnWriteArrayList rtList = new CopyOnWriteArrayList<>(); + rtList.add((int) rt); + map.put(RT_LIST, rtList); + apiRtAndTpsMap.put(apiId, map); + + } else { + //已存在该接口记录,相关rt记录必然已初始化 + apiRtAndTpsMap.get(apiId).get(RT_LIST).add((int) rt); + } + + //接口请求次数统计 实际上是tps + ConcurrentHashMap> apiCountMap = totalCountContext.getApiCountMap(); + //总数+1 + if (apiCountMap.get(apiId) == null) { + log.error("api Id API_REQ_TOTAL_T is null:{}", apiId); + } else { + if (recordTps) { + apiCountMap.get(apiId).get(API_REQ_TOTAL_T).increment(); + } + if (success) { + apiCountMap.get(apiId).get(API_REQ_SUCC).increment(); + } else { + apiCountMap.get(apiId).get(API_REQ_FAIL).increment(); + } + } + } + + public static void recordRpsCount(int apiId, SceneTotalCountContext totalCountContext) { + //总次数+1 + //tps 请求数+1 + totalCountContext.getTmpRpsCounter().increment(); + + totalCountContext.getTotalReq().increment(); + //接口请求次数统计 实际上是rps + ConcurrentHashMap> apiCountMap = totalCountContext.getApiCountMap(); + if (!apiCountMap.containsKey(apiId)) { + //首次统计该接口 + ConcurrentHashMap map = new ConcurrentHashMap<>(); + LongAdder total = new LongAdder(); + total.increment(); + map.put(API_REQ_TOTAL_R, total); + map.put(API_REQ_TOTAL_T, new LongAdder()); + map.put(API_REQ_SUCC, new LongAdder()); + map.put(API_REQ_FAIL, new LongAdder()); + + apiCountMap.put(apiId, map); + } else { + //总数+1 + apiCountMap.get(apiId).get(API_REQ_TOTAL_R).increment(); + } + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/common/Safe.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/common/Safe.java new file mode 100644 index 000000000..448c048ab --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/common/Safe.java @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.common; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author goodjava@qq.com + * @date 2022/5/19 + */ +public class Safe { + + private static final Logger log = LoggerFactory.getLogger(Safe.class); + + public interface ExRunnable { + void run() throws Throwable; + + } + + + public static void run(ExRunnable runnable) { + try { + runnable.run(); + } catch (Throwable ex) { + log.error(ex.getMessage()); + } + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/common/TraceUtil.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/common/TraceUtil.java new file mode 100644 index 000000000..4dcf08724 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/common/TraceUtil.java @@ -0,0 +1,30 @@ +package run.mone.mimeter.engine.common; + +import java.util.concurrent.ThreadLocalRandom; + +public class TraceUtil { + static String traceString = "0123456789abcdef"; + + public static String traceId() { + ThreadLocalRandom random = ThreadLocalRandom.current(); + char[] chars = new char[32]; + chars[0] = 'f'; + chars[1] = 'f'; + chars[2] = '8'; + chars[3] = '8'; + for (int i = 4; i < 32; i++) { + chars[i] = traceString.charAt(random.nextInt(16)); + } + return new String(chars); + } + + public static String spanId() { + ThreadLocalRandom random = ThreadLocalRandom.current(); + char[] chars = new char[16]; + for (int i = 0; i < 16; i++) { + chars[i] = traceString.charAt(random.nextInt(16)); + } + return new String(chars); + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/event/EventProcessor.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/event/EventProcessor.java new file mode 100644 index 000000000..55a500402 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/event/EventProcessor.java @@ -0,0 +1,82 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.event; + +import com.google.common.eventbus.Subscribe; +import com.xiaomi.data.push.rpc.RpcClient; +import com.xiaomi.data.push.rpc.protocol.RemotingCommand; +import com.xiaomi.youpin.docean.anno.Component; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import run.mone.mimeter.engine.agent.bo.MibenchCmd; +import run.mone.mimeter.engine.agent.bo.data.AgentReq; +import run.mone.mimeter.engine.agent.bo.stat.SceneTotalCountContextDTO; +import run.mone.mimeter.engine.agent.bo.task.TaskResult; +import run.mone.mimeter.engine.agent.bo.task.TaskStatusBo; + +import javax.annotation.Resource; + +/** + * @author goodjava@qq.com + * @author dongzhenxing + * @date 2022/5/19 + */ +@Component +public class EventProcessor { + + private static final Logger log = LoggerFactory.getLogger(EventProcessor.class); + + @Resource(name = "rpcClient") + private RpcClient rpcClient; + + @Subscribe + public void taskResultCallback(TaskResult tr) { + log.info("tr id:{} success:{} failue:{}", tr.getId(), tr.getSuccess().get(), tr.getFailure().get()); + AgentReq ar = new AgentReq(); + ar.setTaskResult(tr); + ar.setCmd(AgentReq.TASK_RESULT_CMD); + try { + rpcClient.sendMessage(RemotingCommand.createGsonRequestCommand(MibenchCmd.MANAGER, ar)); + } catch (Exception e) { + log.error("taskResultCallback failed,reportId:{},cause by:{}",tr.getReportId(),e.getMessage()); + } + } + + @Subscribe + public void taskTotalContextCallback(SceneTotalCountContextDTO totalCountContextDTO) { + AgentReq ar = new AgentReq(); + ar.setTotalCountContextDTO(totalCountContextDTO); + ar.setCmd(AgentReq.TOTAL_DATA_COUNT_CMD); + try { + rpcClient.sendMessage(RemotingCommand.createGsonRequestCommand(MibenchCmd.MANAGER, ar)); + } catch (Exception e) { + log.error("taskTotalContextCallback failed,reportId:{},cause by:{}",totalCountContextDTO.getReportId(),e.getMessage()); + } + } + + @Subscribe + public void taskStatusCallback(TaskStatusBo taskStatusBo) { + AgentReq ar = new AgentReq(); + ar.setStatusBo(taskStatusBo); + ar.setCmd(AgentReq.UP_TASK_STATUS); + try { + rpcClient.sendMessage(RemotingCommand.createGsonRequestCommand(MibenchCmd.MANAGER, ar)); + } catch (Exception e) { + log.error("taskStatusCallback failed,tasktId:{},cause by:{}",taskStatusBo.getTaskId(),e.getMessage()); + } + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/common/BasePostFilter.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/common/BasePostFilter.java new file mode 100644 index 000000000..dbb8e394c --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/common/BasePostFilter.java @@ -0,0 +1,10 @@ +package run.mone.mimeter.engine.filter.common; + +import run.mone.mimeter.engine.agent.bo.task.Task; + +/** + * @author dongzhenxing + */ +public interface BasePostFilter { + Object doFilter(Task task, Object object); +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/common/BasePreFilter.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/common/BasePreFilter.java new file mode 100644 index 000000000..ed9442db7 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/common/BasePreFilter.java @@ -0,0 +1,13 @@ +package run.mone.mimeter.engine.filter.common; + +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskContext; + +/** + * @author dongzhenxing + */ +public interface BasePreFilter { + CommonReqInfo doFilter(Task task, TaskContext context,CommonReqInfo commonReqInfo); + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/common/FilterOrder.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/common/FilterOrder.java new file mode 100644 index 000000000..3bccbf922 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/common/FilterOrder.java @@ -0,0 +1,15 @@ +package run.mone.mimeter.engine.filter.common; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * @author dongzhenxing + */ +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.ANNOTATION_TYPE}) +public @interface FilterOrder { + int value(); +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/common/MimeterFilterInfo.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/common/MimeterFilterInfo.java new file mode 100644 index 000000000..0ea9acf05 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/common/MimeterFilterInfo.java @@ -0,0 +1,20 @@ +package run.mone.mimeter.engine.filter.common; + +import lombok.Data; + +import java.io.Serializable; + +/** + * @author dongzhenxing + */ +@Data +public class MimeterFilterInfo implements Serializable { + private String filterName; + private int order; + + /** + * 0 前置 + * 1 后置 + */ + private int type; +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/dto/BasicTrafficListDTO.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/dto/BasicTrafficListDTO.java new file mode 100644 index 000000000..29c9941f5 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/dto/BasicTrafficListDTO.java @@ -0,0 +1,30 @@ +/* + * Copyright 2020 XiaoMi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at the following link. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package run.mone.mimeter.engine.filter.dto; + +import com.xiaomi.youpin.tesla.traffic.recording.api.bo.traffic.BasicTrafficList; +import lombok.Data; + +import java.io.Serializable; +import java.util.concurrent.atomic.LongAdder; + +/** + * Created by dongzhenxing on 2023/3/7 4:45 PM + */ +@Data +public class BasicTrafficListDTO extends BasicTrafficList implements Serializable { + private LongAdder pointer = new LongAdder(); +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/IPostFilterChain.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/IPostFilterChain.java new file mode 100644 index 000000000..8588ca7b7 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/IPostFilterChain.java @@ -0,0 +1,10 @@ +package run.mone.mimeter.engine.filter.postFilter; + +import run.mone.mimeter.engine.agent.bo.task.Task; + +/** + * @author dongzhenxing + */ +public interface IPostFilterChain { + Object doPostFilter(Task task, Object resInfo); +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/PostFilter.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/PostFilter.java new file mode 100644 index 000000000..d97648a91 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/PostFilter.java @@ -0,0 +1,11 @@ +package run.mone.mimeter.engine.filter.postFilter; + +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.filter.common.BasePostFilter; + +/** + * @author dongzhenxing + */ +public abstract class PostFilter { + public abstract Object doFilter(Task task, Object res, BasePostFilter filter); +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/PostFilterAnno.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/PostFilterAnno.java new file mode 100644 index 000000000..2638e74fa --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/PostFilterAnno.java @@ -0,0 +1,14 @@ +package run.mone.mimeter.engine.filter.postFilter; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * @author dongzhenxing + */ +@Target({ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +public @interface PostFilterAnno { +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/filters/Demo2PostFilter.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/filters/Demo2PostFilter.java new file mode 100644 index 000000000..8b4df4a99 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/filters/Demo2PostFilter.java @@ -0,0 +1,22 @@ +package run.mone.mimeter.engine.filter.postFilter.filters; + +import com.xiaomi.youpin.docean.anno.Component; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.filter.common.BasePostFilter; +import run.mone.mimeter.engine.filter.common.FilterOrder; +import run.mone.mimeter.engine.filter.postFilter.PostFilter; +import run.mone.mimeter.engine.filter.postFilter.PostFilterAnno; + +/** + * @author dongzhenxing + */ +@Component +@PostFilterAnno +@FilterOrder(2) +public class Demo2PostFilter extends PostFilter { + @Override + public Object doFilter(Task task, Object res, BasePostFilter filter) { + System.out.println(res.toString()); + return filter.doFilter(task,res); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/filters/DemoPostFilter.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/filters/DemoPostFilter.java new file mode 100644 index 000000000..7cf86ec12 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/filters/DemoPostFilter.java @@ -0,0 +1,22 @@ +package run.mone.mimeter.engine.filter.postFilter.filters; + +import com.xiaomi.youpin.docean.anno.Component; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.filter.common.BasePostFilter; +import run.mone.mimeter.engine.filter.common.FilterOrder; +import run.mone.mimeter.engine.filter.postFilter.PostFilter; +import run.mone.mimeter.engine.filter.postFilter.PostFilterAnno; + +/** + * @author dongzhenxing + */ +@Component +@PostFilterAnno +@FilterOrder(1) +public class DemoPostFilter extends PostFilter { + @Override + public Object doFilter(Task task, Object res, BasePostFilter filter) { + System.out.println(res.toString()); + return filter.doFilter(task,res); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/filters/MimeterPostFilter.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/filters/MimeterPostFilter.java new file mode 100644 index 000000000..777ac4a28 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/postFilter/filters/MimeterPostFilter.java @@ -0,0 +1,16 @@ +package run.mone.mimeter.engine.filter.postFilter.filters; + +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.filter.common.BasePostFilter; + +/** + * @author dongzhenxing + */ +public class MimeterPostFilter implements BasePostFilter { + + @Override + public Object doFilter(Task task, Object object) { + System.out.println("MimeterPostFilter"); + return object; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/IPreFilterChain.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/IPreFilterChain.java new file mode 100644 index 000000000..34a3ad66c --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/IPreFilterChain.java @@ -0,0 +1,12 @@ +package run.mone.mimeter.engine.filter.preFilter; + +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskContext; + +/** + * @author dongzhenxing + */ +public interface IPreFilterChain { + CommonReqInfo doPreFilter(Task task, TaskContext context,CommonReqInfo reqInfo); +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/PreFilter.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/PreFilter.java new file mode 100644 index 000000000..0c6c5a7ed --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/PreFilter.java @@ -0,0 +1,14 @@ +package run.mone.mimeter.engine.filter.preFilter; + +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskContext; +import run.mone.mimeter.engine.filter.common.BasePreFilter; + +/** + * @author dongzhenxing + */ +public abstract class PreFilter { + public abstract CommonReqInfo doFilter(Task task, TaskContext context,CommonReqInfo commonReqInfo,BasePreFilter filter); + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/PreFilterAnno.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/PreFilterAnno.java new file mode 100644 index 000000000..bd1bd6dd4 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/PreFilterAnno.java @@ -0,0 +1,14 @@ +package run.mone.mimeter.engine.filter.preFilter; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * @author dongzhenxing + */ +@Target({ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +public @interface PreFilterAnno { +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/filters/DagParamFilter.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/filters/DagParamFilter.java new file mode 100644 index 000000000..483a4ca79 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/filters/DagParamFilter.java @@ -0,0 +1,115 @@ +package run.mone.mimeter.engine.filter.preFilter.filters; + +import com.google.gson.Gson; +import com.xiaomi.youpin.docean.anno.Component; +import common.Replacer; +import common.Util; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.math.NumberUtils; +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.data.ParamType; +import run.mone.mimeter.engine.agent.bo.data.ParamTypeEnum; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskContext; +import run.mone.mimeter.engine.agent.bo.task.TaskType; +import run.mone.mimeter.engine.filter.common.BasePreFilter; +import run.mone.mimeter.engine.filter.common.FilterOrder; +import run.mone.mimeter.engine.filter.preFilter.PreFilter; +import run.mone.mimeter.engine.filter.preFilter.PreFilterAnno; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author dongzhenxing + * x5 Dag参数替换过滤器 + */ +@Component +@FilterOrder(1) +@PreFilterAnno +@Slf4j +public class DagParamFilter extends PreFilter { + + private static final Gson gson = Util.getGson(); + + @Override + public CommonReqInfo doFilter(Task task, TaskContext context, CommonReqInfo commonReqInfo, BasePreFilter filter) { + + //非http任务,直接略过 + if (task.getType() != TaskType.http && task.getType() != TaskType.dubbo) { + return filter.doFilter(task, context, commonReqInfo); + } + Map contextMap = context.getAttachments(); + //替换参数 + updateParam(task, commonReqInfo, contextMap); + return filter.doFilter(task, context, commonReqInfo); + } + + /** + * 用于上游节点接口动态更新本接口实际参数 + * + * @ paramIndex 参数索引位置 + * @ paramName 参数名 若paramName 为例:${name} 类型,则直接替换 + * @ value 上游传下来的实际参数值 + */ + private void updateParam(Task task, CommonReqInfo reqInfo, Map contextMap) { + if (contextMap.get("replacerMap") == null) { + return; + } + Map> replacerMap = (Map>) contextMap.get("replacerMap"); + replacerMap.values().forEach(replacers -> { + replacers.stream().filter(replacer -> replacer.getTaskId() == task.getId()).forEach(replacer -> { + if (replacer.getParamName().startsWith("${")) { + try { + //post请求 的 json参数 + String valStr = replacer.getValue().toString(); + + if (!replacer.isForceStr()) { + if (NumberUtils.isNumber(valStr)) { + if (isNumeric(valStr)) { + replacer.setValue(Long.parseLong(valStr)); + } else { + replacer.setValue(Double.parseDouble(valStr)); + } + } else { + replacer.setValue(valStr); + } + } + + reqInfo.setParamJson(Util.Parser.parse$(Util.getElKey(replacer.getParamName()).getKey(), reqInfo.getParamJson(), replacer.getValue())); + } catch (Exception e) { + log.error("updateParam param after,body:{},time:{},error:{}", reqInfo.getParamJson(), System.currentTimeMillis(), e); + } + } else { + if (task.getHttpData().getTypes().size() > replacer.getParamIndex()) { + ParamType type = task.getHttpData().getTypes().get(replacer.getParamIndex()); + //对象类型 + if (type.getTypeEnum().equals(ParamTypeEnum.pojo)) { + ConcurrentHashMap params = (ConcurrentHashMap) task.getHttpData().getParams().get(replacer.getParamIndex()); + params.put(replacer.getParamName(), replacer.getValue()); + } + //基本类型 + if (type.getTypeEnum().equals(ParamTypeEnum.primary)) { + task.getHttpData().getParams().set(replacer.getParamIndex(), replacer.getValue()); + } + reqInfo.setQueryParamMap(task.getHttpData().httpGetTmpParams(task.getHttpData().getParams())); + if (task.isDebug()) { + reqInfo.setParamJson(gson.toJson(reqInfo.getQueryParamMap())); + } + } + } + }); + }); + } + + private static boolean isNumeric(String str) { + for (int i = str.length(); --i >= 0; ) { + int chr = str.charAt(i); + if (chr < 48 || chr > 57) + return false; + } + return true; + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/filters/MimeterPreFilter.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/filters/MimeterPreFilter.java new file mode 100644 index 000000000..3c750bbcc --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/filters/MimeterPreFilter.java @@ -0,0 +1,18 @@ +package run.mone.mimeter.engine.filter.preFilter.filters; + +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskContext; +import run.mone.mimeter.engine.filter.common.BasePreFilter; + +/** + * @author dongzhenxing + */ +public class MimeterPreFilter implements BasePreFilter { + @Override + public CommonReqInfo doFilter(Task task, TaskContext context, CommonReqInfo commonReqInfo) { + //默认 filter 原样返回 + return commonReqInfo; + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/filters/TrafficFilter.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/filters/TrafficFilter.java new file mode 100644 index 000000000..7f028a726 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/filters/TrafficFilter.java @@ -0,0 +1,111 @@ +package run.mone.mimeter.engine.filter.preFilter.filters; + +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; +import com.google.gson.reflect.TypeToken; +import com.xiaomi.youpin.docean.anno.Component; +import com.xiaomi.youpin.tesla.traffic.recording.api.bo.traffic.BasicTraffic; +import com.xiaomi.youpin.tesla.traffic.recording.api.bo.traffic.BasicTrafficList; +import common.Util; +import lombok.extern.slf4j.Slf4j; +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.data.HttpData; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskContext; +import run.mone.mimeter.engine.agent.bo.task.TaskType; +import run.mone.mimeter.engine.filter.common.BasePreFilter; +import run.mone.mimeter.engine.filter.common.FilterOrder; +import run.mone.mimeter.engine.filter.dto.BasicTrafficListDTO; +import run.mone.mimeter.engine.filter.preFilter.PreFilter; +import run.mone.mimeter.engine.filter.preFilter.PreFilterAnno; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import static common.Const.HTTP_GET; +import static common.Const.HTTP_POST; + +/** + * @author dongzhenxing + * 录制流量参数过滤器 + */ +@Component +@FilterOrder(4) +@PreFilterAnno +@Slf4j +public class TrafficFilter extends PreFilter { + + public static final Gson gson = Util.getGson(); + + /** + * 用于暂存本 agent 当前所用的流量数据 + */ + private static final ConcurrentHashMap trafficCache = new ConcurrentHashMap<>(); + + @Override + public CommonReqInfo doFilter(Task task, TaskContext context,CommonReqInfo commonReqInfo, BasePreFilter filter) { + //非http任务或未启用流量数据,直接略过 + if (task.getType() != TaskType.http || !task.getHttpData().isEnableTraffic()) { + return filter.doFilter(task,context, commonReqInfo); + } + //启用流量数据,替换header、params + HttpData httpData = task.getHttpData(); + if (!trafficCache.containsKey(httpData.getTrafficConfId())) { + //没有缓存流量数据,直接略过 + return filter.doFilter(task,context, commonReqInfo); + } + BasicTrafficListDTO basicTrafficListDTO = trafficCache.get(httpData.getTrafficConfId()); + + //空数组,跳过 + if (basicTrafficListDTO.getList().size() == 0){ + return filter.doFilter(task,context, commonReqInfo); + } + + //取模,循环读取 + BasicTraffic traffic = basicTrafficListDTO.getList().get(basicTrafficListDTO.getPointer().intValue() % basicTrafficListDTO.getList().size()); + //取完后移动游标 + trafficCache.get(httpData.getTrafficConfId()).getPointer().add(1); + + //增加请求头 + commonReqInfo.getHeaders().putAll(traffic.getOriginHeaders()); + + //替换请求体 + if (httpData.getMethod().equalsIgnoreCase(HTTP_POST)) { + String originBody = traffic.getOrginBody(); + commonReqInfo.setParamJson(originBody); + } else if (httpData.getMethod().equalsIgnoreCase(HTTP_GET)) { + String originBody = traffic.getOriginQueryString(); + try { + Map queryMap = gson.fromJson(originBody, new TypeToken>() { + }.getType()); + commonReqInfo.setQueryParamMap(queryMap); + } catch (JsonSyntaxException e) { + log.error("TrafficFilter doFilter error query body"); + } + } + return filter.doFilter(task, context,commonReqInfo); + } + + /** + * 更新暂存加载到的traffic数据 + */ + public static void updateTrafficCache(List trafficLists) { + trafficLists.forEach(trafficList -> { + BasicTrafficListDTO dto = new BasicTrafficListDTO(); + dto.setRecordingConfigId(trafficList.getRecordingConfigId()); + dto.setList(trafficList.getList()); + dto.setPage(trafficList.getPage()); + dto.setPageSize(trafficList.getPageSize()); + dto.setTotal(trafficList.getTotal()); + trafficCache.put(trafficList.getRecordingConfigId(), dto); + }); + } + + /** + * 压测结束后清理流量数据缓存 + */ + public static void cleanTrafficCache(List trafficConfIdList) { + trafficConfIdList.forEach(trafficCache::remove); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/filters/TspAuthFilter.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/filters/TspAuthFilter.java new file mode 100644 index 000000000..0ac5e2160 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/filters/TspAuthFilter.java @@ -0,0 +1,136 @@ +package run.mone.mimeter.engine.filter.preFilter.filters; + +import com.google.gson.Gson; +import com.google.gson.JsonSyntaxException; +import com.xiaomi.youpin.docean.anno.Component; +import common.Util; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.binary.Hex; +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.data.HttpData; +import run.mone.mimeter.engine.agent.bo.data.TspAuthInfoDTO; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskContext; +import run.mone.mimeter.engine.agent.bo.task.TaskType; +import run.mone.mimeter.engine.filter.common.BasePreFilter; +import run.mone.mimeter.engine.filter.common.FilterOrder; +import run.mone.mimeter.engine.filter.preFilter.PreFilter; +import run.mone.mimeter.engine.filter.preFilter.PreFilterAnno; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import static common.Const.*; +import static common.Const.CONTENT_TYPE_APP_FORM2; + +/** + * @author dongzhenxing + * 汽车部 tsp接口鉴权过滤器 + */ +@Component +@PreFilterAnno +@FilterOrder(1) +@Slf4j +public class TspAuthFilter extends PreFilter { + + private static final String ACCESS_KEY = "accesskey"; + + private static final String SIGNATURE = "signature"; + + private static final String TIMESTAMP = "timestamp"; + + public static final Gson gson = Util.getGson(); + + @Override + public CommonReqInfo doFilter(Task task, TaskContext context,CommonReqInfo commonReqInfo, BasePreFilter filter) { + HttpData httpData = task.getHttpData(); + if (task.getType() != TaskType.http || httpData == null) { + return filter.doFilter(task, context,commonReqInfo); + } + //非http启用该功能的接口任务,直接返回不做处理 + TspAuthInfoDTO tspAuth = httpData.getTspAuthInfoDTO(); + + if (tspAuth == null || !tspAuth.isEnableAuth()) { + return filter.doFilter(task, context, commonReqInfo); + } + + long now = System.currentTimeMillis(); + commonReqInfo.getHeaders().put(ACCESS_KEY, tspAuth.getAccessKey()); + commonReqInfo.getHeaders().put(SIGNATURE, genSignature(httpData, commonReqInfo, tspAuth.getAccessKey(), tspAuth.getSecretKey(), now)); + commonReqInfo.getHeaders().put(TIMESTAMP, String.valueOf(now)); + + return filter.doFilter(task, context, commonReqInfo); + } + + private String genSignature(HttpData httpData, CommonReqInfo reqInfo, String accessKey, String secretKey, long now) { + try { + MessageDigest sha512 = MessageDigest.getInstance("SHA-512"); + String tempStringSignature = accessKey + secretKey + now + string2Sign(httpData, reqInfo); + byte[] tempSignature = sha512.digest(tempStringSignature.getBytes()); + return String.valueOf(Hex.encodeHex(tempSignature)); + } catch (NoSuchAlgorithmException e) { + log.warn("TspAuthFilter genSignature failed:{}", e.getMessage()); + return ""; + } + } + + private String string2Sign(HttpData httpData, CommonReqInfo reqInfo) { + StringBuilder sign = new StringBuilder(); + String url = httpData.getUrl(); + String[] urlArr = url.split("//", 2); + url = urlArr[1]; + if (httpData.getMethod().equalsIgnoreCase(HTTP_GET)) { + //get请求 + if (!url.contains("/")) { + sign.append("/"); + } else { + sign.append(url.substring(url.indexOf("/"))); + } + sign.append('+').append("GET"); + Map paramAndValue = reqInfo.getQueryParamMap(); + List keyList = paramAndValue.keySet().stream().sorted(String::compareTo).toList(); + if (keyList.size() != 0) { + sign.append('+'); + } + for (int i = 0; i < keyList.size(); i++) { + sign.append(keyList.get(i).toLowerCase()).append("=").append(paramAndValue.get(keyList.get(i))); + if (i != keyList.size() - 1) { + sign.append("&"); + } + } + } else if (httpData.getMethod().equalsIgnoreCase(HTTP_POST)) { + //post请求 + TreeMap target = new TreeMap<>(); + if (!url.contains("/")) { + sign.append("/"); + } else { + sign.append(url.substring(url.indexOf("/"))); + } + sign.append('+').append("POST"); + + if (httpData.getContentType().equals(CONTENT_TYPE_APP_FORM) || httpData.getContentType().equals(CONTENT_TYPE_APP_FORM2)) { + Map paramAndValue = reqInfo.getQueryParamMap(); + //对 key 排序 + List keyList = paramAndValue.keySet().stream().sorted(String::compareTo).toList(); + if (keyList.size() != 0) { + sign.append('+'); + } + for (String s : keyList) { + target.put(s, paramAndValue.get(s)); + } + } else { + //json + try { + target = gson.fromJson(httpData.getPostParamJson(), TreeMap.class); + } catch (JsonSyntaxException e) { + log.warn("string2Sign gson.fromJson(httpData.getPostParamJson(),TreeMap.class) failed,cause :{}", e.getMessage()); + } + } + sign.append(gson.toJson(target)); + } + return sign.toString(); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/filters/X5Filter.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/filters/X5Filter.java new file mode 100644 index 000000000..ab2e5149a --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/filter/preFilter/filters/X5Filter.java @@ -0,0 +1,114 @@ +package run.mone.mimeter.engine.filter.preFilter.filters; + +import com.google.common.base.Strings; +import com.google.gson.Gson; +import com.xiaomi.data.push.common.SafeRun; +import com.xiaomi.youpin.docean.anno.Component; +import common.Util; +import org.apache.commons.codec.Charsets; +import run.mone.mimeter.dashboard.bo.sceneapi.X5VersionEnum; +import run.mone.mimeter.engine.agent.bo.data.ApiX5InfoDTO; +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.data.HttpData; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskContext; +import run.mone.mimeter.engine.agent.bo.task.TaskType; +import run.mone.mimeter.engine.common.Safe; +import run.mone.mimeter.engine.filter.common.BasePreFilter; +import run.mone.mimeter.engine.filter.common.FilterOrder; +import run.mone.mimeter.engine.filter.preFilter.PreFilter; +import run.mone.mimeter.engine.filter.preFilter.PreFilterAnno; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * @author dongzhenxing + * x5 鉴权过滤器 + */ +@Component +@FilterOrder(2) +@PreFilterAnno +public class X5Filter extends PreFilter { + + private static final Gson gson = Util.getGson(); + + @Override + public CommonReqInfo doFilter(Task task, TaskContext context, CommonReqInfo commonReqInfo, BasePreFilter filter) { + + //非http任务,直接略过 + if (task.getType() != TaskType.http) { + return filter.doFilter(task,context, commonReqInfo); + } + HttpData httpData = task.getHttpData(); + //未开启x5 + if (httpData == null || httpData.getApiX5InfoDTO() == null || !httpData.getApiX5InfoDTO().isEnableX5()){ + return filter.doFilter(task,context,commonReqInfo); + } + //启用x5协议 + commonReqInfo.setParamJson(calculateAndReplaceX5Params(httpData.getApiX5InfoDTO(),commonReqInfo)); + return filter.doFilter(task, context,commonReqInfo); + } + + private String calculateAndReplaceX5Params(ApiX5InfoDTO x5InfoDTO,CommonReqInfo commonReqInfo){ + String x5Version = x5InfoDTO.getX5Version(); + Map body = new HashMap<>(); + SafeRun.run(() ->{ + Map x5Body = new HashMap<>(); + Map x5Header = new HashMap<>(); + x5Header.put("appid", x5InfoDTO.getAppId()); + if (!Strings.isNullOrEmpty(x5InfoDTO.getX5Method())) { + x5Header.put("method", x5InfoDTO.getX5Method()); + } + String sign = genSign(x5InfoDTO.getAppId(), commonReqInfo.getParamJson(), x5InfoDTO.getAppKey()); + x5Header.put("sign", sign); + x5Body.put("header", x5Header); + x5Body.put("body", commonReqInfo.getParamJson()); + String data = gson.toJson(x5Body); + body.put("data", Base64.getEncoder().encodeToString(data.getBytes(StandardCharsets.UTF_8))); + }); + + if (X5VersionEnum.CHINA_VERSION.version.equals(x5Version)) { + return gson.toJson(body); + } else if (X5VersionEnum.INFORMATION_VERISON.version.equals(x5Version)) { + return "\""+body.get("data")+"\""; //todo 临时修改,改造json进制转换== + } + return gson.toJson(body); + } + + + /** + * @param appId 应用id + * @param body 入参 + * @param appKey 密钥 secret + * @return 签名 + */ + private static String genSign(String appId, String body, String appKey) { + String str = appId + body + appKey; + StringBuilder md5CodeBuffer = new StringBuilder(); + try { + MessageDigest md5Digest = MessageDigest.getInstance("MD5"); + md5Digest.update(str.getBytes()); + byte[] strBytesDigest = md5Digest.digest(); + + // 将二进制字节信息转换为十六进制字符串 + String strHexDigest; + for (byte b : strBytesDigest) { + strHexDigest = Integer.toHexString(b & 0XFF); + if (strHexDigest.length() == 1) { + md5CodeBuffer.append("0").append(strHexDigest); + } else { + md5CodeBuffer.append(strHexDigest); + } + } + } catch (Exception e) { + return str; + } + + return md5CodeBuffer.toString().toUpperCase(); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/BenchEngineService.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/BenchEngineService.java new file mode 100644 index 000000000..531d8213a --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/BenchEngineService.java @@ -0,0 +1,491 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.service; + +import com.google.common.base.Stopwatch; +import com.google.common.util.concurrent.RateLimiter; +import com.google.gson.Gson; +import com.xiaomi.youpin.docean.Ioc; +import com.xiaomi.youpin.docean.anno.Service; +import com.xiaomi.youpin.tesla.traffic.recording.api.service.TrafficDubboService; +import common.Const; +import common.Util; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import run.mone.event.Event; +import run.mone.mimeter.engine.agent.bo.data.*; +import run.mone.mimeter.engine.agent.bo.hosts.HostBo; +import run.mone.mimeter.engine.agent.bo.stat.SceneTotalCountContext; +import run.mone.mimeter.engine.client.base.IClient; +import run.mone.mimeter.engine.event.EventProcessor; +import run.mone.mimeter.engine.agent.bo.task.*; +import run.mone.mimeter.engine.filter.preFilter.filters.TrafficFilter; + +import javax.annotation.Resource; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.LongAdder; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static common.Const.*; +import static common.Util.*; + +/** + * @author goodjava@qq.com + * @date 2022/5/19 + */ +@Service +public class BenchEngineService { + + private static final Logger log = LoggerFactory.getLogger(BenchEngineService.class); + + private static final String logPrefix = "[BenchEngineService]"; + + private static final int LOG_SAMPLING_PERCENTAGE = 5; + + private static final int MIN_TASK_EXECUTION_TIME = 10; + + private static final int TIME_ONE_SECOND = 1000; + + private final Gson gson = Util.getGson(); + + @Resource + private Ioc ioc; + + @Resource + private EventProcessor eventProcessor; + + @Resource + private DataCountService dataCountService; + + @Resource + private DatasetService datasetService; + + private final ExecutorService pool = Executors.newVirtualThreadPerTaskExecutor(); + + private static final ConcurrentMap contextMap = new ConcurrentHashMap<>(); + + public void init() { + Event.ins().register(eventProcessor); + } + + public TaskResult submitTask(Context context, Task task) throws InterruptedException { + + Stopwatch dsw = Stopwatch.createStarted(); + + String logMsg = logPrefix + "submitTask "; + String logSuffix = ", task id: " + task.getId() + ", " + "scene id: " + task.getSceneId() + + ", submit type: " + task.getSubmitTaskType() + ", task type: " + task.getType().code + " "; + + TaskResult tr = initTaskResult(task, context); + + IClient client = getClient(task.getType()); + final int taskId = task.getId(); + + //分发任务前,加载,初始化参数、header数据源 + if (task.getType() == TaskType.dag) { + //参数数据源 + TreeMap> dataMap = new TreeMap<>(); + try { + dataMap = datasetService.loadAndInitParamDataset(task); + log.debug("loadAndInitParamDataset data map :{}",gson.toJson(dataMap)); + } catch (Exception e) { + log.warn("load dataset error,e:{}", e.getMessage()); + } finally { + log.debug("task {} load dataset use time:{}", task.getId(), dsw.elapsed(TimeUnit.MILLISECONDS)); + } + task.setDataMap(dataMap); + + //流量参数数据源 + if (task.isEnableTraffic()){ + //若该链路开启流量参数使用 + datasetService.loadTrafficData(task); + } + } + + if (task.isDebug()) { + //单接口调试或场景调试 + log.info("debug task:{} begin", task.getId()); + Result res; + try { + res = singleDebugOnce(task, client); + tr.setCode(res.getCode()); + if (res.getData() != null) { + tr.setResult(res.getData().toString()); + } else { + tr.setResult("500 error"); + } + tr.setOk(res.isOk()); + if (res.getRespHeaders() != null) { + tr.setRespHeaders(res.getRespHeaders()); + } + if (!res.isOk() && res.getTriggerCp() != null) { + tr.setTriggerCpInfo(res.getTriggerCp()); + } + tr.setRt(res.getRt()); + tr.setSize(res.getSize()); + tr.getSuccess().incrementAndGet(); + + } catch (Throwable ex) { + tr.getFailure().incrementAndGet(); + tr.setCode(500); + tr.setOk(false); + tr.setResult(ex.getMessage()); + log.error(logMsg + "single debug error" + logSuffix, ex); + } finally { + //每次调试完需要清理数据 + pool.submit(() ->{ + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + log.error(logMsg + "single debug error" + logSuffix, e); + } + this.cleanLoadedDataCache(task); + }); + log.info("task:{} debug one finish use time:{}", taskId, dsw.elapsed(TimeUnit.SECONDS)); + } + } else { + //执行场景压测 + //更新任务状态 + notifyTaskRunning(taskId); + contextMap.put(task.getId(), tr); + if (task.getTime() == 0) { + task.setTime(MIN_TASK_EXECUTION_TIME); + } + + //总协程计数器 + LongAdder counter = new LongAdder(); + + //dag任务前缀标识 + AtomicInteger dagIndex = new AtomicInteger(); + + int qps = task.getQps(); + RateLimiter limiter = RateLimiter.create(qps); + int finalLogRate = (task.getLogRate() > 0 && task.getLogRate() <= 1000) ? task.getLogRate() : LOG_SAMPLING_PERCENTAGE; + Set logIndices = new HashSet<>(sampleByTimeAndQps(task.getTime(), qps, finalLogRate)); + + //数据记录统计初始化 + SceneTotalCountContext totalCountContext = initCountCtx(task); + + log.info(logMsg + "task begin qps:{} task time:{}" + logSuffix, task.getQps(), task.getTime()); + //每次j循环内都是1s + for (int j = 1; j <= task.getTime(); j++) { + Stopwatch sw = Stopwatch.createStarted(); + + int finalJ = j; + int curQps = qps; + // 手动命令控制 qps + if (context.getTaskQps() != 0){ + curQps = Util.calculateAgentRps(context.getTaskQps(), task.getAgentNum()); + } + if (curQps != 0) { + if (curQps != qps) { + logIndices.clear(); + logIndices.addAll(sampleByTimeAndQps(task.getTime(), curQps, finalLogRate)); + qps = curQps; + } + limiter.setRate(qps); + } + int finalQps = qps; + //当前该链路的实际rps + if (context.getTaskQps() != 0){ + totalCountContext.getDagTaskRps().setRps(context.getTaskQps()); + }else { + totalCountContext.getDagTaskRps().setRps(finalQps * task.getAgentNum()); + } + if (context.getRpsRate() != 0){ + totalCountContext.setRpsRate(context.getRpsRate()); + } + counter.add(finalQps); + log.debug(logMsg + "task begin final qps:{} task time:{}" + logSuffix, finalQps, task.getTime()); + IntStream.range(0, finalQps).forEach(i -> { + limiter.acquire(); + pool.submit(() -> { + TaskContext taskCtx = new TaskContext(); + taskCtx.setNum(dagIndex.incrementAndGet()); + taskCtx.getAttachments().put(TASK_CTX_RECORD_LOG, !logIndices.isEmpty() ? + logIndices.contains(((finalJ - 1) % 10) * finalQps + i) : judgeByRateLog(finalLogRate)); + taskCtx.getAttachments().put(TASK_CTX_LINE_FLAG, (finalJ - 1) * finalQps + i); + taskCtx.getAttachments().put(TASK_CTX_SCENE_QPS, task.getQps()); + try { + //只会是 dag 任务 + log.debug("context line flag: "+taskCtx.getAttachments().get(TASK_CTX_LINE_FLAG)); + Result res = client.call(task, taskCtx, null, totalCountContext); + int code = res.getCode(); + + //链路整体是否执行成功 + if (code == 0 || code == 200) { + tr.getSuccess().incrementAndGet(); + } else { + tr.getFailure().incrementAndGet(); + } + } catch (Throwable ex) { + tr.getFailure().incrementAndGet(); + log.error(logMsg + "---pool submit error, i: " + i + ", " + logSuffix + ex.getMessage()); + } finally { + counter.decrement(); + } + }); + }); + + long time = sw.elapsed(TimeUnit.MILLISECONDS); + if (time < TIME_ONE_SECOND) { + try { + TimeUnit.MILLISECONDS.sleep(TIME_ONE_SECOND - time); + } catch (Throwable ex) { + log.error(logMsg + "sleep error " + logSuffix + " error: " + ex.getMessage()); + } + while (limiter.tryAcquire()) { + //把剩余的令牌都拿走 + } + } + + //proms打点 + dataCountService.recordProms(totalCountContext.getTmpTpsCounter(), totalCountContext.getTmpRpsCounter(), task); + + //推送频率 10s一次 + if (j % PUSH_STAT_RATE == 0) { + dataCountService.pushTotalCountData(totalCountContext, false); + } + + //压测取消 + if (context.isCancel()) { + log.info(logMsg + "---------task is cancelled" + logSuffix); + // 手动结束通知 + tr.setQuitByManual(true); + tr.setCancelType(context.getCancelType()); + //退出阻塞 + counter.reset(); + break; + } + } + + //阻塞主线程直到所有调用结束 + this.aWait(counter); + + //任务结束最后一次把剩余数据都推送完成 + dataCountService.pushTotalCountData(totalCountContext, true); + + context.setFinish(true); + contextMap.remove(task.getId()); + + //压测结束清理缓存数据 + this.cleanLoadedDataCache(task); + + log.info(logMsg + "---------task has finished" + logSuffix); + } + //任务结果回调manager + tr.setAddr(task.getAddr()); + try { + Event.ins().post(tr); + } catch (Exception e) { + log.error("bench engine post finish failed:{}", e.getMessage()); + } + return tr; + } + + /** + * 阻塞主线程 + */ + private void aWait(LongAdder counter) throws InterruptedException { + //阻塞主线程直到所有调用结束 + while (counter.intValue() > 0) { + TimeUnit.MILLISECONDS.sleep(500); + } + } + + private TaskResult initTaskResult(Task task, Context context) { + TaskResult tr = new TaskResult(); + tr.setSceneId(task.getSceneId()); + tr.setDebug(task.isDebug()); + tr.setId(task.getId()); + tr.setReportId(task.getReportId()); + tr.setContext(context); + return tr; + } + + private SceneTotalCountContext initCountCtx(Task task) { + return SceneTotalCountContext.builder(). + reportId(task.getReportId()) + .sceneId(task.getSceneId()) + .sceneType(task.getSceneType().name()) + .taskId(task.getId()). + agentNum(task.getAgentNum()) + .dagTaskRps(task.getDagTaskRps()) + .rpsRate(task.getRpsRate()) + .connectTaskNum(task.getConnectTaskNum()) + .lastTime(false) + .totalReq(new LongAdder()) + .lossConnNum(new LongAdder()) + .totalTCount(new LongAdder()) + .totalSuccReq(new LongAdder()) + .totalErrReq(new LongAdder()) + .tmpRpsCounter(new LongAdder()) + .tmpTpsCounter(new LongAdder()) + .errCounterMap(new ConcurrentHashMap<>()) + .apiRtMap(new ConcurrentHashMap<>()) + .apiCountMap(new ConcurrentHashMap<>()) + .apiRpsMap(new ConcurrentHashMap<>()) + .apiTpsMap(new ConcurrentHashMap<>()).build(); + } + + private void notifyTaskRunning(int taskId) { + TaskStatusBo statusBo = new TaskStatusBo(); + statusBo.setTaskId(taskId); + statusBo.setTaskStatus(TaskStatus.Running); + Event.ins().post(statusBo); + } + + public void cancelTask(Task task) { + task.getIds().forEach(taskId -> { + TaskResult tr = contextMap.get(taskId); + if (tr != null) { + tr.setOpUser(task.getOpUser()); + Optional.of(tr).ifPresent(it -> it.getContext().setCancel(true)); + } + //更新任务状态 + TaskStatusBo statusBo = new TaskStatusBo(); + statusBo.setSceneId(task.getSceneId()); + statusBo.setTaskId(taskId); + statusBo.setTaskStatus(TaskStatus.Stopped); + Event.ins().post(statusBo); + }); + } + + /** + * 手动调速 + */ + public void changeTaskQps(ChangeQpsReq req) { + req.getDagTaskRpsList().forEach(dagTaskRps -> { + TaskResult tr = contextMap.get(dagTaskRps.getTaskId()); + Optional.ofNullable(tr).ifPresent(it -> { + it.getContext().setTaskQps(dagTaskRps.getRps()); + if (req.getRpsRate() != null){ + it.getContext().setRpsRate(req.getRpsRate()); + } + }); + }); + } + + + /** + * 加载 host 文件 + */ + public HostsFileResult loadHostsReq() { + HostsFileResult rt = new HostsFileResult(); + List hostsContent; + try { + hostsContent = HostsService.loadHostFile(); + } catch (Exception e) { + log.error("load hosts file from machine error:{}", e.getMessage()); + hostsContent = new ArrayList<>(); + } + rt.setHostsFile(gson.toJson(hostsContent)); + return rt; + } + + /** + * 修改更新host文件 + */ + public void editHostsFile(List hostReqList) { + + List hostBoList = new ArrayList<>(); + + hostReqList.forEach(hostReq -> { + HostBo hostBo = new HostBo(); + hostBo.setDomain(hostReq.getDomain()); + hostBo.setIp(hostReq.getIp()); + hostBoList.add(hostBo); + }); + + HostsService.updateHostConfig(hostBoList); + } + + /** + * 删除host文件配置 + */ + public void delHostsFile(List hostReqList) { + + List hostBoList = new ArrayList<>(); + + hostReqList.forEach(hostReq -> { + HostBo hostBo = new HostBo(); + hostBo.setDomain(hostReq.getDomain()); + hostBoList.add(hostBo); + }); + + HostsService.deleteDomainsConfig(hostBoList); + } + + private Result singleDebugOnce(Task task, IClient client) { + CommonReqInfo commonReqInfo = new CommonReqInfo(); + if (task.getType() != TaskType.dag) { + commonReqInfo = getDebugParam(task); + } + return client.call(task, new TaskContext(), commonReqInfo, new SceneTotalCountContext()); + } + + private CommonReqInfo getDebugParam(Task task) { + CommonReqInfo commonReqInfo = new CommonReqInfo(); + HttpData httpData = task.getHttpData(); + DubboData dubboData = task.getDubboData(); + if (task.getType() == TaskType.http) { + if (httpData.getMethod().equalsIgnoreCase(Const.HTTP_GET)) { + commonReqInfo.setParamsType(ReqParamType.Http_Get.code); +// commonReqInfo.setGetOrFormParamsList(httpData.getParams()); + commonReqInfo.setQueryParamMap(httpData.httpGetTmpParams(httpData.getParams())); + commonReqInfo.setParamJson(gson.toJson(httpData.httpGetParams())); + commonReqInfo.setHeaders(httpData.getHeaders()); + } else if (httpData.getMethod().equalsIgnoreCase(Const.HTTP_POST)) { + if (httpData.getContentType().equals(CONTENT_TYPE_APP_FORM) || httpData.getContentType().equals(CONTENT_TYPE_APP_FORM2)) { + commonReqInfo.setParamsType(ReqParamType.Http_Post_Form.code); +// commonReqInfo.setGetOrFormParamsList(httpData.getParams()); + commonReqInfo.setQueryParamMap(httpData.httpGetTmpParams(httpData.getParams())); + commonReqInfo.setParamJson(gson.toJson(httpData.httpGetParams())); + } else { + commonReqInfo.setParamsType(ReqParamType.Http_Post_Json.code); + commonReqInfo.setParamJson(httpData.getPostParamJson()); + } + commonReqInfo.setHeaders(httpData.getHeaders()); + } + } else if (task.getType() == TaskType.dubbo) { + commonReqInfo.setParamsType(ReqParamType.Dubbo.code); + commonReqInfo.setParamJson(dubboData.getRequestBody()); + } + return commonReqInfo; + } + + private void cleanLoadedDataCache(Task task){ + //清理文件数据源缓存 + pool.submit(() -> { + try { + datasetService.processDataMapCache(task); + } catch (InterruptedException e) { + log.error("cleanLoadedDataCache task has finished remove data cache error:{}", e.getMessage()); + } + }); + //清理流量数据缓存 + pool.submit(() -> TrafficFilter.cleanTrafficCache(task.getTrafficToPullConfList().getApiTrafficReqList().stream(). + map(PullApiTrafficReq::getTrafficConfigId).collect(Collectors.toList()))); + } + private IClient getClient(TaskType type) { + return ioc.getBean(type.name() + "MClient"); + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/DataCountService.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/DataCountService.java new file mode 100644 index 000000000..0657c9f43 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/DataCountService.java @@ -0,0 +1,191 @@ +package run.mone.mimeter.engine.service; + +import com.xiaomi.youpin.docean.anno.Service; +import common.Const; +import lombok.extern.slf4j.Slf4j; +import run.mone.event.Event; +import run.mone.mimeter.engine.agent.bo.task.HeraContextInfo; +import run.mone.mimeter.engine.agent.bo.stat.SceneTotalCountContext; +import run.mone.mimeter.engine.agent.bo.stat.SceneTotalCountContextDTO; +import run.mone.mimeter.engine.agent.bo.task.Task; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.LongAdder; + +import static common.Const.*; +import static common.Const.METRICS_TYPE_SCENE; +import static run.mone.mimeter.engine.agent.bo.stat.MetricLabelEnum.labelNameSlice; + +@Service +@Slf4j +public class DataCountService { + /** + * 增量统计数据推送 + */ + public void pushTotalCountData(SceneTotalCountContext totalCountContext, boolean lastTime) { + // 结束时最后推送一次统计数据 + if (lastTime) { + totalCountContext.setLastTime(true); + } else { + //兼容,tps不应该是数组了,目前忽略了最后一次剩余的统计 + recordCurTpsAndRps(totalCountContext); + } + try { + //推送数据统计事件 + SceneTotalCountContextDTO dto = copyToDTO(totalCountContext, lastTime); + Event.ins().post(dto); + } catch (Exception e) { + log.warn("push count data error:{}", e.getMessage()); + } + } + + /** + * do copy + */ + private SceneTotalCountContextDTO copyToDTO(SceneTotalCountContext totalCountContext, boolean last) { + SceneTotalCountContextDTO dto = new SceneTotalCountContextDTO(); + //copy 基本信息 + dto.setReportId(totalCountContext.getReportId()); + dto.setSceneId(totalCountContext.getSceneId()); + dto.setSceneType(totalCountContext.getSceneType()); + dto.setTaskId(totalCountContext.getTaskId()); + dto.setAgentNum(totalCountContext.getAgentNum()); + dto.setDagTaskRps(totalCountContext.getDagTaskRps()); + dto.setRpsRate(totalCountContext.getRpsRate()); + dto.setConnectTaskNum(totalCountContext.getConnectTaskNum()); + dto.setLastTime(totalCountContext.isLastTime()); + //copy 总请求数 + synchronized (totalCountContext.getTotalReq()) { + //请求总数 + dto.setTotalReq(totalCountContext.getTotalReq().longValue()); + totalCountContext.getTotalReq().reset(); + //总丢失连接数 + dto.setLossConnNum(totalCountContext.getLossConnNum().longValue()); + totalCountContext.getLossConnNum().reset(); + + //copy 总事务数数 + dto.setTotalTCount(totalCountContext.getTotalTCount().longValue()); + totalCountContext.getTotalTCount().reset(); + //copy 总成功数 + dto.setTotalSuccReq(totalCountContext.getTotalSuccReq().longValue()); + totalCountContext.getTotalSuccReq().reset(); + //copy 总错误数 + dto.setTotalErrReq(totalCountContext.getTotalErrReq().longValue()); + totalCountContext.getTotalErrReq().reset(); + } + + synchronized (totalCountContext.getErrCounterMap()) { + //copy 错误记录map + ConcurrentHashMap> tmpErrCounterMap = new ConcurrentHashMap<>(); + totalCountContext.getErrCounterMap().forEach((errFlag, map) -> { + ConcurrentHashMap subMap = new ConcurrentHashMap<>(); + map.forEach((apiId, adder) -> subMap.put(apiId, adder.longValue())); + tmpErrCounterMap.put(errFlag, subMap); + }); + dto.setCounterMap(tmpErrCounterMap); + if (last) { + totalCountContext.getErrCounterMap().clear(); + } else { + totalCountContext.getErrCounterMap().values().forEach(map -> map.forEach((apiId, value) -> value.reset())); + } + } + + synchronized (totalCountContext.getApiRtMap()) { + //copy rt记录的map + ConcurrentHashMap>> tmpApiRtAndTpsMap = new ConcurrentHashMap<>(); + totalCountContext.getApiRtMap().forEach((apiId, map) -> { + ConcurrentHashMap> subMap = new ConcurrentHashMap<>(); + map.forEach((labFlag, vList) -> subMap.put(labFlag, vList.stream().toList())); + tmpApiRtAndTpsMap.put(apiId, subMap); + }); + + dto.setApiRtMap(tmpApiRtAndTpsMap); + totalCountContext.getApiRtMap().clear(); + } + synchronized (totalCountContext.getApiCountMap()) { + //copy 接口级别统计次数 + ConcurrentHashMap> apiCountMap = new ConcurrentHashMap<>(); + totalCountContext.getApiCountMap().forEach((apiId, map) -> { + ConcurrentHashMap subMap = new ConcurrentHashMap<>(); + map.forEach((type, count) -> subMap.put(type, count.intValue())); + apiCountMap.put(apiId, subMap); + }); + dto.setApiCountMap(apiCountMap); + + if (last) { + totalCountContext.getApiCountMap().clear(); + } else { + totalCountContext.getApiCountMap().values().forEach(it -> { + it.get(API_REQ_TOTAL_T).reset(); + it.get(API_REQ_TOTAL_R).reset(); + it.get(API_REQ_SUCC).reset(); + it.get(API_REQ_FAIL).reset(); + }); + } + } + + synchronized (totalCountContext.getApiRpsMap()) { + //copy api Rps + ConcurrentHashMap apiRpsMap = new ConcurrentHashMap<>(); + totalCountContext.getApiRpsMap().forEach((apiId, avg10Rps) -> apiRpsMap.put(apiId, avg10Rps.intValue())); + dto.setApiRpsMap(apiRpsMap); + totalCountContext.getApiRpsMap().clear(); + } + synchronized (totalCountContext.getApiTpsMap()) { + //copy api Tps + ConcurrentHashMap apiTpsMap = new ConcurrentHashMap<>(); + totalCountContext.getApiTpsMap().forEach((apiId, avg10Tps) -> apiTpsMap.put(apiId, avg10Tps.intValue())); + dto.setApiTpsMap(apiTpsMap); + totalCountContext.getApiTpsMap().clear(); + } + + return dto; + } + + /** + * 计算暂存当前的apis 10s 内tps + */ + private void recordCurTpsAndRps(SceneTotalCountContext totalCountContext) { + totalCountContext.getApiCountMap().forEach((apiId, countMap) -> { + //计算10s 平均 tps + ConcurrentHashMap apiTpsMap = totalCountContext.getApiTpsMap(); + int avg10Tps = countMap.get(API_REQ_TOTAL_T).intValue() / PUSH_STAT_RATE; + if (!apiTpsMap.containsKey(apiId)) { + apiTpsMap.put(apiId, new AtomicInteger(avg10Tps)); + } else { + apiTpsMap.get(apiId).set(avg10Tps); + } + //计算10s 平均 rps + ConcurrentHashMap apiRpsMap = totalCountContext.getApiRpsMap(); + int avg10Rps = countMap.get(API_REQ_TOTAL_R).intValue() / PUSH_STAT_RATE; + if (!apiRpsMap.containsKey(apiId)) { + apiRpsMap.put(apiId, new AtomicInteger(avg10Rps)); + } else { + apiRpsMap.get(apiId).set(avg10Rps); + } + }); + } + + public void recordProms(LongAdder tpsCounter, LongAdder rpsCounter, Task task) { + HeraContextInfo heraContextInfo = task.getHeraContextInfo(); + long rps; + synchronized (rpsCounter) { + rps = rpsCounter.longValue(); + rpsCounter.reset(); + } + MetricsService.recordCounter(Const.METRICS_NAME_RPS, labelNameSlice(4), rps, + String.valueOf(heraContextInfo.getSceneId()), heraContextInfo.getTaskFlag(), String.valueOf(heraContextInfo.getSerialLinkId()), + METRICS_TYPE_SCENE); + + long tps; + synchronized (tpsCounter) { + tps = tpsCounter.longValue(); + tpsCounter.reset(); + } + MetricsService.recordCounter(Const.METRICS_NAME_TPS, labelNameSlice(4), tps, + String.valueOf(heraContextInfo.getSceneId()), heraContextInfo.getTaskFlag(), String.valueOf(heraContextInfo.getSerialLinkId()), + METRICS_TYPE_SCENE); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/DatasetService.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/DatasetService.java new file mode 100644 index 000000000..3fec48a32 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/DatasetService.java @@ -0,0 +1,450 @@ +package run.mone.mimeter.engine.service; + +import com.google.common.base.Stopwatch; +import com.google.gson.Gson; +import com.xiaomi.youpin.docean.anno.Service; +import com.xiaomi.youpin.docean.common.Pair; +import com.xiaomi.youpin.tesla.traffic.recording.api.bo.traffic.BasicTrafficList; +import com.xiaomi.youpin.tesla.traffic.recording.api.bo.traffic.GetTrafficReq; +import com.xiaomi.youpin.tesla.traffic.recording.api.service.TrafficDubboService; +import common.Const; +import common.Util; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang.math.NumberUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.dubbo.annotation.DubboReference; +import run.mone.mimeter.dashboard.bo.common.Result; +import run.mone.mimeter.dashboard.bo.dataset.DatasetLineNum; +import run.mone.mimeter.dashboard.bo.dataset.DatasetLinesReq; +import run.mone.mimeter.dashboard.service.DatasetInfoSercice; +import run.mone.mimeter.engine.agent.bo.data.DubboData; +import run.mone.mimeter.engine.agent.bo.data.HttpData; +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; +import run.mone.mimeter.engine.agent.bo.data.ReqParamType; +import run.mone.mimeter.engine.agent.bo.task.PullApiTrafficReq; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskType; +import run.mone.mimeter.engine.bo.DataMapCache; +import run.mone.mimeter.engine.filter.preFilter.filters.TrafficFilter; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.IntStream; + +import static common.Const.*; + +/** + * 数据源服务 + */ +@Service +@Slf4j +public class DatasetService { + + private final Gson gson = Util.getGson(); + + private static final Pattern EL_PATTERN = Pattern.compile("\\$\\{([^}]*)}"); + + /** + * <"reportId",> + */ + private final ConcurrentHashMap datasetCache = new ConcurrentHashMap<>(); + + @DubboReference(check = false, group = "${mimeter.dashboard.dubbo.group}", version = "${mimeter.dashboard.dubbo.version}", interfaceClass = DatasetInfoSercice.class, timeout = 10000) + private DatasetInfoSercice datasetInfoSercice; + + @DubboReference(check = false, group = "${traffic.dubbo.group}", interfaceClass = TrafficDubboService.class, timeout = 10000) + private TrafficDubboService trafficDubboService; + + /** + * 加载录制的流量数据 + */ + public void loadTrafficData(Task task) { + try { + List pullApiTrafficReqs = task.getTrafficToPullConfList().getApiTrafficReqList(); + + //批量加载流量数据的请求列表 + List getTrafficReqList = new ArrayList<>(); + + pullApiTrafficReqs.forEach(pullApiTrafficReq -> { + GetTrafficReq getTrafficReq = new GetTrafficReq(); + getTrafficReq.setRecordingConfigId(pullApiTrafficReq.getTrafficConfigId()); + getTrafficReq.setUrl(pullApiTrafficReq.getUrl()); + getTrafficReq.setStartTime(pullApiTrafficReq.getFromTime()); + getTrafficReq.setEndTime(pullApiTrafficReq.getToTime()); + getTrafficReqList.add(getTrafficReq); + }); + //获取每个conf的流量记录条数 + Map confTrafficSizeMap = trafficDubboService.getTrafficRecordCountList(getTrafficReqList).getData(); + + //更新每个conf请求的拉取数据范围 + getTrafficReqList.forEach(getTrafficReq -> { + if (!confTrafficSizeMap.containsKey(getTrafficReq.getRecordingConfigId())) { + //must be bug + log.warn("load traffic size failed!"); + } else { + //该配置项当前筛选范围的流量总条数 + int totalSize = confTrafficSizeMap.get(getTrafficReq.getRecordingConfigId()); + //当前页数与大小根据压测机集群机器总数及本机器索引位置计算, + //例如 共5台发压机,流量总量10000,本机 index 为1,则page为2,pageSize为2000 + int page = task.getAgentIndex() + 1; + int pageSize = totalSize / task.getAgentNum(); + //最多允许 1w 行 + if (pageSize > AGENT_MAX_TRAFFIC_SIZE) { + pageSize = AGENT_MAX_TRAFFIC_SIZE; + } + getTrafficReq.setPage(page); + getTrafficReq.setPageSize(pageSize); + } + }); + //拉取具体流量数据列表 + List basicTrafficLists = trafficDubboService.getBasicTrafficList(getTrafficReqList).getData(); + + //更新filter内的缓存流量数据 + TrafficFilter.updateTrafficCache(basicTrafficLists); + } catch (Exception e) { + log.error("DatasetService loadTrafficData error:{}", e.getMessage()); + } + } + + public TreeMap> loadAndInitParamDataset(Task task) { + + if (datasetCache.get(task.getReportId()) != null && !task.isDebug()) { + return datasetCache.get(task.getReportId()).getDataMap(); + } else { + //每台发压机随机取一部分请求数据 + int agentNum = task.getAgentNum(); + + log.info("loadAndInitParamDataset agent num:{}",agentNum); + + //该机器索引位置 + int index = task.getAgentIndex(); + + log.info("loadAndInitParamDataset agent index:{}",index); + + List reqList = new ArrayList<>(); + // 从 dashboard 获取该场景下数据源文件名及对应行数 + List datasetLineNums = datasetInfoSercice.getLineNumBySceneId(task.getSceneId()).getData(); + log.info("loadAndInitParamDataset datasetLineNums :{}",gson.toJson(datasetLineNums)); + + datasetLineNums.forEach(datasetLineNum -> { + DatasetLinesReq req = new DatasetLinesReq(); + req.setDatasetId(datasetLineNum.getDatasetId()); + req.setFileUrl(datasetLineNum.getFileUrl()); + req.setFileKsKey(datasetLineNum.getFileKsKey()); + req.setDefaultParamName(datasetLineNum.getDefaultParamName()); + long lineNum = datasetLineNum.getFileRaw(); + if (datasetLineNum.getIgnoreFirstLine()) { + lineNum -= 1; + } + double range = (double) (lineNum / agentNum); + if (range < 1.0) { + range = 1.0; + } + if (isInt(range)) { + req.setFrom(index * (int) range); + req.setTo((index + 1) * (int) range); + } else { + req.setFrom(index * (int) range); + req.setTo((index + 1) * (int) range + 1); + } + + reqList.add(req); + }); + Stopwatch dsw = Stopwatch.createStarted(); + + Result>> dataRes; + try { + log.info("loadAndInitParamDataset getDatasetMap :{}",gson.toJson(reqList)); + + dataRes = datasetInfoSercice.getDatasetMap(reqList); + } catch (Exception e) { + log.error("getDatasetMap faild,cause by:{}", e.getMessage(),e); + return new TreeMap<>(); + } + log.debug("call dubbo data task {} load dataset use time:{}", task.getId(), dsw.elapsed(TimeUnit.MILLISECONDS)); + if (dataRes == null || dataRes.getData() == null || dataRes.getData().isEmpty()) { + log.error("get dataset error ,data is empty,reqList:{}", Arrays.toString(reqList.toArray())); + return new TreeMap<>(); + } + + //每次压测只拉取缓存一次 + if (!task.isDebug()) { + //调试任务直接拉取,不缓存 + datasetCache.putIfAbsent(task.getReportId(), new DataMapCache(new AtomicInteger(1), dataRes.getData())); + } + return dataRes.getData(); + } + } + + + /** + * 数据过大,二分切分重拉 + */ +// private void reverseAppendData(TreeMap> totalRes, List reqList, run.mone.mibench.dashboard.bo.common.Result>> dataRes) { +// if (dataRes.getCode() == Const.DUBBO_DATA_TOO_LONG_ERR_CODE) { +// List reqListFront = new ArrayList<>(); +// List reqListBack = new ArrayList<>(); +// +// for (DatasetLinesReq req : reqList) { +// DatasetLinesReq frontReq = new DatasetLinesReq(); +// BeanUtils.copyProperties(req, frontReq); +// DatasetLinesReq backReq = new DatasetLinesReq(); +// BeanUtils.copyProperties(req, backReq); +// +// int from = req.getFrom(); +// int to = req.getTo(); +// +// frontReq.setFrom(from); +// frontReq.setTo((to + from) / 2); +// reqListFront.add(frontReq); +// +// backReq.setFrom((to + from) / 2 + 1); +// backReq.setTo(to); +// reqListBack.add(backReq); +// } +// +// run.mone.mibench.dashboard.bo.common.Result>> dataResFront = datasetInfoSercice.getDatasetMap(reqListFront); +// reverseAppendData(totalRes, reqListFront, dataResFront); +// +// run.mone.mibench.dashboard.bo.common.Result>> dataResBack = datasetInfoSercice.getDatasetMap(reqListBack); +// reverseAppendData(totalRes, reqListBack, dataResBack); +// +// } else { +// TreeMap> tempData = dataRes.getData(); +// tempData.keySet().forEach(key -> { +// if (totalRes.containsKey(key)) { +// totalRes.get(key).addAll(tempData.get(key)); +// } else { +// totalRes.put(key, tempData.get(key)); +// } +// }); +// } +// } + + /** + * @param currentTask 当前任务 + * @param lineFlag 读取的行数 + */ + public CommonReqInfo processTaskParam(Task currentTask, int lineFlag, boolean isDebug) throws CloneNotSupportedException { + log.debug("processTaskParam lineFlag :{}, dataMap:{}",lineFlag,currentTask.getDataMap()); + + CommonReqInfo commonReqInfo = new CommonReqInfo(); + TreeMap> dataMap = currentTask.getDataMap(); + //替换数据源参数 + if (currentTask.getType() == TaskType.http) { + HttpData httpData = currentTask.getHttpData().clone(); + //替换路径参数 + if (httpData.getUrl().contains("${")) { + processDatasetUrl(httpData, dataMap, lineFlag); + } + commonReqInfo.setDebugUrl(httpData.getUrl()); + //http场景 + if (httpData.getMethod().equalsIgnoreCase(Const.HTTP_GET)) { + //get请求 + List tmpList = new ArrayList<>(httpData.getParamNum()); + IntStream.range(0, httpData.getParamNum()).forEach(i -> { + String paramValue = httpData.getParams().get(i).toString(); + tmpList.add(getValue(paramValue, dataMap, lineFlag)); + }); +// commonReqInfo.setGetOrFormParamsList(tmpList); + commonReqInfo.setQueryParamMap(httpData.httpGetTmpParams(tmpList)); + if (currentTask.isDebug()) { + commonReqInfo.setParamsType(ReqParamType.Http_Get.code); + commonReqInfo.setParamJson(gson.toJson(httpData.httpGetTmpParams(tmpList))); + } + } else if (httpData.getMethod().equalsIgnoreCase(Const.HTTP_POST)) { + if (httpData.getContentType().equals(CONTENT_TYPE_APP_FORM) || httpData.getContentType().equals(CONTENT_TYPE_APP_FORM2)) { + //post 表单,处理方式同get + List tmpList = new ArrayList<>(httpData.getParamNum()); + + IntStream.range(0, httpData.getParamNum()).forEach(i -> { + String paramValue = httpData.getParams().get(i).toString(); + tmpList.add(getValue(paramValue, dataMap, lineFlag)); + }); + commonReqInfo.setQueryParamMap(httpData.httpGetTmpParams(tmpList)); + if (currentTask.isDebug()) { + commonReqInfo.setParamsType(ReqParamType.Http_Post_Form.code); + commonReqInfo.setParamJson(gson.toJson(httpData.httpGetTmpParams(tmpList))); + } + } else { + //post json 先转json处理,后续考虑是否优化 + String body = httpData.getJsonParam().get(); +// if (currentTask.getHttpData().getUrl().contains("proretail-api.nr.mi.com/api/nstrade/mobile/customer/queryCustomerInfo")){ +// log.info("final before json body:{},time:{}", body, System.currentTimeMillis()); +// } +// 使用完成后恢复为源参数体 +// httpData.recoverJsonParam(); + Matcher m = EL_PATTERN.matcher(body); + while (m.find()) { + String expr = m.group(1); + Object targetV; + String v = getDataValue(expr, dataMap, lineFlag); + if (NumberUtils.isNumber(v)) { + if (isNumeric(v)) { + targetV = Long.parseLong(v); + } else { + targetV = Double.parseDouble(v); + } + } else { + targetV = v; + } +// log.info("final target v:{}", targetV); + if (targetV != null && StringUtils.isNotEmpty(targetV.toString())){ + body = Util.Parser.parse$(expr, body, targetV); + } + } +// log.info("final after json body:{}", body); + httpData.setPostParamJson(body); + commonReqInfo.setParamsType(ReqParamType.Http_Post_Json.code); + commonReqInfo.setParamJson(body); + } + } + //替换header参数 + Map tmpHeaders = new HashMap<>(); + for (Map.Entry headerMap : + httpData.getHeaders().entrySet()) { + tmpHeaders.put(headerMap.getKey(), getValue(headerMap.getValue(), dataMap, lineFlag)); + } + commonReqInfo.setHeaders(tmpHeaders); + } else if (currentTask.getType() == TaskType.dubbo) { + DubboData dubboData = currentTask.getDubboData(); + String body = dubboData.getJsonParam().get(); +// dubboData.recoverJsonParam(); + Matcher m = EL_PATTERN.matcher(body); + while (m.find()) { + String expr = m.group(1); + Object targetV; + String v = getDataValue(expr, dataMap, lineFlag); + if (NumberUtils.isNumber(v)) { + if (isNumeric(v)) { + targetV = Long.parseLong(v); + } else { + targetV = Double.parseDouble(v); + } + } else { + targetV = v; + } + if (targetV != null && StringUtils.isNotEmpty(targetV.toString())){ + body = Util.Parser.parse$(expr, body, targetV); + } + } + commonReqInfo.setParamsType(ReqParamType.Dubbo.code); + commonReqInfo.setParamJson(body); + } + return commonReqInfo; + } + + private void processDatasetUrl(HttpData httpData, TreeMap> dataMap, int n) { + if (dataMap == null) { + return; + } + String k; + String v = null; + String url = httpData.getUrl(); + String tmpUrl = url; + if (null != url && url.length() > 0) { + Matcher m = EL_PATTERN.matcher(url); + while (m.find()) { + try { + k = m.group(1); + if (dataMap.get(k) != null && dataMap.get(k).size() != 0) { + int line = n % dataMap.get(k).size(); + v = dataMap.get(k).get(line); + } + try { + if (v != null) { + tmpUrl = Util.Parser.parse$(k, tmpUrl, v); + } + } catch (Exception e) { + log.error("processDatasetUrl error,k:{}, v:{}, error:{}", k, v, e); + } + } catch (Exception e) { + log.error("processDatasetUrl 2 error, v:{}, error:{}", v, e); + } + } + } + httpData.setUrl(tmpUrl); + } + + private String getValue(String key, TreeMap> dataMap, int n) { + if (dataMap == null) { + return key; + } + //适配 ${key} 格式 + if (key.startsWith("${")) { + Pair pair = getElKey(key, dataMap, n); + return pair.getValue(); + } + return key; + } + + private static Pair getElKey(String key, TreeMap> dataMap, int n) { + String k = null; + String v = null; + if (null != key && key.length() > 0) { + Matcher m = EL_PATTERN.matcher(key); + + if (m.find()) { + k = m.group(1); + if (dataMap.get(k) != null && dataMap.get(k).size() != 0) { + int line = n % dataMap.get(k).size(); + v = dataMap.get(k).get(line); + } + } + } + if (v == null) { + v = k; + } + return Pair.of(k, v); + } + + private static String getDataValue(String key, TreeMap> dataMap, int n) { + log.debug("getDataValue lineFlag :{}, dataMap:{}",n,dataMap); + if (dataMap == null) { + return ""; + } + if (null != key && key.length() > 0) { + if (dataMap.get(key) == null) { + return ""; + } + log.debug("getDataValue size :{}",dataMap.get(key).size()); + + if (dataMap.get(key).size() != 0) { + int line = n % dataMap.get(key).size(); + log.debug("getDataValue line :{},val:{}",line,dataMap.get(key).get(line)); + + return dataMap.get(key).get(line); + } + } + return ""; + } + + public void processDataMapCache(Task task) throws InterruptedException { + if (datasetCache.get(task.getReportId()) != null) { + int currFinLinkNum = datasetCache.get(task.getReportId()).getCountFinLinkNum().addAndGet(1); + if (currFinLinkNum == task.getConnectTaskNum()) { + //最后一条链路完成,延迟100s清理该次压测的缓存数据源 + Thread.sleep(100 * 1000); + datasetCache.remove(task.getReportId()); + } + } + } + + private static boolean isInt(double num) { + return Math.abs(num - Math.round(num)) < Double.MIN_VALUE; + } + + public static boolean isNumeric(String str) { + for (int i = str.length(); --i >= 0; ) { + int chr = str.charAt(i); + if (chr < 48 || chr > 57) + return false; + } + return true; + } +} + diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/HostsService.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/HostsService.java new file mode 100644 index 000000000..0970b609e --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/HostsService.java @@ -0,0 +1,202 @@ +package run.mone.mimeter.engine.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import run.mone.mimeter.engine.agent.bo.hosts.HostBo; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +@Slf4j +public class HostsService { + + /** + * 获取host文件路径 + */ + public static String getHostFile() { + String fileName; + // 判断系统 + if ("linux".equalsIgnoreCase(System.getProperty("os.name")) || "Mac OS X".equalsIgnoreCase(System.getProperty("os.name"))) { + fileName = "/etc/hosts"; +// fileName = "/Users/dongzhenxing/Desktop/hosts"; + } else { + //不支持其他系统 + return ""; + } + return fileName; + } + + public static List loadHostFile() { + // Step1: 获取host文件 + String fileName = getHostFile(); + List hostFileDataLines = new ArrayList<>(); + try { + hostFileDataLines = FileUtils.readLines(new File(fileName)); + } catch (IOException e) { + log.error("Reading host file occurs error: " + e.getMessage()); + } + return (List) hostFileDataLines; + } + + /** + * 根据输入Domain,批量删除host文件中的host配置 + */ + public synchronized static boolean deleteDomainsConfig(List confList) { + if (confList == null || confList.size() == 0) { + return true; + } + confList.forEach(domainMap -> { + if (domainMap.getDomain() == null || domainMap.getDomain().trim().isEmpty()) { + throw new IllegalArgumentException("ERROR: ip & domain must be specified"); + } + }); + // Step1: 获取host文件 + String fileName = getHostFile(); + if (fileName.isEmpty()) { + throw new IllegalArgumentException("ERROR:get host file path failed"); + } + List hostFileDataLines; + try { + hostFileDataLines = FileUtils.readLines(new File(fileName)); + } catch (IOException e) { + log.error("Reading host file occurs error: " + e.getMessage()); + return false; + } + // Step2: 解析host文件,如果指定域名不存在,则Ignore,如果已经存在,则直接删除该行配置 + List newLinesList = new ArrayList<>(); + // 标识本次文件是否有更新,比如如果指定的IP和域名已经在host文件中存在,则不用再写文件 + AtomicBoolean updateFlag = new AtomicBoolean(false); + + List finalHostFile = (List) hostFileDataLines; + + confList.forEach(domainMap -> { + String domain = domainMap.getDomain(); + for (Object line : finalHostFile) { + String strLine = line.toString(); + // host文件中的空行或无效行,直接跳过 + if (StringUtils.isEmpty(strLine) || strLine.trim().equals("#")) { + continue; + } + // 如果没有被注释掉,则 + if (!strLine.trim().startsWith("#")) { + int index = strLine.toLowerCase().indexOf(domain.toLowerCase()); + // 如果行字符可以匹配上指定域名,则针对该行做操作 + if (index != -1) { + // 匹配到相同的域名,直接将整行数据干掉 + updateFlag.set(true); + continue; + } + } + // 如果没有匹配到,直接将当前行加入代码中 + newLinesList.add(strLine); + } + finalHostFile.clear(); + finalHostFile.addAll(newLinesList); + newLinesList.clear(); + }); + //Step3: 将更新后的数据写入host文件中去 + if (updateFlag.get()) { + try { + FileUtils.writeLines(new File(fileName), finalHostFile); + } catch (IOException e) { + log.error("Updating host file occurs error: " + e.getMessage()); + return false; + } + } + return true; + } + + /** + * 根据输入IP和Domain,更新host文件中的某个host配置 + */ + public synchronized static boolean updateHostConfig(List confList) { + if (confList == null || confList.size() == 0) { + return true; + } + // Step1: 获取host文件 + String fileName = getHostFile(); + List hostFileDataLines; + List finalHostFile; + try { + hostFileDataLines = FileUtils.readLines(new File(fileName)); + } catch (IOException e) { + log.error("Reading host file occurs error: " + e.getMessage()); + return false; + } + finalHostFile = (List) hostFileDataLines; + //Step2: 解析host文件,如果指定域名不存在,则追加,如果已经存在,则修改IP进行保存 + List newLinesList = new ArrayList<>(); + // 指定domain是否存在,如果存在,则不追加 + final boolean[] findFlag = {false}; + // 标识本次文件是否有更新,比如如果指定的IP和域名已经在host文件中存在,则不用再写文件 + final AtomicBoolean[] updateFlag = {new AtomicBoolean(false)}; + List finalHostFile1 = finalHostFile; + confList.forEach(domainMap -> { + for (Object line : finalHostFile1) { + String strLine = line.toString(); + // 将host文件中的空行或无效行,直接去掉 + if (StringUtils.isEmpty(strLine) || strLine.trim().equals("#")) { + continue; + } + if (!strLine.startsWith("#")) { + int index = strLine.toLowerCase().indexOf(domainMap.getDomain().toLowerCase()); + // 如果行字符可以匹配上指定域名,则针对该行做操作 + if (index != -1) { + // 如果之前已经找到过一条,则说明当前line的域名已重复, + // 故删除当前line, 不将该条数据放到newLinesList中去 + if (findFlag[0]) { + updateFlag[0].set(true); + continue; + } + // 否则,继续寻找 + String[] array = strLine.trim().split(" "); + //遍历域名 + for (int i = 1; i < array.length; i++) { + if (domainMap.getDomain().equalsIgnoreCase(array[i])) { + findFlag[0] = true; + // IP相同,则不更新该条数据,直接将数据放到newLinesList中去 + if (!array[0].equals(domainMap.getIp())) { + // IP不同,将匹配上的domain的ip 更新成设定好的IP地址 + StringBuilder sb = new StringBuilder(); + sb.append(domainMap.getIp()); + for (int j = 1; i < array.length; i++) { + //array[j] 域名 + sb.append(" ").append(array[j]); + } + strLine = sb.toString(); + updateFlag[0].set(true); + } + } + } + } + } + // 如果有更新,则会直接更新到strLine中去 + // 故这里直接将strLine赋值给newLinesList + newLinesList.add(strLine); + } + // Step3: 如果没有任何Host域名匹配上,则追加 + if (!findFlag[0]) { + newLinesList.add(domainMap.getIp() + " " + domainMap.getDomain()); + } + finalHostFile1.clear(); + finalHostFile1.addAll(newLinesList); + newLinesList.clear(); + findFlag[0] = false; + }); + + //Step4: 写设定文件 + if (updateFlag[0].get() || !findFlag[0]) { + try { + FileUtils.writeLines(new File(fileName), finalHostFile1); + } catch (IOException e) { + log.error("Updating host file occurs error: " + e.getMessage()); + return false; + } + } + return true; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/LoggerBuilder.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/LoggerBuilder.java new file mode 100644 index 000000000..7fc26f0c9 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/LoggerBuilder.java @@ -0,0 +1,26 @@ +package run.mone.mimeter.engine.service; + +import com.xiaomi.sautumn.api.bo.ResourceKey; +import com.xiaomi.sautumn.api.bo.ResourceType; +import com.xiaomi.sautumn.api.service.ResourceService; +import com.xiaomi.youpin.docean.anno.Component; +import lombok.Data; +import org.slf4j.Logger; + +import javax.annotation.Resource; + +@Component +@Data +public class LoggerBuilder { + + @Resource + private ResourceService resourceService; + + private Logger logger; + + public void init(){ + ResourceKey customLogKey = new ResourceKey("customLogKey", ResourceType.customLog); + logger = resourceService.getResource(customLogKey); + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/MetricsService.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/MetricsService.java new file mode 100644 index 000000000..25ee6f642 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/main/java/run/mone/mimeter/engine/service/MetricsService.java @@ -0,0 +1,130 @@ +package run.mone.mimeter.engine.service; + +import com.google.common.collect.Sets; +import com.xiaomi.youpin.docean.anno.Service; +import com.xiaomi.youpin.prometheus.client.Metrics; +import io.prometheus.client.Collector; +import io.prometheus.client.CollectorRegistry; +import io.prometheus.client.exporter.common.TextFormat; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.*; + +import static com.google.common.base.Preconditions.checkArgument; +import static common.Const.*; +import static run.mone.mimeter.engine.agent.bo.stat.MetricLabelEnum.getLabelApiRpsNames; +import static run.mone.mimeter.engine.agent.bo.stat.MetricLabelEnum.getLabelNames; + +/** + * @author Xirui Yang (yangxirui@xiaomi.com) + * @version 1.0 + * @since 2022/7/25 + */ +@Service +public class MetricsService { + + private static final String logPrefix = "[MetricsService]"; + + private static final Logger log = LoggerFactory.getLogger(MetricsService.class); + + private CollectorRegistry defaultRegistry; + + public void init() { + this.defaultRegistry = CollectorRegistry.defaultRegistry; + } + + public String metrics() { + return this.metrics(this.defaultRegistry); + } + + private String metrics(CollectorRegistry registry) { + List list = new ArrayList<>(); + Enumeration samples = registry.filteredMetricFamilySamples(Sets.newHashSet(list)); + StringWriter writer = new StringWriter(); + String str = ""; + + try { + TextFormat.write004(writer, samples); + StringBuffer sb = writer.getBuffer(); + str = sb.toString(); + } catch (IOException e) { + log.error(logPrefix + "metrics writer buffer error", e); + } + return str; + } + + private static void validate(String metricName, long value, String[] labelNames) { + checkArgument(value >= 0, logPrefix + "record invalid value"); + checkArgument(StringUtils.isNotBlank(metricName), logPrefix + "record empty metricName"); + checkArgument(labelNames != null && labelNames.length > 0, logPrefix + "record empty labelNames"); + } + + public static void recordCounter(String metricName, long value, String... labelValues) { + recordCounter(metricName, getLabelNames(), value, labelValues); + } + + public static void recordCounter(String metricName, String[] labelNames, long value, String... labelValues) { + try { + validate(metricName, value, labelNames); + Metrics.getInstance().newCounter(metricName, labelNames).with(labelValues).add(value, labelValues); + } catch (Exception e) { + log.error(logPrefix + "recordCounter exception; " + e.getMessage()); + } + } + + public static void recordRt(String metricName, long value, String... labelValues) { + recordRt(metricName, getLabelNames(), value, labelValues); + } + + public static void recordRt(String metricName, String[] labelNames, long value, String... labelValues) { + double[] buckets = new double[]{10.0, 100.0, 200.0, 300.0, 400.0, 600.0, 800.0, 1000.0, 1400.0, 1800.0, 2200.0, 2600.0, 3000.0, 6000.0}; + recordHistogram(metricName, labelNames, buckets, value, labelValues); + } + + private static void checkLabelValues(String... labelValues) { + boolean hasNull = false; + + for (int i = 0; i < labelValues.length; i++) { + if (labelValues[i] == null) { + hasNull = true; + labelValues[i] = ""; + } + } + if (hasNull) { + log.error(logPrefix + "checkLabelValues null value; " + Arrays.toString(labelValues)); + } + } + + private static void recordHistogram(String metricName, String[] labelNames, double[] buckets, long value, String... labelValues) { + String logSuffix = ""; + + try { + validate(metricName, value, labelNames); + checkArgument(buckets != null && buckets.length > 0, logPrefix + "recordHistogram empty buckets"); + checkLabelValues(labelValues); + + logSuffix = ", label names " + Arrays.toString(labelNames) + ", label values " + Arrays.toString(labelValues) + "; "; + checkArgument(labelValues != null && labelValues.length == labelNames.length, + logPrefix + "recordHistogram invalid label" + logSuffix); + + Metrics.getInstance().newHistogram(metricName, buckets, labelNames).with(labelValues).observe(value, labelValues); + } catch (Exception e) { + log.error(logPrefix + "recordHistogram exception" + logSuffix + e.getMessage()); + } + } + + public static void recordTpsAndRtMetrics(long elapsed, String[] labelVals,boolean recordTps) { + recordRt(METRICS_NAME_RT_API, elapsed, labelVals); + if (recordTps){ + recordCounter(METRICS_NAME_TPS_API, 1, labelVals); + } + } + + public static void recordApiRpsMetrics(String[] labelVals) { + recordCounter(METRICS_NAME_RPS_API, getLabelApiRpsNames(),1, labelVals); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/ConcurrencyTest.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/ConcurrencyTest.java new file mode 100644 index 000000000..85aa63591 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/ConcurrencyTest.java @@ -0,0 +1,120 @@ +package run.mone.mimeter.engine.service.test; + +import com.google.common.base.Stopwatch; +import org.junit.Ignore; +import org.junit.Test; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.IntStream; + +import static common.Util.*; + +/** + * @author Xirui Yang (yangxirui@xiaomi.com) + * @version 1.0 + * @since 2022/8/28 + */ +@Ignore +public class ConcurrencyTest { + + + private String m() { + try { + + } finally { + if (1 == 1) { + return "def"; + } + } + System.out.println("return"); + return "abc"; + } + + + @Test + public void testFinally() { + System.out.println(m()); + } + + @Test + public void testLatch() { + CountDownLatch l = new CountDownLatch(10); + IntStream.range(0,22).forEach(i->{ + l.countDown(); + }); + System.out.println(l.getCount()); + } + + + @Test + public void testMap() { + ConcurrentHashMap m = new ConcurrentHashMap<>(); + m.put("abc", "abc"); + Object v = m.remove("def"); + System.out.println(v); + System.out.println(m); + } + + @Test + public void atomicTest() { + AtomicLong al = new AtomicLong(0L); + modifyAL(al, 1L); + System.out.println(al.get()); + } + + private void modifyAL(AtomicLong al, long newVal) { + al.set(newVal); + } + +// @Test +// public void nullCastTest() { +// Integer val = null; +// System.out.println((long) val); +// } + + @Test + public void sampleRateToCntTest() { + System.out.println(sampleRateToCnt(1, 5)); + } + + @Test + public void samplingTest() { + int[][] lists = new int[][]{{10, 10}, {1000, 100}, {100000, 5000}, {100000, 20000}, {100000, 50000}, {100000, 80000}}; + + for (int[] arr : lists) { + Stopwatch sw = Stopwatch.createStarted(); + sampleIndices(arr[0], arr[1]); + System.out.println("sampling indices takes:" + sw.elapsed(TimeUnit.MILLISECONDS) + "ms"); + } + } + + @Test + public void sampleByTimeAndQpsTest() { + int[][] lists = new int[][]{{8, 1, 10}, {1000, 100, 5}}; + + for (int[] arr : lists) { + int duration = arr[0]; + int qps = arr[1]; + Set logIndices = new HashSet<>(sampleByTimeAndQps(duration, qps, arr[2])); + System.out.println("sampling indices:" + logIndices); + + int hits = 0; + + for (int j = 1; j <= duration; j++) { + for (int i = 0; i < qps; i++) { + if (logIndices.contains(((j - 1) % 10) * qps + i)) { + hits++; + } + } + } + int total = duration * qps; + System.out.println("hits:" + hits + ", total:" + total + ", rate:" + ((100.0d * hits) / total)); + } + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/DagClientTest.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/DagClientTest.java new file mode 100644 index 000000000..663938e1d --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/DagClientTest.java @@ -0,0 +1,372 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.service.test; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.gson.Gson; +import com.xiaomi.data.push.schedule.task.graph.GraphTaskContext; +import com.xiaomi.data.push.schedule.task.graph.TaskEdgeData; +import com.xiaomi.data.push.schedule.task.graph.TaskVertexData; +import com.xiaomi.youpin.docean.Ioc; +import com.xiaomi.youpin.docean.common.StringUtils; +import common.HttpClientV6; +import common.HttpResult; +import common.Util; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import run.mone.mimeter.engine.agent.bo.data.*; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskContext; +import run.mone.mimeter.engine.agent.bo.task.TaskType; +import run.mone.mimeter.engine.client.base.DagClient; + +import java.util.*; +import java.util.concurrent.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.IntStream; + +/** + * @author goodjava@qq.com + * @date 2022/5/30 + */ +public class DagClientTest { + + private static Logger log = LoggerFactory.getLogger(DagClientTest.class); + + private static final Pattern EL_PATTERN = Pattern.compile("\\$\\{(.*)}"); + + @Test + public void test5(){ + Map> map = new HashMap<>(); + List tobeR = new ArrayList<>(); + tobeR.add("dzx"); + tobeR.add("dp"); + tobeR.add("yxr"); + + map.put("username",tobeR); + + List age = new ArrayList<>(); + age.add("18"); + age.add("19"); + age.add("20"); + map.put("age",age); + + String body = "[\n" + + " {\n" + + " \"id\": \"${username}\",\n" + + " \"obj\": {\n" + + " \"name\": \"${username}\",\n" + + " \"age\": \"${age}\"\n" + + " }\n" + + " }\n" + + "]"; + + Matcher m = EL_PATTERN.matcher(body); + Random random = new Random(); + int lineFlag = random.nextInt(100); + while (m.find()) { + String expr = m.group(1); + body = Util.Parser.parse$(expr, body, null); + } + System.out.println(body); + } + + @Test + public void testDagClient() { + Ioc ioc = Ioc.ins(); + ioc.putBean(ioc).init("run.mone.mibench.engine"); + DagClient client = ioc.getBean("dagClient"); + Task task = new Task(); + task.setDagInfo(getContext()); + log.info("{}", task.getDagInfo()); + client.call(task, new TaskContext(),null,null); + } + + /** + * 测试纯http的dag链路任务(图的一个形式,一个链表) + * 先获取token 然后根据token获取id 最后通过id获取age + */ + @Test + public void testDagClientHttp() throws InterruptedException { + Ioc ioc = Ioc.ins(); + ioc.putBean(ioc).init("run.mone.mibench.engine"); + DagClient client = ioc.getBean("dagClient"); + Task task = new Task(); + task.setTimeout(5000); + task.setDagInfo(getHttpDagContext()); + + ExprKey key1 = new ExprKey(2,2, "", "params.toMap(){data}", Lists.newArrayList("httpData", "0", "a")); + //获取1的结果中的data + ExprKey key2 = new ExprKey(1,2, "", "params.toMap(){data}", Lists.newArrayList("httpData", "1", "b")); + + Map map = new HashMap<>(); + map.put(key1,""); + map.put(key2,""); + + String j = new Gson().toJson(map); + + log.info("{}", task.getDagInfo()); + client.call(task, new TaskContext(),null,null); + Thread.currentThread().join(); + } + + @Test + public void testClientHttp() { + List headers = new ArrayList<>(); + Map map = new HashMap<>(); + Map headerMap = new HashMap<>(); + headerMap.put("Content-Type","x-www-form-urlencoded"); + headerMap.put("Xm-Forged-Req","true"); + + map.put("invoice_type","4"); + map.put("invoice_title","person"); + map.put("use_red_packet", "1"); + map.put("shipment_id", "2"); + HttpResult res = null; + try { + Gson gson = new Gson(); + //http://api2.order.mi.com/buy/submit + res = HttpClientV6.httpPost(null,"http://www.test.com/submit",headerMap,map,"UTF-8",5000); + String json = gson.toJson(res.content); + System.out.println(json); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Test + public void testDivide(){ + int totalErrCount = 10; + int totalCount = 141; + double rate = (1.0 * totalErrCount / totalCount)*100; + + System.out.println(Math.round(rate)); + } + @Test + public void testC() { + double v = Double.parseDouble("99"); + System.out.println((int) v); + } + + + static class T { + public String id; + + public String parent; + + public List childList = new ArrayList<>(); + + public CountDownLatch latch = null; + + public T(String id, String parent) { + this.id = id; + this.parent = parent; + if (!StringUtils.isEmpty(parent)) { + latch = new CountDownLatch(1); + } else { + latch = new CountDownLatch(0); + } + } + + public void execute() { + System.out.println("execute:" + this.id); + childList.stream().forEach(it -> { + it.latch.countDown(); + }); + } + + + } + + + @Test + public void testList() throws InterruptedException { + ExecutorService pool = Executors.newVirtualThreadPerTaskExecutor(); + T t2 = new T("2", "1"); + T t1 = new T("1", ""); + t1.childList.add(t2); + List list = Lists.newArrayList(t2, t1); + list.stream().parallel().forEach(it -> { + System.out.println("run:" + it.id); + CompletableFuture.supplyAsync(() -> { + try { + it.latch.await(); + it.execute(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return it; + }, pool); + }); + Thread.currentThread().join(); + } + + @Test + public void testLatch() throws InterruptedException { + CountDownLatch latch = new CountDownLatch(1); + new Thread(()->{ + try { + TimeUnit.SECONDS.sleep(5); + latch.countDown(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + }).start(); + latch.await(); + System.out.println(latch.getCount()); + } + + @Test + public void testCountdownLatch() throws InterruptedException { + CountDownLatch countDownLatch = new CountDownLatch(0); + countDownLatch.await(); + + CountDownLatch countDownLatch1 = new CountDownLatch(10); + + IntStream.range(0, 12).forEach(it -> { + System.out.println(it); + countDownLatch1.countDown(); + }); + countDownLatch1.await(); + + System.out.println("finish"); + } + + public GraphTaskContext getContext() { + GraphTaskContext taskContext = new GraphTaskContext<>(); + taskContext.setTaskList(Lists.newArrayList(createVertex(0, 0, new ExprKey(0, 2,"1_p", "params", Lists.newArrayList()), "0"), + createVertex(1, 3, new ExprKey(0, 2,"1_p", "params", Lists.newArrayList()), "1"), + //直接把0的结果放到2的参数中(map->i) + createVertex(2, 0, new ExprKey(0,2, "2_p", "params", Lists.newArrayList("demoData", "data", "i")), "2"), + createVertex(3, 0, new ExprKey(2, 2,"3_p", "params", Lists.newArrayList("demoData", "data", "i")), "3"))); + //1依赖0 2依赖于0 3依赖于2 (1的执行时间比较久3秒,所以会先输出3的结果) + taskContext.setDependList(Lists.newArrayList(new TaskEdgeData(0, 1), new TaskEdgeData(0, 2), new TaskEdgeData(2, 3))); + return taskContext; + } + + + /** + * 执行三步 获取 token -> 获取 id -> 获取 agent -> 求和 + * + * @return + */ + public GraphTaskContext getHttpDagContext() { + GraphTaskContext taskContext = new GraphTaskContext<>(); + taskContext.setTaskList(Lists.newArrayList( + //获取token + createHttpCalVertex(0, "post", Lists.newArrayList(new ParamType(ParamTypeEnum.pojo, "pojo")), + Lists.newArrayList(new ExprKey(0, 2,"", "", Lists.newArrayList())), + "token", "http://127.0.0.1:7777/token", ImmutableMap.of("uuid", "123")), + //换取id + createHttpCalVertex(1, "post", Lists.newArrayList(new ParamType(ParamTypeEnum.pojo, "pojo")), + Lists.newArrayList(new ExprKey(0, 2,"", "params.toMap(){data}", Lists.newArrayList("httpData", "0", "token"))), + "id", "http://127.0.0.1:7777/id", ImmutableMap.of("uuid", "456")), + //获取age + createHttpCalVertex(2, "post", Lists.newArrayList(new ParamType(ParamTypeEnum.pojo, "pojo")), + Lists.newArrayList(new ExprKey(1, 2,"", "params.toMap(){data}", Lists.newArrayList("httpData", "0", "id"))), + "age", "http://127.0.0.1:7777/age", ImmutableMap.of("uuid", "789")), + //求和(多参数) + createHttpCalVertex(3, "get", Lists.newArrayList( + new ParamType(ParamTypeEnum.primary, "a"), + new ParamType(ParamTypeEnum.primary, "b") + ), + Lists.newArrayList( + //获取2的结果中的data + new ExprKey(2,2, "", "params.toMap(){data}", Lists.newArrayList("httpData", "0", "a")), + //获取1的结果中的data + new ExprKey(1,2, "", "params.toMap(){data}", Lists.newArrayList("httpData", "1", "b")) + ), + "sum", "http://127.0.0.1:7777/sum", ImmutableMap.of("uuid", "799")) + ) + ); + //任务是有依赖关系的 + taskContext.setDependList(Lists.newArrayList(new TaskEdgeData(0, 1), new TaskEdgeData(1, 2), new TaskEdgeData(2, 3), new TaskEdgeData(1, 3))); + return taskContext; + } + + + private TaskVertexData createHttpCalVertex(int id, String method, List types, List keys, String resultName, String url, Map d) { + int time = 100; + TaskVertexData data = new TaskVertexData<>(); + NodeInfo nodeInfo = new NodeInfo(); + nodeInfo.setId(id); + nodeInfo.setResultName(resultName); + Task task = new Task(); + task.setDebug(true); + task.setId(id); + task.setType(TaskType.http); + task.setTimeout(time); + HttpData httpData = new HttpData(); + httpData.setUrl(url); + httpData.setMethod(method); + httpData.initTypeList(types); + if (httpData.getParamNum() == 1) { + Map m = (Map) httpData.getParams().get(0); + m.putAll(d); + } + + task.setHttpData(httpData); + nodeInfo.setTask(task); + keys.forEach(key -> nodeInfo.getExprMap().put(key, "")); + data.setData(nodeInfo); + data.setIndex(id); + return data; + } + + + private TaskVertexData createVertex(int id, int sleepTime, ExprKey key, String resultName) { + TaskVertexData data = new TaskVertexData<>(); + NodeInfo nodeInfo = new NodeInfo(); + nodeInfo.setId(id); + nodeInfo.setResultName(resultName); + Task task = new Task(); + task.setDebug(true); + task.getAttachments().put("sleep", String.valueOf(sleepTime)); + task.setId(id); + task.setType(TaskType.demo); + task.setDemoData(new DemoData()); + nodeInfo.setTask(task); + nodeInfo.getExprMap().put(key, ""); + data.setData(nodeInfo); + data.setIndex(id); + return data; + } + + @Test + public void testFuture() throws ExecutionException, InterruptedException { + ExecutorService pool = Executors.newVirtualThreadPerTaskExecutor(); + CompletableFuture f = CompletableFuture.supplyAsync(() -> { + try { + TimeUnit.SECONDS.sleep(3); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return "abc"; + }, pool).handle((str, e) -> { + log.info("str:{}", str); + return str; + }); + + System.out.println("get"); + + System.out.println(f.get()); + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/EngineTest.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/EngineTest.java new file mode 100644 index 000000000..fe9cf8bc3 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/EngineTest.java @@ -0,0 +1,46 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.service.test; + +import run.mone.mimeter.engine.service.BenchEngineService; +import org.junit.Test; +import run.mone.mimeter.engine.agent.bo.task.Context; +import run.mone.mimeter.engine.agent.bo.data.HttpData; +import run.mone.mimeter.engine.agent.bo.task.Task; + +/** + * @author goodjava@qq.com + * @date 2022/5/19 + */ +public class EngineTest { + + + @Test + public void testEngine() throws InterruptedException { + BenchEngineService engineService = new BenchEngineService(); + Context context = new Context(); + Task task = new Task(); + task.setId(1); + task.setQps(1000); + task.setTime(10); + HttpData data = new HttpData(); + data.setUrl("http://www.baidu.com"); + task.setHttpData(data); + engineService.submitTask(context, task); + + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/ExprTest.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/ExprTest.java new file mode 100644 index 000000000..a94e7450c --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/ExprTest.java @@ -0,0 +1,105 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.service.test; + +import com.xiaomi.data.push.antlr.expr.Expr; +import common.Util; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.junit.Test; +import run.mone.mimeter.engine.agent.bo.data.HttpData; + +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * @author goodjava@qq.com + * @date 2022/6/2 + */ +public class ExprTest { + + @Data + @AllArgsConstructor + @NoArgsConstructor + private class D { + private int id; + private String name; + } + + @Data + private class Result { + private int code; + private String message; + private D data; + } + + + @Test + public void testResultExpr() { + Map m = new HashMap<>(); + Result r = new Result<>(); + r.setData("data:" + System.currentTimeMillis()); + m.put("id", r); + String v = Expr.result(m, "result{id}.data"); + System.out.println(v); + } + + + @Test + public void t() { + List list = new ArrayList<>(2); + list.add("dsds"); + System.out.println(list.get(0)); + } + + + private static final Pattern EL_PATTERN = Pattern.compile("\\$\\{([^}]*)}"); + + private void processDatasetUrl(HttpData httpData, TreeMap> dataMap, int n) { + if (dataMap == null) { + return; + } + String k; + String v = null; + String url = httpData.getUrl(); + String tmpUrl = url; + if (null != url && url.length() > 0) { + Matcher m = EL_PATTERN.matcher(url); + while (m.find()) { + k = m.group(1); + if (dataMap.get(k) != null && dataMap.get(k).size() != 0) { + int line = n % dataMap.get(k).size(); + v = dataMap.get(k).get(line); + } + tmpUrl = Util.Parser.parse$(k, tmpUrl, v); + } + } + httpData.setUrl(tmpUrl); + } + + @Test + public void testExpr() { + + String json = "{\"a\":\"aaa\",\"c\":2,\"b\":\"ccc\"}"; + + TreeMap treeMap = Util.getGson().fromJson(json, TreeMap.class); + System.out.println(treeMap.get("")); + + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/FutureTest.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/FutureTest.java new file mode 100644 index 000000000..76eaa4d5a --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/FutureTest.java @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.service.test; + +import org.junit.Test; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +/** + * @author goodjava@qq.com + * @date 2022/6/5 + */ +public class FutureTest { + + + @Test + public void testFuture() throws ExecutionException, InterruptedException { + CompletableFuture f = CompletableFuture.supplyAsync(() -> { + try { + TimeUnit.SECONDS.sleep(2); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + System.out.println("finish"); + return "ok"; + }).completeOnTimeout("timeout",1, TimeUnit.SECONDS).handle((v,e)->{ + System.out.println(v); + return v; + }); + System.out.println("aa"); + Thread.currentThread().join(); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/HostUtilTest.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/HostUtilTest.java new file mode 100644 index 000000000..93278239e --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/HostUtilTest.java @@ -0,0 +1,49 @@ +package run.mone.mimeter.engine.service.test; + +import org.junit.Test; +import run.mone.mimeter.engine.agent.bo.hosts.HostBo; +import run.mone.mimeter.engine.service.HostsService; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author dongzhenxing + */ +public class HostUtilTest { + + @Test + public void test() throws InterruptedException { + System.out.println(System.getProperty("os.name")); + } + + @Test + public void testUpdateHosts() { + List hostBoList = new ArrayList<>(); + HostBo hostBo = new HostBo(); + hostBo.setIp("127.0.0.1"); + hostBo.setDomain("test.com"); + + HostBo hostBo2 = new HostBo(); + hostBo2.setIp("127.0.0.2"); + hostBo2.setDomain("test2.com"); + hostBoList.add(hostBo); + hostBoList.add(hostBo2); + HostsService.updateHostConfig(hostBoList); + } + + @Test + public void testDelHosts() { + List hostBoList = new ArrayList<>(); + HostBo hostBo = new HostBo(); + hostBo.setDomain("test.com"); + + HostBo hostBo2 = new HostBo(); + hostBo2.setDomain("test2.com"); + hostBoList.add(hostBo); + hostBoList.add(hostBo2); + HostsService.deleteDomainsConfig(hostBoList); + } + + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/RateLimiterTest.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/RateLimiterTest.java new file mode 100644 index 000000000..b1d7fd620 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/RateLimiterTest.java @@ -0,0 +1,40 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.service.test; + +import com.google.common.util.concurrent.RateLimiter; +import org.junit.Test; + +import java.util.Date; + +/** + * @author goodjava@qq.com + * @date 2022/5/23 + */ +public class RateLimiterTest { + + + @Test + public void testRateLimiter() { + RateLimiter limiter = RateLimiter.create(5); + + for (int i = 0; i < 20; i++) { + limiter.acquire(); + System.out.println(new Date()); + } + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/WaitTest.java b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/WaitTest.java new file mode 100644 index 000000000..7283edb71 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent-service/src/test/java/run/mone/mimeter/engine/service/test/WaitTest.java @@ -0,0 +1,21 @@ +package run.mone.mimeter.engine.service.test; + +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * @author goodjava@qq.com + * @date 2022/9/17 10:17 + */ +public class WaitTest { + + @Test + public void testCountdownLatch() throws InterruptedException { + CountDownLatch latch = new CountDownLatch(1); +// latch.await(); + latch.await(2, TimeUnit.SECONDS); + System.out.println("finish"); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent/README.md b/ozburst-all/mimeter-engine/mimeter-agent/README.md new file mode 100644 index 000000000..dbe1357d2 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent/README.md @@ -0,0 +1,21 @@ ++ 压测agent ++ 使用jdk19-loom ++ 启动函数 ++ com.xiaomi.sautumn.server.bootstrap.SautumnApplication ++ env ++ function.App=mibench-agent;mione.faas.func.env=staging;mione.faas.func.env.id=92;mione.faas.func.id=65;mione.faas.local.jar.path=/Users/zzy/IdeaProjects/mibench-engine/mibench-agent/target/mibench-agent-1.0-SNAPSHOT.jar;mione.faas.test.jar.env=true ++ jvm 参数 ++ --add-opens java.base/java.util=ALL-UNNAMED ++ --add-opens java.base/java.lang=ALL-UNNAMED ++ --add-opens java.base/java.math=ALL-UNNAMED ++ --add-opens java.xml/com.sun.org.apache.xerces.internal.impl.dv.util=ALL-UNNAMED ++ 配置名称 ++ mione.faas.func.env.id=1;mione.faas.func.id=1 (agent) ++ serviceName=new_mibench_manager ++ nacosAddr=127.0.0.1:80 ++ mione.faas.web.port=8080 ++ 支持压测类型 + + http + + dubbo + + demo(用来测试代码) ++ 支持debug模式(可以获取到结果) diff --git a/ozburst-all/mimeter-engine/mimeter-agent/dependency-reduced-pom.xml b/ozburst-all/mimeter-engine/mimeter-agent/dependency-reduced-pom.xml new file mode 100644 index 000000000..f92edef2e --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent/dependency-reduced-pom.xml @@ -0,0 +1,97 @@ + + + + mimeter-engine + run.mone + 1.0-SNAPSHOT + + 4.0.0 + mimeter-agent + + + + maven-shade-plugin + 3.1.1 + + + package + + shade + + + + + + + + + + staging + + + src/main/resources/config/staging.properties + + + + staging + + + + online + + + src/main/resources/config/online.properties + + + + online + + + + + + junit + junit + 4.12 + test + + + hamcrest-core + org.hamcrest + + + + + javax.annotation + javax.annotation-api + 1.3.2 + provided + + + run.mone + docean-plugin-rpc + 1.4-SNAPSHOT + provided + + + rpc + run.mone + + + docean-plugin-config + run.mone + + + + + org.projectlombok + lombok + 1.18.24 + provided + + + + 20 + 20 + + diff --git a/ozburst-all/mimeter-engine/mimeter-agent/pom.xml b/ozburst-all/mimeter-engine/mimeter-agent/pom.xml new file mode 100644 index 000000000..3cb5eb665 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent/pom.xml @@ -0,0 +1,107 @@ + + + + mimeter-engine + run.mone + 1.0-SNAPSHOT + + 4.0.0 + + mimeter-agent + + + 20 + 20 + + + + + + + + junit + junit + 4.12 + test + + + + javax.annotation + javax.annotation-api + 1.3.2 + provided + + + + run.mone + mimeter-api + 1.0-SNAPSHOT + + + + run.mone + mimeter-agent-service + 1.0-SNAPSHOT + + + + + run.mone + docean-plugin-rpc + 1.4-SNAPSHOT + provided + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.1 + + + + + package + + shade + + + + + + + + + + staging + + staging + + + true + + + + src/main/resources/config/staging.properties + + + + + online + + online + + + + src/main/resources/config/online.properties + + + + + \ No newline at end of file diff --git a/ozburst-all/mimeter-engine/mimeter-agent/src/main/java/run/mone/mimeter/engine/agent/bo/Version.java b/ozburst-all/mimeter-engine/mimeter-agent/src/main/java/run/mone/mimeter/engine/agent/bo/Version.java new file mode 100644 index 000000000..fdd6664aa --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent/src/main/java/run/mone/mimeter/engine/agent/bo/Version.java @@ -0,0 +1,29 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo; + +/** + * @author goodjava@qq.com + * @date 2022/5/20 + */ +public class Version { + + @Override + public String toString() { + return "agent:0.0.2:2023-09-19"; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent/src/main/java/run/mone/mimeter/engine/agent/handler/AgentHandler.java b/ozburst-all/mimeter-engine/mimeter-agent/src/main/java/run/mone/mimeter/engine/agent/handler/AgentHandler.java new file mode 100644 index 000000000..5bfdf6ac4 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent/src/main/java/run/mone/mimeter/engine/agent/handler/AgentHandler.java @@ -0,0 +1,64 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.handler; + +import run.mone.mimeter.engine.agent.bo.Version; +import com.xiaomi.youpin.docean.anno.Controller; +import com.xiaomi.youpin.docean.anno.RequestMapping; +import run.mone.mimeter.engine.service.MetricsService; + +import javax.annotation.Resource; + +import java.security.Security; + +/** + * @author goodjava@qq.com + * @author dongzhenxing + * @date 2022/5/19 + */ +@Controller +public class AgentHandler { + + @Resource + private MetricsService metricsService; + + /** + * 控制发压机jvm缓存dns时间,减短缓存时间 + * + * 一般情况下我们不需要完全取消JVM的DNS缓存,只需要调小有效时间,经过一些测试发现一下结论: + * 1)在高并发时,不做DNS缓存时的CPU耗用比做了3s缓存的CPU耗用要高3/4倍,实时DNS请求相当耗用CPU; + * 2)3s和30s缓存有效时间对dns查询响应时间的影响差别不大,cpu内存占用都比较接近; + * 3)建议使用3秒缓存,兼顾运维和性能; + */ + public void init(){ + Security.setProperty("networkaddress.cache.ttl", "3"); + Security.setProperty("networkaddress.cache.negative.ttl", "0"); + } + + @RequestMapping(path = "/info") + public String info() { + return "agent:" + new Version() + " jvm:" + System.getProperty("java.version"); + } + + + @RequestMapping(path = "/metrics", method = "get") + public String metrics() { + return this.metricsService.metrics(); + } + + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent/src/main/java/run/mone/mimeter/engine/agent/processor/AgentProcessor.java b/ozburst-all/mimeter-engine/mimeter-agent/src/main/java/run/mone/mimeter/engine/agent/processor/AgentProcessor.java new file mode 100644 index 000000000..d944bce5e --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent/src/main/java/run/mone/mimeter/engine/agent/processor/AgentProcessor.java @@ -0,0 +1,119 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.processor; + +import com.google.gson.Gson; +import com.xiaomi.data.push.rpc.netty.NettyRequestProcessor; +import com.xiaomi.data.push.rpc.protocol.RemotingCommand; +import com.xiaomi.youpin.docean.anno.Component; +import common.Util; +import io.netty.channel.ChannelHandlerContext; +import run.mone.mimeter.engine.agent.bo.MibenchCmd; +import run.mone.mimeter.engine.agent.bo.data.AgentReq; +import run.mone.mimeter.engine.agent.bo.task.Context; +import run.mone.mimeter.engine.agent.bo.task.HostsFileResult; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskResult; +import run.mone.mimeter.engine.service.BenchEngineService; + +import javax.annotation.Resource; +import java.nio.charset.StandardCharsets; + +/** + * @author goodjava@qq.com + * @date 2022/5/11 + */ +@Component +public class AgentProcessor implements NettyRequestProcessor { + + @Resource + private BenchEngineService engineService; + + private static final Gson gson = Util.getGson(); + + @Override + public RemotingCommand processRequest(ChannelHandlerContext channelHandlerContext, RemotingCommand remotingCommand) throws Exception { + AgentReq req = remotingCommand.getReq(AgentReq.class); + Task task = req.getTask(); + //manager server 地址 + if (task != null){ + task.setAddr(req.getAddr()); + } + if (req.getCmd().equals(AgentReq.SUBMIT_TASK_CMD)) { + if (task.isDebug()){ + TaskResult tr = engineService.submitTask(new Context(), task); + RemotingCommand resp = RemotingCommand.createResponseCommand(MibenchCmd.TASK); + resp.setBody(gson.toJson(tr).getBytes(StandardCharsets.UTF_8)); + return resp; + } + engineService.submitTask(new Context(), task); + } + + //停止任务 + if (req.getCmd().equals(AgentReq.CANCEL_TASK_CMD)) { + engineService.cancelTask(task); + } + + //变更任务rps + if (req.getCmd().equals(AgentReq.CHANGE_TASK_QPS)) { + engineService.changeTaskQps(req.getChangeQpsReq()); + } + + //修改本地host文件 + if (req.getCmd().equals(AgentReq.EDIT_HOST_CMD)) { + engineService.editHostsFile(req.getAgentHostReqList()); + RemotingCommand resp = RemotingCommand.createResponseCommand(MibenchCmd.TASK); + TaskResult tr = new TaskResult(); + tr.setOk(true); + resp.setBody(gson.toJson(tr).getBytes(StandardCharsets.UTF_8)); + return resp; + } + + //删除本地某项host配置 + if (req.getCmd().equals(AgentReq.DEL_HOST_CMD)){ + engineService.delHostsFile(req.getAgentHostReqList()); + RemotingCommand resp = RemotingCommand.createResponseCommand(MibenchCmd.TASK); + TaskResult tr = new TaskResult(); + tr.setOk(true); + resp.setBody(gson.toJson(tr).getBytes(StandardCharsets.UTF_8)); + return resp; + } + + //加载本地某项host配置 + if (req.getCmd().equals(AgentReq.LOAD_HOST_CMD)){ + RemotingCommand resp = RemotingCommand.createResponseCommand(MibenchCmd.TASK); + HostsFileResult hostsFileResult = engineService.loadHostsReq(); + resp.setBody(gson.toJson(hostsFileResult).getBytes(StandardCharsets.UTF_8)); + return resp; + } + + + return RemotingCommand.createResponseCommand(MibenchCmd.TASK); + } + + @Override + public boolean rejectRequest() { + return false; + } + + @Override + public int cmdId() { + return MibenchCmd.TASK; + } + + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent/src/main/java/run/mone/mimeter/engine/agent/task/PingTask.java b/ozburst-all/mimeter-engine/mimeter-agent/src/main/java/run/mone/mimeter/engine/agent/task/PingTask.java new file mode 100644 index 000000000..086d4adcc --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent/src/main/java/run/mone/mimeter/engine/agent/task/PingTask.java @@ -0,0 +1,90 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.task; + +import com.google.gson.Gson; +import com.xiaomi.data.push.rpc.protocol.RemotingCommand; +import com.xiaomi.data.push.task.Task; +import com.xiaomi.youpin.docean.anno.Component; +import com.xiaomi.youpin.docean.common.StringUtils; +import common.NetUtils; +import common.Util; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import run.mone.mimeter.engine.agent.bo.MibenchCmd; +import run.mone.mimeter.engine.agent.bo.Version; +import run.mone.mimeter.engine.agent.bo.data.AgentInfoDTO; +import run.mone.mimeter.engine.agent.bo.data.AgentReq; +import run.mone.mimeter.engine.agent.bo.data.User; + +import java.nio.charset.StandardCharsets; + + +/** + * @author goodjava@qq.com + * @date 2022/5/20 + */ +@Component +public class PingTask extends Task { + + private static final Logger log = LoggerFactory.getLogger(PingTask.class); + + private final Gson gson = Util.getGson(); + + private final String name = System.currentTimeMillis() + ""; + + private static final String SERVER_NAME = "mimeter"; + + public void init() { + this.setRunnable(() -> { + String msg = "ping:" + new Version(); + log.info(msg); + AgentReq agentReq = new AgentReq(); + User user = new User(); + user.setName(name); + agentReq.setUser(user); + agentReq.setAgentInfoDTO(getApiInfoDto()); + RemotingCommand req = RemotingCommand.createRequestCommand(MibenchCmd.PING); + req.setBody(gson.toJson(agentReq).getBytes(StandardCharsets.UTF_8)); + String addr = this.getClient().getServerAddrs(); + if (StringUtils.isNotEmpty(addr)) { + this.getClient().sendMessage(addr, req, responseFuture -> log.info("pong")); + } + }); + //单位秒 + this.setDelay(5); + } + + private AgentInfoDTO getApiInfoDto() { + AgentInfoDTO agentInfoDTO = new AgentInfoDTO(); + agentInfoDTO.setIp(NetUtils.getLocalHost()); + + agentInfoDTO.setPort(0); + agentInfoDTO.setHostname(NetUtils.getHostName()); + //cpu mem信息从proms 同步 + agentInfoDTO.setMem(0); + agentInfoDTO.setUseMem(0); + agentInfoDTO.setCpu(0); + agentInfoDTO.setUseCpu(0); + agentInfoDTO.setServerName(SERVER_NAME); + String selectZone = System.getenv("SELECT_ZONE"); + log.info("[PingTask.getApiInfoDto], selectZone: {}", selectZone); + agentInfoDTO.setDesc(StringUtils.isEmpty(selectZone) ? "default" : selectZone); + return agentInfoDTO; + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent/src/main/resources/config/online.properties b/ozburst-all/mimeter-engine/mimeter-agent/src/main/resources/config/online.properties new file mode 100644 index 000000000..e69de29bb diff --git a/ozburst-all/mimeter-engine/mimeter-agent/src/main/resources/config/staging.properties b/ozburst-all/mimeter-engine/mimeter-agent/src/main/resources/config/staging.properties new file mode 100644 index 000000000..e69de29bb diff --git a/ozburst-all/mimeter-engine/mimeter-agent/src/test/java/run/mone/mimeter/engine/test/CountdownLatchTest.java b/ozburst-all/mimeter-engine/mimeter-agent/src/test/java/run/mone/mimeter/engine/test/CountdownLatchTest.java new file mode 100644 index 000000000..92b9e6aa3 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent/src/test/java/run/mone/mimeter/engine/test/CountdownLatchTest.java @@ -0,0 +1,33 @@ +package run.mone.mimeter.engine.test; + +import org.junit.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +public class CountdownLatchTest { + + + @Test + public void testLatch() throws InterruptedException { + CountDownLatch latch = new CountDownLatch(1); + System.out.println(latch.getCount()); + latch.await(2,TimeUnit.SECONDS); + System.out.println(latch.getCount()); + } + + @Test + public void testCountdownLatch() throws InterruptedException { + CountDownLatch latch = new CountDownLatch(1); + new Thread(()->{ + try { + TimeUnit.SECONDS.sleep(4); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + latch.countDown(); + }).start(); + latch.await(10, TimeUnit.SECONDS); + System.out.println("finish"); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent/src/test/java/run/mone/mimeter/engine/test/ThreadWaitAndNotify.java b/ozburst-all/mimeter-engine/mimeter-agent/src/test/java/run/mone/mimeter/engine/test/ThreadWaitAndNotify.java new file mode 100644 index 000000000..2c0386a52 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent/src/test/java/run/mone/mimeter/engine/test/ThreadWaitAndNotify.java @@ -0,0 +1,46 @@ +package run.mone.mimeter.engine.test; + +import org.junit.Test; + +import java.util.concurrent.TimeUnit; + +public class ThreadWaitAndNotify { + static final Object object=new Object(); + public static class ThreadWait extends Thread{ + @Override + public void run(){ + synchronized (object){ + System.out.println(System.currentTimeMillis()+" A开始运行"); + try { + object.wait(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + System.out.println(System.currentTimeMillis()+" 重新获取到监视器,继续执行run方法中代码块——A结束运行"); + } + } + } + public static class ThreadNotify extends Thread{ + @Override + public void run(){ + synchronized (object){ + System.out.println(System.currentTimeMillis()+"B开始运行"); + object.notify(); + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + System.out.println(System.currentTimeMillis()+"释放监视器——B结束运行"); + } + } + } + + @Test + public void test() throws InterruptedException { + new ThreadWait().start(); + new ThreadNotify().start(); + + TimeUnit.SECONDS.sleep(10); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-agent/src/test/java/run/mone/mimeter/engine/test/VirturalThreadTest.java b/ozburst-all/mimeter-engine/mimeter-agent/src/test/java/run/mone/mimeter/engine/test/VirturalThreadTest.java new file mode 100644 index 000000000..55aaafcf8 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-agent/src/test/java/run/mone/mimeter/engine/test/VirturalThreadTest.java @@ -0,0 +1,147 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.test; + +import org.junit.Test; + +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.LongAdder; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.IntStream; + +/** + * @author goodjava@qq.com + * @date 2022/5/19 + */ +public class VirturalThreadTest { + + static final Object obj = new Object(); + + @Test + public void test1() throws InterruptedException { + System.out.println("123"); + new Thread(() -> { + System.out.println("123"); + }).start(); + Thread.currentThread().join(); + } + + @Test + public void test3() { + Random random = new Random(); + for (int i = 0; i < 10; i++) { + System.out.println(random.nextInt(3)); + } + } + + @Test + public void test5() { + AtomicInteger counter = new AtomicInteger(10); + long before = System.currentTimeMillis(); +// + new Thread(() -> { + try { + synchronized (obj) { + TimeUnit.SECONDS.sleep(3000); + obj.notify(); + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + }).start(); + + System.out.println("123"); + synchronized (obj) { + try { + obj.wait(8000); + } catch (InterruptedException e) { + e.fillInStackTrace(); + } + } + + long now = System.currentTimeMillis(); + System.out.println("time:" + (now - before)); + System.out.println("456"); + } + + @Test + public void test4() { + System.out.println(isInt(1.10)); + } + + + @Test + public void test6(){ + Pattern EL_PATTERN = Pattern.compile("\\$\\{([^}]*)}"); + + String json = "{\"userInfo\":\"${username}\"}"; + Matcher m = EL_PATTERN.matcher(json); + while (m.find()) { + String expr = m.group(1); +// body = Util.Parser.parse$(expr, body, getDataValue(expr, dataMap, lineFlag)); + System.out.println(expr); + } + } + + public static boolean isInt(double num) { + return Math.abs(num - Math.round(num)) < Double.MIN_VALUE; + } + + @Test + public void test2() throws InterruptedException { + + LongAdder total = new LongAdder(); + + total.add(1000*1000); + + LongAdder counter = new LongAdder(); + + ExecutorService pool = Executors.newVirtualThreadPerTaskExecutor(); + IntStream.range(0, 1000).forEach(i -> { + for (int j = 0; j < 1000; j++) { + pool.submit(() -> { + counter.increment(); + try { + Thread.sleep(1000); + total.decrement(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + System.out.println("fiber-" + i); + }); + } + }); + + aWait(total); + System.out.println("!!!!"+counter.longValue()); + + } + + private void aWait(LongAdder counter) throws InterruptedException { + long before = System.currentTimeMillis(); + //阻塞主线程直到所有调用结束 + while (counter.intValue() != 0) { + TimeUnit.MILLISECONDS.sleep(500); + } + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/pom.xml b/ozburst-all/mimeter-engine/mimeter-api/pom.xml new file mode 100644 index 000000000..9b6df0338 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/pom.xml @@ -0,0 +1,42 @@ + + + + mimeter-engine + run.mone + 1.0-SNAPSHOT + + 4.0.0 + + mimeter-api + + + 20 + 20 + + + + + + run.mone + struct + 1.4-SNAPSHOT + + + + org.nutz + nutz + 1.r.68-youpin-SNAPSHOT + + + + com.xiaomi.mone + 1.0.0-SNAPSHOT + sautumn-api + + + + + + \ No newline at end of file diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/Const.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/Const.java new file mode 100644 index 000000000..64f23032a --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/Const.java @@ -0,0 +1,185 @@ +package common; + +public class Const { + + public static final boolean sendLog = true; + + public static final String HEAR_HEADER_KEY = "heracontext"; + + public static final String DUBBO_TRACE_HEADER_KEY = "traceparent"; + + public static final String MIMETER_UA_KEY = "mimeter"; + + public static final int SINGLE_API_DEBUG = 0; + public static final int SCENE_DEBUG = 1; + public static final int SINGLE_BENCH = 2; + + public static final String METRICS_NAME_TPS = "tps"; + public static final String METRICS_NAME_RPS = "rps"; + + public static final String METRICS_TYPE_SCENE = "scene"; + + + public static final String METRICS_NAME_TPS_API = "tps_api"; + + public static final String METRICS_NAME_RPS_API = "rps_api"; + + public static final String METRICS_NAME_RT_API = "rt_api"; + + public static final String TASK_CTX_RECORD_LOG = "recordLog"; + + public static final String TASK_CTX_LINE_FLAG = "lineFlag"; + + public static final String TASK_CTX_SCENE_QPS = "sceneQps"; + + public static final int PUSH_STAT_RATE = 10; + + public static final int API_TYPE_HTTP = 1; + public static final int API_TYPE_DUBBO = 3; + + public static final int SCENE_TYPE_HTTP = 0; + public static final int SCENE_TYPE_DUBBO = 1; + + public static final int HTTP_REQ_GET = 0; + public static final int HTTP_REQ_POST = 1; + + public static final int FEI_SHU_ALARM = 0; + + public static final int SMS_ALARM = 1; + public static final String HTTP_GET = "get"; + public static final String HTTP_POST = "post"; + + public static final String CONTENT_TYPE_APP_JSON = "application/json"; + public static final String CONTENT_TYPE_APP_FORM = "application/x-www-form-urlencoded"; + public static final String CONTENT_TYPE_APP_FORM2 = "x-www-form-urlencoded"; + + public static final String HERA_SCENE_TASK = "scene_task"; + public static final String HERA_SERIAL_LINK = "serial_link"; + public static final String HERA_API_ID = "api"; + + + public static final String SuccessRate = "SuccessRate"; + + public static final String P99ResponseTime = "P99ResponseTime"; + public static final String AvgResponseTime = "AvgResponseTime"; + public static final String RequestPerSecond = "RequestPerSecond"; + public static final String CpuUtilization = "CpuUtilization"; + public static final String MemoryUtilization = "MemoryUtilization"; + public static final String Load5Average = "Load5Average"; + public static final String Load5Max = "Load5Max"; + + public static final int STATUS_CODE = 1; + public static final int HEADER_CODE = 2; + public static final int OUTPUT_CODE = 3; + + public static final int BIGGER = 1; + public static final int BIGGER_AND_EQ = 2; + public static final int SMLLER = 3; + public static final int SMALLER_AND_EQ = 4; + public static final int EQ = 5; + public static final int CONTAIN = 6; + public static final int NOT_CONTAIN = 7; + public static final int NOT_EQ = 8; + + + /** + * http 错误状态码前缀 + */ + public static final String ERR_STATUS_CODE_PREFIX = "sCode_"; + + /** + * dubbo 错误状态 + */ + public static final String ERR_STATUS_DUBBO_PREFIX = "dubbo_call_fail"; + + /** + * 检查点 错误规则前缀 + */ + public static final String ERR_CHECKPOINT_PREFIX = "cpId_"; + + /** + * http 错误状态码前缀 + */ + public static final int ERR_STATUS_CODE_TYPE = 0; + + /** + * 检查点 错误规则前缀 + */ + public static final int ERR_CHECKPOINT_TYPE = 1; + + + /** + * dubbo调用错误类型 + */ + public static final int ERR_DUBBO_CALL_TYPE = 2; + + public static final int DUBBO_DATA_TOO_LONG_ERR_CODE = 200012; + + /** + * 记录每个接口的rt列表 + */ + public static final String RT_LIST = "rt_list"; + + + /** + * 记录每个接口的平均rt + */ + public static final String AVG_RT = "avg_rt"; + + /** + * 记录每个接口的最大rt + */ + public static final String MAX_RT = "max_rt"; + + /** + * 记录每个接口的平均 tps + */ + public static final String AVG_TPS = "avg_tps"; + + /** + * 记录每个接口的平均 rps + */ + public static final String AVG_RPS = "avg_rps"; + + /** + * 记录每个接口的最大tps + */ + public static final String MAX_TPS = "max_tps"; + + /** + * 记录每个接口的最大rps + */ + public static final String MAX_RPS = "max_rps"; + + /** + * 接口发压请求总次数(返回 用于rps) + */ + public static final String API_REQ_TOTAL_R = "api_req_total_r"; + + /** + * 接口请求总次数(返回 用于tps) + */ + public static final String API_REQ_TOTAL_T = "api_req_total_t"; + + /** + * 接口请求成功数 + */ + public static final String API_REQ_SUCC = "api_req_succ"; + + /** + * 接口请求失败数 + */ + public static final String API_REQ_FAIL = "api_req_fail"; + + /** + * 单台agent最大承受rps量 + */ + public static final int AGENT_MAX_RPS = 15000; + + public static final String DISABLE_URL_ENCODE = "disable-url-encode"; + + /** + * 单台机器单个配置项最多允许使用10000条数据,防止OOM + */ + public static final int AGENT_MAX_TRAFFIC_SIZE = 10000; +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/HttpClientV6.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/HttpClientV6.java new file mode 100644 index 000000000..cc6e51719 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/HttpClientV6.java @@ -0,0 +1,325 @@ +// +// Source code recreated from a .class file by IntelliJ IDEA +// (powered by FernFlower decompiler) +// + +package common; + +import com.google.common.collect.Lists; +import com.google.common.io.ByteStreams; + +import java.io.*; +import java.net.HttpURLConnection; +import java.net.InetAddress; +import java.net.URL; +import java.net.URLEncoder; +import java.security.SecureRandom; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.zip.GZIPInputStream; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.HttpsURLConnection; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLSession; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +import com.google.gson.Gson; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Slf4j +public class HttpClientV6 { +// private static final Logger log = LoggerFactory.getLogger(HttpClientV6.class); + + private final Gson gson = Util.getGson(); + + static TrustManager[] trustAllCerts = new TrustManager[]{new X509TrustManager() { + public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { + } + + public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { + } + + public X509Certificate[] getAcceptedIssuers() { + return null; + } + }}; + + public HttpClientV6() { + } + + public static HttpResult postRt(TpsRecord needRecordTps, String url, byte[] body, Map headers, int timeout) { + HttpURLConnection conn = null; + + HttpResult var14; + try { + conn = (HttpURLConnection) (new URL(url)).openConnection(); + conn.setRequestMethod("POST"); + conn.setConnectTimeout(timeout); + conn.setReadTimeout(timeout); + conn.setDoOutput(true); + conn.setDoInput(true); + if (null != headers) { + Iterator var7 = headers.entrySet().iterator(); + + while (var7.hasNext()) { + Map.Entry entry = (Map.Entry) var7.next(); + conn.addRequestProperty((String) entry.getKey(), (String) entry.getValue()); + } + } + + conn.addRequestProperty("Connection", "close"); + conn.getOutputStream().write(body); + + var14 = getResult(conn, false, null); + } catch (Exception var12) { + log.warn("http client v2 error:{}", var12.getCause()); + needRecordTps.setNeedRecordTps(false); + var14 = new HttpResult(500, var12.getMessage(), Collections.emptyMap()); + } finally { + if (null != conn) { + conn.disconnect(); + } + } + + return var14; + } + + public static HttpResult httpPost(TpsRecord needRecordTps, String url, Map headers, Map body, String encoding, int readTimeout) { + List list = Lists.newArrayList(); + if (null != headers) { + headers.entrySet().stream().forEach((it) -> { + list.add(it.getKey()); + list.add(it.getValue()); + }); + } + + return request(needRecordTps, url, list, body, encoding, "POST", readTimeout, false, null); + } + + public static HttpResult httpGet(TpsRecord needRecordTps, String url, List headers, Map paramValues, String encoding, int readTimeout) { + return request(needRecordTps, url, headers, paramValues, encoding, "GET", readTimeout, false, (File) null); + } + + public static HttpResult request(TpsRecord needRecordTps, String url, List headers, Map paramValues, String encoding, String method, int readTimeout, boolean download, File file) { + HttpURLConnection conn = null; + OutputStreamWriter wr = null; + HttpResult var12; + try { + String encodedContent = encodingParams(paramValues, encoding); + + if (method.equalsIgnoreCase(Const.HTTP_GET)){ + url = url + "?" + encodedContent; + } + if (url.startsWith("https")) { + HttpsURLConnection.setDefaultHostnameVerifier(new NullHostNameVerifier()); + SSLContext sc = SSLContext.getInstance("TLS"); + sc.init(null, trustAllCerts, new SecureRandom()); + HttpsURLConnection.setDefaultSSLSocketFactory(sc.getSocketFactory()); + } + conn = (HttpURLConnection) (new URL(url)).openConnection(); + conn.setConnectTimeout(readTimeout); + conn.setReadTimeout(readTimeout); + conn.setRequestMethod(method); + setHeaders(conn, headers, encoding); + conn.addRequestProperty("Connection", download ? "keep-alive" : "close"); + + // 向服务器发送请求 + if (!method.equalsIgnoreCase(Const.HTTP_GET)){ + conn.setDoOutput(true); + wr = new OutputStreamWriter(conn.getOutputStream()); + wr.write(encodedContent); + wr.flush(); + } + conn.connect(); + log.debug("Request from server: " + url); + var12 = getResult(conn, download, file); + return var12; + } catch (Exception var18) { + try { + if (conn != null) { + log.warn("failed to request " + conn.getURL() + " from " + InetAddress.getByName(conn.getURL().getHost()).getHostAddress()); + } + } catch (Exception var17) { + log.warn("failed to request:{}", var17.getMessage()); + } + + log.warn("failed to request:{}", var18.getMessage()); + needRecordTps.setNeedRecordTps(false); + var12 = new HttpResult(500, var18.getMessage(), Collections.emptyMap()); + } finally { + if (conn != null) { + conn.disconnect(); + } + if (wr != null){ + try { + wr.close(); + } catch (IOException e) { + log.error("failed to request,failed to close wr"); + } + } + } + return var12; + } + + private static HttpResult getResult(HttpURLConnection conn, boolean download, File file) throws IOException { + int respCode = conn.getResponseCode(); + Object inputStream; + if (200 != respCode && 304 != respCode && 204 != respCode && 201 != respCode) { + inputStream = conn.getErrorStream(); + } else { + inputStream = conn.getInputStream(); + } + + Map respHeaders = new HashMap(conn.getHeaderFields().size()); + Iterator var6 = conn.getHeaderFields().entrySet().iterator(); + + while (var6.hasNext()) { + Map.Entry> entry = (Map.Entry) var6.next(); + respHeaders.put(entry.getKey(), (String) ((List) entry.getValue()).get(0)); + } + + String encodingGzip = "gzip"; + if (encodingGzip.equals(respHeaders.get("Content-Encoding"))) { + inputStream = new GZIPInputStream((InputStream) inputStream); + } + + if (null != file) { + boolean res = getBytes((InputStream) inputStream, file); + return new HttpResult(respCode, res ? "57" : "", respHeaders); + } else { + HttpResult res; + if (inputStream != null) { + byte[] data = download ? getBytes((InputStream) inputStream) : ByteStreams.toByteArray((InputStream) inputStream); + res = new HttpResult(respCode, new String(data, getCharset(conn)), respHeaders); + res.data = data; + } else { + res = new HttpResult(respCode, String.valueOf(respCode), respHeaders); + res.data = String.valueOf(respCode).getBytes(); + } + return res; + } + } + + public static byte[] getBytes(InputStream is) throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + byte[] data = new byte[8192]; + + int nRead; + while ((nRead = is.read(data)) != -1) { + if (Thread.currentThread().isInterrupted()) { + log.warn("interrupted"); + return new byte[0]; + } + + buffer.write(data, 0, nRead); + } + + byte[] res = buffer.toByteArray(); + is.close(); + buffer.close(); + return res; + } + + public static boolean getBytes(InputStream is, File file) throws IOException { + FileOutputStream fos = new FileOutputStream(file); + byte[] data = new byte[8192]; + + try { + int len; + while ((len = is.read(data)) != -1) { + if (Thread.currentThread().isInterrupted()) { + log.warn("interrupted download:{}", file); + boolean var5 = false; + return var5; + } + + fos.write(data, 0, len); + } + + fos.flush(); + } finally { + is.close(); + fos.close(); + } + + return true; + } + + private static String getCharset(HttpURLConnection conn) { + String contentType = conn.getContentType(); + if (null != contentType && !contentType.equals("")) { + String[] values = contentType.split(";"); + if (values.length == 0) { + return "UTF-8"; + } else { + String charset = "UTF-8"; + String[] var4 = values; + int var5 = values.length; + + for (int var6 = 0; var6 < var5; ++var6) { + String value = var4[var6]; + value = value.trim(); + if (value.toLowerCase().startsWith("charset=")) { + charset = value.substring("charset=".length()); + } + } + + return charset; + } + } else { + return "UTF-8"; + } + } + + private static void setHeaders(HttpURLConnection conn, List headers, String encoding) { + if (null != headers) { + Iterator iter = headers.iterator(); + + while (iter.hasNext()) { + conn.addRequestProperty((String) iter.next(), (String) iter.next()); + } + } + + conn.addRequestProperty("Accept-Charset", encoding); + } + + private static String encodingParams(Map params, String encoding) throws UnsupportedEncodingException { + StringBuilder sb = new StringBuilder(); + if (null != params && !params.isEmpty()) { + params.put("encoding", encoding); + + for (Map.Entry stringStringEntry : params.entrySet()) { + if (null != stringStringEntry.getValue() && !stringStringEntry.getValue().equals("")) { + sb.append(stringStringEntry.getKey()).append("="); + if (!encoding.isEmpty()) { + sb.append(URLEncoder.encode(stringStringEntry.getValue(), encoding)); + } else { + sb.append(stringStringEntry.getValue()); + } + sb.append("&"); + } + } + + return sb.toString(); + } else { + return ""; + } + } + + public static class NullHostNameVerifier implements HostnameVerifier { + public NullHostNameVerifier() { + } + + public boolean verify(String arg0, SSLSession arg1) { + return true; + } + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/HttpResult.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/HttpResult.java new file mode 100644 index 000000000..21df46c38 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/HttpResult.java @@ -0,0 +1,25 @@ +package common; + +import java.util.Map; + +public class HttpResult { + public final int code; + public final String content; + public byte[] data; + private final Map respHeaders; + + public HttpResult(int code, String content, Map respHeaders) { + this.code = code; + this.content = content; + this.respHeaders = respHeaders; + } + + public String getHeader(String name) { + return this.respHeaders.get(name); + } + + public Map getHeaders() { + return this.respHeaders; + } + + } \ No newline at end of file diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/MapTypeAdapter.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/MapTypeAdapter.java new file mode 100644 index 000000000..e777d4bc5 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/MapTypeAdapter.java @@ -0,0 +1,112 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package common; + +import com.google.gson.Gson; +import com.google.gson.TypeAdapter; +import com.google.gson.TypeAdapterFactory; +import com.google.gson.internal.LinkedTreeMap; +import com.google.gson.internal.bind.ObjectTypeAdapter; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; +import com.google.gson.stream.JsonWriter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public final class MapTypeAdapter extends TypeAdapter { + public static final TypeAdapterFactory FACTORY = new TypeAdapterFactory() { + @SuppressWarnings("unchecked") + @Override + public TypeAdapter create(Gson gson, TypeToken type) { + if (type.getRawType() == Object.class) { + return (TypeAdapter) new MapTypeAdapter(gson); + } + return null; + } + }; + + private final Gson gson; + + private MapTypeAdapter(Gson gson) { + this.gson = gson; + } + + @Override + public Object read(JsonReader in) throws IOException { + JsonToken token = in.peek(); + //判断字符串的实际类型 + switch (token) { + case BEGIN_ARRAY: + List list = new ArrayList<>(); + in.beginArray(); + while (in.hasNext()) { + list.add(read(in)); + } + in.endArray(); + return list; + + case BEGIN_OBJECT: + Map map = new LinkedTreeMap<>(); + in.beginObject(); + while (in.hasNext()) { + map.put(in.nextName(), read(in)); + } + in.endObject(); + return map; + case STRING: + return in.nextString(); + case NUMBER: + String s = in.nextString(); + if (s.contains(".")) { + return Double.valueOf(s); + } else { + try { + return Integer.valueOf(s); + } catch (Exception e) { + return Long.valueOf(s); + } + } + case BOOLEAN: + return in.nextBoolean(); + case NULL: + in.nextNull(); + return null; + default: + throw new IllegalStateException(); + } + } + + @Override + public void write(JsonWriter out, Object value) throws IOException { + if (value == null) { + out.nullValue(); + return; + } + //noinspection unchecked + TypeAdapter typeAdapter = (TypeAdapter) gson.getAdapter(value.getClass()); + if (typeAdapter instanceof ObjectTypeAdapter) { + out.beginObject(); + out.endObject(); + return; + } + typeAdapter.write(out, value); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/NetUtils.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/NetUtils.java new file mode 100644 index 000000000..db5a41db8 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/NetUtils.java @@ -0,0 +1,248 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package common; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; + +import java.io.IOException; +import java.net.*; +import java.util.*; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * @author goodjava@qq.com + */ +@Slf4j +public class NetUtils { + + + private static volatile InetAddress LOCAL_ADDRESS = null; + + public static final String LOCALHOST = "127.0.0.1"; + public static final String ANYHOST = "0.0.0.0"; + public static final String DOCKERHOST = "172.17.0.1"; + private static final Pattern IP_PATTERN = Pattern.compile("\\d{1,3}(\\.\\d{1,3}){3,5}$"); + + private static final int RND_PORT_START = 30000; + + private static final int MAX_PORT = 65535; + + private static final Random RANDOM = new Random(System.currentTimeMillis()); + + private static final int RND_PORT_RANGE = 10000; + + + public static String getHostName() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + log.error("get host name error:{}", e.getMessage()); + return ""; + } + } + + + public static String getLocalHost() { + InetAddress address = getLocalAddress(); + return address == null ? LOCALHOST : address.getHostAddress(); + } + + public static InetAddress getLocalAddress() { + if (LOCAL_ADDRESS != null) { + return LOCAL_ADDRESS; + } + InetAddress localAddress = getLocalAddress0(); + LOCAL_ADDRESS = localAddress; + return localAddress; + } + + private static InetAddress getLocalAddress0() { + InetAddress localAddress = null; + try { + localAddress = InetAddress.getLocalHost(); + if (localAddress instanceof Inet6Address) { + Inet6Address address = (Inet6Address) localAddress; + if (isValidV6Address(address)) { + return normalizeV6Address(address); + } + } else if (isValidAddress(localAddress)) { + return localAddress; + } + } catch (Throwable e) { + log.warn(e.getMessage()); + } + try { + Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); + if (null == interfaces) { + return localAddress; + } + while (interfaces.hasMoreElements()) { + try { + NetworkInterface network = interfaces.nextElement(); + Enumeration addresses = network.getInetAddresses(); + while (addresses.hasMoreElements()) { + try { + InetAddress address = addresses.nextElement(); + if (address instanceof Inet6Address) { + Inet6Address v6Address = (Inet6Address) address; + if (isValidV6Address(v6Address)) { + return normalizeV6Address(v6Address); + } + } else if (isValidAddress(address)) { + return address; + } + } catch (Throwable e) { + log.warn(e.getMessage()); + } + } + } catch (Throwable e) { + log.warn(e.getMessage()); + } + } + } catch (Throwable e) { + log.warn(e.getMessage()); + } + return localAddress; + } + + static boolean isValidV6Address(Inet6Address address) { + boolean preferIpv6 = Boolean.getBoolean("java.net.preferIPv6Addresses"); + if (!preferIpv6) { + return false; + } + try { + return address.isReachable(100); + } catch (IOException e) { + // ignore + } + return false; + } + + static InetAddress normalizeV6Address(Inet6Address address) { + String addr = address.getHostAddress(); + int i = addr.lastIndexOf('%'); + if (i > 0) { + try { + return InetAddress.getByName(addr.substring(0, i) + '%' + address.getScopeId()); + } catch (UnknownHostException e) { + // ignore + log.debug("Unknown IPV6 address: ", e); + } + } + return address; + } + + + static boolean isValidAddress(InetAddress address) { + if (address == null || address.isLoopbackAddress()) { + return false; + } + String name = address.getHostAddress(); + return (name != null + && !ANYHOST.equals(name) + && !LOCALHOST.equals(name) + && !DOCKERHOST.equals(name) + && IP_PATTERN.matcher(name).matches()); + } + + + public static int getAvailablePort(int port) { + if (port <= 0) { + return getAvailablePort(); + } + for (int i = port; i < MAX_PORT; i++) { + ServerSocket ss = null; + try { + ss = new ServerSocket(i); + return i; + } catch (IOException e) { + // continue + } finally { + if (ss != null) { + try { + ss.close(); + } catch (IOException e) { + } + } + } + } + return port; + } + + public static int getAvailableUdpPort(int port) { + if (port <= 0) { + return getUdpAvailablePort(); + } + for (int i = port; i < MAX_PORT; i++) { + DatagramSocket ss = null; + try { + ss = new DatagramSocket(i); + return i; + } catch (IOException e) { + // continue + } finally { + if (ss != null) { + ss.close(); + } + } + } + return port; + } + + + public static int getAvailablePort() { + ServerSocket ss = null; + try { + ss = new ServerSocket(); + ss.bind(null); + return ss.getLocalPort(); + } catch (IOException e) { + return getRandomPort(); + } finally { + if (ss != null) { + try { + ss.close(); + } catch (IOException e) { + } + } + } + } + + public static int getUdpAvailablePort() { + DatagramSocket ss = null; + try { + ss = new DatagramSocket(); + ss.bind(null); + return ss.getLocalPort(); + } catch (IOException e) { + return getRandomPort(); + } finally { + if (ss != null) { + ss.close(); + } + } + } + + + public static int getRandomPort() { + return RND_PORT_START + RANDOM.nextInt(RND_PORT_RANGE); + } + + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/Replacer.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/Replacer.java new file mode 100644 index 000000000..f5ae36746 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/Replacer.java @@ -0,0 +1,40 @@ +/* + * Copyright 2020 XiaoMi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at the following link. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package common; + +import lombok.Data; + +import java.io.Serializable; + +/** + * Created by dongzhenxing on 2023/4/4 1:24 AM + */ +@Data +public class Replacer implements Serializable { + int taskId; + int paramIndex; + String paramName; + Object value; + boolean forceStr; + + public Replacer(int taskId, int paramIndex, String paramName, Object value, boolean forceStr) { + this.taskId = taskId; + this.paramIndex = paramIndex; + this.paramName = paramName; + this.value = value; + this.forceStr = forceStr; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/TpsRecord.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/TpsRecord.java new file mode 100644 index 000000000..9b107bed9 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/TpsRecord.java @@ -0,0 +1,12 @@ +package common; + +import lombok.Data; + +@Data +public class TpsRecord { + boolean needRecordTps; + + public TpsRecord(boolean needRecordTps) { + this.needRecordTps = needRecordTps; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/Util.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/Util.java new file mode 100644 index 000000000..bc2c094bb --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/common/Util.java @@ -0,0 +1,242 @@ +package common; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonArray; +import com.google.gson.TypeAdapterFactory; +import com.google.gson.internal.bind.ObjectTypeAdapter; +import com.xiaomi.youpin.docean.common.Pair; + +import java.lang.reflect.Field; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class Util { + + private static final Pattern EL_PATTERN = Pattern.compile("\\$\\{(.*)\\}"); + + public static Gson getGson() { + Gson gson = new GsonBuilder().serializeNulls().enableComplexMapKeySerialization().disableHtmlEscaping().create(); + try { + Field factories = Gson.class.getDeclaredField("factories"); + factories.setAccessible(true); + Object o = factories.get(gson); + Class[] declaredClasses = Collections.class.getDeclaredClasses(); + for (Class c : declaredClasses) { + if ("java.util.Collections$UnmodifiableList".equals(c.getName())) { + Field listField = c.getDeclaredField("list"); + listField.setAccessible(true); + List list = (List) listField.get(o); + int i = list.indexOf(ObjectTypeAdapter.FACTORY); + list.set(i, MapTypeAdapter.FACTORY); + break; + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return gson; + } + + public static class Parser { + + /** + * 将字符串text中由openToken和closeToken组成的占位符依次替换为args数组中的值 + */ + public static String parse(String openToken, String closeToken, String text, Object... args) { + if (args == null || args.length <= 0) { + return text; + } + int argsIndex = 0; + + if (text == null || text.isEmpty()) { + return ""; + } + char[] src = text.toCharArray(); + int offset = 0; + // search open token + int start = text.indexOf(openToken, offset); + if (start == -1) { + return text; + } + final StringBuilder builder = new StringBuilder(); + StringBuilder expression = null; + int replaceCount = 0; + while (start > -1) { + if (start > 0 && src[start - 1] == '\\') { + // this open token is escaped. remove the backslash and continue. + builder.append(src, offset, start - offset - 1).append(openToken); + offset = start + openToken.length(); + } else { + // found open token. let's search close token. + if (expression == null) { + expression = new StringBuilder(); + } else { + expression.setLength(0); + } + builder.append(src, offset, start - offset); + offset = start + openToken.length(); + int end = text.indexOf(closeToken, offset); + while (end > -1) { + if (end > offset && src[end - 1] == '\\') { + // this close token is escaped. remove the backslash and continue. + expression.append(src, offset, end - offset - 1).append(closeToken); + offset = end + closeToken.length(); + end = text.indexOf(closeToken, offset); + } else { + expression.append(src, offset, end - offset); + break; + } + } + if (end == -1) { + // close token was not found. + builder.append(src, start, src.length - start); + offset = src.length; + } else { + String value = (argsIndex <= args.length - 1) ? + (args[argsIndex] == null ? "" : args[argsIndex].toString()) : expression.toString(); + //数组数据源标识 ["${list}"] "LF1,2,3" + boolean listNum = false; + if (value.startsWith("LF")) { + if (value.charAt(2) != '"') { + //数组 纯数字 + value = value.substring(2); + listNum = true; + } else { + //数组 字符串 + value = value.substring(3, value.length() - 1); + } + } + if (value.startsWith("s^")) { + value = value.substring(2); + } + assert args[0] != null; + if (args[0].getClass().getName().startsWith("java.lang.Long") || args[0].getClass().getName().startsWith("java.lang.Double") || listNum) { + builder.deleteCharAt(builder.length() - 1); + offset = end + closeToken.length() + 1; + } else { + offset = end + closeToken.length(); + } + builder.append(value); + argsIndex++; + } + } + start = text.indexOf(openToken, offset); + replaceCount++; + if (replaceCount >= args.length) { + break; + } + } + if (offset < src.length) { + builder.append(src, offset, src.length - offset); + } + return builder.toString(); + } + + public static String parse$(String key, String text, Object... args) { + return Parser.parse("${" + key, "}", text, args); + } + + + public static String parse1(String text, Object... args) { + return Parser.parse("{", "}", text, args); + } + } + + + public static Pair getElKey(String key) { + String k = null; + String v = null; + if (null != key && key.length() > 0) { + Matcher m = EL_PATTERN.matcher(key); + + if (m.find()) { + k = m.group(1); + v = ""; + } + } + return Pair.of(k, v); + } + + /** + * 按比率判断 + * + * @param rate 0~100 + * @return + */ + public static boolean judgeByRate(int rate) { + Random random = new Random(); + return random.nextInt(100) <= rate; + } + + /** + * 按比率判断 + * + * @param rate 0~1000 + * @return + */ + public static boolean judgeByRateLog(int rate) { + Random random = new Random(); + return random.nextInt(1000) <= rate; + } + + public static int sampleRateToCnt(int n, int rate) { + return (int) Math.ceil(n * rate / 1000.0d); + } + + // fix inaccurate log rate due to low qps, so use (qps * Math.min(10, duration)) as base + public static List sampleByTimeAndQps(int duration, int qps, int rate) { + int base = Math.min(10, duration) * qps; + return sampleIndices(base, sampleRateToCnt(base, rate)); + } + + /** + * 从[0, n)随机抽取出k个数, k <= n + */ + public static List sampleIndices(int n, int k) { + List output = new ArrayList<>(); + + // will use judgeByRate() as backup + if (k <= 0 || n < k) { + return output; + } + int i, j; + + for (i = 0; i < k; i++) { + output.add(i); + } + Random rand = new Random(); + + for (j = i; j < n; j++) { + int index = rand.nextInt(j + 1); + + if (index < k) { + output.set(index, j); + } + } + return output; + } + + public static Object getListValByType(JsonArray valList, int index, String type) { + Object value; + switch (type) { + case "string" -> value = valList.get(index).getAsString(); + case "int" -> value = valList.get(index).getAsInt(); + case "double" -> value = valList.get(index).getAsDouble(); + default -> value = valList.get(index); + } + return value; + } + + public static int calculateAgentRps(int totalRps, int agentNum) { + int agentQps = totalRps / agentNum; + if (agentQps < 1) { + agentQps = 1; + } else if (agentQps > Const.AGENT_MAX_RPS) { + //单台最多给15000 rps + agentQps = Const.AGENT_MAX_RPS; + } + return agentQps; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/MibenchCmd.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/MibenchCmd.java new file mode 100644 index 000000000..3afb1fce2 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/MibenchCmd.java @@ -0,0 +1,35 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo; + +/** + * @author goodjava@qq.com + * @date 2022/5/22 + */ +public class MibenchCmd { + + + public static final int PING = 12345; + + public static final int TASK = 12445; + + + public static final int TASK_RESULT = 12446; + + public static final int MANAGER = 12545; + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/AgentHostReq.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/AgentHostReq.java new file mode 100644 index 000000000..0be2f64cb --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/AgentHostReq.java @@ -0,0 +1,12 @@ +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class AgentHostReq implements Serializable { + String domain; + + String ip; +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/AgentInfoDTO.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/AgentInfoDTO.java new file mode 100644 index 000000000..b03239a58 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/AgentInfoDTO.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.io.Serializable; + +/** + * @author goodjava@qq.com + * @date 2022/5/19 + */ +@Data +public class AgentInfoDTO implements Serializable { + + private String id; + + private String ip; + + private int port; + + private String hostname; + + private int cpu; + + private long mem; + + private int useCpu; + + private long useMem; + + private String serverName; + + private String desc; + + private Long utime; + + private int status; + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/AgentReq.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/AgentReq.java new file mode 100644 index 000000000..5c5623754 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/AgentReq.java @@ -0,0 +1,111 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; +import run.mone.mimeter.engine.agent.bo.stat.SceneTotalCountContextDTO; +import run.mone.mimeter.engine.agent.bo.task.ChangeQpsReq; +import run.mone.mimeter.engine.agent.bo.task.Task; +import run.mone.mimeter.engine.agent.bo.task.TaskResult; +import run.mone.mimeter.engine.agent.bo.task.TaskStatusBo; + +import java.io.Serializable; +import java.util.List; + +/** + * @author goodjava@qq.com + * @date 2022/5/19 + */ +@Data +public class AgentReq implements Serializable { + + private User user; + + private String addr; + + private AgentInfoDTO agentInfoDTO; + + private List agentHostReqList; + + private String cmd; + + /** + * 修改压测 qps + */ + private ChangeQpsReq changeQpsReq; + + /** + * 任务信息 + */ + private Task task; + + /** + * 任务结果 + */ + private TaskResult taskResult; + + /** + * 打点统计 + */ + private SceneTotalCountContextDTO totalCountContextDTO; + + /** + * 状态记录 + */ + private TaskStatusBo statusBo; + + public static final String TASK_RESULT_CMD = "taskResult"; + + + public static final String TOTAL_DATA_COUNT_CMD = "totalCountContext"; + + /** + * 提交任务 + */ + public static final String SUBMIT_TASK_CMD = "submitTask"; + + /** + * 提交修改host命令 + */ + public static final String EDIT_HOST_CMD = "editHost"; + + /** + * 提交加载host文件命令 + */ + public static final String LOAD_HOST_CMD = "loadHost"; + + /** + * 提交删除host配置命令 + */ + public static final String DEL_HOST_CMD = "delHost"; + + /** + * 取消任务 + */ + public static final String CANCEL_TASK_CMD = "cancelTask"; + + /** + * 变更任务qps + */ + public static final String CHANGE_TASK_QPS = "upQpsTask"; + /** + * 更新任务状态 + */ + public static final String UP_TASK_STATUS = "updateTaskStatus"; + + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/ApiX5InfoDTO.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/ApiX5InfoDTO.java new file mode 100644 index 000000000..c73591e1e --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/ApiX5InfoDTO.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020 XiaoMi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at the following link. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.io.Serializable; + +/** + * Created by dongzhenxing on 2023/3/6 11:22 AM + */ +@Data +public class ApiX5InfoDTO implements Serializable { + + private boolean enableX5; + + private String appId; + + private String appKey; + + private String x5Method; + + private String x5Version; +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/Attachment.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/Attachment.java new file mode 100644 index 000000000..f59ff1d77 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/Attachment.java @@ -0,0 +1,29 @@ +/* + * Copyright 2020 XiaoMi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at the following link. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.io.Serializable; + +/** + * Created by dongzhenxing on 2023/6/21 4:36 PM + */ +@Data +public class Attachment implements Serializable { + private String paramKey; + private String paramValue; +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/BaseData.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/BaseData.java new file mode 100644 index 000000000..8574cdace --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/BaseData.java @@ -0,0 +1,145 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.data; + +import com.google.gson.Gson; +import common.Replacer; +import common.Util; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.math.NumberUtils; + +import java.io.Serializable; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicReference; + +/** + * @author goodjava@qq.com + * @date 2022/6/2 + */ +@Data +@Slf4j +public class BaseData implements Serializable { + + private int paramNum; + + private List types = new CopyOnWriteArrayList<>(); + + protected List params = new CopyOnWriteArrayList<>(); + + /** + * 源json参数体 + */ + private String originJsonParam = ""; + + private AtomicReference jsonParam = new AtomicReference<>(""); + + + private CopyOnWriteArrayList outputParams; + + /** + * 检查点 + */ + private CopyOnWriteArrayList checkPointInfoList; + + /** + * 过滤条件 + */ + private CopyOnWriteArrayList filterCondition; + + private static final Gson gson = Util.getGson(); + + public BaseData() { + } + + public void initTypeList(List list) { + this.types.addAll(list); + this.paramNum = this.types.size(); + + list.forEach(it -> { + //若接口入参是对象 + if (it.getTypeEnum().equals(ParamTypeEnum.pojo)) { + params.add(new ConcurrentHashMap<>()); + } + }); + } + + /** + * 用于上游节点接口动态更新本接口实际参数 + * + * @ paramIndex 参数索引位置 + * @ paramName 参数名 若paramName 为例:${name} 类型,则直接替换 + * @ value 上游传下来的实际参数值 + */ + public void updateParam(List replacerList) { + AtomicReference body = new AtomicReference<>(getOriginJsonParam()); + + replacerList.forEach(replacer -> { + if (replacer.getParamName().startsWith("${")) { + try { + //post请求 的 json参数 + String valStr = replacer.getValue().toString(); + + if (!replacer.isForceStr()) { + if (NumberUtils.isNumber(valStr)) { + if (isNumeric(valStr)) { + replacer.setValue(Long.parseLong(valStr)); + } else { + replacer.setValue(Double.parseDouble(valStr)); + } + } else { + replacer.setValue(valStr); + } + } +// log.debug("updateParam param,name:{},body:{},value:{}", paramName, body, value); + + body.set(Util.Parser.parse$(Util.getElKey(replacer.getParamName()).getKey(), body.get(), replacer.getValue())); + } catch (Exception e) { + log.error("updateParam param after,body:{},time:{},error:{}", body.get(), System.currentTimeMillis(), e); + } + log.debug("updateParam param after,body:{},time:{}", body, System.currentTimeMillis()); + } else { + ParamType type = types.get(replacer.getParamIndex()); + //对象类型 + if (type.getTypeEnum().equals(ParamTypeEnum.pojo)) { + ConcurrentHashMap params = (ConcurrentHashMap) this.params.get(replacer.getParamIndex()); + params.put(replacer.getParamName(), replacer.getValue()); + } + //基本类型 + if (type.getTypeEnum().equals(ParamTypeEnum.primary)) { + params.set(replacer.getParamIndex(), replacer.getValue()); + } + } + }); + getJsonParam().set(body.get()); + } + +// public synchronized void recoverJsonParam() { +// setJsonParam(getOriginJsonParam()); +// } + + public static boolean isNumeric(String str) { + for (int i = str.length(); --i >= 0; ) { + int chr = str.charAt(i); + if (chr < 48 || chr > 57) + return false; + } + return true; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/CheckFilterConditionRes.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/CheckFilterConditionRes.java new file mode 100644 index 000000000..f6c7a0d1f --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/CheckFilterConditionRes.java @@ -0,0 +1,11 @@ +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class CheckFilterConditionRes implements Serializable { + private boolean match; + private String triggerFilterCondition; +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/CheckPointInfo.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/CheckPointInfo.java new file mode 100644 index 000000000..f908e03a4 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/CheckPointInfo.java @@ -0,0 +1,13 @@ +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; +import java.io.Serializable; + +@Data +public class CheckPointInfo implements Serializable { + private Integer id; + private Integer checkType; + private String checkObj; + private Integer checkCondition; + private String checkContent; +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/CommonReqInfo.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/CommonReqInfo.java new file mode 100644 index 000000000..76ec44615 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/CommonReqInfo.java @@ -0,0 +1,40 @@ +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +@Data +public class CommonReqInfo implements Serializable { + + /** + * 1:http get + * 2:http post form + * 3:http post json + * 4:dubbo + */ + private int paramsType; + + private String debugUrl; + + private String paramJson; + +// private List getOrFormParamsList; + + private Map queryParamMap; + + private Map headers; + + @Override + public String toString() { + return "CommonReqInfo{" + + "paramsType=" + paramsType + + ", debugUrl='" + debugUrl + '\'' + + ", paramJson='" + paramJson + '\'' + + ", queryParamMap=" + queryParamMap + + ", headers=" + headers + + '}'; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/DebugSceneApiInfoReq.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/DebugSceneApiInfoReq.java new file mode 100644 index 000000000..0f3028350 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/DebugSceneApiInfoReq.java @@ -0,0 +1,52 @@ +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.io.Serializable; + +/** + * @author dongzhenxing + * 用于单独调试的接口信息 + */ +@Data +public class DebugSceneApiInfoReq implements Serializable { + + private Integer apiType; + + private String apiUrl; + + private Integer requestMethod; + + private Integer requestTimeout; + + private String contentType; + + private String serviceName; + + private String methodName; + + private String attachments; + + private String paramTypeList; + + private String dubboGroup; + + private String dubboVersion; + + private String dubboMavenVersion; + + private String apiHeader; + + private TspAuthInfoDTO apiTspAuth; + + private String requestParamInfo; + + private String requestBody; + + private String dubboParamJson; + + private String checkPointInfoListStr; + + private String outputParamInfosStr; + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/DemoData.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/DemoData.java new file mode 100644 index 000000000..e1371dfa9 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/DemoData.java @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.io.Serializable; + +/** + * @author goodjava@qq.com + * @date 2022/6/2 + */ +@Data +public class DemoData extends BaseData implements Serializable { + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/DubboData.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/DubboData.java new file mode 100644 index 000000000..0f93d5336 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/DubboData.java @@ -0,0 +1,56 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author goodjava@qq.com + * @date 2022/5/23 + */ +@Data +public class DubboData extends BaseData implements Serializable,Cloneable { + + private String serviceName; + + private String methodName; + + private String group; + + private String version; + + private String mavenVersion; + + private String dubboEnv; + + private List requestParamTypeList; + + private volatile String requestBody; + + private Integer requestTimeout; + + private ConcurrentHashMap attachments = new ConcurrentHashMap<>(); + + @Override + public DubboData clone() throws CloneNotSupportedException { + return (DubboData) super.clone(); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/ExprKey.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/ExprKey.java new file mode 100644 index 000000000..58df33003 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/ExprKey.java @@ -0,0 +1,66 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +/** + * @author goodjava@qq.com + * @date 2022/6/2 + */ +@Data +public class ExprKey implements Serializable { + + /** + * 从那个顶点取值 + */ + private int index; + + /** + * 类型 1 txt 2 json + */ + private int origin; + + /** + * 取完值的名字 + */ + private String name; + + /** + * 取值表达式 + */ + private String expr; + + /** + * 放入task data中的表达式规则 demoData->0->i + */ + private List putValueExpr; + + public ExprKey() { + } + + public ExprKey(int index, int origin, String name, String expr, List putValueExpr) { + this.index = index; + this.origin = origin; + this.name = name; + this.expr = expr; + this.putValueExpr = putValueExpr; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/HeaderInfo.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/HeaderInfo.java new file mode 100644 index 000000000..e8bb050fe --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/HeaderInfo.java @@ -0,0 +1,11 @@ +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class HeaderInfo implements Serializable { + private String headerName; + private String headerValue; +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/HttpData.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/HttpData.java new file mode 100644 index 000000000..9810855d2 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/HttpData.java @@ -0,0 +1,71 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.IntStream; + +/** + * @author goodjava@qq.com + * @date 2022/5/19 + */ +@Data +public class HttpData extends BaseData implements Serializable,Cloneable{ + + private String url; + + private String method; + + private String contentType; + + private int timeout; + + private ConcurrentHashMap headers = new ConcurrentHashMap<>(); + + private TspAuthInfoDTO tspAuthInfoDTO; + + private ApiX5InfoDTO apiX5InfoDTO; + + private volatile String postParamJson; + + private boolean enableTraffic; + + private int trafficConfId; + + public Map httpGetParams() { + Map map = new HashMap<>(); + IntStream.range(0, this.getParamNum()).forEach(i -> map.put(this.getTypes().get(i).getName(), this.params.get(i).toString())); + return map; + } + + public Map httpGetTmpParams(List params) { + Map map = new HashMap<>(); + IntStream.range(0, this.getParamNum()).forEach(i -> map.put(this.getTypes().get(i).getName(), params.get(i).toString())); + return map; + } + + @Override + public HttpData clone() throws CloneNotSupportedException { + return (HttpData) super.clone(); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/HttpResult.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/HttpResult.java new file mode 100644 index 000000000..dcb0161b8 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/HttpResult.java @@ -0,0 +1,56 @@ +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.util.Objects; + +@Data +public class HttpResult { + private int code; + + private String data; + + private String message; + + private String method; + + public HttpResult(int code, String data, String message) { + this.code = code; + this.data = data; + this.message = message; + } + + public static HttpResult fail(int errorCode, String message,String data) { + return new HttpResult(errorCode, message,data); + } + + public static HttpResult success(String data) { + return new HttpResult(200, data,"ok"); + } + + @Override + public String toString() { + return "HttpResult{" + + "code=" + code + + ", data='" + data + '\'' + + ", message='" + message + '\'' + + ", method='" + method + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + HttpResult that = (HttpResult) o; + return code == that.code && + Objects.equals(data, that.data) && + Objects.equals(message, that.message) && + Objects.equals(method, that.method); + } + + @Override + public int hashCode() { + return Objects.hash(code, data, message, method); + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/NodeInfo.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/NodeInfo.java new file mode 100644 index 000000000..880ea21be --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/NodeInfo.java @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; +import run.mone.mimeter.engine.agent.bo.task.Task; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + + +/** + * @author goodjava@qq.com + */ +@Data +public class NodeInfo implements Serializable { + + private Integer id; + + private Task task; + + private String resultName; + + private Map exprMap = new HashMap<>(); + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/OutputParam.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/OutputParam.java new file mode 100644 index 000000000..1c02d7001 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/OutputParam.java @@ -0,0 +1,19 @@ +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class OutputParam implements Serializable { + + private int origin; + private String paramName; + private String parseExpr; + + public OutputParam(int origin,String paramName, String parseExpr) { + this.origin = origin; + this.paramName = paramName; + this.parseExpr = parseExpr; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/ParamType.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/ParamType.java new file mode 100644 index 000000000..57db90b8d --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/ParamType.java @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +/** + * @author goodjava@qq.com + * @date 2022/6/5 + */ +@Data +public class ParamType { + + /** + * 入参类型 + * 对象为 pojo + * 基本类型为 primary + */ + private ParamTypeEnum typeEnum; + + /** + * 入参名 + */ + private String name; + + public ParamType(ParamTypeEnum typeEnum, String name) { + this.typeEnum = typeEnum; + this.name = name; + } + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/ParamTypeEnum.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/ParamTypeEnum.java new file mode 100644 index 000000000..4cb15a4d8 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/ParamTypeEnum.java @@ -0,0 +1,11 @@ +package run.mone.mimeter.engine.agent.bo.data; + +/** + * @author goodjava@qq.com + * @date 2022/6/5 + */ +public enum ParamTypeEnum { + pojo, + primary, + list, +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/ReqParamType.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/ReqParamType.java new file mode 100644 index 000000000..f764664e7 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/ReqParamType.java @@ -0,0 +1,31 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.data; + +public enum ReqParamType { + + Http_Get(1), + Http_Post_Form(2), + Http_Post_Json(3), + Dubbo(4); + + public final int code; + + private ReqParamType(int code) { + this.code = code; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/Result.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/Result.java new file mode 100644 index 000000000..25648ddbd --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/Result.java @@ -0,0 +1,83 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.io.Serializable; +import java.util.Map; + +/** + * @author goodjava@qq.com + * @date 2022/5/19 + */ +@Data +public class Result implements Serializable { + + private int code; + + private String message; + + private D data; + + private boolean ok; + + private CommonReqInfo commonReqInfo; + + private Map respHeaders; + + private String triggerCp; + + /** + * 触发的检查点条件 + */ + private String triggerFilterCondition; + + private long rt; + + //bytes + private long size; + + public Result() { + } + + public Result(int code, String message, D data) { + this.code = code; + this.message = message; + this.data = data; + } + + public static Result cancel() { + return new Result<>(501, "cancel", "cancel"); + } + + public static Result fail(int errorCode, String message,D data) { + return new Result(errorCode, message,data); + } + + public static Result success(D data) { + return new Result(200, "ok",data); + } + + public boolean isSuccess() { + return checkSuccess(this.code); + } + + public static boolean checkSuccess(final int code) { + return code == 0 || code == 200; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/TspAuthInfoDTO.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/TspAuthInfoDTO.java new file mode 100644 index 000000000..4d3c63c57 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/TspAuthInfoDTO.java @@ -0,0 +1,16 @@ +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +@Data +public class TspAuthInfoDTO { + private boolean enableAuth; + private String accessKey; + private String secretKey; + + public TspAuthInfoDTO(boolean enableAuth, String accessKey, String secretKey) { + this.enableAuth = enableAuth; + this.accessKey = accessKey; + this.secretKey = secretKey; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/User.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/User.java new file mode 100644 index 000000000..8b9acd59d --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/data/User.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.data; + +import lombok.Data; + +import java.io.Serializable; +import java.util.Map; + +/** + * @author goodjava@qq.com + * @date 2022/5/19 + * + * 用户信息 + */ +@Data +public class User implements Serializable { + + private String name; + + private String type; + + private Map attachments; + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/hosts/HostBo.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/hosts/HostBo.java new file mode 100644 index 000000000..8feaec1e3 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/hosts/HostBo.java @@ -0,0 +1,18 @@ +package run.mone.mimeter.engine.agent.bo.hosts; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class HostBo implements Serializable { + /** + * 绑定的ip + */ + private String ip; + + /** + * 绑定的域名 + */ + private String domain; +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/DubboResultCheckInfo.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/DubboResultCheckInfo.java new file mode 100644 index 000000000..b3c09c784 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/DubboResultCheckInfo.java @@ -0,0 +1,10 @@ +package run.mone.mimeter.engine.agent.bo.stat; + +import lombok.Data; + +@Data +public class DubboResultCheckInfo extends ResultCheckInfo { + + private String triggerCpInfo; + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/HttpResultCheckInfo.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/HttpResultCheckInfo.java new file mode 100644 index 000000000..afb5b9fa4 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/HttpResultCheckInfo.java @@ -0,0 +1,14 @@ +package run.mone.mimeter.engine.agent.bo.stat; + +import lombok.Data; + +@Data +public class HttpResultCheckInfo extends ResultCheckInfo { + /** + * http状态码 + */ + private String httpStatusCode; + + private String triggerCpInfo; + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/MetricLabelEnum.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/MetricLabelEnum.java new file mode 100644 index 000000000..fffb55241 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/MetricLabelEnum.java @@ -0,0 +1,74 @@ +package run.mone.mimeter.engine.agent.bo.stat; + +import java.util.Arrays; + +import static com.google.common.base.Preconditions.checkArgument; + +/** + * @author Xirui Yang (yangxirui@xiaomi.com) + * @version 1.0 + * @since 2022/8/26 + * + * 注意:只许往后加枚举值 + */ +public enum MetricLabelEnum { + + SCENE_ID(0, "sceneid"), + REPORT_ID(1, "reportid"), + SERIAL_ID(2, "serialid"), + TYPE(3, "type"), + CODE(4, "code"), + URI(5, "uri"), + METHOD(6, "method"), + API_ID(7, "apiid"), + SUCCESS(8, "success"); + + private final int index; + + private final String name; + + private static final String[] labelNames = new String[MetricLabelEnum.values().length]; + + private static final String[] labelApiRpsNames = new String[8]; + + static { + int i = 0; + for (MetricLabelEnum label : MetricLabelEnum.values()) { + labelNames[i++] = label.getName(); + } + labelApiRpsNames[0] = SCENE_ID.name; + labelApiRpsNames[1] = REPORT_ID.name; + labelApiRpsNames[2] = SERIAL_ID.name; + labelApiRpsNames[3] = TYPE.name; + labelApiRpsNames[4] = URI.name; + labelApiRpsNames[5] = METHOD.name; + labelApiRpsNames[6] = API_ID.name; + labelApiRpsNames[7] = SUCCESS.name; + } + + public static String[] getLabelNames() { + return labelNames; + } + + public static String[] getLabelApiRpsNames() { + return labelApiRpsNames; + } + + public static String[] labelNameSlice(int len) { + checkArgument(len > 0 && len <= labelNames.length, "MetricLabelEnum labelNames invalid len " + len); + return Arrays.asList(labelNames).subList(0, len).toArray(new String[]{}); + } + + MetricLabelEnum(int index, String name) { + this.index = index; + this.name = name; + } + + public int getIndex() { + return index; + } + + public String getName() { + return name; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/ResultCheckInfo.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/ResultCheckInfo.java new file mode 100644 index 000000000..3fe9c80c8 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/ResultCheckInfo.java @@ -0,0 +1,17 @@ +package run.mone.mimeter.engine.agent.bo.stat; + +import lombok.Data; + +@Data +public class ResultCheckInfo { + /** + * 是否成功 + */ + private boolean ok; + + /** + * 对应的检查点id + */ + private Integer checkPointId; + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/SceneErrorCountContext.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/SceneErrorCountContext.java new file mode 100644 index 000000000..006669747 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/SceneErrorCountContext.java @@ -0,0 +1,54 @@ +package run.mone.mimeter.engine.agent.bo.stat; + + +import lombok.Data; + +import java.io.Serializable; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.LongAdder; + +@Data +public class SceneErrorCountContext implements Serializable { + private String reportId; + private Integer sceneId; + + private Integer taskId; + + private Integer agentNum; + + /** + * 关联任务数 + */ + private Integer connectTaskNum; + + private boolean lastTime; + + private LongAdder tmpCount; + /** + * 全场景请求次数 + */ + private LongAdder totalReq; + + /** + * 全场景错误次数 + */ + private LongAdder totalErrReq; + + /** + * <"s_code_errorCode",> or <"cp_id_checkpointId",> + * <"s_code_404",<2342,10>> or <"cp_id_141242",<2342,10>> + */ + private ConcurrentMap> counterMap; + + public SceneErrorCountContext(String reportId, Integer sceneId, Integer taskId, Integer agentNum, Integer connectTaskNum, boolean lastTime, LongAdder totalReq, LongAdder totalErrReq, ConcurrentMap> counterMap) { + this.reportId = reportId; + this.sceneId = sceneId; + this.taskId = taskId; + this.agentNum = agentNum; + this.connectTaskNum = connectTaskNum; + this.lastTime = lastTime; + this.totalReq = totalReq; + this.totalErrReq = totalErrReq; + this.counterMap = counterMap; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/SceneTotalCountContext.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/SceneTotalCountContext.java new file mode 100644 index 000000000..0778b3ab4 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/SceneTotalCountContext.java @@ -0,0 +1,117 @@ +package run.mone.mimeter.engine.agent.bo.stat; + + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import run.mone.mimeter.engine.agent.bo.task.DagTaskRps; + +import java.io.Serializable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.LongAdder; + +@Data +@AllArgsConstructor +@Builder +public class SceneTotalCountContext implements Serializable { + private String reportId; + private Integer sceneId; + + /** + * 场景类型,用于调用获取p99 p95 + */ + private String sceneType; + + private Integer taskId; + + private Integer agentNum; + + /** + * linkId -> dag task id and rps + */ + DagTaskRps dagTaskRps; + + /** + * 可控的场景发压比例 + */ + private Integer rpsRate; + + /** + * 关联任务数 + */ + private Integer connectTaskNum; + + private boolean lastTime; + + /** + * 全场景发压请求次数 + */ + private LongAdder totalReq; + + /** + * 丢失连接数 + */ + private LongAdder lossConnNum; + + /** + * 全场景业务处理次数 + */ + private LongAdder totalTCount; + + /** + * 全场景成功次数 + */ + private LongAdder totalSuccReq; + + /** + * 全场景错误次数 + */ + private LongAdder totalErrReq; + + + /** + * 用于临时proms打点链路tps统计,每秒清零,消耗较大 + */ + private LongAdder tmpTpsCounter; + + /** + * 用于临时proms打点链路tps统计,每秒清零,消耗较大 + */ + private LongAdder tmpRpsCounter; + + /** + * 记录接口错误统计数据 + * <"s_code_errorCode",> or <"cp_id_checkpointId",> + * <"s_code_404",<2342,10>> or <"cp_id_141242",<2342,10>> + */ + private ConcurrentMap> errCounterMap; + + /** + * 记录接口Rt数据 + *

+ * 例:> 每次调用统计 + */ + private ConcurrentHashMap>> apiRtMap; + + /** + * 记录接口请求次数,成功失败数 tps + * > + */ + private ConcurrentHashMap> apiCountMap; + + /** + * 记录接口平均rps + */ + private ConcurrentHashMap apiRpsMap; + + /** + * 记录接口平均tps + */ + private ConcurrentHashMap apiTpsMap; + + public SceneTotalCountContext() { + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/SceneTotalCountContextDTO.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/SceneTotalCountContextDTO.java new file mode 100644 index 000000000..6c2c4b4cf --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/SceneTotalCountContextDTO.java @@ -0,0 +1,104 @@ +package run.mone.mimeter.engine.agent.bo.stat; + + +import lombok.Data; +import run.mone.mimeter.engine.agent.bo.task.DagTaskRps; + +import java.io.Serializable; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +@Data +public class SceneTotalCountContextDTO implements Serializable { + + private String reportId; + + private Integer sceneId; + + private Integer taskId; + + /** + * 场景类型,用于调用获取p99 p95 + */ + private String sceneType; + + /** + * 使用的发压机数量 + */ + private Integer agentNum; + + /** + * 链路实际 rps + */ + DagTaskRps dagTaskRps; + + /** + * 可控的场景发压比例 + */ + private Integer rpsRate; + + /** + * 关联任务数 + */ + private Integer connectTaskNum; + + private boolean lastTime; + + /** + * 全场景请求次数 + */ + private long totalReq; + + /** + * 丢失连接数 + */ + private long lossConnNum; + + /** + * 全场景业务处理次数 + */ + private long totalTCount; + + /** + * 全场景成功次数 + */ + private long totalSuccReq; + + /** + * 全场景错误次数 + */ + private long totalErrReq; + + /** + * 记录接口错误统计 + * <"s_code_errorCode",> or <"cp_id_checkpointId",> + * <"s_code_404",<2342,10>> or <"cp_id_141242",<2342,10>> + */ + private ConcurrentHashMap> counterMap; + + + /** + * 记录接口Rt与Tps数据 + * + * 例:> 每次调用统计rt + */ + private ConcurrentHashMap>> apiRtMap; + + /** + * 记录接口请求次数,成功失败数 + * > + */ + private ConcurrentHashMap> apiCountMap; + + + /** + * 记录接口平均rps + */ + private ConcurrentHashMap apiRpsMap; + + /** + * 记录接口平均tps + */ + private ConcurrentHashMap apiTpsMap; + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/SysMonitorType.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/SysMonitorType.java new file mode 100644 index 000000000..023e1c120 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/SysMonitorType.java @@ -0,0 +1,32 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.stat; + +public enum SysMonitorType { + Cpu_Usage("cpuUsage"), + Mem_Used("memUsage"), + Load_Avg("load_avg"), + Load_Max("load_max"), + + ; + + public final String name; + + SysMonitorType(String name) { + this.name = name; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/TotalCounterStatistic.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/TotalCounterStatistic.java new file mode 100644 index 000000000..d792322ed --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/stat/TotalCounterStatistic.java @@ -0,0 +1,127 @@ +package run.mone.mimeter.engine.agent.bo.stat; + + +import lombok.Data; +import run.mone.mimeter.engine.agent.bo.task.DagTaskRps; + +import java.io.Serializable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.LongAdder; +@Data +public class TotalCounterStatistic implements Serializable { + + private AtomicInteger finishAgentNum; + + /** + * 可控的场景发压比例 + */ + private volatile int rpsRate; + + /** + * 全场景总请求次数 + */ + private LongAdder totalReq; + + /** + * 丢失连接数 + */ + private LongAdder lossConnNum; + + /** + * 全场景业务处理次数 + */ + private LongAdder totalTCount; + + /** + * 全场景成功次数 + */ + private LongAdder totalSuccReq; + + /** + * 全场景总错误请求次数 + */ + private LongAdder totalErrReq; + + /** + * 全场景平均rt + */ + private volatile int avgRt; + + /** + * 全场景最大rt + */ + private volatile int maxRt; + + /** + * 全场景平均tps + */ + private volatile int avgTps; + + /** + * 全场景最大tps + */ + private volatile int maxTps; + + /** + * 全场景最大rps + */ + private volatile int maxRps; + + /** + * 全场景平均rps + */ + private volatile int avgRps; + + /** + * 错误数据统计 + * <"s_code_errorCode",> or <"cp_id_checkpointId",> + * <"s_code_404",<2342,10>> or <"cp_id_141242",<2342,10>> + */ + private ConcurrentMap> counterMap; + + /** + * 记录接口Rt与Tps数据 + *

+ * 例:> + * > + * > + * > + * > + * > + * > + */ + private ConcurrentHashMap> apiRtAndTpsMap; + + /** + * 场景下每个链路的实时Rps + */ + private ConcurrentHashMap reportLinkRps; + + /** + * 记录接口请求次数,成功失败数 + * > + */ + private ConcurrentHashMap> apiCountMap; + + public TotalCounterStatistic(AtomicInteger finishAgentNum, LongAdder totalReq, + LongAdder lossConnNum, + LongAdder totalTCount,LongAdder totalSuccReq, LongAdder totalErrReq, + ConcurrentMap> counterMap, + ConcurrentHashMap> apiRtAndTpsMap, + ConcurrentHashMap> apiCountMap, + ConcurrentHashMap reportLinkRps + ) { + this.finishAgentNum = finishAgentNum; + this.totalReq = totalReq; + this.lossConnNum = lossConnNum; + this.totalTCount = totalTCount; + this.totalSuccReq = totalSuccReq; + this.totalErrReq = totalErrReq; + this.counterMap = counterMap; + this.apiRtAndTpsMap = apiRtAndTpsMap; + this.apiCountMap = apiCountMap; + this.reportLinkRps = reportLinkRps; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/CancelType.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/CancelType.java new file mode 100644 index 000000000..a00af6195 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/CancelType.java @@ -0,0 +1,28 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.task; + +public enum CancelType { + Manual(0), + BySla(1); + + public final int code; + + private CancelType(int code) { + this.code = code; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/ChangeQpsReq.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/ChangeQpsReq.java new file mode 100644 index 000000000..6f1859064 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/ChangeQpsReq.java @@ -0,0 +1,18 @@ +package run.mone.mimeter.engine.agent.bo.task; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +@Data +public class ChangeQpsReq implements Serializable { + + Integer rpsRate; + + List dagTaskRpsList; + + public ChangeQpsReq(List dagTaskRpsList) { + this.dagTaskRpsList = dagTaskRpsList; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/Context.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/Context.java new file mode 100644 index 000000000..c364f833c --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/Context.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.task; + +import lombok.Data; + +import java.io.Serializable; + +/** +* @author goodjava@qq.com +* @date 2022/5/19 +*/ +@Data +public class Context implements Serializable { + + /** + * 完成 + */ + private volatile boolean finish; + + /** + * 被取消掉 + */ + private volatile boolean cancel; + + /** + * 取消的方式 + */ + private volatile int cancelType; + + /** + * 可控的图任务qps + */ + private volatile int taskQps; + + /** + * 可控的场景发压比例 + */ + private volatile int rpsRate; +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/DagTaskRps.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/DagTaskRps.java new file mode 100644 index 000000000..5dece6634 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/DagTaskRps.java @@ -0,0 +1,20 @@ +package run.mone.mimeter.engine.agent.bo.task; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class DagTaskRps implements Serializable { + String reportId; + Integer linkId; + Integer taskId; + Integer rps; + + public DagTaskRps(String reportId, Integer linkId, Integer taskId, Integer rps) { + this.reportId = reportId; + this.linkId = linkId; + this.taskId = taskId; + this.rps = rps; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/HeraContextInfo.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/HeraContextInfo.java new file mode 100644 index 000000000..0cb7e9d6f --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/HeraContextInfo.java @@ -0,0 +1,20 @@ +package run.mone.mimeter.engine.agent.bo.task; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class HeraContextInfo implements Serializable { + private Integer sceneId; + private Integer serialLinkId; + private Integer sceneApiId; + private String taskFlag; + + public HeraContextInfo(Integer sceneId, Integer serialLinkId, Integer sceneApiId, String taskFlag) { + this.sceneId = sceneId; + this.serialLinkId = serialLinkId; + this.sceneApiId = sceneApiId; + this.taskFlag = taskFlag; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/HostsFileResult.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/HostsFileResult.java new file mode 100644 index 000000000..9100d9ac2 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/HostsFileResult.java @@ -0,0 +1,10 @@ +package run.mone.mimeter.engine.agent.bo.task; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class HostsFileResult implements Serializable { + private String hostsFile; +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/PullApiTrafficReq.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/PullApiTrafficReq.java new file mode 100644 index 000000000..315c6fb8c --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/PullApiTrafficReq.java @@ -0,0 +1,46 @@ +/* + * Copyright 2020 XiaoMi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at the following link. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package run.mone.mimeter.engine.agent.bo.task; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +/** + * Created by dongzhenxing on 2023/2/24 7:00 PM + */ +@Data +public class PullApiTrafficReq implements Serializable { + + private int trafficConfigId; + + private String url; + + private long fromTime; + + private long toTime; + + public PullApiTrafficReq() { + } + + public PullApiTrafficReq(int trafficConfigId, String url, long fromTime, long toTime) { + this.trafficConfigId = trafficConfigId; + this.url = url; + this.fromTime = fromTime; + this.toTime = toTime; + } +} \ No newline at end of file diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/PullTrafficReqBase.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/PullTrafficReqBase.java new file mode 100644 index 000000000..1ea22ce02 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/PullTrafficReqBase.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020 XiaoMi. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at the following link. + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package run.mone.mimeter.engine.agent.bo.task; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +/** + * Created by dongzhenxing on 2023/2/24 7:00 PM + */ +@Data +public class PullTrafficReqBase implements Serializable { + +// /** +// * 开启使用流量数据的接口id列表 +// */ +// private List useTrafficApiIds; + + /** + * 每个接口所用的流量配置及筛选范围 + */ + private List apiTrafficReqList; + +} \ No newline at end of file diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/SubmitTaskRes.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/SubmitTaskRes.java new file mode 100644 index 000000000..b549041c9 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/SubmitTaskRes.java @@ -0,0 +1,27 @@ +package run.mone.mimeter.engine.agent.bo.task; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +@Data +public class SubmitTaskRes implements Serializable { + private String reportId; + private TaskResult taskResult; + + /** + * 链路id与生成的dag 任务id映射 + */ + private Map linkTaskIdMap; + + private List agentIpList; + + public SubmitTaskRes(String reportId, TaskResult taskResult, Map linkTaskIdMap, List agentIpList) { + this.reportId = reportId; + this.taskResult = taskResult; + this.linkTaskIdMap = linkTaskIdMap; + this.agentIpList = agentIpList; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/Task.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/Task.java new file mode 100644 index 000000000..e9e8cd2a2 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/Task.java @@ -0,0 +1,184 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.task; + +import com.xiaomi.data.push.schedule.task.graph.GraphTaskContext; +import lombok.Data; +import run.mone.mimeter.engine.agent.bo.data.DebugSceneApiInfoReq; +import run.mone.mimeter.engine.agent.bo.data.DemoData; +import run.mone.mimeter.engine.agent.bo.data.DubboData; +import run.mone.mimeter.engine.agent.bo.data.HttpData; +import run.mone.mimeter.engine.agent.bo.data.NodeInfo; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; + +/** + * @author goodjava@qq.com + * @date 2022/5/19 + */ +@Data +public class Task implements Serializable { + + /** + * 单接口任务id + */ + private int id; + + /** + * 用于取消任务的任务id列表 + */ + private List ids; + + /** + * 取消任务的类型 0:手动,1:由sla触发 + */ + private int cancelType; + + /** + * 由sla触发结束的规则内容 + */ + private String cancelBySlaRule; + + /** + * 场景定义id + */ + private int sceneId; + + /** + * 用于hera context 传参 + */ + private HeraContextInfo heraContextInfo; + + /** + * 0:单接口调试 + * 1:场景调试 + * 2:场景压测 + */ + private int submitTaskType; + + /** + * 用于调试的接口信息 + */ + private DebugSceneApiInfoReq apiInfo; + + private String reportId; + + private String opUser; + + //********** 以上字段由dashboard传,以下取场景数据构造// + + /** + * 是否启用流量参数 + */ + private boolean enableTraffic; + + /** + * 待拉取流量配置列表 + */ + private PullTrafficReqBase trafficToPullConfList; + + /** + * 发压机数量 + */ + private int agentNum; + + /** + * 该发压机器索引序号,从 0 开始 + */ + private int agentIndex; + + private int connectTaskNum; + + private int qps; + + private int logRate; + + /** + * 持续多少秒 + */ + private int time; + + /** + * 压力递增模式 + */ + private Integer incrementMode; + + /** + * 本任务 + */ + DagTaskRps dagTaskRps; + + /** + * 自定义成功状态码 + */ + private String successCode; + + /** + * 接口发压信息 + */ + private String apiBenchInfos; + + /** + * 超时时间(毫秒) + */ + private int timeout = 1000; + + private TaskType type; + + private TaskType sceneType; + + private String addr; + + private HttpData httpData; + + private DubboData dubboData; + + private DemoData demoData; + + /** + * 可控的场景发压比例 + */ + private Integer rpsRate; + + private ConcurrentHashMap attachments = new ConcurrentHashMap<>(); + + private TreeMap> dataMap; + + /** + * debug只会调用一次,并且需要拿到返回结果 + */ + private boolean debug; + + private GraphTaskContext dagInfo; + + /** + * 提值表达式(需要从依赖的任务里提值) + * + * dag.0.map{data}{name} + */ + private Map exprMap; + + /** + * 打点记录链路id + */ + private Integer serialLinkID; + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/TaskContext.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/TaskContext.java new file mode 100644 index 000000000..55d826902 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/TaskContext.java @@ -0,0 +1,58 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.task; + +import lombok.Data; +import run.mone.mimeter.engine.agent.bo.stat.SceneTotalCountContext; + +import java.io.Serializable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * @author goodjava@qq.com + * @date 2022/6/2 + */ +@Data +public class TaskContext implements Serializable { + + private int num; + + /** + * 是否发生了错误 + */ + private volatile boolean error; + + /** + * 完成任务数 + */ + private AtomicInteger finishTaskNum = new AtomicInteger(0); + + + private ConcurrentHashMap attachments = new ConcurrentHashMap<>(); + + /** + * 存储各个node的结果 + */ + private ConcurrentHashMap resultMap = new ConcurrentHashMap<>(); + + /** + * 统计错误信息 + */ + private SceneTotalCountContext sceneTotalCountContext = new SceneTotalCountContext(); + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/TaskResult.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/TaskResult.java new file mode 100644 index 000000000..f4704c5bf --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/TaskResult.java @@ -0,0 +1,87 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.task; + +import lombok.Data; +import run.mone.mimeter.engine.agent.bo.data.CommonReqInfo; + +import java.io.Serializable; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author goodjava@qq.com + * @date 2022/5/19 + */ +@Data +public class TaskResult implements Serializable { + + /** + * 任务id + */ + private int id; + + private Context context; + + private String addr; + + private String opUser; + + private AtomicLong success = new AtomicLong(); + + private AtomicLong failure = new AtomicLong(); + + private int code; + + private boolean ok; + + private String result; + + private CommonReqInfo commonReqInfo; + + /** + * 返回的header + */ + private Map respHeaders; + + private String debugUrl; + + private long rt; + + //bytes + private long size; + + private String triggerCpInfo; + + /** + * 触发的检查点条件 + */ + private String triggerFilterCondition; + + private boolean debug; + + private String reportId; + + private Integer sceneId; + + private boolean quitByManual; + + private int cancelType; + + public String cancelBySlaRule; + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/TaskStatus.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/TaskStatus.java new file mode 100644 index 000000000..e9a896e16 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/TaskStatus.java @@ -0,0 +1,33 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mimeter.engine.agent.bo.task; + +public enum TaskStatus { + Init(0), + Success(1), + Failure(2), + Retry(3), + Running(4), + + Stopped(5); + + public final int code; + + private TaskStatus(int code) { + this.code = code; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/TaskStatusBo.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/TaskStatusBo.java new file mode 100644 index 000000000..3ea5d0667 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/TaskStatusBo.java @@ -0,0 +1,16 @@ +package run.mone.mimeter.engine.agent.bo.task; + +import lombok.Data; + +import java.io.Serializable; + +@Data +public class TaskStatusBo implements Serializable { + + private int taskId; + + private int sceneId; + + private TaskStatus taskStatus; + +} diff --git a/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/TaskType.java b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/TaskType.java new file mode 100644 index 000000000..a0f36ea8e --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-api/src/main/java/run/mone/mimeter/engine/agent/bo/task/TaskType.java @@ -0,0 +1,20 @@ +package run.mone.mimeter.engine.agent.bo.task; + +/** + * @author goodjava@qq.com + * @date 2022/5/19 + */ +public enum TaskType { + + http(1), + dubbo(2), + dag(3), + demo(4); + + + public final int code; + + private TaskType(int code) { + this.code = code; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-test/pom.xml b/ozburst-all/mimeter-engine/mimeter-test/pom.xml new file mode 100644 index 000000000..3187f82be --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-test/pom.xml @@ -0,0 +1,87 @@ + + + + mimeter-engine + run.mone + 1.0-SNAPSHOT + + 4.0.0 + + mimeter-test + + + 20 + 20 + + + + + + com.xiaomi.mone + sautumn-server + 1.0.0-SNAPSHOT + + + com.xiaomi.youpin + rpc + + + com.xiaomi.youpin + docean + + + com.xiaomi.youpin + gwdash-api + + + tools + com.sun + + + com.xiaomi.youpin + struct + + + com.xiaomi.youpin + log + + + maven-pmd-plugin + org.apache.maven.plugins + + + jackson-annotations + com.fasterxml.jackson.core + + + + + + mone-tenant + mone-tenant-api + 1.0-SNAPSHOT + + + + run.mone + struct + 1.4-SNAPSHOT + + + + run.mone + docean + 1.4-SNAPSHOT + + + + ch.qos.logback + logback-classic + 1.1.2 + + + + + \ No newline at end of file diff --git a/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/Bootstrap.java b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/Bootstrap.java new file mode 100644 index 000000000..a126d4134 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/Bootstrap.java @@ -0,0 +1,44 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mibench.test; + +import com.xiaomi.youpin.docean.Ioc; +import com.xiaomi.youpin.docean.Mvc; +import com.xiaomi.youpin.docean.config.HttpServerConfig; +import com.xiaomi.youpin.docean.mvc.DoceanHttpServer; + +/** + * @author goodjava@qq.com + * @date 2022/6/2 + */ +public class Bootstrap { + + + /** + * 起一个http服务+dubbo服务,用来测试dag任务 + * + * @param args + */ + public static void main(String[] args) throws InterruptedException { + Ioc.ins().init("run.mone.mibench.test", "com.xiaomi.youpin.docean.plugin"); + Mvc.ins(); + DoceanHttpServer server = new DoceanHttpServer(HttpServerConfig.builder().port(7777).websocket(false).build()); + server.start(); + } + + +} diff --git a/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/controller/Request.java b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/controller/Request.java new file mode 100644 index 000000000..11de3018a --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/controller/Request.java @@ -0,0 +1,57 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mibench.test.controller; + +/** + * @author goodjava@qq.com + * @date 2022/6/2 + */ +public class Request { + + private String token; + + private String id; + + private String uuid; + + public Request() { + } + + public String getToken() { + return token; + } + + public void setToken(String token) { + this.token = token; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/controller/Response.java b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/controller/Response.java new file mode 100644 index 000000000..ed4cc5000 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/controller/Response.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mibench.test.controller; +/** + * @author goodjava@qq.com + * @date 2022/6/2 + */ +public class Response { + + private int code; + + private String message; + + private D data; + + public Response() { + } + + public Response(int code, String message, D data) { + this.code = code; + this.message = message; + this.data = data; + } + + public int getCode() { + return code; + } + + public void setCode(int code) { + this.code = code; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public D getData() { + return data; + } + + public void setData(D data) { + this.data = data; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/controller/Test.java b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/controller/Test.java new file mode 100644 index 000000000..e2e96d2bb --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/controller/Test.java @@ -0,0 +1,30 @@ +package run.mone.mibench.test.controller; + +import com.xiaomi.youpin.docean.anno.RequestMapping; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class Test { + + /** + * //Athena:用token换id的方法注释 + * 用token换id + * + * @param request + * @return + */ + @RequestMapping(path = "/id") + //Athena:定义请求映射路径的注释 + public Response getId(Request request) { + //Athena:记录请求信息的注释 + log.info("/id req:{}", request); + //Athena:创建响应对象的注释 + Response r = new Response<>(); + //Athena:设置响应消息的注释 + r.setMessage(request.getId()); + //Athena:设置响应数据的注释 + r.setData("1"); + //Athena:返回响应对象的注释 + return r; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/controller/TestController.java b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/controller/TestController.java new file mode 100644 index 000000000..6bdc9f468 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/controller/TestController.java @@ -0,0 +1,109 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mibench.test.controller; + +import com.xiaomi.youpin.docean.anno.Controller; +import com.xiaomi.youpin.docean.anno.RequestMapping; +import com.xiaomi.youpin.docean.anno.RequestParam; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.TimeUnit; + +/** + * @date 2022/6/2 + */ +@Controller +public class TestController { + + private static Logger log = LoggerFactory.getLogger(TestController.class); + + /** + //Athena:获取一个token + * 换取一个token + * + * @param request + * @return + */ + //Athena:映射请求路径为/token + @RequestMapping(path = "/token") + //Athena:处理请求并返回响应 + public Response getToken(Request request) { + //Athena:打印请求信息 + log.info("/token req:{}", request); + //Athena:模拟延迟 + sleep(); + //Athena:创建响应对象 + Response r = new Response<>(); + //Athena:设置响应消息 + r.setMessage(request.getId()); + //Athena:设置响应数据 + r.setData("abcdefaaff"); + //Athena:返回响应 + return r; + } + + private void sleep() { + try { + TimeUnit.SECONDS.sleep(2); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + + /** + * 用token换id + * + * @param request + * @return + */ + @RequestMapping(path = "/id") + public Response getId(Request request) { + log.info("/id req:{}", request); + Response r = new Response<>(); + r.setMessage(request.getId()); + r.setData("1"); + return r; + } + + /** + * 用id查年龄 + * + * @param request + * @return + */ + @RequestMapping(path = "/age") + public Response getAge(Request request) { + log.info("/age req:{}", request); + Response r = new Response<>(); + r.setMessage(request.getId()); + r.setData("22"); + return r; + } + + + @RequestMapping(path = "/sum", method = "get") + public Response sum(@RequestParam("a") int a, @RequestParam("b") int b) { + log.info("/sum req:{} {}", a, b); + Response r = new Response<>(); + r.setData("" + (a + b)); + return r; + } + + +} diff --git a/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/service/MBReq.java b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/service/MBReq.java new file mode 100644 index 000000000..56995c0d4 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/service/MBReq.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mibench.test.service; + +import java.io.Serializable; + +/** + * @author goodjava@qq.com + * @date 2022/6/4 + */ +public class MBReq implements Serializable { + + private int id; + + public MBReq() { + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/service/MBRes.java b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/service/MBRes.java new file mode 100644 index 000000000..20395c833 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/service/MBRes.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mibench.test.service; + +import java.io.Serializable; + +/** + * @author goodjava@qq.com + * @date 2022/6/4 + */ +public class MBRes implements Serializable { + + private int id; + + public MBRes() { + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/service/MBService.java b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/service/MBService.java new file mode 100644 index 000000000..1959f1181 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/service/MBService.java @@ -0,0 +1,30 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mibench.test.service; + +/** + * @author goodjava@qq.com + * @date 2022/6/4 + */ +public interface MBService { + + + int sum(int a, int b); + + MBRes call(MBReq req); + +} diff --git a/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/service/MbServiceImpl.java b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/service/MbServiceImpl.java new file mode 100644 index 000000000..1c799d411 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-test/src/main/java/run/mone/mibench/test/service/MbServiceImpl.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Xiaomi + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package run.mone.mibench.test.service; + + +import com.xiaomi.youpin.docean.plugin.dubbo.anno.Service; + +/** + * @author goodjava@qq.com + * @date 2022/6/4 + */ +@Service(interfaceClass = MBService.class) +public class MbServiceImpl implements MBService{ + @Override + public int sum(int a, int b) { + return a+b; + } + + @Override + public MBRes call(MBReq req) { + MBRes res = new MBRes(); + res.setId(req.getId()); + return res; + } +} diff --git a/ozburst-all/mimeter-engine/mimeter-test/src/main/resources/config.properties b/ozburst-all/mimeter-engine/mimeter-test/src/main/resources/config.properties new file mode 100644 index 000000000..fec8e3ce9 --- /dev/null +++ b/ozburst-all/mimeter-engine/mimeter-test/src/main/resources/config.properties @@ -0,0 +1,2 @@ +dubbo_app_name=mibench_test +dubbo_reg_address=nacos://127.0.0.1:80 \ No newline at end of file diff --git a/ozburst-all/mimeter-engine/pom.xml b/ozburst-all/mimeter-engine/pom.xml new file mode 100644 index 000000000..df36d8d2b --- /dev/null +++ b/ozburst-all/mimeter-engine/pom.xml @@ -0,0 +1,95 @@ + + + 4.0.0 + + run.mone + mimeter-engine + pom + 1.0-SNAPSHOT + + mimeter-api + mimeter-agent + mimeter-agent-service + mimeter-agent-manager + mimeter-test + + + + 20 + 20 + + + + + org.projectlombok + lombok + 1.18.24 + provided + + + + com.xiaomi.youpin + docean-plugin-dubbo-serverless + 1.4-SNAPSHOT + + + run.mone + docean + + + org.apache.dubbo + dubbo + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.10.1 + + 20 + 20 + false + + --enable-preview + -J--add-opens=java.base/java.util=ALL-UNNAMED + -J--add-opens=java.base/java.lang=ALL-UNNAMED + -J--add-opens=java.base/java.math=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.jvm=ALL-UNNAMED + -J--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED + + + + + + + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + + + + \ No newline at end of file