diff --git a/docker/docker-compose-arm64.yml b/docker/docker-compose-arm64.yml index 4da8fe4..55adc8e 100644 --- a/docker/docker-compose-arm64.yml +++ b/docker/docker-compose-arm64.yml @@ -16,6 +16,82 @@ version: "3" services: + redis-cluster: + image: redis + command: redis-cli --cluster create 172.22.225.9:6371 172.22.225.9:6372 172.22.225.9:6373 172.22.225.9:6374 172.22.225.9:6375 172.22.225.9:6376 --cluster-replicas 1 --cluster-yes + depends_on: + - redis-6371 + - redis-6372 + - redis-6373 + - redis-6374 + - redis-6375 + - redis-6376 + redis-6371: # 服务名称 + image: redis # 创建容器时所需的镜像 + container_name: redis-6371 # 容器名称 + restart: always # 容器总是重新启动 + ports: + - "6371:6371" + - "16371:16371" + volumes: # 数据卷，目录挂载 + - ./redis/redis1/redis1.conf:/usr/local/etc/redis/redis.conf + - ./redis/redis1/data:/data + command: redis-server /usr/local/etc/redis/redis.conf # 覆盖容器启动后默认执行的命令 + + redis-6372: + image: redis + container_name: redis-6372 + ports: + - "6372:6372" + - "16372:16372" + volumes: + - ./redis/redis2/redis2.conf:/usr/local/etc/redis/redis.conf + - ./redis/redis2/data:/data + command: redis-server /usr/local/etc/redis/redis.conf + + redis-6373: + image: redis + container_name: redis-6373 + ports: + - "6373:6373" + - "16373:16373" + volumes: + - ./redis/redis3/redis3.conf:/usr/local/etc/redis/redis.conf + - ./redis/redis3/data:/data + command: redis-server /usr/local/etc/redis/redis.conf + + redis-6374: + image: redis + container_name: redis-6374 + ports: + - "6374:6374" + - "16374:16374" + volumes: + - ./redis/redis4/redis4.conf:/usr/local/etc/redis/redis.conf + - ./redis/redis4/data:/data + command: redis-server /usr/local/etc/redis/redis.conf + + redis-6375: + image: redis + container_name: redis-6375 + ports: + - "6375:6375" + - "16375:16375" + volumes: + - ./redis/redis5/redis5.conf:/usr/local/etc/redis/redis.conf + - ./redis/redis5/data:/data + command: redis-server /usr/local/etc/redis/redis.conf + + redis-6376: + image: redis + container_name: redis-6376 + ports: + - "6376:6376" + - "16376:16376" + volumes: + - ./redis/redis6/redis6.conf:/usr/local/etc/redis/redis.conf + - ./redis/redis6/data:/data + command: redis-server /usr/local/etc/redis/redis.conf moonbox-mysql: platform: linux/amd64 container_name: moonbox-mysql diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 0fc7570..3d43ca8 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -16,6 +16,82 @@ version: "3" services: + redis-cluster: + image: redis + command: redis-cli --cluster create 172.22.225.9:6371 172.22.225.9:6372 172.22.225.9:6373 172.22.225.9:6374 172.22.225.9:6375 172.22.225.9:6376 --cluster-replicas 1 --cluster-yes + depends_on: + - redis-6371 + - redis-6372 + - redis-6373 + - redis-6374 + - redis-6375 + - redis-6376 + redis-6371: # 服务名称 + image: redis # 创建容器时所需的镜像 + container_name: redis-6371 # 容器名称 + restart: always # 容器总是重新启动 + ports: + - "6371:6371" + - "16371:16371" + volumes: # 数据卷，目录挂载 + - ./redis/redis1/redis1.conf:/usr/local/etc/redis/redis.conf + - ./redis/redis1/data:/data + command: redis-server /usr/local/etc/redis/redis.conf # 覆盖容器启动后默认执行的命令 + + redis-6372: + image: redis + container_name: redis-6372 + ports: + - "6372:6372" + - "16372:16372" + volumes: + - ./redis/redis2/redis2.conf:/usr/local/etc/redis/redis.conf + - ./redis/redis2/data:/data + command: redis-server /usr/local/etc/redis/redis.conf + + redis-6373: + image: redis + container_name: redis-6373 + ports: + - "6373:6373" + - "16373:16373" + volumes: + - ./redis/redis3/redis3.conf:/usr/local/etc/redis/redis.conf + - ./redis/redis3/data:/data + command: redis-server /usr/local/etc/redis/redis.conf + + redis-6374: + image: redis + container_name: redis-6374 + ports: + - "6374:6374" + - "16374:16374" + volumes: + - ./redis/redis4/redis4.conf:/usr/local/etc/redis/redis.conf + - ./redis/redis4/data:/data + command: redis-server /usr/local/etc/redis/redis.conf + + redis-6375: + image: redis + container_name: redis-6375 + ports: + - "6375:6375" + - "16375:16375" + volumes: + - ./redis/redis5/redis5.conf:/usr/local/etc/redis/redis.conf + - ./redis/redis5/data:/data + command: redis-server /usr/local/etc/redis/redis.conf + + redis-6376: + image: redis + container_name: redis-6376 + ports: + - "6376:6376" + - "16376:16376" + volumes: + - ./redis/redis6/redis6.conf:/usr/local/etc/redis/redis.conf + - ./redis/redis6/data:/data + command: redis-server /usr/local/etc/redis/redis.conf moonbox-mysql: container_name: moonbox-mysql build: diff --git a/docker/redis/redis1/data/nodes.conf b/docker/redis/redis1/data/nodes.conf new file mode 100644 index 0000000..e283fc6 --- /dev/null +++ b/docker/redis/redis1/data/nodes.conf @@ -0,0 +1,4 @@ +2c71ee68db63234165e93d1619ff02a1641eb2f8 172.22.225.9:6372@16372,,tls-port=0,shard-id=a4b1bdfa0c58a6fbcd97d8a843d46286dc365e8f master - 0 1709196638000 2 connected 5461-10922 +3926e4b60debec1282741e15953a74a6c4d3eb2a 172.22.225.9:6371@16371,,tls-port=0,shard-id=dad52cafaa52b017aedb5bedc18b1dbc459ff13d myself,master - 0 1709196638000 1 connected 0-5460 +e8fcf70df6cfd3838303d18d8e88f20f372120eb 172.22.225.9:6373@16373,,tls-port=0,shard-id=0e55c09c228fd825377ca07f972e371d7e2e5913 master - 1709196638126 1709196638030 6 disconnected 10923-16383 +vars currentEpoch 6 lastVoteEpoch 0 diff --git a/docker/redis/redis1/redis1.conf b/docker/redis/redis1/redis1.conf new file mode 100644 index 0000000..9e7083f --- /dev/null +++ b/docker/redis/redis1/redis1.conf @@ -0,0 +1,10 @@ +port 6371 +protected-mode no +daemonize no +appendonly yes +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 15000 +cluster-announce-ip 172.22.225.9 +cluster-announce-port 6371 +cluster-announce-bus-port 16371 \ No newline at end of file diff --git a/docker/redis/redis2/data/nodes.conf b/docker/redis/redis2/data/nodes.conf new file mode 100644 index 0000000..1e8029b --- /dev/null +++ b/docker/redis/redis2/data/nodes.conf @@ -0,0 +1,4 @@ +e8fcf70df6cfd3838303d18d8e88f20f372120eb 172.22.225.9:6373@16373,,tls-port=0,shard-id=0e55c09c228fd825377ca07f972e371d7e2e5913 master - 1709196637740 1709196637645 6 disconnected 10923-16383 +2c71ee68db63234165e93d1619ff02a1641eb2f8 172.22.225.9:6372@16372,,tls-port=0,shard-id=a4b1bdfa0c58a6fbcd97d8a843d46286dc365e8f myself,master - 0 1709196638000 2 connected 5461-10922 +3926e4b60debec1282741e15953a74a6c4d3eb2a 172.22.225.9:6371@16371,,tls-port=0,shard-id=dad52cafaa52b017aedb5bedc18b1dbc459ff13d master - 0 1709196638146 1 connected 0-5460 +vars currentEpoch 6 lastVoteEpoch 0 diff --git a/docker/redis/redis2/redis2.conf b/docker/redis/redis2/redis2.conf new file mode 100644 index 0000000..0779fb8 --- /dev/null +++ b/docker/redis/redis2/redis2.conf @@ -0,0 +1,10 @@ +port 6372 +protected-mode no +daemonize no +appendonly yes +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 15000 +cluster-announce-ip 172.22.225.9 +cluster-announce-port 6372 +cluster-announce-bus-port 16372 diff --git a/docker/redis/redis3/data/nodes.conf b/docker/redis/redis3/data/nodes.conf new file mode 100644 index 0000000..f3397f5 --- /dev/null +++ b/docker/redis/redis3/data/nodes.conf @@ -0,0 +1,4 @@ +3926e4b60debec1282741e15953a74a6c4d3eb2a 172.22.225.9:6371@16371,,tls-port=0,shard-id=dad52cafaa52b017aedb5bedc18b1dbc459ff13d master - 0 1709196411483 1 connected 0-5460 +e8fcf70df6cfd3838303d18d8e88f20f372120eb 172.22.225.9:6373@16373,,tls-port=0,shard-id=0e55c09c228fd825377ca07f972e371d7e2e5913 myself,master - 0 1709196402000 3 connected 10923-16383 +2c71ee68db63234165e93d1619ff02a1641eb2f8 172.22.225.9:6372@16372,,tls-port=0,shard-id=a4b1bdfa0c58a6fbcd97d8a843d46286dc365e8f master - 1709196403317 1709196402321 2 connected 5461-10922 +vars currentEpoch 6 lastVoteEpoch 0 diff --git a/docker/redis/redis3/redis3.conf b/docker/redis/redis3/redis3.conf new file mode 100644 index 0000000..6626abe --- /dev/null +++ b/docker/redis/redis3/redis3.conf @@ -0,0 +1,10 @@ +port 6373 +protected-mode no +daemonize no +appendonly yes +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 15000 +cluster-announce-ip 172.22.225.9 +cluster-announce-port 6373 +cluster-announce-bus-port 16373 diff --git a/docker/redis/redis4/data/nodes.conf b/docker/redis/redis4/data/nodes.conf new file mode 100644 index 0000000..9fee9b0 --- /dev/null +++ b/docker/redis/redis4/data/nodes.conf @@ -0,0 +1,4 @@ +2c71ee68db63234165e93d1619ff02a1641eb2f8 172.22.225.9:6372@16372,,tls-port=0,shard-id=a4b1bdfa0c58a6fbcd97d8a843d46286dc365e8f master - 0 1709196486419 2 connected 5461-10922 +e8fcf70df6cfd3838303d18d8e88f20f372120eb 172.22.225.9:6374@16374,,tls-port=0,shard-id=0e55c09c228fd825377ca07f972e371d7e2e5913 myself,master - 0 1709196486000 4 connected +3926e4b60debec1282741e15953a74a6c4d3eb2a 172.22.225.9:6371@16371,,tls-port=0,shard-id=dad52cafaa52b017aedb5bedc18b1dbc459ff13d master - 1709196486410 1709196486324 1 connected 0-5460 +vars currentEpoch 6 lastVoteEpoch 0 diff --git a/docker/redis/redis4/redis4.conf b/docker/redis/redis4/redis4.conf new file mode 100644 index 0000000..ff5a7fd --- /dev/null +++ b/docker/redis/redis4/redis4.conf @@ -0,0 +1,10 @@ +port 6374 +protected-mode no +daemonize no +appendonly yes +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 15000 +cluster-announce-ip 172.22.225.9 +cluster-announce-port 6374 +cluster-announce-bus-port 16374 diff --git a/docker/redis/redis5/data/nodes.conf b/docker/redis/redis5/data/nodes.conf new file mode 100644 index 0000000..21d50b4 --- /dev/null +++ b/docker/redis/redis5/data/nodes.conf @@ -0,0 +1,4 @@ +2c71ee68db63234165e93d1619ff02a1641eb2f8 172.22.225.9:6372@16372,,tls-port=0,shard-id=a4b1bdfa0c58a6fbcd97d8a843d46286dc365e8f master - 0 1709196516484 2 connected 5461-10922 +3926e4b60debec1282741e15953a74a6c4d3eb2a 172.22.225.9:6371@16371,,tls-port=0,shard-id=dad52cafaa52b017aedb5bedc18b1dbc459ff13d master - 1709196516477 1709196516385 1 connected 0-5460 +e8fcf70df6cfd3838303d18d8e88f20f372120eb 172.22.225.9:6375@16375,,tls-port=0,shard-id=0e55c09c228fd825377ca07f972e371d7e2e5913 myself,master - 0 1709196516000 5 connected +vars currentEpoch 6 lastVoteEpoch 0 diff --git a/docker/redis/redis5/redis5.conf b/docker/redis/redis5/redis5.conf new file mode 100644 index 0000000..97e5b91 --- /dev/null +++ b/docker/redis/redis5/redis5.conf @@ -0,0 +1,10 @@ +port 6375 +protected-mode no +daemonize no +appendonly yes +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 15000 +cluster-announce-ip 172.22.225.9 +cluster-announce-port 6375 +cluster-announce-bus-port 16375 diff --git a/docker/redis/redis6/data/nodes.conf b/docker/redis/redis6/data/nodes.conf new file mode 100644 index 0000000..ac271dc --- /dev/null +++ b/docker/redis/redis6/data/nodes.conf @@ -0,0 +1,4 @@ +2c71ee68db63234165e93d1619ff02a1641eb2f8 172.22.225.9:6372@16372,,tls-port=0,shard-id=a4b1bdfa0c58a6fbcd97d8a843d46286dc365e8f master - 0 1709196437505 2 connected 5461-10922 +e8fcf70df6cfd3838303d18d8e88f20f372120eb 172.22.225.9:6376@16376,,tls-port=0,shard-id=0e55c09c228fd825377ca07f972e371d7e2e5913 myself,master - 0 1709196437000 6 connected +3926e4b60debec1282741e15953a74a6c4d3eb2a 172.22.225.9:6371@16371,,tls-port=0,shard-id=dad52cafaa52b017aedb5bedc18b1dbc459ff13d master - 0 1709196437504 1 connected 0-5460 +vars currentEpoch 6 lastVoteEpoch 0 diff --git a/docker/redis/redis6/redis6.conf b/docker/redis/redis6/redis6.conf new file mode 100644 index 0000000..4f78695 --- /dev/null +++ b/docker/redis/redis6/redis6.conf @@ -0,0 +1,10 @@ +port 6376 +protected-mode no +daemonize no +appendonly yes +cluster-enabled yes +cluster-config-file nodes.conf +cluster-node-timeout 15000 +cluster-announce-ip 172.22.225.9 +cluster-announce-port 6376 +cluster-announce-bus-port 16376 diff --git a/local-agent/monbox-agent.tar b/local-agent/monbox-agent.tar index 30d0917..e00c4eb 100644 Binary files a/local-agent/monbox-agent.tar and b/local-agent/monbox-agent.tar differ diff --git a/moonbox-agent/moonbox-java-agent/moonbox-core/pom.xml b/moonbox-agent/moonbox-java-agent/moonbox-core/pom.xml index e8e43eb..89dc5b0 100644 --- a/moonbox-agent/moonbox-java-agent/moonbox-core/pom.xml +++ b/moonbox-agent/moonbox-java-agent/moonbox-core/pom.xml @@ -17,6 +17,11 @@ lombok provided + + org.apache.httpcomponents + httpcore + 4.4.14 + com.google.guava guava diff --git a/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/hbase-plugin/pom.xml b/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/hbase-plugin/pom.xml new file mode 100644 index 0000000..3b801fc --- /dev/null +++ b/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/hbase-plugin/pom.xml @@ -0,0 +1,44 @@ + + + + repeater-plugins + com.vivo.jvm.sandbox + 1.0.0-SNAPSHOT + + + + org.projectlombok + lombok + 1.18.24 + compile + + + 4.0.0 + + hbase-plugin + + + ${project.name}-${project.version} + + + org.apache.maven.plugins + maven-assembly-plugin + + + + attached + + package + + + jar-with-dependencies + + + + + + + + \ No newline at end of file diff --git a/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/hbase-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/hbase/HbasePlugin.java b/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/hbase-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/hbase/HbasePlugin.java new file mode 100644 index 0000000..85d3305 --- /dev/null +++ b/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/hbase-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/hbase/HbasePlugin.java @@ -0,0 +1,108 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +package com.alibaba.jvm.sandbox.repeater.plugin.hbase; + +import com.alibaba.jvm.sandbox.api.event.Event; +import com.alibaba.jvm.sandbox.repeater.plugin.api.InvocationProcessor; +import com.alibaba.jvm.sandbox.repeater.plugin.core.impl.AbstractInvokePluginAdapter; +import com.alibaba.jvm.sandbox.repeater.plugin.core.model.EnhanceModel; +import com.alibaba.jvm.sandbox.repeater.plugin.domain.InvokeType; +import com.alibaba.jvm.sandbox.repeater.plugin.spi.InvokePlugin; +import com.google.common.collect.Lists; +import org.kohsuke.MetaInfServices; + +import java.util.List; + +/** + * HBase增强处理插件 + */ +@MetaInfServices(InvokePlugin.class) +public class HbasePlugin extends AbstractInvokePluginAdapter { + + /** + * 获取增强模型列表 + * + * @return 增强模型列表 + */ + @Override + protected List getEnhanceModels() { + EnhanceModel.MethodPattern getMethod = EnhanceModel.MethodPattern.builder() + .methodName("get") + .parameterType(new String[]{"org.apache.hadoop.hbase.client.Get"}) + .build(); + + EnhanceModel.MethodPattern getMethod1 = EnhanceModel.MethodPattern.builder() + .methodName("get") + .parameterType(new String[]{List.class.getCanonicalName()}) + .build(); + + EnhanceModel.MethodPattern putMethod = EnhanceModel.MethodPattern.builder() + .methodName("put") + .build(); + + EnhanceModel.MethodPattern deleteMethod = EnhanceModel.MethodPattern.builder() + .methodName("delete") + .build(); + EnhanceModel.MethodPattern appendMethod = EnhanceModel.MethodPattern.builder() + .methodName("append") + .build(); + + EnhanceModel.MethodPattern incrementMethod = EnhanceModel.MethodPattern.builder() + .methodName("increment") + .build(); + + EnhanceModel.MethodPattern existsMethod = EnhanceModel.MethodPattern.builder() + .methodName("exists") + .build(); + + EnhanceModel.MethodPattern existsAllMethod = EnhanceModel.MethodPattern.builder() + .methodName("existsAll") + .build(); + + EnhanceModel.MethodPattern batchMethod = EnhanceModel.MethodPattern.builder() + .methodName("batch") + .parameterType(new String[]{List.class.getCanonicalName(),Object[].class.getCanonicalName()}) + .build(); + + EnhanceModel hTable = EnhanceModel.builder() + .classPattern("org.apache.hadoop.hbase.client.HTable") + .methodPatterns(new EnhanceModel.MethodPattern[]{getMethod,getMethod1,putMethod,deleteMethod,appendMethod,incrementMethod,existsMethod,existsAllMethod,batchMethod}) + .watchTypes(Event.Type.BEFORE, Event.Type.RETURN, Event.Type.THROWS) + .build(); + return Lists.newArrayList(hTable); + } + + + @Override + protected InvocationProcessor getInvocationProcessor() { + return new HbaseProcessor(getType()); + } + + @Override + public InvokeType getType() { + return InvokeType.HBASE; + } + + @Override + public String identity() { + return "hbase"; + } + + @Override + public boolean isEntrance() { + return false; + } +} diff --git a/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/hbase-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/hbase/HbaseProcessor.java b/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/hbase-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/hbase/HbaseProcessor.java new file mode 100644 index 0000000..94a72b8 --- /dev/null +++ b/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/hbase-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/hbase/HbaseProcessor.java @@ -0,0 +1,161 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +package com.alibaba.jvm.sandbox.repeater.plugin.hbase; + +import com.alibaba.jvm.sandbox.api.event.BeforeEvent; +import com.alibaba.jvm.sandbox.repeater.plugin.core.impl.api.DefaultInvocationProcessor; +import com.alibaba.jvm.sandbox.repeater.plugin.domain.Identity; +import com.alibaba.jvm.sandbox.repeater.plugin.domain.Invocation; +import com.alibaba.jvm.sandbox.repeater.plugin.domain.InvokeType; +import com.alibaba.jvm.sandbox.repeater.plugin.utils.Bytes; +import com.alibaba.jvm.sandbox.repeater.plugin.utils.ParameterTypesUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.apache.commons.lang3.reflect.MethodUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + + +/** + * hbase 处理器 + */ +@Slf4j +class HbaseProcessor extends DefaultInvocationProcessor { + + HbaseProcessor(InvokeType type) { + super(type); + } + + /** + * 组装Identity对象，用于标识HBase表的操作 + * + * @param event BeforeEvent事件对象 + * @return Identity对象，包含HBase表的操作信息 + */ + @Override + public Identity assembleIdentity(BeforeEvent event) { + try { + Object hTable = event.target; + Object table = FieldUtils.readField(hTable, "tableName", true); + String tableName = (String) MethodUtils.invokeMethod(table, "getNameAsString"); + String operation = event.javaMethodName; + return new Identity(InvokeType.HBASE.name(), tableName, operation + ParameterTypesUtil.getTypesStrByObjects(event.argumentArray), null); + } catch (Exception e) { + log.error("hbaseProcessor-assembleIdentity failed,event:{}", event, e); + return new Identity(InvokeType.HBASE.name(), "unknown" + ":" + "unknown", "unknown", null); + } + } + + + /** + * 组装请求参数 + * + * @param event 方法调用前事件 + * @return 请求参数数组，如果参数为空则返回null + */ + @Override + public Object[] assembleRequest(BeforeEvent event) { + if (event.argumentArray == null || event.argumentArray.length < 1) { + return null; + } + Object argument = event.argumentArray[0]; + if (argument instanceof List) { + List list = (List) argument; + List requests = list.stream().map((Function) this::assembleRequest).collect(Collectors.toList()); + if (event.javaMethodName.contains("batch")) { + return new Object[]{requests, event.argumentArray[1]}; + } else { + return new Object[]{requests}; + } + } else { + return new Object[]{assembleRequest(argument)}; + } + } + + /** + * 组装模拟响应 + * + * @param event 前置事件对象 + * @param invocation 调用对象 + * @return 组装后的模拟响应，如果uri包含"batch"则返回null，否则返回调用对象的响应结果 + */ + @Override + public Object assembleMockResponse(BeforeEvent event, Invocation invocation) { + String uri = invocation.getIdentity().getUri(); + if (uri.contains("batch")) { + Object[] recordResults = (Object[]) invocation.getRequest()[1]; + Object[] objects = (Object[]) event.argumentArray[1]; + System.arraycopy(recordResults, 0, objects, 0, objects.length); + return null; + } + return invocation.getResponse(); + } + + @Override + public boolean inTimeSerializeRequest(Invocation invocation, BeforeEvent event) { + return false; + } + + + /** + * 组装请求参数 + * + * @param data 请求数据对象 + * @return 组装好的请求参数Map，包含row和familyMap两个键值对，其中familyMap是一个嵌套的Map结构， + * key为列族名称，value为该列族下所有列的键值对。 + */ + @SuppressWarnings("unchecked") + private Map assembleRequest(Object data) { + try { + Map getReq = new HashMap<>(); + byte[] row = (byte[]) FieldUtils.readField(data, "row", true); + Map> familyMap = (Map>) FieldUtils.readField(data, "familyMap", true); + Map> familyQualifierValueMap = new HashMap<>(); + if (familyMap != null && !familyMap.isEmpty()) { + familyMap.forEach((bytes, objects) -> { + Map qualifierValueMap = new HashMap<>(); + familyQualifierValueMap.put(Bytes.toString(bytes), qualifierValueMap); + if (objects != null && !objects.isEmpty()) { + objects.forEach(object -> { + try { + int qualifierOffset = (int) MethodUtils.invokeMethod(object, "getQualifierOffset"); + int qualifierLength = (int) MethodUtils.invokeMethod(object, "getQualifierLength"); + int valueOffset = (int) MethodUtils.invokeMethod(object, "getValueOffset"); + int valueLength = (int) MethodUtils.invokeMethod(object, "getValueLength"); + byte[] value = (byte[]) MethodUtils.invokeMethod(object, "getValueArray"); + qualifierValueMap.put(Bytes.toString(value, qualifierOffset, qualifierLength), + Bytes.toString(value, valueOffset, valueLength)); + } catch (Exception ignored) { + + } + + }); + } + }); + } + getReq.put("row", Bytes.toString(row)); + getReq.put("familyMap", familyQualifierValueMap); + return getReq; + } catch (Exception e) { + log.error("hbaseProcessor-assembleRequest failed,data:{}", data, e); + return null; + } + } +} diff --git a/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/tars-plugin/pom.xml b/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/tars-plugin/pom.xml new file mode 100644 index 0000000..d2c2605 --- /dev/null +++ b/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/tars-plugin/pom.xml @@ -0,0 +1,44 @@ + + + + repeater-plugins + com.vivo.jvm.sandbox + 1.0.0-SNAPSHOT + + 4.0.0 + + tars-plugin + + + com.google.protobuf + protobuf-java + 2.3.0 + provided + + + + ${project.name}-${project.version} + + + org.apache.maven.plugins + maven-assembly-plugin + + + + attached + + package + + + jar-with-dependencies + + + + + + + + + \ No newline at end of file diff --git a/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/tars-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/tars/TarsClientPlugin.java b/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/tars-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/tars/TarsClientPlugin.java new file mode 100644 index 0000000..85db80b --- /dev/null +++ b/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/tars-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/tars/TarsClientPlugin.java @@ -0,0 +1,69 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +package com.alibaba.jvm.sandbox.repeater.plugin.tars; + +import com.alibaba.jvm.sandbox.api.event.Event; +import com.alibaba.jvm.sandbox.repeater.plugin.api.InvocationProcessor; +import com.alibaba.jvm.sandbox.repeater.plugin.core.impl.AbstractInvokePluginAdapter; +import com.alibaba.jvm.sandbox.repeater.plugin.core.model.EnhanceModel; +import com.alibaba.jvm.sandbox.repeater.plugin.domain.InvokeType; +import com.alibaba.jvm.sandbox.repeater.plugin.spi.InvokePlugin; +import com.google.common.collect.Lists; +import org.kohsuke.MetaInfServices; + +import java.util.List; + +/** + * TarsClientPlugin - tars客户端增强器 + */ +@MetaInfServices(InvokePlugin.class) +public class TarsClientPlugin extends AbstractInvokePluginAdapter { + /** + * 获取增强模型列表 + * + * @return 增强模型列表 + */ + @Override + protected List getEnhanceModels() { + EnhanceModel enhanceModel = EnhanceModel.builder().classPattern("com.qq.tars.client.ObjectProxy") + .methodPatterns(EnhanceModel.MethodPattern.transform("invoke")) + .watchTypes(Event.Type.BEFORE, Event.Type.RETURN, Event.Type.THROWS) + .build(); + + return Lists.newArrayList(enhanceModel); + } + + @Override + protected InvocationProcessor getInvocationProcessor() { + return new TarsInvocationProcessor(getType()); + } + + @Override + public InvokeType getType() { + return InvokeType.TARS_CLIENT; + } + + @Override + public String identity() { + return "tars-client"; + } + + @Override + public boolean isEntrance() { + return false; + } +} diff --git a/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/tars-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/tars/TarsInvocationProcessor.java b/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/tars-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/tars/TarsInvocationProcessor.java new file mode 100644 index 0000000..808c272 --- /dev/null +++ b/moonbox-agent/moonbox-java-agent/moonbox-plugins/moonbox/tars-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/tars/TarsInvocationProcessor.java @@ -0,0 +1,70 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ + +package com.alibaba.jvm.sandbox.repeater.plugin.tars; + +import com.alibaba.jvm.sandbox.api.event.BeforeEvent; +import com.alibaba.jvm.sandbox.repeater.plugin.core.impl.api.DefaultInvocationProcessor; +import com.alibaba.jvm.sandbox.repeater.plugin.core.util.LogUtil; +import com.alibaba.jvm.sandbox.repeater.plugin.domain.Identity; +import com.alibaba.jvm.sandbox.repeater.plugin.domain.InvokeType; + +import java.lang.reflect.Method; + +/** + * TarsInvocationProcessor - tars增强逻辑处理器 + */ +public class TarsInvocationProcessor extends DefaultInvocationProcessor { + public TarsInvocationProcessor(InvokeType type) { + super(type); + } + + /** + * 组装身份信息 + * + * @param event 事件对象 + * @return 身份信息 + */ + @Override + public Identity assembleIdentity(BeforeEvent event) { + try { + if(event.argumentArray == null || event.argumentArray.length !=3){ + return null; + } + Method method = (Method) event.argumentArray[1]; + String serviceName =method.getDeclaringClass().getCanonicalName(); + String methodName=method.getName(); + return new Identity(InvokeType.TARS_CLIENT.name(), serviceName, methodName, null); + }catch (Throwable e){ + LogUtil.error(e.getMessage(),e); + return new Identity(InvokeType.TARS_CLIENT.name(), "getObjectName", null, null); + } + } + + /** + * 组装请求参数 + * + * @param event 事件对象 + * @return 请求参数数组，如果参数不符合要求则返回null + */ + @Override + public Object[] assembleRequest(BeforeEvent event) { + if(event.argumentArray == null || event.argumentArray.length !=3){ + return null; + } + return (Object[]) event.argumentArray[2]; + } +} diff --git a/moonbox-agent/moonbox-java-agent/moonbox-plugins/pom.xml b/moonbox-agent/moonbox-java-agent/moonbox-plugins/pom.xml index 9a69291..0acd568 100644 --- a/moonbox-agent/moonbox-java-agent/moonbox-plugins/pom.xml +++ b/moonbox-agent/moonbox-java-agent/moonbox-plugins/pom.xml @@ -28,7 +28,8 @@ moonbox/spring-session-plugin moonbox/universal-plugin moonbox/motan-plugin - + moonbox/tars-plugin + moonbox/hbase-plugin repeater/dubbo-plugin repeater/hibernate-plugin repeater/http-plugin diff --git a/moonbox-agent/moonbox-java-agent/moonbox-plugins/repeater/redis-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/redis/RedissonPlugin.java b/moonbox-agent/moonbox-java-agent/moonbox-plugins/repeater/redis-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/redis/RedissonPlugin.java new file mode 100644 index 0000000..ab776fc --- /dev/null +++ b/moonbox-agent/moonbox-java-agent/moonbox-plugins/repeater/redis-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/redis/RedissonPlugin.java @@ -0,0 +1,71 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +package com.alibaba.jvm.sandbox.repeater.plugin.redis; + +import com.alibaba.jvm.sandbox.api.event.Event.Type; +import com.alibaba.jvm.sandbox.repeater.plugin.api.InvocationProcessor; +import com.alibaba.jvm.sandbox.repeater.plugin.core.impl.spi.AbstractInvokePluginAdapter; +import com.alibaba.jvm.sandbox.repeater.plugin.core.model.EnhanceModel; +import com.alibaba.jvm.sandbox.repeater.plugin.spi.InvokePlugin; +import com.google.common.collect.Lists; +import com.vivo.internet.moonbox.common.api.model.InvokeType; +import org.kohsuke.MetaInfServices; + +import java.util.List; + +/** + * Redisson功能插件 + */ +@MetaInfServices(InvokePlugin.class) +public class RedissonPlugin extends AbstractInvokePluginAdapter { + + /** + * 获取增强模型列表 + * + * @return 增强模型列表 + */ + @Override + protected List getEnhanceModels() { + EnhanceModel em = EnhanceModel.builder() + .classPattern("org.redisson.RedissonBucket") + .methodPatterns(EnhanceModel.MethodPattern.transform("get")) + .watchTypes(Type.BEFORE, Type.RETURN, Type.THROWS) + .build(); + + return Lists.newArrayList(em); + } + + @Override + protected InvocationProcessor getInvocationProcessor() { + return new RedissonProcessor(getType()); + } + + @Override + public InvokeType getType() { + return InvokeType.REDIS; + } + + @Override + public String identity() { + return "redis"; + } + + @Override + public boolean isEntrance() { + return false; + } + +} diff --git a/moonbox-agent/moonbox-java-agent/moonbox-plugins/repeater/redis-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/redis/RedissonProcessor.java b/moonbox-agent/moonbox-java-agent/moonbox-plugins/repeater/redis-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/redis/RedissonProcessor.java new file mode 100644 index 0000000..f89a3b3 --- /dev/null +++ b/moonbox-agent/moonbox-java-agent/moonbox-plugins/repeater/redis-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/redis/RedissonProcessor.java @@ -0,0 +1,57 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +package com.alibaba.jvm.sandbox.repeater.plugin.redis; + +import com.alibaba.jvm.sandbox.api.event.BeforeEvent; +import com.alibaba.jvm.sandbox.repeater.plugin.core.impl.api.DefaultInvocationProcessor; +import com.alibaba.jvm.sandbox.repeater.plugin.core.utils.MoonboxLogUtils; +import com.vivo.internet.moonbox.common.api.model.Invocation; +import com.vivo.internet.moonbox.common.api.model.InvokeType; +import org.apache.commons.lang3.reflect.FieldUtils; + +/** + * RedissonProcessor 新的处理器 + */ +public class RedissonProcessor extends DefaultInvocationProcessor { + public RedissonProcessor(InvokeType type) { + super(type); + } + + /** + * 组装请求参数 + * + * @param event 事件对象 + * @return 请求参数数组，包含目标对象的名称 + */ + @Override + public Object[] assembleRequest(BeforeEvent event) { + Object target = event.target; + try { + Object name = FieldUtils.readField(target, "name", true); + return new Object[]{name}; + } catch (Exception e) { + MoonboxLogUtils.warn("redisson assembleRequest failed.", e); + return new Object[]{}; + } + } + + + @Override + public boolean inTimeSerializeRequest(Invocation invocation, BeforeEvent event) { + return false; + } + +} diff --git a/moonbox-agent/moonbox-java-agent/moonbox-plugins/repeater/redis-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/redis/StringRedisPlugin.java b/moonbox-agent/moonbox-java-agent/moonbox-plugins/repeater/redis-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/redis/StringRedisPlugin.java new file mode 100644 index 0000000..543f4a4 --- /dev/null +++ b/moonbox-agent/moonbox-java-agent/moonbox-plugins/repeater/redis-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/redis/StringRedisPlugin.java @@ -0,0 +1,77 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +package com.alibaba.jvm.sandbox.repeater.plugin.redis; + +import com.alibaba.jvm.sandbox.api.event.Event.Type; +import com.alibaba.jvm.sandbox.repeater.plugin.api.InvocationProcessor; +import com.alibaba.jvm.sandbox.repeater.plugin.core.impl.spi.AbstractInvokePluginAdapter; +import com.alibaba.jvm.sandbox.repeater.plugin.core.model.EnhanceModel; +import com.alibaba.jvm.sandbox.repeater.plugin.spi.InvokePlugin; +import com.google.common.collect.Lists; +import com.vivo.internet.moonbox.common.api.model.InvokeType; +import org.kohsuke.MetaInfServices; + +import java.util.List; + +/** + * string增强插件 + */ +@MetaInfServices(InvokePlugin.class) +public class StringRedisPlugin extends AbstractInvokePluginAdapter { + + /** + * 获取增强模型列表 + * + * @return 增强模型列表 + */ + @Override + protected List getEnhanceModels() { + EnhanceModel em = EnhanceModel.builder() + .classPattern("org.springframework.data.redis.core.DefaultValueOperations") + .methodPatterns(EnhanceModel.MethodPattern.transform("get","getAndSet","increment","decrement","append","multiGet","size")) + .watchTypes(Type.BEFORE, Type.RETURN, Type.THROWS) + .build(); + + EnhanceModel em1 = EnhanceModel.builder() + .classPattern("org.springframework.data.redis.core.DefaultHashOperations") + .methodPatterns(EnhanceModel.MethodPattern.transform("get","hasKey","increment","decrement","append","keys","size","entries")) + .watchTypes(Type.BEFORE, Type.RETURN, Type.THROWS) + .build(); + + return Lists.newArrayList(em,em1); + } + + @Override + protected InvocationProcessor getInvocationProcessor() { + return new StringRedisProcessor(getType()); + } + + @Override + public InvokeType getType() { + return InvokeType.REDIS; + } + + @Override + public String identity() { + return "redis"; + } + + @Override + public boolean isEntrance() { + return false; + } + +} diff --git a/moonbox-agent/moonbox-java-agent/moonbox-plugins/repeater/redis-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/redis/StringRedisProcessor.java b/moonbox-agent/moonbox-java-agent/moonbox-plugins/repeater/redis-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/redis/StringRedisProcessor.java new file mode 100644 index 0000000..ea4881d --- /dev/null +++ b/moonbox-agent/moonbox-java-agent/moonbox-plugins/repeater/redis-plugin/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/redis/StringRedisProcessor.java @@ -0,0 +1,37 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +package com.alibaba.jvm.sandbox.repeater.plugin.redis; + +import com.alibaba.jvm.sandbox.api.event.BeforeEvent; +import com.alibaba.jvm.sandbox.repeater.plugin.core.impl.api.DefaultInvocationProcessor; +import com.vivo.internet.moonbox.common.api.model.Invocation; +import com.vivo.internet.moonbox.common.api.model.InvokeType; + +/** + * RedissonProcessor 新的处理器 + */ +public class StringRedisProcessor extends DefaultInvocationProcessor { + + public StringRedisProcessor(InvokeType type) { + super(type); + } + + @Override + public boolean inTimeSerializeRequest(Invocation invocation, BeforeEvent event) { + return false; + } + +} diff --git a/moonbox-common/moonbox-data/src/main/java/com/vivo/internet/moonbox/common/api/model/AbstractRecordInterface.java b/moonbox-common/moonbox-data/src/main/java/com/vivo/internet/moonbox/common/api/model/AbstractRecordInterface.java index 14bdbba..eb5a4d6 100644 --- a/moonbox-common/moonbox-data/src/main/java/com/vivo/internet/moonbox/common/api/model/AbstractRecordInterface.java +++ b/moonbox-common/moonbox-data/src/main/java/com/vivo/internet/moonbox/common/api/model/AbstractRecordInterface.java @@ -21,10 +21,6 @@ /** * AbstractRecordInterface - {@link AbstractRecordInterface} - * - * @author yanjiang.liu - * @version 1.0 - * @since 2022/8/23 14:39 */ @Data public abstract class AbstractRecordInterface implements Serializable { @@ -33,9 +29,21 @@ public abstract class AbstractRecordInterface implements Serializable { private String sampleRate = "10000"; + private String analysisFields; + + /** + * 流量去重配置字段 + */ + private String uniqRecordDataFields; + + /** + * 流量去重配置字段（根据响应字段） + */ + private String uniqResponseDataFields; + /** * 获取接口唯一配置 - * + * * @return 唯一键 */ public abstract String getUniqueKey(); diff --git a/moonbox-common/moonbox-tools/pom.xml b/moonbox-common/moonbox-tools/pom.xml index 5003faf..9ffd42b 100644 --- a/moonbox-common/moonbox-tools/pom.xml +++ b/moonbox-common/moonbox-tools/pom.xml @@ -16,6 +16,12 @@ org.apache.commons commons-lang3 + + commons-codec + commons-codec + 1.11 + compile + com.google.guava guava @@ -24,6 +30,11 @@ com.alibaba fastjson + + com.fasterxml.jackson.core + jackson-databind + ${jackson-databind.version} + org.kohsuke.metainf-services metainf-services @@ -38,5 +49,11 @@ org.slf4j slf4j-api + + org.codehaus.jackson + jackson-mapper-asl + 1.9.13 + compile + \ No newline at end of file diff --git a/moonbox-common/moonbox-tools/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/util/EncodeUtils.java b/moonbox-common/moonbox-tools/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/util/EncodeUtils.java new file mode 100644 index 0000000..8e322bd --- /dev/null +++ b/moonbox-common/moonbox-tools/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/util/EncodeUtils.java @@ -0,0 +1,70 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +package com.alibaba.jvm.sandbox.repeater.plugin.util; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.codec.digest.DigestUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.charset.StandardCharsets; + +public final class EncodeUtils { + + private static final Logger LOG = LoggerFactory.getLogger(EncodeUtils.class); + + /** + * 计算给定数据的MD5哈希值 + * + * @param data 给定的数据 + * @return 数据的MD5哈希值，如果输入数据为null则返回null + */ + public static byte[] md5(byte[] data) { + if (data == null) { + return null; + } + return DigestUtils.md5(data); + } + + /** + * 计算给定字符串的MD5哈希值 + * + * @param data 给定的字符串 + * @return 字符串的MD5哈希值 + */ + public static String md5Hex(String data) { + if (data == null){ + return null; + } + return DigestUtils.md5Hex(data.getBytes(StandardCharsets.UTF_8)); + } + + /** + * 将字节数组进行Base64编码 + * + * @param data 待编码的字节数组 + * @return 编码后的字符串，若输入为null则返回null + */ + public static String base64Encode(byte[] data) { + if (data == null){ + return null; + } + return Base64.encodeBase64String(data); + + } + private EncodeUtils() { + } +} diff --git a/moonbox-common/moonbox-tools/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/util/JacksonUtils.java b/moonbox-common/moonbox-tools/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/util/JacksonUtils.java new file mode 100644 index 0000000..0eeba85 --- /dev/null +++ b/moonbox-common/moonbox-tools/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/util/JacksonUtils.java @@ -0,0 +1,84 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +package com.alibaba.jvm.sandbox.repeater.plugin.util; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.extern.slf4j.Slf4j; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; + +/** + * JacksonUtils - {@link JacksonUtils} + */ +@SuppressWarnings({"rawtypes", "unchecked"}) +@Slf4j +public class JacksonUtils { + + private static final ObjectMapper mapper = new ObjectMapper(); + + static { + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false); + mapper.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); + mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + mapper.setDateFormat(sdf); + } + + /** + * 将对象序列化为字符串 + * + * @param object 要序列化的对象 + * @return 序列化后的字符串 + * @throws Exception 序列化异常 + */ + public static String serialize(Object object) throws Exception { + return serialize(object, true); + } + + /** + * 将对象序列化为JSON字符串 + * + * @param object 需要序列化的对象 + * @param pretty 是否需要格式化输出 + * @return 序列化后的JSON字符串 + * @throws Exception 序列化异常 + */ + public static String serialize(Object object, boolean pretty) throws Exception { + try { + return pretty ? mapper.writerWithDefaultPrettyPrinter().writeValueAsString(object) : mapper.writeValueAsString(object); + } catch (JsonProcessingException e) { + log.error("jackSonUtil-serialize failed,object:{}, pretty:{}", object, pretty, e); + throw new Exception("jackson-serialize-error", e); + } + } + + /** + * 获取集合类型的JavaType对象 + * + * @param elementClasses 集合元素的Class对象数组 + * @return JavaType对象，表示指定集合类型和元素类型的集合 + */ + @SuppressWarnings("deprecation") + private static JavaType getCollectionType(Class... elementClasses) { + return mapper.getTypeFactory().constructParametrizedType(ArrayList.class, ArrayList.class, elementClasses); + } + +} diff --git a/moonbox-common/moonbox-tools/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/util/JsonUtils.java b/moonbox-common/moonbox-tools/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/util/JsonUtils.java new file mode 100644 index 0000000..cb3b92a --- /dev/null +++ b/moonbox-common/moonbox-tools/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/util/JsonUtils.java @@ -0,0 +1,109 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +package com.alibaba.jvm.sandbox.repeater.plugin.util; + +import com.google.common.base.Strings; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.JsonParser; +import org.codehaus.jackson.map.DeserializationConfig; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.type.JavaType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +public class JsonUtils { + + private static final Logger logger = LoggerFactory.getLogger(JsonUtils.class); + + private static final ObjectMapper objectMapper; + + static { + objectMapper = new ObjectMapper(); + objectMapper.configure(JsonParser.Feature.ALLOW_SINGLE_QUOTES, true); + objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); + objectMapper.configure(JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, true); + objectMapper.configure(JsonParser.Feature.ALLOW_BACKSLASH_ESCAPING_ANY_CHARACTER, true); + objectMapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); + objectMapper.configure(DeserializationConfig.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true); + } + + public static T readObject(String json, Class clz)throws Exception{ + if (Strings.isNullOrEmpty(json)) { + return null; + } + try { + return getJsonMapper().readValue(json, clz); + } catch (Exception e) { + logger.error("jackSonUtil-serialize failed,json:{}, clz:{}", json, clz, e); + throw new Exception("readObject出错：" + e.getMessage() + " json:" + json + " clz:" + clz, e); + } + } + + public static T readObject(String json,Classclazz0,Class...clazz) throws IOException { + JavaType javaType = getJsonMapper().getTypeFactory().constructParametricType(clazz0,clazz); + return getJsonMapper().readValue(json,javaType); + } + + + public static ObjectMapper getJsonMapper() { + return objectMapper; + } + + @SuppressWarnings("unchecked") + public static T getProperty(String json, String propertyName, Class clazz) throws Exception { + try { + JsonNode readTree = getJsonMapper().readTree(json); + String[] hirarchyProperties = propertyName.split("\\."); + int i=0; + for (String property : hirarchyProperties) { + i++; + if(readTree.has(property)){ + readTree = readTree.get(property); + }else{ + if(i!=hirarchyProperties.length){ + logger.error("encounter unexpected internal Property:" + propertyName + " json: " + json + " clz:" + clazz.getName()); + throw new Exception("enconter unexpected internal internal property!"); + }else{ + return null; + } + } + } + if (clazz == Long.class) { + return (T) Long.valueOf(readTree.asLong()); + } else if (clazz == Integer.class) { + return (T) Integer.valueOf(readTree.asInt()); + } else if (clazz == Float.class) { + return (T) new Float(readTree.asDouble()); + } else if (clazz == Double.class) { + return (T) new Double(readTree.asDouble()); + } else if (clazz == String.class) { + return (T) readTree.asText(); + } else if(clazz == Boolean.class) { + return (T) Boolean.valueOf(readTree.asBoolean()); + }else{ + return (T) readTree; + } + } catch (Exception e) { + logger.error("getProperty properites:" + propertyName + " json: " + json + " clz:" + clazz.getName(), e); + throw new Exception("getProperty failed",e); + } + } + + + +} \ No newline at end of file diff --git a/moonbox-common/moonbox-tools/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/util/UriUtils.java b/moonbox-common/moonbox-tools/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/util/UriUtils.java new file mode 100644 index 0000000..aa754c3 --- /dev/null +++ b/moonbox-common/moonbox-tools/src/main/java/com/alibaba/jvm/sandbox/repeater/plugin/util/UriUtils.java @@ -0,0 +1,52 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +package com.alibaba.jvm.sandbox.repeater.plugin.util; + + +import org.apache.commons.lang3.StringUtils; + +/** + * UriUtils - 接口类 + */ +public class UriUtils { + + /** + * 获取接口类型 + * + * @param uri 通过uri来获取接口类型 + * @return + */ + public static String getUriType(String uri) { + if (StringUtils.isNotBlank(uri) && uri.contains("://")) { + return uri.substring(0, uri.indexOf("://")); + } + return ""; + } + + + /** + * 获取接口类型 + * + * @param uri 通过uri来获取接口类型 + * @return + */ + public static String getUriPath(String uri) { + if (uri.contains("//")) { + return uri.substring(uri.indexOf("//")+2); + } + return uri; + } +} diff --git a/moonbox-server/moonbox-service-agent/pom.xml b/moonbox-server/moonbox-service-agent/pom.xml index 54063f2..1039082 100644 --- a/moonbox-server/moonbox-service-agent/pom.xml +++ b/moonbox-server/moonbox-service-agent/pom.xml @@ -21,6 +21,11 @@ com.vivo.internet 1.0.0-SNAPSHOT + + moonbox-service-data-redis + com.vivo.internet + 1.0.0-SNAPSHOT + moonbox-service-data com.vivo.internet diff --git a/moonbox-server/moonbox-service-agent/src/main/java/com/vivo/internet/moonbox/service/agent/record/service/impl/RecordServiceImpl.java b/moonbox-server/moonbox-service-agent/src/main/java/com/vivo/internet/moonbox/service/agent/record/service/impl/RecordServiceImpl.java index 9e284a0..3603d4d 100644 --- a/moonbox-server/moonbox-service-agent/src/main/java/com/vivo/internet/moonbox/service/agent/record/service/impl/RecordServiceImpl.java +++ b/moonbox-server/moonbox-service-agent/src/main/java/com/vivo/internet/moonbox/service/agent/record/service/impl/RecordServiceImpl.java @@ -15,11 +15,15 @@ */ package com.vivo.internet.moonbox.service.agent.record.service.impl; +import com.vivo.internet.moonbox.common.api.serialize.Serializer; +import com.vivo.internet.moonbox.common.api.serialize.SerializerProvider; +import com.vivo.internet.moonbox.redis.RecordRedisService; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.BeanUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.Assert; +import com.alibaba.jvm.sandbox.repeater.plugin.util.JacksonUtils; import com.alibaba.fastjson.JSON; import com.vivo.internet.moonbox.common.api.dto.MoonBoxResult; @@ -36,10 +40,6 @@ /** * RecordServiceImpl - 流量录制服务 - * - * @author xu.kai - * @version 1.0 - * @since 2022/9/5 15:49 */ @Service @Slf4j @@ -49,6 +49,9 @@ public class RecordServiceImpl implements RecordService { private final RecordDataService recordDataService; + @Autowired(required = false) + private RecordRedisService recordRedisService; + @Autowired public RecordServiceImpl(TaskConfigService taskConfigService, RecordDataService recordDataService) { this.taskConfigService = taskConfigService; @@ -84,6 +87,20 @@ public MoonBoxResult saveRecord(String body) { log.error("deserialize response body failed, response:{}.",wrapper.getEntranceInvocation().getResponseSerialized(), e); } + Object[] objects = SerializerProvider.instance().provide(Serializer.Type.HESSIAN).deserialize(wrapper.getEntranceInvocation() + .getRequestSerialized(), Object[].class); + try{ + entity.setRequest(JacksonUtils.serialize(objects)); + } catch (Exception e) { + log.error("deserialize request body failed, body:{}.",wrapper.getEntranceInvocation().getRequestSerialized(), e); + } + + //请求重复校验(不同类型、不同接口的核心字段配置请在web后台任务创建时自行选择) + String errMsg = recordRedisService == null ? "" :recordRedisService.judgeSave(wrapper, agentConfig.getRecordAgentConfig(), entity); + if (StringUtils.isNotEmpty(errMsg)) { + return MoonBoxResult.createFailResponse(errMsg); + } + boolean isSuccess = recordDataService.saveData(entity); return isSuccess ? MoonBoxResult.createSuccess("-/-") : MoonBoxResult.createFailResponse("failed"); } diff --git a/moonbox-server/moonbox-service-data-redis/pom.xml b/moonbox-server/moonbox-service-data-redis/pom.xml new file mode 100644 index 0000000..458e206 --- /dev/null +++ b/moonbox-server/moonbox-service-data-redis/pom.xml @@ -0,0 +1,61 @@ + + + 4.0.0 + + com.vivo.internet + moonbox-service-data-redis + 1.0.0-SNAPSHOT + + + + + org.springframework.boot + spring-boot-starter-data-redis + 2.5.12 + + + + redis.clients + jedis + 3.7.0 + + + + com.jayway.jsonpath + json-path + 2.2.0 + + + + com.vivo.internet + moonbox-data + 1.0.0-SNAPSHOT + + + + com.vivo.internet + moonbox-service-data + 1.0.0-SNAPSHOT + compile + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.5.1 + + 8 + 8 + UTF-8 + + + + + + \ No newline at end of file diff --git a/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/RecordRedisService.java b/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/RecordRedisService.java new file mode 100644 index 0000000..64750f0 --- /dev/null +++ b/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/RecordRedisService.java @@ -0,0 +1,70 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +package com.vivo.internet.moonbox.redis; + +import com.vivo.internet.moonbox.common.api.model.RecordAgentConfig; +import com.vivo.internet.moonbox.common.api.model.RecordWrapper; +import com.vivo.internet.moonbox.redis.dto.UniqModel; +import com.vivo.internet.moonbox.service.data.model.record.RecordWrapperEntity; + +import java.util.List; + +public interface RecordRedisService { + + /** + * 判断是否需要保存流量记录 + * + * @param taskConfigResult 接口配置信息 + * @param esRecordEntity 请求对象实体 + * @return 判断结果，true表示需要保存，false表示不需要保存 + */ + UniqModel judgeNeedSaveRecord(RecordWrapper recordWrapper, RecordAgentConfig taskConfigResult, RecordWrapperEntity esRecordEntity); + + /** + * 更新唯一字符串到Redis中 + * + * @param uniqModel 唯一字符串模型 + */ + void updateUniqueStringToRedis(UniqModel uniqModel); + + /** + * 根据返回值判断是否需要保存流量记录 + * + * @param recordWrapper 流量记录包装器 + * @param taskConfig 接口配置 + * @param esRecordEntity 请求对象 + * @return 是否需要保存流量记录的唯一模型 + */ + String judgeSave(RecordWrapper recordWrapper, RecordAgentConfig taskConfig, RecordWrapperEntity esRecordEntity); + + /** + * 将任务执行记录的追踪信息保存到Redis中 + * + * @param taskRunId 任务执行记录ID + * @param traceId 追踪信息ID + */ + void saveRecordTraceToRedis(String taskRunId, String traceId); + + /** + * 从Redis中获取录制任务的记录轨迹 + * + * @param replayTaskRunId 回放任务运行ID + * @param recordTaskRunId 录制任务运行ID + * @param size 获取的记录轨迹数量 + * @return 记录轨迹列表 + */ + List getRecordTracesFromRedis(String replayTaskRunId, String recordTaskRunId, int size); +} diff --git a/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/config/JedisClusterConfig.java b/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/config/JedisClusterConfig.java new file mode 100644 index 0000000..9f661dc --- /dev/null +++ b/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/config/JedisClusterConfig.java @@ -0,0 +1,63 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +package com.vivo.internet.moonbox.redis.config; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.JedisCluster; + +import javax.annotation.Resource; +import java.util.HashSet; +import java.util.Set; +@Slf4j +@Configuration +public class JedisClusterConfig { + + @Resource + private RedisProperties redisProperties; + + /** + * 获取JedisCluster实例 + * 注意：返回的JedisCluster是单例的，并且可以直接注入到其他类中去使用 + * 当前只实现了redis集群模式，如果单节点部署，需要自行添加 + * @return JedisCluster实例 + */ + @Bean + @Conditional(RedisPropCondition.class) + public JedisCluster getJedisCluster() { + try { + //获取服务器数组(这里要相信自己的输入，所以没有考虑空指针问题) + String[] serverArray = redisProperties.getClusterNodes().split(","); + Set nodes = new HashSet<>(); + + // 遍历每个节点的地址，将其解析成主机名和端口号，并添加到nodes中保存 + for (String ipPort : serverArray) { + String[] ipPortPair = ipPort.split(":"); + nodes.add(new HostAndPort(ipPortPair[0].trim(), Integer.parseInt(ipPortPair[1].trim()))); + } + // 创建一个JedisCluster实例，并设置相关的参数，比如nodes、commandTimeout、poolConfig等 + return new JedisCluster(nodes, redisProperties.getCommandTimeout(), 1000, 1, + redisProperties.getPassword(), new GenericObjectPoolConfig<>()); + } catch (Exception e) { + log.error("redis config init failed", e); + return null; + } + } +} \ No newline at end of file diff --git a/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/config/RedisPropCondition.java b/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/config/RedisPropCondition.java new file mode 100644 index 0000000..669ecb1 --- /dev/null +++ b/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/config/RedisPropCondition.java @@ -0,0 +1,36 @@ +package com.vivo.internet.moonbox.redis.config; + +import org.springframework.boot.autoconfigure.condition.ConditionOutcome; +import org.springframework.boot.autoconfigure.condition.SpringBootCondition; +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; + +import java.util.regex.Pattern; + +/** + * 条件 + */ +public class RedisPropCondition extends SpringBootCondition implements Condition { + + //IP-PORT校验 + private static final String IP_PORT_PATTERN = + "^((\\d{1,3}\\.){3}\\d{1,3}:\\d{1,5},)*(\\d{1,3}\\.){3}\\d{1,3}:\\d{1,5}$"; + + /** + * 获取匹配结果 + * + * @param context 条件上下文 + * @param metadata 注释类型元数据 + * @return 匹配结果 + */ + @Override + public ConditionOutcome getMatchOutcome(ConditionContext context, AnnotatedTypeMetadata metadata) { + Environment env = context.getEnvironment(); + // 使用环境变量来决定条件是否满足 + String myProperty = env.getProperty("spring.redis.clusterNodes"); + //IP-PORT校验 + return new ConditionOutcome(Pattern.matches(IP_PORT_PATTERN, myProperty), ""); + } +} diff --git a/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/config/RedisProperties.java b/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/config/RedisProperties.java new file mode 100644 index 0000000..58295bd --- /dev/null +++ b/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/config/RedisProperties.java @@ -0,0 +1,37 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +package com.vivo.internet.moonbox.redis.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * JEDIS的属性配置，目前只给了jedis的客户端配置 + * 使用cluster模式，单节点需自行修改 + * 默认无密码方式，可以更改配置 + */ +@Data +@Component +@ConfigurationProperties(prefix = "spring.redis") +public class RedisProperties { + + private int expireSeconds; + private String clusterNodes; + private String password = null; + private String host; + private int commandTimeout; +} \ No newline at end of file diff --git a/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/dto/UniqModel.java b/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/dto/UniqModel.java new file mode 100644 index 0000000..927582d --- /dev/null +++ b/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/dto/UniqModel.java @@ -0,0 +1,31 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +package com.vivo.internet.moonbox.redis.dto; + +import lombok.Builder; +import lombok.Data; + +/** + * UniqModel - 去重信息 + */ +@Data +@Builder +public class UniqModel { + + private boolean save; + + private String uniqKey; +} diff --git a/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/impl/RecordRedisServiceImpl.java b/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/impl/RecordRedisServiceImpl.java new file mode 100644 index 0000000..a81fd8d --- /dev/null +++ b/moonbox-server/moonbox-service-data-redis/src/main/java/com/vivo/internet/moonbox/redis/impl/RecordRedisServiceImpl.java @@ -0,0 +1,265 @@ +/* +Copyright 2022 vivo Communication Technology Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + */ +package com.vivo.internet.moonbox.redis.impl; + +import com.alibaba.fastjson.JSON; +import com.google.common.base.Splitter; +import com.google.common.collect.Lists; +import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.JsonPath; +import com.vivo.internet.moonbox.common.api.model.*; +import com.vivo.internet.moonbox.common.api.serialize.Serializer; +import com.vivo.internet.moonbox.common.api.serialize.SerializerProvider; +import com.vivo.internet.moonbox.redis.RecordRedisService; +import com.vivo.internet.moonbox.redis.config.RedisPropCondition; +import com.vivo.internet.moonbox.redis.dto.UniqModel; +import com.alibaba.jvm.sandbox.repeater.plugin.util.EncodeUtils; +import com.alibaba.jvm.sandbox.repeater.plugin.util.JsonUtils; +import com.alibaba.jvm.sandbox.repeater.plugin.util.UriUtils; +import com.vivo.internet.moonbox.service.data.model.record.RecordWrapperEntity; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Conditional; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; +import redis.clients.jedis.JedisCluster; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 上报信息防重复缓存实现，面向多场景，目前唯一性校验规则写死，可后期扩展 + * 采用摘要方式避免redisIO抖动 + */ +@Service +@Slf4j +@Conditional(RedisPropCondition.class) +public class RecordRedisServiceImpl implements RecordRedisService { + + @Autowired(required = false) + private JedisCluster jedisCluster; + + private static final Splitter sp= Splitter.on(',').omitEmptyStrings().trimResults(); + + /** + * 判断是否需要保存记录 + * + * @param recordWrapper 记录包装器 + * @param recordTaskConfig 记录任务配置 + * @param esRecordEntity ES记录实体 + * @return 唯一性模型，包括是否需要保存、唯一键等信息 + */ + @Override + public UniqModel judgeNeedSaveRecord(RecordWrapper recordWrapper, RecordAgentConfig recordTaskConfig, RecordWrapperEntity esRecordEntity) { + AbstractRecordInterface recordInterface = findConfig(recordTaskConfig, esRecordEntity); + if(recordInterface == null){ + return UniqModel.builder().save(true).build(); + } + if(StringUtils.isBlank(recordInterface.getUniqRecordDataFields())){ + return UniqModel.builder().save(true).build(); + } + List fields =sp.splitToList(recordInterface.getUniqRecordDataFields()); + try { + Listvalues=getRequestFields(recordWrapper, fields,esRecordEntity.getRequest()); + //构造redis的key + String key= getKey(esRecordEntity.getTaskRunId(),esRecordEntity.getEntranceDesc(),values); + return UniqModel.builder().save( !jedisCluster.exists(key)).uniqKey(key).build(); + }catch (Throwable e){ + log.error("taskRunId:"+recordWrapper.getTaskRunId()+" entranceUri:"+recordInterface.getUniqueKey()+" "+e.getMessage(),e); + return UniqModel.builder().save(true).build(); + } + } + + /** + * 判断是否需要保存记录 + * + * @param recordWrapper 记录包装器 + * @param recordTaskConfig 记录任务配置 + * @param esRecordEntity ES记录实体 + * @return 是否需要保存记录以及唯一键信息 + */ + @Override + public String judgeSave(RecordWrapper recordWrapper, RecordAgentConfig recordTaskConfig, RecordWrapperEntity esRecordEntity) { + AbstractRecordInterface recordInterface = findConfig(recordTaskConfig, esRecordEntity); + if(recordInterface == null){ + return null; + } + if(StringUtils.isBlank(recordInterface.getUniqResponseDataFields()) && StringUtils.isBlank(recordInterface.getUniqRecordDataFields())){ + return null; + } + try { + String str = recordInterface.getUniqRecordDataFields(); + if(!StringUtils.isBlank(str)) { + List fields = sp.splitToList(str); + List values = getRequestFields(recordWrapper, fields, esRecordEntity.getRequest()); + //构造redis的key + String key = getKey(esRecordEntity.getTaskRunId(), esRecordEntity.getEntranceDesc(), values); + //生成一个结果集 + UniqModel uniqModel = UniqModel.builder().save(!jedisCluster.exists(key)).uniqKey(key).build(); + if (!uniqModel.isSave()) { + return "uniq request skip"; + } + //持久化 + updateUniqueStringToRedis(uniqModel); + } + + str = recordInterface.getUniqResponseDataFields(); + if(!StringUtils.isBlank(str)) { + List fields = sp.splitToList(str); + List values = getResponseFields(recordWrapper, fields, esRecordEntity.getResponse()); + //构造redis的key + String key = getResponseKey(esRecordEntity.getTaskRunId(), esRecordEntity.getEntranceDesc(), values); + //生成一个结果集 + UniqModel uniqModel = UniqModel.builder().save(!jedisCluster.exists(key)).uniqKey(key).build(); + if (!uniqModel.isSave()) { + return "uniq response skip"; + } + //持久化 + updateUniqueStringToRedis(uniqModel); + } + return null; + } catch (Throwable e) { + log.error("taskRunId:" + recordWrapper.getTaskRunId() + " entranceUri:" + recordInterface.getUniqueKey() + " " + e.getMessage(), e); + return null; + } + } + + + @Override + public void updateUniqueStringToRedis(UniqModel uniqModel) { + jedisCluster.setex(uniqModel.getUniqKey(),24 * 60 * 60,"XXXX"); + } + + @Override + public void saveRecordTraceToRedis(String taskRunId, String traceId) { + String cacheKey = "record_trace_prefix_" + taskRunId; + jedisCluster.rpush(cacheKey, traceId); + } + + @Override + public List getRecordTracesFromRedis(String replayTaskRunId, String recordTaskRunId, int size) { + String replayKey = "replay_position_cache_prefix_" + replayTaskRunId; + String position = jedisCluster.get(replayKey); + long startPos = 0L; + if (StringUtils.isNotBlank(position)) { + startPos = Long.parseLong(position); + } + long stopPos=startPos+size-1; + String recordKey = "record_trace_prefix_" + recordTaskRunId; + List result = jedisCluster.lrange(recordKey, startPos, stopPos); + + log.info("getRecordTracesFromRedis taskRunId:{},startPos:{},endPos:{},resultSize:{}",recordTaskRunId + ,startPos,stopPos,result.size()); + + if (CollectionUtils.isEmpty(result)) { + return result; + } + jedisCluster.incrBy(replayKey, result.size()); + return result; + } + + + private String getKey(String taskRunId,String entranceUri,Listvalues) { + return "unique_filter_" + EncodeUtils.md5Hex(taskRunId+"_"+entranceUri+"_"+JSON.toJSONString(values)); + } + private String getResponseKey(String taskRunId,String entranceUri,Listvalues) { + return "unique_filter_response" + EncodeUtils.md5Hex(taskRunId+"_"+entranceUri+"_"+JSON.toJSONString(values)); + } + + + /** + * 获取请求参数字段的值 + * + * @param recordWrapper 记录包装器 + * @param dataFields 请求参数字段列表 + * @param jsonRequest JSON格式的请求体 + * @return 请求参数字段的值列表 + * @throws Exception 异常信息 + */ + private static List getRequestFields(RecordWrapper recordWrapper,ListdataFields,String jsonRequest)throws Exception{ + log.info("getRequestFieldsStartParam"); + Listvalues; + if (recordWrapper.getEntranceInvocation().getType().getInvokeName().equals(InvokeType.HTTP.getInvokeName())) { + Serializer hessian = SerializerProvider.instance().provide(Serializer.Type.HESSIAN); + String requestSerialized = recordWrapper.getEntranceInvocation().getRequestSerialized(); + Object[] request = hessian.deserialize(requestSerialized, Object[].class); + Map requestMap = (Map) request[0]; + Map parseMap = new HashMap<>(); + parseMap.put("headers", requestMap.get("headers")); + parseMap.put("params", requestMap.get("paramsMap")); + String body = (String) requestMap.get("body"); + Map bodyMap = JsonUtils.readObject(body, Map.class); + parseMap.put("body", bodyMap); + String jsonData = JSON.toJSONString(parseMap); + DocumentContext documentContext = JsonPath.parse(jsonData); + List fieldValues = Lists.newArrayList(); + dataFields.forEach(s -> fieldValues.add(documentContext.read(s))); + return fieldValues; + } else { + DocumentContext documentContext = JsonPath.parse(jsonRequest); + values = Lists.newArrayList(); + List finalValues = values; + dataFields.forEach(s -> finalValues.add(documentContext.read(s))); + } + log.info("getRequestFieldsEndParam:{}",values); + return values; + + } + + private List getResponseFields(RecordWrapper recordWrapper,ListdataFields,String jsonRequest)throws Exception{ + Listvalues = Lists.newArrayList(); + DocumentContext documentContext = JsonPath.parse(jsonRequest); + dataFields.forEach(s -> values.add(documentContext.read(s))); + return values; + + } + + private AbstractRecordInterface findConfig(RecordAgentConfig recordTaskConfig, RecordWrapperEntity esRecordEntity){ + String entranceDesc = esRecordEntity.getEntranceDesc(); + String type = UriUtils.getUriType(entranceDesc); + String path = UriUtils.getUriPath(entranceDesc); + + if(InvokeType.HTTP.getInvokeName().equalsIgnoreCase(type)){ + for(HttpRecordInterface httpRecordInterface:recordTaskConfig.getHttpRecordInterfaces()){ + if(path.equalsIgnoreCase(httpRecordInterface.getUri())){ + return httpRecordInterface; + } + } + } + if(InvokeType.DUBBO.getInvokeName().equalsIgnoreCase(type)){ + for(DubboRecordInterface dubboRecordInterface:recordTaskConfig.getDubboRecordInterfaces()){ + String tmpPath= path.substring(0,path.indexOf("(")); + if(tmpPath.equals(dubboRecordInterface.getInterfaceName()+"/"+dubboRecordInterface.getMethodName())){ + return dubboRecordInterface; + } + } + } + if(InvokeType.JAVA.getInvokeName().equals(type)) { + for (JavaRecordInterface jri : recordTaskConfig.getJavaRecordInterfaces()) { + for (String methodName : jri.getMethodPatterns()) { + if(path.equals(jri.getClassPattern() + "/" + methodName + "()")){ + return jri; + } + } + } + } + return null; + } + + +} diff --git a/moonbox-server/moonbox-service-data/src/main/java/com/vivo/internet/moonbox/service/data/model/record/RecordWrapperEntity.java b/moonbox-server/moonbox-service-data/src/main/java/com/vivo/internet/moonbox/service/data/model/record/RecordWrapperEntity.java index 0fe84f4..0505294 100644 --- a/moonbox-server/moonbox-service-data/src/main/java/com/vivo/internet/moonbox/service/data/model/record/RecordWrapperEntity.java +++ b/moonbox-server/moonbox-service-data/src/main/java/com/vivo/internet/moonbox/service/data/model/record/RecordWrapperEntity.java @@ -21,10 +21,6 @@ /** * RecordWrapperEntity - 录制数据保存实体 - * - * @author xu.kai - * @version 1.0 - * @since 2022/9/6 17:36 */ @EqualsAndHashCode(callSuper = true) @Data @@ -36,4 +32,6 @@ public class RecordWrapperEntity extends RecordWrapper { private String response; + private String request; + } \ No newline at end of file diff --git a/moonbox-server/moonbox-web/src/main/resources/application.properties b/moonbox-server/moonbox-web/src/main/resources/application.properties index 1ed54a0..de8d0eb 100644 --- a/moonbox-server/moonbox-web/src/main/resources/application.properties +++ b/moonbox-server/moonbox-web/src/main/resources/application.properties @@ -23,4 +23,21 @@ sandbox.agent.download.uri=/api/agent/downLoadSandBoxZipFile moonbox.agent.download.uri=/api/agent/downLoadMoonBoxZipFile spring.servlet.multipart.max-file-size=100MB -spring.servlet.multipart.max-request-size=100MB \ No newline at end of file +spring.servlet.multipart.max-request-size=100MB + + +#redis +spring.redis.clusterNodes=172.22.225.9:6371,172.22.225.9:6372,172.22.225.9:6373 +#spring.redis.port=6379 +spring.redis.database=0 +#spring.redis.password= + +# Connection pool configuration +# Adjust these values based on your Redis server capacity and application needs +spring.redis.jedis.pool.max-active=8 +spring.redis.jedis.pool.max-idle=8 +spring.redis.jedis.pool.min-idle=2 +spring.redis.jedis.pool.max-wait=-1 + +# Enable auto-configuration for Spring Data Redis +spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration \ No newline at end of file diff --git a/moonbox-server/pom.xml b/moonbox-server/pom.xml index c22a0d3..4e1cfd8 100644 --- a/moonbox-server/pom.xml +++ b/moonbox-server/pom.xml @@ -19,6 +19,7 @@ moonbox-service-agent moonbox-service-data moonbox-service-data-es + moonbox-service-data-redis