diff --git a/custom/src/main/java/co/elastic/otel/ElasticAutoConfigurationCustomizerProvider.java b/custom/src/main/java/co/elastic/otel/ElasticAutoConfigurationCustomizerProvider.java index fd5d79a0..ebaa569f 100644 --- a/custom/src/main/java/co/elastic/otel/ElasticAutoConfigurationCustomizerProvider.java +++ b/custom/src/main/java/co/elastic/otel/ElasticAutoConfigurationCustomizerProvider.java @@ -18,6 +18,7 @@ */ package co.elastic.otel; +import co.elastic.otel.config.DynamicInstrumentation; import com.google.auto.service.AutoService; import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizer; import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider; @@ -49,6 +50,12 @@ public class ElasticAutoConfigurationCustomizerProvider public void customize(AutoConfigurationCustomizer autoConfiguration) { autoConfiguration.addPropertiesCustomizer( ElasticAutoConfigurationCustomizerProvider::propertiesCustomizer); + autoConfiguration.addTracerProviderCustomizer( + (providerBuilder, properties) -> { + DynamicInstrumentation.setTracerConfigurator( + providerBuilder, DynamicInstrumentation.UpdatableConfigurator.INSTANCE); + return providerBuilder; + }); } static Map propertiesCustomizer(ConfigProperties configProperties) { diff --git a/custom/src/main/java/co/elastic/otel/config/DynamicInstrumentation.java b/custom/src/main/java/co/elastic/otel/config/DynamicInstrumentation.java new file mode 100644 index 00000000..799b8fc0 --- /dev/null +++ b/custom/src/main/java/co/elastic/otel/config/DynamicInstrumentation.java @@ -0,0 +1,269 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package co.elastic.otel.config; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.TracerProvider; +import io.opentelemetry.sdk.common.InstrumentationScopeInfo; +import io.opentelemetry.sdk.internal.ComponentRegistry; +import io.opentelemetry.sdk.internal.ScopeConfigurator; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import io.opentelemetry.sdk.trace.internal.TracerConfig; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Notes: 1. The instrumentation can't have been disabled by configuration, eg using + * -Dotel.instrumentation.[name].enabled=false as in that case it is never initialized so can't be + * "re-enabled" 2. The specific instrumentation name is used, you can see these by setting this + * class logging level to j.u.l.Level.CONFIG 3. The disable/re-enable is eventually consistent, + * needing the application to pass a synchronization barrier to take effect - but for most + * applications these are very frequent + */ +public class DynamicInstrumentation { + + private static final Logger logger = Logger.getLogger(DynamicInstrumentation.class.getName()); + public static final String INSTRUMENTATION_NAME_PREPEND = "io.opentelemetry."; + // note the option can't be an env because no OSes support changing envs while the program runs + public static final String INSTRUMENTATION_DISABLE_OPTION = + "elastic.otel.java.disable_instrumentations"; + + private static Object getField(String fieldname, Object target) { + try { + Field field = target.getClass().getDeclaredField(fieldname); + field.setAccessible(true); + return field.get(target); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new IllegalStateException( + "Error getting " + fieldname + " from " + target.getClass(), e); + } + } + + private static Object call(String methodname, Object target) { + try { + Method method = target.getClass().getDeclaredMethod(methodname); + method.setAccessible(true); + return method.invoke(target); + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + throw new IllegalStateException( + "Error calling " + methodname + " on " + target.getClass(), e); + } + } + + private static Object call( + String methodname, Object target, T arg1, Class arg1Class) { + try { + Method method = target.getClass().getDeclaredMethod(methodname, arg1Class); + method.setAccessible(true); + return method.invoke(target, arg1); + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + throw new IllegalStateException( + "Error calling " + methodname + " on " + target.getClass() + "(" + arg1Class + ")", e); + } + } + + // SdkTracerProviderBuilder.setTracerConfigurator(ScopeConfigurator<> configurator) + // here because it's not currently public + public static SdkTracerProviderBuilder setTracerConfigurator( + SdkTracerProviderBuilder sdkTracerProviderBuilder, + ScopeConfigurator configurator) { + call("setTracerConfigurator", sdkTracerProviderBuilder, configurator, ScopeConfigurator.class); + return sdkTracerProviderBuilder; + } + + // SdkTracerProvider.getTracerConfig(InstrumentationScopeInfo instrumentationScopeInfo) + // here because it's not currently public + private static TracerConfig getTracerConfig( + SdkTracerProvider provider, InstrumentationScopeInfo instrumentationScopeInfo) { + return (TracerConfig) + call("getTracerConfig", provider, instrumentationScopeInfo, InstrumentationScopeInfo.class); + } + + // SdkTracer.getInstrumentationScopeInfo() + // here because it's not currently public + private static InstrumentationScopeInfo getInstrumentationScopeInfo(Tracer sdkTracer) + throws NoSuchFieldException, IllegalAccessException { + return (InstrumentationScopeInfo) call("getInstrumentationScopeInfo", sdkTracer); + } + + // Not an existing method + // SdkTracerProvider.updateTracerConfigurations() + // updates all tracers with the current SdkTracerProvider.tracerConfigurator + // Code implementation equivalent to + // this.tracerSdkComponentRegistry + // .getComponents() + // .forEach( + // sdkTracer -> + // sdkTracer.updateTracerConfig( + // getTracerConfig(sdkTracer.getInstrumentationScopeInfo()))); + // where SdkTracer.updateTracerConfig(TracerConfig tracerConfig) is equivalent to + // this.tracerEnabled = tracerConfig.isEnabled(); + private static void updateTracerConfigurations(TracerProvider provider) { + if (!(provider instanceof SdkTracerProvider)) { + provider = (TracerProvider) getField("delegate", provider); + } + ComponentRegistry tracerSdkComponentRegistry = + (ComponentRegistry) getField("tracerSdkComponentRegistry", provider); + SdkTracerProvider finalProvider = (SdkTracerProvider) provider; + final List activatedTracers; + if (logger.isLoggable(Level.CONFIG)) { + activatedTracers = new ArrayList<>(); + } else { + activatedTracers = null; + } + tracerSdkComponentRegistry + .getComponents() + .forEach( + sdkTracer -> { + try { + InstrumentationScopeInfo instrumentationScopeInfo = + getInstrumentationScopeInfo(sdkTracer); + TracerConfig tConfig = getTracerConfig(finalProvider, instrumentationScopeInfo); + Field tracerEnabledField = sdkTracer.getClass().getDeclaredField("tracerEnabled"); + tracerEnabledField.setAccessible(true); + // Update is synced but the reader is NOT necessarily so this is eventual + // consistency, takes effect when the application passes a sync boundary + synchronized (sdkTracer) { + tracerEnabledField.set(sdkTracer, tConfig.isEnabled()); + } + if (logger.isLoggable(Level.CONFIG)) { + String name = instrumentationScopeInfo.getName(); + if (name.startsWith(INSTRUMENTATION_NAME_PREPEND)) { + name = name.substring(INSTRUMENTATION_NAME_PREPEND.length()); + } + activatedTracers.add(name); + activatedTracers.add(tConfig.isEnabled() ? "enabled" : "disabled"); + } + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException(e); + } + }); + if (logger.isLoggable(Level.CONFIG)) { + logger.log(Level.CONFIG, "Activated Tracers: " + activatedTracers); + } + } + + public static void reenableTracesFor(String instrumentationName) { + UpdatableConfigurator.INSTANCE.put( + InstrumentationScopeInfo.create(INSTRUMENTATION_NAME_PREPEND + instrumentationName), + TracerConfig.enabled()); + updateTracerConfigurations(GlobalOpenTelemetry.getTracerProvider()); + } + + public static void disableTracesFor(String instrumentationName) { + UpdatableConfigurator.INSTANCE.put( + InstrumentationScopeInfo.create(INSTRUMENTATION_NAME_PREPEND + instrumentationName), + TracerConfig.disabled()); + updateTracerConfigurations(GlobalOpenTelemetry.getTracerProvider()); + } + + public static class UpdatableConfigurator implements ScopeConfigurator { + public static final UpdatableConfigurator INSTANCE = new UpdatableConfigurator(); + private final ConcurrentMap map = new ConcurrentHashMap<>(); + + private UpdatableConfigurator() {} + + @Override + public TracerConfig apply(InstrumentationScopeInfo scopeInfo) { + return map.getOrDefault(scopeInfo.getName(), TracerConfig.defaultConfig()); + } + + public void put(InstrumentationScopeInfo scope, TracerConfig tracerConfig) { + map.put(scope.getName(), tracerConfig); + } + } + + static { + if ("true".equals(System.getProperty(INSTRUMENTATION_DISABLE_OPTION + ".checker")) + || "true".equals(System.getenv("ELASTIC_OTEL_JAVA_DISABLE_INSTRUMENTATIONS_CHECKER"))) { + Thread checker = new Thread(new OptionChecker(), "Elastic dynamic_instrumentation checker"); + checker.setDaemon(true); + checker.start(); + } + } + + static class OptionChecker implements Runnable { + private Map alreadyDisabled = new HashMap<>(); + + // Note that if the property and the API are both used to specify enablement + // for a particular instrument, and this thread is executing, the property + // will take priority if the instrument is in the property - by virtue of running + // more frequently; but won't if the instrument is removed from the property! + // TODO define priority of enablement by source of disabler + @Override + public void run() { + while (true) { + String disableList; + synchronized (this) { + disableList = System.getProperty(INSTRUMENTATION_DISABLE_OPTION); + } + if (disableList != null && !disableList.trim().isEmpty()) { + // some values in the disable_instrumentations list + Set toBeEnabled = null; + if (!alreadyDisabled.isEmpty()) { + toBeEnabled = new HashSet<>(alreadyDisabled.keySet()); + } + for (String toBeDisabled : disableList.split(",")) { + toBeDisabled = toBeDisabled.trim(); + if (alreadyDisabled.containsKey(toBeDisabled)) { + // already disabled and keep it that way + if (toBeEnabled != null) { + toBeEnabled.remove(toBeDisabled); + } + } else { + DynamicInstrumentation.disableTracesFor(toBeDisabled); + alreadyDisabled.put(toBeDisabled, Boolean.TRUE); + } + } + if (toBeEnabled != null) { + for (String instrumentation : toBeEnabled) { + DynamicInstrumentation.reenableTracesFor(instrumentation); + alreadyDisabled.remove(instrumentation); + } + } + } else { + // empty list so anything currently disabled should be re-enabled + if (!alreadyDisabled.isEmpty()) { + for (String instrumentation : new HashSet<>(alreadyDisabled.keySet())) { + DynamicInstrumentation.reenableTracesFor(instrumentation); + alreadyDisabled.remove(instrumentation); + } + } + } + try { + Thread.sleep(1000L); + } catch (InterruptedException ignored) { + } + } + } + } +} diff --git a/custom/src/test/java/co/elastic/otel/config/DynamicInstrumentationTest.java b/custom/src/test/java/co/elastic/otel/config/DynamicInstrumentationTest.java new file mode 100644 index 00000000..2d3399b7 --- /dev/null +++ b/custom/src/test/java/co/elastic/otel/config/DynamicInstrumentationTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package co.elastic.otel.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import io.opentelemetry.sdk.common.InstrumentationScopeInfo; +import io.opentelemetry.sdk.internal.ScopeConfigurator; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import org.junit.jupiter.api.Test; + +public class DynamicInstrumentationTest { + @Test + // Functional testing is in DynamicInstrumentationSmokeTest + // These tests are so that when the SDK implementation stops being + // experimental, we switch from reflection to actual method calls + public void checkForPublicImplementations() throws NoSuchMethodException, ClassNotFoundException { + + Method method1 = + SdkTracerProviderBuilder.class.getDeclaredMethod( + "setTracerConfigurator", ScopeConfigurator.class); + assertThat(Modifier.toString(method1.getModifiers())).isNotEqualTo("public"); + + Method method2 = + SdkTracerProvider.class.getDeclaredMethod( + "getTracerConfig", InstrumentationScopeInfo.class); + assertThat(Modifier.toString(method2.getModifiers())).isNotEqualTo("public"); + + Class sdkTracer = Class.forName("io.opentelemetry.sdk.trace.SdkTracer"); + Method method3 = sdkTracer.getDeclaredMethod("getInstrumentationScopeInfo"); + assertThat(Modifier.toString(method3.getModifiers())).isNotEqualTo("public"); + + assertThatThrownBy( + () -> SdkTracerProvider.class.getDeclaredMethod("updateTracerConfigurations")) + .isInstanceOf(NoSuchMethodException.class); + } +} diff --git a/smoke-tests/src/test/java/com/example/javaagent/smoketest/DynamicInstrumentationSmokeTest.java b/smoke-tests/src/test/java/com/example/javaagent/smoketest/DynamicInstrumentationSmokeTest.java new file mode 100644 index 00000000..3aa3de44 --- /dev/null +++ b/smoke-tests/src/test/java/com/example/javaagent/smoketest/DynamicInstrumentationSmokeTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package com.example.javaagent.smoketest; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.protobuf.ByteString; +import io.opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest; +import io.opentelemetry.proto.trace.v1.Span; +import java.util.List; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +class DynamicInstrumentationSmokeTest extends TestAppSmokeTest { + + @BeforeAll + public static void start() { + startTestApp( + (container) -> { + container.addEnv( + "OTEL_INSTRUMENTATION_METHODS_INCLUDE", + "co.elastic.otel.test.DynamicInstrumentationController[flipMethods]"); + container.addEnv("ELASTIC_OTEL_JAVA_DISABLE_INSTRUMENTATIONS_CHECKER", "true"); + container.addEnv("OTEL_JAVAAGENT_DEBUG", "true"); + }); + } + + @AfterAll + public static void end() { + stopApp(); + } + + @Test + public void flipMethodInstrumentation() throws InterruptedException { + doRequest(getUrl("/dynamic"), okResponseBody("enabled")); + List traces = waitForTraces(); + List spans = getSpans(traces).toList(); + assertThat(spans) + .hasSize(2) + .extracting("name") + .containsOnly("GET /dynamic", "DynamicInstrumentationController.flipMethods"); + ByteString firstTraceID = spans.get(0).getTraceId(); + + Thread.sleep(2000L); // give the flip time to be applied + + doRequest(getUrl("/dynamic"), okResponseBody("disabled")); + traces = waitForTraces(); + spans = getSpans(traces).dropWhile(span -> span.getTraceId().equals(firstTraceID)).toList(); + assertThat(spans).hasSize(1).extracting("name").containsOnly("GET /dynamic"); + ByteString secondTraceID = spans.get(0).getTraceId(); + + Thread.sleep(2000L); // give the flip time to be applied + + doRequest(getUrl("/dynamic"), okResponseBody("enabled")); + traces = waitForTraces(); + spans = + getSpans(traces) + .dropWhile( + span -> + span.getTraceId().equals(firstTraceID) + || span.getTraceId().equals(secondTraceID)) + .toList(); + assertThat(spans) + .hasSize(2) + .extracting("name") + .containsOnly("GET /dynamic", "DynamicInstrumentationController.flipMethods"); + } +} diff --git a/smoke-tests/test-app/src/main/java/co/elastic/otel/test/DynamicInstrumentationController.java b/smoke-tests/test-app/src/main/java/co/elastic/otel/test/DynamicInstrumentationController.java new file mode 100644 index 00000000..bf054ab7 --- /dev/null +++ b/smoke-tests/test-app/src/main/java/co/elastic/otel/test/DynamicInstrumentationController.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package co.elastic.otel.test; + +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/dynamic") +public class DynamicInstrumentationController { + public static final String INSTRUMENTATION_DISABLE_OPTION = + "elastic.otel.java.disable_instrumentations"; + + // note synchronized to make enable/disable faster with DynamicInstrumentation + @GetMapping + public synchronized String flipMethods() { + String old = System.getProperty(INSTRUMENTATION_DISABLE_OPTION, ""); + if (old.isEmpty()) { + System.setProperty(INSTRUMENTATION_DISABLE_OPTION, "methods"); + } else { + System.setProperty(INSTRUMENTATION_DISABLE_OPTION, ""); + } + return old.isEmpty() ? "enabled" : "disabled"; + } +}