Merge pull request #11 from newbrough/feature/use-agent

agent and config changes

gumshoe-hooks/README.rst

@@ -8,27 +8,47 @@ Overview
 Gumshoe adds a hook in the JVM to monitor socket and file I/O.  The monitored JVM must be run
 using the commandline:
 
-    java -bootclasspath/p gumshoe-hooks.jar ... 
+    java -Xjavaagent:/PATH/TO/gumshoe-agent.jar ... 
 
 
 More Detail
 -----------
 
 Gumshoe Load Investigator measures socket and file I/O using the sun.misc.IoTrace class.
 The built-in implementation has several empty methods that are called before and after each I/O
-operation.   This package overrides this implementation with one that allows gumshoe to handle
-these calls and collect statistics.  
-
-Because it has to override a built-in class from rt.jar, the contents of this module are included 
-in the bootclasspath before rt.jar.  Then any application that wants to receive the IoTrace callbacks
-can implement an interface IoTraceDelegate and install it with IoTraceUtil.addTrace().
+operation.   This package replaces this implementation with one that allows gumshoe to handle
+these calls and collect statistics.  Then any application that wants to receive the IoTrace callbacks
+can implement an interface IoTraceListener and install it with IoTraceHandler.addTrace().
 
 
 Performance Note
 ----------------
 
-The IoTrace callbacks execute before and after every read or write operation on every socket or file.
-It can affect performance.  Specifically, it will add CPU and memory overhead to track the I/O.
-For applications that are constrained by I/O performance, this is not usually a problem. 
-Regardless, this overhead can be removed by omitting the -bootclasspath/p option. The original 
-IoTrace class from rt.jar is used, resulting in no additional system load per I/O operation.
+The IoTrace callbacks occur before and after every read or write operation on every socket or file.
+Without using gumshoe at all, there is CPU overhead from two empty method calls per I/O operation.
+With them gumshoe agent installed but not collecting socket or file I/O, this increases slightly --
+the empty method is now replaced by a method with one line that calls an empty delegate method.
+So even in CPU-constrained applications, the agent alone should not add much overhead.
+
+When monitoring is enabled, there is additional CPU and memory overhead involved in collecting
+and accumulating I/O statistics.  In many cases this is not an issue; when there is a
+network or file I/O bottleneck, additional CPU and memory can be used without affecting
+overall system performance.  
+
+If CPU or memory are an issue, overhead can be reduced in several ways:
+
+- Divide and conquer: use fewer probes
+
+  Collect file I/O in one pass, than and network I/O at a different time (instead of collecting both at once).
+
+- Divide and conquer: limit targets collected
+
+  Select by IP address or directory to collect less information at a time. For example, collect network data 
+  from your LAN in one pass, then collect samples from outside your LAN in another pass.
+
+- Reduce clutter: use stack filters
+
+  Stack filters can reduce the size of the stacks collected significantly and make it much easier to
+  spot the parts of code resulting in the I/O measured.  Stack filters cause some additional CPU overhead
+  when each sample is collected, but can reduce memory significantly.
+

gumshoe-hooks/pom.xml

@@ -4,7 +4,7 @@
 
 	<artifactId>gumshoe-hooks</artifactId>
 
-    <description>bootclasspath JVM hooks for gumshoe diagnostic tools</description>
+    <description>gumshoe JVM hook for collecting statistics</description>
 
     <build>
         <plugins>
@@ -17,6 +17,32 @@
                       <target>1.7</target>
                   </configuration>
               </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>attached</goal>
+                        </goals>
+                        <phase>package</phase>
+                        <configuration>
+                            <descriptorRefs>
+                                <descriptorRef>jar-with-dependencies</descriptorRef>
+                            </descriptorRefs>
+                            <archive>
+                                <manifestEntries>
+                                    <Premain-Class>com.dell.gumshoe.hook.Agent</Premain-Class>
+                                    <Can-Redefine-Classes>true</Can-Redefine-Classes>
+                                    <Boot-Class-Path>gumshoe-agent.jar</Boot-Class-Path>
+                                </manifestEntries>
+                            </archive>
+                            <finalName>gumshoe-agent</finalName>
+                            <appendAssemblyId>false</appendAssemblyId>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 

gumshoe-hooks/src/main/java/com/dell/gumshoe/hook/Agent.java

@@ -0,0 +1,93 @@
+package com.dell.gumshoe.hook;
+
+import sun.misc.IoTrace;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.instrument.ClassDefinition;
+import java.lang.instrument.Instrumentation;
+import java.util.jar.JarEntry;
+import java.util.jar.JarInputStream;
+
+/** install our version of IoTrace to capture socket and file I/O activity
+ */
+public class Agent {
+    private static final String CLASS_FILENAME = "sun/misc/IoTrace.class";
+
+    public static void premain(String args, Instrumentation inst) throws Exception {
+        final byte[] alternate = getAlternate();
+        if(alternate==null) {
+            System.out.println("GUMSHOE ERROR: failed to locate IoTrace hook");
+            return;
+        }
+        try {
+            inst.redefineClasses(new ClassDefinition(IoTrace.class, alternate));
+            if(Boolean.getBoolean("gumshoe.verbose")) {
+                System.out.println("GUMSHOE: installed IoTrace hook");
+            }
+        } catch(Exception e) {
+            System.out.println("GUMSHOE ERROR: failed to install IoTrace hook");
+            e.printStackTrace();
+        }
+    }
+
+    /** loop over classpath */
+    private static byte[] getAlternate() throws IOException {
+        final String[] classpath = System.getProperty("java.class.path").split(System.getProperty("path.separator"));
+        for(String entry : classpath) {
+            final File file = new File(entry);
+            if( ! file.canRead()) continue;
+
+            if(file.isFile()) {
+                final JarInputStream in = new JarInputStream(new FileInputStream(file));
+                try {
+                    final byte[] contents = getFromJar(in);
+                    if(contents!=null) { return contents; }
+                } finally {
+                    in.close();
+                }
+            } else {
+                // this case happens during development
+                if(file.isDirectory()) {
+                    final File classFile = new File(file, CLASS_FILENAME);
+                    if(classFile.isFile()) {
+                        return getFromFile(classFile);
+                    }
+                }
+            }
+        }
+        return null;
+    }
+
+    private static byte[] getFromJar(JarInputStream jarIn) throws IOException {
+        JarEntry entry;
+        while((entry = jarIn.getNextJarEntry())!=null) {
+            if(CLASS_FILENAME.equals(entry.getName())) {
+                return getFromStream(jarIn, entry.getSize());
+            }
+        }
+        return null;
+    }
+
+    private static byte[] getFromFile(File file) throws IOException {
+        final InputStream in = new BufferedInputStream(new FileInputStream(file));
+        try {
+            return getFromStream(in, file.length());
+        } finally {
+            in.close();
+        }
+    }
+
+    private static byte[] getFromStream(InputStream in, long size) throws IOException {
+        final byte[] contents = new byte[(int)size];
+        int pos = 0;
+        int len;
+        while(size-pos>0 && (len=in.read(contents, pos, (int)size-pos))!=-1) {
+            pos+=len;
+        }
+        return contents;
+    }
+}

gumshoe-hooks/src/main/java/com/dell/gumshoe/IoTraceAdapter.java → gumshoe-hooks/src/main/java/com/dell/gumshoe/hook/IoTraceAdapter.java

@@ -1,11 +1,11 @@
-package com.dell.gumshoe;
+package com.dell.gumshoe.hook;
 
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.SocketAddress;
 
-/** adapter class to simplify implementation of IoTraceDelegate */
-public class IoTraceAdapter implements IoTraceDelegate {
+/** adapter class to simplify implementation of IoTraceListener */
+public class IoTraceAdapter implements IoTraceListener {
     public Object socketReadBegin() { return null; }
     public void socketReadEnd(Object context, InetAddress address, int port, int timeout, long bytesRead) { }
     public Object socketWriteBegin() { return null; }

gumshoe-hooks/src/main/java/com/dell/gumshoe/hook/IoTraceHandler.java

@@ -0,0 +1,104 @@
+package com.dell.gumshoe.hook;
+
+import java.net.InetAddress;
+import java.net.SocketAddress;
+
+public class IoTraceHandler {
+    private static IoTraceListener NULL_OBJECT = new IoTraceAdapter();
+    private static IoTraceListener DELEGATE = NULL_OBJECT;
+
+    public synchronized static void addTrace(IoTraceListener delegate) throws Exception {
+        if(DELEGATE==NULL_OBJECT) {
+            DELEGATE = delegate;
+        } else if(DELEGATE instanceof IoTraceMultiplexer) {
+            final IoTraceMultiplexer multi = (IoTraceMultiplexer) DELEGATE;
+            multi.addDelegate(delegate);
+        } else {
+            final IoTraceMultiplexer multi = new IoTraceMultiplexer();
+            multi.addDelegate(DELEGATE);
+            multi.addDelegate(delegate);
+            DELEGATE = multi;
+        }
+    }
+
+    public static void removeTrace(IoTraceListener delegate) throws Exception {
+        if(DELEGATE==delegate) {
+            DELEGATE = NULL_OBJECT;
+        } else if(DELEGATE instanceof IoTraceMultiplexer) {
+            final IoTraceMultiplexer multi = (IoTraceMultiplexer) DELEGATE;
+            multi.removeDelegate(delegate);
+        } else {
+            throw new IllegalArgumentException("unable to remove, that IoTraceListener was not installed: " + delegate);
+        }
+    }
+
+    public static IoTraceListener getTrace() {
+        return DELEGATE;
+    }
+
+    /////
+
+    public static Object socketReadBegin() {
+        return DELEGATE.socketReadBegin();
+    }
+
+    public static void socketReadEnd(Object context, InetAddress address, int port,
+            int timeout, long bytesRead) {
+        DELEGATE.socketReadEnd(context, address, port, timeout, bytesRead);
+    }
+
+    public static Object socketWriteBegin() {
+        return DELEGATE.socketWriteBegin();
+    }
+
+    public static void socketWriteEnd(Object context, InetAddress address, int port,
+            long bytesWritten) {
+        DELEGATE.socketWriteEnd(context, address, port, bytesWritten);
+    }
+
+    public static void socketReadEnd(Object context, SocketAddress address,
+            long bytesRead) {
+        DELEGATE.socketReadEnd(context, address, bytesRead);
+    }
+
+    public static void socketWriteEnd(Object context, SocketAddress address,
+            long bytesWritten) {
+        DELEGATE.socketWriteEnd(context, address, bytesWritten);
+    }
+
+    public static Object datagramReadBegin() {
+        return DELEGATE.datagramReadBegin();
+    }
+
+    public static void datagramReadEnd(Object context, SocketAddress address,
+            long bytesRead) {
+        DELEGATE.datagramReadEnd(context, address, bytesRead);
+    }
+
+    public static Object datagramWriteBegin() {
+        return DELEGATE.datagramWriteBegin();
+    }
+
+    public static void datagramWriteEnd(Object context, SocketAddress address,
+            long bytesWritten) {
+        DELEGATE.datagramWriteEnd(context, address, bytesWritten);
+    }
+
+    public static Object fileReadBegin(String path) {
+        return DELEGATE.fileReadBegin(path);
+    }
+
+    public static void fileReadEnd(Object context, long bytesRead) {
+        DELEGATE.fileReadEnd(context, bytesRead);
+    }
+
+    public static Object fileWriteBegin(String path) {
+        return DELEGATE.fileWriteBegin(path);
+    }
+
+    public static void fileWriteEnd(Object context, long bytesWritten) {
+        DELEGATE.fileWriteEnd(context, bytesWritten);
+    }
+
+
+}

gumshoe-hooks/src/main/java/com/dell/gumshoe/IoTraceDelegate.java → gumshoe-hooks/src/main/java/com/dell/gumshoe/hook/IoTraceListener.java

@@ -1,10 +1,10 @@
-package com.dell.gumshoe;
+package com.dell.gumshoe.hook;
 
 import java.net.InetAddress;
 import java.net.SocketAddress;
 
 /** public interface for delegate to plug into sun.misc.IoTrace */
-public interface IoTraceDelegate {
+public interface IoTraceListener {
         public Object socketReadBegin();
         public void socketReadEnd(Object context, InetAddress address, int port, int timeout, long bytesRead);
         public Object socketWriteBegin();
@@ -22,4 +22,9 @@ public interface IoTraceDelegate {
         public void fileReadEnd(Object context, long bytesRead);
         public Object fileWriteBegin(String path);
         public void fileWriteEnd(Object context, long bytesWritten);
+
+        // marking interfaces
+        public interface SocketListener extends IoTraceListener { }
+        public interface DatagramListener extends IoTraceListener { }
+        public interface FileListener extends IoTraceListener { }
 }