all: export service-specific log files

This commit allows any service to export a list of service-specific log files that are relevant for a certain node role. The following service-specific log files are now exported:

- php: PHP access log, PHP error log (for role php), nginx proxy access log, nginx proxy error log (for role proxy), nginx static access log, nginx static error log (for role web)

- java: Tomcat access log, Tomcat output (for role Tomcat), nginx proxy access log, nginx proxy error log (for role proxy), nginx static access log, nginx static error log (for role web)

- mysql: log of the mysql daemon mysql.log (for role mysql) – I was not able to find a log file of the Galera Load Balancer

- xtreemfs: logs for DIR, MRC, OSD (one log file each of these roles)

- generic: standard output and standard error of the uploaded scripts

Links for the service-specific log files are now exposed in the web frontend inside the rendering of each instance.

Using cps-tools, the user can get the logs using the command 'cps-tools service get_agent_log …'. To check the log files available for each instance, the user can use the 'cps-tools service list_agent_logs …' command.

Other improvements:
- frontend: the flicker when refreshing a service page was removed;
- cps-tools: fixed the suggestions that appear when the agent_id parameter is invalid;
- mysql: moved the mysql.log file to /var/cache/cpsagent/ directory.

conpaas-frontend/www/css/conpaas.css

@@ -810,15 +810,14 @@ input.button {
 }
 
 .agent-logs {
-	padding-right: 12px;
-	padding-left: 12px;
+	width: 115px;
 	font-weight: normal;
 	color: #666;
 	font-size: 10px;
 }
 
 .agent-ip-address {
-	width: 110px;
+	width: 105px;
 	text-align: right;
 }
 
@@ -885,7 +884,7 @@ input.button {
 }
 
 .generic-instance-name {
-	width: 485px;
+	width: 450px;
 }
 
 .generic-script-status {

conpaas-frontend/www/lib/service/Service.php

@@ -251,7 +251,7 @@ public function fetchLog() {
  	}
 
 	public function fetchAgentLog($params) {
-		$json = $this->application->managerRequest('get', 'get_agent_log', 0, $params);
+		$json = $this->application->managerRequest('get', 'get_agent_log', $this->sid, $params);
 		$log = json_decode($json, true);
 		return $log['result']['log'];
 	}

conpaas-frontend/www/lib/ui/LinkUI.php

@@ -49,7 +49,8 @@ public function addClass($class) {
 
 	private function renderSymbol() {
 		return
-			'<img src="'.$this->iconURL.'" style="vertical-align: middle;" />';
+			'<img src="'.$this->iconURL.'" width="14" height="14"'
+				.' style="vertical-align: middle;" />';
 	}
 
 	public function __toString() {

conpaas-frontend/www/lib/ui/Role.php

@@ -81,8 +81,8 @@ public static function getInfo($role) {
 			'backend' => '',
 			'php' => 'PHP server',
 			'java' => 'Apache Tomcat servlet container',
-			'web' => 'static web server (NGINX)',
-			'proxy' => 'load balancer (NGINX)',
+			'web' => 'static web server (Nginx)',
+			'proxy' => 'load balancer (Nginx)',
 
 			// mysql
 			'mysql'=>'MySQL with Galera extensions',

conpaas-frontend/www/lib/ui/instance/Instance.php

@@ -5,62 +5,64 @@
 
 class Instance {
 
-	protected $info;
+    protected $info;
 
-	public function __construct($info) {
-		$this->info = $info;
-	}
+    public function __construct($info) {
+        $this->info = $info;
+    }
 
     protected function renderAgentLogs() {
-        $linkAgentLogs = LinkUI('agent log',
-            'viewlog.php?aid='.$_SESSION['aid']
-            .'&sid='.$this->info['sid']
-            .'&agentId='.$this->info['id']
-        )->setExternal(true);
-        $html = '<div class="right agent-logs">'
-					.$linkAgentLogs
-				.'</div>';
+        $html = '<div class="right agent-logs">';
+        foreach ($this->info['logs'] as $log) {
+            $html .= LinkUI($log['description'],
+                'viewlog.php?aid='.$_SESSION['aid']
+                .'&sid='.$this->info['sid']
+                .'&agentId='.$this->info['id']
+                .'&filename='.$log['filename']
+            )->setExternal(true);
+        }
+        $html .= '</div>';
         return $html;
     }
 
-	public function render() {
-		return
-		'<div class="instance dualbox">'
-			.'<div class="left">'
-				.'<i class="title">Instance '.$this->info['id'].'</i>'
-				.$this->renderCapabs()
-				.'<div class="cloud-name" title="cloud provider">'
-					.$this->info['cloud']
-				.'</div>'
-			.'</div>'
-			.'<div class="right agent-ip-address">'
-				.'<i class="address">'.$this->info['ip'].'</i>'
-			.'</div>'
-			.$this->renderAgentLogs()
-			.'<div class="clear"></div>'
-		.'</div>';
-	}
+    public function render() {
+        return
+        '<div class="instance dualbox">'
+            .'<div class="left">'
+                .'<i class="title">Instance '.$this->info['id'].'</i>'
+                .$this->renderCapabs()
+                .'<div class="cloud-name" title="cloud provider">'
+                    .$this->info['cloud']
+                .'</div>'
+            .'</div>'
+            .'<div class="right agent-ip-address">'
+                .'<i class="address">'.$this->info['ip'].'</i>'
+            .'</div>'
+            .$this->renderAgentLogs()
+            .'<div class="clear"></div>'
+        .'</div>';
+    }
 
-	public function renderInCluster() {
-		return
-		'<div class="instance dualbox">'
-			.'<div class="left">'
-				.'<i class="title">Instance '.$this->info['id'].'</i>'
-				.'<i class="cloud-name" title="cloud provider">'
-					.$this->info['cloud']
-				.'</i>'
-			.'</div>'
-			.'<div class="right agent-ip-address">'
-				.'<i class="address">'.$this->info['ip'].'</i>'
-			.'</div>'
-			.$this->renderAgentLogs()
-			.'<div class="clear"></div>'
-		.'</div>';
-	}
+    public function renderInCluster() {
+        return
+        '<div class="instance dualbox">'
+            .'<div class="left">'
+                .'<i class="title">Instance '.$this->info['id'].'</i>'
+                .'<i class="cloud-name" title="cloud provider">'
+                    .$this->info['cloud']
+                .'</i>'
+            .'</div>'
+            .'<div class="right agent-ip-address">'
+                .'<i class="address">'.$this->info['ip'].'</i>'
+            .'</div>'
+            .$this->renderAgentLogs()
+            .'<div class="clear"></div>'
+        .'</div>';
+    }
 
-	public function getSize() {
-		return 1;
-	}
+    public function getSize() {
+        return 1;
+    }
 
 }
 

conpaas-frontend/www/lib/ui/instance/generic/__init__.php

@@ -161,41 +161,15 @@ public static function renderScriptStatusTable($scriptStatus) {
         return $html;
     }
 
-    protected function renderAgentLogs() {
-        $linkAgentLogs = LinkUI('agent log',
-            'viewlog.php?aid='.$_SESSION['aid']
-            .'&sid='.$this->info['sid']
-            .'&agentId='.$this->info['id']
-        )->setExternal(true);
-        $linkAgentOut = LinkUI('agent output',
-            'viewlog.php?aid='.$_SESSION['aid']
-            .'&sid='.$this->info['sid']
-            .'&agentId='.$this->info['id']
-            .'&filename=agent.out'
-        )->setExternal(true);
-        $linkAgentErr = LinkUI('agent error',
-            'viewlog.php?aid='.$_SESSION['aid']
-            .'&sid='.$this->info['sid']
-            .'&agentId='.$this->info['id']
-            .'&filename=agent.err'
-        )->setExternal(true);
-        $html = '<div class="right agent-logs">'
-                    .$linkAgentLogs
-                    .$linkAgentOut
-                    .$linkAgentErr
-				.'</div>';
-        return $html;
-    }
-
     public function renderInCluster() {
         return
         '<div class="instance dualbox">'
             .'<div class="left">'
                 .'<div class="left generic-instance-name">'
                     .'<i class="title">Instance '.$this->info['id'].'</i>'
-                    .'<span class="cloud-name" title="cloud provider">'
+                    .'<i class="cloud-name" title="cloud provider">'
                         .$this->info['cloud']
-                    .'</span>'
+                    .'</i>'
                 .'</div>'
                 .'<div class="clear"></div>'
                 .'<div class="left">'
@@ -212,9 +186,7 @@ public function renderInCluster() {
             .'<div class="right agent-ip-address">'
                 .'<i class="address">'.$this->info['ip'].'</i>'
             .'</div>'
-            .'<div class="right agent-logs">'
-                .$this->renderAgentLogs()
-            .'</div>'
+            .$this->renderAgentLogs()
             .'<div class="clear"></div>'
             .$this->renderVolumeCreateButton()
         .'</div>';

conpaas-services/src/conpaas/core/agent.py

@@ -80,16 +80,12 @@ def check_process(self, kwargs):
     @expose('GET')
     def get_log(self, kwargs):
         """Return the contents of a logfile"""
-        other_log_files = [ 'agent.out', 'agent.err' ]
-        exp_params = [('filename', is_in_list(other_log_files), self.LOG_FILE)]
+        exp_params = [('filename', is_string, self.LOG_FILE)]
         try:
             filename = check_arguments(exp_params, kwargs)
         except Exception as ex:
             return HttpErrorResponse("%s" % ex)
 
-        if filename in other_log_files:
-            filename = os.path.join(self.ROOT_DIR, filename)
-
         try:
             return HttpJsonResponse({'log': file_get_contents(filename)})
         except:

conpaas-services/src/conpaas/core/manager.py

@@ -226,6 +226,19 @@ def get_sninfo(self, node_roles, cloud):
                 sninfo.append(self.get_role_sninfo(role, cloud))
         return sninfo
 
+    def get_role_logs(self, role):
+        """Returns a list containing the logs that are be exposed by an agent
+        of the specified type (role). Each log is represented by a dict
+        containing the name of the log file, the description and the file
+        path inside the agent.
+
+        It should be overwritten by the service managers if specific roles
+        need to expose additional log files.
+        """
+        return [ {'filename': 'cpsagent.log',
+                  'description': 'agent log',
+                  'path': '/var/log/cpsagent.log'} ]
+
     def get_context_replacement(self):
         """Returns a dict used to fill in the values of variables in the
         VM contextualization script.
@@ -294,6 +307,37 @@ def get_manager_log(self, kwargs):
         except:
             return HttpErrorResponse('Failed to read log')
 
+    @expose('GET')
+    def get_agent_log(self, kwargs):
+        """Return logfile"""
+        node_ids = [ str(node.id) for node in self.nodes ]
+        exp_params = [('agentId', is_in_list(node_ids)),
+                      ('filename', is_string, None)]
+        try:
+            agent_id, filename = check_arguments(exp_params, kwargs)
+
+            # Get the node that has the specified agent_id
+            node = [ node for node in self.nodes if node.id == agent_id ][0]
+
+            # If a filename was specified...
+            if filename:
+                # Get the logs for the node's role
+                logs = self.get_role_logs(node.role)
+
+                # Check that the filename is valid for that role
+                filenames = map(lambda log: log['filename'], logs)
+                exp_params = [('filename', is_in_list(filenames))]
+                check_arguments(exp_params, dict(filename=filename))
+
+                # Replace the filename with its corresponding path in the agent
+                filename = [ log['path'] for log in logs
+                                         if log['filename'] == filename ][0]
+
+            res = self.fetch_agent_log(node.ip, self.AGENT_PORT, filename)
+            return HttpJsonResponse(res)
+        except Exception as ex:
+            return HttpErrorResponse("%s" % ex)
+
     # the following two methods are the base agent client
     def _check(self, response):
         code, body = response
@@ -919,22 +963,6 @@ def get_startup_script(self, kwargs):
         except IOError:
             return HttpErrorResponse('No startup script')
 
-    @expose('GET')
-    def get_agent_log(self, kwargs):
-        """Return logfile"""
-        node_ids = [ str(node.id) for node in self.nodes ]
-        exp_params = [('agentId', is_in_list(node_ids)),
-                      ('filename', is_string, None)]
-        try:
-            agent_id, filename = check_arguments(exp_params, kwargs)
-
-            node_ip = [ node.ip for node in self.nodes
-                        if node.id == agent_id ][0]
-            res = self.fetch_agent_log(node_ip, self.AGENT_PORT, filename)
-            return HttpJsonResponse(res)
-        except Exception as ex:
-            return HttpErrorResponse("%s" % ex)
-
     @expose('POST')
     def git_push_hook(self, kwargs):
         try:

conpaas-services/src/conpaas/services/generic/manager/manager.py

@@ -122,6 +122,18 @@ def get_node_roles(self):
     def get_starting_nodes(self):
         return { self.ROLE_MASTER: 1 }
 
+    def get_role_logs(self, role):
+        logs = BaseManager.get_role_logs(self, role)
+
+        logs.extend([{'filename': 'agent.out',
+                      'description': 'agent output',
+                      'path': '/root/agent.out'},
+                     {'filename': 'agent.err',
+                      'description': 'agent error',
+                      'path': '/root/agent.err'}]);
+
+        return logs
+
     def _create_initial_configuration(self):
         self.logger.info("Creating initial configuration")
 
@@ -320,7 +332,10 @@ def get_node_info(self, kwargs):
                 'ip': serviceNode.ip,
                 'vmid': serviceNode.vmid,
                 'cloud': serviceNode.cloud_name,
-                'is_master': self.__is_master(serviceNode)
+                'is_master': self.__is_master(serviceNode),
+                'role': serviceNode.role,
+                'logs': self.get_role_logs(serviceNode.role)
+
             }
         })
 

conpaas-services/src/conpaas/services/helloworld/manager/manager.py

@@ -104,7 +104,9 @@ def get_node_info(self, kwargs):
                             'id': serviceNode.id,
                             'ip': serviceNode.ip,
                             'vmid': serviceNode.vmid,
-                            'cloud': serviceNode.cloud_name
+                            'cloud': serviceNode.cloud_name,
+                            'role': serviceNode.role,
+                            'logs': self.get_role_logs(serviceNode.role)
                             }
             })
 

conpaas-services/src/conpaas/services/mysql/agent/internals.py

@@ -99,8 +99,6 @@ def stop(self, kwargs):
 
         No parameters.
         """
-        if len(kwargs) > 0:
-            self.logger.warning('MySQL agent "stop" was called with arguments that will be ignored: "%s"' % kwargs)
         try:
             exp_params = []
             check_arguments(exp_params, kwargs)
@@ -263,8 +261,6 @@ def getLoad(self, kwargs):
         Returns the local load of the single nodes.
 
         """
-	if len(kwargs) > 0:
-            self.logger.warning('MySQL agent "stop" was called with arguments that will be ignored: "%s"' % kwargs)
         try:
 	    exp_params = []
 	    check_arguments(exp_params, kwargs)