From 27b6a3b9c58473d7b4735ce8beaf7c92cce6d10d Mon Sep 17 00:00:00 2001 From: tcrivat Date: Fri, 22 Jul 2016 23:12:33 +0200 Subject: [PATCH] all: export service-specific log files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit allows any service to export a list of service-specific log files that are relevant for a certain node role. The following service-specific log files are now exported: - php: PHP access log, PHP error log (for role php), nginx proxy access log, nginx proxy error log (for role proxy), nginx static access log, nginx static error log (for role web) - java: Tomcat access log, Tomcat output (for role Tomcat), nginx proxy access log, nginx proxy error log (for role proxy), nginx static access log, nginx static error log (for role web) - mysql: log of the mysql daemon mysql.log (for role mysql) – I was not able to find a log file of the Galera Load Balancer - xtreemfs: logs for DIR, MRC, OSD (one log file each of these roles) - generic: standard output and standard error of the uploaded scripts Links for the service-specific log files are now exposed in the web frontend inside the rendering of each instance. Using cps-tools, the user can get the logs using the command 'cps-tools service get_agent_log …'. To check the log files available for each instance, the user can use the 'cps-tools service list_agent_logs …' command. Other improvements: - frontend: the flicker when refreshing a service page was removed; - cps-tools: fixed the suggestions that appear when the agent_id parameter is invalid; - mysql: moved the mysql.log file to /var/cache/cpsagent/ directory. --- conpaas-frontend/www/css/conpaas.css | 7 +- conpaas-frontend/www/lib/service/Service.php | 2 +- conpaas-frontend/www/lib/ui/LinkUI.php | 3 +- conpaas-frontend/www/lib/ui/Role.php | 4 +- .../www/lib/ui/instance/Instance.php | 98 ++++++++++--------- .../www/lib/ui/instance/generic/__init__.php | 34 +------ conpaas-services/src/conpaas/core/agent.py | 6 +- conpaas-services/src/conpaas/core/manager.py | 60 +++++++++--- .../services/generic/manager/manager.py | 17 +++- .../services/helloworld/manager/manager.py | 4 +- .../conpaas/services/mysql/agent/internals.py | 4 - .../src/conpaas/services/mysql/agent/role.py | 4 +- .../conpaas/services/mysql/manager/config.py | 21 ++-- .../conpaas/services/mysql/manager/manager.py | 20 +++- .../services/webservers/manager/config.py | 3 +- .../webservers/manager/internal/__init__.py | 80 ++++++++++++--- .../webservers/manager/internal/java.py | 26 +++++ .../webservers/manager/internal/php.py | 20 ++++ .../services/xtreemfs/manager/manager.py | 37 ++++++- cps-tools/src/cps_tools/service.py | 32 ++++-- 20 files changed, 327 insertions(+), 155 deletions(-) diff --git a/conpaas-frontend/www/css/conpaas.css b/conpaas-frontend/www/css/conpaas.css index 3d9e7544..d3d197d8 100644 --- a/conpaas-frontend/www/css/conpaas.css +++ b/conpaas-frontend/www/css/conpaas.css @@ -810,15 +810,14 @@ input.button { } .agent-logs { - padding-right: 12px; - padding-left: 12px; + width: 115px; font-weight: normal; color: #666; font-size: 10px; } .agent-ip-address { - width: 110px; + width: 105px; text-align: right; } @@ -885,7 +884,7 @@ input.button { } .generic-instance-name { - width: 485px; + width: 450px; } .generic-script-status { diff --git a/conpaas-frontend/www/lib/service/Service.php b/conpaas-frontend/www/lib/service/Service.php index 262a4025..14d7d69b 100644 --- a/conpaas-frontend/www/lib/service/Service.php +++ b/conpaas-frontend/www/lib/service/Service.php @@ -251,7 +251,7 @@ public function fetchLog() { } public function fetchAgentLog($params) { - $json = $this->application->managerRequest('get', 'get_agent_log', 0, $params); + $json = $this->application->managerRequest('get', 'get_agent_log', $this->sid, $params); $log = json_decode($json, true); return $log['result']['log']; } diff --git a/conpaas-frontend/www/lib/ui/LinkUI.php b/conpaas-frontend/www/lib/ui/LinkUI.php index f18e9b21..ec8ae7ba 100644 --- a/conpaas-frontend/www/lib/ui/LinkUI.php +++ b/conpaas-frontend/www/lib/ui/LinkUI.php @@ -49,7 +49,8 @@ public function addClass($class) { private function renderSymbol() { return - ''; + ''; } public function __toString() { diff --git a/conpaas-frontend/www/lib/ui/Role.php b/conpaas-frontend/www/lib/ui/Role.php index b35ae8d9..a360e0be 100644 --- a/conpaas-frontend/www/lib/ui/Role.php +++ b/conpaas-frontend/www/lib/ui/Role.php @@ -81,8 +81,8 @@ public static function getInfo($role) { 'backend' => '', 'php' => 'PHP server', 'java' => 'Apache Tomcat servlet container', - 'web' => 'static web server (NGINX)', - 'proxy' => 'load balancer (NGINX)', + 'web' => 'static web server (Nginx)', + 'proxy' => 'load balancer (Nginx)', // mysql 'mysql'=>'MySQL with Galera extensions', diff --git a/conpaas-frontend/www/lib/ui/instance/Instance.php b/conpaas-frontend/www/lib/ui/instance/Instance.php index 6868c9e2..e9b34e8a 100644 --- a/conpaas-frontend/www/lib/ui/instance/Instance.php +++ b/conpaas-frontend/www/lib/ui/instance/Instance.php @@ -5,62 +5,64 @@ class Instance { - protected $info; + protected $info; - public function __construct($info) { - $this->info = $info; - } + public function __construct($info) { + $this->info = $info; + } protected function renderAgentLogs() { - $linkAgentLogs = LinkUI('agent log', - 'viewlog.php?aid='.$_SESSION['aid'] - .'&sid='.$this->info['sid'] - .'&agentId='.$this->info['id'] - )->setExternal(true); - $html = '
' - .$linkAgentLogs - .'
'; + $html = '
'; + foreach ($this->info['logs'] as $log) { + $html .= LinkUI($log['description'], + 'viewlog.php?aid='.$_SESSION['aid'] + .'&sid='.$this->info['sid'] + .'&agentId='.$this->info['id'] + .'&filename='.$log['filename'] + )->setExternal(true); + } + $html .= '
'; return $html; } - public function render() { - return - '
' - .'
' - .'Instance '.$this->info['id'].'' - .$this->renderCapabs() - .'
' - .$this->info['cloud'] - .'
' - .'
' - .'
' - .''.$this->info['ip'].'' - .'
' - .$this->renderAgentLogs() - .'
' - .'
'; - } + public function render() { + return + '
' + .'
' + .'Instance '.$this->info['id'].'' + .$this->renderCapabs() + .'
' + .$this->info['cloud'] + .'
' + .'
' + .'
' + .''.$this->info['ip'].'' + .'
' + .$this->renderAgentLogs() + .'
' + .'
'; + } - public function renderInCluster() { - return - '
' - .'
' - .'Instance '.$this->info['id'].'' - .'' - .$this->info['cloud'] - .'' - .'
' - .'
' - .''.$this->info['ip'].'' - .'
' - .$this->renderAgentLogs() - .'
' - .'
'; - } + public function renderInCluster() { + return + '
' + .'
' + .'Instance '.$this->info['id'].'' + .'' + .$this->info['cloud'] + .'' + .'
' + .'
' + .''.$this->info['ip'].'' + .'
' + .$this->renderAgentLogs() + .'
' + .'
'; + } - public function getSize() { - return 1; - } + public function getSize() { + return 1; + } } diff --git a/conpaas-frontend/www/lib/ui/instance/generic/__init__.php b/conpaas-frontend/www/lib/ui/instance/generic/__init__.php index 3cb89f25..6da3cdf3 100644 --- a/conpaas-frontend/www/lib/ui/instance/generic/__init__.php +++ b/conpaas-frontend/www/lib/ui/instance/generic/__init__.php @@ -161,41 +161,15 @@ public static function renderScriptStatusTable($scriptStatus) { return $html; } - protected function renderAgentLogs() { - $linkAgentLogs = LinkUI('agent log', - 'viewlog.php?aid='.$_SESSION['aid'] - .'&sid='.$this->info['sid'] - .'&agentId='.$this->info['id'] - )->setExternal(true); - $linkAgentOut = LinkUI('agent output', - 'viewlog.php?aid='.$_SESSION['aid'] - .'&sid='.$this->info['sid'] - .'&agentId='.$this->info['id'] - .'&filename=agent.out' - )->setExternal(true); - $linkAgentErr = LinkUI('agent error', - 'viewlog.php?aid='.$_SESSION['aid'] - .'&sid='.$this->info['sid'] - .'&agentId='.$this->info['id'] - .'&filename=agent.err' - )->setExternal(true); - $html = '
' - .$linkAgentLogs - .$linkAgentOut - .$linkAgentErr - .'
'; - return $html; - } - public function renderInCluster() { return '
' .'
' .'
' .'Instance '.$this->info['id'].'' - .'' + .'' .$this->info['cloud'] - .'' + .'' .'
' .'
' .'
' @@ -212,9 +186,7 @@ public function renderInCluster() { .'
' .''.$this->info['ip'].'' .'
' - .'
' - .$this->renderAgentLogs() - .'
' + .$this->renderAgentLogs() .'
' .$this->renderVolumeCreateButton() .'
'; diff --git a/conpaas-services/src/conpaas/core/agent.py b/conpaas-services/src/conpaas/core/agent.py index ec7edc4d..c52ead63 100755 --- a/conpaas-services/src/conpaas/core/agent.py +++ b/conpaas-services/src/conpaas/core/agent.py @@ -80,16 +80,12 @@ def check_process(self, kwargs): @expose('GET') def get_log(self, kwargs): """Return the contents of a logfile""" - other_log_files = [ 'agent.out', 'agent.err' ] - exp_params = [('filename', is_in_list(other_log_files), self.LOG_FILE)] + exp_params = [('filename', is_string, self.LOG_FILE)] try: filename = check_arguments(exp_params, kwargs) except Exception as ex: return HttpErrorResponse("%s" % ex) - if filename in other_log_files: - filename = os.path.join(self.ROOT_DIR, filename) - try: return HttpJsonResponse({'log': file_get_contents(filename)}) except: diff --git a/conpaas-services/src/conpaas/core/manager.py b/conpaas-services/src/conpaas/core/manager.py index 2c340b33..9caccdc9 100644 --- a/conpaas-services/src/conpaas/core/manager.py +++ b/conpaas-services/src/conpaas/core/manager.py @@ -226,6 +226,19 @@ def get_sninfo(self, node_roles, cloud): sninfo.append(self.get_role_sninfo(role, cloud)) return sninfo + def get_role_logs(self, role): + """Returns a list containing the logs that are be exposed by an agent + of the specified type (role). Each log is represented by a dict + containing the name of the log file, the description and the file + path inside the agent. + + It should be overwritten by the service managers if specific roles + need to expose additional log files. + """ + return [ {'filename': 'cpsagent.log', + 'description': 'agent log', + 'path': '/var/log/cpsagent.log'} ] + def get_context_replacement(self): """Returns a dict used to fill in the values of variables in the VM contextualization script. @@ -294,6 +307,37 @@ def get_manager_log(self, kwargs): except: return HttpErrorResponse('Failed to read log') + @expose('GET') + def get_agent_log(self, kwargs): + """Return logfile""" + node_ids = [ str(node.id) for node in self.nodes ] + exp_params = [('agentId', is_in_list(node_ids)), + ('filename', is_string, None)] + try: + agent_id, filename = check_arguments(exp_params, kwargs) + + # Get the node that has the specified agent_id + node = [ node for node in self.nodes if node.id == agent_id ][0] + + # If a filename was specified... + if filename: + # Get the logs for the node's role + logs = self.get_role_logs(node.role) + + # Check that the filename is valid for that role + filenames = map(lambda log: log['filename'], logs) + exp_params = [('filename', is_in_list(filenames))] + check_arguments(exp_params, dict(filename=filename)) + + # Replace the filename with its corresponding path in the agent + filename = [ log['path'] for log in logs + if log['filename'] == filename ][0] + + res = self.fetch_agent_log(node.ip, self.AGENT_PORT, filename) + return HttpJsonResponse(res) + except Exception as ex: + return HttpErrorResponse("%s" % ex) + # the following two methods are the base agent client def _check(self, response): code, body = response @@ -919,22 +963,6 @@ def get_startup_script(self, kwargs): except IOError: return HttpErrorResponse('No startup script') - @expose('GET') - def get_agent_log(self, kwargs): - """Return logfile""" - node_ids = [ str(node.id) for node in self.nodes ] - exp_params = [('agentId', is_in_list(node_ids)), - ('filename', is_string, None)] - try: - agent_id, filename = check_arguments(exp_params, kwargs) - - node_ip = [ node.ip for node in self.nodes - if node.id == agent_id ][0] - res = self.fetch_agent_log(node_ip, self.AGENT_PORT, filename) - return HttpJsonResponse(res) - except Exception as ex: - return HttpErrorResponse("%s" % ex) - @expose('POST') def git_push_hook(self, kwargs): try: diff --git a/conpaas-services/src/conpaas/services/generic/manager/manager.py b/conpaas-services/src/conpaas/services/generic/manager/manager.py index 36a9bd31..e28a83c2 100644 --- a/conpaas-services/src/conpaas/services/generic/manager/manager.py +++ b/conpaas-services/src/conpaas/services/generic/manager/manager.py @@ -122,6 +122,18 @@ def get_node_roles(self): def get_starting_nodes(self): return { self.ROLE_MASTER: 1 } + def get_role_logs(self, role): + logs = BaseManager.get_role_logs(self, role) + + logs.extend([{'filename': 'agent.out', + 'description': 'agent output', + 'path': '/root/agent.out'}, + {'filename': 'agent.err', + 'description': 'agent error', + 'path': '/root/agent.err'}]); + + return logs + def _create_initial_configuration(self): self.logger.info("Creating initial configuration") @@ -320,7 +332,10 @@ def get_node_info(self, kwargs): 'ip': serviceNode.ip, 'vmid': serviceNode.vmid, 'cloud': serviceNode.cloud_name, - 'is_master': self.__is_master(serviceNode) + 'is_master': self.__is_master(serviceNode), + 'role': serviceNode.role, + 'logs': self.get_role_logs(serviceNode.role) + } }) diff --git a/conpaas-services/src/conpaas/services/helloworld/manager/manager.py b/conpaas-services/src/conpaas/services/helloworld/manager/manager.py index 65f50e7d..f6d95383 100755 --- a/conpaas-services/src/conpaas/services/helloworld/manager/manager.py +++ b/conpaas-services/src/conpaas/services/helloworld/manager/manager.py @@ -104,7 +104,9 @@ def get_node_info(self, kwargs): 'id': serviceNode.id, 'ip': serviceNode.ip, 'vmid': serviceNode.vmid, - 'cloud': serviceNode.cloud_name + 'cloud': serviceNode.cloud_name, + 'role': serviceNode.role, + 'logs': self.get_role_logs(serviceNode.role) } }) diff --git a/conpaas-services/src/conpaas/services/mysql/agent/internals.py b/conpaas-services/src/conpaas/services/mysql/agent/internals.py index 45ae7069..56fb8bb5 100644 --- a/conpaas-services/src/conpaas/services/mysql/agent/internals.py +++ b/conpaas-services/src/conpaas/services/mysql/agent/internals.py @@ -99,8 +99,6 @@ def stop(self, kwargs): No parameters. """ - if len(kwargs) > 0: - self.logger.warning('MySQL agent "stop" was called with arguments that will be ignored: "%s"' % kwargs) try: exp_params = [] check_arguments(exp_params, kwargs) @@ -263,8 +261,6 @@ def getLoad(self, kwargs): Returns the local load of the single nodes. """ - if len(kwargs) > 0: - self.logger.warning('MySQL agent "stop" was called with arguments that will be ignored: "%s"' % kwargs) try: exp_params = [] check_arguments(exp_params, kwargs) diff --git a/conpaas-services/src/conpaas/services/mysql/agent/role.py b/conpaas-services/src/conpaas/services/mysql/agent/role.py index 86dd8eb9..42e83d76 100644 --- a/conpaas-services/src/conpaas/services/mysql/agent/role.py +++ b/conpaas-services/src/conpaas/services/mysql/agent/role.py @@ -65,6 +65,8 @@ def __init__(self, config, nodes, device_name): # Get MySQL configuration parameters self.manager_ip = config.get('agent', 'MANAGER_IP') self.mysqldump_path = config.get('agent', 'MYSQLDUMP_PATH') + self.var_cache = config.get('agent', 'VAR_CACHE') + self.error_log = join(self.var_cache, 'mysql.log') self.mycnf_filepath = config.get("MySQL_configuration", "my_cnf_file") self.path_mysql_ssr = config.get("MySQL_configuration", "path_mysql_ssr") @@ -87,7 +89,7 @@ def __init__(self, config, nodes, device_name): if "datadir" in line: f.write("datadir = %s/data/\n" % self.mount_point) elif "log_error" in line: - f.write("log_error = /root/mysql.log\n") + f.write("log_error = %s\n" % self.error_log) #elif "socket" in line: # f.write("socket = %s/data/\n" % self.mount_point) elif "bind-address" in line: diff --git a/conpaas-services/src/conpaas/services/mysql/manager/config.py b/conpaas-services/src/conpaas/services/mysql/manager/config.py index 99f767f9..a4458932 100755 --- a/conpaas-services/src/conpaas/services/mysql/manager/config.py +++ b/conpaas-services/src/conpaas/services/mysql/manager/config.py @@ -40,7 +40,7 @@ class SQLServiceNode(ServiceNode): def __init__(self, node): ServiceNode.__init__(self, node.vmid, node.ip, node.private_ip, - node.cloud_name) + node.cloud_name, role=node.role) self.port = 5555 '''String representation of the ServiceNode. @@ -65,7 +65,7 @@ class GLBServiceNode(ServiceNode): def __init__(self, node): ServiceNode.__init__(self, node.vmid, node.ip, node.private_ip, - node.cloud_name) + node.cloud_name, role=node.role) self.port = 5555 '''String representation of the ServiceNode. @@ -112,13 +112,12 @@ def get_glb_nodes(self): return [serviceNode for serviceNode in self.glb_service_nodes.values()] def getMySQLNode(self, id): - # self.logger.debug('Entering getMysqlNodes id= %s ' % id) - if self.serviceNodes.has_key(id) : - node = self.serviceNodes[id] - else: - node = self.glb_service_nodes[id] - # self.logger.debug('Exit getMySQlNode node = %s' %node) - return node + if self.serviceNodes.has_key(id): + node = self.serviceNodes[id] + else: + node = self.glb_service_nodes[id] + + return node def addGLBServiceNodes(self, nodes): ''' @@ -126,7 +125,7 @@ def addGLBServiceNodes(self, nodes): ''' self.logger.debug('Entering addGLBServiceNodes') for node in nodes: - self.glb_service_nodes[node.id] = GLBServiceNode(node) + self.glb_service_nodes[str(node.id)] = GLBServiceNode(node) self.logger.debug('Exiting addGLBServiceNodes') def removeGLBServiceNode(self, id): @@ -144,7 +143,7 @@ def addMySQLServiceNodes(self, nodes): Add new Service Node to the server (configuration). ''' for node in nodes: - self.serviceNodes[node.id] = SQLServiceNode(node) + self.serviceNodes[str(node.id)] = SQLServiceNode(node) def removeMySQLServiceNode(self, id): ''' diff --git a/conpaas-services/src/conpaas/services/mysql/manager/manager.py b/conpaas-services/src/conpaas/services/mysql/manager/manager.py index eb856354..b692ab9f 100644 --- a/conpaas-services/src/conpaas/services/mysql/manager/manager.py +++ b/conpaas-services/src/conpaas/services/mysql/manager/manager.py @@ -68,6 +68,16 @@ def get_role_sninfo(self, role, cloud): else: return BaseManager.get_role_sninfo(self, role, cloud) + def get_role_logs(self, role): + logs = BaseManager.get_role_logs(self, role) + + if role == self.ROLE_MYSQL: + logs.extend([{'filename': 'mysql.log', + 'description': 'MySQL log', + 'path': '/var/cache/cpsagent/mysql.log'}]); + + return logs + def get_context_replacement(self): if not self.root_pass: # self.root_pass='password' @@ -164,21 +174,21 @@ def get_node_info(self, kwargs): name of cloud provider """ try: - exp_params = [('serviceNodeId', is_string)] + node_ids = self.config.serviceNodes.keys() + self.config.glb_service_nodes.keys() + exp_params = [('serviceNodeId', is_in_list(node_ids))] serviceNodeId = check_arguments(exp_params, kwargs) except Exception as ex: return HttpErrorResponse("%s" % ex) - if serviceNodeId not in self.config.serviceNodes and serviceNodeId not in self.config.glb_service_nodes : - return HttpErrorResponse('Unknown "serviceNodeId" %s, should be one of %s.' - % (serviceNodeId, self.config.serviceNodes.keys())) serviceNode = self.config.getMySQLNode(serviceNodeId) return HttpJsonResponse({'serviceNode': {'id': serviceNode.id, 'ip': serviceNode.ip, 'vmid': serviceNode.vmid, 'cloud': serviceNode.cloud_name, 'isNode': serviceNode.isNode, - 'isGlb_node': serviceNode.isGlb_node + 'isGlb_node': serviceNode.isGlb_node, + 'role': serviceNode.role, + 'logs': self.get_role_logs(serviceNode.role) } }) diff --git a/conpaas-services/src/conpaas/services/webservers/manager/config.py b/conpaas-services/src/conpaas/services/webservers/manager/config.py index 1ebfc65a..917c7155 100755 --- a/conpaas-services/src/conpaas/services/webservers/manager/config.py +++ b/conpaas-services/src/conpaas/services/webservers/manager/config.py @@ -29,7 +29,8 @@ def __init__(self, node, weightWeb=DEFAULT_WEIGHT, runBackend=False): ServiceNode.__init__(self, node.vmid, node.ip, node.private_ip, - node.cloud_name, weightBackend) + node.cloud_name, weightBackend, + node.role) self.isRunningProxy = runProxy self.isRunningWeb = runWeb self.isRunningBackend = runBackend diff --git a/conpaas-services/src/conpaas/services/webservers/manager/internal/__init__.py b/conpaas-services/src/conpaas/services/webservers/manager/internal/__init__.py index 06db84e9..afb33e64 100755 --- a/conpaas-services/src/conpaas/services/webservers/manager/internal/__init__.py +++ b/conpaas-services/src/conpaas/services/webservers/manager/internal/__init__.py @@ -143,6 +143,39 @@ def get_starting_nodes(self): self._configuration_set(config) return node_roles + def get_role_logs(self, role, add_default=True): + if add_default: + logs = BaseManager.get_role_logs(self, role) + else: + logs = [] + + if role == self.ROLE_WEB: + logs.extend([{'filename': 'nginx-static-access.log', + 'description': 'Nginx web access', + 'path': '/var/cache/cpsagent/nginx-static-access.log'}, + {'filename': 'nginx-static-error.log', + 'description': 'Nginx web error', + 'path': '/var/cache/cpsagent/nginx-static-error.log'}]); + elif role == self.ROLE_PROXY: + logs.extend([{'filename': 'nginx-proxy-access.log', + 'description': 'Nginx proxy access', + 'path': '/var/cache/cpsagent/nginx-proxy-access.log'}, + {'filename': 'nginx-proxy-error.log', + 'description': 'Nginx proxy error', + 'path': '/var/cache/cpsagent/nginx-proxy-error.log'}]); + elif role == self.ROLE_PROXY_BACKEND_WEB: + logs.extend(self.get_role_logs(self.ROLE_PROXY, False)) + logs.extend(self.get_role_logs(self.ROLE_WEB, False)) + logs.extend(self.get_role_logs(self.ROLE_BACKEND, False)) + elif role == self.ROLE_PROXY_BACKEND: + logs.extend(self.get_role_logs(self.ROLE_PROXY, False)) + logs.extend(self.get_role_logs(self.ROLE_BACKEND, False)) + elif role == self.ROLE_PROXY_WEB: + logs.extend(self.get_role_logs(self.ROLE_PROXY, False)) + logs.extend(self.get_role_logs(self.ROLE_WEB, False)) + + return logs + def _configuration_get(self): return self.service_config @@ -159,6 +192,21 @@ def _create_initial_configuration(self): # def _adapting_get_count(self): # return self.memcache.get('adapting_count') + def _update_roles(self, nodes): + for node in nodes: + if node.isRunningProxy and node.isRunningWeb and node.isRunningBackend: + node.role = self.ROLE_PROXY_BACKEND_WEB + elif node.isRunningProxy and node.isRunningBackend: + node.role = self.ROLE_PROXY_BACKEND + elif node.isRunningProxy and node.isRunningWeb: + node.role = self.ROLE_PROXY_WEB + elif node.isRunningBackend: + node.role = self.ROLE_BACKEND + elif node.isRunningProxy: + node.role = self.ROLE_PROXY + elif node.isRunningWeb: + node.role = self.ROLE_WEB + def _start_scalaris(self, config, nodes, first_start): # If needed, must be overridden by extending classes pass @@ -249,7 +297,7 @@ def on_start(self, nodes): elif node.role == self.ROLE_BACKEND: kwargs = {'runBackend': True} - config.serviceNodes[node.id] = WebServiceNode(node, **kwargs) + config.serviceNodes[str(node.id)] = WebServiceNode(node, **kwargs) config.update_mappings() @@ -456,26 +504,25 @@ def on_add_nodes(self, nodes): for i in backendNodesKill: i.isRunningBackend = False - node_instances = nodes - # node_instances = [] + # nodes = [] # try: # self._adapting_set_count( # len(proxyNodesNew) + len(webNodesNew) + len(backendNodesNew)) # if proxy > 0: - # node_instances += self.controller.create_nodes( + # nodes += self.controller.create_nodes( # len(proxyNodesNew), client.check_agent_process, self.AGENT_PORT, cloud) # if web > 0 and vm_web_type is None: - # node_instances += self.controller.create_nodes( + # nodes += self.controller.create_nodes( # len(webNodesNew), client.check_agent_process, self.AGENT_PORT, cloud) # elif web > 0: - # node_instances += self.controller.create_nodes( + # nodes += self.controller.create_nodes( # len(webNodesNew), client.check_agent_process, self.AGENT_PORT, cloud, inst_type=vm_web_type) # if backend > 0 and vm_backend_type is None: - # node_instances += self.controller.create_nodes( + # nodes += self.controller.create_nodes( # len(backendNodesNew), client.check_agent_process, self.AGENT_PORT, cloud) # elif backend > 0: - # node_instances += self.controller.create_nodes( + # nodes += self.controller.create_nodes( # len(backendNodesNew), client.check_agent_process, self.AGENT_PORT, cloud, inst_type=vm_backend_type) # except: # self.logger.exception( @@ -490,11 +537,12 @@ def on_add_nodes(self, nodes): i = 0 newNodes = [] for kwargs in proxyNodesNew + webNodesNew + backendNodesNew: - config.serviceNodes[node_instances[i].id] = WebServiceNode( - node_instances[i], self.DEFAULT_NODE_WEIGHT, self.DEFAULT_NODE_WEIGHT, **kwargs) - newNodes += [config.serviceNodes[node_instances[i].id]] + config.serviceNodes[str(nodes[i].id)] = WebServiceNode( + nodes[i], self.DEFAULT_NODE_WEIGHT, self.DEFAULT_NODE_WEIGHT, **kwargs) + newNodes += [config.serviceNodes[nodes[i].id]] i += 1 config.update_mappings() + self._update_roles(config.serviceNodes.values()) # start the Scalaris process to keep the session data self._start_scalaris(config, nodes, False) @@ -522,8 +570,8 @@ def on_add_nodes(self, nodes): self._update_proxy( config, [i for i in config.serviceNodes.values() if i.isRunningProxy and i not in newNodes]) - # remove_nodes old ones + # remove_nodes old ones self._stop_backend(config, backendNodesKill) self._stop_web(config, webNodesKill) @@ -850,6 +898,7 @@ def on_remove_nodes(self, node_roles): packingNode.isRunningWeb = True config.update_mappings() + self._update_roles(config.serviceNodes.values()) self.logger.info("Remove_nodes " + str(config.serviceNodes)) # new nodes if packBackend: @@ -1171,7 +1220,7 @@ def list_nodes(self, kwargs): @expose('GET') def get_node_info(self, kwargs): config = self._configuration_get() - exp_params = [('serviceNodeId', is_in_list(config.serviceNodes))] + exp_params = [('serviceNodeId', is_in_list(config.serviceNodes.keys()))] try: serviceNodeId = check_arguments(exp_params, kwargs) except Exception as ex: @@ -1187,7 +1236,9 @@ def get_node_info(self, kwargs): 'isRunningWeb': serviceNode.isRunningWeb, 'isRunningBackend': serviceNode.isRunningBackend, 'weightWeb': serviceNode.weightWeb, - 'weightBackend': serviceNode.weightBackend + 'weightBackend': serviceNode.weightBackend, + 'role': serviceNode.role, + 'logs': self.get_role_logs(serviceNode.role) } }) @@ -1315,6 +1366,7 @@ def _do_migrate_nodes(self, migration_plan, delay): raise ex finally: config.update_mappings() + self._update_roles(config.serviceNodes.values()) config.proxy_count = len(config.getProxyServiceNodes()) config.backend_count = len(config.getBackendServiceNodes()) if config.backend_count == 1 and config.getBackendServiceNodes()[0] in config.getProxyServiceNodes(): diff --git a/conpaas-services/src/conpaas/services/webservers/manager/internal/java.py b/conpaas-services/src/conpaas/services/webservers/manager/internal/java.py index b828115e..64b7ba12 100755 --- a/conpaas-services/src/conpaas/services/webservers/manager/internal/java.py +++ b/conpaas-services/src/conpaas/services/webservers/manager/internal/java.py @@ -12,12 +12,14 @@ import tempfile import stat import os.path +import datetime from conpaas.services.webservers.manager.config import CodeVersion, JavaServiceConfiguration from conpaas.services.webservers.agent import client from conpaas.services.webservers.misc import archive_open, archive_get_members from conpaas.core.https.server import HttpErrorResponse, HttpJsonResponse +from conpaas.core.manager import BaseManager from . import BasicWebserversManager, ManagerException from conpaas.core.expose import expose from conpaas.core import git @@ -35,6 +37,30 @@ def __init__(self, config_parser, **kwargs): if kwargs['reset_config']: self._create_initial_configuration() + def get_role_logs(self, role, add_default=True): + if add_default: + logs = BaseManager.get_role_logs(self, role) + else: + logs = [] + + if role == self.ROLE_BACKEND: + now = datetime.datetime.now() + access_log = ('localhost_access_log.%d-%02d-%02d.txt' % + (now.year, now.month, now.day)) + + logs.extend([{'filename': 'access_log.txt', + 'description': 'Tomcat access', + 'path': '/var/cache/cpsagent/tomcat_instance/logs/' + '%s' % access_log}, + {'filename': 'catalina.out', + 'description': 'Tomcat output', + 'path': '/var/cache/cpsagent/tomcat_instance/logs/' + 'catalina.out'}]); + else: + logs.extend(BasicWebserversManager.get_role_logs(self, role, False)) + + return logs + def _update_code(self, config, nodes): for serviceNode in nodes: # Push the current code version via GIT if necessary diff --git a/conpaas-services/src/conpaas/services/webservers/manager/internal/php.py b/conpaas-services/src/conpaas/services/webservers/manager/internal/php.py index c8b2cd2f..5d35fb09 100755 --- a/conpaas-services/src/conpaas/services/webservers/manager/internal/php.py +++ b/conpaas-services/src/conpaas/services/webservers/manager/internal/php.py @@ -14,6 +14,8 @@ from conpaas.services.webservers.manager.config import CodeVersion, PHPServiceConfiguration from conpaas.services.webservers.agent import client from conpaas.core.https.server import HttpErrorResponse, HttpJsonResponse + +from conpaas.core.manager import BaseManager from . import BasicWebserversManager, ManagerException from conpaas.core.expose import expose @@ -49,6 +51,24 @@ def __init__(self, config_parser, **kwargs): except Exception as ex: self.logger.exception('Failed to initialize the Provisioning Manager %s' % str(ex)) + def get_role_logs(self, role, add_default=True): + if add_default: + logs = BaseManager.get_role_logs(self, role) + else: + logs = [] + + if role == self.ROLE_BACKEND: + logs.extend([{'filename': 'fpm-access.log', + 'description': 'PHP access', + 'path': '/var/cache/cpsagent/fpm-access.log'}, + {'filename': 'fpm-error.log', + 'description': 'PHP error', + 'path': '/var/cache/cpsagent/fpm-error.log'}]); + else: + logs.extend(BasicWebserversManager.get_role_logs(self, role, False)) + + return logs + def _render_scalaris_node(self, node, role): ip = node.ip.replace('.', ',') return '{{' + ip + '},14195,' + role + '}' diff --git a/conpaas-services/src/conpaas/services/xtreemfs/manager/manager.py b/conpaas-services/src/conpaas/services/xtreemfs/manager/manager.py index 3a0ee34f..a7629abe 100644 --- a/conpaas-services/src/conpaas/services/xtreemfs/manager/manager.py +++ b/conpaas-services/src/conpaas/services/xtreemfs/manager/manager.py @@ -53,6 +53,9 @@ class XtreemFSManager(BaseManager): ROLE_MRC = 'mrc' # Metadata and Replica Catalog ROLE_OSD = 'osd' # Object Storage Device + # Packed node types + ROLE_DIR_MRC_OSD = 'dir_mrc_osd' # DIR + MRC + OSD + def __init__(self, config_parser, **kwargs): BaseManager.__init__(self, config_parser) @@ -96,14 +99,42 @@ def get_node_roles(self): def get_default_role(self): return self.ROLE_OSD + def get_starting_nodes(self): + return { self.ROLE_DIR_MRC_OSD: 1 } + def get_role_sninfo(self, role, cloud): - if role == self.ROLE_OSD: + if role == self.ROLE_OSD or role == self.ROLE_DIR_MRC_OSD: return self.get_standard_sninfo_with_volume( role, cloud, 'osd-%(vm_id)s', self.osd_volume_size) else: return BaseManager.get_role_sninfo(self, role, cloud) + def get_role_logs(self, role, add_default=True): + if add_default: + logs = BaseManager.get_role_logs(self, role) + else: + logs = [] + + if role == self.ROLE_DIR: + logs.extend([{'filename': 'dir.log', + 'description': 'DIR log', + 'path': '/var/log/xtreemfs/dir.log'}]); + elif role == self.ROLE_MRC: + logs.extend([{'filename': 'mrc.log', + 'description': 'MRC log', + 'path': '/var/log/xtreemfs/mrc.log'}]); + elif role == self.ROLE_OSD: + logs.extend([{'filename': 'osd.log', + 'description': 'OSD log', + 'path': '/var/log/xtreemfs/osd.log'}]); + elif role == self.ROLE_DIR_MRC_OSD: + logs.extend(self.get_role_logs(self.ROLE_DIR, False)) + logs.extend(self.get_role_logs(self.ROLE_MRC, False)) + logs.extend(self.get_role_logs(self.ROLE_OSD, False)) + + return logs + def __get__uuid(self, node_id, node_type): if node_type == self.ROLE_DIR: node_map = self.dir_node_uuid_map @@ -683,7 +714,9 @@ def get_node_info(self, kwargs): 'cloud': serviceNode.cloud_name, 'dir': serviceNode in self.dirNodes, 'mrc': serviceNode in self.mrcNodes, - 'osd': serviceNode in self.osdNodes + 'osd': serviceNode in self.osdNodes, + 'role': serviceNode.role, + 'logs': self.get_role_logs(serviceNode.role) } }) diff --git a/cps-tools/src/cps_tools/service.py b/cps-tools/src/cps_tools/service.py index 7b8d5e5a..d2ad6e51 100644 --- a/cps-tools/src/cps_tools/service.py +++ b/cps-tools/src/cps_tools/service.py @@ -42,6 +42,7 @@ def __init__(self, serv_parser, client, service_type=None, roles=None, self._add_get_script() self._add_set_script() self._add_get_log() + self._add_list_agent_logs() self._add_get_agent_log() self._add_get_history() self._add_add_nodes() @@ -367,11 +368,7 @@ def list_nodes(self, args): params = { 'serviceNodeId': node } details = self.client.call_manager_get(app_id, service_id, "get_node_info", params) except Exception: - print "WARNING: got node identifier from list_nodes but failed on get_node_info", - if type(details) is dict and 'error' in details: - print ": %s" % details['error'] - else: - print "." + print "WARNING: got node identifier from list_nodes but failed on get_node_info." row = { 'aid': app_id, 'sid': service_id, @@ -584,6 +581,26 @@ def delete_volume(self, args): else: print "FAILED!" + # ========== list_agent_logs + def _add_list_agent_logs(self): + subparser = self.add_parser('list_agent_logs', + help="list the log files available for an agent") + subparser.set_defaults(run_cmd=self.list_agent_logs, parser=subparser) + subparser.add_argument('app_name_or_id', + help="Name or identifier of an application") + subparser.add_argument('serv_name_or_id', + help="Name or identifier of a service") + subparser.add_argument('agent_id', + help="Id of the agent (use list_nodes to find it)") + + def list_agent_logs(self, args): + app_id, service_id = self.check_service(args.app_name_or_id, args.serv_name_or_id) + + params = { 'serviceNodeId': args.agent_id } + res = self.client.call_manager_get(app_id, service_id, "get_node_info", params) + + print self.client.prettytable(('filename', 'description'), res['serviceNode']['logs']) + # ========== get_agent_log def _add_get_agent_log(self): subparser = self.add_parser('get_agent_log', @@ -596,7 +613,8 @@ def _add_get_agent_log(self): subparser.add_argument('agent_id', help="Id of the agent (use list_nodes to find it)") subparser.add_argument('-f', '--filename', metavar='FILENAME', - default=None, help="log file name") + default=None, help="log file name (use " + "list_agent_logs to find it)") def get_agent_log(self, args): app_id, service_id = self.check_service(args.app_name_or_id, args.serv_name_or_id) @@ -604,7 +622,7 @@ def get_agent_log(self, args): params = { 'agentId': args.agent_id } if args.filename: params['filename'] = args.filename - res = self.client.call_manager_get(app_id, 0, "get_agent_log", params) + res = self.client.call_manager_get(app_id, service_id, "get_agent_log", params) print res['log']