Replace psi python module with psutil

.travis.yml

@@ -43,7 +43,7 @@ python:
 
 install:
   - pip install --user --upgrade pip
-  - pip install --user --pre psi
+  - pip install --user --pre psutil
   - pip install --user lockfile
   - pip install --user paramiko
   - pip install --user setuptools

README.amazon_linux

@@ -43,7 +43,7 @@ sudo make install
 sudo pip install --upgrade pip
 as root
     pip install epydoc
-    pip install psi
+    pip install psutil
     pip install --upgrade lockfile
 
 # init GPDB

README.debian

@@ -27,8 +27,8 @@ pip install lockfile
 pip install paramiko
 pip install setuptools
 pip install epydoc
-pip install psi
-Note: debian8 required pip install --pre psi
+pip install psutil
+Note: debian8 required pip install --pre psutil
 
 sysctl.conf 
 kernel.shmmax = 500000000

README.md

@@ -40,7 +40,7 @@ to the segments, and collects the results.
   You will need to add the following Python modules (2.7 & 2.6 are
   supported) into your installation
 
-  * psi
+  * psutil
   * lockfile (>= 0.9.1)
   * paramiko
   * setuptools

gpAux/gpperfmon/src/gpperfmon.py

@@ -14,8 +14,8 @@
 import shutil
 import ConfigParser
 import re
-import psi.process
 import getpass
+import psutil
 from gppylib.db import dbconn
 
 GPPERFMONHOME=os.getenv('GPPERFMONHOME')
@@ -244,12 +244,10 @@ def lighty_stop(instance):
     status = lighty_status(instance)
     if status == RUNNING or status == STRANDED_LIGHT or status == ONLY_LIGHT:
         try:
-            ptable = psi.process.ProcessTable()
-            for pid in ptable:
-                process = ptable[pid]
+            for process in psutil.process_iter():
 
                 try:
-                    process_args  = process.args
+                    process_args  = process.cmdline()
                 except:
                     continue
                     # we do not have privalege to view/kill this process, so skip it
@@ -261,8 +259,8 @@ def lighty_stop(instance):
                 if not re.search(instance_conf, process_args[2]):
                     continue
 
-                print "killing pid %d" % pid
-                os.kill(pid, signal.SIGTERM)
+                print "killing pid %d" % process.pid
+                os.kill(process.pid, signal.SIGTERM)
 
             # lighty shuts down quickly, but give it a little bit of time.
             time.sleep(1)
@@ -289,22 +287,19 @@ def lighty_stop(instance):
 def kill_orphaned_cgi_scripts():
     "Search for any gpmonws.py that are stranded and do not belong to lighttpd"
 
-    ptable = psi.process.ProcessTable()
-    for pid in ptable:
-        process = ptable[pid]
-
+    for process in psutil.process_iter():
         try:
-            process_args  = process.args
+            process_args  = process.cmdline()
         except:
             continue
             # we do not have privalege to view/kill this process, so skip it
 
         if len(process_args) >= 2:
             if process_args[0] == 'python':
                 if re.search("gpmonws.py", process_args[1]):
-                    if process.ppid == 1:
-                        print "Killing stranded gpmonws.py process with pid %d" % pid
-                        os.kill(pid, signal.SIGTERM)
+                    if process.ppid() == 1:
+                        print "Killing stranded gpmonws.py process with pid %d" % process.pid
+                        os.kill(process.pid, signal.SIGTERM)
 
 #################
 def lighty_restart(instance):
@@ -346,12 +341,10 @@ def lighty_status(instance):
         pass
 
     try:
-        ptable = psi.process.ProcessTable()
-        for pid in ptable:
-            process = ptable[pid]
+        for process in psutil.process_iter():
 
             try:
-                process_args  = process.args
+                process_args  = process.cmdline()
             except:
                 continue
                 # we do not have privalege to view/kill this process, so skip it
@@ -361,7 +354,7 @@ def lighty_status(instance):
 
             # lighttpd process
             if re.search(light_bin, process_args[0]):
-                if lightpid != 0 and pid == lightpid:
+                if lightpid != 0 and process.pid == lightpid:
                     foundLighthttp = True
                 elif len(process_args) >= 3:
                     if re.search(instance_conf, process_args[2]):
@@ -372,7 +365,7 @@ def lighty_status(instance):
                 if len(process_args) < 2:
                     continue
                 if re.search(instance_mon, process_args[1]):
-                    if lightpid != 0 and process.ppid == lightpid:
+                    if lightpid != 0 and process.ppid() == lightpid:
                         foundGpmonws = True
                     else:
                         foundStrandedPython = True

gpMgmt/bin/Makefile

@@ -31,7 +31,7 @@ PYLIB_DIR=$(SRC)/ext
 core: pygresql subprocess32
 
 ifneq "$(wildcard $(CURDIR)/pythonSrc/ext/*.tar.gz)" ""
-all: core lockfile paramiko pycrypto stream pychecker psi unittest2
+all: core lockfile paramiko pycrypto stream pychecker psutil unittest2
 else
 all: core stream
 endif
@@ -119,16 +119,16 @@ pycrypto:
 	cp -r $(PYLIB_SRC_EXT)/$(PYCRYPTO_DIR)/build/lib.*/Crypto $(PYLIB_DIR)
 
 #
-# PSI
+# PSUTIL
 #
-PSI_VERSION=0.3b2_gp
-PSI_DIR=PSI-$(PSI_VERSION)
-
-psi:
-	@echo "--- psi"
-	cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PSI_DIR).tar.gz
-	cd $(PYLIB_SRC_EXT)/$(PSI_DIR)/ && CC="$(CC)" python setup.py build
-	cp -r $(PYLIB_SRC_EXT)/$(PSI_DIR)/build/lib.*/psi $(PYLIB_DIR)
+PSUTIL_VERSION=4.0.0
+PSUTIL_DIR=psutil-$(PSUTIL_VERSION)
+
+psutil:
+	@echo "--- psutil"
+	cd $(PYLIB_SRC_EXT)/ && $(TAR) xzf $(PSUTIL_DIR).tar.gz
+	cd $(PYLIB_SRC_EXT)/$(PSUTIL_DIR)/ && CC="$(CC)" python setup.py build
+	cp -r $(PYLIB_SRC_EXT)/$(PSUTIL_DIR)/build/lib.*/psutil $(PYLIB_DIR)
 
 
 #

gpMgmt/bin/gpdebug

@@ -8,8 +8,7 @@
 """
 import os, signal, time
 
-from psi import AttrInsufficientPrivsError
-from psi.process import Process, ProcessTable, NoSuchProcessError
+import psutil
 
 from gppylib import gplog, pgconf
 from gppylib.db import dbconn
@@ -339,10 +338,11 @@ class GpDebugMain(YamlMain):
         ""
         self.logger.info("Era: %s" % self.options.era)
         self.orphans = []
-        for p in ProcessTable().values():
+        for p in psutil.process_iter():
             try:
-                d = dict(p.env)                
-            except AttrInsufficientPrivsError:
+                d = p.environ()
+                command = ' '.join(p.cmdline())
+            except psutil.Error:
                 continue
             e = d.get('GPERA')
             s = d.get('GPSESSID')
@@ -352,7 +352,7 @@ class GpDebugMain(YamlMain):
             if e is None:
                 continue
             if e != self.options.era:
-                self.orphans.append((e, s, str(p.pid), p.command.strip()))
+                self.orphans.append((e, s, str(p.pid), command.strip()))
                 continue
 
             # look for process in same era with a session id not in
@@ -369,7 +369,7 @@ class GpDebugMain(YamlMain):
                 if s == '0' or s in (self.options.sessionids or []):
                     continue
 
-            self.orphans.append((e, s, str(p.pid), p.command.strip()))
+            self.orphans.append((e, s, str(p.pid), command.strip()))
 
     def return_orphan_details(self):
         ""

gpMgmt/bin/gppylib/commands/unix.py

@@ -8,11 +8,11 @@
 
 """
 import os
+import psutil
 import platform
 import socket
 import sys
 import signal
-import psi.process
 
 from gppylib.gplog import *
 from gppylib.commands.base import *
@@ -961,21 +961,15 @@ def file_uid(self):
 #--------------get list of desecendant processes -------------------
 
 def getDescendentProcesses(pid):
-    ''' return all process which are descendant from the given processid ''' 
+    ''' return all process pids which are descendant from the given processid ''' 
 
-    children = list()
-    grandchildren = list()
+    children_pids = []
 
-    for p in psi.process.ProcessTable().values():
+    for p in psutil.Process(pid).children(recursive=True):
+        if p.is_running():
+            children_pids.append(p.pid)
 
-        if int(p.ppid) == int(pid):
-            children.append(int(p.pid))
-
-    # recursion
-    for child in children:
-        grandchildren.extend( getDescendentProcesses(child) )
-
-    return children + grandchildren
+    return children_pids 
 
 
 #--------------global variable initialization ----------------------

gpMgmt/bin/gppylib/programs/kill.py

@@ -8,7 +8,7 @@
 from gppylib.mainUtils import addStandardLoggingAndHelpOptions, ProgramArgumentValidationException
 from gppylib.commands.unix import kill_sequence, check_pid 
 from gppylib.operations.package import dereference_symlink
-from psi.process import Process, NoSuchProcessError
+from psutil import Process, NoSuchProcess
 
 logger = gplog.get_default_logger()
 
@@ -94,12 +94,13 @@ def examine_process(self, pid):
 
         try:    
             proc = Process(pid=pid)
-        except NoSuchProcessError, e:
+        except NoSuchProcess, e:
             raise KillError('Process with pid(%s) does not exist' % pid)
 
-        logger.info('process %s is %s' % (pid, proc.command.strip()))
+        command = ' '.join(proc.cmdline())
+        logger.info('process %s is %s' % (pid, command.strip()))
 
-        return proc.command.strip()
+        return command.strip()
 
     def terminate_process(self, pid):
 

gpMgmt/bin/gppylib/test/regress/test_regress_gpssh.py

@@ -2,7 +2,8 @@
 
 import os, signal, time, re
 import unittest2 as unittest
-import psi.process, subprocess
+import psutil
+from subprocess import PIPE
 
 class GpsshTestCase(unittest.TestCase):
 
@@ -12,12 +13,12 @@ def searchForProcessOrChildren(self):
         euid = os.getuid()
         count = 0
 
-        for p in psi.process.ProcessTable().values():
-            if p.euid != euid:
+        for p in psutil.process_iter():
+            if p.uids().effective != euid:
                 continue
-            if not re.search('ssh', p.command):
+            if not re.search('ssh', ' '.join(p.cmdline())):
                 continue
-            if p.ppid != 1:
+            if p.ppid() != 1:
                 continue
 
             count += 1
@@ -32,7 +33,7 @@ def test00_gpssh_sighup(self):
 
         before_count = self.searchForProcessOrChildren()
 
-        p = subprocess.Popen("gpssh -h localhost", shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        p = psutil.Popen("gpssh -h localhost", shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
         pid = p.pid
 
         time.sleep(3)

vagrant/centos/vagrant-setup.sh

@@ -20,7 +20,7 @@ sudo yum -y install perl-Env
 sudo yum -y install ccache
 wget https://bootstrap.pypa.io/get-pip.py
 sudo python get-pip.py
-sudo pip install psi lockfile paramiko setuptools epydoc
+sudo pip install psutil lockfile paramiko setuptools epydoc
 rm get-pip.py
 
 # Misc

vagrant/debian/vagrant-setup.sh

@@ -14,7 +14,7 @@ pip install lockfile
 pip install paramiko
 pip install setuptools
 pip install epydoc
-pip install --pre psi
+pip install --pre psutil
 
 sudo service docker start
 sudo useradd postgres