diff --git a/Makefile.am b/Makefile.am index 911e4e6318..31f8a64869 100644 --- a/Makefile.am +++ b/Makefile.am @@ -3,6 +3,23 @@ include $(top_srcdir)/buildutils/config.mk ACLOCAL_AMFLAGS = -I buildutils SUBDIRS = src doc contrib/blcr contrib/init.d contrib/systemd +noinst_SCRIPTS = torque.setup + +edit_scripts = sed \ + -e "s|@exec_prefix[@]|$(exec_prefix)|g" \ + -e "s|@PBS_HOME[@]|$(PBS_SERVER_HOME)|g" + +torque.setup: Makefile + rm -f $@ $@.tmp + my_srcdir=''; \ + test -f ./$@.in || my_srcdir=$(srcdir)/; \ + $(edit_scripts) $${my_srcdir}$@.in >$@.tmp + chmod +x $@.tmp + chmod a-w $@.tmp + mv $@.tmp $@ + +torque.setup: torque.setup.in + EXTRA_DIST = acinclude.m4 \ CHANGELOG \ configure.ac \ @@ -25,7 +42,7 @@ EXTRA_DIST = acinclude.m4 \ README.new_in_51 \ Release_Notes \ run_report.py \ - torque.setup \ + torque.setup.in \ torque.spec \ buildutils/config.mk \ buildutils/determine.distro \ @@ -68,6 +85,7 @@ EXTRA_DIST = acinclude.m4 \ bin_SCRIPTS = pbs-config +CLEANFILES = torque.setup DISTCLEANFILES = pbs-config current_hash MOSTLYCLEANFILES = *.gcda *.gcno *.gcov diff --git a/buildutils/torque-build b/buildutils/torque-build index fa71c500b3..ed6782d48e 100755 --- a/buildutils/torque-build +++ b/buildutils/torque-build @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash POSIXLY_CORRECT=1 @@ -29,7 +29,8 @@ usage() { echo " /opt/coverity/default/bin/cov-build)" echo " -C|--no-coverity Do not run coverity (default)" echo "" - echo " -g|--enable-cgroups Enable cgroups" + echo " -g|--enable-cgroups Enable cgroups (default)" + echo " -G|--disable-cgroups Do not enable cgroups" echo " -u|--unit-tests Run unit-tests (default)" echo " -U|--no-unit-tests Do not run unit tests" echo "" @@ -134,6 +135,11 @@ do cgroups=1 shift ;; + -G|--disable-cgroups) + cgroups=0 + with_option="$with_option --without cgroups --without hwloc" + shift + ;; *) echo "Invalid argument '${1}' specified." >&2 usage @@ -225,6 +231,7 @@ then cgroupsopts="--enable-cgroups" configure_options="$configure_options --enable-cgroups --with-hwloc-path=$hwloc_path" else + hwloc_path=none cgroupsopts="--disable-cgroups" fi @@ -289,8 +296,10 @@ then mkdir -p buildutils/rpmbuild/SOURCES mkdir -p buildutils/rpmbuild/SPECS mkdir -p buildutils/rpmbuild/RPMS/{i386,i686,noarch,x86_64} - # get the distro - distro=$(buildutils/determine.distro -r) + + # ignore SP version suffix for SLES12 + echo "${distro}" | fgrep -q sles12. && distro=sles12 + if [ "${distro}" = el7 ]; then cp buildutils/torque.adaptive.spec.el7 \ buildutils/rpmbuild/SPECS/torque.adaptive.spec diff --git a/buildutils/torque.adaptive.spec.el6 b/buildutils/torque.adaptive.spec.el6 index 7fc84ab6c9..2c369a54c2 100644 --- a/buildutils/torque.adaptive.spec.el6 +++ b/buildutils/torque.adaptive.spec.el6 @@ -63,9 +63,12 @@ unset CONFFLAGS \ CONFFLAGS_FILE=`mktemp %{_tmppath}/confflags.XXXX` \ ./configure --help 2>&1 > ${CONFFLAGS_FILE} \ - if grep -q -- '--with-hwloc-path' "${CONFFLAGS_FILE}" \ + if test %{_hwlocpath} != none \ then \ - CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ + if grep -q -- '--with-hwloc-path' "${CONFFLAGS_FILE}" \ + then \ + CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ + fi \ fi \ if grep -q -- '--enable-cgroups' "${CONFFLAGS_FILE}" \ then \ @@ -348,7 +351,7 @@ ### Macro variables %{!?torque_user:%global torque_user root} %{!?torque_home:%global torque_home %{_var}/spool/%{project_name}} -%{!?sendmail_path:%global sendmail_path %{_sbindir}/sendmail} +%{!?sendmail_path:%global sendmail_path /usr/sbin/sendmail} # This ensures that the debugging symbols are not stripped %define __os_install_post /usr/lib/rpm/brp-compress @@ -600,10 +603,10 @@ echo '$logevent 225 # bitmap of which events to log' > \ %{buildroot}%{torque_sysconfdir}/mom_priv/config %{__mkdir_p} %{buildroot}/etc/profile.d -echo 'test -d ${_bindir} && export PATH=%{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.sh +echo 'test -d %{_bindir} && export PATH=%{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.sh echo 'test `id -u` = 0 && test -d %{_sbindir} && export PATH=%{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.sh -echo 'test -d ${_bindir} && setenv PATH %{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.csh -echo 'test `id -u` = 0 && test-d ${_sbindir} && setenv PATH %{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.csh +echo 'test -d %{_bindir} && setenv PATH %{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.csh +echo 'test `id -u` = 0 && test -d %{_sbindir} && setenv PATH %{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.csh # Moab requires libtorque.so.0, but works with libtorque.so.2, so fudge it %{__ln_s} libtorque.so.2 %{buildroot}%{_libdir}/libtorque.so.0 @@ -674,8 +677,6 @@ TIMESTAMP="`date +%%Y.%%m.%%d_%%H.%%M.%%S`" %{pre_add_back_up_file %{_bindir}/printjob %{client_sub}-${TIMESTAMP}} %{pre_add_back_up_file %{_bindir}/printserverdb %{client_sub}-${TIMESTAMP}} %{pre_add_back_up_file %{_bindir}/printtracking %{client_sub}-${TIMESTAMP}} -%{pre_add_back_up_file %{_bindir}/printtracking %{client_sub}-${TIMESTAMP}} -%{pre_add_back_up_file %{_bindir}/printtracking %{client_sub}-${TIMESTAMP}} for file in %{_bindir}/q* do %{pre_add_back_up_file ${file} %{client_sub}-${TIMESTAMP}} diff --git a/buildutils/torque.adaptive.spec.el7 b/buildutils/torque.adaptive.spec.el7 index 74dd1b2065..344bc13523 100644 --- a/buildutils/torque.adaptive.spec.el7 +++ b/buildutils/torque.adaptive.spec.el7 @@ -63,9 +63,12 @@ unset CONFFLAGS \ CONFFLAGS_FILE=`mktemp %{_tmppath}/confflags.XXXX` \ ./configure --help 2>&1 > ${CONFFLAGS_FILE} \ - if grep -q -- '--with-hwloc-path' "${CONFFLAGS_FILE}" \ + if test %{_hwlocpath} != none \ then \ - CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ + if grep -q -- '--with-hwloc-path' "${CONFFLAGS_FILE}" \ + then \ + CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ + fi \ fi \ if grep -q -- '--enable-cgroups' "${CONFFLAGS_FILE}" \ then \ @@ -348,7 +351,7 @@ ### Macro variables %{!?torque_user:%global torque_user root} %{!?torque_home:%global torque_home %{_var}/spool/%{project_name}} -%{!?sendmail_path:%global sendmail_path %{_sbindir}/sendmail} +%{!?sendmail_path:%global sendmail_path /usr/sbin/sendmail} # This ensures that the debugging symbols are not stripped %define __os_install_post /usr/lib/rpm/brp-compress @@ -599,10 +602,10 @@ echo '$logevent 225 # bitmap of which events to log' > \ %{buildroot}%{torque_sysconfdir}/mom_priv/config %{__mkdir_p} %{buildroot}/etc/profile.d -echo 'test -d ${_bindir} && export PATH=%{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.sh +echo 'test -d %{_bindir} && export PATH=%{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.sh echo 'test `id -u` = 0 && test -d %{_sbindir} && export PATH=%{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.sh -echo 'test -d ${_bindir} && setenv PATH %{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.csh -echo 'test `id -u` = 0 && test-d ${_sbindir} && setenv PATH %{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.csh +echo 'test -d %{_bindir} && setenv PATH %{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.csh +echo 'test `id -u` = 0 && test -d %{_sbindir} && setenv PATH %{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.csh # Moab requires libtorque.so.0, but works with libtorque.so.2, so fudge it %{__ln_s} libtorque.so.2 %{buildroot}%{_libdir}/libtorque.so.0 diff --git a/buildutils/torque.adaptive.spec.sles12 b/buildutils/torque.adaptive.spec.sles12 index 74dd1b2065..63215bc56f 100644 --- a/buildutils/torque.adaptive.spec.sles12 +++ b/buildutils/torque.adaptive.spec.sles12 @@ -63,9 +63,12 @@ unset CONFFLAGS \ CONFFLAGS_FILE=`mktemp %{_tmppath}/confflags.XXXX` \ ./configure --help 2>&1 > ${CONFFLAGS_FILE} \ - if grep -q -- '--with-hwloc-path' "${CONFFLAGS_FILE}" \ + if test %{_hwlocpath} != none \ then \ - CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ + if grep -q -- '--with-hwloc-path' "${CONFFLAGS_FILE}" \ + then \ + CONFFLAGS='--with-hwloc-path=%{_hwlocpath}' \ + fi \ fi \ if grep -q -- '--enable-cgroups' "${CONFFLAGS_FILE}" \ then \ @@ -348,7 +351,7 @@ ### Macro variables %{!?torque_user:%global torque_user root} %{!?torque_home:%global torque_home %{_var}/spool/%{project_name}} -%{!?sendmail_path:%global sendmail_path %{_sbindir}/sendmail} +%{!?sendmail_path:%global sendmail_path /usr/sbin/sendmail} # This ensures that the debugging symbols are not stripped %define __os_install_post /usr/lib/rpm/brp-compress @@ -599,10 +602,10 @@ echo '$logevent 225 # bitmap of which events to log' > \ %{buildroot}%{torque_sysconfdir}/mom_priv/config %{__mkdir_p} %{buildroot}/etc/profile.d -echo 'test -d ${_bindir} && export PATH=%{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.sh +echo 'test -d %{_bindir} && export PATH=%{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.sh echo 'test `id -u` = 0 && test -d %{_sbindir} && export PATH=%{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.sh -echo 'test -d ${_bindir} && setenv PATH %{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.csh -echo 'test `id -u` = 0 && test-d ${_sbindir} && setenv PATH %{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.csh +echo 'test -d %{_bindir} && setenv PATH %{_bindir}:${PATH}' > %{buildroot}/etc/profile.d/torque.csh +echo 'test `id -u` = 0 && test -d %{_sbindir} && setenv PATH %{_sbindir}:${PATH}' >> %{buildroot}/etc/profile.d/torque.csh # Moab requires libtorque.so.0, but works with libtorque.so.2, so fudge it %{__ln_s} libtorque.so.2 %{buildroot}%{_libdir}/libtorque.so.0 diff --git a/buildutils/torque.spec.in b/buildutils/torque.spec.in index 31f2aae4ce..6f24df39c0 100644 --- a/buildutils/torque.spec.in +++ b/buildutils/torque.spec.in @@ -13,6 +13,13 @@ %define bcond_without() %{expand:%%{!?_without_%{1}:%%global with_%{1} 1}} %endif +# If _initrddir is defined, which indicates that we are on a system with SYSV +# init.d services, define _initrddir to be /etc/init.d so that the source +# tarball will work across both Red Hat and SUSE distros. +%if %{?_initrddir:1}0 +%define _initrddir /etc/init.d +%endif + # some binaries under /bin in el5|6 are only provided in /usr/bin in el7+. # the el7 rpms actually provide the old location, but can't be used as a rpm requirement # introducing the rootbin macro for this, as none seems to exists @@ -387,6 +394,9 @@ test "x$RPM_BUILD_ROOT" != "x/" && rm -rf $RPM_BUILD_ROOT %doc INSTALL INSTALL.GNU CHANGELOG PBS_License.txt README.* Release_Notes src/pam/README.pam %doc doc/READ_ME doc/doc_fonts doc/soelim.c doc/ers %config(noreplace) %{torque_home}/pbs_environment +%config(noreplace) /etc/ld.so.conf.d/torque.conf +%config(noreplace) /etc/profile.d/torque.sh +%config(noreplace) /etc/profile.d/torque.csh %config(noreplace) %{torque_home}/server_name %if %{?_unitdir:1}0 %{_unitdir}/trqauthd.service diff --git a/configure.ac b/configure.ac index 69468113f9..195d6bbe23 100644 --- a/configure.ac +++ b/configure.ac @@ -33,7 +33,7 @@ dnl added library functionality to testing dnl by John Rosenquist (jrosenquist@adaptivecomputing.com) AC_PREREQ(2.53) -AC_INIT([torque], [master], [torqueusers@supercluster.org]) +AC_INIT([torque], [6.1.2], [torqueusers@supercluster.org]) AC_REVISION($Revision$) AC_CONFIG_SRCDIR([src/cmds/qrun.c]) AC_CONFIG_AUX_DIR([buildutils]) @@ -486,6 +486,7 @@ m4_ifdef([AM_SILENT_RULES],[ src/test/checkpoint/Makefile src/test/cray_cpa/Makefile src/test/cray_energy/Makefile + src/test/cray_taskstats/Makefile src/test/generate_alps_status/Makefile src/test/mom_job_func/Makefile src/test/mom_comm/Makefile @@ -494,6 +495,7 @@ m4_ifdef([AM_SILENT_RULES],[ src/test/mom_server/Makefile src/test/mom_process_request/Makefile src/test/mom_req_quejob/Makefile + src/test/nvidia/Makefile src/test/parse_config/Makefile src/test/pbs_demux/Makefile src/test/pmix_interface/Makefile @@ -506,6 +508,7 @@ m4_ifdef([AM_SILENT_RULES],[ src/test/tmsock_recov/Makefile src/test/mom_mach/Makefile src/test/mom_start/Makefile + src/test/trq_cgroups/Makefile src/resmom/linux/test/Makefile src/resmom/linux/test/cpuset/Makefile src/resmom/linux/test/sys_file/Makefile @@ -767,7 +770,7 @@ fi dnl if using gcc, we can be very strict AC_ARG_ENABLE(gcc_warnings, [ --disable-gcc-warnings Disable gcc strictness and warnings. If using - gcc, default is to not error on any warnings]) + gcc, default is to error on all warnings]) if test "x$GCC" = "xyes" ;then AC_MSG_CHECKING([whether to disable strict gcc warnings]) if test "${enable_gcc_warnings}" = "no" ; then @@ -979,6 +982,9 @@ build_l26_cpuset=no cpuset=0 AC_MSG_CHECKING([whether to enable cpusets]) if test "x$enable_cpuset" = "xyes" ; then + if test "${build_linux_cgroups}" = "yes" ; then + AC_MSG_ERROR([cpuset and cgroups are mutually exclusive options]) + fi AC_MSG_RESULT([yes]) build_l26_cpuset=yes cpuset=1 @@ -990,6 +996,7 @@ else AC_MSG_RESULT([no]) RPM_AC_OPTS="$RPM_AC_OPTS --without cpuset" fi + AM_CONDITIONAL([BUILD_L26_CPUSETS], test "$build_l26_cpuset" = yes) @@ -1001,6 +1008,9 @@ AC_ARG_ENABLE(geometry_requests, [ if test "x$GCC" = "xyes" ;then AC_MSG_CHECKING([whether to allow geometry requests]) if test "${enable_geometry_requests}" = "yes" ; then + if test "${build_linux_cgroups}" = "yes" ; then + AC_MSG_ERROR([geometry-requests and cgroups are mutually exclusive options]) + fi AC_MSG_RESULT([yes]) build_l26_cpuset=yes AC_DEFINE(PENABLE_LINUX26_CPUSETS, 1, [Define to enable Linux 2.6 cpusets]) @@ -1015,6 +1025,16 @@ dnl AC_DEFINE(GEOMETRY_REQUESTS, 1, [Define to allow job specific geometry re fi fi + +dnl Don't allow the specification of --disable-cpuset and --enable-geometry-requests +case "${ac_configure_args}" in + *disable-cpuset*) + case "${ac_configure_args}" in + *enable-geometry-requests*) + AC_MSG_ERROR([--enable-geometry-requests requires the use of cpusets. You may not specify both --disable-cpuset and --enable-geometry-requests]) + esac +esac + dnl compile for NUMA systems - allows the site to configure: dnl 1. cpusets and multi-moms dnl 2. memory fences to only access close memory @@ -1178,7 +1198,8 @@ AC_ARG_ENABLE(nvidia-gpus, [ --enable-nvidia-gpus enable Nvidia gpu support Nvidia gpu support requires the use of the Nvidia Management Library (NVML) When using --enable-nvidia-gpus, you must also specify --with-nvml-lib=DIR and - --with-nvml-include=DIR. hwloc 1.9 or later is also required. + --with-nvml-include=DIR. + hwloc 1.9 or later is also required if implementing cgroups along with GPU support. See --with-hwloc-path. ], enable_nvidia=$enableval,enable_nvidia=no) @@ -1473,11 +1494,12 @@ AC_ARG_ENABLE(align_memory, [ --enable-align-memory Change compile flags to get rid of unaligned memory access errors]) if test "x$GCC" = "xyes" ;then AC_MSG_CHECKING([whether add memory alignment flags]) - if test "${enable_align_memory}" = "yes" ; then - AC_MSG_RESULT([yes]) - CFLAGS="$CFLAGS -fstack-protector -Wformat -Wformat-security -DFORTIFY_SOURCE=2" - else + if test "$x{enable_align_memory}" = "xno" ; then AC_MSG_RESULT([no]) + else + AC_MSG_RESULT([default yes]) + CFLAGS="$CFLAGS -fstack-protector -Wformat -Wformat-security -DFORTIFY_SOURCE=2" + CXXFLAGS="$CXXFLAGS -fstack-protector -Wformat -Wformat-security -DFORTIFY_SOURCE=2" fi fi @@ -1704,7 +1726,7 @@ case "$jobcreatedir" in esac AC_MSG_RESULT([$jobcreatedir]) -AUTH_TYPE="classic (pbs_iff)" +AUTH_TYPE="trqauthd" dnl compile for munge authorization - allows sites to use munge dnl as a means to validate user identity from remote hosts. (executable based implementation) @@ -1836,6 +1858,18 @@ esac +dnl Make a bind() call before making a connect() call +AC_MSG_CHECKING([whether to build with bind() calls before connect() calls]) +AC_ARG_ENABLE(bind-outbound-sockets, + [ --enable-bind-outbound-sockets enable build with bind() calls before connect() calls], + BIND_OUTBOUND_SOCKETS=$enableval,BIND_OUTBOUND_SOCKETS=yes) +AC_MSG_RESULT($BIND_OUTBOUND_SOCKETS) +if test "$BIND_OUTBOUND_SOCKETS" = "yes" ; then + AC_DEFINE(BIND_OUTBOUND_SOCKETS, 1, [turns on the compilation of BIND_OUTBOUND_SOCKETS code]) +fi + + + dnl dnl ###################################################################### dnl Scheduler settings @@ -1971,10 +2005,10 @@ AC_ARG_ENABLE(spool, [ $TORQUEHOME/spool and copy them to the users home directory when the job completes.], [case "${enableval}" in - yes) NO_SPOOL_OUTPUT=0 ;; - no) NO_SPOOL_OUTPUT=1 ;; + yes) NO_SPOOL_OUTPUT=0 ; RPM_AC_OPTS="$RPM_AC_OPTS --with spool" ;; + no) NO_SPOOL_OUTPUT=1 ; RPM_AC_OPTS="$RPM_AC_OPTS --without spool" ;; *) AC_MSG_ERROR(--enable-spool cannot take a value) ;; -esac],[NO_SPOOL_OUTPUT=0])dnl +esac],[NO_SPOOL_OUTPUT=0 ; RPM_AC_OPTS="$RPM_AC_OPTS --with spool"])dnl AC_DEFINE_UNQUOTED(NO_SPOOL_OUTPUT, ${NO_SPOOL_OUTPUT}, [directly use homedirs instead of $TORQUEHOME/spool]) AC_ARG_ENABLE(shell-use-argv, [ @@ -2114,6 +2148,38 @@ AC_DEFINE_UNQUOTED(TCP_RETRY_LIMIT,${TCP_RETRY_LIMIT},[Define to set a max retry +AC_ARG_WITH(reserved_port_start, [ + --with-reserved-port-start=NUMBER + Set the starting reserved port number to be used. Must be a number + greater than 143 and less than 824. Please be careful when setting this, + as restricting the number too much may not leave enough privileged ports + available to pbs_server, which can potential slow down the server + immensely], + [RESERVED_PORT_START=${withval}],[RESERVED_PORT_START=144]) + +dnl Make sure it's a completely numeric string +if [[ $(expr "x${RESERVED_PORT_START}" : "x[0-9]*$") -gt 0 ]]; then + echo "Setting reserved port to get ports from ${RESERVED_PORT_START} to 1023 inclusive." +else + echo "Value ${RESERVER_PORT_START} is not permitted for --with-reserved-port-start because it contains non-numeric characters." + exit -1 +fi + +dnl Make sure it isn't less than 144 +if [[ $RESERVED_PORT_START -lt 144 ]]; then + echo "Value ${RESERVED_PORT_START} is not permitted for --with-reserved-port-start because it is less than 144" + exit -1 +fi + +dnl Make sure it's less than 824 so that there are at least 200 reserved ports available for the daemons. +if [[ $RESERVED_PORT_START -gt 823 ]]; then + echo "Value ${RESERVED_PORT_START} is not permitted for --with-reserved-port-start because it is greater than 823 and we need a range of at least 200 possible privileged ports." + exit -1 +fi +AC_DEFINE_UNQUOTED(RESERVED_PORT_START, ${RESERVED_PORT_START},[Define to specify a different reserved port starting point]) + + + AC_ARG_WITH(default_server, [ --with-default-server=HOSTNAME set the name of the computer that clients will @@ -2308,6 +2374,7 @@ AC_ARG_WITH(xauth, ], [ TestPath="$PATH" + TestPath="${TestPath}${PATH_SEPARATOR}/usr/bin" TestPath="${TestPath}${PATH_SEPARATOR}/usr/X/bin" TestPath="${TestPath}${PATH_SEPARATOR}/usr/bin/X11" TestPath="${TestPath}${PATH_SEPARATOR}/usr/X11R6/bin" @@ -2553,6 +2620,10 @@ dnl AC_CREATE_GENERIC_CONFIG([pbs],[-ltorque]) +# Cause make rpm to default to use the prefix resulting from configure +RPM_AC_OPTS="--define '_prefix ${prefix}'$RPM_AC_OPTS" +AC_SUBST(RPM_AC_OPTS) + AC_SUBST(ALPS_LIBS) dnl these are quoted, and this removes them @@ -2560,7 +2631,6 @@ SPEC_NAME=$PACKAGE_TARNAME SPEC_VERSION=$PACKAGE_VERSION AC_SUBST(SPEC_NAME) AC_SUBST(SPEC_VERSION) -AC_SUBST(RPM_AC_OPTS) dnl Head and bottom for the config-file AH_TOP([#ifndef _PBS_CONFIG_H_ diff --git a/contrib/init.d/pbs_server.in b/contrib/init.d/pbs_server.in index 22f48ff457..4a4ae63852 100644 --- a/contrib/init.d/pbs_server.in +++ b/contrib/init.d/pbs_server.in @@ -48,21 +48,29 @@ kill_pbs_server() { pid=`pidof_pbs_server` if [ $? -ne 0 ]; then - return 0; + return 0 fi - # Note that qterm shouldn't be used since it will - # kill the master pbs_server which may be on another node (in HA mode) - kill -TERM $pid + # Use a qterm that only acts locally + $BIN_PATH/qterm -l + + # If our local qterm couldn't do anything, then send the signal as a last resort + if [ $? -ne 0 ]; then + kill -TERM $pid + fi # wait for process to end for i in {1..5}; do - kill -0 $pid &>/dev/null || return 0 - sleep 1 + kill -0 $pid &>/dev/null || return 0 + sleep 1 done # waited too long, try again - killproc pbs_server -TERM + $BIN_PATH/qterm -l + if [ $? -ne 0 ]; then + killproc pbs_server -TERM + fi + return $? } diff --git a/contrib/pestat-1.0/pestat.c b/contrib/pestat-1.0/pestat.c index d55d02f930..e3dfba3ff3 100644 --- a/contrib/pestat-1.0/pestat.c +++ b/contrib/pestat-1.0/pestat.c @@ -195,7 +195,7 @@ int main( } /* END switch (c) */ } - con = cnt2server(def_server); + con = cnt2server(def_server, false); if (con <= 0) { diff --git a/contrib/systemd/pbs_mom.service.in b/contrib/systemd/pbs_mom.service.in index 5bb8cd32b2..f832852a02 100644 --- a/contrib/systemd/pbs_mom.service.in +++ b/contrib/systemd/pbs_mom.service.in @@ -8,9 +8,12 @@ After=network.target local-fs.target syslog.target Type=simple User=root +Delegate=yes + LimitNOFILE=32768 LimitMEMLOCK=infinity LimitSTACK=12582912 +LimitCORE=infinity Environment=PBS_HOME=@PBS_HOME@ Environment=PBS_ARGS= @@ -18,5 +21,16 @@ EnvironmentFile=-/etc/sysconfig/pbs_mom ExecStart=@sbindir@/pbs_mom -F -d $PBS_HOME $PBS_ARGS +KillMode=process + +# attempt graceful shutdown +ExecStop=/bin/bash -c " \ + for i in {1..5}; do \ + kill -0 $MAINPID &>/dev/null || exit 0; \ + @sbindir@/momctl -s && exit; \ + sleep 1; \ + done \ +" + [Install] WantedBy=multi-user.target diff --git a/contrib/systemd/pbs_server.service.in b/contrib/systemd/pbs_server.service.in index 30e5197459..0cdb79a911 100644 --- a/contrib/systemd/pbs_server.service.in +++ b/contrib/systemd/pbs_server.service.in @@ -8,6 +8,8 @@ After=trqauthd.service network.target local-fs.target rsyslog.target Type=simple User=root +LimitCORE=infinity + # Let systemd guess the pid. # # PIDFile should not be set to PBS_HOME/server_priv/server.lock diff --git a/contrib/systemd/trqauthd.service.in b/contrib/systemd/trqauthd.service.in index 7c29a68e7f..0930010d60 100644 --- a/contrib/systemd/trqauthd.service.in +++ b/contrib/systemd/trqauthd.service.in @@ -8,6 +8,8 @@ After=network.target local-fs.target syslog.target Type=simple User=root +LimitCORE=infinity + ExecStart=@sbindir@/trqauthd -F ExecStop=-@sbindir@/trqauthd -d diff --git a/doc/ers/pbs_resources_all.so b/doc/ers/pbs_resources_all.so index 6690f6d893..ad8d0ffa04 100644 --- a/doc/ers/pbs_resources_all.so +++ b/doc/ers/pbs_resources_all.so @@ -93,13 +93,13 @@ Each node_spec is an of nodes required of the type declared in the node_spec and a .I name or one or more -.I properity -or properities desired for the nodes. -The number, the name, and each properity in the node_spec are separated by +.I property +or properties desired for the nodes. +The number, the name, and each property in the node_spec are separated by a colon ':'. If no number is specified, one (1) is assumed. Units: string. .IP -The name of a node is its hostname. The properities of nodes are: +The name of a node is its hostname. The properties of nodes are: .RS .IP . 2 ppn=# @@ -107,7 +107,7 @@ specifying the number of processors per node requested. Defaults to 1. .IP . 2 arbitrary string assigned by the system administrator, please check with your administrator as to the node names -and properities available to you. +and properties available to you. .RE .IP Examples: diff --git a/doc/man1/qsub.1.in b/doc/man1/qsub.1.in index ddb3728806..955d291f4e 100644 --- a/doc/man1/qsub.1.in +++ b/doc/man1/qsub.1.in @@ -83,11 +83,11 @@ qsub \- submit pbs job .SH SYNOPSIS qsub [\-a date_time] [\-A account_string] [\-b secs] [\-c checkpoint_options] [\-C directive_prefix] -[\-d path] [\-D path] [\-e path] [\-f] [\-h] [\-i idle_slot_limit] [\-I] [\-j join] [\-k keep] [\-K kill_delay] -[\-l resource_list] [\-m mail_options] [\-M user_list] [\-n node_exclusive] [\-N name] +[\-d path] [\-D path] [\-e path] [\-f] [\-F] [\-h] [\-i idle_slot_limit] [\-I] [\-j join] [\-k keep] [\-K kill_delay] +[\-l resource_list] [\-L NUMA_resource_list] [\-m mail_options] [\-M user_list] [\-n node_exclusive] [\-N name] [\-o path] [\-p priority] [\-P proxy_username[:group]] [\-q destination] [\-r c] [\-S path_list] [\-t array_request] [\-T prologue/epilogue script_name] -[\-u user_list] [\-v variable_list] [\-V] [\-w] path +[\-u user_list] [\-v variable_list] [\-V] [\-w path] [\-W additional_attributes] [\-x] [\-X] [\-z] [script] .SH DESCRIPTION To create a job is to submit an executable script to a batch server. @@ -392,6 +392,13 @@ attribute will be set to true, which indicates that the job can survive the loss of a mom other than the "mother superior" mom (the first node in the exec hosts ) .Ig +.IP "\-F" 8 +Specifies the arguments that will be passed to the job script when the script is launched. +.br +The accepted syntax is: +.br +qsub -F "myarg1 myarg2 myarg3=myarg3value" myscript2.sh +.Ig .IP "\-h" 8 Specifies that a user hold be applied to the job at submission time. .if !\n(Pb .ig Ig @@ -535,6 +542,23 @@ Other than syntax, qsub performs no resource or value checking. The checking is performed by the execution server. .Ig .RE +.IP "\-L NUMA_resource_list" 8 +Defines the NUMA-aware resource requests for NUMA hardware. This option will work with non-NUMA hardware. +Syntax for +.Ar NUMA_resource_list +is: +.br +.Ty "\ \ \ \ tasks=#[:lprocs=#|all] +.br +.Ty "\ \ \ \ [:{usecores|usethreads|allowthreads}] +.br +.Ty "\ \ \ \ [:place={socket|numanode|core|thread}[=#]{node}][:memory=#][:swap=#][:maxtpn=#][:gpus=#[:]][:mics=#] +.br +.Ty "\ \ \ \ [:gres=][:feature=] +.br +.Ty "\ \ \ \ [[:{cpt|cgroup_per_task}]|[:{cph|cgroup_per_host}]] +.Ig +.RE .IP "\-m mail_options " 8 Defines the set of conditions under which the execution server will send a mail message about the job. The mail_options diff --git a/doc/man7/pbs_server_attributes.7.in b/doc/man7/pbs_server_attributes.7.in index 22215f2f81..d8cac0dd0a 100644 --- a/doc/man7/pbs_server_attributes.7.in +++ b/doc/man7/pbs_server_attributes.7.in @@ -789,14 +789,7 @@ ignored by pbs_server. [internal type: string] .Ig .Al scheduler_iteration -The time, in seconds, between iterations of attempts by the batch server -to schedule jobs. On each iteration, the server examines the available -resources and runnable jobs to see if a job can be initiated. This -examination also occurs whenever a running batch job terminates or a -new job is placed in the queued state in an execution queue. -Format: integer seconds; default value: 10 minutes, set by -.Sc PBS_SCHEDULE_CYCLE -in server_limits.h. +Has no effect. .if !\n(Pb .ig Ig [internal type: integer, displays as name defined below] .Ig diff --git a/doc/man8/pbs_mom.8.in b/doc/man8/pbs_mom.8.in index c3c0e97b91..39e6d1ed38 100644 --- a/doc/man8/pbs_mom.8.in +++ b/doc/man8/pbs_mom.8.in @@ -81,8 +81,10 @@ .SH NAME pbs_mom \- start a pbs batch execution mini-server .SH SYNOPSIS -pbs_mom [\^\-a\ alarm\^] [\^\-C\ chkdirectory\^] [\^\-c\ config\^] [\^\-D\^] [\^\-d\ directory] [\^\-F\^] [\^\-h\ help] [\^\-H\ hostname] [\^\-L\ logfile] [\^\-M\ MOMport\^] [\^\-R\ RPPport\^] [\^\-p\^|\^\-q\^|\^\-r] [\^\-x] -.SH DESCRIPTION +pbs_mom [\^\-a\ alarm\^] [\^\-C\ chkdirectory\^] [\^\-c\ config\^] [\^\-D\^] [\^\-d\ directory] [\^\-F\^] [\^\-h\ help] [\^\-H\ hostname] [\^\-L\ logfile] [\^\-M\ MOMport\^] [\^\-R\ RPPport\^] [\^\-p\^|\^\-q\^|\^\-r] [\^\-w] +[\^\-x] +.LP +SH DESCRIPTION The .B pbs_mom command starts the operation of a batch \fBM\fPachine \fBO\fPriented @@ -218,6 +220,8 @@ MOM may kill a process that is not a batch session. Specifies the port number on which the pbs_server is listening for requests. If pbs_server is started with a \-p option, pbs_mom will need to use the \-S option and match the port value which was used to start pbs_server. +.IP "\-w" 16 +When started with -w, pbs_moms wait until they get their MOM hierarchy file from pbs_server to send their first update, or until 10 minutes pass. This reduces network traffic on startup and can bring up clusters faster. .IP "\-x" 16 Disables the check for privileged port resource monitor connections. This is used mainly for testing since the privileged port is the only diff --git a/src/cmds/pbsnodes.c b/src/cmds/pbsnodes.c index 894755bd31..7fa23acac6 100644 --- a/src/cmds/pbsnodes.c +++ b/src/cmds/pbsnodes.c @@ -131,7 +131,7 @@ enum note_flags {unused, set, list, append}; -int quiet = 0; +bool quiet = false; char *progname; @@ -228,7 +228,7 @@ static int set_all_nodeattrs( NULL, &local_errno); - if (rc && !quiet) + if (rc && (!quiet)) { fprintf(stderr, "Error setting note attribute for %s - ", nodename); @@ -252,7 +252,7 @@ static int set_all_nodeattrs( NULL, &local_errno); - if (rc && !quiet) + if (rc && (!quiet)) { fprintf(stderr, "Error setting node power state for %s - ", nodename); @@ -825,7 +825,7 @@ int main( case 'q': - quiet = 1; + quiet = true; break; @@ -911,7 +911,7 @@ int main( if ((optarg != NULL) && !strcmp(optarg, "version")) { - fprintf(stderr, "Version: %s\nCommit: %s\n", + fprintf(stdout, "Version: %s\nCommit: %s\n", PACKAGE_VERSION, GIT_HASH); exit(0); @@ -920,6 +920,13 @@ int main( { TShowAbout_exit(); } + else if ((optarg != NULL) && !strcmp(optarg, "xml")) + { + flag = ALLI; + DisplayXML = TRUE; + + break; + } errflg = 1; @@ -963,7 +970,7 @@ int main( exit(1); } - con = cnt2server(specified_server); + con = cnt2server(specified_server, quiet); if (con <= 0) { diff --git a/src/cmds/qalter.c b/src/cmds/qalter.c index 023481cbf5..a43c981582 100644 --- a/src/cmds/qalter.c +++ b/src/cmds/qalter.c @@ -681,7 +681,7 @@ int main( cnt: - connect = cnt2server(server_name.c_str()); + connect = cnt2server(server_name.c_str(), false); if (connect <= 0) { diff --git a/src/cmds/qchkpt.c b/src/cmds/qchkpt.c index fca647cfd1..445654322d 100644 --- a/src/cmds/qchkpt.c +++ b/src/cmds/qchkpt.c @@ -50,7 +50,7 @@ int main(int argc, char **argv) /* qchkpt */ cnt: - connect = cnt2server(server_name.c_str()); + connect = cnt2server(server_name.c_str(), false); if (connect <= 0) { diff --git a/src/cmds/qdel.c b/src/cmds/qdel.c index b5f43b2f32..909018da66 100644 --- a/src/cmds/qdel.c +++ b/src/cmds/qdel.c @@ -310,7 +310,7 @@ int qdel_main( cnt: - connect = cnt2server(server_name.c_str()); + connect = cnt2server(server_name.c_str(), false); if (connect <= 0) { diff --git a/src/cmds/qdisable.c b/src/cmds/qdisable.c index 33124c2191..df8948d4ca 100644 --- a/src/cmds/qdisable.c +++ b/src/cmds/qdisable.c @@ -106,7 +106,7 @@ static void execute( NULL, (char *)"enabled", NULL, (char *)"FALSE", SET }; - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { merr = pbs_manager_err(ct, MGR_CMD_SET, MGR_OBJ_QUEUE, queue, &attr, NULL, &local_errno); diff --git a/src/cmds/qenable.c b/src/cmds/qenable.c index d57faf83b6..8517de8ba5 100644 --- a/src/cmds/qenable.c +++ b/src/cmds/qenable.c @@ -109,7 +109,7 @@ static void execute( NULL, (char *)"enabled", NULL, (char *)"TRUE", SET }; - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { merr = pbs_manager_err(ct, MGR_CMD_SET, MGR_OBJ_QUEUE, queue, &attr, NULL, &local_errno); diff --git a/src/cmds/qgpumode.c b/src/cmds/qgpumode.c index 376f85f7ab..6303b1a59e 100644 --- a/src/cmds/qgpumode.c +++ b/src/cmds/qgpumode.c @@ -132,7 +132,7 @@ static void execute( /* The request to change mode */ - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { merr = pbs_gpumode_err(ct, node, gpuid, mode, &local_errno); diff --git a/src/cmds/qgpureset.c b/src/cmds/qgpureset.c index 629da1091c..cb07bf09de 100644 --- a/src/cmds/qgpureset.c +++ b/src/cmds/qgpureset.c @@ -144,7 +144,7 @@ static void execute( /* The request to change mode */ - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { merr = pbs_gpureset_err(ct, node, gpuid, ecc_perm, ecc_vol, &local_errno); diff --git a/src/cmds/qhold.c b/src/cmds/qhold.c index 0f34e2bad6..cce4be7e87 100644 --- a/src/cmds/qhold.c +++ b/src/cmds/qhold.c @@ -137,7 +137,7 @@ int main( cnt: - connect = cnt2server(server_name.c_str()); + connect = cnt2server(server_name.c_str(), false); if (connect <= 0) { diff --git a/src/cmds/qmgr.c b/src/cmds/qmgr.c index 5ee1effc4b..1bde061923 100644 --- a/src/cmds/qmgr.c +++ b/src/cmds/qmgr.c @@ -677,7 +677,7 @@ struct server *make_connection( struct server *svr = NULL; - if ((connection = cnt2server(name)) > 0) + if ((connection = cnt2server(name, false)) > 0) { svr = new_server(); diff --git a/src/cmds/qmove.c b/src/cmds/qmove.c index 6118d5ca8c..977bbdea3e 100644 --- a/src/cmds/qmove.c +++ b/src/cmds/qmove.c @@ -66,7 +66,7 @@ int main(int argc, char **argv) /* qmove */ cnt: - connect = cnt2server(server_name.c_str()); + connect = cnt2server(server_name.c_str(), false); if (connect <= 0) { diff --git a/src/cmds/qmsg.c b/src/cmds/qmsg.c index b6c70d9400..9f046e3475 100644 --- a/src/cmds/qmsg.c +++ b/src/cmds/qmsg.c @@ -94,7 +94,7 @@ int main( cnt: - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); if (connect <= 0) { diff --git a/src/cmds/qorder.c b/src/cmds/qorder.c index b9a7fc7990..947050de87 100644 --- a/src/cmds/qorder.c +++ b/src/cmds/qorder.c @@ -135,7 +135,7 @@ int main( if (pn != NULL) *pn = ':'; /* restore : if it was present */ - connect = cnt2server(svrtmp); + connect = cnt2server(svrtmp, false); if (connect <= 0) { diff --git a/src/cmds/qrerun.c b/src/cmds/qrerun.c index 5edeae6bed..f665252d19 100644 --- a/src/cmds/qrerun.c +++ b/src/cmds/qrerun.c @@ -148,7 +148,7 @@ int main( cnt: - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); if (connect <= 0) { diff --git a/src/cmds/qrls.c b/src/cmds/qrls.c index b7a2e89ba2..e274c340a3 100644 --- a/src/cmds/qrls.c +++ b/src/cmds/qrls.c @@ -151,7 +151,7 @@ int main( cnt: - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); if (connect <= 0) { diff --git a/src/cmds/qrun.c b/src/cmds/qrun.c index 17cd441b47..64d13a9ee3 100644 --- a/src/cmds/qrun.c +++ b/src/cmds/qrun.c @@ -155,7 +155,7 @@ static void execute( cnt: - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { char job_id[PBS_MAXSVRJOBID]; diff --git a/src/cmds/qselect.c b/src/cmds/qselect.c index c534c860b0..1b4f509f20 100644 --- a/src/cmds/qselect.c +++ b/src/cmds/qselect.c @@ -514,7 +514,7 @@ int main( } } - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); if (connect <= 0) { diff --git a/src/cmds/qsig.c b/src/cmds/qsig.c index 52f0f14e47..edcfa9a267 100644 --- a/src/cmds/qsig.c +++ b/src/cmds/qsig.c @@ -89,7 +89,7 @@ int main( cnt: - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); if (connect <= 0) { diff --git a/src/cmds/qstart.c b/src/cmds/qstart.c index 43cb18dc8b..e257331a55 100644 --- a/src/cmds/qstart.c +++ b/src/cmds/qstart.c @@ -107,7 +107,7 @@ static void execute( NULL, (char *)"started", NULL, (char *)"TRUE", SET }; - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { merr = pbs_manager_err(ct, MGR_CMD_SET, MGR_OBJ_QUEUE, queue, &attr, NULL, &local_errno); diff --git a/src/cmds/qstat.c b/src/cmds/qstat.c index e6d1e936a5..e6f819d5e3 100644 --- a/src/cmds/qstat.c +++ b/src/cmds/qstat.c @@ -2817,7 +2817,7 @@ int process_commandline_opts( if ((optarg != NULL) && !strcmp(optarg, "version")) { - fprintf(stderr, "Version: %s\nCommit: %s\n", + fprintf(stdout, "Version: %s\nCommit: %s\n", PACKAGE_VERSION, GIT_HASH); exit(0); @@ -2828,6 +2828,16 @@ int process_commandline_opts( TShowAbout_exit(); } + if ((optarg != NULL) && !strcmp(optarg, "xml")) + { + DisplayXML = true; + + /* We want to return all attributes */ + attrib = NULL; + + break; + } + /* unexpected '--' option received */ errflg = 1; @@ -3097,7 +3107,7 @@ int run_job_mode( while (retry_count < MAX_RETRIES) { - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); any_failed = -1 * connect; if (connect <= 0 ) @@ -3295,7 +3305,7 @@ int run_queue_mode( while(retry_count < MAX_RETRIES) { - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); if (connect <= 0) { @@ -3389,7 +3399,7 @@ int run_server_mode( while (retry_count < MAX_RETRIES) { - connect = cnt2server(server_out); + connect = cnt2server(server_out, false); if (connect <= 0) { diff --git a/src/cmds/qstop.c b/src/cmds/qstop.c index 1f83a75646..73fb5674ee 100644 --- a/src/cmds/qstop.c +++ b/src/cmds/qstop.c @@ -104,7 +104,7 @@ static void execute( NULL, (char *)"started", NULL, (char *)"FALSE", SET }; - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { merr = pbs_manager_err(ct, MGR_CMD_SET, MGR_OBJ_QUEUE, queue, &attr, NULL, &local_errno); diff --git a/src/cmds/qsub_functions.c b/src/cmds/qsub_functions.c index ea8981a669..9843557415 100644 --- a/src/cmds/qsub_functions.c +++ b/src/cmds/qsub_functions.c @@ -39,6 +39,13 @@ #include #include +#include + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include + #ifdef sun #include #endif /* sun */ @@ -47,10 +54,6 @@ #include #endif -#if defined(FD_SET_IN_SYS_SELECT_H) -#include -#endif - #include "libcmds.h" /* TShowAbout_exit */ #include "cmds.h" #include "net_connect.h" @@ -88,10 +91,14 @@ char *host_name_suffix = NULL; int J_opt = FALSE; int P_opt = FALSE; +const int MAX_BIND_RETRIES = 20; const char *checkpoint_strings = "n,c,s,u,none,shutdown,periodic,enabled,interval,depth,dir"; char *alternate_dependency; int alternate_data_type; +int interactive_port_min; +int interactive_port_max; complete_req cr; +std::string L_request; /* adapted from openssh */ /* The parameter was EMsg, but was never used. @@ -788,17 +795,16 @@ int validate_submit_filter( void validate_pbs_o_workdir( - job_data_container *job_attr) + const job_info *ji) { - job_data *tmp_job_info = NULL; + job_data *tmp_job_info = NULL; const char *the_val = NULL; - char null_val[] = "\0"; char tmp_dir[MAXPATHLEN] = {""}; - if (hash_find(job_attr, ATTR_init_work_dir, &tmp_job_info) == FALSE) + if (hash_find(ji->job_attr, ATTR_init_work_dir, &tmp_job_info) == FALSE) { - if (hash_find(job_attr, "PWD", &tmp_job_info)) + if (hash_find(ji->job_attr, "PWD", &tmp_job_info)) the_val = tmp_job_info->value.c_str(); else { @@ -806,14 +812,35 @@ void validate_pbs_o_workdir( if ((the_dir = getcwd(tmp_dir, MAXPATHLEN)) != NULL) the_val = the_dir; else - the_val = null_val; + { + // Current working directory is deleted. Fail + fprintf(stderr, "qsub: Cannot obtain the current directory.\nPlease submit from a valid directory.\n"); + exit(3); + } } } else + { + // save the value of the work dir job attribute the_val = tmp_job_info->value.c_str(); - hash_add_or_exit(job_attr, ATTR_pbs_o_workdir, the_val, ENV_DATA); - hash_add_or_exit(job_attr, ATTR_init_work_dir, the_val, ENV_DATA); + if (hash_find(ji->client_attr, "validate_path", &tmp_job_info)) + { + // validate local existence of '-w' working directory + + struct stat sb; + + if ((stat(the_val, &sb) != 0) || + (!(S_ISDIR(sb.st_mode)))) + { + fprintf(stderr, "qsub: Requested working directory '%s' is not a valid directory\nPlease specify a valid working directory.\n", the_val); + exit(3); + } + } + } + + hash_add_or_exit(ji->job_attr, ATTR_pbs_o_workdir, the_val, ENV_DATA); + hash_add_or_exit(ji->job_attr, ATTR_init_work_dir, the_val, ENV_DATA); } /* END validate_pbs_o_workdir() */ @@ -883,6 +910,11 @@ void validate_qsub_host_pbs_o_server( } /* END validate_qsub_host_pbs_o_server() */ + +/* + * are_mpp_present() + */ + int are_mpp_present( job_data_container *resources, @@ -983,6 +1015,86 @@ bool is_resource_request_valid( +/* + * is_memory_request_valid() + * For cgroups, swap cannot be less than phsyical memory, so reject such jobs. + * + * @param ji (I) - the job information + * @param err_msg (O) - the error message + * @return true if the request is valid or cgroups aren't enabled, false otherwise. + */ + +bool is_memory_request_valid( + + job_info *ji, + std::string &err_msg) + + { + bool valid = true; + +#ifdef PENABLE_LINUX_CGROUPS + job_data_container *resources = ji->res_attr; + job_data *jd; + std::string mem_str; + std::string vmem_str; + unsigned long mem = 0; + unsigned long vmem = 0; + + if (hash_find(resources, "vmem", &jd) == 0) + { + if (hash_find(resources, "pvmem", &jd)) + { + read_mem_value(jd->value.c_str(), vmem); + vmem_str = jd->value; + err_msg = "qsub: requested pvmem='"; + err_msg += vmem_str; + } + } + else + { + read_mem_value(jd->value.c_str(), vmem); + vmem_str = jd->value; + err_msg = "qsub: requested vmem='"; + err_msg += vmem_str; + } + + if (vmem != 0) + { + if (hash_find(resources, "mem", &jd)) + { + read_mem_value(jd->value.c_str(), mem); + mem_str = jd->value; + + if (vmem < mem) + { + err_msg += "' which is less than mem='"; + err_msg += mem_str; + err_msg += "'. If p/vmem is set to a non-zero value, it must be greater than or equal to mem.\n"; + valid = false; + } + } + else if (hash_find(resources, "pmem", &jd)) + { + read_mem_value(jd->value.c_str(), mem); + mem_str = jd->value; + + if (vmem < mem) + { + err_msg += "' which is less than pmem='"; + err_msg += mem_str; + err_msg += "'. If p/vmem is set to a non-zero value, it must be greater than or equal to pmem.\n"; + valid = false; + } + } + } + +#endif + + return(valid); + } // END is_memory_request_valid() + + + void validate_basic_resourcing( job_info *ji) @@ -996,6 +1108,12 @@ void validate_basic_resourcing( exit(4); } + if (is_memory_request_valid(ji, err_msg) == false) + { + fprintf(stderr, "%s", err_msg.c_str()); + exit(4); + } + } /* END validate_basic_resourcing() */ @@ -1074,7 +1192,7 @@ void validate_join_options ( void post_check_attributes(job_info *ji, char *script_tmp) { - validate_pbs_o_workdir(ji->job_attr); + validate_pbs_o_workdir(ji); validate_qsub_host_pbs_o_server(ji->job_attr); validate_basic_resourcing(ji); @@ -1107,6 +1225,7 @@ void add_new_request_if_present( std::string req_str; cr.toString(req_str); hash_add_or_exit(ji->job_attr, ATTR_req_information, req_str.c_str(), CMDLINE_DATA); + hash_add_or_exit(ji->job_attr, ATTR_L_request, L_request.c_str(), CMDLINE_DATA); } } // END add_new_request_if_present() @@ -1198,6 +1317,8 @@ static int get_script( directive_prefix[0] = '\\'; strcat(directive_prefix, ArgV[index]); directive_prefix_on = false; + + cfilter += directive_prefix; continue; } @@ -1549,15 +1670,27 @@ void do_dir( } /* END do_dir() */ + +unsigned int get_port_in_range() + + { + int span = interactive_port_max - interactive_port_min + 1; + return(rand() % span + interactive_port_min); + } // END get_port_in_range() + + + /* * The following bunch of functions support the "Interactive Job" * capability of PBS. */ + /* * interactive_port - get a socket to listen to for "interactive" job * When the "interactive" job is run, its standard in, out, and error * will be connected to this socket. */ + char *interactive_port( int *sock) @@ -1568,6 +1701,7 @@ char *interactive_port( struct sockaddr_in myaddr; unsigned short port; + int bind_rc = -1; *sock = socket(AF_INET, SOCK_STREAM, 0); @@ -1580,11 +1714,20 @@ char *interactive_port( myaddr.sin_family = AF_INET; myaddr.sin_addr.s_addr = INADDR_ANY; - if (bind(*sock, (struct sockaddr *)&myaddr, namelen) < 0) + // Since the requested port range could be in use, retry throughout the port range. + for (int bind_retry = 0; bind_retry < MAX_BIND_RETRIES; bind_retry++) + { + if (interactive_port_min != 0) + myaddr.sin_port = htons(get_port_in_range()); + + if ((bind_rc = bind(*sock, (struct sockaddr *)&myaddr, namelen)) >= 0) + break; + } + + if (bind_rc < 0) print_qsub_usage_exit("qsub: unable to bind to socket"); /* get port number assigned */ - if (getsockname(*sock, (struct sockaddr *)&myaddr, &namelen) < 0) print_qsub_usage_exit("qsub: unable to get port number"); @@ -2033,7 +2176,7 @@ void bailout(void) printf("Job %s is being deleted\n", new_jobname); - c = cnt2server(server_out); + c = cnt2server(server_out, false); if (c <= 0) { @@ -2137,20 +2280,18 @@ void x11handler( int param_sock) { - struct pfwdsock *socks; + std::vector *socks = new std::vector(NUM_SOCKS); int n; char *display; - calloc_or_fail((char **)&socks, sizeof(struct pfwdsock) * NUM_SOCKS, "x11handler"); - for (n = 0;n < NUM_SOCKS;n++) - (socks + n)->active = 0; + socks->at(n).active = 0; - (socks + 0)->sock = param_sock; + socks->at(0).sock = param_sock; - (socks + 0)->active = 1; + socks->at(0).active = 1; - (socks + 0)->listening = 1; + socks->at(0).listening = 1; /* Try to open a socket for the local X server. */ display = getenv("DISPLAY"); @@ -2167,6 +2308,133 @@ void x11handler( exit(EXIT_FAILURE); } +/* + * wait_for_read_ready() + * + * Wait for a file descriptor to be ready to read before timeout elapsed + * @param fd - file descriptor + * @param timeout_sec - time in seconds to wait for read activity + * @return -1 if error, 0 if not ready, >1 if ready + */ + +int wait_for_read_ready( + + int fd, + int timeout_sec) + + { + struct pollfd PollArray; + struct timespec ts; + int n; + + PollArray.fd = fd; + PollArray.events = POLLIN; + PollArray.revents = 0; + + ts.tv_sec = timeout_sec; + ts.tv_nsec = 0; + + // wait for data ready to read + n = ppoll(&PollArray, 1, &ts, NULL); + + if (n > 0) + { + // events returned, fd ready for reading? + if ((PollArray.revents & POLLIN) == 0) + { + // none ready to read after timeout + n = 0; + } + } + else if ((n == -1) && (errno == EINTR)) + { + // ignore signals -- none ready to read + n = 0; + } + + return(n); + } + + + +/* + * get_interactive_job_id() + * + */ + +int get_interactive_job_id( + + int &news, + char *momjobid, + int momjobid_size) + + { + int rc = PBSE_NONE; + torque_socklen_t fromlen; + char cur_server[PBS_MAXSERVERNAME + PBS_MAXPORTNUM + 2]; + + struct sockaddr_in from; + + // Accept connection on socket set up earlier + while (true) + { + int rc; + + if ((rc = wait_for_read_ready(inter_sock, 30)) < 0) + print_qsub_usage_exit("qsub: poll failed"); + + if (rc > 0) + break; + + /* connect to server, status job to see if still there */ + + if (!locate_job(new_jobname, server_out, cur_server)) + { + fprintf(stderr, "qsub: job %s apparently deleted\n", + new_jobname); + + exit(1); + } + } + + // apparently someone is attempting to connect to us + fromlen = sizeof(from); + + if ((news = accept(inter_sock, (struct sockaddr *) & from, &fromlen)) < 0) + print_qsub_usage_exit("qsub: accept error"); + + // When MOM connects, she will send the job id for us to verify + int amt = momjobid_size; + char *pc = momjobid; + + while (amt > 0) + { + int amt_read = read_ac_socket(news, pc, amt); + + if (amt_read <= 0) + break; + + pc += amt_read; + amt -= amt_read; + + if (*(pc - 1) == '\0') + break; + } + + if ((pc - momjobid == 1) && + (momjobid[0] == 'C')) + return(-2); + + // Return -1 if we didn't read anything + if (amt == momjobid_size) + { + shutdown(news, 2); + rc = -1; + } + + return(rc); + } // END get_interactive_job_id() + /* @@ -2177,24 +2445,14 @@ void interactive( job_data_container *client_attr) { - int amt; - char cur_server[PBS_MAXSERVERNAME + PBS_MAXPORTNUM + 2]; - char momjobid[LOG_BUF_SIZE+1]; int news; - int nsel; - char *pc; - fd_set selset; struct sigaction act; - struct sockaddr_in from; - torque_socklen_t fromlen; - - struct timeval timeout; - struct winsize wsz; job_data *tmp_job_info; + bool read_jobid = false; /* Catch SIGINT and SIGTERM, and */ /* setup to catch Death of child */ @@ -2228,75 +2486,28 @@ void interactive( } - printf("qsub: waiting for job %s to start\n", - - new_jobname); + printf("qsub: waiting for job %s to start\n", new_jobname); - /* Accept connection on socket set up earlier */ - - nsel = 0; - - while (nsel == 0) + for (int retry = 0; retry < 3; retry++) { - FD_ZERO(&selset); - FD_SET(inter_sock, &selset); - - timeout.tv_usec = 0; - timeout.tv_sec = 30; - - nsel = select(FD_SETSIZE, &selset, NULL, NULL, &timeout); - - if (nsel > 0) + int rc = get_interactive_job_id(news, momjobid, sizeof(momjobid)); + if (rc == PBSE_NONE) { + read_jobid = true; break; } - else if (nsel == -1) - { - if (errno == EINTR) - { - nsel = 0; - } - else - print_qsub_usage_exit("qsub: select failed"); - } - - /* connect to server, status job to see if still there */ - - if (!locate_job(new_jobname, server_out, cur_server)) + else if (rc == -2) { - fprintf(stderr, "qsub: job %s apparently deleted\n", - new_jobname); - + // The job was canceled while waiting to start + fprintf(stderr, "qsub: Job %s was canceled while waiting to start\n", new_jobname); exit(1); } } - /* apparently someone is attempting to connect to us */ - - fromlen = sizeof(from); - - if ((news = accept(inter_sock, (struct sockaddr *) & from, &fromlen)) < 0) - print_qsub_usage_exit("qsub: accept error"); - - /* When MOM connects, she will send the job id for us to verify */ - - amt = LOG_BUF_SIZE + 1; - - pc = momjobid; - - while (amt > 0) + if (read_jobid == false) { - fromlen = read_ac_socket(news, pc, amt); - - if (fromlen <= 0) - break; - - pc += fromlen; - - if (*(pc - 1) == '\0') - break; - - amt -= fromlen; + fprintf(stderr, "qsub: received 3 connections, but couldn't read a valid job id\n"); + exit(1); } if (strncmp(momjobid, "PBS:", 4) == 0) @@ -2415,8 +2626,6 @@ void interactive( } /* END interactive() */ - - int validate_group_list( char *glist) @@ -2424,14 +2633,11 @@ int validate_group_list( { /* check each group to determine if it is a valid group that the user can be a part of. * group list is of the form group[@host][,group[@host]...] */ - char *buf = NULL; char *groups = strdup(glist); const char *delims = ","; - char *tmp_group = strtok(groups, delims); + char *tmp_group = strtok(groups, delims); char *at; char *u_name; - char **pmem; - struct group *grent; struct passwd *pwent; if ((pwent = getpwuid(getuid())) == NULL) @@ -2446,46 +2652,23 @@ int validate_group_list( { if ((at = strchr(tmp_group,'@')) != NULL) *at = '\0'; - - if ((grent = getgrnam_ext(&buf, tmp_group)) == NULL) - { - free(groups); - return(FALSE); - } - - pmem = grent->gr_mem; - free_grname(grent, buf); - - if (pmem == NULL) - { - free(groups); - return(FALSE); - } - - while (*pmem != NULL) - { - if (!strcmp(*pmem,u_name)) - break; - pmem++; - } - - if (*pmem == NULL) - { - /* match not found */ + if (is_group_member(u_name, tmp_group) == FALSE) + { free(groups); - return(FALSE); - } - + return(false); + } + tmp_group = strtok(NULL,delims); } free(groups); - return(TRUE); + return(true); } + bool came_from_moab( const char *src, @@ -2534,6 +2717,12 @@ void process_opt_L( { char err_buf[MAXLINE*2]; + if (L_request.size() > 0) + L_request += " "; + + L_request += "-L "; + L_request += cmd_arg; + if (strncmp(cmd_arg, "tasks=", 6)) { snprintf(err_buf, sizeof(err_buf), "qsub: illegal -L value: '%s'", cmd_arg); @@ -3376,7 +3565,8 @@ void process_opts( if (dependency_options.size() > 1) { alternate_dependency = strdup(dependency_options[1].c_str()); - alternate_data_type = data_type; + // Make this overwrite the previous value if needed + alternate_data_type = data_type - 1; } } else if (!strcmp(keyword, ATTR_job_radix)) @@ -3814,7 +4004,7 @@ void process_opts( case 'l': - if (add_verify_resources(ji->res_attr, vptr, data_type)) + if (add_verify_resources(ji->res_attr, vptr, FILTER_DATA)) print_qsub_usage_exit("qsub: illegal -l value"); break; @@ -4041,6 +4231,29 @@ void process_config_file( strcpy(host_name_suffix, param_val); } } + + if ((param_val = get_trq_param("INTERACTIVE_PORT_RANGE", torque_cfg_buf)) != NULL) + { + // Must be - + char *ptr = param_val; + bool valid = false; + interactive_port_min = strtol(ptr, &ptr, 10); + if (*ptr == '-') + { + ptr++; + interactive_port_max = strtol(ptr, &ptr, 10); + + if ((interactive_port_max > interactive_port_min) && + (interactive_port_min > 1024)) + valid = true; + } + + if (valid == false) + { + interactive_port_min = 0; + interactive_port_max = 0; + } + } } /* END if (load_config(torque_cfg_buf,sizeof(torque_cfg_buf)) == 0) */ } @@ -4218,7 +4431,7 @@ void process_early_opts( TShowAbout_exit(); else if (strcmp(name, "version") == 0) { - fprintf(stderr, "Version: %s\nCommit: %s\n", PACKAGE_VERSION, GIT_HASH); + fprintf(stdout, "Version: %s\nCommit: %s\n", PACKAGE_VERSION, GIT_HASH); exit(0); } } @@ -4323,6 +4536,8 @@ void main_func( int debug = FALSE; job_info ji; + srand(time(NULL)); + /** * Before we go to the trouble of allocating memory, initializing structures, * and setting up for ordinary workflow, check options to see if we'll be @@ -4610,7 +4825,7 @@ void main_func( } } - sock_num = cnt2server(server_out); + sock_num = cnt2server(server_out, false); if (sock_num <= 0) { diff --git a/src/cmds/qterm.c b/src/cmds/qterm.c index 30795a37bd..d05bd42d4e 100644 --- a/src/cmds/qterm.c +++ b/src/cmds/qterm.c @@ -33,6 +33,33 @@ int exitstatus = 0; /* Exit Status */ static void execute(int, const char *); + +/* + * is_local() - tells you if the specified server is local or not + * + * @param active_server - the server that we need to determine if is local or not + * @return true if the specified server is local + */ + +bool is_local( + + const char *active_server) + + { + int err = 0; + char local_hostname[PBS_MAXHOSTNAME]; + pbs_net_t addr = get_hostaddr(&err, active_server); + + if ((gethostname(local_hostname, sizeof(local_hostname)) == -1) || + (get_fullhostname(local_hostname, local_hostname, sizeof(local_hostname), NULL) == -1) || + (addr == 0)) + return(false); + + return(get_hostaddr(&err, local_hostname) == addr); + } + + + int main( int argc, /* I */ @@ -43,12 +70,13 @@ int main( * This routine sends a Server Shutdown request to the batch server. */ - static char opts[] = "t:"; /* See man getopt */ + static char opts[] = "lt:"; /* See man getopt */ int s; /* The execute line option */ static char usage[] = "Usage: qterm [-t quick] [server ...]\n"; char *type = NULL; /* Pointer to the type of termination */ int manner; /* The type of termination */ int errflg = 0; /* Error flag */ + bool local = false; /* Command line options */ @@ -57,6 +85,10 @@ int main( switch (s) { + case 'l': + + local = true; + case 't': type = optarg; @@ -116,8 +148,20 @@ int main( else { /* shutdown default server */ - - execute(manner, ""); + if (local == true) + { + char *active_server; + int port = 0; + if (get_active_pbs_server(&active_server, &port) == PBSE_NONE) + { + if (is_local(active_server)) + execute(manner, active_server); + else + exitstatus = -1; + } + } + else + execute(manner, ""); } exit(exitstatus); @@ -146,7 +190,7 @@ int main( static void execute( - int manner, /* I */ + int manner, /* I */ const char *server) /* I */ { @@ -155,7 +199,7 @@ static void execute( char *errmsg; /* Error message from pbs_terminate */ int local_errno = 0; - if ((ct = cnt2server(server)) > 0) + if ((ct = cnt2server(server, false)) > 0) { err = pbs_terminate_err(ct, manner, NULL, &local_errno); diff --git a/src/daemon_client/trq_auth_daemon.c b/src/daemon_client/trq_auth_daemon.c index 248a09c7ad..f6e2ff808c 100644 --- a/src/daemon_client/trq_auth_daemon.c +++ b/src/daemon_client/trq_auth_daemon.c @@ -43,14 +43,15 @@ bool down_server = false; bool use_log = true; bool daemonize_server = true; static int changed_msg_daem = 0; -static char *active_pbs_server; +std::string active_pbs_server; +std::string log_file_path = ""; /* Get the name of the active pbs_server */ int load_trqauthd_config( - char **default_server_name, - int *t_port, - char **trqauthd_unix_domain_port) + std::string &default_server_name, + int *t_port, + char **trqauthd_unix_domain_port) { int rc = PBSE_NONE; @@ -69,7 +70,7 @@ int load_trqauthd_config( * the client utilities determine the pbs_server port) */ printf("hostname: %s\n", tmp_name); - *default_server_name = tmp_name; + default_server_name = tmp_name; PBS_get_server(tmp_name, (unsigned int *)t_port); if (*t_port == 0) *t_port = PBS_BATCH_SERVICE_PORT; @@ -106,7 +107,7 @@ void initialize_globals_for_log(const char *port) strcpy(pbs_current_user, "trqauthd"); if ((msg_daemonname = strdup(pbs_current_user))) changed_msg_daem = 1; - log_set_hostname_sharelogging(active_pbs_server, port); + log_set_hostname_sharelogging(active_pbs_server.c_str(), port); } int init_trqauth_log(const char *server_port) @@ -129,7 +130,14 @@ int init_trqauth_log(const char *server_port) log_get_set_eventclass(&eventclass, SETV); initialize_globals_for_log(server_port); - sprintf(path_log, "%s/%s", path_home, TRQ_LOGFILES); + if(log_file_path == "") + { + sprintf(path_log, "%s/%s", path_home, TRQ_LOGFILES); + } + else + { + sprintf(path_log, "%s", log_file_path.c_str()); + } if ((mkdir(path_log, 0755) == -1) && (errno != EEXIST)) { openlog("daemonize_trqauthd", LOG_PID | LOG_NOWAIT, LOG_DAEMON); @@ -264,6 +272,7 @@ void parse_command_line(int argc, char **argv) {"about", no_argument, 0, 0 }, {"help", no_argument, 0, 0 }, {"version", no_argument, 0, 0 }, + {"logfile_dir", required_argument,0, 0 }, {0, 0, 0, 0 } }; @@ -295,6 +304,9 @@ void parse_command_line(int argc, char **argv) fprintf(stderr, "Version: %s \nCommit: %s\n", VERSION, GIT_HASH); exit(0); break; + case 3: /* logfile */ + log_file_path = optarg; + break; } break; @@ -429,7 +441,7 @@ int trq_main( return(rc); } - if ((rc = load_trqauthd_config(&active_pbs_server, &trq_server_port, &daemon_port)) != PBSE_NONE) + if ((rc = load_trqauthd_config(active_pbs_server, &trq_server_port, &daemon_port)) != PBSE_NONE) { fprintf(stderr, "Failed to load configuration. Make sure the $TORQUE_HOME/server_name file exists\n"); } diff --git a/src/drmaa/src/session.c b/src/drmaa/src/session.c index 86d0a931bb..f5deeddcae 100644 --- a/src/drmaa/src/session.c +++ b/src/drmaa/src/session.c @@ -553,7 +553,8 @@ int drmaa_job_ps( static struct attropl attribs[2]; - if (strcmp(attribs[0].name, "job_state")) + if ((attribs[0].name == NULL) || + (strcmp(attribs[0].name, "job_state"))) init_attribs(attribs); #if 0 { NULL, "exit_status", NULL, NULL, 0 } diff --git a/src/gui/Ccode/xpbs_datadump.c b/src/gui/Ccode/xpbs_datadump.c index b7074b12d1..a7e4e8bfb4 100644 --- a/src/gui/Ccode/xpbs_datadump.c +++ b/src/gui/Ccode/xpbs_datadump.c @@ -1622,7 +1622,7 @@ int main( /* qstat */ if (sigsetjmp(env_alrm, 1) == 0) { alarm(timeout_secs); - myconnection = cnt2server(server_out); + myconnection = cnt2server(server_out, false); } alarm(0); diff --git a/src/include/allocation.hpp b/src/include/allocation.hpp index 5dc7b82b55..a63e67f14b 100644 --- a/src/include/allocation.hpp +++ b/src/include/allocation.hpp @@ -87,6 +87,8 @@ extern const int MEM_INDICES; extern const int CPU_INDICES; +extern const int GPU_INDICES; +extern const int MIC_INDICES; extern const int exclusive_node; extern const int exclusive_socket; @@ -97,6 +99,20 @@ extern const int exclusive_legacy; extern const int exclusive_legacy2; extern const int exclusive_none; + +class cgroup_info + { + public: + + std::string mem_string; + std::string cpu_string; + std::string gpu_string; + std::string mic_string; + + cgroup_info() : mem_string(), cpu_string(), gpu_string(), mic_string() {} + }; + + // forward declare req class req; @@ -133,6 +149,7 @@ class allocation void set_cput_used(const unsigned long cput_used); void set_memory_used(const unsigned long long mem_used); void place_indices_in_string(std::string &output, int which); + void place_indices_in_string(cgroup_info &output); void set_place_type(const std::string &place); void get_place_type(int &place_type); void write_task_information(std::string &task_info) const; @@ -148,6 +165,8 @@ class allocation bool fully_placed() const; bool partially_placed(const req &r) const; void clear(); + void adjust_for_spread(unsigned int quantity, bool use_modulo); + void adjust_for_remainder(allocation &remainder); }; #endif diff --git a/src/include/array.h b/src/include/array.h index fdb064e251..74050d0a25 100644 --- a/src/include/array.h +++ b/src/include/array.h @@ -160,7 +160,7 @@ class job_array ~job_array(); int parse_array_request(const char *request); - int set_slot_limit(char *request); + int set_slot_limit(const char *request); int set_idle_slot_limit(long requested_limit); void set_submit_host(const char *submit_host); void set_owner(const char *owner); @@ -175,6 +175,7 @@ class job_array bool need_to_update_slot_limits() const; void mark_deleted(); bool is_deleted() const; + bool mark_end_of_subjob(job *pjob); }; diff --git a/src/include/attribute.h b/src/include/attribute.h index 15aa7c97b6..8fbff163b8 100644 --- a/src/include/attribute.h +++ b/src/include/attribute.h @@ -147,7 +147,6 @@ typedef struct svrattrl svrattrl; #define al_op al_atopl.op - /* * The value of an attribute is contained in the following structure. * @@ -384,6 +383,7 @@ void attr_atomic_kill(pbs_attribute *temp, attribute_def *pdef, int); void overwrite_complete_req(pbs_attribute *attr, pbs_attribute *new_attr); +long time_str_to_seconds(const char *str); int decode_b(pbs_attribute *patr, const char *name, const char *rn, const char *val, int); int decode_c(pbs_attribute *patr, const char *name, const char *rn, const char *val, int); int decode_l(pbs_attribute *patr, const char *name, const char *rn, const char *val, int); diff --git a/src/include/batch_request.h b/src/include/batch_request.h index e9aa75ba68..ede71794d3 100644 --- a/src/include/batch_request.h +++ b/src/include/batch_request.h @@ -319,8 +319,10 @@ struct rq_jobobit * so here is the union ... */ -struct batch_request + +class batch_request { + public: list_link rq_link; /* linkage of all requests */ int rq_type; /* type of request */ int rq_perm; /* access permissions for the user */ @@ -333,10 +335,10 @@ struct batch_request char rq_host[PBS_MAXHOSTNAME+1]; /* name of host sending request */ int rq_refcount; void *rq_extra; /* optional ptr to extra info */ - int rq_noreply; /* Set true if no reply is required */ + bool rq_noreply; /* Set true if no reply is required */ int rq_failcode; char *rq_extend; /* request "extension" data */ - char *rq_id; /* the batch request's id */ + std::string rq_id; /* the batch request's id */ struct batch_reply rq_reply; /* the reply area for this request */ @@ -394,9 +396,16 @@ struct batch_request struct rq_jobobit rq_jobobit; } rq_ind; - }; -typedef struct batch_request batch_request; + batch_request(); + batch_request(int type); + batch_request(const batch_request &other); + ~batch_request(); + batch_request &operator =(const batch_request &other); + + void update_object_id(int job_index); + int copy_attribute_list(const batch_request &other); + }; typedef container::item_container batch_request_holder; typedef container::item_container::item_iterator batch_request_holder_iterator; @@ -404,7 +413,6 @@ typedef container::item_container::item_iterator batch_reques extern batch_request_holder brh; -batch_request *alloc_br (int type); extern void reply_ack (struct batch_request *); extern void req_reject (int code, int aux, struct batch_request *, const char *, const char *); extern void reply_badattr (int code, int aux, svrattrl *, struct batch_request *); @@ -419,9 +427,9 @@ int process_request(struct tcp_chan *chan); int get_batch_request_id(batch_request *preq); int insert_batch_request(batch_request *preq); -batch_request *get_batch_request(char *br_id); -batch_request *get_remove_batch_request(char *br_id); -int remove_batch_request(char *br_id); +batch_request *get_batch_request(const char *br_id); +batch_request *get_remove_batch_request(const char *br_id); +int remove_batch_request(const char *br_id); int req_gpuctrl_svr(struct batch_request *preq); @@ -484,5 +492,4 @@ extern int encode_DIS_svrattrl (struct tcp_chan *chan, svrattrl *); extern int dis_request_read (struct tcp_chan *chan, struct batch_request *); extern int dis_reply_read (struct tcp_chan *chan, struct batch_reply *); -batch_request *duplicate_request(batch_request *preq, int job_index = -1); #endif /* BATCH_REQUEST_H */ diff --git a/src/include/cmds.h b/src/include/cmds.h index 436868a6a4..f93737386b 100644 --- a/src/include/cmds.h +++ b/src/include/cmds.h @@ -130,7 +130,7 @@ void prt_error(char *, char *, int); extern "C" { void set_attr(struct attrl **, const char *, const char *); -int cnt2server(const char *server); +int cnt2server(const char *server, bool silence); } int set_resources(struct attrl **, char *, int); void prt_job_err(const char *, int, const char *); diff --git a/src/include/container.hpp b/src/include/container.hpp index 89d77445cc..0b4e46ca3a 100644 --- a/src/include/container.hpp +++ b/src/include/container.hpp @@ -166,8 +166,8 @@ class item_container #endif pContainer = pCtner; iter = -1; - pContainer->initialize_ra_iterator(&iter); reversed = reverse; + pContainer->initialize_ra_iterator(&iter, this->reversed); endHit = false; } void reset(void) //Reset the iterator; @@ -183,7 +183,7 @@ class item_container } #endif iter = -1; - pContainer->initialize_ra_iterator(&iter); + pContainer->initialize_ra_iterator(&iter, this->reversed); endHit = false; } private: @@ -256,7 +256,11 @@ class item_container item *pItem = new item(id,it); if (insert_thing(pItem) < 0) + { + delete pItem; return false; + } + return true; } @@ -280,7 +284,10 @@ class item_container item *pItem = new item(id,it); if (insert_thing_after(pItem,index) < 0) + { + delete pItem; return false; + } return true; } @@ -299,7 +306,7 @@ class item_container return false; int iter = -1; - initialize_ra_iterator(&iter); + initialize_ra_iterator(&iter, false); while(index--) { item *pItem = next_thing(&iter); @@ -310,7 +317,10 @@ class item_container item *pItem = new item(id,it); if (insert_thing_before(pItem,iter) < 0) + { + delete pItem; return false; + } return true; } @@ -349,7 +359,10 @@ class item_container item *pItem = new item(id,it); if (insert_thing_before(pItem,index) != PBSE_NONE) + { + delete pItem; return false; + } return true; } @@ -664,6 +677,7 @@ class item_container /* update the last index */ slots[last].next = rc; last = rc; + slots[ALWAYS_EMPTY_INDEX].prev = last; /* update the new item's next index */ slots[rc].next = ALWAYS_EMPTY_INDEX; @@ -710,11 +724,7 @@ class item_container next = slots[index].next; slots[rc].next = next; slots[index].next = rc; - - if (next != 0) - { - slots[next].prev = rc; - } + slots[next].prev = rc; /* update the last index if needed */ if (last == index) @@ -806,7 +816,9 @@ class item_container int prev = slots[index].prev; int next = slots[index].next; - map.erase(slots[index].pItem->id); + if (map.find(slots[index].pItem->id) != map.end()) + map.erase(slots[index].pItem->id); + slots[index].prev = ALWAYS_EMPTY_INDEX; slots[index].next = ALWAYS_EMPTY_INDEX; delete slots[index].pItem; @@ -1050,10 +1062,14 @@ class item_container */ void initialize_ra_iterator( - int *iter) + int *iter, + bool reversed) { - *iter = slots[ALWAYS_EMPTY_INDEX].next; + if (reversed == false) + *iter = slots[ALWAYS_EMPTY_INDEX].next; + else + *iter = slots[ALWAYS_EMPTY_INDEX].prev; } /* END initialize_ra_iterator() */ diff --git a/src/include/lib_ifl.h b/src/include/lib_ifl.h index 27043c0750..ad933faccf 100644 --- a/src/include/lib_ifl.h +++ b/src/include/lib_ifl.h @@ -27,7 +27,7 @@ int parse_response_svr(int sock, char **msg); int build_response_client(int code, char *msg, char **send_message); int get_trq_server_addr(char *server_name, char **server_addr, int *server_addr_len); void *process_svr_conn(void *sock); -int validate_server(char *active_server_name, int t_server_port, char *ssh_key, char **sign_key); +int validate_server(std::string &active_server_name, int t_server_port, char *ssh_key, char **sign_key); int set_active_pbs_server(const char *, const int); int get_active_pbs_server(char **, int *); int validate_active_pbs_server(char **); @@ -294,6 +294,7 @@ int parse_daemon_response(long long code, long long len, char *buf); ssize_t send_unix_creds(int sd); #endif int pbs_original_connect(char *server); +int pbs_original_connect_ext(char *server, bool silence); int pbs_disconnect_socket(int socket); int pbs_connect_with_retry(char *server_name_ptr, int retry_seconds); void initialize_connections_table(); diff --git a/src/include/log.h b/src/include/log.h index 173a75e83c..0d90ce8516 100644 --- a/src/include/log.h +++ b/src/include/log.h @@ -94,7 +94,7 @@ /* ** Set up a debug print macro. */ -#ifdef NDEBUG +#if defined(NDEBUG) || (DEBUG == 0) #define DBPRT(x) #else #define DBPRT(x) printf x; diff --git a/src/include/machine.hpp b/src/include/machine.hpp index 13b293d688..d0fdf631d8 100644 --- a/src/include/machine.hpp +++ b/src/include/machine.hpp @@ -2,6 +2,7 @@ #define __MACHINE_HPP__ #include #include +#include #include #include #ifdef NVIDIA_GPUS @@ -59,7 +60,7 @@ class PCI_Device #endif #ifdef NVIDIA_GPUS - void initializeGpu(int idx, hwloc_topology_t topology); + void initializeGpu(int idx); #endif void displayAsString(stringstream &out) const; void setName(const string &name); @@ -141,12 +142,14 @@ class Chip Chip(); Chip(int execution_slots, int &es_remainder, int &per_numa_remainder); Chip(const Json::Value &layout, std::vector &valid_ids); + Chip(const std::string &legacy_layout, std::vector &valid_ids); Chip(const Chip &other); Chip &operator=(const Chip &other); ~Chip(); int get_id() const; int getTotalCores() const; int getTotalThreads() const; + int get_total_gpus() const; int getAvailableCores() const; int getAvailableThreads() const; hwloc_uint64_t getAvailableMemory() const; @@ -166,6 +169,9 @@ class Chip void parse_values_from_json_string(const Json::Value &layout, std::string &cores, std::string &threads, std::string &gpus, std::string &mics, std::vector &valid_ids); + void legacy_parse_values_from_json_string(const std::string &legacy_layout, std::string &cores, + std::string &threads, std::string &gpus, std::string &mics, + std::vector &valid_ids); #ifdef NVIDIA_GPUS #ifdef NVML_API @@ -182,12 +188,13 @@ class Chip void setCores(int cores); // used for unit tests void setThreads(int threads); // used for unit tests void setChipAvailable(bool available); - float how_many_tasks_fit(const req &r, int place_type) const; + void set_gpus(int gpus); // used for unit tests + double how_many_tasks_fit(const req &r, int place_type) const; bool has_socket_exclusive_allocation() const; bool task_will_fit(const req &r, int place_type) const; int free_core_count() const; void calculateStepCounts(const int lprocs_per_task, const int pu_per_task, int &step, int &step_rem, int &place_count, int &place_count_rem); - bool spread_place(req &r, allocation &master, int execution_slots_per, int &remainder); + bool spread_place(req &r, allocation &master, allocation &to_place, allocation &remainder); bool spread_place_cores(req &r, allocation &master, int &remaining_cores, int &remaining_lprocs, int &gpus, int &mics); bool spread_place_threads(req &r, allocation &master, int &remaining_cores, int &remaining_lprocs, int &gpus, int &mics); void place_all_execution_slots(req &r, allocation &task_alloc); @@ -207,6 +214,8 @@ class Chip int reserve_accelerator(int type); void free_accelerators(allocation &a); void free_accelerator(int index, int type); + void legacy_initialize_allocations(char *layout, std::vector &valid_ids); + void legacy_initialize_allocation(char *layout, std::vector &valid_ids); void initialize_allocations(const Json::Value &layout, std::vector &valid_ids); void initialize_allocation(const Json::Value &layout, std::vector &valid_ids); void aggregate_allocation(allocation &a); @@ -222,6 +231,7 @@ class Chip bool reserve_chip_thread(int core_index, allocation &a); bool reserve_place_thread(int core_index, allocation &a); bool reserve_chip_place_thread(int core_index, allocation &a); + void save_allocations(const Chip &other); }; @@ -246,6 +256,7 @@ class Socket Socket(); Socket(int execution_slots, int numa_nodes, int &es_remainder); Socket(const Json::Value &layout, std::vector &valid_ids); + Socket(const std::string &legacy_layout, std::vector &valid_ids); ~Socket(); Socket &operator=(const Socket &other); int initializeSocket(hwloc_obj_t obj); @@ -254,6 +265,8 @@ class Socket int getTotalChips() const; int getTotalCores() const; int getTotalThreads() const; + int get_total_gpus() const; + int get_available_gpus() const; int getAvailableChips() const; int getAvailableCores() const; int get_free_cores() const; @@ -269,9 +282,9 @@ class Socket void displayAsJson(Json::Value &out, bool include_jobs) const; void setId(int id); void addChip(); // used for unit tests - float how_many_tasks_fit(const req &r, int place_type) const; + double how_many_tasks_fit(const req &r, int place_type) const; void place_all_execution_slots(req &r, allocation &task_alloc); - bool spread_place(req &r, allocation &master, int execution_slots_per, int &remainder, bool chips); + bool spread_place(req &r, allocation &master, allocation &to_place, allocation &remainder, bool chips); bool spread_place_pu(req &r, allocation &task_alloc, int &cores, int &lprocs, int &gpus, int &mics); int place_task(req &r, allocation &a, int to_place, const char *hostname); bool free_task(const char *jobid); @@ -283,6 +296,7 @@ class Socket void update_internal_counts(vector &allocs); int get_gpus_remaining(); int get_mics_remaining(); + void save_allocations(const Socket &other); }; @@ -307,11 +321,12 @@ class Machine vector allocations; #ifdef NVIDIA_GPUS #ifdef NVML_API - hwloc_obj_t get_non_nvml_device(hwloc_topology_t topology, nvmlDevice_t device); + hwloc_obj_t get_non_nvml_device(hwloc_topology_t topology, nvmlDevice_t device, std::set &); #endif #endif void initialize_from_json(const string &json_str, vector &valid_ids); + int legacy_initialize_from_json(const string &json_str, vector &valid_ids); public: Machine& operator=(const Machine& newMachine); @@ -328,9 +343,10 @@ class Machine int getTotalChips() const; int getTotalCores() const; int getTotalThreads() const; + int get_total_gpus() const; int getAvailableSockets() const; int getAvailableChips() const; - int getAvailableMemory() const; + hwloc_uint64_t getAvailableMemory() const; int getAvailableCores() const; int getAvailableThreads() const; int getDedicatedSockets() const; @@ -342,16 +358,16 @@ class Machine void displayAsString(stringstream &out) const; void displayAsJson(stringstream &out, bool include_jobs) const; void insertNvidiaDevice(PCI_Device& device); - void store_device_on_appropriate_chip(PCI_Device &device); + void store_device_on_appropriate_chip(PCI_Device &device, bool no_info); void place_all_execution_slots(req &r, allocation &master, const char *hostname); int spread_place(req &r, allocation &master, int tasks_for_node, const char *hostname); int spread_place_pu(req &r, allocation &master, int tasks_for_node, const char *hostname); - int place_job(job *pjob, string &cpu_string, string &mem_string, const char *hostname, bool legacy_vmem); + int place_job(job *pjob, cgroup_info &cgi, const char *hostname, bool legacy_vmem); void setMemory(long long mem); void addSocket(int count); // used for unit tests void setIsNuma(bool is_numa); // used for unit tests + void save_allocations(const Machine &other); void free_job_allocation(const char *jobid); - int get_jobs_cpusets(const char *jobid, string &cpus, string &mems); int fit_tasks_within_sockets(req &r, allocation &job_alloc, const char *hostname, int &remaining_tasks); void place_remaining(req &to_split, allocation &master, int &remaining_tasks, const char *hostname); int how_many_tasks_can_be_placed(req &r) const; @@ -360,6 +376,8 @@ class Machine bool check_if_possible(int &sockets, int &numa_nodes, int &cores, int &threads) const; bool is_initialized() const; void reinitialize_from_json(const std::string &json_layout, std::vector &valid_ids); + void compare_remaining_values(allocation &remaining, const char *caller) const; + void clear(); }; extern Machine this_node; diff --git a/src/include/mail_throttler.hpp b/src/include/mail_throttler.hpp index 3966ccb859..466e772bed 100644 --- a/src/include/mail_throttler.hpp +++ b/src/include/mail_throttler.hpp @@ -7,20 +7,28 @@ #include #include +#include "pbs_job.h" +#include "resource.h" class mail_info { public: - std::string mailto; - std::string exec_host; - std::string jobid; - std::string jobname; - std::string text; /* additional optional text */ - std::string errFile; - std::string outFile; - int mail_point; + std::string mailto; + std::string exec_host; + std::string jobid; + std::string jobname; + std::string text; // additional optional text + std::string errFile; + std::string outFile; + int mail_point; + std::string queue_name; + std::string owner; + std::string working_directory; + std::vector resources_requested; + std::vector resources_used; mail_info(); + mail_info(job *pjob); mail_info(const mail_info &other); mail_info &operator =(const mail_info &other); }; diff --git a/src/include/mom_config.h b/src/include/mom_config.h index 5634cfd2c6..ace8931004 100644 --- a/src/include/mom_config.h +++ b/src/include/mom_config.h @@ -116,6 +116,7 @@ extern char *auto_ideal_load; extern char *auto_max_load; extern int exec_with_exec; extern int ServerStatUpdateInterval; +extern int varattr_tv; extern char *AllocParCmd; extern char PBSNodeCheckPath[]; extern int PBSNodeCheckProlog; @@ -174,6 +175,10 @@ extern int resend_join_job_wait_time; extern int mom_hierarchy_retry_time; extern int MOMJobDirStickySet; extern std::string presetup_prologue; +extern unsigned long max_memory; +extern unsigned long max_swap; +extern bool get_cray_taskstats; +extern u_long pbsclient; struct specials { @@ -193,6 +198,8 @@ unsigned long setenablemomrestart(const char *value); unsigned long setrcpcmd(const char *value); unsigned long setjobdirectorysticky(const char *value); unsigned long setcudavisibledevices(const char *value); +unsigned long setnodecheckonjobstart(const char *value); +unsigned long setnodecheckonjobend(const char *value); #ifdef PENABLE_LINUX26_CPUSETS unsigned long setmempressthr(const char *); diff --git a/src/include/mom_func.h b/src/include/mom_func.h index 6d3c8f09a0..069f70198f 100644 --- a/src/include/mom_func.h +++ b/src/include/mom_func.h @@ -257,7 +257,7 @@ extern proc_stat_t *get_proc_stat(int pid); extern void term_job(job *); int TTmpDirName(job *, char *, int); -extern bool check_pwd(job *); +extern int check_pwd(job *); extern int task_save(task *) ; extern void DIS_rpp_reset(void); extern void checkret(char **, long); @@ -268,7 +268,7 @@ void clear_servers(); void set_jobs_substate(job *pjob, int new_substate); -int become_the_user(job *pjob); +int become_the_user(job *pjob, bool want_effective); bool am_i_mother_superior(const job &pjob); diff --git a/src/include/mom_memory.h b/src/include/mom_memory.h index 2e7891283e..7c872a0017 100644 --- a/src/include/mom_memory.h +++ b/src/include/mom_memory.h @@ -14,6 +14,7 @@ unsigned long long swap_total; unsigned long long swap_used; unsigned long long swap_free; + time_t timestamp; }proc_mem_t; proc_mem_t *get_proc_mem_from_path(const char *path); diff --git a/src/include/net_connect.h b/src/include/net_connect.h index 1555f351a0..7638ad7c6d 100644 --- a/src/include/net_connect.h +++ b/src/include/net_connect.h @@ -213,7 +213,6 @@ void net_close(int); int wait_request(time_t waittime, long *); void net_add_close_func(int, void(*func)(int)); int get_max_num_descriptors(void); -int get_fdset_size(void); char * netaddr_pbs_net_t(pbs_net_t); #ifdef __cplusplus } diff --git a/src/include/pbs_error_db.h b/src/include/pbs_error_db.h index 4df4b695c5..b531953c3e 100644 --- a/src/include/pbs_error_db.h +++ b/src/include/pbs_error_db.h @@ -212,7 +212,7 @@ PbsErrClient(PBSE_NOJOBARRAYS, (char *)"Queue does not allow job arrays") /* 15090 */ PbsErrClient(PBSE_RELAYED_TO_MOM, (char *)"request was relayed to a MOM") /* */ PbsErrClient(PBSE_MEM_MALLOC, (char *)"Error allocating memory - out of memory") -PbsErrClient(PBSE_MUTEX, (char *)"Error allocating controling mutex (lock/unlock)") +PbsErrClient(PBSE_MUTEX, (char *)"Error allocating controlling mutex (lock/unlock)") PbsErrClient(PBSE_THREADATTR, (char *)"Error setting thread attributes") PbsErrClient(PBSE_THREAD, (char *)"Error creating thread") PbsErrClient(PBSE_SELECT, (char *)"Error in socket select") @@ -298,6 +298,8 @@ PbsErrClient(PBSE_CGROUP_CREATE_FAIL, (char *)"Could not create all of the cgrou PbsErrClient(PBSE_EOF, (char *)"This stream has already been closed. End of File.") PbsErrClient(PBSE_GPU_PROHIBITED_MODE, (char *)"Invalid gpu mode requested. Prohibited mode is not allowed. Check the spelling of the mode request for errors") PbsErrClient(PBSE_NODE_DELETED, (char *)"Node was deleted during work") +PbsErrClient(PBSE_STATE_SLOT_LIMIT, (char *)"The requested state or substate can't be set from the job's current state (slot limit in effect).") +PbsErrClient(PBSE_BAD_GROUP, (char *)"Bad group entry") /* pbs client errors ceiling (max_client_err + 1) */ PbsErrClient(PBSE_CEILING, (char*)0) #endif @@ -313,8 +315,8 @@ PbsErrRm(PBSE_RMBADPARAM, (char *)"parameter could not be used") PbsErrRm(PBSE_RMNOPARAM, (char *)"a parameter needed did not exist") /* something specified didn't exist */ PbsErrRm(PBSE_RMEXIST, (char *)"something specified didn't exist") -/* a system error occured */ -PbsErrRm(PBSE_RMSYSTEM, (char *)"a system error occured") +/* a system error occurred */ +PbsErrRm(PBSE_RMSYSTEM, (char *)"a system error occurred") /* only part of reservation made */ PbsErrRm(PBSE_RMPART, (char *)"only part of reservation made") /* pbs rm errors ceiling (max_rm_err + 1) */ diff --git a/src/include/pbs_ifl.h b/src/include/pbs_ifl.h index f10eed1e18..c2c86d9fed 100644 --- a/src/include/pbs_ifl.h +++ b/src/include/pbs_ifl.h @@ -341,6 +341,7 @@ #define ATTR_pass_cpu_clock "pass_cpu_clock" #define ATTR_request_version "request_version" #define ATTR_req_information "req_information" +#define ATTR_sendmail_path "sendmail_path" /* additional node "attributes" names */ #define ATTR_NODE_state "state" @@ -380,6 +381,9 @@ #define ATTR_cpustr "cpuset_string" #define ATTR_memstr "memset_string" +#define ATTR_L_request "L_Request" +#define ATTR_gpus_reserved "gpus_reserved" +#define ATTR_mics_reserved "mics_reserved" #define ATTR_user_kill_delay "user_kill_delay" #define ATTR_idle_slot_limit "idle_slot_limit" #define ATTR_default_gpu_mode "default_gpu_mode" @@ -653,6 +657,9 @@ int pbs_asyrunjob(int c, char *jobid, char *location, char *extend); int pbs_alterjob_async(int connect, char *job_id, struct attrl *attrib, char *extend); int pbs_alterjob(int connect, char *job_id, struct attrl *attrib, char *extend); int pbs_connect(char *server); +#ifdef __cplusplus +int pbs_connect_ext(char *server, bool silence); +#endif int pbs_query_max_connections(); char *pbs_default(void); char *pbs_fbserver(void); diff --git a/src/include/pbs_job.h b/src/include/pbs_job.h index 985ac92416..48363548c1 100644 --- a/src/include/pbs_job.h +++ b/src/include/pbs_job.h @@ -436,6 +436,9 @@ enum job_atr JOB_ATR_memset_string, JOB_ATR_user_kill_delay, JOB_ATR_idle_slot_limit, + JOB_ATR_LRequest, + JOB_ATR_gpus_reserved, + JOB_ATR_mics_reserved, JOB_ATR_UNKN, /* the special "unknown" type */ JOB_ATR_LAST /* This MUST be LAST */ }; @@ -527,6 +530,18 @@ typedef std::set job_pid_set_t; #ifdef PBS_MOM // forward declare task so it can be part of the job class task; + +class task_usage_info + { + public: + + unsigned long cput; + unsigned long long mem; + + task_usage_info() : cput(0), mem(0) + { + } + }; #endif #define COUNTED_GLOBALLY 0x0001 @@ -806,6 +821,7 @@ class job unsigned ji_queue_counted; bool ji_being_deleted; int ji_commit_done; /* req_commit has completed. If in routing queue job can now be routed */ + bool ji_routed; // false if the job is an array template that is stuck in a routing queue. /* * fixed size internal data - maintained via "quick save" @@ -842,6 +858,7 @@ class job void encode_plugin_resource_usage(tlist_head *phead) const; void add_plugin_resource_usage(std::string &acct_data) const; + size_t number_of_plugin_resources() const; }; #endif @@ -1139,6 +1156,7 @@ class send_job_request //#define JOB_SVFLG_RescAssn 0x2000 // Not used on the mom //#define JOB_SVFLG_CHECKPOINT_COPIED 0x4000 // Not used on the mom #define JOB_SVFLG_INTERMEDIATE_MOM 0x8000 // This is for job_radix. I am an intermediate mom +#define JOB_SVFLG_PROLOGUES_RAN 0x10000 // job has run prologues #endif @@ -1285,6 +1303,7 @@ dir so that job can be restarted */ #define JOB_EXEC_OVERLIMIT_WT -11 /* job exceeded a walltime limit */ #define JOB_EXEC_OVERLIMIT_CPUT -12 /* job exceeded a cpu time limit */ #define JOB_EXEC_RETRY_CGROUP -13 /* couldn't create the job's cgroups */ +#define JOB_EXEC_RETRY_PROLOGUE -14 /* prologue failed */ extern void depend_clrrdy(job *); extern int depend_on_que(pbs_attribute *, void *, int); @@ -1307,7 +1326,8 @@ job *job_clone(job *,struct job_array *, int, bool); job *svr_find_job(const char *jobid, int get_subjob); job *svr_find_job_by_id(int internal_job_id); job *find_job_by_array(all_jobs *aj, const char *job_id, int get_subjob, bool locked); -bool job_id_exists(const std::string &job_id_string, int *rcode); +bool job_id_exists(const std::string &job_id_string); +void *svr_job_purge_task(void *vp); bool internal_job_id_exists(int internal_id); #else extern job *mom_find_job(const char *); @@ -1337,6 +1357,8 @@ bool have_reservation(job *, struct pbs_queue *); int lock_ji_mutex(job *pjob, const char *id, const char *msg, int logging); int unlock_ji_mutex(job *pjob, const char *id, const char *msg, int logging); +int issue_signal(job **, const char *, void(*)(struct batch_request *), void *, char *); + #ifdef BATCH_REQUEST_H extern job *chk_job_request(char *, struct batch_request *); extern int net_move(job *, struct batch_request *); @@ -1345,7 +1367,6 @@ extern int svr_chk_owner(struct batch_request *, job *); extern struct batch_request *cpy_stage(struct batch_request *, job *, enum job_atr, int); extern struct batch_request *setup_cpyfiles(struct batch_request *, job *, char *, char *, int, int); extern struct batch_request *cpy_checkpoint(struct batch_request *, job *, enum job_atr, int); -int issue_signal(job **, const char *, void(*)(struct batch_request *), void *, char *); #endif /* BATCH_REQUEST_H */ diff --git a/src/include/policy_values.h b/src/include/policy_values.h index ec6a179227..b4ed28810a 100644 --- a/src/include/policy_values.h +++ b/src/include/policy_values.h @@ -2,4 +2,7 @@ extern bool cray_enabled; extern bool ghost_array_recovery; +extern int default_gpu_mode; + +int set_default_gpu_mode_int(const char *gpu_mode_str); diff --git a/src/include/port_forwarding.h b/src/include/port_forwarding.h index ef88000b79..04bbff0847 100644 --- a/src/include/port_forwarding.h +++ b/src/include/port_forwarding.h @@ -6,6 +6,8 @@ #define MAX_DISPLAYS 500 #define X11OFFSET 50 +#include + /* derived from XF4/xc/lib/dps/Xlibnet.h */ #ifndef X_UNIX_PATH @@ -33,7 +35,7 @@ struct pfwdsock char buff[BUF_SIZE]; }; -void port_forwarder(struct pfwdsock *, int(*connfunc)(char *phost, long pport, char *), char*, int, char *); +void port_forwarder(std::vector *, int(*connfunc)(char *phost, long pport, char *), char*, int, char *); void set_nodelay(int); int connect_local_xsocket(u_int); int x11_connect_display(char *, long, char *); diff --git a/src/include/qmgr.h b/src/include/qmgr.h index 595139fccd..fc312fa2ad 100644 --- a/src/include/qmgr.h +++ b/src/include/qmgr.h @@ -306,7 +306,7 @@ void disconnect_from_server(); "resources_cost - the cost factors of resources. Used for sync. job starting\n" \ "resources_default - the default resource value when the job does not specify\n" \ "resource_max - the maximum amount of resources that are on the system\n" \ - "scheduler_iteration - the amount of seconds between timed scheduler iterations\n" \ + "scheduler_iteration - has no effect\n" \ "scheduling - when true the server should tell the scheduler to run\n" \ "system_cost - arbitrary value factored into resource costs\n" \ "use_jobs_subdirs - when true divide storage of jobs into subdirectories in $PBS_HOME/server_priv/{jobs,arrays}\n" \ diff --git a/src/include/qmgr_svr_public.h b/src/include/qmgr_svr_public.h index ba2b95732b..c767944cbb 100644 --- a/src/include/qmgr_svr_public.h +++ b/src/include/qmgr_svr_public.h @@ -204,3 +204,4 @@ ATTR_ghost_array_recovery, ATTR_cgroup_per_task, ATTR_idle_slot_limit, ATTR_default_gpu_mode, +ATTR_sendmail_path, diff --git a/src/include/req.hpp b/src/include/req.hpp index 7220238926..080e66447d 100644 --- a/src/include/req.hpp +++ b/src/include/req.hpp @@ -116,8 +116,10 @@ extern const char *place_legacy2; class req { int execution_slots; - unsigned long mem; - unsigned long swap; + unsigned long total_mem; + unsigned long mem_per_task; + unsigned long total_swap; + unsigned long swap_per_task; unsigned long disk; int nodes; int socket; @@ -172,8 +174,8 @@ class req void get_values(std::vector &names, std::vector &values) const; void toString(std::string &str) const; int getExecutionSlots() const; - unsigned long getMemory() const; - unsigned long getSwap() const; + unsigned long get_total_memory() const; + unsigned long get_total_swap() const; int get_cores() const; int get_threads() const; int get_sockets() const; @@ -182,7 +184,7 @@ class req int getPlaceThreads() const; unsigned long getDisk() const; int getMaxtpn() const; - int getGpus() const; + int get_gpus() const; int getMics() const; std::string getGpuMode() const; std::string getGres() const; @@ -202,9 +204,9 @@ class req bool cgroup_preference_set() const; int get_task_allocation(unsigned int index, allocation &task_allocation) const; unsigned long long get_memory_for_host(const std::string &host) const; - unsigned long long get_memory_per_task(); + unsigned long long get_memory_per_task() const; unsigned long long get_swap_for_host(const std::string &host) const; - unsigned long long get_swap_per_task(); + unsigned long long get_swap_per_task() const; void get_task_stats(std::vector &task_index, std::vector &cput_used, std::vector &mem_used); int get_execution_slots() const; @@ -224,4 +226,7 @@ class req void set_task_usage_stats(int task_index, unsigned long cput_used, unsigned long long mem_used); }; + +int read_mem_value(const char *value, unsigned long &parsed); + #endif /* _REQ_HPP */ diff --git a/src/include/resmon.h b/src/include/resmon.h index 6a674092e0..9c8995c716 100644 --- a/src/include/resmon.h +++ b/src/include/resmon.h @@ -106,11 +106,12 @@ struct config #define RM_NPARM 32 /* max number of parameters for child */ -#define RM_CMD_CLOSE 1 -#define RM_CMD_REQUEST 2 -#define RM_CMD_CONFIG 3 -#define RM_CMD_SHUTDOWN 4 -#define RM_CMD_LAYOUT 5 +#define RM_CMD_CLOSE 1 +#define RM_CMD_REQUEST 2 +#define RM_CMD_CONFIG 3 +#define RM_CMD_SHUTDOWN 4 +#define RM_CMD_LAYOUT 5 +#define RM_CMD_UPDATE_LAYOUT 6 #define RM_RSP_OK 100 #define RM_RSP_ERROR 999 diff --git a/src/include/server.h b/src/include/server.h index 9f3215bab1..43171243f1 100644 --- a/src/include/server.h +++ b/src/include/server.h @@ -229,6 +229,7 @@ enum srv_atr SRV_ATR_CgroupPerTask, SRV_ATR_IdleSlotLimit, SRV_ATR_DefaultGpuMode, + SRV_ATR_SendmailPath, /* This must be last */ SRV_ATR_LAST diff --git a/src/include/trq_cgroups.h b/src/include/trq_cgroups.h index 389633fa4b..d140c890dc 100644 --- a/src/include/trq_cgroups.h +++ b/src/include/trq_cgroups.h @@ -40,4 +40,6 @@ void trq_cg_delete_job_cgroups(const char *job_id, bool successfully_created); bool have_incompatible_dash_l_resource(pbs_attribute *pattr); int trq_cg_add_devices_to_cgroup(job *pjob); int init_torque_cgroups(); +void trq_cg_signal_tasks(const std::string& cgroup_path, int signal); +int find_range_in_cpuset_string(std::string &source, std::string &output); #endif /* _TRQ_CGROUPS_H_ */ diff --git a/src/include/u_hash_map_structs.h b/src/include/u_hash_map_structs.h index ca6db3ceb5..ca050bcc5c 100644 --- a/src/include/u_hash_map_structs.h +++ b/src/include/u_hash_map_structs.h @@ -123,6 +123,8 @@ typedef job_data_container::item_iterator job_data_iterator; int hash_add_item(job_data_container *head, const char *name, const char *value, int var_type, int op_type); +void hash_priority_add_or_exit(job_data_container *head, const char *name, const char *value, int var_type); + void hash_add_or_exit(job_data_container *head, const char *name, const char *value, int var_type); int hash_del_item(job_data_container *head, const char *name); diff --git a/src/include/utils.h b/src/include/utils.h index 1aaafe6fca..e1a3fbd815 100644 --- a/src/include/utils.h +++ b/src/include/utils.h @@ -145,6 +145,7 @@ extern char *OriginalPath; /* group functions in u_groups.c */ extern void free_grname(struct group *, char *); extern void free_pwnam(struct passwd *pwdp, char *buf); +extern bool is_group_member(char *user_name, char *group_name); extern struct group *getgrnam_ext (char **, char *); extern struct group *getgrgid_ext (char **, gid_t); @@ -209,5 +210,8 @@ int rmdir_ext(const char *dir, int retry_limit = 20); int unlink_ext(const char *filename, int retry_limit = 20); int mkdir_wrapper(const char *pathname, mode_t mode); +/* from parse_config.c */ +int setbool(const char *value); + #endif /* END #ifndef UTILS_H */ diff --git a/src/include/work_task.h b/src/include/work_task.h index e24e9fbb6f..9a44729a80 100644 --- a/src/include/work_task.h +++ b/src/include/work_task.h @@ -176,7 +176,7 @@ int insert_timed_task(all_tasks *, time_t task_time, work_task *); struct batch_request; -int issue_to_svr(const char *server_name, batch_request **preq, void (*f)(struct work_task *)); +int issue_to_svr(const char *server_name, batch_request *preq, void (*f)(struct work_task *)); #define TASKS_TO_REMOVE 1000 diff --git a/src/lib/Libattr/attr_fn_arst.c b/src/lib/Libattr/attr_fn_arst.c index 811410f6c1..c63281637e 100644 --- a/src/lib/Libattr/attr_fn_arst.c +++ b/src/lib/Libattr/attr_fn_arst.c @@ -1135,7 +1135,9 @@ char *arst_string( return(NULL); } - if ((pattr->at_type != ATR_TYPE_ARST) || !(pattr->at_flags & ATR_VFLAG_SET)) + if ((pattr->at_type != ATR_TYPE_ARST) || + !(pattr->at_flags & ATR_VFLAG_SET) || + (pattr->at_val.at_arst == NULL)) { /* bad type or value not set */ diff --git a/src/lib/Libattr/attr_fn_complete_req.c b/src/lib/Libattr/attr_fn_complete_req.c index 358fa5780c..c50a2ccb16 100644 --- a/src/lib/Libattr/attr_fn_complete_req.c +++ b/src/lib/Libattr/attr_fn_complete_req.c @@ -152,8 +152,8 @@ int decode_complete_req( unsigned long long swap; req &r = cr->get_req(i); - mem = r.getMemory(); - swap = r.getSwap(); + mem = r.get_total_memory(); + swap = r.get_total_swap(); if (swap != 0) { @@ -220,8 +220,8 @@ int decode_complete_req( unsigned long long swap; req &r = cr->get_req(i); - mem = r.getMemory(); - swap = r.getSwap(); + mem = r.get_total_memory(); + swap = r.get_total_swap(); if (swap != 0) { diff --git a/src/lib/Libattr/attr_fn_hold.c b/src/lib/Libattr/attr_fn_hold.c index 3fed470150..16ad88d406 100644 --- a/src/lib/Libattr/attr_fn_hold.c +++ b/src/lib/Libattr/attr_fn_hold.c @@ -144,6 +144,14 @@ int decode_hold( patr->at_val.at_long |= HOLD_s; break; + case 'a': + patr->at_val.at_long |= HOLD_a; + break; + + case 'l': + patr->at_val.at_long |= HOLD_l; + break; + default: return (PBSE_BADATVAL); } @@ -208,6 +216,12 @@ int encode_hold( if (attr->at_val.at_long & HOLD_u) *(pal->al_value + i++) = 'u'; + + if (attr->at_val.at_long & HOLD_a) + *(pal->al_value + i++) = 'a'; + + if (attr->at_val.at_long & HOLD_l) + *(pal->al_value + i++) = 'l'; } pal->al_flags = attr->at_flags; diff --git a/src/lib/Libattr/attr_fn_resc.c b/src/lib/Libattr/attr_fn_resc.c index a8c54aaa0b..3998a6e7cd 100644 --- a/src/lib/Libattr/attr_fn_resc.c +++ b/src/lib/Libattr/attr_fn_resc.c @@ -173,15 +173,9 @@ int decode_resc( if (prdef == NULL) { - /* - * didn't find resource with matching name, use unknown; - * but return PBSE_UNKRESC in case caller doesn`t wish to - * accept unknown resources - */ - + // Don't accept unknown resources as this strands jobs. rc = PBSE_UNKRESC; - - prdef = svr_resc_def + (svr_resc_size - 1); + return(rc); } prsc = find_resc_entry(patr, prdef); @@ -220,8 +214,6 @@ int decode_resc( - - /* * encode_resc - encode attr of type ATR_TYPE_RESR into attr_extern form * @@ -763,6 +755,8 @@ int action_resc( int actmode) { + int rc = PBSE_NONE; + if (pattr->at_val.at_ptr != NULL) { std::vector *resources = (std::vector *)pattr->at_val.at_ptr; @@ -773,12 +767,17 @@ int action_resc( if ((r.rs_value.at_flags & ATR_VFLAG_MODIFY) && (r.rs_defin->rs_action)) - r.rs_defin->rs_action(&r, pattr, actmode); + { + int tmp_rc = r.rs_defin->rs_action(&r, pattr, actmode); + if ((tmp_rc != PBSE_NONE) && + (rc == PBSE_NONE)) + rc = tmp_rc; + } resources->at(i).rs_value.at_flags &= ~ATR_VFLAG_MODIFY; } } - return(PBSE_NONE); + return(rc); } // END action_resc() diff --git a/src/lib/Libattr/attr_fn_time.c b/src/lib/Libattr/attr_fn_time.c index 261f6c8d25..cb6d5ac124 100644 --- a/src/lib/Libattr/attr_fn_time.c +++ b/src/lib/Libattr/attr_fn_time.c @@ -111,80 +111,64 @@ * value type "long" * -------------------------------------------------- */ +#define PBS_MAX_TIME (LONG_MAX - 1) + /* - * decode_time - decode time into into pbs_attribute structure of type ATR_TYPE_LONG + * time_str_to_seconds() * - * Returns: 0 if ok - * >0 error number if error - * *patr elements set + * Determines a number of seconds from a time string in the format: + * [DD:]HH:MM:SS[.MS] + * + * @param str - the time string + * @return the number of seconds or -1 if the format is invalid */ -#define PBS_MAX_TIME (LONG_MAX - 1) - -int decode_time( +long time_str_to_seconds( - pbs_attribute *patr, /* I/O (modified) */ - const char * UNUSED(name), /* I - pbs_attribute name (not used) */ - const char * UNUSED(rescn), /* I - resource name (not used) */ - const char *val, /* I - pbs_attribute value */ - int UNUSED(perm)) /* only used for resources */ + const char *str) { - int i; + long time_val = 0; + + char *workval = strdup(str); char msec[4]; + + char *workvalsv = workval; + bool use_days = false; int ncolon = 0; - int use_days = 0; int days = 0; - char *pc; - long rv = 0; - char *workval; - char *workvalsv; - - if ((val == NULL) || (strlen(val) == 0)) - { - patr->at_flags = (patr->at_flags & ~ATR_VFLAG_SET) | ATR_VFLAG_MODIFY; - - patr->at_val.at_long = 0; - - /* SUCCESS */ - - return(0); - } - - /* FORMAT: [DD]:HH:MM:SS[.MS] */ - - workval = strdup(val); - - workvalsv = workval; if (workvalsv == NULL) { /* FAILURE - cannot alloc memory */ - goto badval; + return(-1); } - for (i = 0;i < 3;++i) + for (int i = 0;i < 3;++i) msec[i] = '0'; - msec[i] = '\0'; + msec[3] = '\0'; - for (pc = workval;*pc;++pc) + for (char *pc = workval; *pc; ++pc) { if (*pc == ':') { if (++ncolon > 3) - goto badval; + { + free(workvalsv); + return(-1); + } /* are days specified? */ if (ncolon > 2) - use_days = 1; + use_days = true; } } - for (pc = workval;*pc;++pc) + for (char *pc = workval; *pc; ++pc) { if (*pc == ':') { @@ -193,12 +177,12 @@ int decode_time( if (use_days) { - days = atoi(workval); - use_days = 0; + days = strtol(workval, NULL, 10); + use_days = false; } else { - rv = (rv * 60) + atoi(workval); + time_val = (time_val * 60) + strtol(workval, NULL, 10); } workval = pc + 1; @@ -208,43 +192,83 @@ int decode_time( { *pc++ = '\0'; - for (i = 0; (i < 3) && *pc; ++i) + for (int i = 0; (i < 3) && *pc; ++i) msec[i] = *pc++; break; } else if (!isdigit((int)*pc)) { - goto badval; /* bad value */ + free(workvalsv); + return(-1); /* bad value */ } } - rv = (rv * 60) + atoi(workval); + time_val = (time_val * 60) + strtol(workval, NULL, 10); if (days > 0) - rv = rv + (days * 24 * 3600); + time_val = time_val + (days * 24 * 3600); - if (rv > PBS_MAX_TIME) - goto badval; + if (time_val > PBS_MAX_TIME) + { + free(workvalsv); + return(-1); + } - if (atoi(msec) >= 500) - rv++; + if (strtol(msec, NULL, 10) >= 500) + time_val++; - patr->at_val.at_long = rv; + free(workvalsv); - patr->at_flags |= ATR_VFLAG_SET | ATR_VFLAG_MODIFY; + return(time_val); + } // END time_string_to_seconds() - free(workvalsv); - /* SUCCESS */ - return(0); +/* + * decode_time - decode time into into pbs_attribute structure of type ATR_TYPE_LONG + * + * Returns: 0 if ok + * >0 error number if error + * *patr elements set + */ + -badval: +int decode_time( - free(workvalsv); + pbs_attribute *patr, /* I/O (modified) */ + const char * UNUSED(name), /* I - pbs_attribute name (not used) */ + const char * UNUSED(rescn), /* I - resource name (not used) */ + const char *val, /* I - pbs_attribute value */ + int UNUSED(perm)) /* only used for resources */ + + { + long rv = 0; + int rc = PBSE_NONE; + + if ((val == NULL) || (strlen(val) == 0)) + { + patr->at_flags = (patr->at_flags & ~ATR_VFLAG_SET) | ATR_VFLAG_MODIFY; + + patr->at_val.at_long = 0; + + /* SUCCESS */ + + return(rc); + } + + rv = time_str_to_seconds(val); + + if (rv >= 0) + { + // SUCCESS + patr->at_val.at_long = rv; + patr->at_flags |= ATR_VFLAG_SET | ATR_VFLAG_MODIFY; + } + else + rc = PBSE_BADATVAL; - return(PBSE_BADATVAL); + return(rc); } /* END decode_time() */ diff --git a/src/lib/Libattr/attr_func.c b/src/lib/Libattr/attr_func.c index eb69af002c..31af7c16b9 100644 --- a/src/lib/Libattr/attr_func.c +++ b/src/lib/Libattr/attr_func.c @@ -405,7 +405,6 @@ svrattrl *attrlist_create( - /* * free_attrlist - free the space allocated to a list of svrattrl * structures diff --git a/src/lib/Libattr/attr_str_conversion.c b/src/lib/Libattr/attr_str_conversion.c index 7a755b4f4b..b30a59b87c 100644 --- a/src/lib/Libattr/attr_str_conversion.c +++ b/src/lib/Libattr/attr_str_conversion.c @@ -177,6 +177,65 @@ int size_to_str( +/* + * resource_index_to_string() + * + * Writes the resource at index as a string: name=value + * + * @param output - the string output + * @param resources - the resource vector + * @param index - the index of the resource to print + * @return PBSE_NONE on success or -1 if index if bad + */ + +int resource_index_to_string( + + std::string &output, + std::vector &resources, + size_t index) + + { + if (index >= resources.size()) + return(-1); + + char buf[MAXLINE]; + int rc = PBSE_NONE; + + output = resources[index].rs_defin->rs_name; + output += "="; + + switch (resources[index].rs_value.at_type) + { + case ATR_TYPE_LONG: + + sprintf(buf, "%ld", resources[index].rs_value.at_val.at_long); + output += buf; + + break; + + case ATR_TYPE_STR: + + output += resources[index].rs_value.at_val.at_str; + + break; + + case ATR_TYPE_SIZE: + + size_to_str(resources[index].rs_value.at_val.at_size, buf, sizeof(buf)); + output += buf; + + break; + + default: + + rc = -1; + break; + + } + + return(rc); + } // END resource_index_to_string() + /* diff --git a/src/lib/Libattr/complete_req.cpp b/src/lib/Libattr/complete_req.cpp index 039cb29740..ca8db949d7 100644 --- a/src/lib/Libattr/complete_req.cpp +++ b/src/lib/Libattr/complete_req.cpp @@ -73,13 +73,12 @@ complete_req::complete_req( int task_count = 0; int execution_slots = 0; unsigned long long mem_values[4]; - int active_index[2]; + bool using_pmem = false; + bool using_pvmem = false; if (resources == NULL) return; - active_index[0] = _MEM_; - active_index[1] = _VMEM_; memset(mem_values, 0, sizeof(mem_values)); for (size_t i = 0; i < resources->size(); i++) @@ -144,25 +143,49 @@ complete_req::complete_req( } } - // Set mem and swap from mem_values - unsigned long long mem = mem_values[_MEM_]; - if ((legacy_vmem == false) && - (task_count != 0)) - mem /= task_count; - if (mem_values[_PMEM_] > mem) + if (task_count == 0) + task_count = 1; + + // Set mem_per_task and swap_per_task from mem_values + unsigned long long mem_per_task = mem_values[_MEM_]; + mem_per_task /= task_count; + + if (this->reqs.size() > 0) + { + // Nodes request - check if pmem * ppn > mem_per_task + req &r = this->reqs[0]; + + if (mem_values[_PMEM_] * r.getExecutionSlots() > mem_per_task) + { + mem_per_task = mem_values[_PMEM_]; + using_pmem = true; + } + } + else if (mem_values[_PMEM_] > mem_per_task) { - active_index[0] = _PMEM_; - mem = mem_values[_PMEM_]; + mem_per_task = mem_values[_PMEM_]; + using_pmem = true; } - unsigned long long vmem = mem_values[_VMEM_]; - if ((legacy_vmem == false) && - (task_count != 0)) - vmem /= task_count; - if (mem_values[_PVMEM_] > vmem) + unsigned long long vmem_per_task = mem_values[_VMEM_]; + if (legacy_vmem == false) + vmem_per_task /= task_count; + + if (this->reqs.size() > 0) { - active_index[1] = _PVMEM_; - vmem = mem_values[_PVMEM_]; + // Nodes request - check if pvmem * ppn > vmem_per_task + req &r = this->reqs[0]; + + if (mem_values[_PVMEM_] * r.getExecutionSlots() > vmem_per_task) + { + vmem_per_task = mem_values[_PVMEM_]; + using_pvmem = true; + } + } + else if (mem_values[_PVMEM_] > vmem_per_task) + { + vmem_per_task = mem_values[_PVMEM_]; + using_pvmem = true; } if (this->reqs.size() == 0) @@ -170,14 +193,14 @@ complete_req::complete_req( // Handle the case where no -lnodes request was made // Distribute the memory across the tasks as -l memory is per job req r; - if (task_count != 0) + if (task_count > 1) { r.set_task_count(task_count); r.set_placement_type(place_legacy); } - r.set_memory(mem); - r.set_swap(vmem); + r.set_memory(mem_per_task * task_count); + r.set_swap(vmem_per_task * task_count); if (execution_slots != 0) r.set_execution_slots(execution_slots); @@ -187,48 +210,29 @@ complete_req::complete_req( else { // Handle the case where a -lnodes request was made - if (mem != 0) + if (mem_per_task != 0) { - if (active_index[0] == _MEM_) + for (unsigned int i = 0; i < this->reqs.size(); i++) { - for (unsigned int i = 0; i < this->reqs.size(); i++) - { - req &r = this->reqs[i]; - r.set_memory(mem); - } - } - else if (active_index[0] == _PMEM_) - { - for (unsigned int i = 0; i < this->reqs.size(); i++) - { - req &r = this->reqs[i]; - int ppn_per_req = r.get_execution_slots(); - - r.set_memory(mem * ppn_per_req); - } + req &r = this->reqs[i]; + // When using pmem, multiply by ppn + if (using_pmem == true) + r.set_memory(mem_per_task * r.getTaskCount() * r.getExecutionSlots()); + else + r.set_memory(mem_per_task * r.getTaskCount()); } } - if (vmem != 0) + if (vmem_per_task != 0) { // Handle the case where a -lnodes request was made - if (active_index[1] == _VMEM_) - { - for (unsigned int i = 0; i < this->reqs.size(); i++) - { - req &r = this->reqs[i]; - r.set_swap(vmem); - } - } - else if (active_index[1] == _PVMEM_) + for (unsigned int i = 0; i < this->reqs.size(); i++) { - for (unsigned int i = 0; i < this->reqs.size(); i++) - { - req &r = this->reqs[i]; - int ppn_per_req = r.get_execution_slots(); - - r.set_swap(vmem * ppn_per_req); - } + req &r = this->reqs[i]; + if (using_pvmem == true) + r.set_swap(vmem_per_task * r.getTaskCount() * r.getExecutionSlots()); + else + r.set_swap(vmem_per_task * r.getTaskCount()); } } } diff --git a/src/lib/Libattr/req.cpp b/src/lib/Libattr/req.cpp index ede978d970..4240265d17 100644 --- a/src/lib/Libattr/req.cpp +++ b/src/lib/Libattr/req.cpp @@ -34,12 +34,12 @@ const char *place_thread = "thread"; const char *place_legacy = "legacy"; const char *place_legacy2 = "legacy2"; -req::req() : execution_slots(1), mem(0), swap(0), disk(0), nodes(0), - socket(0), numa_nodes(0), cores(0), threads(0), thread_usage_policy(ALLOW_THREADS), - thread_usage_str(allow_threads), gpus(0), mics(0), maxtpn(0), gres(), os(), arch(), - node_access_policy(), features(), placement_str(), req_attr(), gpu_mode(), - placement_type(exclusive_none), task_count(1), pack(false), single_job_access(false), - per_task_cgroup(-1), index(0), hostlist(), task_allocations() +req::req() : execution_slots(1), total_mem(0), mem_per_task(0), total_swap(0), swap_per_task(0), + disk(0), nodes(0), socket(0), numa_nodes(0), cores(0), threads(0), + thread_usage_policy(ALLOW_THREADS), thread_usage_str(allow_threads), gpus(0), mics(0), + maxtpn(0), gres(), os(), arch(), node_access_policy(), features(), placement_str(), + req_attr(), gpu_mode(), placement_type(exclusive_none), task_count(1), pack(false), + single_job_access(false), per_task_cgroup(-1), index(0), hostlist(), task_allocations() { } @@ -48,12 +48,12 @@ req::req() : execution_slots(1), mem(0), swap(0), disk(0), nodes(0), req::req( - char *work_str) : execution_slots(1), mem(0), swap(0), disk(0), nodes(0), socket(0), - numa_nodes(0), cores(0), threads(0), thread_usage_policy(ALLOW_THREADS), - thread_usage_str(allow_threads), gpus(0), mics(0), maxtpn(0), gres(), - os(), arch(), node_access_policy(), features(), placement_str(), gpu_mode(), - task_count(1), pack(false), single_job_access(false), per_task_cgroup(-1), - index(0), hostlist(), task_allocations() + char *work_str) : execution_slots(1), total_mem(0), mem_per_task(0), total_swap(0), swap_per_task(0), + disk(0), nodes(0), socket(0), numa_nodes(0), cores(0), threads(0), + thread_usage_policy(ALLOW_THREADS), thread_usage_str(allow_threads), gpus(0), + mics(0), maxtpn(0), gres(), os(), arch(), node_access_policy(), features(), + placement_str(), gpu_mode(), task_count(1), pack(false), single_job_access(false), + per_task_cgroup(-1), index(0), hostlist(), task_allocations() { char *ptr = work_str; @@ -109,8 +109,10 @@ req::req( req::req( const req &other) : execution_slots(other.execution_slots), - mem(other.mem), - swap(other.swap), + total_mem(other.total_mem), + mem_per_task(other.mem_per_task), + total_swap(other.total_swap), + swap_per_task(other.swap_per_task), disk(other.disk), nodes(other.nodes), socket(other.socket), @@ -535,7 +537,12 @@ int req::set_name_value_pair( rc = parse_positive_integer(value, this->execution_slots); } else if (!strcmp(name, "memory")) - rc = read_mem_value(value, this->mem); + { + rc = read_mem_value(value, this->mem_per_task); + + if (rc == PBSE_NONE) + this->total_mem = this->mem_per_task * this->task_count; + } else if (!strcmp(name, "gpus")) rc = parse_positive_integer(value, this->gpus); else if (!strcmp(name, "mics")) @@ -560,7 +567,12 @@ int req::set_name_value_pair( else if (!strcmp(name, "reqattr")) this->req_attr = value; else if (!strcmp(name, "swap")) - rc = read_mem_value(value, this->swap); + { + rc = read_mem_value(value, this->swap_per_task); + + if (rc == PBSE_NONE) + this->total_swap = this->swap_per_task * this->task_count; + } else return(PBSE_BAD_PARAMETER); @@ -1111,15 +1123,16 @@ int req::set_from_submission_string( req::req( - const std::string &resource_request) : execution_slots(1), mem(0), swap(0), disk(0), - nodes(0), socket(0), numa_nodes(0), - cores(0), threads(0), thread_usage_policy(ALLOW_THREADS), - thread_usage_str(allow_threads), gpus(0), mics(0), - maxtpn(0), gres(), os(), arch(), node_access_policy(), - features(), placement_str(), req_attr(), gpu_mode(), - task_count(1), pack(false), single_job_access(false), - per_task_cgroup(-1), index(-1), hostlist(), - task_allocations() + const std::string &resource_request) : execution_slots(1), total_mem(0), mem_per_task(0), + total_swap(0), swap_per_task(0), disk(0), nodes(0), socket(0), + numa_nodes(0), cores(0), threads(0), + thread_usage_policy(ALLOW_THREADS), + thread_usage_str(allow_threads), gpus(0), mics(0), + maxtpn(0), gres(), os(), arch(), node_access_policy(), + features(), placement_str(), req_attr(), gpu_mode(), + task_count(1), pack(false), single_job_access(false), + per_task_cgroup(-1), index(-1), hostlist(), + task_allocations() { char *work_str = strdup(resource_request.c_str()); @@ -1145,8 +1158,10 @@ req &req::operator =( return(*this); this->execution_slots = other.execution_slots; - this->mem = other.mem; - this->swap = other.swap; + this->total_mem = other.total_mem; + this->mem_per_task = other.mem_per_task; + this->total_swap = other.total_swap; + this->swap_per_task = other.swap_per_task; this->disk = other.disk; this->nodes = other.nodes; this->socket = other.socket; @@ -1211,15 +1226,27 @@ void req::toString( } } - if (this->mem != 0) + if (this->total_mem != 0) + { + snprintf(buf, sizeof(buf), " total_mem: %lukb\n", this->total_mem); + str += buf; + } + + if (this->mem_per_task != 0) + { + snprintf(buf, sizeof(buf), " mem: %lukb\n", this->mem_per_task); + str += buf; + } + + if (this->total_swap != 0) { - snprintf(buf, sizeof(buf), " mem: %lukb\n", this->mem); + snprintf(buf, sizeof(buf), " total_swap: %lukb\n", this->total_swap); str += buf; } - if (this->swap != 0) + if (this->swap_per_task != 0) { - snprintf(buf, sizeof(buf), " swap: %lukb\n", this->swap); + snprintf(buf, sizeof(buf), " swap: %lukb\n", this->swap_per_task); str += buf; } @@ -1360,6 +1387,9 @@ void req::toString( * * Gets the name and value of each part of this class that is set * + * NOTE: these values are usually passed to encode_complete_req() and often + * later consumed by decode_complete_req() which calls set_value() + * * For each value that is set: * name=:index value=mem != 0) + if (this->mem_per_task != 0) { - snprintf(buf, sizeof(buf), "memory.%d", this->index); + snprintf(buf, sizeof(buf), "total_memory.%d", this->index); names.push_back(buf); - snprintf(buf, sizeof(buf), "%lukb", this->mem); + snprintf(buf, sizeof(buf), "%lukb", this->total_mem); values.push_back(buf); + + if (this->mem_per_task != 0) + { + snprintf(buf, sizeof(buf), "memory.%d", this->index); + names.push_back(buf); + snprintf(buf, sizeof(buf), "%lukb", this->mem_per_task); + values.push_back(buf); + } } - if (this->swap != 0) + if (this->swap_per_task != 0) { - snprintf(buf, sizeof(buf), "swap.%d", this->index); + snprintf(buf, sizeof(buf), "total_swap.%d", this->index); names.push_back(buf); - snprintf(buf, sizeof(buf), "%lukb", this->swap); + snprintf(buf, sizeof(buf), "%lukb", this->total_swap); values.push_back(buf); + + if (this->swap_per_task != 0) + { + snprintf(buf, sizeof(buf), "swap.%d", this->index); + names.push_back(buf); + snprintf(buf, sizeof(buf), "%lukb", this->swap_per_task); + values.push_back(buf); + } } if (this->disk != 0) @@ -1577,6 +1623,7 @@ void req::get_values( } // END get_values() + void req::get_task_stats( std::vector &task_index, @@ -1665,8 +1712,10 @@ char *capture_until_newline_and_advance( * req[] * task count: * [lprocs: ] - * [mem: kb] - * [swap: kb] + * [total_mem: kb] + * [mem: kb] + * [total_swap: kb] + * [swap: kb] * [disk: kb] * [socket: ] * [numa chips: ] @@ -1712,7 +1761,8 @@ void req::set_from_string( } current += 2; // move past the ': ' - this->task_count = strtol(current, ¤t, 10); + if ((this->task_count = strtol(current, ¤t, 10)) == 0) + this->task_count = 1; while (is_whitespace(*current)) current++; @@ -1733,21 +1783,45 @@ void req::set_from_string( move_past_whitespace(¤t); } + if (!strncmp(current, "total_mem", 9)) + { + // found total mem + current += 11; // move past 'total_mem: ' + this->total_mem = strtoll(current, ¤t, 10); + this->mem_per_task = this->total_mem / this->task_count; + current += 2; // move past 'kb' + + move_past_whitespace(¤t); + } + if (!strncmp(current, "mem", 3)) { // found mem current += 5; // move past 'mem: ' - this->mem = strtoll(current, ¤t, 10); + this->mem_per_task = strtoll(current, ¤t, 10); + this->total_mem = this->mem_per_task * this->task_count; current += 2; // move past 'kb' move_past_whitespace(¤t); } + if (!strncmp(current, "total_swap", 10)) + { + // found total_swap + current += 12; // move past 'total_swap: ' + this->total_swap = strtoll(current, ¤t, 10); + this->swap_per_task = this->total_swap / this->task_count; + current += 2; // move past kb + + move_past_whitespace(¤t); + } + if (!strncmp(current, "swap", 4)) { // found swap current += 6; // move past 'swap: ' - this->swap = strtoll(current, ¤t, 10); + this->swap_per_task = strtoll(current, ¤t, 10); + this->total_swap = this->swap_per_task * this->task_count; current += 2; // move past kb move_past_whitespace(¤t); @@ -1990,6 +2064,8 @@ void req::set_from_string( * set_value() * * Sets a value on this class identified by name + * NOTE: this is called by decode_complete_req() and should match what is produced by get_values(), + * since get_values() is called by encode_complete_req() * @param name - the name of the value to be set * @param value - the value * @return PBSE_NONE if the name is valid or PBSE_BAD_PARAMETER for an invalid name @@ -2025,22 +2101,44 @@ int req::set_value( } } } + else if (!strncmp(name, "total_memory", 12)) + { + if ((!is_default) || + (this->mem_per_task == 0)) + { + if ((rc = read_mem_value(value, this->total_mem)) != PBSE_NONE) + return(rc); + } + } else if (!strncmp(name, "memory", 6)) { if ((!is_default) || - (this->mem == 0)) + (this->mem_per_task == 0)) { - if ((rc = read_mem_value(value, this->mem)) != PBSE_NONE) + if ((rc = read_mem_value(value, this->mem_per_task)) != PBSE_NONE) + return(rc); + else + this->total_mem = this->mem_per_task * this->task_count; + } + } + else if (!strncmp(name, "total_swap", 10)) + { + if ((!is_default) || + (this->mem_per_task == 0)) + { + if ((rc = read_mem_value(value, this->total_swap)) != PBSE_NONE) return(rc); } } else if (!strncmp(name, "swap", 4)) { if ((!is_default) || - (this->swap == 0)) + (this->swap_per_task == 0)) { - if ((rc = read_mem_value(value, this->swap)) != PBSE_NONE) + if ((rc = read_mem_value(value, this->swap_per_task)) != PBSE_NONE) return(rc); + else + this->total_swap = this->swap_per_task * this->task_count; } } else if (!strncmp(name, "disk", 4)) @@ -2153,7 +2251,7 @@ int req::set_value( /* - * set_value() + * set_task_value() * * Sets a value on this class identified by name * @param name - the name of the value to be set @@ -2204,16 +2302,16 @@ int req::getExecutionSlots() const return(this->execution_slots); } -unsigned long req::getMemory() const +unsigned long req::get_total_memory() const { - return(this->mem); + return(this->total_mem); } -unsigned long req::getSwap() const +unsigned long req::get_total_swap() const { - return(this->swap); + return(this->total_swap); } unsigned long req::getDisk() const @@ -2294,12 +2392,6 @@ int req::getMaxtpn() const return(this->maxtpn); } -std::string req::getGpuMode() const - - { - return(this->gpu_mode); - } - std::string req::getReqAttr() const { @@ -2318,7 +2410,7 @@ int req::getIndex() const return(this->index); } -int req::getGpus() const +int req::get_gpus() const { return(this->gpus); @@ -2419,7 +2511,6 @@ int req::get_num_tasks_for_host( } task_count = num_ppn / this->execution_slots; - } } } @@ -2483,10 +2574,10 @@ int req::get_task_allocation( * */ -unsigned long long req::get_swap_per_task() +unsigned long long req::get_swap_per_task() const { - return(this->swap); + return(this->swap_per_task); } @@ -2497,10 +2588,10 @@ unsigned long long req::get_swap_per_task() * */ -unsigned long long req::get_memory_per_task() +unsigned long long req::get_memory_per_task() const { - return(this->mem); + return(this->mem_per_task); } /* @@ -2516,9 +2607,7 @@ unsigned long long req::get_swap_for_host( { int num_tasks = this->get_num_tasks_for_host(host); - unsigned long long swap = this->swap * num_tasks; - - return(swap); + return(this->swap_per_task * num_tasks); } // END get_swap_for_host() @@ -2535,10 +2624,8 @@ unsigned long long req::get_memory_for_host( const std::string &host) const { - int num_tasks = this->get_num_tasks_for_host(host); - unsigned long long mem = this->mem * num_tasks; - - return(mem); + int num_tasks = this->get_num_tasks_for_host(host); + return(this->mem_per_task * num_tasks); } // END get_memory_for_host() @@ -2557,6 +2644,7 @@ void req::clear_allocations() { this->task_allocations.clear(); + this->hostlist.clear(); } // END clear_allocations() @@ -2588,7 +2676,8 @@ void req::set_memory( unsigned long mem) { - this->mem = mem; + this->total_mem = mem; + this->mem_per_task = this->total_mem / this->task_count; } void req::set_swap( @@ -2596,7 +2685,8 @@ void req::set_swap( unsigned long swap) { - this->swap = swap; + this->total_swap = swap; + this->swap_per_task = this->total_swap / this->task_count; } int req::get_execution_slots() const diff --git a/src/lib/Libcmds/add_verify_resources.c b/src/lib/Libcmds/add_verify_resources.c index 4ebc0851bf..04be1a0f34 100644 --- a/src/lib/Libcmds/add_verify_resources.c +++ b/src/lib/Libcmds/add_verify_resources.c @@ -109,9 +109,10 @@ /* hostlist=name1+name2,mppnodes=name1+name2,param=val1+val23+val24+val25*/ int add_verify_resources( + job_data_container *res_attr, /* M */ - char *resources, /* I */ - int p_type) /* I */ + char *resources, /* I */ + int p_type) /* I */ { char *r; @@ -289,7 +290,7 @@ int add_verify_resources( else snprintf(value, vlen, "%s", v); - hash_add_or_exit(res_attr, name, value, p_type); + hash_priority_add_or_exit(res_attr, name, value, p_type); } else { diff --git a/src/lib/Libcmds/cnt2server.c b/src/lib/Libcmds/cnt2server.c index a9d2bfe850..1a558d84e7 100644 --- a/src/lib/Libcmds/cnt2server.c +++ b/src/lib/Libcmds/cnt2server.c @@ -84,14 +84,15 @@ * * Synopsis: * - * int cnt2server(const char *server) + * int cnt2server(const char *server, bool silence) * * server The name of the server to connect to. A NULL or null string * for the default server. + * silence Suppress stderr? * * Returns: * - * The connection returned by pbs_connect(). + * The connection returned by pbs_connect_ext(). */ #include /* the master config generated by configure */ @@ -133,7 +134,8 @@ extern "C" { int cnt2server( - const char *SpecServer) /* I (optional) */ + const char *SpecServer, /* I (optional) */ + bool silence) /* I (write error messages?) */ { int connect; @@ -207,13 +209,13 @@ int cnt2server( } - /* NOTE: env vars PBS_DEFAULT and PBS_SERVER will be checked and applied w/in pbs_connect() */ + /* NOTE: env vars PBS_DEFAULT and PBS_SERVER will be checked and applied w/in pbs_connect_ext() */ start: - connect = pbs_connect(Server); + connect = pbs_connect_ext(Server, silence); - /* if pbs_connect failed maybe the active server is down. validate the active + /* if pbs_connect_ext failed maybe the active server is down. validate the active server and try again */ if ((connect <= 0) && ((SpecServer == NULL) || (SpecServer[0] == '\0')) && (one_host_only == 0)) { @@ -222,9 +224,9 @@ int cnt2server( if (rc != PBSE_NONE) return(connect); - connect = pbs_connect(valid_server); + connect = pbs_connect_ext(valid_server, silence); - if (connect >= 0) + if ((!silence) && (connect >= 0)) fprintf(stderr, "New active server is %s\n", valid_server); free(valid_server); } @@ -239,22 +241,25 @@ int cnt2server( { case PBSE_BADHOST: - if (Server[0] == '\0') + if (!silence) { - fprintf(stderr, "Cannot resolve default server host '%s' - check server_name file.\n", - pbs_default()); - } - else - { - fprintf(stderr, "Cannot resolve specified server host '%s'.\n", - Server); + if (Server[0] == '\0') + { + fprintf(stderr, "Cannot resolve default server host '%s' - check server_name file.\n", + pbs_default()); + } + else + { + fprintf(stderr, "Cannot resolve specified server host '%s'.\n", + Server); + } } break; case PBSE_NOCONNECTS: - if (thistime == 0) + if ((!silence) && (thistime == 0)) fprintf(stderr, "Too many open connections.\n"); if (cnt2server_retry != 0) @@ -264,13 +269,14 @@ int cnt2server( case PBSE_NOSERVER: - fprintf(stderr, "No default server name - check server_name file.\n"); + if (!silence) + fprintf(stderr, "No default server name - check server_name file.\n"); break; case PBSE_SYSTEM: - if (thistime == 0) + if ((!silence) && (thistime == 0)) fprintf(stderr, "System call failure.\n"); if (cnt2server_retry != 0) @@ -280,7 +286,7 @@ int cnt2server( case PBSE_PERM: - if (thistime == 0) + if ((!silence) && (thistime == 0)) fprintf(stderr, "No Permission.\n"); if (cnt2server_retry != 0) @@ -289,13 +295,15 @@ int cnt2server( break; case PBSE_IFF_NOT_FOUND: - - fprintf(stderr, "Unable to authorize user.\n"); + + if (!silence) + fprintf(stderr, "Unable to authorize user.\n"); break; case PBSE_PROTOCOL: - fprintf(stderr, "protocol failure.\n"); + if (!silence) + fprintf(stderr, "protocol failure.\n"); break; case PBSE_SOCKET_FAULT: @@ -325,15 +333,18 @@ int cnt2server( break; } - connect = pbs_connect(new_server_name); - if (connect <= 0) + connect = pbs_connect_ext(new_server_name, silence); + if (!silence) { - if (thistime == 0) - fprintf(stderr, "Communication failure.\n"); - } - else - { - fprintf(stderr, "New active server is %s\n", new_server_name); + if (connect <= 0) + { + if (thistime == 0) + fprintf(stderr, "Communication failure.\n"); + } + else + { + fprintf(stderr, "New active server is %s\n", new_server_name); + } } free(new_server_name); @@ -344,7 +355,7 @@ int cnt2server( default: - if (thistime == 0) + if ((!silence) && (thistime == 0)) fprintf(stderr, "Communication failure.\n"); if (cnt2server_retry != 0) @@ -378,10 +389,11 @@ int cnt2server( goto start; } - fprintf(stderr, "Cannot connect to default server host '%s' - check pbs_server daemon and/or trqauthd.\n", - pbs_default()); + if (!silence) + fprintf(stderr, "Cannot connect to default server host '%s' - check pbs_server daemon and/or trqauthd.\n", + pbs_default()); } - else + else if (!silence) { fprintf(stderr, "Cannot connect to specified server host '%s'.\n", Server); @@ -402,7 +414,7 @@ int cnt2server( retry: - if (thistime == 0) + if ((!silence) && (thistime == 0)) { fprintf(stderr, "Retrying for %d seconds\n", (int)cnt2server_retry); diff --git a/src/lib/Libcmds/get_server.c b/src/lib/Libcmds/get_server.c index 643c979685..e4e2ec57a8 100644 --- a/src/lib/Libcmds/get_server.c +++ b/src/lib/Libcmds/get_server.c @@ -158,6 +158,11 @@ int get_server_and_job_ids( if (notNULL(parent_server)) { + char *colon = strchr(parent_server, ':'); + + if (colon != NULL) + *colon = '\0'; + snprintf(def_server, PBS_MAXSERVERNAME, "%s", parent_server); c = def_server; diff --git a/src/lib/Libcmds/lib_cmds.h b/src/lib/Libcmds/lib_cmds.h index 7c6b38571d..f359a3633e 100644 --- a/src/lib/Libcmds/lib_cmds.h +++ b/src/lib/Libcmds/lib_cmds.h @@ -13,7 +13,7 @@ int check_job_name(char *name, int chk_alpha); /* from file cnt2server.c */ int cnt2server_conf(long retry); -int cnt2server(const char *SpecServer); +int cnt2server(const char *SpecServer, bool silence); /* from file cvtdate.c */ time_t cvtdate(char *datestr); diff --git a/src/lib/Libifl/dec_svrattrl.c b/src/lib/Libifl/dec_svrattrl.c index f44e2ba3fd..369e92e69a 100644 --- a/src/lib/Libifl/dec_svrattrl.c +++ b/src/lib/Libifl/dec_svrattrl.c @@ -207,14 +207,15 @@ int decode_DIS_svrattrl( psvrat->al_op = (enum batch_op)disrui(chan, &rc); - if (rc) break; + if (rc) + break; append_link(phead, &psvrat->al_link, psvrat); } if (rc) { - (void)free(psvrat); + free(psvrat); } return (rc); diff --git a/src/lib/Libifl/lib_ifl.h b/src/lib/Libifl/lib_ifl.h index 38f99a4ab1..a380bb77a4 100644 --- a/src/lib/Libifl/lib_ifl.h +++ b/src/lib/Libifl/lib_ifl.h @@ -291,6 +291,7 @@ int parse_daemon_response(long long code, long long len, char *buf); ssize_t send_unix_creds(int sd); #endif int pbs_original_connect(char *server); +int pbs_original_connect_ext(char *server, bool silence); int pbs_disconnect_socket(int socket); int pbs_connect_with_retry(char *server_name_ptr, int retry_seconds); void initialize_connections_table(); diff --git a/src/lib/Libifl/pbsD_connect.c b/src/lib/Libifl/pbsD_connect.c index b4591aa2e1..4fcad0f630 100644 --- a/src/lib/Libifl/pbsD_connect.c +++ b/src/lib/Libifl/pbsD_connect.c @@ -887,6 +887,13 @@ int trq_set_preferred_network_interface( +int pbs_original_connect( + + char *server) /* I (FORMAT: NULL | '\0' | HOSTNAME | HOSTNAME:PORT )*/ + + { + return(pbs_original_connect_ext(server, false)); + } /* returns socket descriptor or negative value (-1) on failure */ @@ -896,9 +903,10 @@ int trq_set_preferred_network_interface( /* NOTE: 0 is not a valid return value */ -int pbs_original_connect( +int pbs_original_connect_ext( - char *server) /* I (FORMAT: NULL | '\0' | HOSTNAME | HOSTNAME:PORT )*/ + char *server, /* I (FORMAT: NULL | '\0' | HOSTNAME | HOSTNAME:PORT )*/ + bool silence) /* I */ { struct sockaddr_in server_addr; @@ -1405,9 +1413,12 @@ int pbs_original_connect( if (rc > 0) tmp_err_msg = pbs_strerror(rc); - fprintf(stderr, - "ERROR: cannot authenticate connection to server \"%s\", errno=%d (%s)\n", - server, rc, tmp_err_msg); + if (!silence) + { + fprintf(stderr, + "ERROR: cannot authenticate connection to server \"%s\", errno=%d (%s)\n", + server, rc, tmp_err_msg); + } } local_errno = PBSE_SOCKET_FAULT; @@ -1429,7 +1440,8 @@ int pbs_original_connect( if (rc != PBSE_NONE) { - fprintf(stderr, "%s\n", err_msg.c_str()); + if (!silence) + fprintf(stderr, "%s\n", err_msg.c_str()); goto cleanup_conn; } } /* END if !use_unixsock */ @@ -1566,12 +1578,10 @@ void print_server_port_to_stderr( } /** - * This is a new version of this function that allows - * connecting to a list of servers. It is backwards - * compatible with the previous version in that it - * will accept a single server name. + * Wrapper function for pbs_connect_ext(). * * @param server_name_ptr A pointer to a server name or server name list. + * @param silence Boolean - print error output? * @returns A file descriptor number. */ @@ -1579,6 +1589,25 @@ int pbs_connect( char *server_name_ptr) /* I (optional) */ + { + return(pbs_connect_ext(server_name_ptr, false)); + } + +/** + * Function connects to a list of servers. It is backwards + * compatible with the previous version in that it + * will accept a single server name. + * + * @param server_name_ptr A pointer to a server name or server name list. + * @param silence Boolean - print error output? + * @returns A file descriptor number. + */ + +int pbs_connect_ext( + + char *server_name_ptr, /* I (optional) */ + bool silence) /* I */ + { int connect = -1; int i, list_len; @@ -1620,7 +1649,7 @@ int pbs_connect( current_name); } - if ((connect = pbs_original_connect(current_name)) >= 0) + if ((connect = pbs_original_connect_ext(current_name, silence)) >= 0) { if (getenv("PBSDEBUG")) { @@ -1629,7 +1658,7 @@ int pbs_connect( return(connect); /* Success, we have a connection, return it. */ } - else + else if (!silence) print_server_port_to_stderr(current_name); } } diff --git a/src/lib/Libifl/tcp_dis.c b/src/lib/Libifl/tcp_dis.c index bb1e880be1..35b072a600 100644 --- a/src/lib/Libifl/tcp_dis.c +++ b/src/lib/Libifl/tcp_dis.c @@ -107,7 +107,7 @@ #endif #define MAX_SOCKETS 65536 -time_t pbs_tcp_timeout = 300; +time_t pbs_tcp_timeout = 300; // seconds diff --git a/src/lib/Libifl/trq_auth.c b/src/lib/Libifl/trq_auth.c index a50ebf4a83..05be9780b1 100644 --- a/src/lib/Libifl/trq_auth.c +++ b/src/lib/Libifl/trq_auth.c @@ -70,7 +70,7 @@ int set_active_pbs_server( ) { - strncpy(active_pbs_server, new_active_server, PBS_MAXSERVERNAME); + snprintf(active_pbs_server, sizeof(active_pbs_server), "%s", new_active_server); active_pbs_server_port = new_active_port; return(PBSE_NONE); } @@ -331,6 +331,19 @@ int trq_simple_connect( int optval = 1; std::string server(server_name); +#ifdef BIND_OUTBOUND_SOCKETS + struct sockaddr_in local; + + /* Bind to the IP address associated with the hostname, in case there are + * muliple possible source IPs for this destination.*/ + if (get_local_address(local) != PBSE_NONE) + { + fprintf(stderr, "could not determine local IP address: %s", strerror(errno)); + return(PBSE_SYSTEM); + } + +#endif + memset(&hints, 0, sizeof(hints)); /* set the hints so we get a STREAM socket */ hints.ai_family = AF_UNSPEC; /* allow for IPv4 or IPv6 */ @@ -365,6 +378,23 @@ int trq_simple_connect( continue; } +#ifdef BIND_OUTBOUND_SOCKETS + + // only bind if not localhost + if (!islocalhostaddr(&local)) + { + rc = bind(sock, (struct sockaddr *)&local, sizeof(sockaddr_in)); + if (rc != 0) + { + fprintf(stderr, "could not bind local socket: %s", strerror(errno)); + rc = PBSE_SYSTEM; + close(sock); + continue; + } + } + +#endif + rc = connect(sock, addr_info->ai_addr, addr_info->ai_addrlen); if (rc != 0) { @@ -401,10 +431,10 @@ int trq_simple_connect( * server is made the active server */ int validate_server( - char *active_server_name, - int t_server_port, - char *ssh_key, - char **sign_key) + std::string &active_server_name, + int t_server_port, + char *ssh_key, + char **sign_key) { int rc = PBSE_NONE; @@ -420,8 +450,8 @@ int validate_server( we stick with the active_server_name. If another server in teh server_name_list responds that will become the active_pbs_server */ - if (active_server_name != NULL) - rc = trq_simple_connect(active_server_name, t_server_port, &sd); + if (active_server_name.size() > 0) + rc = trq_simple_connect(active_server_name.c_str(), t_server_port, &sd); else rc = PBSE_UNKREQ; /* If we don't have a server name we want to process everyting. */ @@ -520,19 +550,24 @@ int parse_terminate_request( int parse_request_client( - int sock, - char **server_name, - int *server_port, - int *auth_type, - char **user, - int *user_pid, - int *user_sock) + int sock, + std::string &server_name, + int *server_port, + int *auth_type, + char **user, + int *user_pid, + int *user_sock) { - int rc = PBSE_NONE; - long long tmp_val = 0, tmp_port = 0, tmp_auth_type = 0, tmp_sock = 0, tmp_pid = 0; + int rc = PBSE_NONE; + long long tmp_val = 0; + long long tmp_port = 0; + long long tmp_auth_type = 0; + long long tmp_sock = 0; + long long tmp_pid = 0; + char *ptr = NULL; - if ((rc = socket_read_str(sock, server_name, &tmp_val)) != PBSE_NONE) + if ((rc = socket_read_str(sock, &ptr, &tmp_val)) != PBSE_NONE) { } else if ((rc = socket_read_num(sock, &tmp_port)) != PBSE_NONE) @@ -557,7 +592,14 @@ int parse_request_client( *user_pid = (int)tmp_pid; *user_sock = (int)tmp_sock; } - return rc; + + if (ptr != NULL) + { + server_name = ptr; + free(ptr); + } + + return(rc); } @@ -627,7 +669,8 @@ int build_active_server_response( if (len == 0) { - validate_server(NULL, 0, NULL, NULL); + std::string empty; + validate_server(empty, 0, NULL, NULL); len = strlen(active_pbs_server); } @@ -760,9 +803,9 @@ int parse_response_svr( int get_trq_server_addr( - char *server_name, - char **server_addr, - int *server_addr_len) + const char *server_name, + char **server_addr, + int *server_addr_len) { int rc = PBSE_NONE; @@ -845,7 +888,7 @@ int authorize_socket( int local_socket, std::string &message, char *msg_buf, - char **server_name_ptr, + std::string &server_name, char **user_name_ptr, std::string &err_msg) @@ -877,11 +920,8 @@ int authorize_socket( * 0|0|| */ - if ((rc = parse_request_client(local_socket, server_name_ptr, &server_port, &auth_type, user_name_ptr, &user_pid, &user_sock)) != PBSE_NONE) + if ((rc = parse_request_client(local_socket, server_name, &server_port, &auth_type, user_name_ptr, &user_pid, &user_sock)) != PBSE_NONE) { - if (*server_name_ptr != NULL) - free(*server_name_ptr); - if (*user_name_ptr != NULL) free(*user_name_ptr); @@ -890,7 +930,6 @@ int authorize_socket( else { int retries = 0; - char *server_name = *server_name_ptr; while (retries < MAX_RETRIES) { @@ -905,7 +944,7 @@ int authorize_socket( usleep(20000); continue; } - else if ((rc = get_trq_server_addr(server_name, &trq_server_addr, &trq_server_addr_len)) != PBSE_NONE) + else if ((rc = get_trq_server_addr(server_name.c_str(), &trq_server_addr, &trq_server_addr_len)) != PBSE_NONE) { disconnect_svr = false; retries++; @@ -977,11 +1016,11 @@ int authorize_socket( if (debug_mode == TRUE) { fprintf(stderr, "Conn to %s port %d success. Conn %d authorized\n", - server_name, server_port, user_sock); + server_name.c_str(), server_port, user_sock); } sprintf(msg_buf, - "User %s at IP:port %s:%d logged in", *user_name_ptr, server_name, server_port); + "User %s at IP:port %s:%d logged in", *user_name_ptr, server_name.c_str(), server_port); log_record(PBSEVENT_CLIENTAUTH | PBSEVENT_FORCE, PBS_EVENTCLASS_TRQAUTHD, className, msg_buf); } @@ -1010,7 +1049,7 @@ void *process_svr_conn( { const char *className = "trqauthd"; int rc = PBSE_NONE; - char *server_name = NULL; + std::string server_name; int server_port = 0; char *user_name = NULL; int user_pid = 0; @@ -1074,7 +1113,7 @@ void *process_svr_conn( case TRQ_AUTH_CONNECTION: { - rc = authorize_socket(local_socket, message, msg_buf, &server_name, &user_name, error_string); + rc = authorize_socket(local_socket, message, msg_buf, server_name, &user_name, error_string); break; } default: @@ -1113,25 +1152,25 @@ void *process_svr_conn( if (debug_mode == TRUE) { - if (server_name != NULL) + if (server_name.size() > 0) fprintf(stderr, "Conn to %s port %d Fail. Conn %d not authorized (Err Num %d)\n", - server_name, server_port, user_sock, rc); + server_name.c_str(), server_port, user_sock, rc); } if (error_string.size() == 0) { - if (server_name != NULL) + if (server_name.size() > 0) snprintf(msg_buf, sizeof(msg_buf), "User %s at IP:port %s:%d login attempt failed --no message", (user_name) ? user_name : "null", - server_name, server_port); + server_name.c_str(), server_port); } else { snprintf(msg_buf, sizeof(msg_buf), "User %s at IP:port %s:%d login attempt failed --%s", (user_name) ? user_name : "null", - (server_name) ? server_name : "null", server_port, + (server_name.size()) ? server_name.c_str() : "null", server_port, error_string.c_str()); } log_record(PBSEVENT_CLIENTAUTH | PBSEVENT_FORCE, PBS_EVENTCLASS_TRQAUTHD, @@ -1141,9 +1180,6 @@ void *process_svr_conn( if (message.length() != 0) rc = socket_write(local_socket, message.c_str(), message.length()); - if (server_name != NULL) - free(server_name); - if (user_name != NULL) free(user_name); diff --git a/src/lib/Libnet/lib_net.h b/src/lib/Libnet/lib_net.h index 85b789b2c0..beca183ab0 100644 --- a/src/lib/Libnet/lib_net.h +++ b/src/lib/Libnet/lib_net.h @@ -52,6 +52,8 @@ int socket_read_str(int socket, char **the_str, long long *str_len); int socket_close(int socket); int pbs_getaddrinfo(const char *pNode,struct addrinfo *pHints,struct addrinfo **ppAddrInfoOut); int connect_to_trqauthd(int *sock); +int get_local_address(struct sockaddr_in &new_sockaddr); +bool islocalhostaddr(struct sockaddr_in*); /* from file server_core.c */ @@ -64,15 +66,10 @@ int start_listener_addrinfo(char *host_name, int server_port, void *(*process_me int bindresvport(int sd, struct sockaddr_in *sin); #endif int get_max_num_descriptors(void); -int get_fdset_size(void); /* static int await_connect(long timeout, int sockd); */ int client_to_svr(pbs_net_t hostaddr, unsigned int port, int local_port, char *EMsg); /* from file net_server.c */ -void global_sock_add(int new_sock); -void global_sock_rem(int new_sock); -fd_set *global_sock_getlist(); -int global_sock_verify(); void netcounter_incr(void); int get_num_connections(); void netcounter_get(int netrates[]); @@ -80,7 +77,6 @@ int init_network(unsigned int port, void *(*readfunc)(void *)); int check_network_port(unsigned int port); int check_trqauthd_unix_domain_port(const char *); int ping_trqauthd(const char *); -int thread_func(int active_sockets, fd_set *select_set); int wait_request(time_t waittime, long *SState); /* static void accept_conn(void *new_conn); */ void globalset_add_sock(int sock); @@ -96,7 +92,7 @@ char *netaddr_pbs_net_t(pbs_net_t ipadd); void net_add_close_func(int, void (*func)(int)); /* from file port_forwarding.c */ -void port_forwarder(struct pfwdsock *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, char *EMsg); +void port_forwarder(std::vector *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, char *EMsg); void set_nodelay(int fd); int connect_local_xsocket(u_int dnr); int x11_connect_display(char *display, long alsounused, char *EMsg); diff --git a/src/lib/Libnet/net_client.c b/src/lib/Libnet/net_client.c index ad220e6e6f..8f8542320b 100644 --- a/src/lib/Libnet/net_client.c +++ b/src/lib/Libnet/net_client.c @@ -95,6 +95,7 @@ #include #include #include +#include #ifdef _AIX #include @@ -124,48 +125,41 @@ int get_max_num_descriptors(void) return(max_num_descriptors); } /* END get_num_max_descriptors() */ -/** - * Returns the number of bytes needed to allocate - * a fd_set array that can hold all of the possible - * socket descriptors. +/* + * wait_for_write_ready + * + * Wait for file descriptor to be ready for writing + * within timeout period. + * + * @param fd - file descriptor + * @param timeout_ms - timeout in milliseconds + * @return true if file descriptor ready for writing, false otherwise */ -int get_fdset_size(void) - - { - unsigned int MaxNumDescriptors = 0; - int NumFDSetsNeeded = 0; - int NumBytesInFDSet = 0; - int Result = 0; - - MaxNumDescriptors = get_max_num_descriptors(); +bool wait_for_write_ready( - NumBytesInFDSet = sizeof(fd_set); - NumFDSetsNeeded = MaxNumDescriptors / FD_SETSIZE; + int fd, + int timeout_ms) - if (MaxNumDescriptors < FD_SETSIZE) - { - /* the default size already provides sufficient space */ + { + struct pollfd PollArray; - Result = NumBytesInFDSet; - } - else if ((MaxNumDescriptors % FD_SETSIZE) > 0) - { - /* we need to allocate more memory to cover extra - * bits--add an extra FDSet worth of memory to the size */ + // initialize + PollArray.fd = fd; + PollArray.events = POLLOUT; + PollArray.revents = 0; - Result = (NumFDSetsNeeded + 1) * NumBytesInFDSet; - } - else + // wait + if ((poll(&PollArray, 1, timeout_ms) == 1) && + ((PollArray.revents & POLLOUT) != 0)) { - /* division was exact--we know exactly how many bytes we need */ - - Result = NumFDSetsNeeded * NumBytesInFDSet; + // ready to write + return(true); } - return(Result); - } /* END get_fdset_size() */ - + // not ready to write + return(false); + } /* ** wait for connect to complete. We use non-blocking sockets, @@ -174,45 +168,19 @@ int get_fdset_size(void) static int await_connect( - long timeout, /* I */ + int timeout_ms, /* I - milliseconds */ int sockd) /* I */ { - int n; int val; int rc; - fd_set *BigFDSet = NULL; - - struct timeval tv; - torque_socklen_t len; - /* - * some operating systems (like FreeBSD) cannot have a value for tv.tv_usec - * larger than 1,000,000 so we need to split up the timeout duration between - * seconds and microseconds - */ - - tv.tv_sec = timeout / 1000000; - tv.tv_usec = timeout % 1000000; - - /* calculate needed size for fd_set in select() */ - - BigFDSet = (fd_set *)calloc(1,sizeof(char) * get_fdset_size()); - if (!BigFDSet) - { - log_err(ENOMEM,__func__,"Could not allocate memory to set file descriptor"); - return -1; - } - - FD_SET(sockd, BigFDSet); - - if ((n = select(sockd+1,0,BigFDSet,0,&tv)) != 1) + if (wait_for_write_ready(sockd, timeout_ms) == false) { /* FAILURE: socket not ready for write */ - free(BigFDSet); return(-1); } @@ -224,7 +192,6 @@ static int await_connect( { /* SUCCESS: no failures detected */ - free(BigFDSet); return(0); } @@ -232,7 +199,6 @@ static int await_connect( /* FAILURE: socket error detected */ - free(BigFDSet); return(-1); } /* END await_connect() */ @@ -240,7 +206,7 @@ static int await_connect( /* global */ -long MaxConnectTimeout = 5000000; /* in microseconds */ +int MaxConnectTimeout = 5000; /* in milliseconds */ /* * client_to_svr - connect to a server @@ -257,7 +223,7 @@ long MaxConnectTimeout = 5000000; /* in microseconds */ * hosts with the same port. Let the caller keep the addresses around * rather than look it up each time. * - * NOTE: will wait up to MaxConnectTimeout microseconds for transient network failures + * NOTE: will wait up to MaxConnectTimeout milliseconds for transient network failures */ /* NOTE: create new connection on reserved port to validate root/trusted authority */ @@ -270,7 +236,6 @@ int client_to_svr( char *EMsg) /* O (optional,minsize=1024) */ { - const char id[] = "client_to_svr"; struct sockaddr_in local; struct sockaddr_in remote; int sock; @@ -286,9 +251,8 @@ int client_to_svr( int trycount = 0; struct timespec rem; -#define STARTPORT 144 #define ENDPORT (IPPORT_RESERVED - 1) -#define NPORTS (ENDPORT - STARTPORT + 1) +#define NPORTS (ENDPORT - RESERVED_PORT_START + 1) #define SHUFFLE_COUNT 3 if (EMsg != NULL) @@ -324,7 +288,7 @@ int client_to_svr( if (EMsg != NULL) sprintf(EMsg, "cannot create socket in %s - errno: %d %s", - id, + __func__, errno, err_buf); @@ -335,7 +299,7 @@ int client_to_svr( { if (EMsg != NULL) sprintf(EMsg, "PBS_NET_MAX_CONNECTIONS exceeded in %s", - id); + __func__); close(sock); /* too many connections */ @@ -398,7 +362,7 @@ int client_to_svr( /* bindresvport could not get a privileged port */ if (EMsg != NULL) sprintf(EMsg, "cannot bind to reserved port in %s - errno: %d %s", - id, + __func__, errno, err_buf); @@ -413,7 +377,7 @@ int client_to_svr( #else /* HAVE_BINDRESVPORT */ /* Pseudo-casual shuffling of tryport */ - tryport = (rand() % NPORTS) + STARTPORT; + tryport = (rand() % NPORTS) + RESERVED_PORT_START; #endif /* HAVE_BINDRESVPORT */ } @@ -423,7 +387,7 @@ int client_to_svr( if (tryport > ENDPORT) { - tryport = STARTPORT; + tryport = RESERVED_PORT_START; } } @@ -468,7 +432,7 @@ int client_to_svr( err_buf[0] = '\0'; sprintf(EMsg, "cannot bind to reserved port in %s - errno: %d %s", - id, + __func__, errno, err_buf); } @@ -540,7 +504,7 @@ int client_to_svr( if (EMsg != NULL) sprintf(EMsg, "cannot connect to port %d in %s - connection refused.\n Check if trqauthd should be running\n", port, - id); + __func__); close(sock); @@ -562,7 +526,7 @@ int client_to_svr( if (await_connect(MaxConnectTimeout, sock) == 0) { - /* socket not ready for writing after MaxConnectTimeout microseconds timeout */ + /* socket not ready for writing after MaxConnectTimeout milliseconds timeout */ /* no network failures detected */ break; @@ -600,7 +564,7 @@ int client_to_svr( sprintf(EMsg, "cannot connect to port %d in %s - errno:%d %s", port, - id, + __func__, errno, err_buf); } @@ -629,7 +593,7 @@ int client_to_svr( if (EMsg != NULL) sprintf(EMsg, "cannot connect to port %d in %s - errno:%d %s", tryport, - id, + __func__, errno, err_buf); diff --git a/src/lib/Libnet/net_common.c b/src/lib/Libnet/net_common.c index 7672ea5b14..7b36d31aa1 100644 --- a/src/lib/Libnet/net_common.c +++ b/src/lib/Libnet/net_common.c @@ -20,7 +20,12 @@ #include /* errno */ #include /* fcntl, F_GETFL */ #include /* gettimeofday */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif #include /* poll functionality */ + #include #include #include "../lib/Liblog/pbs_log.h" /* log_err */ @@ -200,13 +205,22 @@ int socket_get_tcp_priv() int priv_port = 0; int flags; #endif - - memset(&local, 0, sizeof(struct sockaddr_in)); - local.sin_family = AF_INET; - + if ((local_socket = socket_get_tcp()) < 0) return(-1); +#ifdef BIND_OUTBOUND_SOCKETS + if (get_local_address(local) != PBSE_NONE) + { + close(local_socket); + return(-1); + } +#else + memset(&local, 0, sizeof(struct sockaddr_in)); + local.sin_family = AF_INET; +#endif + + #ifndef NOPRIVPORTS /* According to the notes in the previous code: * bindresvport seems to cause connect() failures in some odd corner case @@ -500,7 +514,7 @@ int process_and_save_socket_error( * socket_wait_for_write() * * connect failed, this function determines why. For non-blocking sockets, - * EINPROGRESS is almost always set, and you need to select the socket and then + * EINPROGRESS is almost always set, and you need to poll the socket and then * read error using getsockopt(), as is done below. * * if the failure is a permanent failure, pass that back to the caller @@ -516,29 +530,31 @@ int socket_wait_for_write( int socket) { - int rc = PBSE_NONE; - int write_soc = 0; - int sock_errno = 0; - socklen_t len = sizeof(int); - fd_set wfd; - struct timeval timeout; + int rc = PBSE_NONE; + int write_soc = 0; + int sock_errno = 0; + socklen_t len = sizeof(int); + struct pollfd PollArray; + struct timespec ts; - timeout.tv_sec = pbs_tcp_timeout; - timeout.tv_usec = 0; + PollArray.fd = socket; + PollArray.events = POLLOUT; + PollArray.revents = 0; - FD_ZERO(&wfd); - FD_SET(socket, &wfd); + ts.tv_sec = pbs_tcp_timeout; + ts.tv_nsec = 0; - if ((write_soc = select(socket+1, 0, &wfd, 0, &timeout)) != 1) + if ((write_soc = ppoll(&PollArray, 1, &ts, NULL)) != 1) { /* timeout is now seen as a permanent failure */ rc = PERMANENT_SOCKET_FAIL; } - else if (((rc = getsockopt(socket, SOL_SOCKET, SO_ERROR, &sock_errno, &len)) == 0) && + else if ((PollArray.revents & POLLOUT) && + ((rc = getsockopt(socket, SOL_SOCKET, SO_ERROR, &sock_errno, &len)) == 0) && (sock_errno == 0)) - { - rc = PBSE_NONE; - } + { + rc = PBSE_NONE; + } else { rc = process_and_save_socket_error(sock_errno); @@ -579,18 +595,22 @@ int socket_wait_for_xbytes( int socket_wait_for_read( int socket, - unsigned int timeout) + unsigned int timeout) // seconds { - int rc = PBSE_NONE; - int ret; - struct pollfd pfd; + int rc = PBSE_NONE; + int ret; + struct pollfd pfd; + struct timespec ts; pfd.fd = socket; pfd.events = POLLIN | POLLHUP; /* | POLLRDNORM; */ pfd.revents = 0; - ret = poll(&pfd, 1, timeout * 1000); /* poll's timeout is in milliseconds */ + ts.tv_sec = timeout; + ts.tv_nsec = 0; + + ret = ppoll(&pfd, 1, &ts, NULL); if (ret > 0) { char buf[8]; @@ -941,7 +961,7 @@ int pbs_getaddrinfo( if ((*ppAddrInfoOut = get_cached_addrinfo_full(pNode)) != NULL) { - return 0; + return(PBSE_NONE); } if (pHints == NULL) @@ -955,20 +975,20 @@ int pbs_getaddrinfo( { if (addrFound) { - rc = 0; + rc = PBSE_NONE; } else { rc = getaddrinfo(pNode,NULL,pHints,ppAddrInfoOut); } - if (rc == 0) + if (rc == PBSE_NONE) { addrFound = TRUE; *ppAddrInfoOut = insert_addr_name_info(*ppAddrInfoOut,pNode); if (*ppAddrInfoOut != NULL) { - return 0; + return(PBSE_NONE); } rc = EAI_AGAIN; } @@ -984,6 +1004,7 @@ int pbs_getaddrinfo( } /* END pbs_getaddrinfo() */ + int connect_to_trqauthd( int *sock) @@ -1018,4 +1039,61 @@ int connect_to_trqauthd( return(rc); } - +/* + * islocalhostaddr() - check if sockaddr_in holds the locahost address + * + */ +bool islocalhostaddr(struct sockaddr_in *saip) + { + if (saip == NULL) + return(false); + + // is 127.0.0.1? + return((saip->sin_addr.s_addr == (uint32_t)((127 << 24) + 1))); + } + +/* + * get_local_address() - lookup the local IP address associated with the local, + * public hostname. + * + * This function will cache the IP address in each process to speed up + * frequent lookups. + */ +int get_local_address( + + struct sockaddr_in &new_sockaddr) + + { + static struct addrinfo *local_addr_info = NULL; + struct addrinfo hints; + int rc; + char hostname[PBS_MAXHOSTNAME]; + hostname[PBS_MAXHOSTNAME - 1] = '\0'; + + // If we already have the local address cached, return (copy) that value. + if (local_addr_info != NULL) + { + new_sockaddr = *((sockaddr_in*)local_addr_info->ai_addr); + + return PBSE_NONE; + } + + /* Bind to the IP address associated with the hostname, in case there are + * muliple possible source IPs for this destination.*/ + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + + gethostname(hostname, sizeof(hostname)); + + rc = getaddrinfo(hostname, NULL, &hints, &local_addr_info); + if (rc != 0) + { + return(rc); + } + + new_sockaddr = *((sockaddr_in*)local_addr_info->ai_addr); + + return PBSE_NONE; + } + diff --git a/src/lib/Libnet/net_server.c b/src/lib/Libnet/net_server.c index f1a3f8e344..591cbfa910 100644 --- a/src/lib/Libnet/net_server.c +++ b/src/lib/Libnet/net_server.c @@ -102,9 +102,10 @@ #include #include /* thread_func functions */ -#if defined(FD_SET_IN_SYS_SELECT_H) -#include +#ifndef _GNU_SOURCE +#define _GNU_SOURCE #endif +#include #if defined(NTOHL_NEEDS_ARPA_INET_H) && defined(HAVE_ARPA_INET_H) #include #endif @@ -153,7 +154,7 @@ struct connection svr_conn[PBS_NET_MAX_CONNECTIONS]; static int max_connection = PBS_NET_MAX_CONNECTIONS; static int num_connections = 0; pthread_mutex_t *num_connections_mutex = NULL; -static fd_set *GlobalSocketReadSet = NULL; +static struct pollfd *GlobalSocketReadArray = NULL; static u_long *GlobalSocketAddrSet = NULL; static u_long *GlobalSocketPortSet = NULL; pthread_mutex_t *global_sock_read_mutex = NULL; @@ -263,7 +264,7 @@ void netcounter_get( /** * init_network - initialize the network interface * allocate a socket and bind it to the service port, - * add the socket to the readset for select(), + * add the socket to the read set for poll(), * add the socket to the connection structure and set the * processing function to accept_conn() */ @@ -323,7 +324,7 @@ int init_network( disiui_(); - for (i = 0;i < PBS_NET_MAX_CONNECTIONS;i++) + for (i = 0; i < PBS_NET_MAX_CONNECTIONS; i++) { svr_conn[i].cn_mutex = (pthread_mutex_t *)calloc(1, sizeof(pthread_mutex_t)); pthread_mutex_init(svr_conn[i].cn_mutex,NULL); @@ -334,10 +335,41 @@ int init_network( pthread_mutex_unlock(svr_conn[i].cn_mutex); } - /* initialize global "read" socket FD bitmap */ - GlobalSocketReadSet = (fd_set *)calloc(1,sizeof(char) * get_fdset_size()); - GlobalSocketAddrSet = (u_long *)calloc(sizeof(ulong),get_max_num_descriptors()); - GlobalSocketPortSet = (u_long *)calloc(sizeof(ulong),get_max_num_descriptors()); + /* initialize global "read" socket array */ + MaxNumDescriptors = get_max_num_descriptors(); + + GlobalSocketReadArray = (struct pollfd *)malloc(MaxNumDescriptors * sizeof(struct pollfd)); + if (GlobalSocketReadArray == NULL) + { + return(-1); // no memory + } + + // set initial values + for (i = 0; i < MaxNumDescriptors; i++) + { + GlobalSocketReadArray[i].fd = -1; + GlobalSocketReadArray[i].events = 0; + GlobalSocketReadArray[i].revents = 0; + } + + GlobalSocketAddrSet = (u_long *)calloc(MaxNumDescriptors, sizeof(ulong)); + if (GlobalSocketAddrSet == NULL) + { + free(GlobalSocketReadArray); + GlobalSocketReadArray = NULL; + return(-1); // no memory + } + + GlobalSocketPortSet = (u_long *)calloc(MaxNumDescriptors, sizeof(ulong)); + if (GlobalSocketPortSet == NULL) + { + free(GlobalSocketReadArray); + GlobalSocketReadArray = NULL; + + free(GlobalSocketAddrSet); + GlobalSocketAddrSet = NULL; + return(-1); // no memory + } global_sock_read_mutex = (pthread_mutex_t *)calloc(1, sizeof(pthread_mutex_t)); pthread_mutex_init(global_sock_read_mutex,&t_attr); @@ -365,7 +397,7 @@ int init_network( if (port != 0) { - sock = socket(AF_INET, SOCK_STREAM, 0); + sock = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, 0); if (sock < 0) { @@ -398,7 +430,7 @@ int init_network( return(-1); } - /* record socket in connection structure and select set */ + /* record socket in connection structure and poll set */ add_conn(sock, type, (pbs_net_t)0, 0, PBS_SOCK_INET, accept_conn); @@ -417,7 +449,7 @@ int init_network( { /* setup unix domain socket */ - unixsocket = socket(AF_UNIX, SOCK_STREAM, 0); + unixsocket = socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); if (unixsocket < 0) { @@ -647,7 +679,7 @@ int check_network_port( /* * wait_request - wait for a request (socket with data to read) - * This routine does a select on the readset of sockets, + * This routine does a poll on the read set of sockets, * when data is ready, the processing routine associated with * the socket is invoked. */ @@ -662,58 +694,71 @@ int wait_request( int n; time_t now; - fd_set *SelectSet = NULL; - int SelectSetSize = 0; + struct pollfd *PollArray = NULL; + struct timespec ts; + int PollArraySize = 0; int MaxNumDescriptors = 0; int SetSize = 0; - u_long *SocketAddrSet = NULL; - u_long *SocketPortSet = NULL; + u_long *SocketAddrSet = NULL; + u_long *SocketPortSet = NULL; char tmpLine[1024]; - struct timeval timeout; long OrigState = 0; if (SState != NULL) OrigState = *SState; - timeout.tv_usec = 0; - timeout.tv_sec = waittime; + // set timeout for ppoll() + ts.tv_sec = waittime; + ts.tv_nsec = 0; - SelectSetSize = sizeof(char) * get_fdset_size(); - SelectSet = (fd_set *)calloc(1,SelectSetSize); + MaxNumDescriptors = get_max_num_descriptors(); + + PollArraySize = MaxNumDescriptors * sizeof(struct pollfd); + PollArray = (struct pollfd *)malloc(PollArraySize); - if(SelectSet == NULL) + if (PollArray == NULL) { return(-1); } + + // since we don't want to hold the mutex for too long, + // just make a local copy of the data and act on it + pthread_mutex_lock(global_sock_read_mutex); - - memcpy(SelectSet,GlobalSocketReadSet,SelectSetSize); + + // copy global array to local one + memcpy(PollArray, GlobalSocketReadArray, PollArraySize); - /* selset = readset;*/ /* readset is global */ - MaxNumDescriptors = get_max_num_descriptors(); - SetSize = MaxNumDescriptors*sizeof(u_long); + SetSize = MaxNumDescriptors * sizeof(u_long); SocketAddrSet = (u_long *)malloc(SetSize); SocketPortSet = (u_long *)malloc(SetSize); - if((SocketAddrSet == NULL) || (SocketPortSet == NULL)) + + if ((SocketAddrSet == NULL) || (SocketPortSet == NULL)) { - free(SelectSet); - if(SocketAddrSet != NULL) free(SocketAddrSet); - if(SocketPortSet != NULL) free(SocketPortSet); + free(PollArray); + + if (SocketAddrSet != NULL) + free(SocketAddrSet); + if (SocketPortSet != NULL) + free(SocketPortSet); + pthread_mutex_unlock(global_sock_read_mutex); return (-1); } - memcpy(SocketAddrSet,GlobalSocketAddrSet,SetSize); - memcpy(SocketPortSet,GlobalSocketPortSet,SetSize); + + memcpy(SocketAddrSet, GlobalSocketAddrSet, SetSize); + memcpy(SocketPortSet, GlobalSocketPortSet, SetSize); pthread_mutex_unlock(global_sock_read_mutex); - n = select(MaxNumDescriptors, SelectSet, (fd_set *)0, (fd_set *)0, &timeout); + + n = ppoll(PollArray, MaxNumDescriptors, &ts, NULL); if (n == -1) { - if (errno == EINTR) + if (errno == EINTR) { n = 0; /* interrupted, cycle around */ } @@ -725,38 +770,48 @@ int wait_request( /* check all file descriptors to verify they are valid */ - /* NOTE: selset may be modified by failed select() */ - for (i = 0; i < MaxNumDescriptors; i++) { - if (FD_ISSET(i, SelectSet) == 0) + // skip this entry, fd not set + if (PollArray[i].fd < 0) continue; - if (fstat(i, &fbuf) == 0) + // skip this entry, return events present + if (PollArray[i].revents != 0) continue; - /* clean up SdList and bad sd... */ + // skip this entry, it's a valid descriptor + if (fstat(i, &fbuf) == 0) + continue; + // remove socket from the global set since it's no longer valid globalset_del_sock(i); - } /* END for each socket in global read set */ + } /* END for each socket in global read array */ - free(SelectSet); + free(PollArray); free(SocketAddrSet); free(SocketPortSet); - log_err(errno, __func__, "Unable to select sockets to read requests"); + log_err(errno, __func__, "Unable to poll sockets to read requests"); return(-1); } /* END else (errno == EINTR) */ } /* END if (n == -1) */ - for (i = 0; (i < max_connection) && (n > 0); i++) + for (i = 0; (n > 0) && (i < max_connection); i++) { - if (FD_ISSET(i, SelectSet)) + // skip entry if it has no return events + if (PollArray[i].revents == 0) + continue; + + // decrement the count of structures that have non-zero revents + n--; + + // is there data ready to read? + if ((PollArray[i].revents & POLLIN)) { pthread_mutex_lock(svr_conn[i].cn_mutex); /* this socket has data */ - n--; svr_conn[i].cn_lasttime = time(NULL); @@ -793,7 +848,7 @@ int wait_request( pthread_mutex_lock(num_connections_mutex); - sprintf(tmpLine, "closed connections to fd %d - num_connections=%d (select bad socket)", + sprintf(tmpLine, "closed connections to fd %d - num_connections=%d (poll bad socket)", i, num_connections); @@ -803,7 +858,7 @@ int wait_request( } } /* END for i */ - free(SelectSet); + free(PollArray); free(SocketAddrSet); free(SocketPortSet); @@ -816,7 +871,7 @@ int wait_request( now = time((time_t *)0); - for (i = 0;i < max_connection;i++) + for (i = 0; i < max_connection; i++) { struct connection *cp; @@ -878,7 +933,7 @@ int wait_request( * accept_conn - accept request for new connection * this routine is normally associated with the main socket, * requests for connection on the socket are accepted and - * the new socket is added to the select set and the connection + * the new socket is added to the poll set and the connection * structure - the processing routine is set to the external * function: process_request(socket) * @@ -937,7 +992,7 @@ void *accept_conn( } else { - /* add the new socket to the select set and connection structure */ + /* add the new socket to the poll set and connection structure */ add_conn( newsock, FromClientDIS, @@ -962,7 +1017,9 @@ void globalset_add_sock( { pthread_mutex_lock(global_sock_read_mutex); - FD_SET(sock, GlobalSocketReadSet); + GlobalSocketReadArray[sock].fd = sock; + GlobalSocketReadArray[sock].events = POLLIN; + GlobalSocketReadArray[sock].revents = 0; GlobalSocketAddrSet[sock] = addr; GlobalSocketPortSet[sock] = port; pthread_mutex_unlock(global_sock_read_mutex); @@ -977,7 +1034,9 @@ void globalset_del_sock( { pthread_mutex_lock(global_sock_read_mutex); - FD_CLR(sock, GlobalSocketReadSet); + GlobalSocketReadArray[sock].fd = -1; + GlobalSocketReadArray[sock].events = 0; + GlobalSocketReadArray[sock].revents = 0; GlobalSocketAddrSet[sock] = 0; GlobalSocketPortSet[sock] = 0; pthread_mutex_unlock(global_sock_read_mutex); @@ -1000,7 +1059,7 @@ int add_connection( unsigned int port, /* port number (host order) on connected host */ unsigned int socktype, /* inet or unix */ void *(*func)(void *), /* function to invoke on data rdy to read */ - int add_wait_request) /* True to add into global select set */ + int add_wait_request) /* True to add into global poll set */ { if ((sock < 0) || @@ -1182,10 +1241,10 @@ void close_conn( /* * In the case of a -t cold start, this will be called prior to - * GlobalSocketReadSet being initialized + * GlobalSocketReadArray being initialized */ - if (GlobalSocketReadSet != NULL) + if (GlobalSocketReadArray != NULL) { globalset_del_sock(sd); } @@ -1252,10 +1311,10 @@ void clear_conn( /* * In the case of a -t cold start, this will be called prior to - * GlobalSocketReadSet being initialized + * GlobalSocketReadArray being initialized */ - if (GlobalSocketReadSet != NULL) + if (GlobalSocketReadArray != NULL) { globalset_del_sock(sd); } @@ -1298,7 +1357,7 @@ void net_close( { int i; - for (i = 0;i < max_connection;i++) + for (i = 0; i < max_connection; i++) { if (i != but) { diff --git a/src/lib/Libnet/port_forwarding.c b/src/lib/Libnet/port_forwarding.c index 1a144df346..e03351dea4 100644 --- a/src/lib/Libnet/port_forwarding.c +++ b/src/lib/Libnet/port_forwarding.c @@ -16,7 +16,10 @@ #include #include #include +#include #include "lib_ifl.h" +#include "lib_net.h" +#include #include "port_forwarding.h" @@ -32,85 +35,94 @@ void port_forwarder( - struct pfwdsock *socks, + std::vector *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, char *EMsg) /* O */ { - fd_set rfdset, wfdset, efdset; - int rc, maxsock = 0; - + int n; + int n2; + int sock; + int num_events; + int rc; struct sockaddr_in from; - torque_socklen_t fromlen; - int n, n2, sock; - - fromlen = sizeof(from); + torque_socklen_t fromlen; + std::vector PollArray(NUM_SOCKS); while (1) { - FD_ZERO(&rfdset); - FD_ZERO(&wfdset); - FD_ZERO(&efdset); - maxsock = 0; - for (n = 0; n < NUM_SOCKS; n++) { - if (!(socks + n)->active) + // clear the entry + PollArray.at(n).fd = -1; + PollArray.at(n).events = 0; + PollArray.at(n).revents = 0; + + if (!socks->at(n).active) continue; - if ((socks + n)->listening) + if (socks->at(n).listening) { - FD_SET((socks + n)->sock, &rfdset); + PollArray.at(n).fd = socks->at(n).sock; + PollArray.at(n).events = POLLIN; } else { - if ((socks + n)->bufavail < BUF_SIZE) - FD_SET((socks + n)->sock, &rfdset); - - if ((socks + ((socks + n)->peer))->bufavail - (socks + ((socks + n)->peer))->bufwritten > 0) - FD_SET((socks + n)->sock, &wfdset); + if (socks->at(n).bufavail < BUF_SIZE) + { + PollArray.at(n).fd = socks->at(n).sock; + PollArray.at(n).events = POLLIN; + } - /*FD_SET((socks+n)->sock,&efdset);*/ + if (socks->at(socks->at(n).peer).bufavail - socks->at(socks->at(n).peer).bufwritten > 0) + { + PollArray.at(n).fd = socks->at(n).sock; + PollArray.at(n).events |= POLLOUT; + } } - - maxsock = (socks + n)->sock > maxsock ? (socks + n)->sock : maxsock; } - maxsock++; - - rc = select(maxsock, &rfdset, &wfdset, &efdset, NULL); + num_events = poll(&PollArray[0], NUM_SOCKS, -1); - if ((rc == -1) && (errno == EINTR)) + if ((num_events == -1) && (errno == EINTR)) continue; - if (rc < 0) + if (num_events < 0) { - perror("port forwarding select()"); + perror("port forwarding poll()"); exit(EXIT_FAILURE); } - for (n = 0;n < NUM_SOCKS;n++) + for (n = 0; (num_events > 0) && (n < NUM_SOCKS); n++) { - if (!(socks + n)->active) + // skip entry with no return events + if (PollArray.at(n).revents == 0) + continue; + + // decrement the count of events returned + num_events--; + + if (!socks->at(n).active) continue; - if (FD_ISSET((socks + n)->sock, &rfdset)) + if ((PollArray.at(n).revents & POLLIN)) { - if ((socks + n)->listening) + if (socks->at(n).listening) { int newsock = 0, peersock = 0; - - if ((sock = accept((socks + n)->sock, (struct sockaddr *) & from, &fromlen)) < 0) + fromlen = sizeof(from); + + if ((sock = accept(socks->at(n).sock, (struct sockaddr *) & from, &fromlen)) < 0) { if ((errno == EAGAIN) || (errno == EWOULDBLOCK) || (errno == EINTR) || (errno == ECONNABORTED)) continue; - close((socks + n)->sock); + close(socks->at(n).sock); - (socks + n)->active = 0; + socks->at(n).active = 0; continue; } @@ -119,7 +131,7 @@ void port_forwarder( for (n2 = 0; n2 < NUM_SOCKS; n2++) { - if ((socks + n2)->active || (((socks + n2)->peer != 0) && (socks + ((socks + n2)->peer))->active)) + if (socks->at(n2).active || ((socks->at(n2).peer != 0) && socks->at(socks->at(n2).peer).active)) continue; if (newsock == 0) @@ -130,93 +142,90 @@ void port_forwarder( break; } - (socks + newsock)->sock = (socks + peersock)->remotesock = sock; - (socks + newsock)->listening = (socks + peersock)->listening = 0; - (socks + newsock)->active = (socks + peersock)->active = 1; - (socks + newsock)->peer = (socks + peersock)->sock = connfunc(phost, pport, EMsg); - (socks + newsock)->bufwritten = (socks + peersock)->bufwritten = 0; - (socks + newsock)->bufavail = (socks + peersock)->bufavail = 0; - (socks + newsock)->buff[0] = (socks + peersock)->buff[0] = '\0'; - (socks + newsock)->peer = peersock; - (socks + peersock)->peer = newsock; + socks->at(newsock).sock = socks->at(peersock).remotesock = sock; + socks->at(newsock).listening = socks->at(peersock).listening = 0; + socks->at(newsock).active = socks->at(peersock).active = 1; + socks->at(newsock).peer = socks->at(peersock).sock = connfunc(phost, pport, EMsg); + socks->at(newsock).bufwritten = socks->at(peersock).bufwritten = 0; + socks->at(newsock).bufavail = socks->at(peersock).bufavail = 0; + socks->at(newsock).buff[0] = socks->at(peersock).buff[0] = '\0'; + socks->at(newsock).peer = peersock; + socks->at(peersock).peer = newsock; } else { /* non-listening socket to be read */ rc = read_ac_socket( - (socks + n)->sock, - (socks + n)->buff + (socks + n)->bufavail, - BUF_SIZE - (socks + n)->bufavail); + socks->at(n).sock, + socks->at(n).buff + socks->at(n).bufavail, + BUF_SIZE - socks->at(n).bufavail); if (rc < 1) { - shutdown((socks + n)->sock, SHUT_RDWR); - close((socks + n)->sock); - (socks + n)->active = 0; + shutdown(socks->at(n).sock, SHUT_RDWR); + close(socks->at(n).sock); + socks->at(n).active = 0; } else { - (socks + n)->bufavail += rc; + socks->at(n).bufavail += rc; } } - } /* END if rfdset */ - } /* END foreach fd */ + } /* END if read */ - for (n = 0; n < NUM_SOCKS; n++) - { - if (!(socks + n)->active) - continue; - - if (FD_ISSET((socks + n)->sock, &wfdset)) + if ((PollArray.at(n).revents & POLLOUT)) { - int peer = (socks + n)->peer; + int peer = socks->at(n).peer; rc = write_ac_socket( - (socks + n)->sock, - (socks + peer)->buff + (socks + peer)->bufwritten, - (socks + peer)->bufavail - (socks + peer)->bufwritten); + socks->at(n).sock, + socks->at(peer).buff + socks->at(peer).bufwritten, + socks->at(peer).bufavail - socks->at(peer).bufwritten); if (rc < 1) { - shutdown((socks + n)->sock, SHUT_RDWR); - close((socks + n)->sock); - (socks + n)->active = 0; + shutdown(socks->at(n).sock, SHUT_RDWR); + close(socks->at(n).sock); + socks->at(n).active = 0; } else { - (socks + peer)->bufwritten += rc; + socks->at(peer).bufwritten += rc; } - } /* END if wfdset */ - + } /* END if write */ } /* END foreach fd */ - for (n2 = 0; n2 <= 1;n2++) + for (n2 = 0; n2 <= 1; n2++) { for (n = 0; n < NUM_SOCKS; n++) { int peer; - if (!(socks + n)->active || (socks + n)->listening) + if (!socks->at(n).active || socks->at(n).listening) continue; - peer = (socks + n)->peer; + peer = socks->at(n).peer; - if ((socks + n)->bufwritten == (socks + n)->bufavail) + if (socks->at(n).bufwritten == socks->at(n).bufavail) { - (socks + n)->bufwritten = (socks + n)->bufavail = 0; + socks->at(n).bufwritten = socks->at(n).bufavail = 0; } - if (!(socks + peer)->active && ((socks + peer)->bufwritten == (socks + peer)->bufavail)) + if (!socks->at(peer).active && (socks->at(peer).bufwritten == socks->at(peer).bufavail)) { - shutdown((socks + n)->sock, SHUT_RDWR); - close((socks + n)->sock); + shutdown(socks->at(n).sock, SHUT_RDWR); + close(socks->at(n).sock); - (socks + n)->active = 0; + socks->at(n).active = 0; } } } } /* END while(1) */ + + // never executed but should satisfy code profilers looking + // for a malloc/free pairing + delete(socks); } /* END port_forwarder() */ @@ -319,6 +328,9 @@ int x11_connect_display( char buf[1024], *cp; struct addrinfo hints, *ai, *aitop; +#ifdef BIND_OUTBOUND_SOCKETS + struct sockaddr_in local; +#endif char strport[NI_MAXSERV]; @@ -397,6 +409,14 @@ int x11_connect_display( return -1; } +#ifdef BIND_OUTBOUND_SOCKETS + if (get_local_address(local) != PBSE_NONE) + { + fprintf(stderr, "could not determine local IP address: %s", strerror(errno)); + return(-1); + } +#endif + for (ai = aitop; ai; ai = ai->ai_next) { /* Create a socket. */ @@ -408,6 +428,23 @@ int x11_connect_display( continue; } +#ifdef BIND_OUTBOUND_SOCKETS + /* Bind to the IP address associated with the hostname, in case there are + * muliple possible source IPs for this destination.*/ + + // don't bind localhost addr + if (!islocalhostaddr(&local)) + { + if (bind(sock, (struct sockaddr *)&local, sizeof(sockaddr_in))) + { + fprintf(stderr, "could not bind local socket: %s", strerror(errno)); + close(sock); + continue; + } + } + +#endif + /* Connect it to the display. */ if (connect(sock, ai->ai_addr, ai->ai_addrlen) < 0) { diff --git a/src/lib/Libnet/rm.c b/src/lib/Libnet/rm.c index a7dcfbc951..79551b9761 100644 --- a/src/lib/Libnet/rm.c +++ b/src/lib/Libnet/rm.c @@ -96,6 +96,7 @@ #include #include #include +#include #include "pbs_ifl.h" #include "pbs_error.h" @@ -206,6 +207,7 @@ int openrm( struct sockaddr_in addr; struct addrinfo *addr_info; + struct sockaddr_in local; if (pbs_getaddrinfo(host, NULL, &addr_info) != 0) { @@ -215,18 +217,36 @@ int openrm( return(ENOENT * -1); } - memset(&addr, '\0', sizeof(addr)); +#ifdef BIND_OUTBOUND_SOCKETS + /* Bind to the IP address associated with the hostname, in case there are + * muliple possible source IPs for this destination.*/ + if (get_local_address(local) != PBSE_NONE) + { + DBPRT(("could not determine local IP address: %s", strerror(errno))); + close(stream); + return(-1); + } - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = htonl(INADDR_ANY); +#else + + memset(&local, 0, sizeof(struct sockaddr_in)); + + local.sin_family = AF_INET; + +#endif while (retries++ < MAX_RETRIES) { - rc = bindresvport(stream, &addr); + /* Attempt to bind to any reserved port that is free. If BIND_OUTBOUND_SOCKETS + * is defined, we need to make sure the socket is bound to a specific + * local IP address - passed in using the `local` structure. + */ + rc = bindresvport(stream, &local); if (rc != 0) { if (retries >= MAX_RETRIES) { + DBPRT(("could not bind local socket: %s", strerror(errno))); close(stream); return(-1*errno); } @@ -1043,75 +1063,79 @@ extern "C" int activereq(void) { -#ifndef NDEBUG - static char id[] = "activereq"; -#endif + int i; + int num; + int PollArraySize; - int i, num; + struct pollfd *PollArray; - struct timeval tv; - - fd_set *FDSet; + flushreq(); - int MaxNumDescriptors = 0; + // initialize the poll array + PollArraySize = get_max_num_descriptors(); + PollArray = (struct pollfd *)malloc(PollArraySize * sizeof(struct pollfd)); - flushreq(); + if (PollArray == NULL) + { + // no memory + DBPRT(("%s: malloc %d %s\n", __func__, errno, pbs_strerror(errno))) + return(-1); + } - MaxNumDescriptors = get_max_num_descriptors(); - FDSet = (fd_set *)calloc(1,sizeof(char) * get_fdset_size()); + // set initial values + for (i = 0; i < PollArraySize; i++) + { + PollArray[i].fd = -1; + PollArray[i].events = 0; + PollArray[i].revents = 0; + } + // now include the sockets to read for (i = 0; i < HASHOUT; i++) { - struct out *op; - op = outs[i]; - - while (op) + for (op = outs[i]; op != NULL; op = op->next) { - FD_SET(op->chan->sock, FDSet); - op = op->next; + PollArray[op->chan->sock].fd = op->chan->sock; + PollArray[op->chan->sock].events = POLLIN; } } - tv.tv_sec = 15; - - tv.tv_usec = 0; - - num = select(MaxNumDescriptors, FDSet, NULL, NULL, &tv); + // poll with 15sec timeout + num = poll(PollArray, PollArraySize, 15000); if (num == -1) { - DBPRT(("%s: select %d %s\n", id, errno, pbs_strerror(errno))) - free(FDSet); + DBPRT(("%s: poll %d %s\n", __func__, errno, pbs_strerror(errno))) + free(PollArray); return -1; } else if (num == 0) { - free(FDSet); - return -2; + free(PollArray); + return -2; } - for (i = 0; i < HASHOUT; i++) + for (i = 0; (num > 0) && (i < PollArraySize); i++) { + // skip entry with no return events + if (PollArray[i].revents == 0) + continue; - struct out *op; - - op = outs[i]; + // decrement count of structures with return events + num--; - while (op) + // something to read? + if ((PollArray[i].revents & POLLIN)) { - if (FD_ISSET(op->chan->sock, FDSet)) - { - free(FDSet); - return op->chan->sock; - } - - op = op->next; + // return socket id that is ready for reading + free(PollArray); + return(i); } } - free(FDSet); + free(PollArray); return(-2); } /* END activereq() */ diff --git a/src/lib/Libutils/allocation.cpp b/src/lib/Libutils/allocation.cpp index 1349b84a0c..c0f2a9a9f2 100644 --- a/src/lib/Libutils/allocation.cpp +++ b/src/lib/Libutils/allocation.cpp @@ -88,6 +88,8 @@ const int MEM_INDICES = 0; const int CPU_INDICES = 1; +const int GPU_INDICES = 2; +const int MIC_INDICES = 3; const int exclusive_none = 0; const int exclusive_node = 1; @@ -157,8 +159,8 @@ allocation::allocation( { this->cpus = r.getExecutionSlots(); - this->memory = r.getMemory(); - this->gpus = r.getGpus(); + this->memory = r.get_memory_per_task(); + this->gpus = r.get_gpus(); this->mics = r.getMics(); if (r.getThreadUsageString() == use_cores) @@ -233,10 +235,41 @@ void allocation::place_indices_in_string( int which) { - if (which == MEM_INDICES) - translate_vector_to_range_string(output, this->mem_indices); - else - translate_vector_to_range_string(output, this->cpu_indices); + switch (which) + { + case MEM_INDICES: + + translate_vector_to_range_string(output, this->mem_indices); + break; + + case CPU_INDICES: + + translate_vector_to_range_string(output, this->cpu_indices); + break; + + case GPU_INDICES: + + translate_vector_to_range_string(output, this->gpu_indices); + break; + + case MIC_INDICES: + + translate_vector_to_range_string(output, this->mic_indices); + break; + } + } // END place_indices_in_string() + + + +void allocation::place_indices_in_string( + + cgroup_info &cgi) + + { + this->place_indices_in_string(cgi.mem_string, MEM_INDICES); + this->place_indices_in_string(cgi.cpu_string, CPU_INDICES); + this->place_indices_in_string(cgi.gpu_string, GPU_INDICES); + this->place_indices_in_string(cgi.mic_string, MIC_INDICES); } // END place_indices_in_string() @@ -289,12 +322,27 @@ void allocation::write_task_information( { std::string cpus; std::string mems; + std::string gpus; + std::string mics; char buf[256]; translate_vector_to_range_string(cpus, this->cpu_indices); translate_vector_to_range_string(mems, this->mem_indices); task_info = "{\"task\":{\"cpu_list\":\"" + cpus; task_info += "\",\"mem_list\":\"" + mems; + + if (this->gpu_indices.size() > 0) + { + translate_vector_to_range_string(gpus, this->gpu_indices); + task_info += "\",\"gpu_list\":\"" + gpus; + } + + if (this->mic_indices.size() > 0) + { + translate_vector_to_range_string(mics, this->mic_indices); + task_info += "\",\"mic_list\":\"" + mics; + } + if (this->task_cput_used != 0) { unsigned long cput_used = this->task_cput_used; @@ -554,11 +602,82 @@ bool allocation::partially_placed( { return ((this->cpus != r.getExecutionSlots()) || - (this->memory != r.getMemory()) || - (this->gpus != r.getGpus()) || + (this->memory != r.get_memory_per_task()) || + (this->gpus != r.get_gpus()) || (this->mics != r.getMics()) || ((r.getPlaceCores() > 0) && (this->place_cpus != r.getPlaceCores())) || ((r.getPlaceThreads() > 0) && (this->place_cpus != r.getPlaceThreads()))); } + + + +/* + * adjust_for_spread() + * + * Reduces the amount of resources to place for this allocation according to how + * many things it is being spread across. + * + * @param quantity - the number of things it is being spread across + * @param finding_remainder - tells us whether or not we're dividing or finding a remainder + */ + +void allocation::adjust_for_spread( + + unsigned int quantity, + bool finding_remainder) + + { + if (quantity != 0) + { + if (finding_remainder == false) + { + this->cpus /= quantity; + this->gpus /= quantity; + this->mics /= quantity; + } + else + { + this->cpus %= quantity; + this->gpus %= quantity; + this->mics %= quantity; + } + } + } // END adjust_for_spread() + + + +/* + * adjust_for_remainder() + * + * Optionally increases the amount of resources to place for this allocation according to the + * remainder parameter. Also, reduces remainder to account for the adjustment. + * + * @param remainder - the remainder that needs to be accounted for. + */ + +void allocation::adjust_for_remainder( + + allocation &remainder) + + { + if (remainder.cpus > 0) + { + this->cpus++; + remainder.cpus--; + } + + if (remainder.mics > 0) + { + this->mics++; + remainder.mics--; + } + + if (remainder.gpus > 0) + { + this->gpus++; + remainder.gpus--; + } + } // END adjust_for_remainder() + diff --git a/src/lib/Libutils/lib_utils.h b/src/lib/Libutils/lib_utils.h index 22688f14d3..a20d77fd52 100644 --- a/src/lib/Libutils/lib_utils.h +++ b/src/lib/Libutils/lib_utils.h @@ -41,6 +41,7 @@ int MXMLFromString(mxml_t **EP, char *XMLString, char **Tail, char *EMsg, /* u_groups.c */ void free_grname(struct group *, char *); +bool is_group_member(char *user_name, char *group_name); struct group * getgrnam_ext(char **user_buf, char * grp_name ); struct group * getgrgid_ext(char **user_buf, gid_t grp_id ); diff --git a/src/lib/Libutils/machine.cpp b/src/lib/Libutils/machine.cpp index 1baf0a636d..0e191f66a6 100644 --- a/src/lib/Libutils/machine.cpp +++ b/src/lib/Libutils/machine.cpp @@ -2,7 +2,9 @@ #include #include #include + #include "pbs_config.h" +#include "log.h" #ifdef PENABLE_LINUX_CGROUPS #include @@ -30,7 +32,6 @@ using namespace std; const int ALL_TASKS = -1; - /* 26 August 2014 * Currently Intel and AMD NUMA hardware * put their NUMA node and Socket in a different order. @@ -155,6 +156,75 @@ void Machine::update_internal_counts() +/* + * legacy_initialize_from_json() + * + * Exists to help people transition from 6.1.0. Deprecated. + */ + +int Machine::legacy_initialize_from_json( + + const std::string &json_str, + std::vector &valid_ids) + + { + int rc = PBSE_NONE; + + const char *socket_str = "\"socket\":{"; + std::size_t socket_begin = json_str.find(socket_str); + + while (socket_begin != std::string::npos) + { + std::size_t next = json_str.find(socket_str, socket_begin + 1); + std::string one_socket = json_str.substr(socket_begin, next - socket_begin); + + Socket s(one_socket, valid_ids); + + this->sockets.push_back(s); + this->totalSockets++; + + socket_begin = next; + } + + if (this->totalSockets == 0) + rc = -1; + else + { + update_internal_counts(); + this->initialized = true; + } + + return(rc); + } // END legacy_initialize_from_json() + + + +/* + * Initialize everything in machine to 0 + */ + +void Machine::clear() + + { + memset(allowed_cpuset_string, 0, MAX_CPUSET_SIZE); + memset(allowed_nodeset_string, 0, MAX_NODESET_SIZE); + this->hardwareStyle = 0; + this->totalMemory = 0; + this->totalSockets = 0; + this->totalChips = 0; + this->totalCores = 0; + this->totalThreads = 0; + this->availableSockets = 0; + this->availableChips = 0; + this->availableCores = 0; + this->availableThreads = 0; + this->sockets.clear(); + this->NVIDIA_device.clear(); + this->allocations.clear(); + } + + + void Machine::initialize_from_json( const std::string &json_str, @@ -176,16 +246,37 @@ void Machine::initialize_from_json( this->sockets.push_back(s); this->totalSockets++; } - + update_internal_counts(); - this->initialized = true; + + if (this->totalCores == 0) + { + this->clear(); + + if (this->legacy_initialize_from_json(json_str, valid_ids) != PBSE_NONE) + { + char log_buf[LOCAL_LOG_BUF_SIZE]; + + snprintf(log_buf, sizeof(log_buf), "Couldn't initialize from json: '%s'", json_str.c_str()); + log_err(-1, __func__, log_buf); + } + } + else + { + this->initialized = true; + } } catch (...) { - char log_buf[LOCAL_LOG_BUF_SIZE]; + this->clear(); - snprintf(log_buf, sizeof(log_buf), "Couldn't initialize from json: '%s'", json_str.c_str()); - log_err(-1, __func__, log_buf); + if (this->legacy_initialize_from_json(json_str, valid_ids) != PBSE_NONE) + { + char log_buf[LOCAL_LOG_BUF_SIZE]; + + snprintf(log_buf, sizeof(log_buf), "Couldn't initialize from json: '%s'", json_str.c_str()); + log_err(-1, __func__, log_buf); + } } } // END initialize_from_json() @@ -197,21 +288,7 @@ void Machine::reinitialize_from_json( std::vector &valid_ids) { - memset(allowed_cpuset_string, 0, MAX_CPUSET_SIZE); - memset(allowed_nodeset_string, 0, MAX_NODESET_SIZE); - this->hardwareStyle = 0; - this->totalMemory = 0; - this->totalSockets = 0; - this->totalChips = 0; - this->totalCores = 0; - this->totalThreads = 0; - this->availableSockets = 0; - this->availableChips = 0; - this->availableCores = 0; - this->availableThreads = 0; - this->sockets.clear(); - this->NVIDIA_device.clear(); - this->allocations.clear(); + this->clear(); this->initialize_from_json(json_layout, valid_ids); } // END reinitialize_from_json() @@ -426,11 +503,31 @@ hwloc_uint64_t Machine::getTotalMemory() const return(this->totalMemory); } +hwloc_uint64_t Machine::getAvailableMemory() const + { + hwloc_uint64_t available = 0; + + for (size_t i = 0; i < this->sockets.size(); i++) + available += this->sockets[i].getAvailableMemory(); + + return(available); + } + int Machine::getTotalSockets() const { return(this->totalSockets); } +int Machine::get_total_gpus() const + { + int total_gpus = 0; + + for (size_t i = 0; i < this->sockets.size(); i++) + total_gpus += this->sockets[i].get_total_gpus(); + + return(total_gpus); + } + int Machine::getTotalChips() const { return(this->totalChips); @@ -610,6 +707,61 @@ void Machine::place_all_execution_slots( +/* + * compare_remaining_values() + * + * Looks at what couldn't be placed in the allocation remaining and logs an appropriate message. + * @param remaining - the allocation that couldn't be completely placed. + * @param caller - the name of the calling function. + */ + +void Machine::compare_remaining_values( + + allocation &remaining, + const char *caller) const + + { + char log_buf[LOCAL_LOG_BUF_SIZE]; + std::string error; + + if (remaining.cpus != 0) + { + sprintf(log_buf, "Couldn't place %d cpus, have %d/%d cores and threads available. ", + remaining.cpus, this->getAvailableCores(), this->getAvailableThreads()); + error += log_buf; + } + + if (remaining.memory != 0) + { + sprintf(log_buf, "Couldn't place %lukb memory, have %lukb available. ", + remaining.memory, (unsigned long)this->getAvailableMemory()); + error += log_buf; + } + + if (remaining.mics != 0) + { + sprintf(log_buf, "Couldn't place %d mics. ", remaining.mics); + error += log_buf; + } + + if (remaining.gpus != 0) + { + sprintf(log_buf, "Couldn't place %d gpus. ", remaining.gpus); + error += log_buf; + } + + if (remaining.place_cpus != 0) + { + sprintf(log_buf, "Couldn't place %d place cpus, have %d/%d cores and threads available. ", + remaining.place_cpus, this->getAvailableCores(), this->getAvailableThreads()); + error += log_buf; + } + + log_err(-1, caller, error.c_str()); + } // END compare_remaining_values() + + + /* * place_remaining() * @@ -726,9 +878,12 @@ void Machine::place_remaining( master.set_place_type(place_legacy); } + // If it didn't get placed, log what prevented it from being placed if (fit_somewhere == false) + { + compare_remaining_values(remaining, __func__); break; - + } remaining_tasks--; } @@ -770,7 +925,7 @@ int Machine::spread_place_pu( pu_per_task_remaining = pu_per_task; lprocs_per_task_remaining = lprocs_per_task; - gpus_remaining = r.getGpus(); + gpus_remaining = r.get_gpus(); mics_remaining = r.getMics(); for (unsigned int j = 0; j < this->totalSockets; j++) @@ -853,6 +1008,7 @@ int Machine::spread_place( bool chips = false; int quantity = r.get_sockets(); int tasks_placed = 0; + char log_buf[LOCAL_LOG_BUF_SIZE]; if (quantity == 0) { @@ -869,7 +1025,7 @@ int Machine::spread_place( else { // Make sure we are grabbing enough memory - unsigned long mem_needed = r.getMemory(); + unsigned long mem_needed = r.get_memory_per_task(); unsigned long mem_count = 0; int mem_quantity = 0; @@ -896,27 +1052,33 @@ int Machine::spread_place( // If we don't have enough memory, reject the job if (mem_needed > mem_count) { + snprintf(log_buf, sizeof(log_buf), + "Job %s requires at least %lukb memory from host %s, but only %lukb is available", + master.jobid.c_str(), mem_needed, hostname, mem_count); + log_err(-1, __func__, log_buf); + return(PBSE_RESCUNAV); } } // Spread the placement evenly across the number of sockets or numa nodes - int execution_slots_per = r.getExecutionSlots() / quantity; - int execution_slots_remainder = r.getExecutionSlots() % quantity; - for (int i = 0; i < tasks_for_node; i++) { bool partial_place = false; allocation task_alloc(master.jobid.c_str()); task_alloc.set_place_type(r.getPlacementType()); + allocation remaining(r); + allocation remainder(r); + + remaining.adjust_for_spread(quantity, false); + remainder.adjust_for_spread(quantity, true); for (int j = 0; j < quantity; j++) { for (unsigned int s = 0; s < this->sockets.size(); s++) { - if (this->sockets[s].spread_place(r, task_alloc, execution_slots_per, - execution_slots_remainder, chips)) + if (this->sockets[s].spread_place(r, task_alloc, remaining, remainder, chips)) { partial_place = true; @@ -943,7 +1105,14 @@ int Machine::spread_place( } if (tasks_placed != tasks_for_node) + { + snprintf(log_buf, sizeof(log_buf), + "Job %s needed to place %d tasks on node %s, but could only place %d", + master.jobid.c_str(), tasks_for_node, hostname, tasks_placed); + log_err(-1, __func__, log_buf); + return(PBSE_IVALREQ); + } return(PBSE_NONE); } // END spread_place() @@ -987,11 +1156,10 @@ int Machine::fit_tasks_within_sockets( int Machine::place_job( - job *pjob, - string &cpu_string, - string &mem_string, - const char *hostname, - bool legacy_vmem) + job *pjob, + cgroup_info &cgi, + const char *hostname, + bool legacy_vmem) { int rc = PBSE_NONE; @@ -1001,6 +1169,7 @@ int Machine::place_job( vector partially_place; allocation job_alloc(pjob->ji_qs.ji_jobid); vector to_split; + char log_buf[LOCAL_LOG_BUF_SIZE]; // See if the tasks fit completely on a socket, and if they do then place them there for (int i = 0; i < num_reqs; i++) @@ -1039,7 +1208,14 @@ int Machine::place_job( // Placing 0 tasks is an error if (this->sockets[j].place_task(r, job_alloc, tasks_for_node, hostname) == 0) + { + snprintf(log_buf, sizeof(log_buf), + "Socket %u for node %s was thought to fit at least %d tasks, but couldn't fit any", + j, hostname, tasks_for_node); + log_err(-1, __func__, log_buf); + return(-1); + } break; } @@ -1079,12 +1255,16 @@ int Machine::place_job( if (remaining_tasks > 0) { // this allocation so that the cleanup happens correctly + snprintf(log_buf, sizeof(log_buf), + "Job %s could not place %d tasks from req %d on host %s", + pjob->ji_qs.ji_jobid, remaining_tasks, partially_place[i], hostname); + log_err(-1, __func__, log_buf); + rc = -1; } } - job_alloc.place_indices_in_string(mem_string, MEM_INDICES); - job_alloc.place_indices_in_string(cpu_string, CPU_INDICES); + job_alloc.place_indices_in_string(cgi); this->allocations.push_back(job_alloc); @@ -1093,40 +1273,6 @@ int Machine::place_job( -/* - * get_jobs_cpusets() - * - * Gets the cpusets corresponding to this job_id so that the child can place them - * @param job_id (I) - the id of the job - * @param cpus (O) - put the string representing the cpus here - * @param mems (O) - put the string representing the memory nodes here - * @return PBSE_NONE if we found the job, -1 otherwise - */ - -int Machine::get_jobs_cpusets( - - const char *job_id, - string &cpus, - string &mems) - - { - int rc = -1; - - for (unsigned int i = 0; i < this->allocations.size(); i++) - { - if (this->allocations[i].jobid == job_id) - { - this->allocations[i].place_indices_in_string(mems, MEM_INDICES); - this->allocations[i].place_indices_in_string(cpus, CPU_INDICES); - rc = PBSE_NONE; - } - } - - return(rc); - } // END get_jobs_cpusets() - - - void Machine::free_job_allocation( const char *job_id) @@ -1158,10 +1304,12 @@ void Machine::free_job_allocation( void Machine::store_device_on_appropriate_chip( - PCI_Device &device) + PCI_Device &device, + bool no_info) { - if (this->isNUMA == false) + if ((this->isNUMA == false) || + (no_info == true)) { this->sockets[0].store_pci_device_appropriately(device, true); } @@ -1259,4 +1407,19 @@ bool Machine::is_initialized() const +void Machine::save_allocations( + + const Machine &other) + + { + this->allocations = other.allocations; + + for (size_t s = 0; s < other.sockets.size() && s < this->sockets.size(); s++) + { + this->sockets[s].save_allocations(other.sockets[s]); + } + } // END save_allocations() + + + #endif /* PENABLE_LINUX_CGROUPS */ diff --git a/src/lib/Libutils/numa_chip.cpp b/src/lib/Libutils/numa_chip.cpp index 32c8dc199f..2e9447493f 100644 --- a/src/lib/Libutils/numa_chip.cpp +++ b/src/lib/Libutils/numa_chip.cpp @@ -122,6 +122,70 @@ Chip::Chip( +/* + * legacy_parse_values_from_json_string() + * + * Deprecated. This exists to help people upgrade. + */ + +void Chip::legacy_parse_values_from_json_string( + + const std::string &json_layout, + std::string &cores, + std::string &threads, + std::string &gpus, + std::string &mics, + std::vector &valid_ids) + + { + char *work_str = strdup(json_layout.c_str()); + char *ptr = strstr(work_str, "os_index\":"); + char *val = work_str; + + if (ptr != NULL) + { + val = ptr + strlen("os_index\":"); + this->id = strtol(val, &val, 10); + } + + if ((ptr = strstr(val, "cores\":")) != NULL) + { + val = ptr + strlen("cores\":") + 1; // add 1 for the open quote + capture_until_close_character(&val, cores, '"'); + } + + if ((ptr = strstr(val, "threads\":")) != NULL) + { + val = ptr + strlen("threads\":") + 1; // add 1 for the open quote + capture_until_close_character(&val, threads, '"'); + } + + if ((ptr = strstr(val, "mem\":")) != NULL) + { + val = ptr + strlen("mem\":"); + this->memory = strtol(val, &val, 10); + this->available_memory = this->memory; + } + + if ((ptr = strstr(val, "gpus\":")) != NULL) + { + val = ptr + strlen("gpus\":") + 1; + capture_until_close_character(&val, gpus, '"'); + } + + if ((ptr = strstr(val, "mics\":")) != NULL) + { + val = ptr + strlen("mics\":") + 1; + capture_until_close_character(&val, mics, '"'); + } + + initialize_allocations(val, valid_ids); + + free(work_str); + } // END legacy_parse_values_from_json_string() + + + /* * parse_values_from_json_string() */ @@ -165,12 +229,15 @@ void Chip::initialize_cores_from_strings( { std::vector core_indices; std::vector thread_indices; - int ratio; + int ratio = 0; translate_range_string_to_vector(cores_str.c_str(), core_indices); translate_range_string_to_vector(threads_str.c_str(), thread_indices); - ratio = thread_indices.size() / core_indices.size(); + // Check if this is a memory-only node + if (core_indices.size() > 0) + ratio = thread_indices.size() / core_indices.size(); + unsigned int j = 0; for (unsigned int i = 0; i < core_indices.size(); i++) @@ -196,6 +263,139 @@ void Chip::initialize_cores_from_strings( +/* + * legacy_initialize_allocation() + * + * This function is only to support people upgrading from 6.1.0. Deprecated. We should + * delete it as soon as it's reasonable to do so. + */ + +void Chip::legacy_initialize_allocation( + + char *allocation_str, + std::vector &valid_ids) + + { + allocation a; + char *ptr = strstr(allocation_str, "jobid\":"); + char *val = allocation_str; + std::string tmp_val; + + if (ptr != NULL) + { + val = ptr + 8; // move past "jobid\":\"" + capture_until_close_character(&val, tmp_val, '"'); + a.jobid = tmp_val; + } + + // check if the id is valid + bool id_valid = false; + for (size_t i = 0; i < valid_ids.size(); i++) + { + if (valid_ids[i] == a.jobid) + { + id_valid = true; + break; + } + } + + if (id_valid == true) + { + ptr = strstr(val, "cpus\":"); + if (ptr != NULL) + { + val = ptr + 7; // move past "cpus\":\"" + capture_until_close_character(&val, tmp_val, '"'); + translate_range_string_to_vector(tmp_val.c_str(), a.cpu_indices); + } + + ptr = strstr(val, "mem\":"); + if (ptr != NULL) + { + val = ptr + 5; // move past "mem\":" + a.memory = strtol(val, &val, 10); + } + + ptr = strstr(val, "exclusive\":"); + if (ptr != NULL) + { + val = ptr + 11; // move past "exclusive\":" + a.place_type = strtol(val, &val, 10); + } + + ptr = strstr(val, "cores_only\":"); + if (ptr != NULL) + { + val = ptr + 12; // move past "cores_only\":" + a.cores_only = (bool)strtol(val, &val, 10); + } + + ptr = strstr(val, "gpus\":"); + if (ptr != NULL) + { + val = ptr + 7; // move past "gpus\":\" + capture_until_close_character(&val, tmp_val, '"'); + translate_range_string_to_vector(tmp_val.c_str(), a.gpu_indices); + } + + ptr = strstr(val, "mics\":"); + if (ptr != NULL) + { + val = ptr + 7; // move past "mics\":\" + capture_until_close_character(&val, tmp_val, '"'); + translate_range_string_to_vector(tmp_val.c_str(), a.mic_indices); + } + + a.mem_indices.push_back(this->id); + this->allocations.push_back(a); + } + + } // END legacy_initialize_allocation() + + + +/* + * legacy_initialize_allocations() + * + * This function is only to support people upgrading from 6.1.0. Deprecated. We should + * delete it as soon as it's reasonable to do so. + */ + +void Chip::legacy_initialize_allocations( + + char *allocations, + std::vector &valid_ids) + + { + static const char *allocation_start = "allocation\":{"; + static const int allocation_start_len = strlen(allocation_start); + + if ((allocations == NULL) || + (*allocations == '\0')) + return; + + char *current = strstr(allocations, allocation_start); + char *next; + + while (current != NULL) + { + current += allocation_start_len; + next = strstr(current, allocation_start); + if (next != NULL) + { + // Make sure there's a termination to the current string + *next = '\0'; + } + + initialize_allocation(current, valid_ids); + + current = next; + } + + } // END legacy_initialize_allocations() + + + /* * initialize_allocation() * @@ -288,7 +488,7 @@ void Chip::initialize_allocations( /* - * reserce_allocation_resources() + * reserve_allocation_resources() * * @param a - the allocation that needs to be */ @@ -419,6 +619,42 @@ void Chip::initialize_accelerators_from_strings( +/* + * This is a legacy constructor for those transitioning from 6.1.0. Delete ASAP + */ + +Chip::Chip( + + const std::string &json_layout, + std::vector &valid_ids) : id(0), totalCores(0), totalThreads(0), availableCores(0), + availableThreads(0), total_gpus(0), available_gpus(0), + total_mics(0), available_mics(0), chip_exclusive(false), + memory(0), available_memory(0), cores(), devices(), + allocations() + + { + memset(chip_cpuset_string, 0, MAX_CPUSET_SIZE); + memset(chip_nodeset_string, 0, MAX_NODESET_SIZE); + + if (json_layout.size() == 0) + return; + + std::string cores; + std::string threads; + std::string gpus; + std::string mics; + + legacy_parse_values_from_json_string(json_layout, cores, threads, gpus, mics, valid_ids); + + initialize_cores_from_strings(cores, threads); + + initialize_accelerators_from_strings(gpus, mics); + + adjust_open_resources(); + } + + + /* * Creates a numa chip from this json * @@ -462,7 +698,6 @@ Chip::Chip( std::string gpus; std::string mics; - parse_values_from_json_string(layout, cores, threads, gpus, mics, valid_ids); initialize_cores_from_strings(cores, threads); @@ -574,6 +809,11 @@ int Chip::getTotalThreads() const return(this->totalThreads); } +int Chip::get_total_gpus() const + { + return(this->total_gpus); + } + int Chip::getAvailableCores() const { if (this->chip_exclusive == true) @@ -812,6 +1052,21 @@ void Chip::setCores( this->availableCores = cores; } +void Chip::set_gpus( + + int gpus) + + { + for (int i = 0; i < gpus; i++) + { + PCI_Device p; + p.set_type(GPU); + p.setId(i); + this->total_gpus++; + this->devices.push_back(p); + } + } + void Chip::setThreads( int threads) @@ -925,16 +1180,16 @@ int Chip::free_core_count() const * @return the number of tasks that fit. This can be 0 */ -float Chip::how_many_tasks_fit( +double Chip::how_many_tasks_fit( const req &r, int place_type) const { - float cpu_tasks; - float gpu_tasks; - float mic_tasks; - float mem_tasks = 0; + double cpu_tasks; + double gpu_tasks; + double mic_tasks; + double mem_tasks = 0; // Consider exclusive socket and node the same as exclusive chip for our purposes if ((place_type == exclusive_socket) || @@ -946,7 +1201,7 @@ float Chip::how_many_tasks_fit( (this->chipIsAvailable()) == true)) { // Need to handle place={core|thread}[=x] - float max_cpus = r.getExecutionSlots(); + double max_cpus = r.getExecutionSlots(); if (r.getPlaceCores() > 0) max_cpus = r.getPlaceCores(); else if (r.getPlaceThreads() > 0) @@ -959,7 +1214,7 @@ float Chip::how_many_tasks_fit( else cpu_tasks = this->availableThreads / max_cpus; - long long memory = r.getMemory(); + unsigned long memory = r.get_memory_per_task(); // Memory isn't required for submission if (memory != 0) @@ -973,7 +1228,7 @@ float Chip::how_many_tasks_fit( else mem_tasks = cpu_tasks; - float gpus = r.getGpus(); + double gpus = r.get_gpus(); if (gpus > 0) { gpu_tasks = this->available_gpus / gpus; @@ -981,7 +1236,7 @@ float Chip::how_many_tasks_fit( mem_tasks = gpu_tasks; } - float mics = r.getMics(); + double mics = r.getMics(); if (mics > 0) { mic_tasks = this->available_mics / mics; @@ -1016,7 +1271,6 @@ bool Chip::getOpenThreadVector( int execution_slots_per_task) { - unsigned int j = 0; int i = execution_slots_per_task; bool fits = false; @@ -1025,24 +1279,21 @@ bool Chip::getOpenThreadVector( are allocated */ if (execution_slots_per_task == 0) return(true); - slots.clear(); - i = execution_slots_per_task; - j = 0; + /* Can't get contiguous threads. Just get them where you can find them */ // Get the thread indices we will use - do + for (size_t core_index = 0; core_index < this->cores.size() && i != 0; core_index++) { - for (unsigned int x = 0; x < this->cores[j].indices.size(); x++) + for (size_t thread_index = 0; + thread_index < this->cores[core_index].indices.size(); + thread_index++) { - int thread_index; - if (this->cores[j].is_index_busy[x] == true) + if (this->cores[core_index].is_index_busy[thread_index] == true) continue; - thread_index = this->cores[j].indices[x]; - - slots.push_back(thread_index); + slots.push_back(this->cores[core_index].indices[thread_index]); i--; - if ((i == 0) || ((x + 1) == this->cores[j].indices.size())) + if ((i == 0) || ((thread_index + 1) == this->cores[core_index].indices.size())) { /* We fit if all of the execution slots have been filled or it we have used all the chip */ @@ -1050,12 +1301,14 @@ bool Chip::getOpenThreadVector( break; } } - j++; - }while((i != 0) && (j < this->cores.size())); - + } + return(fits); - } + } // END getOpenThreadVector() + + + /* * getContiguousThreadVector * @@ -1073,8 +1326,7 @@ bool Chip::getContiguousThreadVector( int execution_slots_per_task) { - unsigned int j = 0; - int i = execution_slots_per_task; + int i = execution_slots_per_task; bool fits = false; /* this makes it so users can request gpus and mics @@ -1084,15 +1336,15 @@ bool Chip::getContiguousThreadVector( return(true); /* First try to get contiguous threads */ - do + for (size_t core_index = 0; core_index < this->cores.size() && i != 0; core_index++) { - for (unsigned int x = 0; x < this->cores[j].indices.size(); x++) + for (size_t thread_index = 0; + thread_index < this->cores[core_index].indices.size(); + thread_index++) { - int thread_index; - /* if this thread is busy and we have already started creating a list, clear the list and start over; otherwise, continue to the next thread */ - if (this->cores[j].is_index_busy[x] == true) + if (this->cores[core_index].is_index_busy[thread_index] == true) { if (slots.size() > 0) { @@ -1103,9 +1355,7 @@ bool Chip::getContiguousThreadVector( continue; } - thread_index = this->cores[j].indices[x]; - - slots.push_back(thread_index); + slots.push_back(this->cores[core_index].indices[thread_index]); i--; if (i == 0) { @@ -1114,45 +1364,18 @@ bool Chip::getContiguousThreadVector( break; } } - j++; - - }while((i != 0) && (j < this->cores.size())); + } if (fits == false) { slots.clear(); - i = execution_slots_per_task; - j = 0; - /* Can't get contiguous threads. Just get them where you can find them */ - // Get the thread indices we will use - do - { - for (unsigned int x = 0; x < this->cores[j].indices.size(); x++) - { - int thread_index; - if (this->cores[j].is_index_busy[x] == true) - continue; - - thread_index = this->cores[j].indices[x]; - - slots.push_back(thread_index); - i--; - if ((i == 0) || ((x + 1) == this->cores[j].indices.size())) - { - /* We fit if all of the execution slots have been filled - or it we have used all the chip */ - fits = true; - break; - } - } - j++; - - }while((i != 0) && (j < this->cores.size())); + fits = getOpenThreadVector(slots, execution_slots_per_task); } return(fits); } + /* * getContiguousCoreVector * @@ -1170,9 +1393,8 @@ bool Chip::getContiguousCoreVector( int execution_slots_per_task) { - unsigned int j = 0; - int i = execution_slots_per_task; - bool fits = false; + int i = execution_slots_per_task; + bool fits = true; /* this makes it so users can request gpus and mics from numanodes which are not where the cores or threads @@ -1181,50 +1403,51 @@ bool Chip::getContiguousCoreVector( return(true); /* First try to get contiguous cores */ - do + for (size_t core_index = 0; core_index < this->cores.size() && i != 0; core_index++) { - if (this->cores[j].is_free() == true) + if (this->cores[core_index].is_free() == true) { - slots.push_back(j); + slots.push_back(core_index); i--; - j++; - if ((i ==0) || (j == this->cores.size())) - { - /* We fit if all of the execution slots have been filled - or it we have used all the chip */ - fits = true; - } } else { - i = execution_slots_per_task; - j++; + if (i != execution_slots_per_task) + { + fits = false; + i = execution_slots_per_task; + } + slots.clear(); } - }while((i != 0) && (j < this->cores.size())); + } + + if (i == execution_slots_per_task) + fits = false; if (fits == false) { /* Can't get contiguous cores. Just get them where you can find them */ // Get the core indices we will use - j = 0; + size_t core_index = 0; for (int i = 0; i < execution_slots_per_task; i++) { - while (j < this->cores.size()) + while (core_index < this->cores.size()) { - if (this->cores[j].is_free() == true) + if (this->cores[core_index].is_free() == true) { - slots.push_back(j); - j++; + slots.push_back(core_index); + core_index++; break; } else - j++; + core_index++; } } } + return(fits); - } + } // END getContiguousCoreVector() /* @@ -1364,8 +1587,8 @@ bool Chip::task_will_fit( { bool fits = false; int max_cpus = r.getExecutionSlots(); - hwloc_uint64_t mem_per_task = r.getMemory(); - int gpus_per_task = r.getGpus(); + hwloc_uint64_t mem_per_task = r.get_memory_per_task(); + int gpus_per_task = r.get_gpus(); int mics_per_task = r.getMics(); bool cores_only = (r.getThreadUsageString() == use_cores); @@ -1735,7 +1958,8 @@ bool Chip::spread_place_threads( return(placed); - } + } // END spread_place_threads() + /* spread_place_cores @@ -1858,7 +2082,7 @@ bool Chip::spread_place_cores( return(placed); - } + } // END spread_place_cores() @@ -1879,40 +2103,37 @@ bool Chip::spread_place( req &r, allocation &task_alloc, - int execution_slots_per, - int &execution_slots_remainder) + allocation &remaining, + allocation &remainder) { bool task_placed = false; if ((this->is_completely_free() == true) && - ((execution_slots_per + execution_slots_remainder) <= this->totalThreads)) + ((remaining.cpus + remainder.cpus) <= this->totalThreads)) { allocation from_this_chip(task_alloc.jobid.c_str()); int placed = 0; - int to_add = 0; + int execution_slots = remaining.cpus; - if (execution_slots_remainder > 0) - to_add = 1; + if (remainder.cpus > 0) + { + execution_slots++; + remainder.cpus--; + } from_this_chip.place_type = task_alloc.place_type; - if (this->totalCores >= execution_slots_per + to_add) + if (this->totalCores >= execution_slots) { - if (to_add == 1) - { - execution_slots_per++; - execution_slots_remainder--; - } - // Spread over just the cores float step = 1.0; - if (execution_slots_per > 0) - step = this->totalCores / (float)execution_slots_per; + if (execution_slots > 0) + step = this->totalCores / (float)execution_slots; from_this_chip.cores_only = true; - for (float i = 0.0; placed < execution_slots_per; i+= step) + for (float i = 0.0; placed < execution_slots; i+= step) { this->reserve_core(std::floor(i + 0.5), from_this_chip); placed++; @@ -1922,17 +2143,10 @@ bool Chip::spread_place( { // Spread over the cores and the threads - this option doesn't really make any // sense but I suppose we have to code for it - int per_core = execution_slots_per / this->totalCores; - int remainder = execution_slots_per % this->totalCores; - - // Place one extra if we still have a remainder - if (execution_slots_remainder > 0) - { - placed--; - execution_slots_remainder--; - } + int per_core = execution_slots / this->totalCores; + int leftover = execution_slots % this->totalCores; - for (unsigned int i = 0; i < this->cores.size() && placed < execution_slots_per; i++) + for (unsigned int i = 0; i < this->cores.size() && placed < execution_slots; i++) { for (int j = 0; j < per_core; j++) { @@ -1941,14 +2155,14 @@ bool Chip::spread_place( } } - while (remainder > 0) + while (leftover > 0) { - for (unsigned int i = 0; i < this->cores.size() && placed < execution_slots_per; i++) + for (unsigned int i = 0; i < this->cores.size() && placed < execution_slots; i++) { if (this->reserve_thread(i, from_this_chip) == true) { placed++; - remainder--; + leftover--; } } } @@ -1956,6 +2170,8 @@ bool Chip::spread_place( from_this_chip.mem_indices.push_back(this->id); + place_accelerators(remaining, from_this_chip); + place_accelerators(remainder, from_this_chip); task_placed = true; this->chip_exclusive = true; this->aggregate_allocation(from_this_chip); @@ -1989,7 +2205,7 @@ int Chip::place_task( allocation chip_alloc(master.jobid.c_str()); int tasks_placed = 0; int execution_slots_per_task = r.getExecutionSlots(); - hwloc_uint64_t mem_per_task = r.getMemory(); + hwloc_uint64_t mem_per_task = r.get_memory_per_task(); int practical_place = master.place_type; chip_alloc.place_type = master.place_type; @@ -2043,7 +2259,7 @@ int Chip::place_task( else { int threads_to_rsv = execution_slots_per_task; - if(r.getPlaceThreads() > 0) + if (r.getPlaceThreads() > 0) threads_to_rsv = r.getPlaceThreads(); place_tasks_execution_slots(execution_slots_per_task, threads_to_rsv, task_alloc, THREAD_INT); @@ -2587,4 +2803,21 @@ bool Chip::has_socket_exclusive_allocation() const } // END has_socket_exclusive_allocation() + +/* + * Preserves all relevant information from the allocations for Chip other on this Chip + * This is called when replacing one Machine object with another, but we are trying to + * keep the information about running jobs. + */ + +void Chip::save_allocations( + + const Chip &other) + + { + this->allocations = other.allocations; + this->adjust_open_resources(); + } // END save_allocations() + + #endif /* PENABLE_LINUX_CGROUPS */ diff --git a/src/lib/Libutils/numa_pci_device.cpp b/src/lib/Libutils/numa_pci_device.cpp index b6a8f9dc30..45d8d4feda 100644 --- a/src/lib/Libutils/numa_pci_device.cpp +++ b/src/lib/Libutils/numa_pci_device.cpp @@ -68,22 +68,50 @@ PCI_Device::~PCI_Device() int PCI_Device::initializePCIDevice(hwloc_obj_t device_obj, int idx, hwloc_topology_t topology) { + // see if we are initializing an NVIDIA PCI device only + if (device_obj == NULL) + { +#if defined(NVIDIA_GPUS) && defined(NVML_API) + // use NVML library to look up info about the device + + nvmlDevice_t gpu; + char buf[NVML_DEVICE_NAME_BUFFER_SIZE]; + + // get the device handle so the name can be looked up + if (nvmlDeviceGetHandleByIndex(idx, &gpu) != NVML_SUCCESS) + return(-1); + + // look up the name + if (nvmlDeviceGetName(gpu, buf, sizeof(buf)) != NVML_SUCCESS) + return(-1); - id = device_obj->logical_index; - name = device_obj->name; - if (device_obj->infos != NULL) + // copy the name + name = buf; + + // copy the id + id = idx; +#else + // NULL hwloc object passed in + return(-1); +#endif + } + else { - info_name = device_obj->infos->name; - info_value = device_obj->infos->value; + id = device_obj->logical_index; + name = device_obj->name; + if (device_obj->infos != NULL) + { + info_name = device_obj->infos->name; + info_value = device_obj->infos->value; + } } - #ifdef MIC this->initializeMic(idx, topology); #endif #ifdef NVIDIA_GPUS - this->initializeGpu(idx, topology); + this->initializeGpu(idx); #endif return(PBSE_NONE); } @@ -93,7 +121,10 @@ void PCI_Device::displayAsString( stringstream &out) const { - out << " pci " << this->id << " " << this->name << "\n"; + if (this->type == GPU) + out << " GPU " << this->id << " " << this->name << "\n"; + else + out << " MIC " << this->id << " " << this->name << "\n"; } // end displayasstring() void PCI_Device::setName( diff --git a/src/lib/Libutils/numa_socket.cpp b/src/lib/Libutils/numa_socket.cpp index 6162fa12aa..cd34515c56 100644 --- a/src/lib/Libutils/numa_socket.cpp +++ b/src/lib/Libutils/numa_socket.cpp @@ -46,6 +46,49 @@ Socket::Socket( +/* + * Legacy constructor for those upgrading from 6.1.0 + */ + +Socket::Socket( + + const std::string &json_layout, + std::vector &valid_ids) : id(0), memory(0), totalCores(0), totalThreads(0), + availableCores(0), availableThreads(0), chips(), + socket_exclusive(false) + + { + const char *chip_str = "\"numanode\":{"; + const char *os_str = "\"os_index\":"; + std::size_t chip_begin = json_layout.find(chip_str); + std::size_t os_begin = json_layout.find(os_str); + + memset(socket_cpuset_string, 0, MAX_CPUSET_SIZE); + memset(socket_nodeset_string, 0, MAX_NODESET_SIZE); + + if ((os_begin == std::string::npos) || + (os_begin > chip_begin)) + return; + else + { + std::string os = json_layout.substr(os_begin + strlen(os_str)); + this->id = strtol(os.c_str(), NULL, 10); + } + + while (chip_begin != std::string::npos) + { + std::size_t next = json_layout.find(chip_str, chip_begin + 1); + std::string one_chip = json_layout.substr(chip_begin, next - chip_begin); + + Chip c(one_chip, valid_ids); + this->chips.push_back(c); + + chip_begin = next; + } + } + + + /* * Builds a copy of the machine's layout in from json which has no whitespace but * if it did it'd look like: @@ -311,6 +354,26 @@ int Socket::getTotalChips() const return(this->chips.size()); } +int Socket::get_total_gpus() const + { + int total_gpus = 0; + + for (size_t i = 0; i < this->chips.size(); i++) + total_gpus += this->chips[i].get_total_gpus(); + + return(total_gpus); + } + +int Socket::get_available_gpus() const + { + int available_gpus = 0; + + for (size_t i = 0; i < this->chips.size(); i++) + available_gpus += this->chips[i].get_available_gpus(); + + return(available_gpus); + } + int Socket::getAvailableChips() const { int available_numa_nodes = 0; @@ -467,13 +530,13 @@ void Socket::update_internal_counts( * @return - the number of tasks from r that could be placed on this socket */ -float Socket::how_many_tasks_fit( +double Socket::how_many_tasks_fit( const req &r, int place_type) const { - float num_that_fit = 0; + double num_that_fit = 0; if ((this->socket_exclusive == false) && ((place_type != exclusive_socket) || @@ -573,15 +636,13 @@ bool Socket::spread_place( req &r, allocation &task_alloc, - int execution_slots_per, - int &execution_slots_remainder, + allocation &remaining, + allocation &remainder, bool chip) { bool placed = false; int numa_nodes_required = 1; - int per_numa = execution_slots_per; - int per_numa_remainder = 0; // We must either be completely free or be placing on just one chip if ((this->is_completely_free()) || @@ -591,33 +652,33 @@ bool Socket::spread_place( { // If we're placing at the socket level, divide execution_slots_per by the number // of chips and place multiple times - per_numa_remainder = per_numa % this->chips.size(); - per_numa /= this->chips.size(); numa_nodes_required = this->chips.size(); - this->socket_exclusive = true; } - + + // In order to evenly spread out resources across nodes, we divide number of nodes + // by resources required and are left with number of resources per node plus a + // remainder. As we iterate over the nodes and allocate resources, we take one + // resource from the remainder every step until there is nothing left in the remainder. + allocation numa_remainder(remaining); + numa_remainder.adjust_for_spread(numa_nodes_required, true); + for (int c = 0; c < numa_nodes_required; c++) { - for (unsigned int i = 0; i < this->chips.size(); i++) + allocation per_numa(remaining); + per_numa.adjust_for_spread(numa_nodes_required, false); + + per_numa.adjust_for_remainder(numa_remainder); + + // This inner loop is for the case where we are not reserving the whole + // socket, but a single chip. In the case of a single chip, we iterate + // over all chips to see if there is a chip which is completely free. + for (unsigned int i = c; i < this->chips.size(); i++) { - if (per_numa_remainder > 0) + if (this->chips[i].spread_place(r, task_alloc, per_numa, remainder)) { - if (this->chips[i].spread_place(r, task_alloc, per_numa + 1, execution_slots_remainder)) - { - placed = true; - per_numa_remainder--; - break; - } - } - else - { - if (this->chips[i].spread_place(r, task_alloc, per_numa, execution_slots_remainder)) - { - placed = true; - break; - } + placed = true; + break; } } } @@ -834,17 +895,20 @@ bool Socket::fits_on_socket( if (remaining.place_cpus > 0) max_cpus = remaining.place_cpus; - if ((remaining.cores_only == true) && - (this->get_free_cores() >= max_cpus)) - fits = true; - else if (remaining.place_type == exclusive_legacy) + if (this->get_available_gpus() >= remaining.gpus) { - if (this->get_free_cores() >= max_cpus) + if ((remaining.cores_only == true) && + (this->get_free_cores() >= max_cpus)) + fits = true; + else if (remaining.place_type == exclusive_legacy) + { + if (this->get_free_cores() >= max_cpus) + fits = true; + } + else if ((remaining.cores_only == false) && + (this->getAvailableThreads() >= max_cpus)) fits = true; } - else if ((remaining.cores_only == false) && - (this->getAvailableThreads() >= max_cpus)) - fits = true; } return(fits); @@ -928,4 +992,15 @@ int Socket::get_mics_remaining() } + +void Socket::save_allocations( + + const Socket &other) + + { + for (size_t c = 0; c < this->chips.size() && c < other.chips.size(); c++) + this->chips[c].save_allocations(other.chips[c]); + } + + #endif /* PENABLE_LINUX_CGROUPS */ diff --git a/src/lib/Libutils/u_groups.c b/src/lib/Libutils/u_groups.c index 5d64622ef0..0708479452 100644 --- a/src/lib/Libutils/u_groups.c +++ b/src/lib/Libutils/u_groups.c @@ -1,6 +1,9 @@ #include "license_pbs.h" /* See here for the software license */ #include +#include +#include #include "utils.h" +#include "u_wrapper.h" void free_grname( @@ -23,6 +26,96 @@ void free_grname( } + /** + * is_group_member - checks to see if user_name is a member of group_name + * + * @param user_name (I) - name of user being checked for group membership + * @param group_name (I) - group name being checked for user membership + * @return true if user_name is a member of group_name. Otherwise return false + * + **/ + bool is_group_member(char *user_name, char *group_name) + { + int j, ngroups; + gid_t *groups; + struct passwd *pw = NULL; + struct group *gr = NULL; + char *user_buffer = NULL; + char log_buf[LOCAL_LOG_BUF_SIZE]; + + ngroups = 0; + + // You can malloc a 0 byte buffer + groups = (gid_t *)malloc(ngroups * sizeof(gid_t)); + if (groups == NULL) + { + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, "failed to allocate memory"); + return(false); + } + + pw = getpwnam_wrapper(&user_buffer, user_name); + if (pw == NULL) + { + free(groups); + if (user_buffer != NULL) + { + free(user_buffer); + } + sprintf(log_buf, "user %s not found.", user_name); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); + return(false); + } + + // The first call to getgrouplist is intended to get the number of + // groups for user_name. It will fail but set ngroups to the right number + if (getgrouplist(user_name, pw->pw_gid, groups, &ngroups) == -1) + { + free(groups); + if (ngroups > 0) + { + groups = (gid_t*)malloc(ngroups * sizeof(gid_t)); + if (groups == NULL) + { + free(pw); + free(user_buffer); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, "failed to allocate memory for getgrouplist"); + return(false); + } + if (getgrouplist(user_name, pw->pw_gid, groups, &ngroups) == -1) + { + sprintf(log_buf, " Call to getgrouplist failed: %s", strerror(errno)); + free(pw); + free(user_buffer); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); + return(false); + } + } + } + + for (j = 0; j < ngroups; j++) + { + gr = getgrgid(groups[j]); + if (gr == NULL) + { + continue; + } + + if (!strcasecmp(gr->gr_name, group_name)) + { + free(pw); + free(user_buffer); + free(groups); + return(true); + } + } + + free(pw); + free(user_buffer); + free(groups); + return(false); + } + + /** * getgrnam_ext - first calls getgrnam, and if this call doesn't return * anything, then it checks if the name is actually a group id by calling getgrgid diff --git a/src/lib/Libutils/u_hash_map_structs.c b/src/lib/Libutils/u_hash_map_structs.c index aac532f401..5d6f6dc35f 100644 --- a/src/lib/Libutils/u_hash_map_structs.c +++ b/src/lib/Libutils/u_hash_map_structs.c @@ -116,6 +116,36 @@ int hash_add_item( +void hash_priority_add_or_exit( + + job_data_container *head, + const char *name, + const char *value, + int var_type) // lower values have higher priority (see "_DATA" suffixed macros in pbs_contants.h) + + { + bool should_add = true; + job_data *old_item = head->find(name); + + if (old_item != NULL) + { + // Only call insert if we have priority over the old item + // Note that items with the same priority will be added + if (var_type > old_item->var_type) + should_add = false; + } + + if (should_add) + { + if (hash_add_item(head, name, value, var_type, SET) == FALSE) + { + fprintf(stderr, "Error allocating memory for hash (%s)-(%s)\n", name, value); + exit(1); + } + } + } // END hash_priority_add_or_exit() + + /* A wrapper for hash to accomodate for memory allocation errors */ diff --git a/src/lib/Libutils/u_misc.c b/src/lib/Libutils/u_misc.c index 2992db463b..ca24ed1320 100644 --- a/src/lib/Libutils/u_misc.c +++ b/src/lib/Libutils/u_misc.c @@ -84,6 +84,7 @@ #include #include #include +#include #include "utils.h" #include "resource.h" @@ -336,11 +337,14 @@ int translate_range_string_to_vector( std::vector &indices) { - char *str = strdup(range_string); - char *ptr = str; - int prev = 0; - int curr; - int rc = PBSE_NONE; + char *str = strdup(range_string); + char *ptr = str; + int prev = 0; + int curr; + int rc = PBSE_NONE; + // use set to hold values added to vector + // since less expensive to check for duplicate entries + std::set vector_values; while (is_whitespace(*ptr)) ptr++; @@ -360,21 +364,53 @@ int translate_range_string_to_vector( if (*ptr == '-') { ptr++; + + if (!isdigit(*ptr)) + { + // Must be a digit at this point in time + rc = -1; + break; + } + curr = strtol(ptr, &ptr, 10); + if (prev >= curr) + { + // invalid range + rc = -1; + break; + } + while (prev <= curr) { + if (vector_values.insert(prev).second == false) + { + // duplicate entry + rc = -1; + break; + } + indices.push_back(prev); prev++; } + if (rc != PBSE_NONE) + break; + while ((*ptr == ',') || (is_whitespace(*ptr))) ptr++; } else { + if (vector_values.insert(prev).second == false) + { + // duplicate entry + rc = -1; + break; + } + indices.push_back(prev); while ((*ptr == ',') || diff --git a/src/lib/Libutils/u_wrapper.c b/src/lib/Libutils/u_wrapper.c index 72d103d769..d419d2b51f 100644 --- a/src/lib/Libutils/u_wrapper.c +++ b/src/lib/Libutils/u_wrapper.c @@ -1,14 +1,6 @@ #include "license_pbs.h" /* See here for the software license */ -#include /* sprintf */ -#include /* calloc */ -#include -#include -#include -#include -#include -#include - +#include "u_wrapper.h" #include "log.h" #include "pbs_error.h" diff --git a/src/lib/Libutils/u_wrapper.h b/src/lib/Libutils/u_wrapper.h new file mode 100644 index 0000000000..5bf2659f01 --- /dev/null +++ b/src/lib/Libutils/u_wrapper.h @@ -0,0 +1,42 @@ +#include "license_pbs.h" /* See here for the software license */ + +#include /* sprintf */ +#include /* calloc */ +#include +#include +#include +#include +#include +#include + + +/** + * getpwnam_wrapper - Get the passwd structure of a user + * + * getpwnam_wrapper allocates all the needed memory and makes a call to getpwnam_r. + * + * @param user_buffer - A pointer to the buffer containing the string fields pointed to by the passwd structure + * @param user_name - The name of the user for which user information is being retrieved. + * + * @returns a struct passwd pointer. The string fields in the passwd structure point to strings in user_buffer + */ +struct passwd *getpwnam_wrapper( + char **user_buffer, /* getpwnam_r uses a buffer which must be freed later. This is a pointer to that buffer */ + const char *user_name + ); + +struct passwd *getpwuid_wrapper( + char **user_buf, + uid_t uid); + +int rmdir_ext( + const char *dir, + int retry_limit); + +int unlink_ext( + const char *filename, + int retry_limit); + +int mkdir_wrapper( + const char *pathname, + mode_t mode); diff --git a/src/momctl/momctl.c b/src/momctl/momctl.c index 081dfa8c13..68c75661a5 100644 --- a/src/momctl/momctl.c +++ b/src/momctl/momctl.c @@ -57,7 +57,8 @@ enum MOMCmdEnum momQuery, momReconfig, momShutdown, - momLayout + momLayout, + momUpdateLayout }; enum MOMCmdEnum CmdIndex = momNONE; @@ -126,7 +127,7 @@ int main( char **ArgV) /* I */ { - const char *OptString = "c:Cd:f:h:lp:q:r:sv"; + const char *OptString = "c:Cd:f:h:lp:q:r:svu"; char HostList[65536]; @@ -371,6 +372,12 @@ int main( break; + case 'u': + + CmdIndex = momUpdateLayout; + + break; + case 'v': /* report verbose logging */ @@ -841,6 +848,18 @@ int do_mom( break; + case momUpdateLayout: + + if ((send_command(chan, RM_CMD_UPDATE_LAYOUT) != PBSE_NONE) || + (check_success(chan) != PBSE_NONE)) + { + fprintf(stdout, "Update layout command failed to send to mom\n"); + return(-1); + } + + + break; + case momQuery: default: diff --git a/src/resmom/Makefile.am b/src/resmom/Makefile.am index 70d7492bb0..18a9aabcf9 100644 --- a/src/resmom/Makefile.am +++ b/src/resmom/Makefile.am @@ -44,7 +44,7 @@ pbs_mom_SOURCES = catch_child.c mom_comm.c mom_inter.c mom_main.c \ release_reservation.c generate_alps_status.c \ parse_config.c node_frequency.cpp cray_energy.c \ accelerators_numa.cpp pmix_interface.c pmix_tracker.cpp \ - pmix_operation.cpp \ + pmix_operation.cpp cray_taskstats.cpp \ ../server/attr_recov.c ../server/dis_read.c \ ../server/job_attr_def.c ../server/job_recov.c \ ../server/reply_send.c ../server/resc_def_all.c \ @@ -63,7 +63,7 @@ endif pbs_demux_SOURCES = pbs_demux.c # use LDADD here so that we don't have to link MOM with libcr -pbs_demux_LDADD = $(BLCR_LDFLAGS) @PBS_MACH@/libmommach.a $(MOMLIBS) $(PBS_LIBS) +pbs_demux_LDADD = $(BLCR_LDFLAGS) @PBS_MACH@/libmommach.a $(PBS_LIBS) pbs_demux_CPPFLAGS = $(BLCR_CPPFLAGS) $(AM_CPPFLAGS) install-exec-hook: diff --git a/src/resmom/accelerators_numa.cpp b/src/resmom/accelerators_numa.cpp index 44b314e8b5..28fcfa25f5 100644 --- a/src/resmom/accelerators_numa.cpp +++ b/src/resmom/accelerators_numa.cpp @@ -21,50 +21,25 @@ void log_nvml_error(nvmlReturn_t rc, char* gpuid, const char* id); #ifdef NVML_API -hwloc_obj_t Machine::get_non_nvml_device(hwloc_topology_t topology, nvmlDevice_t device) - { - hwloc_obj_t osdev; - nvmlReturn_t nvres; - nvmlPciInfo_t pci; - - if (!hwloc_topology_is_thissystem(topology)) - { - errno = EINVAL; - return NULL; - } - - nvres = nvmlDeviceGetPciInfo(device, &pci); - if (NVML_SUCCESS != nvres) - return NULL; - - osdev = NULL; - while ((osdev = hwloc_get_next_osdev(topology, osdev)) != NULL) - { - hwloc_obj_t pcidev = osdev->parent; - if (strncmp(osdev->name, "card", 4)) - continue; - if (pcidev - && pcidev->type == HWLOC_OBJ_PCI_DEVICE - && pcidev->attr->pcidev.domain == pci.domain - && pcidev->attr->pcidev.bus == pci.bus - && pcidev->attr->pcidev.dev == pci.device - && pcidev->attr->pcidev.func == 0) - return osdev; - } - - return(NULL); - } +int Machine::initializeNVIDIADevices( + + hwloc_obj_t machine_obj, + hwloc_topology_t topology) -int Machine::initializeNVIDIADevices(hwloc_obj_t machine_obj, hwloc_topology_t topology) { nvmlReturn_t rc; + int rc_init = PBSE_NONE; + unsigned int device_count; /* Initialize the NVML handle. * * nvmlInit should be called once before invoking any other methods in the NVML library. * A reference count of the number of initializations is maintained. Shutdown only occurs * when the reference count reaches zero. + * + * This routine does not use hwloc since as of <= 1.11.7 it is known to fail to identifiy NVIDIA devices + * on some systems. * */ rc = nvmlInit(); if (rc != NVML_SUCCESS && rc != NVML_ERROR_ALREADY_INITIALIZED) @@ -73,49 +48,24 @@ int Machine::initializeNVIDIADevices(hwloc_obj_t machine_obj, hwloc_topology_t t return(PBSE_NONE); } - unsigned int device_count = 0; - /* Get the device count. */ rc = nvmlDeviceGetCount(&device_count); if (rc == NVML_SUCCESS) { nvmlDevice_t gpu; + std::set identified; /* Get the nvml device handle at each index */ for (unsigned int idx = 0; idx < device_count; idx++) { - rc = nvmlDeviceGetHandleByIndex(idx, &gpu); + PCI_Device new_device; - if (rc != NVML_SUCCESS) + if (new_device.initializePCIDevice(NULL, idx, topology) != PBSE_NONE) { - /* TODO: get gpuid from nvmlDevice_t struct */ - log_nvml_error(rc, NULL, __func__); - } - - /* Use the hwloc library to determine device locality */ - hwloc_obj_t gpu_obj; - hwloc_obj_t ancestor_obj; - int is_in_tree; - - gpu_obj = hwloc_nvml_get_device_osdev(topology, gpu); - if (gpu_obj == NULL) - { - /* This was not an nvml device. We will look for a "card" device (GeForce or Quadra) */ - gpu_obj = this->get_non_nvml_device(topology, gpu); - if (gpu_obj == NULL) - continue; - } - - /* The ancestor was not a numa chip. Is it the machine? */ - ancestor_obj = hwloc_get_ancestor_obj_by_type(topology, HWLOC_OBJ_MACHINE, gpu_obj); - if (ancestor_obj != NULL) - { - PCI_Device new_device; - - new_device.initializePCIDevice(gpu_obj, idx, topology); - - store_device_on_appropriate_chip(new_device); + rc_init = -1; + break; } + store_device_on_appropriate_chip(new_device, false); } } else @@ -136,7 +86,7 @@ int Machine::initializeNVIDIADevices(hwloc_obj_t machine_obj, hwloc_topology_t t log_nvml_error(rc, NULL, __func__); } - return(PBSE_NONE); + return(rc_init); } #endif @@ -152,7 +102,6 @@ int Chip::initializeMICDevices(hwloc_obj_t chip_obj, hwloc_topology_t topology) { hwloc_obj_t mic_obj; hwloc_obj_t ancestor_obj; - int is_in_tree; mic_obj = hwloc_intel_mic_get_device_osdev_by_index(topology, idx); if (mic_obj == NULL) @@ -218,16 +167,33 @@ void PCI_Device::initializeMic( #ifdef NVIDIA_GPUS +/* + * due to an hwloc limitation, it can't be depended on to report the cpu information about an nvml device + * so we work around that by getting the cpulist information in a different way (sysfs) + */ + void PCI_Device::initializeGpu( - int idx, - hwloc_topology_t topology) + int idx) { int rc; nvmlDevice_t gpu_device; - + nvmlPciInfo_t pci; + char cpulist_path[PATH_MAX]; + FILE *fp; + char *p; + id = idx; + + this->type = GPU; + + // look up nearest cpuset of device + // Can do this one of 3 ways: + // 1) use hwloc (broken in 1.11--doesn't id some gpu devices) + // 2) use nvmlDeviceGetCpuAffinity() (need to convert bitmap to string) + // 3) look up in sysfs (no conversion needed so use this method) + rc = nvmlDeviceGetHandleByIndex(idx, &gpu_device); if (rc != NVML_SUCCESS) { @@ -236,28 +202,38 @@ void PCI_Device::initializeGpu( buf = "nvmlDeviceGetHandleByIndex failed for nvidia gpus"; buf = buf + name.c_str(); log_err(-1, __func__, buf.c_str()); + return; } - else - { - nearest_cpuset = hwloc_bitmap_alloc(); - if (nearest_cpuset != NULL) - { - rc = hwloc_nvml_get_device_cpuset(topology, gpu_device, nearest_cpuset); - if (rc != 0) - { - string buf; - buf = "could not get cpuset of "; - buf = buf + name.c_str(); - log_err(-1, __func__, buf.c_str()); - } + // get the PCI info from the NVML identified device + rc = nvmlDeviceGetPciInfo(gpu_device, &pci); + if (rc != NVML_SUCCESS) + { + snprintf(log_buffer, sizeof(log_buffer), "nvmlDeviceGetPciInfo failed with %d for index %d", + rc, idx); + log_err(-1, __func__, log_buffer); + return; + } + + // build path to cpulist for this PCI device + snprintf(cpulist_path, sizeof(cpulist_path), "/sys/bus/pci/devices/%s/local_cpulist", + pci.busId); - hwloc_bitmap_list_snprintf(cpuset_string, MAX_CPUSET_SIZE, nearest_cpuset); - } + // open cpulist + if ((fp = fopen(cpulist_path, "r")) == NULL) + { + snprintf(log_buffer, sizeof(log_buffer), "could not open %s", cpulist_path); + log_err(-1, __func__, log_buffer); + return; } - this->type = GPU; + // read cpulist + fgets(cpuset_string, MAX_CPUSET_SIZE, fp); + fclose(fp); + // delete the trailing newline + if ((p = strchr(cpuset_string, '\n')) != NULL) + *p = '\0'; } #endif diff --git a/src/resmom/catch_child.c b/src/resmom/catch_child.c index 8d080abe8c..72ed7857e3 100644 --- a/src/resmom/catch_child.c +++ b/src/resmom/catch_child.c @@ -108,6 +108,7 @@ int send_back_std_and_staged_files(job *pjob, int exit_status); /* END external prototypes */ +void update_jobs_usage(job *pjob); void exit_mom_job(job *pjob, int mom_radix); /* @@ -1069,6 +1070,12 @@ int run_epilogues( int rc; + if (!(pjob->ji_qs.ji_svrflags & JOB_SVFLG_PROLOGUES_RAN)) + { + log_err(-1, __func__, "prologues were not run; skipping epilogues"); + return(PBSE_NONE); + } + if ((pjob->ji_wattr[JOB_ATR_interactive].at_flags & ATR_VFLAG_SET) && pjob->ji_wattr[JOB_ATR_interactive].at_val.at_long) { @@ -1174,7 +1181,7 @@ int send_job_obit( /* send the job obiturary notice to the server */ - preq = alloc_br(PBS_BATCH_JobObit); + preq = new batch_request(PBS_BATCH_JobObit); if (preq == NULL) { @@ -1202,9 +1209,17 @@ int send_job_obit( resc_access_perm = ATR_DFLAG_RDACC; - if (check_rur == true) + if (is_login_node == TRUE) { - get_energy_used(pjob); + if (check_rur == true) + { + get_energy_used(pjob); + } + + if (get_cray_taskstats) + { + update_jobs_usage(pjob); + } } encode_used(pjob, resc_access_perm, NULL, &preq->rq_ind.rq_jobobit.rq_attr); @@ -1490,7 +1505,7 @@ void *obit_reply( /* read and decode the reply */ - if ((preq = alloc_br(PBS_BATCH_JobObit)) == NULL) + if ((preq = new batch_request(PBS_BATCH_JobObit)) == NULL) return(NULL); CLEAR_HEAD(preq->rq_ind.rq_jobobit.rq_attr); @@ -1754,10 +1769,9 @@ void init_abort_jobs( DBPRT(("init_abort_jobs: setting grpcache for job %s\n", pj->ji_qs.ji_jobid)); - bool good; + - good = check_pwd(pj); - if (good == false) + if (check_pwd(pj) != PBSE_NONE) { /* somehow a job that was legally executing (had a password entry) * no longer has a password entry?? */ diff --git a/src/resmom/cray_taskstats.cpp b/src/resmom/cray_taskstats.cpp new file mode 100644 index 0000000000..ad505d412b --- /dev/null +++ b/src/resmom/cray_taskstats.cpp @@ -0,0 +1,220 @@ + +#include +#include +#include +#include +#include +#include +#include + +#include "pbs_job.h" +#include "resource.h" +#include "pbs_error.h" +#include "resmon.h" + +unsigned int total_chars_read = 0; +ssize_t last_line = 0; +std::vector task_lines; +// usage_information maps the job id to the usage reported +std::map usage_information; + + + +/* + * update_usage_information() + * + * Updates our map with the latest and greatest usage information + */ + +void update_usage_information( + + std::string &jobid, + task_usage_info &tui) + + { + std::map::iterator it = usage_information.find(jobid); + + if (it != usage_information.end()) + { + if (tui.cput > it->second.cput) + usage_information[jobid].cput = tui.cput; + + if (tui.mem > it->second.mem) + usage_information[jobid].mem = tui.mem; + } + else + usage_information[jobid] = tui; + } // END update_usage_information() + + +/* + * parse_rur_line() + * + * Parses a line of an rur file to find the stats for this job. + * @param line - the line of the file + * @param pjob - the job whose stats we're finding + */ + +// RUR usage lines look like this +// 2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 0, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 10000000, 'stime', 0, 'max_rss', 940, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0] + + +int parse_rur_line( + + const std::string &line, + std::set &jobs_this_iteration) + + { + size_t pos = line.find("jobid: "); + + if (pos != std::string::npos) + { + std::string jobid(line.substr(pos + 7)); // + 7 to pass 'jobid: ' + pos = jobid.find(","); + + if (pos != std::string::npos) + { + jobid.erase(pos); + + if (jobs_this_iteration.find(jobid) != jobs_this_iteration.end()) + { + // We're reading the file from the end, or most recent first, so there is no reason to + // look at a second entry for a job. + return(PBSE_NONE); + } + + pos = line.find("taskstats ["); + + if (pos != std::string::npos) + { + // Add 11 for 'taskstats [' + std::string stats = line.substr(pos + 11, std::string::npos); + char *work_str = strdup(stats.c_str()); + char *ptr = work_str; + char *tmp_ptr = strstr(ptr, "utime',"); + task_usage_info tui; + + if (tmp_ptr != NULL) + { + ptr = tmp_ptr + 7; // move past "utime'," + + if (*ptr == ' ') + ptr++; + + tui.cput = strtol(ptr, &ptr, 10); + } + + if ((tmp_ptr = strstr(ptr, "stime',")) != NULL) + { + ptr = tmp_ptr + 7; // move past "stime'," + + if (*ptr == ' ') + ptr++; + + tui.cput += strtol(ptr, &ptr, 10); + } + + if ((ptr = strstr(ptr, "max_rss',")) != NULL) + { + ptr += 9; // move past "max_rss'," + + if (*ptr == ' ') + ptr++; + + tui.mem = strtoll(ptr, NULL, 10); + } + + jobs_this_iteration.insert(jobid); + update_usage_information(jobid, tui); + + free(work_str); + } + } + } + + + return(PBSE_NONE); + } // parse_rur_line() + + + +/* + * append_date() + * + * Appends the date format to the end of the task_stats path + */ + +void append_date( + + std::string &path) + + { + // NYI: find out the date format and append it + // NYI: add logic to notice when it is a new day + } // END append_date() + + + +void read_rur_stats_file( + + const char *basepath) + + { + std::string path(basepath); + + append_date(path); + + std::ifstream task_file(path.c_str()); + + if (task_file.is_open()) + { + std::set jobs_this_iteration; + task_file.seekg(total_chars_read); + last_line = task_lines.size(); + + // Read in all new lines + for (std::string line; std::getline(task_file, line); ) + { + total_chars_read += line.size() + 1; // Add one for the newline that gets eaten. + task_lines.push_back(line); + } + + for (ssize_t i = task_lines.size() - 1; i >= last_line; i--) + parse_rur_line(task_lines[i], jobs_this_iteration); + } + } // END read_rur_stats_file() + + + +void update_jobs_usage( + + job *pjob) + + { + std::string numeric_jobid(pjob->ji_qs.ji_jobid); + size_t pos = numeric_jobid.find("."); + if (pos != std::string::npos) + numeric_jobid.erase(pos); + + std::map::iterator it = usage_information.find(numeric_jobid); + if (it != usage_information.end()) + { + resource_def *rd; + resource *pres; + + rd = find_resc_def(svr_resc_def, "cput", svr_resc_size); + pres = add_resource_entry(pjob->ji_wattr + JOB_ATR_resc_used, rd); + pres->rs_value.at_flags |= ATR_VFLAG_SET; + pres->rs_value.at_type = ATR_TYPE_LONG; + pres->rs_value.at_val.at_long = MAX(it->second.cput, pres->rs_value.at_val.at_long); + + rd = find_resc_def(svr_resc_def, "mem", svr_resc_size); + pres = add_resource_entry(pjob->ji_wattr + JOB_ATR_resc_used, rd); + pres->rs_value.at_flags |= ATR_VFLAG_SET; + pres->rs_value.at_type = ATR_TYPE_SIZE; + pres->rs_value.at_val.at_size.atsv_shift = 10; + pres->rs_value.at_val.at_size.atsv_units = ATR_SV_BYTESZ; + pres->rs_value.at_val.at_size.atsv_num = MAX(it->second.mem, pres->rs_value.at_val.at_size.atsv_num); + } + } // END update_jobs_usage() + diff --git a/src/resmom/linux/mom_mach.c b/src/resmom/linux/mom_mach.c index 837f4e3bbd..acaf81d467 100644 --- a/src/resmom/linux/mom_mach.c +++ b/src/resmom/linux/mom_mach.c @@ -157,9 +157,9 @@ extern char path_meminfo[MAX_LINE]; ** local functions and data */ static const char *resi (struct rm_attribute *); -static const char *totmem (struct rm_attribute *); +const char *totmem (struct rm_attribute *); static const char *availmem (struct rm_attribute *); -static const char *physmem (struct rm_attribute *); +const char *physmem (struct rm_attribute *); static const char *ncpus (struct rm_attribute *); static const char *walltime (struct rm_attribute *); static const char *quota (struct rm_attribute *); @@ -769,6 +769,11 @@ proc_mem_t *get_proc_mem(void) #else proc_mem_t *mem; #endif + + time_now = time(NULL); + + if (time_now - ret_mm.timestamp < ServerStatUpdateInterval) + return(&ret_mm); #ifdef NUMA_SUPPORT ret_mm.mem_total = 0; @@ -810,6 +815,8 @@ proc_mem_t *get_proc_mem(void) free(mem); #endif + ret_mm.timestamp = time_now; + return(&ret_mm); } /* END get_proc_mem() */ @@ -1600,7 +1607,7 @@ unsigned long long resi_sum( } } - full_cgroup_path = cg_memory_path + pjob->ji_qs.ji_jobid + "/memory.max_usage_in_bytes"; + full_cgroup_path = cg_memory_path + pjob->ji_qs.ji_jobid + "/memory.stat"; fd = open(full_cgroup_path.c_str(), O_RDONLY); if (fd <= 0) @@ -1624,10 +1631,19 @@ unsigned long long resi_sum( } else if (rc != 0) { - int hardwareStyle; - unsigned long long mem_read; + int hardwareStyle; + unsigned long long mem_read; + char *buf2; + buf2 = strstr(buf, "\nrss "); + if (buf2 == NULL) + { + sprintf(buf, "read failed finding rss %s", full_cgroup_path.c_str()); + log_err(errno, __func__, buf); + close(fd); + return(0); + } - mem_read = strtoull(buf, NULL, 10); + mem_read = strtoull(buf2 + 5, NULL, 10); hardwareStyle = this_node.getHardwareStyle(); @@ -4393,13 +4409,19 @@ const char *nusers( - +/* + * totmem - Determine the total amount of memory (RAM + Swap) on this host + * + * @return the total memory in kb as a string in the format: kb, or NULL on error + */ const char *totmem( struct rm_attribute *attrib) { - proc_mem_t *mm; + proc_mem_t *mm; + unsigned long total = 0; + unsigned long swap; if (attrib) { @@ -4428,16 +4450,32 @@ const char *totmem( log_record(PBSEVENT_SYSTEM, 0, __func__, log_buffer); } - sprintf(ret_string, "%lukb", + total = mm->mem_total >> 10; // Convert from bytes to kb + + if ((max_memory > 0) && + (max_memory < total)) + total = max_memory; + + swap = mm->swap_total >> 10; // Convert from bytes to kb + + if ((max_swap > 0) && + (max_swap < swap)) + total += max_swap; + else + total += swap; - (ulong)((mm->mem_total >> 10) + (mm->swap_total >> 10))); /* KB */ + sprintf(ret_string, "%lukb", total); return(ret_string); } /* END totmem() */ - +/* + * availmem() - Get the amount of available RAM + Swap for this host + * + * @return the available memory in kb as a string in the format: kb, or NULL on error + */ const char *availmem( @@ -4462,7 +4500,7 @@ const char *availmem( rm_errno = RM_ERR_SYSTEM; return(NULL); - } /* END availmem() */ + } if (LOGLEVEL >= 6) { @@ -4482,7 +4520,6 @@ const char *availmem( - const char *ncpus( struct rm_attribute *attrib) @@ -4588,25 +4625,19 @@ int find_file( +/* + * physmem() - Determines the amount of RAM on this machine + * + * @return - the amount of RAM in kb as a string in the format: amount, or NULL on error + */ - -static const char *physmem( +const char *physmem( struct rm_attribute *attrib) { - char tmpBuf[PMEMBUF_SIZE]; - - char *tmp_ptr; - char *BPtr; - int BSpace; - - unsigned long long mem; - unsigned long long mem_total; - FILE *fp; -#ifdef NUMA_SUPPORT - int i; -#endif + proc_mem_t *mm; + unsigned long long mem_total; if (attrib != NULL) { @@ -4617,78 +4648,19 @@ static const char *physmem( return(NULL); } - mem_total = 0; - -#ifdef NUMA_SUPPORT - - for (i = 0; i < node_boards[numa_index].num_nodes; i++) -#endif /* NUMA_SUPPORT */ + if ((mm = get_proc_mem()) == NULL) { -#ifdef NUMA_SUPPORT - if (!(fp = fopen(node_boards[numa_index].path_meminfo[i],"r"))) -#else - if (!(fp = fopen(path_meminfo, "r"))) -#endif - { - rm_errno = RM_ERR_SYSTEM; - - return(NULL); - } - - BPtr = tmpBuf; - - BSpace = sizeof(tmpBuf); - - BPtr[0] = '\0'; - - while (!feof(fp)) - { - if (fgets(BPtr, BSpace, fp) == NULL) - { - break; - } - - BSpace -= strlen(BPtr); - - BPtr += strlen(BPtr); - } - - fclose(fp); - - /* FORMAT: '...\nMemTotal: XXX kB\n' */ - - if ((tmp_ptr = strstr(tmpBuf, "MemTotal:")) != NULL) - { - BPtr = tmp_ptr + strlen("MemTotal:"); - - if (sscanf(BPtr, "%llu", - &mem) != 1) - { - rm_errno = RM_ERR_SYSTEM; - - return(NULL); - } - - /* value specified in kb */ - } - else - { - /* attempt to load first numeric value */ - - if (sscanf(BPtr, "%*s %llu", - &mem) != 1) - { - rm_errno = RM_ERR_SYSTEM; + log_err(errno, __func__, "get_proc_mem"); - return(NULL); - } + rm_errno = RM_ERR_SYSTEM; + return(NULL); + } - /* value specified in bytes */ + mem_total = mm->mem_total >> 10; // Convert from bytes to kb - mem >>= 10; - } - mem_total += mem; - } + if ((max_memory > 0) && + (max_memory < mem_total)) + mem_total = max_memory; sprintf(ret_string, "%llukb", mem_total); @@ -4697,9 +4669,6 @@ static const char *physmem( - - - char *size_fs( char *param) @@ -4729,10 +4698,10 @@ char *size_fs( return(NULL); } -#ifdef RPT_BAVAIL -#define RPT_STATFS_MEMBER f_bavail -#else +#ifdef RPT_BFREE #define RPT_STATFS_MEMBER f_bfree +#else +#define RPT_STATFS_MEMBER f_bavail #endif sprintf(ret_string, "%lukb:%lukb", diff --git a/src/resmom/mom_comm.c b/src/resmom/mom_comm.c index 826200b567..f75733f841 100644 --- a/src/resmom/mom_comm.c +++ b/src/resmom/mom_comm.c @@ -99,6 +99,7 @@ #include #endif #include +#include #include "libpbs.h" #include "list_link.h" @@ -209,7 +210,7 @@ struct routefd unsigned short r_fd; }; -fd_set readset; +struct pollfd *readset = NULL; /* external functions */ @@ -2628,10 +2629,7 @@ int im_join_job_as_sister( } } - bool good; - - good = check_pwd(pjob); - if (good == false) + if (check_pwd(pjob) != PBSE_NONE) { /* log_buffer populated in check_pwd() */ @@ -4979,7 +4977,31 @@ void create_contact_list( if (node_addr != ipaddr_connect) sister_list.insert(i); } - } + } // END create_contact_list() + + + +/* + * process_kill_or_abort_error_from_ms() + * + * I sent a kill or abort to mother superior and got an error back; mother superior + * must've already cleaned up the job, so I should too. + * + * @param pjob - the job we're about to delete + * @return PBSE_NONE + */ + +int process_kill_or_abort_error_from_ms( + + job *pjob) + + { + int rc = PBSE_NONE; + + mom_deljob(pjob); + + return(rc); + } // END process_kill_or_abort_error_from_ms() @@ -5157,6 +5179,22 @@ int im_poll_error( +/* + * is_mother_superior() + * + * @return true if np describes mother superior for its job, false otherwise + */ + +bool is_mother_superior( + + hnodent *np) + + { + return(np->hn_node == 0); + } // END is_mother_superior() + + + /* * process_error_reply @@ -5246,6 +5284,10 @@ int process_error_reply( case IM_KILL_JOB: rc = process_end_job_error_reply(pjob, np, pSockAddr, errcode); + + if ((rc == IM_FAILURE) && + (is_mother_superior(np) == true)) + rc = process_kill_or_abort_error_from_ms(pjob); break; @@ -6349,7 +6391,7 @@ void im_request( chan->sock = -1; snprintf(log_buffer, LOCAL_LOG_BUF_SIZE, - "Error response received from client %s (%d) jobid %s", + "response received from client %s (%d) jobid %s", netaddr(pSockAddr), sender_port, jobid); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, jobid, log_buffer); @@ -8319,7 +8361,7 @@ static int adoptSession( char jobid_copy[PBS_MAXSVRJOBID+1]; int other_id_len; -#ifdef PENABLE_LINUX26_CPUSETS +#if !defined(PENABLE_LINUX_CGROUPS) and defined(PENABLE_LINUX26_CPUSETS) unsigned int len; FILE *fp; @@ -8838,6 +8880,8 @@ int run_prologue_scripts( ret = PBSE_NONE; done: + pjob->ji_qs.ji_svrflags |= JOB_SVFLG_PROLOGUES_RAN; + return(ret); } /* END run_prologue_scripts() */ @@ -8856,6 +8900,10 @@ int readit( char buf[READ_BUF_SIZE]; size_t ret; + // confirm readset has been initialized + if (readset == NULL) + return(-2); + if ((amt = recv(sock, buf, READ_BUF_SIZE, 0)) > 0) { ret = send(fd, buf, amt, 0); @@ -8863,14 +8911,21 @@ int readit( { close(sock); close(fd); - FD_CLR(sock, &readset); + + // remove sock from readset + readset[sock].fd = -1; + readset[sock].events = 0; + readset[sock].revents = 0; } } else { close(sock); - FD_CLR(sock, &readset); + // remove sock from readset + readset[sock].fd = -1; + readset[sock].events = 0; + readset[sock].revents = 0; } return(amt); @@ -8895,7 +8950,6 @@ void fork_demux( { pid_t cpid; - struct timeval timeout; int i; int retries; int maxfd; @@ -8905,7 +8959,6 @@ void fork_demux( int fd2; int im_mom_stdout; int im_mom_stderr; - fd_set selset; pid_t parent; u_long ipaddr; struct sigaction act; @@ -8915,6 +8968,8 @@ void fork_demux( int pipes[2]; int pipe_failed = FALSE; char buf[MAXLINE]; + int pollset_size_bytes; + struct pollfd *pollset; if ((maxfd = sysconf(_SC_OPEN_MAX)) < 0) { @@ -9011,9 +9066,35 @@ void fork_demux( /* maxfd = sysconf(_SC_OPEN_MAX); */ - FD_ZERO(&readset); - FD_SET(im_mom_stdout, &readset); - FD_SET(im_mom_stderr, &readset); + pollset_size_bytes = maxfd * sizeof(struct pollfd); + + readset = (struct pollfd *)malloc(maxfd * sizeof(struct pollfd)); + if (readset == NULL) + { + perror("failed to malloc memory for readset"); + _exit(5); + } + + // set initial values + for (i = 0; i < maxfd; i++) + { + readset[i].fd = -1; + readset[i].events = 0; + readset[i].revents = 0; + } + + readset[im_mom_stdout].fd = im_mom_stdout; + readset[im_mom_stdout].events = POLLIN; + + readset[im_mom_stderr].fd = im_mom_stderr; + readset[im_mom_stderr].events = POLLIN; + + pollset = (struct pollfd *)malloc(pollset_size_bytes); + if (pollset == NULL) + { + perror("failed to malloc memory for pollset"); + _exit(5); + } if (listen(im_mom_stdout, TORQUE_LISTENQUEUE) < 0) { @@ -9094,11 +9175,11 @@ void fork_demux( while (1) { - selset = readset; - timeout.tv_usec = 0; - timeout.tv_sec = 20; - - n = select(FD_SETSIZE, &selset, (fd_set *)0, (fd_set *)0, &timeout); + // copy readset to local set for poll + memcpy(pollset, readset, pollset_size_bytes); + + // wait for up to 20sec + n = poll(pollset, maxfd, 20000); if (n == -1) { @@ -9108,7 +9189,7 @@ void fork_demux( } else { - perror("fork_demux: select failed\n"); + perror("fork_demux: poll failed\n"); close(im_mom_stdout); close(im_mom_stderr); close(fd1); @@ -9131,12 +9212,18 @@ void fork_demux( } /* END else if (n == 0) */ - for (i = 0;(n != 0) && (i < maxfd);++i) + for (i = 0; (n > 0) && (i < maxfd); i++) { - if (FD_ISSET(i, &selset)) + // skip entry with no return events + if (pollset[i].revents == 0) + continue; + + // decrement count of structures that have return events + n--; + + if ((pollset[i].revents & POLLIN)) { /* this socket has data */ - n--; switch (routem[i].r_which) { @@ -9159,8 +9246,11 @@ void fork_demux( routem[newsock].r_which = routem[i].r_which == listen_out ? new_out : new_err; routem[newsock].r_fd = newsock; open_sockets++; - - FD_SET(newsock, &readset); + + // add new socket to readset for future polling + readset[newsock].fd = newsock; + readset[newsock].events = POLLIN; + readset[newsock].revents = 0; break; diff --git a/src/resmom/mom_inter.c b/src/resmom/mom_inter.c index 898ab6f70e..c8b1d0cf94 100644 --- a/src/resmom/mom_inter.c +++ b/src/resmom/mom_inter.c @@ -538,31 +538,22 @@ int x11_create_display( FILE *f; pid_t childpid; - struct pfwdsock *socks; + std::vector *socks = new std::vector(NUM_SOCKS); const char *homeenv = "HOME"; *display = '\0'; - if ((socks = (struct pfwdsock *)calloc(sizeof(struct pfwdsock), NUM_SOCKS)) == NULL) - { - /* FAILURE - cannot alloc memory */ - - fprintf(stderr,"ERROR: could not calloc!\n"); - - return(-1); - } - if (put_env_var(homeenv, homedir)) { /* FAILURE */ - free(socks); + delete(socks); fprintf(stderr, "ERROR: could not insert %s into environment\n", homeenv); return(-1); } for (n = 0;n < NUM_SOCKS;n++) - socks[n].active = 0; + socks->at(n).active = 0; x11proto[0] = x11data[0] = '\0'; @@ -578,7 +569,7 @@ int x11_create_display( n, strerror(errno)); - free(socks); + delete(socks); return(-1); } @@ -601,7 +592,7 @@ int x11_create_display( fprintf(stderr, "getaddrinfo: %.100s\n", gai_strerror(gaierr)); - free(socks); + delete(socks); return(-1); } @@ -626,7 +617,7 @@ int x11_create_display( fprintf(stderr, "socket: %.100s\n", strerror(errno)); - free(socks); + delete(socks); freeaddrinfo(aitop); return (-1); @@ -666,7 +657,7 @@ int x11_create_display( for (n = 0; n < num_socks; n++) { - close(socks[n].sock); + close(socks->at(n).sock); } num_socks = 0; @@ -674,8 +665,8 @@ int x11_create_display( break; } - socks[num_socks].sock = sock; - socks[num_socks].active = 1; + socks->at(num_socks).sock = sock; + socks->at(num_socks).active = 1; num_socks++; #ifndef DONT_TRY_OTHER_AF @@ -706,7 +697,7 @@ int x11_create_display( { fprintf(stderr, "Failed to allocate internet-domain X11 display socket.\n"); - free(socks); + delete(socks); return(-1); } @@ -716,21 +707,21 @@ int x11_create_display( for (n = 0;n < num_socks;n++) { DBPRT(("listening on fd %d\n", - socks[n].sock)); + socks->at(n).sock)); - if (listen(socks[n].sock,TORQUE_LISTENQUEUE) < 0) + if (listen(socks->at(n).sock,TORQUE_LISTENQUEUE) < 0) { fprintf(stderr,"listen: %.100s\n", strerror(errno)); - close(socks[n].sock); + close(socks->at(n).sock); - free(socks); + delete(socks); return(-1); } - socks[n].listening = 1; + socks->at(n).listening = 1; } /* END for (n) */ /* setup local xauth */ @@ -766,14 +757,14 @@ int x11_create_display( fprintf(stderr, "could not run %s\n", cmd); - free(socks); + delete(socks); return(-1); } if ((childpid = fork()) > 0) { - free(socks); + delete(socks); DBPRT(("successful x11 init, returning display %d\n", display_number)); @@ -785,7 +776,7 @@ int x11_create_display( { fprintf(stderr, "failed x11 init fork\n"); - free(socks); + delete(socks); return(-1); } diff --git a/src/resmom/mom_main.c b/src/resmom/mom_main.c index b52e453a3e..2d8453a15f 100644 --- a/src/resmom/mom_main.c +++ b/src/resmom/mom_main.c @@ -192,6 +192,7 @@ char *path_undeliv; char *path_aux; char *path_home = (char *)PBS_SERVER_HOME; char *mom_home; +char mom_ipaddr[INET_ADDRSTRLEN]; bool use_path_home = false; @@ -279,9 +280,10 @@ int use_nvidia_gpu = TRUE; pjobexec_t TMOMStartInfo[TMAX_JE]; - +const char *taskstats_basepath = "/var/opt/cray/log/partition-current/messages-"; /* prototypes */ +void read_rur_stats_file(const char *basepath); void read_mom_hierarchy(); void sort_paths(); void resend_things(); @@ -299,12 +301,12 @@ extern int check_for_mics(uint32_t& mic_count); #ifdef NVIDIA_GPUS #ifdef NVML_API -extern int init_nvidia_nvml(unsigned int &device_count); extern int shut_nvidia_nvml(); #endif /* NVML_API */ extern int check_nvidia_setup(); #endif /* NVIDIA_GPUS */ +int read_all_devices(); int send_join_job_to_a_sister(job *pjob, int stream, eventent *ep, tlist_head phead, int node_id); void prepare_child_tasks_for_delete(); static void mom_lock(int fds, int op); @@ -344,6 +346,7 @@ char *path_log; int LOGLEVEL = 0; /* valid values (0 - 10) */ int DEBUGMODE = 0; bool daemonize_mom = true; +bool force_layout_update = false; long TJobStartTimeout = PBS_PROLOG_TIME; /* seconds to wait for job to launch before purging */ @@ -710,10 +713,6 @@ void die( cleanup(); -#if defined(NVIDIA_GPUS) && defined(NVML_API) - shut_nvidia_nvml(); -#endif /* NVIDIA_GPUS and NVML_API */ - log_close(1); exit(1); @@ -1647,7 +1646,8 @@ void add_diag_header( { output << "\nHost: " << mom_short_name << "/" << mom_host << " Version: "; - output << PACKAGE_VERSION << " PID: " << getpid() << "\n"; + output << PACKAGE_VERSION << " IP address: " << mom_ipaddr; + output << " PID: " << getpid() << "\n"; } /* END add_diag_header() */ @@ -2501,6 +2501,45 @@ void set_report_mom_cuda_visible_devices( +void set_report_node_check_on_job_start( + + std::stringstream &output, + char *curr) + + { + int enable; + + if ((*curr == '=') && ((*curr) + 1 != '\0')) + { + if ((enable = setbool(curr + 1)) != -1) + PBSNodeCheckProlog = enable; + } + + output << "node_check_on_job_start=" << PBSNodeCheckProlog; + + } /* END set_node_check_on_job_start() */ + + + +void set_report_node_check_on_job_end( + + std::stringstream &output, + char *curr) + + { + int enable; + + if ((*curr == '=') && ((*curr) + 1 != '\0')) + { + if ((enable = setbool(curr + 1)) != -1) + PBSNodeCheckEpilog = enable; + } + + output << "node_check_on_job_end=" << PBSNodeCheckEpilog; + } /* END set_node_check_on_job_end() */ + + + void set_report_rcpcmd( std::stringstream &output, @@ -2763,6 +2802,14 @@ int process_rm_cmd_request( { set_report_mom_cuda_visible_devices(output, curr); } + else if (!strncasecmp(name, "node_check_on_job_start", strlen("node_check_on_job_start"))) + { + set_report_node_check_on_job_start(output, curr); + } + else if (!strncasecmp(name, "node_check_on_job_end", strlen("node_check_on_job_end"))) + { + set_report_node_check_on_job_end(output, curr); + } else { report_other_configured_attribute(output, name, curr, cp, restrictrm); @@ -3083,10 +3130,6 @@ int rm_request( cleanup(); -#if defined(NVIDIA_GPUS) && defined(NVML_API) - shut_nvidia_nvml(); -#endif /* NVIDIA_GPUS and NVML_API */ - /* We use delete_job_files_sem to make sure there are no outstanding job cleanup routines in progress before we exit. delete_job_files_sem @@ -3128,6 +3171,15 @@ int rm_request( break; + case RM_CMD_UPDATE_LAYOUT: + + // Force an over-write of the layout on the next successful send to pbs_server + force_layout_update = true; + diswsi(chan, RM_RSP_OK); + DIS_tcp_wflush(chan); + + break; + default: sprintf(log_buffer, "unknown command %d", @@ -3342,13 +3394,14 @@ int do_tcp( default: { - struct sockaddr_in *addr = NULL; struct sockaddr s_addr; unsigned int len = sizeof(s_addr); if (getpeername(chan->sock, &s_addr, &len) == 0) { - addr = (struct sockaddr_in *)&s_addr; +#if DEBUG > 0 + struct sockaddr_in *addr = (struct sockaddr_in *)&s_addr; +#endif DBPRT(("%s: unknown request %d from %s", __func__, proto, netaddr(addr))) } @@ -3834,9 +3887,9 @@ int job_over_limit( total = (index == 0) ? gettime(useresc) : getsize(useresc); #ifndef NUMA_SUPPORT - for (i = 0;i < pjob->ji_numnodes - 1;i++) + for (int j = 0; j < pjob->ji_numnodes - 1; j++) { - noderes *nr = &pjob->ji_resources[i]; + noderes *nr = &pjob->ji_resources[j]; total += ((index == 0) ? nr->nr_cput : nr->nr_mem); } @@ -4226,7 +4279,7 @@ void parse_command_line( errflg = 0; - while ((c = getopt(argc, argv, "a:A:c:C:d:DFhH:l:L:mM:pPqrR:s:S:vwx-:")) != -1) + while ((c = getopt(argc, argv, "a:A:c:C:d:DfFhH:l:L:mM:pPqrR:s:S:vwx-:")) != -1) { switch (c) { @@ -4246,6 +4299,7 @@ void parse_command_line( printf("installdir: %s\n", PBS_INSTALL_DIR); printf("serverhome: %s\n", PBS_SERVER_HOME); printf("version: %s\n", PACKAGE_VERSION); + printf("Commit: %s\n", GIT_HASH); exit(0); } @@ -4334,6 +4388,12 @@ void parse_command_line( break; + case 'f': // force layout update + + force_layout_update = true; + + break; + case 'F': /* do not fork */ // use when running under systemd @@ -4700,6 +4760,8 @@ int cg_initialize_hwloc_topology() hwloc_free_xmlbuffer(topology, xml_buf); #endif + read_all_devices(); + unsigned long flags = HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM; flags |= HWLOC_TOPOLOGY_FLAG_IO_DEVICES; @@ -4837,6 +4899,8 @@ int setup_program_environment(void) char *ptr; /* local tmp variable */ int network_retries = 0; + struct sockaddr_in network_addr; + /* must be started with real and effective uid of 0 */ if (IamRoot() == 0) { @@ -5015,6 +5079,12 @@ int setup_program_environment(void) hostc = 1; } + // Get external IP address of local node + if (get_local_address(network_addr) == 0) + { + inet_ntop(AF_INET, &(network_addr.sin_addr), mom_ipaddr, INET_ADDRSTRLEN); + } + if (!multi_mom) { log_init(NULL, mom_host); @@ -5130,6 +5200,13 @@ int setup_program_environment(void) exit(1); } + if(varattr_tv != -1 && varattr_tv < ServerStatUpdateInterval) + { + sprintf(log_buffer, + "WARNING: varattr time value of %i is less than server update interval of ($status_update_time) %i. Varattr will only be run after the server update interval. Suggest setting varattr time to a multiple of server update interval\n", + varattr_tv, ServerStatUpdateInterval); + log_record(PBSE_NONE,PBS_EVENTCLASS_SERVER,msg_daemonname,log_buffer); + } #ifdef PENABLE_LINUX26_CPUSETS rc = initialize_hwloc_topology(); @@ -6466,7 +6543,7 @@ void prepare_child_tasks_for_delete() -time_t calculate_select_timeout() { +time_t calculate_poll_timeout() { time_t tmpTime; extern time_t wait_time; @@ -6608,11 +6685,11 @@ void main_loop(void) time_now = time((time_t *)0); - tmpTime = calculate_select_timeout(); + tmpTime = calculate_poll_timeout(); resend_things(); - /* wait_request does a select and then calls the connection's cn_func for sockets with data */ + /* wait_request does a poll and then calls the connection's cn_func for sockets with data */ if (wait_request(tmpTime, NULL) != 0) { @@ -6637,6 +6714,10 @@ void main_loop(void) { MOMCheckRestart(); /* There are no jobs, see if the server needs to be restarted. */ } + + if (get_cray_taskstats) + read_rur_stats_file(taskstats_basepath); + } /* END while (mom_run_state == MOM_RUN_STATE_RUNNING) */ #ifdef ENABLE_PMIX @@ -7120,6 +7201,8 @@ int main( { use_nvidia_gpu = FALSE; } + else + shut_nvidia_nvml(); #endif /* NVML_API */ if (!check_nvidia_setup()) { @@ -7143,10 +7226,6 @@ int main( /* shutdown mom */ -#if defined(NVIDIA_GPUS) && defined(NVML_API) - shut_nvidia_nvml(); -#endif /* NVIDIA_GPUS and NVML_API */ - mom_close_poll(); net_close(-1); /* close all network connections */ diff --git a/src/resmom/mom_process_request.c b/src/resmom/mom_process_request.c index 46c0909c1a..90c100d036 100644 --- a/src/resmom/mom_process_request.c +++ b/src/resmom/mom_process_request.c @@ -133,7 +133,7 @@ void *mom_process_request( time_now = time(NULL); - if ((request = alloc_br(0)) == NULL) + if ((request = new batch_request(0)) == NULL) { mom_close_client(sfds); return NULL; @@ -502,47 +502,25 @@ static void mom_close_client( +batch_request::batch_request( -/* - * alloc_br - allocate and clear a batch_request structure - */ - -struct batch_request *alloc_br( - - int type) + int type) : rq_type(type), rq_perm(0), rq_fromsvr(0), rq_conn(-1), rq_orgconn(-1), rq_extsz(0), + rq_time(time_now), rq_refcount(0), rq_extra(NULL), rq_noreply(FALSE), rq_failcode(0), + rq_extend(NULL), rq_id() { + CLEAR_LINK(this->rq_link); - struct batch_request *req = NULL; - - req = (struct batch_request *)calloc(1, sizeof(struct batch_request)); - - if (req == NULL) - { - log_err(errno, "alloc_br", msg_err_malloc); - - return(NULL); - } - - req->rq_type = type; - - req->rq_conn = -1; /* indicate not connected */ - req->rq_orgconn = -1; /* indicate not connected */ - req->rq_time = time_now; - req->rq_reply.brp_choice = BATCH_REPLY_CHOICE_NULL; - req->rq_noreply = FALSE; /* indicate reply is needed */ + memset(&this->rq_user, 0, sizeof(this->rq_user)); + memset(&this->rq_host, 0, sizeof(this->rq_host)); + memset(&this->rq_reply, 0, sizeof(this->rq_reply)); - CLEAR_LINK(req->rq_link); - - append_link(&svr_requests, &req->rq_link, req); - - return(req); + this->rq_reply.brp_choice = BATCH_REPLY_CHOICE_NULL; + append_link(&svr_requests, &this->rq_link, this); } - - /* * close_quejob - locate and deal with the new job that was being recevied * when the net connection closed. @@ -586,43 +564,31 @@ static void close_quejob( - - -/* - * free_br - free space allocated to a batch_request structure - * including any sub-structures - */ - -void free_br( - - batch_request *preq) +batch_request::~batch_request() { - if (preq == NULL) - return; - - delete_link(&preq->rq_link); + delete_link(&this->rq_link); - reply_free(&preq->rq_reply); + reply_free(&this->rq_reply); - if (preq->rq_extend) - free(preq->rq_extend); + if (this->rq_extend) + free(this->rq_extend); - if (preq->rq_extra) - free(preq->rq_extra); + if (this->rq_extra) + free(this->rq_extra); - switch (preq->rq_type) + switch (this->rq_type) { case PBS_BATCH_QueueJob: - free_attrlist(&preq->rq_ind.rq_queuejob.rq_attr); + free_attrlist(&this->rq_ind.rq_queuejob.rq_attr); break; case PBS_BATCH_JobCred: - if (preq->rq_ind.rq_jobcred.rq_data) - free(preq->rq_ind.rq_jobcred.rq_data); + if (this->rq_ind.rq_jobcred.rq_data) + free(this->rq_ind.rq_jobcred.rq_data); break; @@ -630,27 +596,27 @@ void free_br( case PBS_BATCH_jobscript: - if (preq->rq_ind.rq_jobfile.rq_data) - free(preq->rq_ind.rq_jobfile.rq_data); + if (this->rq_ind.rq_jobfile.rq_data) + free(this->rq_ind.rq_jobfile.rq_data); break; case PBS_BATCH_HoldJob: - freebr_manage(&preq->rq_ind.rq_hold.rq_orig); + freebr_manage(&this->rq_ind.rq_hold.rq_orig); break; case PBS_BATCH_CheckpointJob: - freebr_manage(&preq->rq_ind.rq_manager); + freebr_manage(&this->rq_ind.rq_manager); break; case PBS_BATCH_MessJob: - if (preq->rq_ind.rq_message.rq_text) - free(preq->rq_ind.rq_message.rq_text); + if (this->rq_ind.rq_message.rq_text) + free(this->rq_ind.rq_message.rq_text); break; @@ -658,7 +624,7 @@ void free_br( case PBS_BATCH_AsyModifyJob: - freebr_manage(&preq->rq_ind.rq_modify); + freebr_manage(&this->rq_ind.rq_modify); break; @@ -671,13 +637,13 @@ void free_br( case PBS_BATCH_StatusSvr: /* DIAGTODO: handle PBS_BATCH_StatusDiag */ - free_attrlist(&preq->rq_ind.rq_status.rq_attr); + free_attrlist(&this->rq_ind.rq_status.rq_attr); break; case PBS_BATCH_JobObit: - free_attrlist(&preq->rq_ind.rq_jobobit.rq_attr); + free_attrlist(&this->rq_ind.rq_jobobit.rq_attr); break; @@ -685,7 +651,7 @@ void free_br( case PBS_BATCH_DelFiles: - freebr_cpyfile(&preq->rq_ind.rq_cpyfile); + freebr_cpyfile(&this->rq_ind.rq_cpyfile); break; @@ -694,11 +660,25 @@ void free_br( /* NO-OP */ break; - } /* END switch (preq->rq_type) */ + } /* END switch (this->rq_type) */ + } // END destructor() - free(preq); - return; + +/* + * free_br - free space allocated to a batch_request structure + * including any sub-structures + */ + +void free_br( + + batch_request *preq) + + { + if (preq == NULL) + return; + + delete preq; } /* END free_br() */ diff --git a/src/resmom/mom_process_request.h b/src/resmom/mom_process_request.h index 123d5ff319..60ce6c73d1 100644 --- a/src/resmom/mom_process_request.h +++ b/src/resmom/mom_process_request.h @@ -10,8 +10,6 @@ void mom_dispatch_request(int sfds, struct batch_request *request); /* static void mom_close_client(int sfds); */ -struct batch_request *alloc_br(int type); - /* static void close_quejob(int sfds); */ void free_br(struct batch_request *preq); diff --git a/src/resmom/mom_req_quejob.c b/src/resmom/mom_req_quejob.c index 3eaf880b05..2211f52d55 100644 --- a/src/resmom/mom_req_quejob.c +++ b/src/resmom/mom_req_quejob.c @@ -754,10 +754,8 @@ void req_mvjobfile( return; } - bool good; - good = check_pwd(pj); if ((pj->ji_grpcache == NULL) && - (good == false)) + (check_pwd(pj) != PBSE_NONE)) { req_reject(PBSE_UNKJOBID, 0, preq, NULL, NULL); diff --git a/src/resmom/mom_server.c b/src/resmom/mom_server.c index 88d7542517..d151f43bc2 100644 --- a/src/resmom/mom_server.c +++ b/src/resmom/mom_server.c @@ -309,6 +309,7 @@ extern mom_hierarchy_t *mh; extern char *stat_string_aggregate; extern unsigned int ssa_index; extern u_long localaddr; +extern bool force_layout_update; extern container::item_container received_statuses; std::vector global_gpu_status; std::vector mom_status; @@ -700,6 +701,11 @@ void gen_layout( std::vector &status) { + if (force_layout_update == true) + { + status.push_back("force_layout_update"); + } + std::stringstream layout; layout << name << "="; this_node.displayAsJson(layout, false); @@ -1758,11 +1764,8 @@ int send_status_through_hierarchy() void mom_server_all_update_stat(void) { - pid_t pid; - int fd_pipe[2]; - int rc; + int rc = -1; char buf[LOCAL_LOG_BUF_SIZE]; - size_t len; time_now = time(NULL); @@ -1793,16 +1796,10 @@ void mom_server_all_update_stat(void) { generate_alps_status(mom_status, apbasil_path, apbasil_protocol); - if (send_update_to_a_server() == PBSE_NONE) - { - ForceServerUpdate = false; - LastServerUpdateTime = time_now; - } + send_update_to_a_server(); } else { - /* The NVIDIA NVML library has a problem when we use it after the first fork. Let's get the gpu status first - and then fork */ #ifdef NVIDIA_GPUS global_gpu_status.clear(); add_gpu_status(global_gpu_status); @@ -1812,101 +1809,27 @@ void mom_server_all_update_stat(void) check_for_mics(global_mic_count); #endif - /* It is possible that pbs_server may get busy and start queing incoming requests and not be able - to process them right away. If pbs_mom is waiting for a reply to a statuys update that has - been queued and at the same time the server makes a request to the mom we can get stuck - in a pseudo live-lock state. That is the server is waiting for a response from the mom and - the mom is waiting for a response from the server. neither of which will come until a request times out. - If we fork the status updates this alleviates the problem by making one less request from the - mom single threaded */ - rc = pipe(fd_pipe); - if (rc != 0) - { - sprintf(buf, "pipe creation failed: %d", errno); - log_err(-1, __func__, buf); - } - - pid = fork(); - - if (pid < 0) - { - log_record(PBSEVENT_SYSTEM, 0, __func__, "Failed to fork stat update process"); - return; - } - - if (pid > 0) - { - // PARENT - close(fd_pipe[1]); - ForceServerUpdate = false; - LastServerUpdateTime = time_now; - UpdateFailCount = 0; - updates_waiting_to_send = 0; - - received_node *rn; - received_statuses.lock(); - container::item_container::item_iterator *iter = received_statuses.get_iterator(); - - // clear cached statuses from hierarchy - while ((rn = iter->get_next_item()) != NULL) - rn->statuses.clear(); - - delete iter; - received_statuses.unlock(); - - len = read(fd_pipe[0], buf, LOCAL_LOG_BUF_SIZE); - - close(fd_pipe[0]); - - if (len <= 0) - { - log_err(-1, __func__, "read of pipe failed for status update"); - return; - } - - if (buf[0] != '0') - num_stat_update_failures++; - else - { - num_stat_update_failures = 0; - for (int sindex = 0; sindex < PBS_MAXSERVER; sindex++) - { - if (mom_servers[sindex].pbs_servername[0] == '\0') - continue; - mom_servers[sindex].MOMLastSendToServerTime = time_now; - } - } - - return; - } - - // CHILD - close(fd_pipe[0]); - #ifdef NUMA_SUPPORT for (numa_index = 0; numa_index < num_node_boards; numa_index++) #endif /* NUMA_SUPPORT */ { update_mom_status(); - if (send_status_through_hierarchy() != PBSE_NONE) + if ((rc = send_status_through_hierarchy()) != PBSE_NONE) rc = send_update_to_a_server(); } - sprintf(buf, "%d", rc); - len = strlen(buf); - write(fd_pipe[1], buf, len); - - exit_called = true; - - exit(0); + if (rc == PBSE_NONE) + { + updates_waiting_to_send = 0; + force_layout_update = false; + } } } /* END mom_server_all_update_stat() */ - /** * power */ @@ -2439,6 +2362,11 @@ void reset_okclients() } } + if (pbsclient != 0) + { + auth_hosts.add_authorized_address(pbsclient, 0, ""); + } + // add localhost auth_hosts.add_authorized_address(localaddr, 0, ""); @@ -3521,7 +3449,7 @@ bool is_for_this_host( void get_device_indices( const char *device_str, - std::vector &device_indices, + std::vector &device_indices, const char *suffix) { @@ -3577,7 +3505,7 @@ void get_device_indices( if (is_for_this_host(host_name_part, suffix) == true) { - unsigned int device_index = atoi(device_index_part.c_str()); + int device_index = atoi(device_index_part.c_str()); device_indices.push_back(device_index); } diff --git a/src/resmom/mom_server_lib.h b/src/resmom/mom_server_lib.h index 775500cc47..cc89cba9a1 100644 --- a/src/resmom/mom_server_lib.h +++ b/src/resmom/mom_server_lib.h @@ -42,7 +42,7 @@ void gen_gen(const char *name, char **BPtr, int *BSpace); #if defined(NVIDIA_GPUS) && defined(NVML_API) void log_nvml_error(nvmlReturn_t rc, char* gpuid, const char* id); -int init_nvidia_nvml(unsigned int &gpu_count); +bool init_nvidia_nvml(unsigned int &gpu_count); int shut_nvidia_nvml(); @@ -67,7 +67,7 @@ void mom_server_update_gpustat(mom_server *pms, char *status_strings); #endif /* NVIDIA_GPUS */ -void get_device_indices(const char *device_str, std::vector &device_indices, const char *suffix); +void get_device_indices(const char *device_str, std::vector &device_indices, const char *suffix); void generate_server_status(std::vector& status); #ifdef NVML_API diff --git a/src/resmom/nvidia.c b/src/resmom/nvidia.c index b4a3285ac6..23637bc586 100644 --- a/src/resmom/nvidia.c +++ b/src/resmom/nvidia.c @@ -95,6 +95,7 @@ #include #include #include +#include #include #include #if defined(NTOHL_NEEDS_ARPA_INET_H) && defined(HAVE_ARPA_INET_H) @@ -143,6 +144,9 @@ extern int use_nvidia_gpu; extern time_t time_now; extern unsigned int global_gpu_count; +#ifdef NVML_API +std::map gpu_minor_to_gpu_index; +#endif /* NVML_API */ int nvidia_gpu_modes[50]; #ifdef NUMA_SUPPORT @@ -258,12 +262,95 @@ void log_nvml_error( } } +/** + * build_gpu_minor_to_gpu_index_map() + * + * Create a mapping from the NVIDIA gpu device minor numbers to the index + * numbers used with the NVML library. + * + * Note that the device index ordering used by the NVML library is based on + * the PCI bus ID. Torque server assumes an ordering based on the device + * minor number (see place_subnodes_in_hostlist()). These two orderings are + * not guaranteed to be the same so a mapping is needed to map the minor + * number to the index number used by the library. + * + * @param device_count - the number of gpu devices on this system + * @return -1 failure, 0 success + * + */ + +int build_gpu_minor_to_gpu_index_map( + + unsigned int device_count) + + { + nvmlDevice_t dev_handle; + unsigned int minor; + unsigned int index; + + // clear the map + gpu_minor_to_gpu_index.clear(); + + // build the map + for (index = 0; index < device_count; index++) + { + // get the device handle + if (nvmlDeviceGetHandleByIndex(index, &dev_handle) != NVML_SUCCESS) + { + gpu_minor_to_gpu_index.clear(); + return(-1); + } + + // look up the device handle's minor number + if (nvmlDeviceGetMinorNumber(dev_handle, &minor) != NVML_SUCCESS) + { + gpu_minor_to_gpu_index.clear(); + return(-1); + } + + // map the minor number to the NVML index + gpu_minor_to_gpu_index[minor] = index; + } + + return(0); + } + +/** + * get_gpu_handle_by_minor() + * + * Get NVIDIA device handle by device minor number. + * + * @param minor - the device minor number + * @param device - reference in which to return the device handle + * @return NVML_ERROR_UNKNOWN if minor number not in map, or an nvmlReturn_t value otherwise + * + */ + +nvmlReturn_t get_gpu_handle_by_minor( + + unsigned int minor, + nvmlDevice_t *device) + + { + std::map::const_iterator it + = gpu_minor_to_gpu_index.find(minor); + + // confirm that minor number is in the map + if (it == gpu_minor_to_gpu_index.end()) + return(NVML_ERROR_UNKNOWN); + + // return the device handle + return(nvmlDeviceGetHandleByIndex(it->second, device)); + } /* * Function to initialize the Nvidia nvml api */ -int init_nvidia_nvml(unsigned int &device_count) +bool init_nvidia_nvml( + + unsigned int &device_count) + { nvmlReturn_t rc; @@ -275,7 +362,16 @@ int init_nvidia_nvml(unsigned int &device_count) if (rc == NVML_SUCCESS) { if ((int)device_count > 0) - return (TRUE); + { + // build map function for minor_to_gpu_index + if (build_gpu_minor_to_gpu_index_map(device_count) != 0) + { + shut_nvidia_nvml(); + return(false); + } + + return(true); + } sprintf(log_buffer,"No Nvidia gpus detected\n"); log_ext(-1, __func__, log_buffer, LOG_DEBUG); @@ -284,14 +380,16 @@ int init_nvidia_nvml(unsigned int &device_count) shut_nvidia_nvml(); - return (FALSE); + return(false); } } log_nvml_error (rc, NULL, __func__); - return (FALSE); - } + return(false); + } // END init_nvidia_nvml() + + /* * Function to shutdown the Nvidia nvml api @@ -304,6 +402,9 @@ int shut_nvidia_nvml() if (!use_nvidia_gpu) return (TRUE); + // clear the map + gpu_minor_to_gpu_index.clear(); + rc = nvmlShutdown(); if (rc == NVML_SUCCESS) @@ -643,14 +744,42 @@ static int gpumodes( return(TRUE); } +/* + * Return the NVML API version + * + * Assumptions: Library has already been initialized (nvmlInit() has been called). + */ + +int get_nvml_version() + { + static int version = -1; + +#ifdef NVML_API + if (version == -1) + { + char version_buf[NVML_SYSTEM_NVML_VERSION_BUFFER_SIZE]; + + // the nvml version is returned as a string in the form x.y.z + // only return major version (x) + if (nvmlSystemGetNVMLVersion(version_buf, sizeof(version_buf)) == NVML_SUCCESS) + version = atoi(version_buf); + } +#endif + + return(version); + } + /* * Function to set gpu mode */ int setgpumode( - int gpuid, - int gpumode) + + int gpuid, + int gpumode, + bool initialized) + { #ifdef NVML_API nvmlReturn_t rc; @@ -658,25 +787,55 @@ int setgpumode( nvmlDevice_t device_hndl; char gpu_id[20]; - if (!check_nvidia_setup()) + if (initialized == false) { - return(PBSE_GPU_NOT_INITIALIZED); + if (!init_nvidia_nvml(global_gpu_count)) + return(PBSE_GPU_NOT_INITIALIZED); + + if (!check_nvidia_setup()) + { + shut_nvidia_nvml(); + return(PBSE_GPU_NOT_INITIALIZED); + } } switch (gpumode) { case gpu_normal: + compute_mode = NVML_COMPUTEMODE_DEFAULT; break; + case gpu_exclusive_thread: - compute_mode = NVML_COMPUTEMODE_EXCLUSIVE_THREAD; + + if (get_nvml_version() >= 8) + { + sprintf(log_buffer, "exclusive thread compute mode is not allowed in NVML version %d. Setting exclusive process compute mode instead.", + get_nvml_version()); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + + compute_mode = NVML_COMPUTEMODE_EXCLUSIVE_PROCESS; + } + else + { + compute_mode = NVML_COMPUTEMODE_EXCLUSIVE_THREAD; + } + break; + case gpu_prohibited: + + if (initialized == false) + shut_nvidia_nvml(); + return (PBSE_IVALREQ); break; + case gpu_exclusive_process: + compute_mode = NVML_COMPUTEMODE_EXCLUSIVE_PROCESS; break; + default: if (LOGLEVEL >= 1) { @@ -684,28 +843,35 @@ int setgpumode( rc); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); } + + if (initialized == false) + shut_nvidia_nvml(); + return (PBSE_IVALREQ); } /* get the device handle */ - - rc = nvmlDeviceGetHandleByIndex(gpuid, &device_hndl); + rc = get_gpu_handle_by_minor(gpuid, &device_hndl); if (device_hndl != NULL) { - if (LOGLEVEL >= 7) - { + if (LOGLEVEL >= 7) + { sprintf(log_buffer, "changing to mode %d for gpu %d", - gpumode, - gpuid); + gpumode, gpuid); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); - } + } rc = nvmlDeviceSetComputeMode(device_hndl, compute_mode); if (rc == NVML_SUCCESS) + { + if (initialized == false) + shut_nvidia_nvml(); + return (PBSE_NONE); + } sprintf(gpu_id, "%d", gpuid); log_nvml_error (rc, gpu_id, __func__); @@ -713,6 +879,10 @@ int setgpumode( sprintf(log_buffer, "Failed to get device handle for gpu id %d, nvml error %d", gpuid, rc); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + + if (initialized == false) + shut_nvidia_nvml(); + return(PBSE_SYSTEM); #else @@ -734,6 +904,16 @@ int setgpumode( } else /* 270 or greater driver */ { + // exclusive thread mode no longer allowed starting with driver version 367 + if ((MOMNvidiaDriverVersion >= 367) && (gpumode == gpu_exclusive_thread)) + { + sprintf(log_buffer, "exclusive thread compute mode is not allowed with NVIDIA driver version %d. Setting exclusive process compute mode instead.", + MOMNvidiaDriverVersion); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + + gpumode = gpu_exclusive_process; + } + sprintf(buf, "nvidia-smi -i %d -c %d 2>&1", gpuid, gpumode); @@ -745,8 +925,8 @@ int setgpumode( log_ext(-1, __func__, log_buffer, LOG_DEBUG); } - if ((fd = popen(buf, "r")) != NULL) - { + if ((fd = popen(buf, "r")) != NULL) + { while (!feof(fd)) { if (fgets(buf, 300, fd)) @@ -779,7 +959,7 @@ int setgpumode( } } pclose(fd); - } + } else { if (LOGLEVEL >= 0) @@ -793,7 +973,7 @@ int setgpumode( return(PBSE_NONE); #endif /* NVML_API */ - } + } // END setgpumode() /* @@ -812,9 +992,13 @@ int resetgpuecc( nvmlEccCounterType_enum counter_type; nvmlDevice_t device_hndl; char gpu_id[20]; + + if (!init_nvidia_nvml(global_gpu_count)) + return(PBSE_GPU_NOT_INITIALIZED); if (!check_nvidia_setup()) { + shut_nvidia_nvml(); return(PBSE_GPU_NOT_INITIALIZED); } @@ -830,25 +1014,25 @@ int resetgpuecc( } /* get the device handle */ - - rc = nvmlDeviceGetHandleByIndex(gpuid, &device_hndl); + rc = get_gpu_handle_by_minor(gpuid, &device_hndl); if (device_hndl != NULL) { - if (LOGLEVEL >= 7) - { - sprintf(log_buffer, "reseting error count %d-%d for gpu %d", - reset_perm, - reset_vol, - gpuid); + if (LOGLEVEL >= 7) + { + sprintf(log_buffer, "resetting error count %d-%d for gpu %d", + reset_perm, reset_vol, gpuid); - log_ext(-1, __func__, log_buffer, LOG_DEBUG); - } + log_ext(-1, __func__, log_buffer, LOG_DEBUG); + } rc = nvmlDeviceClearEccErrorCounts(device_hndl, counter_type); if (rc == NVML_SUCCESS) + { + shut_nvidia_nvml(); return (PBSE_NONE); + } sprintf(gpu_id, "%d", gpuid); log_nvml_error (rc, gpu_id, __func__); @@ -856,6 +1040,7 @@ int resetgpuecc( sprintf(log_buffer, "Failed to get device handle for gpu id %d - nvml error %d", gpuid, rc); log_err(-1, __func__, log_buffer); + shut_nvidia_nvml(); return(PBSE_SYSTEM); #else @@ -981,12 +1166,13 @@ int resetgpuecc( int set_gpu_modes( - std::vector& gpu_indices, - int gpu_flags) + std::vector &gpu_indices, + int gpu_flags) { - int rc = PBSE_NONE; - int gpu_mode = -1; + int rc = PBSE_NONE; + int gpu_mode = -1; + bool initialized = false; gpu_mode = gpu_flags; @@ -1000,7 +1186,21 @@ int set_gpu_modes( gpu_mode -= 1000; } - for (std::vector::iterator it = gpu_indices.begin(); it != gpu_indices.end(); it++) + +#ifdef NVML_API + if (!init_nvidia_nvml(global_gpu_count)) + return(PBSE_GPU_NOT_INITIALIZED); + + if (!check_nvidia_setup()) + { + shut_nvidia_nvml(); + return(PBSE_GPU_NOT_INITIALIZED); + } + + initialized = true; +#endif + + for (std::vector::iterator it = gpu_indices.begin(); it != gpu_indices.end(); it++) { int set_mode_result; /* check to see if we need to reset error counts */ @@ -1008,7 +1208,7 @@ int set_gpu_modes( { if (LOGLEVEL >= 7) { - sprintf(log_buffer, "reseting gpuid %d volatile error counts", + sprintf(log_buffer, "resetting gpuid %d volatile error counts", *it); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); @@ -1016,7 +1216,7 @@ int set_gpu_modes( } } - set_mode_result = setgpumode(*it, gpu_mode); + set_mode_result = setgpumode(*it, gpu_mode, initialized); if (set_mode_result != PBSE_NONE) { sprintf(log_buffer, "could not set mode on gpu %d", *it); @@ -1024,9 +1224,13 @@ int set_gpu_modes( rc = set_mode_result; } } + +#ifdef NVML_API + shut_nvidia_nvml(); +#endif return(rc); - } + } // END set_gpu_modes() /* @@ -1081,22 +1285,22 @@ int get_gpu_mode( int set_gpu_req_modes( - std::vector& gpu_indices, - int gpu_flags, - job *pjob) + std::vector &gpu_indices, + int gpu_flags, + job *pjob) { pbs_attribute *pattr; - int rc; - unsigned int gpu_indices_size = gpu_indices.size(); - std::vector::iterator it = gpu_indices.begin(); + int rc; + int gpu_indices_size = gpu_indices.size(); + size_t gpu_index = 0; + bool initialized = false; /* Is the a -l resource request or a -L resource request */ pattr = &pjob->ji_wattr[JOB_ATR_resource]; if (have_incompatible_dash_l_resource(pattr) == true) { rc = set_gpu_modes(gpu_indices, gpu_flags); - } else { @@ -1115,16 +1319,29 @@ int set_gpu_req_modes( req r; std::string gpu_mode; - for (unsigned int i = 0; i < cr->get_num_reqs() && it != gpu_indices.end(); i++) +#ifdef NVML_API + if (!init_nvidia_nvml(global_gpu_count)) + return(PBSE_GPU_NOT_INITIALIZED); + + if (!check_nvidia_setup()) + { + shut_nvidia_nvml(); + return(PBSE_GPU_NOT_INITIALIZED); + } + + initialized = true; +#endif + + for (int i = 0; i < cr->get_num_reqs() && gpu_index < gpu_indices.size(); i++) { int total_req_gpus; /* Look through each req and see if there is a mode to set */ r = cr->get_req(i); gpu_mode = r.get_gpu_mode(); /* returns a string indicating the gpu mode */ - total_req_gpus = r.getGpus(); + total_req_gpus = r.get_gpus(); - for (unsigned int j = 0; j < total_req_gpus && it != gpu_indices.end(); j++) + for (int j = 0; j < total_req_gpus && gpu_index < gpu_indices.size(); j++) { /* only use as many gpus as requested for the req */ int mode; @@ -1132,36 +1349,54 @@ int set_gpu_req_modes( if (gpu_flags >= 1000) { /* We need to reset error counts */ - resetgpuecc(*it, 0, 1); + resetgpuecc(gpu_indices[gpu_index], 0, 1); } mode = get_gpu_mode(gpu_mode); if (mode == gpu_mode_not_set) + { +#ifdef NVML_API + shut_nvidia_nvml(); +#endif return(PBSE_NONE); + } if (mode == gpu_unknown) { sprintf(log_buffer, "gpu mode unknown: %s", pjob->ji_qs.ji_jobid); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + +#ifdef NVML_API + shut_nvidia_nvml(); +#endif return(PBSE_IVALREQ); } - rc = setgpumode(*it, mode); + rc = setgpumode(gpu_indices[gpu_index], mode, initialized); if (rc != PBSE_NONE) { - sprintf(log_buffer, "could not set mode on gpu %d", *it); + sprintf(log_buffer, "could not set mode on gpu %d", gpu_indices[gpu_index]); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + +#ifdef NVML_API + shut_nvidia_nvml(); +#endif + return(PBSE_IVALREQ); } - it++; + + gpu_index++; } } - - rc = PBSE_NONE; + rc = PBSE_NONE; } } +#ifdef NVML_API + shut_nvidia_nvml(); +#endif + return(rc); } @@ -1179,15 +1414,55 @@ int setup_gpus_for_job( job *pjob) /* I */ { +#ifdef PENABLE_LINUX_CGROUPS + std::string gpus_reserved; + std::string gpu_range; +#else char *gpu_str; +#endif int gpu_flags = 0; int rc; + std::vector gpu_indices; /* if node does not have Nvidia recognized driver version then forget it */ if (MOMNvidiaDriverVersion < 260) return(PBSE_NONE); +#ifdef PENABLE_LINUX_CGROUPS + /* if there are no gpus, do nothing */ + if ((pjob->ji_wattr[JOB_ATR_gpus_reserved].at_flags & ATR_VFLAG_SET) == 0) + return(PBSE_NONE); + + /* if there are no gpu flags, do nothing */ + if ((pjob->ji_wattr[JOB_ATR_gpu_flags].at_flags & ATR_VFLAG_SET) == 0) + return(PBSE_NONE); + + gpus_reserved = pjob->ji_wattr[JOB_ATR_gpus_reserved].at_val.at_str; + + if (gpus_reserved.length() == 0) + return(PBSE_NONE); + + find_range_in_cpuset_string(gpus_reserved, gpu_range); + translate_range_string_to_vector(gpu_range.c_str(), gpu_indices); + + gpu_flags = pjob->ji_wattr[JOB_ATR_gpu_flags].at_val.at_long; + + if (LOGLEVEL >= 7) + { + sprintf(log_buffer, "job %s has gpus_reserved %s gpu_flags %d", + pjob->ji_qs.ji_jobid, gpu_range.c_str(), gpu_flags); + + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + } + + rc = set_gpu_req_modes(gpu_indices, gpu_flags, pjob); + if (rc != PBSE_NONE) + { + sprintf(log_buffer, "Failed to set gpu modes for job %s. error %d", pjob->ji_qs.ji_jobid, rc); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + } +#else /* if there are no gpus, do nothing */ if ((pjob->ji_wattr[JOB_ATR_exec_gpus].at_flags & ATR_VFLAG_SET) == 0) return(PBSE_NONE); @@ -1205,34 +1480,20 @@ int setup_gpus_for_job( if (LOGLEVEL >= 7) { - sprintf(log_buffer, "job %s has exec_gpus %s gpu_flags %d", - pjob->ji_qs.ji_jobid, - gpu_str, - gpu_flags); + sprintf(log_buffer, "job %s has exec_gpus %s gpu_flags %d", + pjob->ji_qs.ji_jobid, gpu_str, gpu_flags); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); } /* tokenize all of the gpu allocations the format of the gpu string is -gpu/index[+-gpu/index...]*/ /* traverse the gpu_str to see what gpus we have assigned */ - - std::vector gpu_indices; get_device_indices(gpu_str, gpu_indices, "-gpu"); -#ifndef PENABLE_LINUX_CGROUPS rc = set_gpu_modes(gpu_indices, gpu_flags); -#else - rc = set_gpu_req_modes(gpu_indices, gpu_flags, pjob); - if (rc != PBSE_NONE) - { - sprintf(log_buffer, "Failed to set gpu modes for job %s. error %d", pjob->ji_qs.ji_jobid, rc); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); - } - -#endif /* PENABLE_LINUX_CGROUPS */ - +#endif return(rc); } /* END setup_gpus_for_job() */ @@ -1261,9 +1522,14 @@ void generate_server_gpustatus_nvml( nvmlUtilization_t util_info; unsigned long long ecc_counts; char tmpbuf[1024+1]; + char log_buf[LOG_BUF_SIZE]; + + if (!init_nvidia_nvml(global_gpu_count)) + return; if (!check_nvidia_setup()) { + shut_nvidia_nvml(); return; } @@ -1272,7 +1538,10 @@ void generate_server_gpustatus_nvml( #ifdef NUMA_SUPPORT // does this node have gpus configured? if (node_boards[numa_index].gpu_end_index < 0) + { + shut_nvidia_nvml(); return; + } #endif /* get timestamp to report */ @@ -1291,7 +1560,9 @@ void generate_server_gpustatus_nvml( } else { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlSystemGetDriverVersion() called from %s", __func__); + log_nvml_error(rc, NULL, log_buf); } #ifndef NUMA_SUPPORT @@ -1300,7 +1571,10 @@ void generate_server_gpustatus_nvml( rc = nvmlDeviceGetCount(&device_count); if (rc != NVML_SUCCESS) { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetCount() called from %s", __func__); + log_nvml_error(rc, NULL, log_buf); + shut_nvidia_nvml(); return; } #endif @@ -1313,11 +1587,13 @@ void generate_server_gpustatus_nvml( for (idx = 0; idx < (int)device_count; idx++) #endif { - rc = nvmlDeviceGetHandleByIndex(idx, &device_hndl); + rc = get_gpu_handle_by_minor(idx, &device_hndl); if (rc != NVML_SUCCESS) { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetHandleByIndex() called from %s (idx=%d)", __func__, idx); + log_nvml_error(rc, NULL, log_buf); continue; } @@ -1338,7 +1614,9 @@ void generate_server_gpustatus_nvml( } else { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetDisplayMode() called from %s (idx=%d)", __func__, idx); + log_nvml_error(rc, NULL, log_buf); gpu_status.push_back("gpu_display=Unknown"); } @@ -1363,7 +1641,9 @@ void generate_server_gpustatus_nvml( } else { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetPciInfo() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the product name */ @@ -1377,7 +1657,9 @@ void generate_server_gpustatus_nvml( } else { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetName() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the fan speed */ @@ -1390,11 +1672,9 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetFanSpeed() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the memory information */ @@ -1410,11 +1690,9 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetMemoryInfo() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the compute mode */ @@ -1460,11 +1738,9 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetComputeMode() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the utilization rates */ @@ -1481,11 +1757,9 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetComputeMode() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the ECC mode */ @@ -1500,11 +1774,9 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetEccMode() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the single bit ECC errors */ @@ -1519,11 +1791,9 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetTotalEccErrors() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the double bit ECC errors */ @@ -1538,11 +1808,9 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetTotalEccErrors() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } /* get the temperature */ @@ -1556,18 +1824,16 @@ void generate_server_gpustatus_nvml( } else if (rc != NVML_ERROR_NOT_SUPPORTED) { - log_nvml_error (rc, NULL, __func__); - } - else if (LOGLEVEL >= 6) - { - log_nvml_error (rc, NULL, __func__); + snprintf(log_buf, sizeof(log_buf), + "nvmlDeviceGetTemperature() called from %s (idx=%d)", __func__, idx); + log_nvml_error (rc, NULL, log_buf); } - } + + shut_nvidia_nvml(); return; - - } + } // END generate_server_gpustatus_nvml() #endif /* NVML_API */ @@ -2073,6 +2339,7 @@ void generate_server_gpustatus_smi( } + void req_gpuctrl_mom( struct batch_request *preq) /* I */ @@ -2131,7 +2398,7 @@ void req_gpuctrl_mom( if (gpumode != -1) { - rc = setgpumode(gpuid, gpumode); + rc = setgpumode(gpuid, gpumode, false); } else if ((reset_perm != -1) || (reset_vol != -1)) { @@ -2202,3 +2469,5 @@ int add_gpu_status( return(PBSE_NONE); } /* END add_gpu_status() */ + + diff --git a/src/resmom/parse_config.c b/src/resmom/parse_config.c index beaf5621a5..8c4bb8585d 100644 --- a/src/resmom/parse_config.c +++ b/src/resmom/parse_config.c @@ -100,6 +100,7 @@ #include "authorized_hosts.hpp" #include "csv.h" #include "json/json.h" +#include "req.hpp" void encode_used(job *pjob, int perm, Json::Value *, tlist_head *phead); void encode_flagged_attrs(job *pjob, int perm, Json::Value *job_info, tlist_head *phead); @@ -127,6 +128,7 @@ double cputfactor = 1.00; float ideal_load_val = -1.0; int exec_with_exec = 0; int ServerStatUpdateInterval = DEFAULT_SERVER_STAT_UPDATES; +int varattr_tv = -1; float max_load_val = -1.0; char *auto_ideal_load = NULL; char *auto_max_load = NULL; @@ -165,7 +167,13 @@ char **maskclient = NULL; /* wildcard connections */ char MOMConfigVersion[64]; int MOMConfigDownOnError = 0; int MOMConfigRestart = 0; +// Setting this variable is redundant for cgroups, so we'll default to false +// It can only cause us pain. +#ifdef PENABLE_LINUX_CGROUPS +int MOMCudaVisibleDevices = 0; +#else int MOMCudaVisibleDevices = 1; +#endif double wallfactor = 1.00; std::vector pcphosts; long pe_alarm_time = PBS_PROLOG_TIME; @@ -189,6 +197,10 @@ int max_join_job_wait_time = MAX_JOIN_WAIT_TIME; int resend_join_job_wait_time = RESEND_WAIT_TIME; int mom_hierarchy_retry_time = NODE_COMM_RETRY_TIME; std::string presetup_prologue; +unsigned long max_memory = 0; +unsigned long max_swap = 0; +bool get_cray_taskstats = false; +u_long pbsclient; @@ -299,6 +311,11 @@ unsigned long setmomhierarchyretrytime(const char *); unsigned long setjobdirectorysticky(const char *); unsigned long setcudavisibledevices(const char *); unsigned long set_presetup_prologue(const char *); +unsigned long set_max_physical_memory(const char *); +unsigned long set_max_swap_memory(const char *); +unsigned long set_get_cray_taskstats(const char *); +unsigned long set_node_check_on_job_start(const char *); +unsigned long set_node_check_on_job_end(const char *); struct specials special[] = { { "force_overwrite", setforceoverwrite}, @@ -384,6 +401,11 @@ struct specials special[] = { { "cuda_visible_devices", setcudavisibledevices}, { "cray_check_rur", setrur }, { "presetup_prologue", set_presetup_prologue}, + { "max_physical_memory", set_max_physical_memory}, + { "max_swap_memory", set_max_swap_memory}, + { "get_cray_taskstats", set_get_cray_taskstats}, + { "node_check_on_job_start", set_node_check_on_job_start }, + { "node_check_on_job_end", set_node_check_on_job_end }, { NULL, NULL } }; @@ -501,6 +523,7 @@ unsigned long setforceoverwrite( return(0); /* error */ } + unsigned long setrur( const char *value) @@ -518,6 +541,64 @@ unsigned long setrur( }/* end setrur() */ + +/* + * set_max_physical_memory() + * + * @param value - the maximum memory value in the format + */ + +unsigned long set_max_physical_memory( + + const char *value) + + { + if (read_mem_value(value, max_memory) != PBSE_NONE) + { + sprintf(log_buffer, "Couldn't parse a memory value from '%s'", value); + log_err(-1, __func__, log_buffer); + return(0); + } + + return(1); + } + + + +unsigned long set_max_swap_memory( + + const char *value) + + { + if (read_mem_value(value, max_swap) != PBSE_NONE) + { + sprintf(log_buffer, "Couldn't parse a memory value from '%s'", value); + log_err(-1, __func__, log_buffer); + return(0); + } + + return(1); + } + + + +unsigned long set_get_cray_taskstats( + + const char *value) + + { + int enable; + + if ((enable = setbool(value)) != -1) + { + get_cray_taskstats = (bool)enable; + return(1); + } + + return(0); /* error */ + } + + unsigned long setidealload( const char *value) @@ -625,8 +706,6 @@ unsigned long setautoidealload( - - unsigned long setallocparcmd( const char *value) /* I */ @@ -641,8 +720,6 @@ unsigned long setallocparcmd( - - unsigned long setautomaxload( const char *value) @@ -747,6 +824,46 @@ unsigned long setnodecheckinterval( +unsigned long set_node_check_on_job_start( + + const char *value) + + { + int enable; + + log_record(PBSEVENT_SYSTEM, PBS_EVENTCLASS_SERVER, __func__, value); + + if ((enable = setbool(value)) != -1) + { + PBSNodeCheckProlog = enable; + return(1); + } + + return(0); + } /* END set_node_check_on_job_start() */ + + + +unsigned long set_node_check_on_job_end( + + const char *value) + + { + int enable; + + log_record(PBSEVENT_SYSTEM, PBS_EVENTCLASS_SERVER, __func__, value); + + if ((enable = setbool(value)) != -1) + { + PBSNodeCheckEpilog = enable; + return(1); + } + + return(0); + } /* END set_node_check_on_job_end() */ + + + unsigned long settimeout( const char *value) @@ -789,8 +906,6 @@ unsigned long setmaxload( - - unsigned long setlogfilemaxsize( const char *value) /* I */ @@ -810,7 +925,6 @@ unsigned long setlogfilemaxsize( - unsigned long setlogfilerolldepth( const char *value) /* I */ @@ -842,7 +956,6 @@ unsigned long setlogdirectory( - unsigned long setlogfilesuffix( const char *value) /* I */ @@ -923,7 +1036,7 @@ u_long setvarattr( ptr = value; pva->va_ttl = strtol(ptr, NULL, 10); - + varattr_tv = pva->va_ttl; /* step forward to end of TTL */ while ((!isspace(*ptr)) && @@ -1446,19 +1559,22 @@ u_long setpbsclient( { /* FAILURE */ - return(1); + return(0); } rc = addclient(value); - if (rc != 0) + // addclient() returns the address on success, 0 on failure + if (rc == 0) { /* FAILURE */ - return(1); + return(0); } - return(0); + pbsclient = rc; + + return(1); } /* END setpbsclient() */ @@ -2287,7 +2403,11 @@ void reset_config_vars() memset(MOMConfigVersion, 0, sizeof(MOMConfigVersion)); MOMConfigDownOnError = 0; MOMConfigRestart = 0; +#ifdef PENABLE_LINUX_CGROUPS + MOMCudaVisibleDevices = 0; +#else MOMCudaVisibleDevices = 1; +#endif wallfactor = 1.00; pcphosts.clear(); pe_alarm_time = PBS_PROLOG_TIME; @@ -3225,7 +3345,6 @@ const char *reqgres( strncat(GResBuf, tmpLine, (sizeof(GResBuf) - strlen(GResBuf) - 1)); } /* END for (cp) */ - return(GResBuf); } /* END reqgres() */ diff --git a/src/resmom/pbs_demux.c b/src/resmom/pbs_demux.c index dece02f257..e3de06577e 100644 --- a/src/resmom/pbs_demux.c +++ b/src/resmom/pbs_demux.c @@ -95,13 +95,11 @@ #include #include #include +#include #ifdef ENABLE_BLCR #include #endif /* ENABLE_BLCR */ -#if defined(FD_SET_IN_SYS_SELECT_H) -# include -#endif #include "lib_ifl.h" #include "pbs_helper.h" @@ -114,7 +112,7 @@ struct routem short r_nl; }; -fd_set readset; +struct pollfd *readset = NULL; void readit( @@ -129,6 +127,10 @@ void readit( int i; char *pc; + // confirm readset has been initialized + if (readset == NULL) + return; + if (prm->r_where == old_out) fil = stdout; else @@ -169,7 +171,10 @@ void readit( prm->r_where = invalid; - FD_CLR(sock, &readset); + // remove socket from readset + readset[sock].fd = -1; + readset[sock].events = 0; + readset[sock].revents = 0; } return; @@ -193,7 +198,6 @@ int main( char *argv[]) { - struct timeval timeout; int i; int maxfd; int main_sock_out = 3; @@ -201,7 +205,9 @@ int main( int n; int newsock; pid_t parent; - fd_set selset; + + struct pollfd *pollset; + int pollset_size_bytes; struct routem *routem; @@ -247,7 +253,7 @@ int main( if (routem == NULL) { - perror("cannot alloc memory"); + perror("cannot alloc memory for routem"); exit(5); } @@ -261,9 +267,38 @@ int main( routem[main_sock_out].r_where = new_out; routem[main_sock_err].r_where = new_err; - FD_ZERO(&readset); - FD_SET(main_sock_out, &readset); - FD_SET(main_sock_err, &readset); + pollset_size_bytes = maxfd * sizeof(struct pollfd); + + readset = (struct pollfd *)malloc(maxfd * sizeof(struct pollfd)); + if (readset == NULL) + { + perror("cannot alloc memory for readset"); + + exit(5); + } + + // set initial values + for (i = 0; i < maxfd; i++) + { + readset[i].fd = -1; + readset[i].events = 0; + readset[i].revents = 0; + } + + readset[main_sock_out].fd = main_sock_out; + readset[main_sock_out].events = POLLIN; + + readset[main_sock_err].fd = main_sock_err; + readset[main_sock_err].events = POLLIN; + + // allocate local pollset to hold a copy of readset + pollset = (struct pollfd *)malloc(pollset_size_bytes); + if (pollset == NULL) + { + perror("cannot alloc memory for pollset"); + + exit(5); + } if (listen(main_sock_out, TORQUE_LISTENQUEUE) < 0) { @@ -281,11 +316,11 @@ int main( while (1) { - selset = readset; - timeout.tv_usec = 0; - timeout.tv_sec = 10; + // copy readset to local set for poll + memcpy(pollset, readset, pollset_size_bytes); - n = select(FD_SETSIZE, &selset, (fd_set *)0, (fd_set *)0, &timeout); + // wait for up to 10sec + n = poll(pollset, maxfd, 10000); if (n == -1) { @@ -295,7 +330,7 @@ int main( } else { - fprintf(stderr, "%s: select failed\n", + fprintf(stderr, "%s: poll failed\n", argv[0]); exit(1); @@ -316,12 +351,17 @@ int main( } } /* END else if (n == 0) */ - for (i = 0;(n != 0) && (i < maxfd);++i) + for (i = 0; (n > 0) && (i < maxfd); i++) { - if (FD_ISSET(i, &selset)) + if (pollset[i].revents == 0) + continue; + + // decrement count of structures with non-zero return events + n--; + + if ((pollset[i].revents & POLLIN)) { /* this socket has data */ - n--; switch ((routem + i)->r_where) { @@ -336,7 +376,10 @@ int main( old_out : old_err; - FD_SET(newsock, &readset); + // add new socket to readset for future polling + readset[newsock].fd = newsock; + readset[newsock].events = POLLIN; + readset[newsock].revents = 0; break; diff --git a/src/resmom/prolog.c b/src/resmom/prolog.c index 63a7424477..6afc41d275 100644 --- a/src/resmom/prolog.c +++ b/src/resmom/prolog.c @@ -1183,7 +1183,7 @@ void prepare_and_run_pelog_as_child( setuid_ext(pbsuser, TRUE); setegid(pbsgroup); - if (become_the_user(pjob) != PBSE_NONE) + if (become_the_user(pjob, false) != PBSE_NONE) { exit(-1); } diff --git a/src/resmom/requests.c b/src/resmom/requests.c index 01aaea402c..a22e01a37a 100644 --- a/src/resmom/requests.c +++ b/src/resmom/requests.c @@ -622,7 +622,7 @@ int return_file( return(0); } - prq = alloc_br(PBS_BATCH_MvJobFile); + prq = new batch_request(PBS_BATCH_MvJobFile); if (prq == NULL) { @@ -1271,7 +1271,7 @@ int message_job( } /* only the child reaches here, become the user for root-squashing as well */ - if (become_the_user(pjob) != PBSE_NONE) + if (become_the_user(pjob, false) != PBSE_NONE) { /* log_buffer is populated by become_the_user */ log_err(errno, __func__, log_buffer); @@ -4333,7 +4333,7 @@ batch_request *initialize_stageout_request( job *pjob) { - batch_request *preq = alloc_br(PBS_BATCH_CopyFiles); + batch_request *preq = new batch_request(PBS_BATCH_CopyFiles); struct rq_cpyfile *pcf = &preq->rq_ind.rq_cpyfile; pcf->rq_dir = STAGE_DIR_OUT; @@ -4397,6 +4397,15 @@ batch_request *get_std_file_info( copy_stdout = false; } + // if files are to be kept, don't copy since they are already at their destination + if (pjob->ji_wattr[JOB_ATR_keep].at_val.at_str != NULL) + { + if (strstr(pjob->ji_wattr[JOB_ATR_keep].at_val.at_str, "o")) + copy_stdout = false; + if (strstr(pjob->ji_wattr[JOB_ATR_keep].at_val.at_str, "e")) + copy_stderr = false; + } + preq = initialize_stageout_request(pjob); struct rq_cpyfile *pcf = &preq->rq_ind.rq_cpyfile; diff --git a/src/resmom/start_exec.c b/src/resmom/start_exec.c index 7ef76a7111..a376a49059 100644 --- a/src/resmom/start_exec.c +++ b/src/resmom/start_exec.c @@ -88,12 +88,21 @@ extern "C" #include #include #include + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include + #include #include #include #include #include #include +#include +#include +#include #if IBM_SP2==2 /* IBM SP with PSSP 3.1 */ #include #endif /* IBM SP */ @@ -375,6 +384,7 @@ extern int use_nvidia_gpu; #endif /* NVIDIA_GPUS */ int exec_job_on_ms(job *pjob); +int remove_leading_hostname(char**); @@ -435,8 +445,8 @@ void no_hang( } /* END no_hang() */ - -bool check_pwd( +/* TODO: change -1 to actual PBSE error codes */ +int check_pwd( job *pjob) /* I (modified) */ @@ -458,7 +468,7 @@ bool check_pwd( sprintf(log_buffer, "no user specified for job"); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); - return(false); + return(-1); } /* we will retry if needed just to cover temporary problems */ @@ -482,14 +492,14 @@ bool check_pwd( ptr); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); - return(false); + return(-1); } #ifdef __CYGWIN__ if (IamUserByName(ptr) == 0) { free_pwnam(pwdp, pwd_buf); - return(false); + return(-1); #endif /* __CYGWIN__ */ if (pjob->ji_grpcache != NULL) @@ -498,7 +508,7 @@ bool check_pwd( /* group cache previously loaded and cached */ free_pwnam(pwdp, pwd_buf); - return(true); + return(PBSE_NONE); } pjob->ji_qs.ji_un_type = JOB_UNION_TYPE_MOM; @@ -514,7 +524,7 @@ bool check_pwd( log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); free_pwnam(pwdp, pwd_buf); - return(false); + return(-1); } strcpy(pjob->ji_grpcache->gc_homedir, pwdp->pw_dir); @@ -564,7 +574,7 @@ bool check_pwd( log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); free_pwnam(pwdp, pwd_buf); - return(false); + return(PBSE_BAD_GROUP); } } /* END if (grpp != NULL) */ } @@ -586,7 +596,7 @@ bool check_pwd( log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); free_pwnam(pwdp, pwd_buf); - return(false); + return(-1); } /* perform site specific check on validatity of account */ @@ -598,16 +608,17 @@ bool check_pwd( log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); free_pwnam(pwdp, pwd_buf); - return(false); + return(-1); } /* SUCCESS */ free_pwnam(pwdp, pwd_buf); - return(true); + return(PBSE_NONE); } /* END check_pwd() */ + /** * @see send_sisters() - child - send ABORT request to sisters * @see start_exec() - parent @@ -683,12 +694,14 @@ void exec_bail( * becomes the user for pjob * * @param pjob - the job whose user we should become + * @param want_effective - whether or not to set euid/eguid rather than uid/gid * @return PBSE_BADUSER on failure */ int become_the_user( - job *pjob) + job *pjob, + bool want_effective) { log_buffer[0] = '\0'; @@ -701,20 +714,41 @@ int become_the_user( (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, strerror(errno)); } - else if (setgid(pjob->ji_qs.ji_un.ji_momt.ji_exgid) != PBSE_NONE) + else if (want_effective) { - snprintf(log_buffer,sizeof(log_buffer), - "PBS: setgid to %lu for UID = %lu failed: %s\n", - (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exgid, - (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, - strerror(errno)); + if (setegid(pjob->ji_qs.ji_un.ji_momt.ji_exgid) != PBSE_NONE) + { + snprintf(log_buffer,sizeof(log_buffer), + "PBS: setegid to %lu for UID = %lu failed: %s\n", + (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exgid, + (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, + strerror(errno)); + } + else if (setuid_ext(pjob->ji_qs.ji_un.ji_momt.ji_exuid, TRUE) < 0) + { + snprintf(log_buffer,sizeof(log_buffer), + "PBS: seteuid to %lu failed: %s\n", + (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, + strerror(errno)); + } } - else if (setuid_ext(pjob->ji_qs.ji_un.ji_momt.ji_exuid, FALSE) < 0) + else { - snprintf(log_buffer,sizeof(log_buffer), - "PBS: setuid to %lu failed: %s\n", - (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, - strerror(errno)); + if (setgid(pjob->ji_qs.ji_un.ji_momt.ji_exgid) != PBSE_NONE) + { + snprintf(log_buffer,sizeof(log_buffer), + "PBS: setgid to %lu for UID = %lu failed: %s\n", + (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exgid, + (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, + strerror(errno)); + } + else if (setuid_ext(pjob->ji_qs.ji_un.ji_momt.ji_exuid, FALSE) < 0) + { + snprintf(log_buffer,sizeof(log_buffer), + "PBS: setuid to %lu failed: %s\n", + (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, + strerror(errno)); + } } if (log_buffer[0] != '\0') @@ -734,7 +768,7 @@ int become_the_user_sjr( struct startjob_rtn *sjr) { - if (become_the_user(pjob) != PBSE_NONE) + if (become_the_user(pjob, false) != PBSE_NONE) { if (write_ac_socket(2, log_buffer, strlen(log_buffer)) == -1) { @@ -773,6 +807,10 @@ int open_demux( torque_socklen_t slen; unsigned short local_port; +#ifdef BIND_OUTBOUND_SOCKETS + struct sockaddr_in local; +#endif + memset(&remote, 0, sizeof(remote)); remote.sin_addr.s_addr = addr; remote.sin_port = htons((unsigned short)port); @@ -789,6 +827,34 @@ int open_demux( return(-1); } +#ifdef BIND_OUTBOUND_SOCKETS + /* Bind to the IP address associated with the hostname, in case there are + * muliple possible source IPs for this destination.*/ + + if (get_local_address(local) != PBSE_NONE) + { + sprintf(log_buffer, "could not determine local IP address: %s", strerror(errno)); + log_err(errno, __func__, log_buffer); + + close(sock); + return(-1); + } + + // do not bind if lochost addr + if (!islocalhostaddr(&local)) + { + if (bind(sock, (struct sockaddr *)&local, sizeof(local))) + { + sprintf(log_buffer, "could not bind local socket: %s", strerror(errno)); + log_err(errno, __func__, log_buffer); + + close(sock); + return(-1); + } + } + +#endif + for (i = 0;i < RETRY;i++) { if (connect(sock, (struct sockaddr *)&remote, sizeof(remote)) == 0) @@ -2127,7 +2193,71 @@ struct radix_buf **allocate_sister_list( } /* END allocate_sister_list() */ +/* + * update the job outpath/errpath attribute with this hostname if needed + * + * @param pjob - the job to check + * @param attr_index - must be JOB_ATR_outpath or JOB_ATR_errpath + * @return -1 on failure + */ + +int update_path_attribute( + + job *pjob, + job_atr attr_index) + + { + char outchar; + + if (pjob == NULL) + return(-1); + + if (attr_index == JOB_ATR_outpath) + outchar = 'o'; + else if (attr_index == JOB_ATR_errpath) + outchar = 'e'; + else + return(-1); + + // if keeping output on this host, update the hostname + if ((pjob->ji_wattr[JOB_ATR_keep].at_flags & ATR_VFLAG_SET) && + (strchr(pjob->ji_wattr[JOB_ATR_keep].at_val.at_str, outchar)) && + (spoolasfinalname == FALSE)) + { + char *p; + std::string full_path; + pbs_attribute *pattr; + // get the current path (may include leading hostname) + p = pjob->ji_wattr[attr_index].at_val.at_str; + + // get just the file path + remove_leading_hostname(&p); + + // build new host:path string + full_path = mom_host; + full_path += ":"; + full_path += p; + + // update the attribute + + pattr = &pjob->ji_wattr[attr_index]; + + job_attr_def[attr_index].at_free(pattr); + + job_attr_def[attr_index].at_decode( + pattr, + NULL, + NULL, + full_path.c_str(), + 0); + + pjob->ji_wattr[attr_index].at_flags = + (ATR_VFLAG_SET | ATR_VFLAG_MODIFY | ATR_VFLAG_SEND); + } + + return(0); + } /* * Used by MOM superior to start the shell process. @@ -2248,7 +2378,7 @@ int TMomFinalizeJob1( * we do this now to save a few things in the job structure */ - if (check_pwd(pjob) == false) + if (check_pwd(pjob) != PBSE_NONE) { log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buffer); @@ -2454,6 +2584,12 @@ int TMomFinalizeJob1( pjob->ji_wattr[JOB_ATR_errpath].at_flags = (ATR_VFLAG_SET | ATR_VFLAG_MODIFY | ATR_VFLAG_SEND); } /* END if (TJE->is_interactive == TRUE) */ + else + { + // update the hostname in the out/err path attributes if needed + update_path_attribute(pjob, JOB_ATR_outpath); + update_path_attribute(pjob, JOB_ATR_errpath); + } #if SHELL_USE_ARGV == 0 #if SHELL_INVOKE == 1 @@ -3206,6 +3342,9 @@ void handle_cpuset_creation( +/* + * handle_reservation() + */ void handle_reservation( @@ -3326,6 +3465,14 @@ void handle_reservation( +/* + * handle_prologs() + * + * Run relevant prologue scripts and notify the mom of a job failure if one of the prologues + * fail. + * + */ + void handle_prologs( job *pjob, @@ -3344,7 +3491,7 @@ void handle_prologs( if ((TJE->is_interactive == FALSE) && (rc != 1)) { - starter_return(TJE->upfds, TJE->downfds, JOB_EXEC_RETRY, sjr); + starter_return(TJE->upfds, TJE->downfds, JOB_EXEC_RETRY_PROLOGUE, sjr); } else { @@ -3363,7 +3510,7 @@ void handle_prologs( if ((TJE->is_interactive == FALSE) && (rc != 1)) { - starter_return(TJE->upfds, TJE->downfds, JOB_EXEC_RETRY, sjr); + starter_return(TJE->upfds, TJE->downfds, JOB_EXEC_RETRY_PROLOGUE, sjr); } else { @@ -3395,7 +3542,7 @@ void handle_prologs( if ((TJE->is_interactive == FALSE) && (rc != 1)) { - starter_return(TJE->upfds, TJE->downfds, JOB_EXEC_RETRY, sjr); + starter_return(TJE->upfds, TJE->downfds, JOB_EXEC_RETRY_PROLOGUE, sjr); } else { @@ -3411,11 +3558,17 @@ void handle_prologs( } } - } /* END handle_prologs() */ + pjob->ji_qs.ji_svrflags |= JOB_SVFLG_PROLOGUES_RAN; + } /* END handle_prologs() */ +/* + * start_interactive_session() + * + * Opens the sockets to talk to the waiting qsub command + */ int start_interactive_session( @@ -3553,6 +3706,9 @@ int start_interactive_session( +/* + * start_interactive_reader() + */ void start_interactive_reader( @@ -3583,6 +3739,9 @@ void start_interactive_reader( +/* + * setup_interacteractive_job() + */ void setup_interactive_job( @@ -4710,15 +4869,16 @@ int TMomFinalizeChild( // Create the cgroups for this job rc = init_torque_cgroups(); - if (rc == PBSE_NONE) { - rc = trq_cg_create_all_cgroups(pjob); - } else { - /* - Send/log an additional error, cleanup and return as before - */ + if (rc == PBSE_NONE) + { + rc = trq_cg_create_all_cgroups(pjob); + } + else + { + // Log an additional error, cleanup and return as before sprintf(log_buffer, "Could not init cgroups for job %s.", pjob->ji_qs.ji_jobid); log_ext(-1, __func__, log_buffer, LOG_ERR); - } + } if (rc != PBSE_NONE) { @@ -5436,10 +5596,10 @@ int setup_process_launch_pipes( (kid_write < 0)) { log_err(-1, __func__, "Couldn't set up pipes to monitor launching a process"); - + return(-1); } - + return(PBSE_NONE); } // END setup_process_launch_pipes() @@ -6122,7 +6282,7 @@ int start_process( * to spawn tasks (ji_grpcache). */ - if (!check_pwd(pjob)) + if (check_pwd(pjob) != PBSE_NONE) { log_err(-1, __func__, log_buffer); @@ -7269,6 +7429,8 @@ int start_exec( execute_presetup_prologue(pjob); } + pjob->ji_qs.ji_svrflags |= JOB_SVFLG_PROLOGUES_RAN; + /* start_exec only executed on mother superior */ pjob->ji_nodeid = 0; /* I'm MS */ nodenum = pjob->ji_numnodes; @@ -7276,7 +7438,7 @@ int start_exec( /* Step 3.0 Validate/Initialize Environment */ /* check creds early because we need the uid/gid for TMakeTmpDir() */ - if (check_pwd(pjob) == false) + if (check_pwd(pjob) != PBSE_NONE) { sprintf(log_buffer, "bad credentials: job id %s", pjob->ji_qs.ji_jobid); log_err(-1, __func__, log_buffer); @@ -7784,6 +7946,90 @@ int remove_leading_hostname( } /* END remove_leading_hostname() */ +#if NO_SPOOL_OUTPUT == 1 +/* + * save_supplementary_group_list + * + * saves the supplementary group list for calling process + * into a buffer + * + * see restore_supplementary_group_list + * + * Note: it is up to the caller to free the allocated space + * malloc'ed in this function. This is normally done in + * restore_supplementary_group_list. + * + * @param *grp_list_size - pointer to size of group list + * @param **grp_list - pointer to address of group list + * @return 0 if group list saved to buffer, -1 otherwise + */ + +int save_supplementary_group_list( + + int *grp_list_size, /* O */ + gid_t **grp_list) /* O */ + + { + int ngroups_expected; + + // get expected number of groups + ngroups_expected = getgroups(0, NULL); + if (ngroups_expected < 0) + return(-1); + + // allocate space for group list + *grp_list = (gid_t *)malloc(ngroups_expected * sizeof(gid_t)); + if (*grp_list == NULL) + return(-1); + + // get the group list + *grp_list_size = getgroups(ngroups_expected, *grp_list); + if (*grp_list_size != ngroups_expected) + { + free(*grp_list); + *grp_list = NULL; + return(-1); + } + + // success + return(0); + } + + +/* + * restore_supplementary_group_list + * + * restores the supplementary group list for calling process + * from a buffer and frees the buffer space + * + * see save_supplementary_group_list + * + * @param grp_list_size - size of group list + * @param *grp_list - pointer to saved group list + * @return 0 if group list restored from buffer, -1 otherwise + */ + +int restore_supplementary_group_list( + + int grp_list_size, /* I */ + gid_t *grp_list) /* I */ + + { + int rc = -1; + + if (grp_list == NULL) + return(rc); + + // restore the supplementary group list + if (setgroups((size_t)grp_list_size, grp_list) == 0) + rc = 0; + + // free the buffer space + free(grp_list); + + return(rc); + } +#endif // NO_SPOOL_OUTPUT /* @@ -8020,6 +8266,9 @@ char *std_file_name( if (spoolasfinalname == FALSE) { + gid_t *grp_list_saved; + int grp_list_saved_size; + /* force all output to user's HOME */ snprintf(path, sizeof(path), "%s", pjob->ji_grpcache->gc_homedir); @@ -8027,14 +8276,67 @@ char *std_file_name( /* if it's not a directory, just use $HOME us usual */ snprintf(path_alt, sizeof(path_alt), "%s/.pbs_spool/", path); - if (setuid_ext(pjob->ji_qs.ji_un.ji_momt.ji_exuid, TRUE) == -1) + // For network file systems like NFS and GPFS and regardless of the root + // squashing mode (enabled or disabled) the user's supplementary group + // list must be set for the caller before doing the stat() call below. + + // save supplementary groups + if (save_supplementary_group_list(&grp_list_saved_size, &grp_list_saved) < 0) + { + snprintf(log_buffer,sizeof(log_buffer), + "failed to save supplementary groups"); + log_err(errno, __func__, log_buffer); + + return(NULL); + } + + // set effective uid, gid + if (become_the_user(pjob, true) != PBSE_NONE) { + restore_supplementary_group_list(grp_list_saved_size, grp_list_saved); + setegid(pbsgroup); + + log_err(errno, __func__, log_buffer); return(NULL); } + // now do the stat() rcstat = stat(path_alt, &myspooldir); - setuid_ext(pbsuser, TRUE); + // restore the euid + if (setuid_ext(pbsuser, TRUE) == -1) + { + restore_supplementary_group_list(grp_list_saved_size, grp_list_saved); + setegid(pbsgroup); + + snprintf(log_buffer,sizeof(log_buffer), + "failed to restore the euid"); + log_err(errno, __func__, log_buffer); + + return(NULL); + } + + // restore supplementary groups + if (restore_supplementary_group_list(grp_list_saved_size, grp_list_saved) < 0) + { + setegid(pbsgroup); + + snprintf(log_buffer,sizeof(log_buffer), + "failed to restore supplementary groups"); + log_err(errno, __func__, log_buffer); + + return(NULL); + } + + // restore the egid + if (setegid(pbsgroup) != 0) + { + snprintf(log_buffer,sizeof(log_buffer), + "failed to restore the egid"); + log_err(errno, __func__, log_buffer); + + return(NULL); + } if ((rcstat == 0) && (S_ISDIR(myspooldir.st_mode))) snprintf(path, sizeof(path), "%s", path_alt); @@ -8204,36 +8506,8 @@ int open_std_file( (geteuid() != pjob->ji_qs.ji_un.ji_momt.ji_exuid)) #endif { - if (setgroups(pjob->ji_grpcache->gc_ngroup,(gid_t *)pjob->ji_grpcache->gc_groups) != PBSE_NONE) + if (become_the_user(pjob, true) != PBSE_NONE) { - snprintf(log_buffer,sizeof(log_buffer), - "setgroups failed for UID = %lu, error: %s\n", - (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, - strerror(errno)); - - log_err(errno, __func__, log_buffer); - } - - if (setegid(pjob->ji_qs.ji_un.ji_momt.ji_exgid) != PBSE_NONE) - { - snprintf(log_buffer, sizeof(log_buffer), - "setegid(%lu) failed for UID = %lu, error: %s\n", - (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exgid, - (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, - strerror(errno)); - - log_err(errno, __func__, log_buffer); - - return(-1); - } - - if (setuid_ext(pjob->ji_qs.ji_un.ji_momt.ji_exuid, TRUE) != PBSE_NONE) - { - snprintf(log_buffer, sizeof(log_buffer), - "seteuid(%lu) failed, error: %s\n", - (unsigned long)pjob->ji_qs.ji_un.ji_momt.ji_exuid, - strerror(errno)); - log_err(errno, __func__, log_buffer); setegid(pbsgroup); @@ -9076,11 +9350,11 @@ int TMomCheckJobChild( int *RC) /* O (return code/errno) */ { - int i; - fd_set fdset; - int rc; - int read_size = sizeof(struct startjob_rtn); - struct timeval timeout; + int i; + struct pollfd PollArray; + struct timespec ts; + int rc; + int read_size = sizeof(struct startjob_rtn); /* NOTE: assume if anything is on pipe, everything is on pipe (may reasult in hang) */ @@ -9090,24 +9364,20 @@ int TMomCheckJobChild( /* read returns the session id or error */ - timeout.tv_sec = Timeout; - timeout.tv_usec = 0; errno = 0; - FD_ZERO(&fdset); + PollArray.fd = TJE->jsmpipe[0]; + PollArray.events = POLLIN; + PollArray.revents = 0; - FD_SET(TJE->jsmpipe[0], &fdset); + ts.tv_sec = Timeout; + ts.tv_nsec = 0; - rc = select( - TJE->jsmpipe[0] + 1, - &fdset, - (fd_set *)NULL, - (fd_set *)NULL, - &timeout); + rc = ppoll(&PollArray, 1, &ts, NULL); - if (rc <= 0) + if ((rc <= 0) || ((PollArray.revents & POLLIN) == 0)) { - /* TIMEOUT - data not yet available */ + /* error, timeout or no data not yet available */ return(FAILURE); } @@ -9637,10 +9907,31 @@ void add_wkm_end( return; } /* END add_wkm_end() */ +#endif /* ENABLE_CSA */ -#endif /* ENABLE_CSA */ +void escape_spaces( + + const char *str, + std::string &escaped) + + { + if (str != NULL) + { + const char *ptr = str; + + while (*ptr != '\0') + { + if (*ptr == ' ') + escaped += '\\'; + + escaped += *ptr++; + } + } + } // escape_spaces() + + /* * @param pjob - used to set up the user's environment if desired @@ -9693,9 +9984,11 @@ int expand_path( environ = vtable.v_envp; } + std::string escaped; + escape_spaces(path_in, escaped); /* expand the path */ - switch (wordexp(path_in, &exp, WRDE_NOCMD | WRDE_UNDEF)) + switch (wordexp(escaped.c_str(), &exp, WRDE_NOCMD | WRDE_UNDEF)) { case 0: diff --git a/src/resmom/start_exec.h b/src/resmom/start_exec.h index f8934372d3..a35950e1d1 100644 --- a/src/resmom/start_exec.h +++ b/src/resmom/start_exec.h @@ -21,8 +21,6 @@ enum csa_chk_cmd #define B_THRESHOLD 2048 #define EXTRA_VARIABLE_SPACE 5120 -bool check_pwd(job *pjob); - int open_demux(u_long addr, int port); int is_joined(job *pjob); diff --git a/src/resmom/trq_cgroups.c b/src/resmom/trq_cgroups.c index ba50b06040..a7a6eae445 100644 --- a/src/resmom/trq_cgroups.c +++ b/src/resmom/trq_cgroups.c @@ -64,6 +64,7 @@ const int MAX_WRITE_RETRIES = 5; #ifdef PENABLE_LINUX_CGROUPS extern Machine this_node; +std::set character_device_ids; /* This array tracks if all of the hierarchies are mounted we need to run our control groups */ @@ -777,6 +778,70 @@ int trq_cg_add_process_to_task_cgroup( +/* + * trq_cg_read_numeric_rss() + * + * Reads and returns the rss value from a memory cgroup stat file. + * @param path - the path to the memory cgroup stat file. + * @param error (O) - set to true if there was an error reading the file. + * @return the RSS memory size recorded in this file, or 0 if none could be read. + */ + +unsigned long long trq_cg_read_numeric_rss( + + string &path, + bool &error) + + { + int fd; + unsigned long long val = 0; + char buf[LOCAL_LOG_BUF_SIZE]; + char *buf2; + + error = false; + + fd = open(path.c_str(), O_RDONLY); + if (fd <= 0) + { + // If we don't have a file to open, just return 0 + if (errno == ENOENT) + return(0); + + sprintf(log_buffer, "failed to open %s: %s", path.c_str(), strerror(errno)); + log_err(errno, __func__, log_buffer); + error = true; + } + else + { + int rc = read(fd, buf, LOCAL_LOG_BUF_SIZE); + + close(fd); + + if (rc == -1) + { + sprintf(buf, "read failed getting value from %s - %s", path.c_str(), strerror(errno)); + log_err(errno, __func__, buf); + error = true; + } + else if (rc != 0) + { + buf2=strstr(buf, "\nrss "); + if (buf2 == NULL) + { + sprintf(buf, "read failed finding rss %s", path.c_str()); + log_err(errno, __func__, buf); + error = true; + } + else + val = strtoull(buf2+5, NULL, 10); + } + } + + return(val); + } // END trq_cg_read_numeric_rss() + + + unsigned long long trq_cg_read_numeric_value( string &path, @@ -849,21 +914,17 @@ int trq_cg_get_task_memory_stats( char req_and_task[256]; int rc = PBSE_NONE; - /* get memory first */ - sprintf(req_and_task, "/%s/R%u.t%u/memory.max_usage_in_bytes", job_id, req_index, task_index); + // get memory first + // We read memory.stat instead of memory.max_usage_in_bytes because the latter counts way more than just + // rss memory. trq_cg_read_numeric_rss() gets us just the resident memory size, which is what we really + // want to limit. + sprintf(req_and_task, "/%s/R%u.t%u/memory.stat", job_id, req_index, task_index); string cgroup_path = cg_memory_path + req_and_task; bool error; - mem_used = trq_cg_read_numeric_value(cgroup_path, error); + mem_used = trq_cg_read_numeric_rss(cgroup_path, error); -/* if (mem_used == 0) - { - sprintf(log_buffer, "peak memory usage read from '%s' is 0, something appears to be incorrect.", - cgroup_path.c_str()); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buffer); - }*/ - if (error) rc = PBSE_SYSTEM; @@ -1880,6 +1941,69 @@ void trq_cg_delete_cgroup_path( } // END trq_cg_delete_cgroup_path() +/* + * trq_cg_signal_tasks() + * + * cgroup_path - the path to the cgroup whose tasks should be signaled + * signal - signal to send to each task in the cgroup + */ + +void trq_cg_signal_tasks( + + const string &cgroup_path, + int signal) + + { + std::string tasks_path(cgroup_path); + int npids; + int slept = 0; + FILE *fp; + char tid_str[1024]; + char log_buf[LOCAL_LOG_BUF_SIZE]; + struct stat statbuf; + + // build path to tasks file + tasks_path += "/tasks"; + + // do not continue if tasks file doesn't exist + if ((lstat(tasks_path.c_str(), &statbuf) != 0) || (!S_ISREG(statbuf.st_mode))) + return; + + do + { + npids = 0; + + // Signal each pid. If it takes more than 5 seconds to kill, give up. + + if ((fp = fopen(tasks_path.c_str(), "r")) != NULL) + { + while ((fgets(tid_str, sizeof(tid_str), fp)) != NULL) + { + int tid = atoi(tid_str); + + if (LOGLEVEL >= 9) + { + snprintf(log_buf, sizeof(log_buf), "sending signal %d to pid %d in cpuset %s", + signal, tid, cgroup_path.c_str()); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); + } + + kill(tid, signal); + npids++; + } + + fclose(fp); + } + + if ((signal == SIGKILL) && (npids)) + { + sleep(1); + slept++; + } + } while((signal == SIGKILL) && (npids > 0) && (slept <= 5)); + + } // END trq_cg_signal_tasks() + /* * trq_cg_delete_job_cgroups() @@ -1898,7 +2022,11 @@ void trq_cg_delete_job_cgroups( trq_cg_delete_cgroup_path(cg_cpu_path + job_id, successfully_created); trq_cg_delete_cgroup_path(cg_cpuacct_path + job_id, successfully_created); - + + // kill any procs in cpuset + trq_cg_signal_tasks(cg_cpuset_path + job_id, SIGKILL); + + // remove directory trq_cg_delete_cgroup_path(cg_cpuset_path + job_id, successfully_created); trq_cg_delete_cgroup_path(cg_memory_path + job_id, successfully_created); @@ -1907,61 +2035,6 @@ void trq_cg_delete_job_cgroups( } // END trq_cg_delete_job_cgroups() -/* - * trq_cg_set_forbidden_devices - * - * Add the indices in the forbidden list to the devices.deny file - * - * @param - forbidden_devices - gpus that are not to be used - * by this job - * @param - job_devices_path - path to devices directory for the job - * - */ - -int trq_cg_set_forbidden_devices(std::vector &forbidden_devices, std::string job_devices_path) - { - int rc; - char log_buf[LOCAL_LOG_BUF_SIZE]; - std::string devices_deny; - FILE *f; - - devices_deny = job_devices_path + '/' + "devices.deny"; - - rc = trq_cg_mkdir(job_devices_path.c_str(), S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); - if ((rc != 0) && - (errno != EEXIST)) - { - sprintf(log_buf, "failed to make directory %s for cgroup: %s", job_devices_path.c_str(), strerror(errno)); - log_err(errno, __func__, log_buf); - return(PBSE_SYSTEM); - } - - for (std::vector::iterator it = forbidden_devices.begin(); it != forbidden_devices.end(); it++) - { - char restricted_gpu[1024]; - -#ifdef NVIDIA_GPUS - sprintf(restricted_gpu, "echo \"c 195:%d rwm\" > %s", *it, devices_deny.c_str()); -#endif - -#ifdef MIC - sprintf(restricted_gpu, "echo \"c 245:%d rwm\" > %s", *it, devices_deny.c_str()); -#endif - - f = popen(restricted_gpu, "r"); - if (f == NULL) - { - sprintf(log_buf, "Failed to add restricted gpu to devices.deny list: index %d in %s", *it, job_devices_path.c_str()); - log_err(errno, __func__, log_buf); - return(PBSE_SYSTEM); - } - - pclose(f); - } - - return(PBSE_NONE); - } - /* * trq_cg_add_devices_to_cgroups @@ -1979,26 +2052,16 @@ int trq_cg_add_devices_to_cgroup( job *pjob) { - std::vector device_indices; + std::vector device_indices; std::vector forbidden_devices; std::string job_devices_path; - char log_buf[LOCAL_LOG_BUF_SIZE]; int rc = PBSE_NONE; - char *device_str; - char suffix[20]; unsigned int device_count = 0; - int index = 0; #ifdef NVIDIA_GPUS device_count = global_gpu_count; - strcpy(suffix, "-gpu"); - index = JOB_ATR_exec_gpus; -#endif - -#ifdef MIC +#elif defined(MIC) device_count = global_mic_count; - strcpy(suffix, "-mic"); - index = JOB_ATR_exec_mics; #endif /* First make sure we have gpus */ @@ -2007,65 +2070,139 @@ int trq_cg_add_devices_to_cgroup( job_devices_path = cg_devices_path + pjob->ji_qs.ji_jobid; - - /* if there are no gpus given, deny all gpus to the job */ - if (((pjob->ji_wattr[index].at_flags & ATR_VFLAG_SET) == 0) || - (pjob->ji_wattr[index].at_val.at_str == NULL)) +#ifdef NVIDIA_GPUS + if (pjob->ji_wattr[JOB_ATR_gpus_reserved].at_val.at_str != NULL) { - for (unsigned int i = 0; i < device_count; i++) - forbidden_devices.push_back(i); - - rc = trq_cg_set_forbidden_devices(forbidden_devices, job_devices_path); - if (rc != PBSE_NONE) - { - sprintf(log_buf, "Failed to write devices.deny list for job %s", pjob->ji_qs.ji_jobid); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); - return(rc); - } - - return(rc); + std::string gpus_reserved(pjob->ji_wattr[JOB_ATR_gpus_reserved].at_val.at_str); + std::string gpu_range; + find_range_in_cpuset_string(gpus_reserved, gpu_range); + translate_range_string_to_vector(gpu_range.c_str(), device_indices); } +#else // We aren't here unless either NVIDIA or MICS are defined + if (pjob->ji_wattr[JOB_ATR_mics_reserved].at_val.at_str != NULL) + { + std::string mics_reserved(pjob->ji_wattr[JOB_ATR_mics_reserved].at_val.at_str); + std::string mic_range; + find_range_in_cpuset_string(mics_reserved, mic_range); + translate_range_string_to_vector(mic_range.c_str(), device_indices); + } +#endif + // device_indices has the list of either gpus or mics for this job + std::string disallow_all("a *:* rwm"); + std::string allow_all_block("b *:* rwm"); - device_str = pjob->ji_wattr[index].at_val.at_str; - - device_indices.clear(); - get_device_indices(device_str, device_indices, suffix); + std::string allowed = job_devices_path + "/devices.allow"; + std::string denied = job_devices_path + "/devices.deny"; + std::string error; + rc = write_to_file(denied.c_str(), disallow_all, error); - /* device_indices has the list of gpus for this job. - make a list of gpus that are not for this job - using that list */ - for (unsigned int i = 0; i < device_count; i++) + if (rc == PBSE_NONE) { - bool found; - - found = false; - for(std::vector::iterator it = device_indices.begin(); it != device_indices.end(); it++) + if ((rc = write_to_file(allowed.c_str(), allow_all_block, error)) != PBSE_NONE) + log_err(errno, __func__, error.c_str()); + else { - if (*it == i) + char allow_device_buf[MAXLINE]; + + for (std::set::iterator it = character_device_ids.begin(); + it != character_device_ids.end(); + it++) { - found = true; - break; + sprintf(allow_device_buf, "c %d:* rwm", *it); + rc = write_to_file(allowed.c_str(), allow_device_buf, error); + if (rc != PBSE_NONE) + { + log_err(errno, __func__, error.c_str()); + break; + } + } + + if (rc == PBSE_NONE) + { + for (size_t i = 0; i < device_indices.size(); i++) + { + // Write the allow statement for this device +#ifdef NVIDIA_GPUS + sprintf(allow_device_buf, "c 195:%d rwm", device_indices[i]); +#else + sprintf(allow_device_buf, "c 245:%d rwm", device_indices[i]); +#endif + rc = write_to_file(allowed.c_str(), allow_device_buf, error); + if (rc != PBSE_NONE) + { + log_err(errno, __func__, error.c_str()); + break; + } + } + + if (rc == PBSE_NONE) + { + // Allow any other non-gpu / mic devices to be used. Things like nvidiactl must be available + // to all. + for (int i = device_count; i < 256; i++) + { +#ifdef NVIDIA_GPUS + sprintf(allow_device_buf, "c 195:%d rwm", i); +#else + sprintf(allow_device_buf, "c 245:%d rwm", i); +#endif + rc = write_to_file(allowed.c_str(), allow_device_buf, error); + if (rc != PBSE_NONE) + { + log_err(errno, __func__, error.c_str()); + break; + } + } + } } } - if (found == false) - forbidden_devices.push_back(i); + } + + return(rc); + } // END trq_cg_add_devices_to_cgroup() - } - if (forbidden_devices.size() == 0) + +int read_all_devices() + + { + int forbidden_device = 0; +#ifdef NVIDIA_GPUS + forbidden_device = 195; +#elif defined(MIC) + forbidden_device = 245; +#endif + + if (forbidden_device == 0) return(PBSE_NONE); - rc = trq_cg_set_forbidden_devices(forbidden_devices, job_devices_path); - if (rc != PBSE_NONE) +#ifdef NVIDIA_GPUS + // force the loading of nvidia-uvm + system("nvidia-modprobe -u -c=0"); +#endif + + std::ifstream infile("/proc/devices"); + std::string line; + + while (std::getline(infile, line)) { - sprintf(log_buf, " 2 Failed to write devices.deny list for job %s", pjob->ji_qs.ji_jobid); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); - return(rc); + if (line == "Character devices:") + continue; + else if (line == "Block devices:") + break; + + int device_index; + std::istringstream iss(line); + + if ((iss >> device_index) && + (device_index != forbidden_device)) + character_device_ids.insert(device_index); } - - return(rc); - } + + return(PBSE_NONE); + } // END read_all_devices() + #endif diff --git a/src/server/accounting.c b/src/server/accounting.c index ed9600d9f9..81d6a45d3e 100644 --- a/src/server/accounting.c +++ b/src/server/accounting.c @@ -217,6 +217,11 @@ int acct_job( sprintf(local_buf, "etime=%ld ", pjob->ji_wattr[JOB_ATR_etime].at_val.at_long); ds += local_buf; + + /* how many times the job has started */ + sprintf(local_buf, "start_count=%ld ", + pjob->ji_wattr[JOB_ATR_start_count].at_val.at_long); + ds += local_buf; /* execution start time */ sprintf(local_buf, "start=%ld ", @@ -274,6 +279,14 @@ int acct_job( free(pal); } /* END while (pal != NULL) */ + if ((pjob->ji_wattr[JOB_ATR_LRequest].at_val.at_str != NULL) && + ((pjob->ji_wattr[JOB_ATR_LRequest].at_flags & ATR_VFLAG_SET) != 0)) + { + ds += "Resource_Request_2.0="; + ds += pjob->ji_wattr[JOB_ATR_LRequest].at_val.at_str; + ds += " "; + } + #ifdef ATTR_X_ACCT /* x attributes */ diff --git a/src/server/array_func.c b/src/server/array_func.c index 5ddd36b99c..b27dcf03e0 100644 --- a/src/server/array_func.c +++ b/src/server/array_func.c @@ -61,9 +61,8 @@ extern int array_upgrade(job_array *, int, int, int *); -extern char *get_correct_jobname(const char *jobid); +extern const char *get_correct_jobname(const char *jobid, std::string &correct); extern int count_user_queued_jobs(pbs_queue *,char *); -extern void post_modify_arrayreq(batch_request *preq); /* global data items used */ @@ -96,25 +95,16 @@ int is_array( char *id) { - int rc = FALSE; + int rc = FALSE; - char *bracket_ptr; - char *end_bracket_ptr; - char *tmpjobid; - char jobid[PBS_MAXSVRJOBID]; - char temp_jobid[PBS_MAXSVRJOBID]; + char *bracket_ptr; + char *end_bracket_ptr; + char jobid[PBS_MAXSVRJOBID]; + char temp_jobid[PBS_MAXSVRJOBID]; + std::string tmpjobid; - tmpjobid = get_correct_jobname(id); - if (tmpjobid == NULL) - { - /* Maybe we should just return ENOMEM? */ - snprintf(jobid, sizeof(jobid), "%s", id); - } - else - { - snprintf(jobid, sizeof(jobid), "%s", tmpjobid); - free(tmpjobid); - } + get_correct_jobname(id, tmpjobid); + snprintf(jobid, sizeof(jobid), "%s", tmpjobid.c_str()); /* Check to see if we have an array dependency */ /* If there is an array dependency count then we will */ @@ -174,12 +164,10 @@ job_array *get_array( const char *id) { - job_array *pa; - char *tmpjobid; + job_array *pa; + std::string tmpjobid; - tmpjobid = get_correct_jobname(id); - if (tmpjobid == NULL) - return(NULL); + get_correct_jobname(id, tmpjobid); allarrays.lock(); @@ -190,8 +178,6 @@ job_array *get_array( allarrays.unlock(); - free(tmpjobid); - return(pa); } /* END get_array() */ @@ -212,12 +198,10 @@ job_array *get_and_remove_array( const char *id) { - job_array *pa = NULL; - char *tmpjobid; + job_array *pa = NULL; + std::string tmpjobid; - tmpjobid = get_correct_jobname(id); - if (tmpjobid == NULL) - return(NULL); + get_correct_jobname(id, tmpjobid); allarrays.lock(); @@ -231,8 +215,6 @@ job_array *get_and_remove_array( allarrays.unlock(); - free(tmpjobid); - return(pa); } // END get_and_remove_array() @@ -1500,6 +1482,7 @@ int delete_array_range( } } + pa->ai_qs.jobs_done += num_deleted; pa->ai_qs.num_failed += num_deleted; return(num_skipped); @@ -1636,7 +1619,10 @@ int delete_whole_array( } if (pa != NULL) + { pa->ai_qs.num_failed += num_deleted; + pa->ai_qs.jobs_done += num_deleted; + } if (num_jobs == 0) return(NO_JOBS_IN_ARRAY); @@ -1801,11 +1787,12 @@ int modify_array_range( } else { - struct batch_request *array_req = duplicate_request(preq, index); + batch_request array_req(*preq); + array_req.update_object_id(index); mutex_mgr pjob_mutex = mutex_mgr(pjob->ji_mutex, true); pthread_mutex_unlock(pa->ai_mutex); - array_req->rq_noreply = TRUE; - modify_job((void **)&pjob, plist, array_req, checkpoint_req, NO_MOM_RELAY); + array_req.rq_noreply = true; + modify_job((void **)&pjob, plist, &array_req, checkpoint_req, NO_MOM_RELAY); pa = get_jobs_array(&pjob); if (pa == NULL) diff --git a/src/server/array_upgrade.c b/src/server/array_upgrade.c index 92849478bd..458eb9d417 100644 --- a/src/server/array_upgrade.c +++ b/src/server/array_upgrade.c @@ -43,6 +43,7 @@ int array_upgrade( { char log_buf[LOCAL_LOG_BUF_SIZE]; + std::string array_id(pa->ai_qs.parent_id); /* reset the file descriptor */ if (lseek(fds, 0, SEEK_SET) != 0) @@ -72,9 +73,9 @@ int array_upgrade( about the incompatibility and inability to upgrade, print a quick and dirty error message and exit */ - sprintf(log_buf, "WARNING, unable to upgrade job array from version %d\n" + sprintf(log_buf, "WARNING, unable to upgrade job array %s from version %d\n" "structs from versions prior to 2.5.0.\n" - "Please downgrade TORQUE and upgrade once no job arrays are queued\n", version); + "Please downgrade TORQUE and upgrade once no job arrays are queued\n",array_id.c_str(), version); log_err(-1, "array_upgrade", log_buf); exit(1); } @@ -93,8 +94,8 @@ int array_upgrade( * know and ignore the array, but don't halt the server. */ sprintf(log_buf, - "WARNING, unable to upgrade job array from version %d. Job array files most likely corrupted.\n" - "Please examine the files at %s for file integrity\n", version, path_arrays); + "WARNING, unable to upgrade job array %s from version %d. Job array files most likely corrupted.\n" + "Please examine the files at %s for file integrity\n", array_id.c_str(), version, path_arrays); log_err(-1, "array_upgrade", log_buf); return 1; } diff --git a/src/server/batch_request.c b/src/server/batch_request.c index 0800a3eb30..c7e95123d9 100644 --- a/src/server/batch_request.c +++ b/src/server/batch_request.c @@ -108,8 +108,7 @@ int get_batch_request_id( snprintf(buf, sizeof(buf), "%d", id); - if ((preq->rq_id = strdup(buf)) == NULL) - return(ENOMEM); + preq->rq_id = buf; insert_batch_request(preq); @@ -126,7 +125,7 @@ int insert_batch_request( brh.lock(); - if(!brh.insert(preq,preq->rq_id)) + if (!brh.insert(preq, preq->rq_id)) { rc = ENOMEM; log_err(rc, __func__, "No memory to resize the array...SYSTEM FAILURE\n"); @@ -150,7 +149,7 @@ int insert_batch_request( batch_request *get_batch_request( - char *br_id) + const char *br_id) { batch_request *preq = NULL; @@ -173,7 +172,7 @@ batch_request *get_batch_request( batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { batch_request *preq = NULL; @@ -194,7 +193,7 @@ batch_request *get_remove_batch_request( int remove_batch_request( - char *br_id) + const char *br_id) { brh.lock(); @@ -209,4 +208,531 @@ int remove_batch_request( +// Constructor with a type +batch_request::batch_request( + int type) : rq_type(type), rq_perm(0), rq_fromsvr(0), rq_conn(-1), rq_orgconn(-1), rq_extsz(0), + rq_time(time(NULL)), rq_refcount(0), rq_extra(NULL), rq_noreply(false), + rq_failcode(0), rq_extend(NULL), rq_id() + + { + memset(this->rq_user, 0, sizeof(this->rq_user)); + memset(this->rq_host, 0, sizeof(this->rq_host)); + memset(&this->rq_ind, 0, sizeof(this->rq_ind)); + memset(&this->rq_reply, 0, sizeof(this->rq_reply)); + + this->rq_reply.brp_choice = BATCH_REPLY_CHOICE_NULL; + } + + + +// Empty constructor +batch_request::batch_request() : rq_type(-1), rq_perm(0), rq_fromsvr(0), rq_conn(-1), + rq_orgconn(-1), rq_extsz(0), rq_time(time(NULL)), rq_refcount(0), + rq_extra(NULL), rq_noreply(false), rq_failcode(0), + rq_extend(NULL), rq_id() + + { + memset(this->rq_user, 0, sizeof(this->rq_user)); + memset(this->rq_host, 0, sizeof(this->rq_host)); + memset(&this->rq_ind, 0, sizeof(this->rq_ind)); + memset(&this->rq_reply, 0, sizeof(this->rq_reply)); + + this->rq_reply.brp_choice = BATCH_REPLY_CHOICE_NULL; + } + + + +int batch_request::copy_attribute_list( + + const batch_request &other) + + { + svrattrl *pal = (svrattrl *)GET_NEXT(other.rq_ind.rq_manager.rq_attr); + tlist_head *phead = &this->rq_ind.rq_manager.rq_attr; + svrattrl *newpal = NULL; + + while (pal != NULL) + { + newpal = (svrattrl *)calloc(1, pal->al_tsize + 1); + if (!newpal) + { + return(PBSE_SYSTEM); + } + + CLEAR_LINK(newpal->al_link); + + newpal->al_atopl.next = 0; + newpal->al_tsize = pal->al_tsize + 1; + newpal->al_nameln = pal->al_nameln; + newpal->al_flags = pal->al_flags; + newpal->al_atopl.name = (char *)newpal + sizeof(svrattrl); + strcpy((char *)newpal->al_atopl.name, pal->al_atopl.name); + newpal->al_nameln = pal->al_nameln; + newpal->al_atopl.resource = newpal->al_atopl.name + newpal->al_nameln; + + if (pal->al_atopl.resource != NULL) + strcpy((char *)newpal->al_atopl.resource, pal->al_atopl.resource); + + newpal->al_rescln = pal->al_rescln; + newpal->al_atopl.value = newpal->al_atopl.name + newpal->al_nameln + newpal->al_rescln; + strcpy((char *)newpal->al_atopl.value, pal->al_atopl.value); + newpal->al_valln = pal->al_valln; + newpal->al_atopl.op = pal->al_atopl.op; + + pal = (struct svrattrl *)GET_NEXT(pal->al_link); + } + + if ((phead != NULL) && + (newpal != NULL)) + append_link(phead, &newpal->al_link, newpal); + + return(PBSE_NONE); + } /* END copy_attribute_list() */ + + + +void batch_request::update_object_id( + + int job_index) + + { + if (job_index >= 0) + { + /* If this is a job array it is possible we only have the array name + and not the individual job. We need to find out what we have and + modify the name if needed */ + char newjobname[PBS_MAXSVRJOBID+1]; + char *ptr1 = strstr(this->rq_ind.rq_manager.rq_objname, "[]"); + + if (ptr1) + { + char *ptr2; + ptr1++; + strcpy(newjobname, this->rq_ind.rq_manager.rq_objname); + + if ((ptr2 = strstr(newjobname, "[]")) != NULL) + { + ptr2++; + *ptr2 = 0; + } + + sprintf(this->rq_ind.rq_manager.rq_objname,"%s%d%s", + newjobname, job_index, ptr1); + } + } + } // END update_object_id() + + + +// Copy constructor +batch_request::batch_request( + + const batch_request &other) : rq_type(other.rq_type), rq_perm(other.rq_perm), + rq_fromsvr(other.rq_fromsvr), rq_conn(other.rq_conn), + rq_orgconn(other.rq_orgconn), rq_extsz(other.rq_extsz), + rq_time(other.rq_time), rq_refcount(other.rq_refcount), + rq_noreply(other.rq_noreply), rq_failcode(other.rq_failcode), + rq_id(other.rq_id) + + { + if (other.rq_extra == NULL) + this->rq_extra = NULL; + else + this->rq_extra = strdup((char *)other.rq_extra); + + if (other.rq_extend == NULL) + this->rq_extend = NULL; + else + this->rq_extend = strdup(other.rq_extend); + + strcpy(this->rq_user, other.rq_user); + strcpy(this->rq_host, other.rq_host); + memcpy(&this->rq_reply, &other.rq_reply, sizeof(this->rq_reply)); + + switch (this->rq_type) + { + /* This function was created for a modify array request (PBS_BATCH_ModifyJob) + the other.rq_ind structure was allocated in dis_request_read. If other + BATCH types are needed refer to that function to see how the rq_ind structure + was allocated and then copy it here. */ + case PBS_BATCH_DeleteJob: + case PBS_BATCH_HoldJob: + case PBS_BATCH_CheckpointJob: + case PBS_BATCH_ModifyJob: + case PBS_BATCH_AsyModifyJob: + + memcpy(this->rq_ind.rq_manager.rq_objname, + other.rq_ind.rq_manager.rq_objname, PBS_MAXSVRJOBID + 1); + + /* based on how decode_DIS_Manage allocates data */ + CLEAR_HEAD(this->rq_ind.rq_manager.rq_attr); + + this->rq_ind.rq_manager.rq_cmd = other.rq_ind.rq_manager.rq_cmd; + this->rq_ind.rq_manager.rq_objtype = other.rq_ind.rq_manager.rq_objtype; + + strcpy(this->rq_ind.rq_manager.rq_objname, other.rq_ind.rq_manager.rq_objname); + this->copy_attribute_list(other); + + break; + + case PBS_BATCH_SignalJob: + + strcpy(this->rq_ind.rq_signal.rq_jid, other.rq_ind.rq_signal.rq_jid); + strcpy(this->rq_ind.rq_signal.rq_signame, other.rq_ind.rq_signal.rq_signame); + + break; + + case PBS_BATCH_MessJob: + + strcpy(this->rq_ind.rq_message.rq_jid, other.rq_ind.rq_message.rq_jid); + this->rq_ind.rq_message.rq_file = other.rq_ind.rq_message.rq_file; + + if (other.rq_ind.rq_message.rq_text != NULL) + this->rq_ind.rq_message.rq_text = strdup(other.rq_ind.rq_message.rq_text); + + break; + + case PBS_BATCH_RunJob: + case PBS_BATCH_AsyrunJob: + + strcpy(this->rq_ind.rq_run.rq_jid, other.rq_ind.rq_run.rq_jid); + + if (other.rq_ind.rq_run.rq_destin) + this->rq_ind.rq_run.rq_destin = strdup(other.rq_ind.rq_run.rq_destin); + + break; + + case PBS_BATCH_Rerun: + + strcpy(this->rq_ind.rq_rerun, other.rq_ind.rq_rerun); + break; + + default: + + break; + } + } + + + +batch_request &batch_request::operator =( + + const batch_request &other) + + { + this->rq_type = other.rq_type; + this->rq_perm = other.rq_perm; + this->rq_fromsvr = other.rq_fromsvr; + this->rq_conn = other.rq_conn; + this->rq_orgconn = other.rq_orgconn; + this->rq_extsz = other.rq_extsz; + this->rq_time = other.rq_time; + this->rq_refcount = other.rq_refcount; + this->rq_noreply = other.rq_noreply; + this->rq_failcode = other.rq_failcode; + + if (other.rq_extra == NULL) + this->rq_extra = NULL; + else + this->rq_extra = strdup((char *)other.rq_extra); + + if (other.rq_extend == NULL) + this->rq_extend = NULL; + else + this->rq_extend = strdup(other.rq_extend); + + this->rq_id = other.rq_id; + + strcpy(this->rq_user, other.rq_user); + strcpy(this->rq_host, other.rq_host); + memcpy(&this->rq_reply, &other.rq_reply, sizeof(this->rq_reply)); + + switch (this->rq_type) + { + /* This function was created for a modify array request (PBS_BATCH_ModifyJob) + the other.rq_ind structure was allocated in dis_request_read. If other + BATCH types are needed refer to that function to see how the rq_ind structure + was allocated and then copy it here. */ + case PBS_BATCH_DeleteJob: + case PBS_BATCH_HoldJob: + case PBS_BATCH_CheckpointJob: + case PBS_BATCH_ModifyJob: + case PBS_BATCH_AsyModifyJob: + + memcpy(this->rq_ind.rq_manager.rq_objname, + other.rq_ind.rq_manager.rq_objname, PBS_MAXSVRJOBID + 1); + + /* based on how decode_DIS_Manage allocates data */ + CLEAR_HEAD(this->rq_ind.rq_manager.rq_attr); + + this->rq_ind.rq_manager.rq_cmd = other.rq_ind.rq_manager.rq_cmd; + this->rq_ind.rq_manager.rq_objtype = other.rq_ind.rq_manager.rq_objtype; + + strcpy(this->rq_ind.rq_manager.rq_objname, other.rq_ind.rq_manager.rq_objname); + this->copy_attribute_list(other); + + break; + + case PBS_BATCH_SignalJob: + + strcpy(this->rq_ind.rq_signal.rq_jid, other.rq_ind.rq_signal.rq_jid); + strcpy(this->rq_ind.rq_signal.rq_signame, other.rq_ind.rq_signal.rq_signame); + + break; + + case PBS_BATCH_MessJob: + + strcpy(this->rq_ind.rq_message.rq_jid, other.rq_ind.rq_message.rq_jid); + this->rq_ind.rq_message.rq_file = other.rq_ind.rq_message.rq_file; + + if (other.rq_ind.rq_message.rq_text != NULL) + this->rq_ind.rq_message.rq_text = strdup(other.rq_ind.rq_message.rq_text); + + break; + + case PBS_BATCH_RunJob: + case PBS_BATCH_AsyrunJob: + + strcpy(this->rq_ind.rq_run.rq_jid, other.rq_ind.rq_run.rq_jid); + if (other.rq_ind.rq_run.rq_destin) + this->rq_ind.rq_run.rq_destin = strdup(other.rq_ind.rq_run.rq_destin); + + break; + + case PBS_BATCH_Rerun: + + strcpy(this->rq_ind.rq_rerun, other.rq_ind.rq_rerun); + break; + + default: + + break; + } + + return(*this); + } // END = operator + + + +void freebr_manage( + + struct rq_manage *pmgr) + + { + free_attrlist(&pmgr->rq_attr); + + return; + } /* END freebr_manage() */ + + + +void freebr_cpyfile( + + struct rq_cpyfile *pcf) + + { + struct rqfpair *ppair; + + while ((ppair = (struct rqfpair *)GET_NEXT(pcf->rq_pair)) != NULL) + { + delete_link(&ppair->fp_link); + + if (ppair->fp_local != NULL) + free(ppair->fp_local); + + if (ppair->fp_rmt != NULL) + free(ppair->fp_rmt); + + free(ppair); + } + + return; + } /* END freebr_cpyfile() */ + + + +void free_rescrq( + + struct rq_rescq *pq) + + { + int i; + + i = pq->rq_num; + + while (i--) + { + if (*(pq->rq_list + i) != NULL) + free(*(pq->rq_list + i)); + } + + if (pq->rq_list != NULL) + free(pq->rq_list); + + return; + } /* END free_rescrq() */ + + + +batch_request::~batch_request() + + { + if (this->rq_id.size() != 0) + { + remove_batch_request(this->rq_id.c_str()); + } + + reply_free(&this->rq_reply); + + if (this->rq_extend) + { + free(this->rq_extend); + this->rq_extend = NULL; + } + + if (this->rq_extra) + { + free(this->rq_extra); + this->rq_extra = NULL; + } + + switch (this->rq_type) + { + case PBS_BATCH_QueueJob: + case PBS_BATCH_QueueJob2: + + free_attrlist(&this->rq_ind.rq_queuejob.rq_attr); + + break; + + case PBS_BATCH_JobCred: + + if (this->rq_ind.rq_jobcred.rq_data) + { + free(this->rq_ind.rq_jobcred.rq_data); + this->rq_ind.rq_jobcred.rq_data = NULL; + } + + break; + + case PBS_BATCH_MvJobFile: + case PBS_BATCH_jobscript: + case PBS_BATCH_jobscript2: + + if (this->rq_ind.rq_jobfile.rq_data) + { + free(this->rq_ind.rq_jobfile.rq_data); + this->rq_ind.rq_jobfile.rq_data = NULL; + } + break; + + case PBS_BATCH_HoldJob: + + freebr_manage(&this->rq_ind.rq_hold.rq_orig); + + break; + + case PBS_BATCH_CheckpointJob: + + freebr_manage(&this->rq_ind.rq_manager); + + break; + + case PBS_BATCH_MessJob: + + if (this->rq_ind.rq_message.rq_text) + { + free(this->rq_ind.rq_message.rq_text); + this->rq_ind.rq_message.rq_text = NULL; + } + + break; + + case PBS_BATCH_ModifyJob: + + case PBS_BATCH_AsyModifyJob: + + freebr_manage(&this->rq_ind.rq_modify); + + break; + + case PBS_BATCH_StatusJob: + + case PBS_BATCH_StatusQue: + + case PBS_BATCH_StatusNode: + + case PBS_BATCH_StatusSvr: + /* DIAGTODO: handle PBS_BATCH_StatusDiag */ + + free_attrlist(&this->rq_ind.rq_status.rq_attr); + + break; + + case PBS_BATCH_JobObit: + + free_attrlist(&this->rq_ind.rq_jobobit.rq_attr); + + break; + + case PBS_BATCH_CopyFiles: + + case PBS_BATCH_DelFiles: + + freebr_cpyfile(&this->rq_ind.rq_cpyfile); + + break; + + case PBS_BATCH_ModifyNode: + + case PBS_BATCH_Manager: + + freebr_manage(&this->rq_ind.rq_manager); + + break; + + case PBS_BATCH_ReleaseJob: + + freebr_manage(&this->rq_ind.rq_release); + + break; + + case PBS_BATCH_Rescq: + + free_rescrq(&this->rq_ind.rq_rescq); + + break; + + case PBS_BATCH_SelectJobs: + + case PBS_BATCH_SelStat: + + free_attrlist(&this->rq_ind.rq_select); + + break; + + case PBS_BATCH_SelStatAttr: + + free_attrlist(&this->rq_ind.rq_select); + free_attrlist(&this->rq_ind.rq_status.rq_attr); + + break; + + case PBS_BATCH_RunJob: + case PBS_BATCH_AsyrunJob: + + if (this->rq_ind.rq_run.rq_destin) + { + free(this->rq_ind.rq_run.rq_destin); + this->rq_ind.rq_run.rq_destin = NULL; + } + break; + + default: + + /* NO-OP */ + + break; + } /* END switch (this->rq_type) */ + } diff --git a/src/server/completed_jobs_map.cpp b/src/server/completed_jobs_map.cpp index 7e50a71bb9..6d22a5fcb8 100644 --- a/src/server/completed_jobs_map.cpp +++ b/src/server/completed_jobs_map.cpp @@ -165,36 +165,19 @@ int completed_jobs_map_class::cleanup_completed_jobs( it != completed_jobs_map.end(); ++it) { - bool rc = false; - // if cleanup time is before/at now, then clean up if (it->second <= now) { - int retcode = PBSE_NONE; work_task *pnew; - // Does job exist - - while((rc = job_id_exists(it->first, &retcode)) == false) + // Does job still exist + if (job_id_exists(it->first) == false) { - if (retcode != 0) - { - // Someone else has a lock we want. give ours up and try again - pthread_mutex_unlock(&completed_jobs_map_mutex); - pthread_mutex_lock(&completed_jobs_map_mutex); - continue; - } - else - break; - } + // Job is gone, mark it for removal + to_remove.push_back(it->first); - if (rc == false) - { - // Job is gone, mark it for removal - to_remove.push_back(it->first); - - continue; - } + continue; + } // create a work task struct to be passed to handle_complete_second_time() if ((pnew = (struct work_task *)calloc(1, sizeof(struct work_task))) == NULL) diff --git a/src/server/geteusernam.c b/src/server/geteusernam.c index f744cfe979..ccc4daf94d 100644 --- a/src/server/geteusernam.c +++ b/src/server/geteusernam.c @@ -315,37 +315,6 @@ bool getegroup( } /* END getegroup() */ -/* - * get_user_host_from_user - * - * @param user_host - receives the host name associated with the user. - * @param user - user name with or without the host name. - * - */ - -void get_user_host_from_user( - - std::string &user_host, - const std::string user) - - { - char *ptr; - char *tmp_name = strdup(user.c_str()); - - user_host.clear(); - ptr = strchr(tmp_name, '@'); - if (ptr != NULL) - { - ptr++; - user_host = ptr; - } - - free(tmp_name); - - } - - - /* * Verifies that the job's user is acceptable @@ -362,14 +331,13 @@ bool is_user_allowed_to_submit_jobs( { char *orighost; - std::string user_host; /* this is a back up to orighost in case we need FQDN */ std::string user(pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str); std::size_t at_pos = user.find('@'); int rc = PBSE_NONE; bool ProxyAllowed = false; bool ProxyRequested = false; bool HostAllowed = false; - char *dptr; + std::string short_host; char log_buf[256]; @@ -386,8 +354,6 @@ bool is_user_allowed_to_submit_jobs( if (EMsg != NULL) EMsg[0] = '\0'; - get_user_host_from_user(user_host, user); - if (at_pos != std::string::npos) user.erase(at_pos); @@ -405,6 +371,13 @@ bool is_user_allowed_to_submit_jobs( return(false); } + short_host = orighost; + size_t pos = short_host.find('.'); + if (pos == std::string::npos) + short_host.clear(); + else + short_host.erase(pos); + /* check to see if we are allowing a proxy user to submit the job */ if ((server.sv_attr[SRV_ATR_AllowProxyUser].at_flags & ATR_VFLAG_SET) && \ (server.sv_attr[SRV_ATR_AllowProxyUser].at_val.at_bool == true)) @@ -432,28 +405,26 @@ bool is_user_allowed_to_submit_jobs( HostAllowed = true; } - /* make short host name */ - if ((dptr = strchr(orighost, '.')) != NULL) + if (HostAllowed == false) { - *dptr = '\0'; - } + bool allowed = false; - if ((HostAllowed == false) && - (is_permitted_by_node_submit(orighost, logging))) - { - /* job submitted from compute host, access allowed */ - if (dptr != NULL) - *dptr = '.'; - - if ((ProxyRequested == false) || - (ProxyAllowed == true)) + if (is_permitted_by_node_submit(orighost, logging) == false) { - return(true); + if (short_host.size() > 0) + allowed = is_permitted_by_node_submit(short_host.c_str(), logging); } + else + allowed = true; - /* host is fine, must validate proxy via ruserok() */ - - HostAllowed = false; + if (allowed == true) + { + if ((ProxyRequested == false) || + (ProxyAllowed == true)) + { + return(true); + } + } } if ((HostAllowed == false) && @@ -472,20 +443,14 @@ bool is_user_allowed_to_submit_jobs( int cmpRet = strcasecmp(testhost, orighost); if ((cmpRet != 0) && - (dptr != NULL)) + (short_host.size() > 0)) { - *dptr = '.'; - cmpRet = strcasecmp(testhost, orighost); - *dptr = '\0'; + cmpRet = strcasecmp(testhost, short_host.c_str()); } if (cmpRet == 0) { /* job submitted from host found in trusted submit host list, access allowed */ - - if (dptr != NULL) - *dptr = '.'; - if ((ProxyRequested == false) || (ProxyAllowed == true)) { @@ -504,12 +469,9 @@ bool is_user_allowed_to_submit_jobs( // Check limited acls if (limited_acls.is_authorized(orighost, user) == true) return(true); - else if (limited_acls.is_authorized(user_host.c_str(), user) == true) + else if (limited_acls.is_authorized(short_host.c_str(), user) == true) return(true); - if (dptr != NULL) - *dptr = '.'; - #ifdef MUNGE_AUTH sprintf(uh, "%s@%s", user.c_str(), orighost); rc = acl_check(&server.sv_attr[SRV_ATR_authusers], uh, ACL_User_Host); diff --git a/src/server/issue_request.c b/src/server/issue_request.c index 788b62b2b8..71c50d2ccd 100644 --- a/src/server/issue_request.c +++ b/src/server/issue_request.c @@ -145,9 +145,9 @@ int issue_Drequest(int conn, struct batch_request *request); int relay_to_mom( - job **pjob_ptr, - struct batch_request *request, /* the request to send */ - void (*func)(struct work_task *)) + job **pjob_ptr, + batch_request *request, /* the request to send */ + void (*func)(struct work_task *)) { int handle; /* a client style connection handle */ @@ -261,6 +261,8 @@ void reissue_to_svr( br_id = (char *)pwt->wt_parm1; preq = get_remove_batch_request(br_id); + free(br_id); + /* if not timed-out, retry send to remote server */ if (preq != NULL) { @@ -274,7 +276,7 @@ void reissue_to_svr( } if (((time_now - preq->rq_time) > PBS_NET_RETRY_LIMIT) || - (issue_to_svr(serverName, &preq, pwt->wt_parmfunc) != PBSE_NONE)) + (issue_to_svr(serverName, preq, pwt->wt_parmfunc) != PBSE_NONE)) { /* either timed-out or got hard error, tell post-function */ @@ -284,6 +286,8 @@ void reissue_to_svr( if (pwt->wt_parmfunc != NULL) (* pwt->wt_parmfunc)(pwt); } + + delete preq; } if (serverName) @@ -302,12 +306,12 @@ void queue_a_retry_task( { /* create a new batch_request because preq is going to be freed when issue_to_svr returns success */ - batch_request *new_preq = duplicate_request(preq, -1); + batch_request *new_preq = new batch_request(*preq); struct work_task *pwt; get_batch_request_id(new_preq); - pwt = set_task(WORK_Timed, (time(NULL) + PBS_NET_RETRY_TIME), reissue_to_svr, new_preq->rq_id, TRUE); + pwt = set_task(WORK_Timed, (time(NULL) + PBS_NET_RETRY_TIME), reissue_to_svr, strdup(new_preq->rq_id.c_str()), TRUE); pwt->wt_parmfunc = replyfunc; @@ -326,13 +330,17 @@ void queue_a_retry_task( * -1 on permanent error (no such host) * * On temporary error, establish a work_task to retry after a delay. + * + * @param servern - the name of the destination server for this request + * @param preq - the batch request we're sending + * @param replyfunc - function that should be called after sending the request */ int issue_to_svr( - const char *servern, /* I */ - struct batch_request **preq_ptr, /* I */ - void (*replyfunc) (struct work_task *)) /* I */ + const char *servern, + batch_request *preq, + void (*replyfunc)(struct work_task *)) { int rc = PBSE_NONE; @@ -342,7 +350,6 @@ int issue_to_svr( pbs_net_t svraddr; char *svrname; unsigned int port = pbs_server_port_dis; - batch_request *preq = *preq_ptr; snprintf(preq->rq_host, sizeof(preq->rq_host), "%s", servern); @@ -372,12 +379,8 @@ int issue_to_svr( if (((rc = issue_Drequest(handle, preq, true)) == PBSE_NONE) && (handle != PBS_LOCAL_CONNECTION)) { - /* preq is already freed if handle == PBS_LOCAL_CONNECTION - a reply - * has always been sent */ rc = preq->rq_reply.brp_code; } - else if (handle == PBS_LOCAL_CONNECTION) - *preq_ptr = NULL; return(rc); } @@ -418,10 +421,10 @@ int que_to_local_svr(struct batch_request *preq) /* I */ preq->rq_fromsvr = 1; preq->rq_perm = ATR_DFLAG_MGRD | ATR_DFLAG_MGWR | ATR_DFLAG_SvWR; - if (preq->rq_id == NULL) + if (preq->rq_id.size() == 0) get_batch_request_id(preq); - set_task(WORK_Immed, 0, reissue_to_svr, preq->rq_id, TRUE); + set_task(WORK_Immed, 0, reissue_to_svr, strdup(preq->rq_id.c_str()), TRUE); return(PBSE_NONE); } /* END que_to_local_svr() */ @@ -439,6 +442,8 @@ int que_to_local_svr(struct batch_request *preq) /* I */ * * THIS SHOULD NOT BE USED IF AN EXTERNAL (CLIENT) REQUEST WAS "relayed". * The request/reply structure is still needed to reply to the client. + * + * FIXME: This also appears to be dead code */ void release_req( @@ -450,7 +455,7 @@ void release_req( char *br_id = (char *)pwt->wt_parm1; if ((preq = get_remove_batch_request(br_id)) != NULL) - free_br(preq); + delete preq; if (pwt->wt_event != -1) svr_disconnect(pwt->wt_event); @@ -491,6 +496,7 @@ int send_request_to_remote_server( int sock = 0; char log_buf[LOCAL_LOG_BUF_SIZE]; struct tcp_chan *chan = NULL; + int old_state = PTHREAD_CANCEL_ENABLE; pthread_mutex_lock(connection[conn].ch_mutex); sock = connection[conn].ch_socket; @@ -498,13 +504,14 @@ int send_request_to_remote_server( request->rq_conn = sock; - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 0); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state); if ((chan = DIS_tcp_setup(sock)) == NULL) { log_err(PBSE_MEM_MALLOC, __func__, "Could not allocate memory for socket buffer"); close_conn(sock, FALSE); + pthread_setcancelstate(old_state, NULL); return(PBSE_MEM_MALLOC); } @@ -769,7 +776,7 @@ int send_request_to_remote_server( if (close_handle == true) svr_disconnect(conn); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(rc); } /* END send_request_to_remote_server() */ diff --git a/src/server/issue_request.h b/src/server/issue_request.h index c9c7cb51fc..e2a3f332c4 100644 --- a/src/server/issue_request.h +++ b/src/server/issue_request.h @@ -12,6 +12,4 @@ void reissue_to_svr(struct work_task *); int handle_local_request(int conn, batch_request *request); -void release_req(struct work_task *pwt); - #endif /* _ISSUE_REQUEST_H */ diff --git a/src/server/job.cpp b/src/server/job.cpp index e56fba5388..5506352d6b 100644 --- a/src/server/job.cpp +++ b/src/server/job.cpp @@ -14,7 +14,7 @@ job::job() : ji_plugin_usage_info(), ji_momstat(0), ji_modified(0), ji_momhandle ji_have_nodes_request(false), ji_external_clone(NULL), ji_cray_clone(NULL), ji_parent_job(NULL), ji_internal_id(-1), ji_being_recycled(false), ji_last_reported_time(0), ji_mod_time(0), - ji_queue_counted(0), ji_being_deleted(false), ji_commit_done(false) + ji_queue_counted(0), ji_being_deleted(false), ji_commit_done(false), ji_routed(true) { memset(this->ji_arraystructid, 0, sizeof(ji_arraystructid)); @@ -170,7 +170,13 @@ void job::set_plugin_resource_usage_from_json( std::vectorkeys = resources.getMemberNames(); for (size_t i = 0; i < keys.size(); i++) - this->ji_plugin_usage_info[keys[i]] = resources[keys[i]].asString(); + { + if (keys[i] != "") + { + if (resources[keys[i]].empty() == false) + this->ji_plugin_usage_info[keys[i]] = resources[keys[i]].asString(); + } + } } @@ -186,3 +192,9 @@ void job::set_plugin_resource_usage_from_json( if (reader.parse(json_str, resources) == true) this->set_plugin_resource_usage_from_json(resources); } + + +size_t job::number_of_plugin_resources() const + { + return(this->ji_plugin_usage_info.size()); + } diff --git a/src/server/job_array.cpp b/src/server/job_array.cpp index c19e4a46dd..47d4da9b4e 100644 --- a/src/server/job_array.cpp +++ b/src/server/job_array.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include "array.h" #include "list_link.h" @@ -130,18 +131,22 @@ void job_array::set_submit_host( int job_array::set_slot_limit( - char *request) + const char *request) { char *pcnt; + char *request_copy; long max_limit; char log_buf[LOCAL_LOG_BUF_SIZE]; + if ((request_copy = strdup(request)) == NULL) + return(-1); + /* check for a max slot limit */ if (get_svr_attr_l(SRV_ATR_MaxSlotLimit, &max_limit) != PBSE_NONE) max_limit = NO_SLOT_LIMIT; - if ((pcnt = strchr(request,'%')) != NULL) + if ((pcnt = strchr(request_copy, '%')) != NULL) { /* remove '%' from the request, or else it can't be parsed */ while (*pcnt == '%') @@ -164,6 +169,7 @@ int job_array::set_slot_limit( max_limit); log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_JOB, this->ai_qs.parent_id, log_buf); + free(request_copy); return(INVALID_SLOT_LIMIT); } } @@ -177,6 +183,7 @@ int job_array::set_slot_limit( this->ai_qs.slot_limit = max_limit; } + free(request_copy); return(PBSE_NONE); } /* END set_slot_limit() */ @@ -239,15 +246,22 @@ int job_array::parse_array_request( long max_array_size; char log_buf[LOCAL_LOG_BUF_SIZE]; this->uncreated_ids.clear(); + std::string request_copy(request); + int loc; + + // remove slot limit (begins with %) if present + if ((loc = request_copy.find_first_of("%")) >= 0) + request_copy.erase(loc); - if ((rc = translate_range_string_to_vector(request, this->uncreated_ids)) != PBSE_NONE) + if ((rc = translate_range_string_to_vector(request_copy.c_str(), this->uncreated_ids)) != PBSE_NONE) return(rc); - this->ai_qs.range_str = request; + this->ai_qs.range_str = request_copy.c_str(); this->ai_qs.num_jobs = this->uncreated_ids.size(); // size of array is the biggest index + 1 - this->ai_qs.array_size = this->uncreated_ids[this->uncreated_ids.size() - 1] + 1; + this->ai_qs.array_size = *std::max_element(this->uncreated_ids.begin(), + this->uncreated_ids.end()) + 1; if (get_svr_attr_l(SRV_ATR_MaxArraySize, &max_array_size) == PBSE_NONE) { @@ -570,3 +584,47 @@ bool job_array::is_deleted() const +/* + * mark_end_of_subjob() + * Takes care of noting that this subjob is being purged from the array + * + * @param pjob - the subjob that is being removed + * @return true if the array has no more subjobs, false otherwise + */ + +bool job_array::mark_end_of_subjob( + + job *pjob) + + { + bool no_more_subjobs = false; + int index; + + if (pjob != NULL) + { + if (this->job_ids != NULL) + { + index = pjob->ji_wattr[JOB_ATR_job_array_id].at_val.at_long; + + if ((index >= 0) && + (index < this->ai_qs.array_size)) + { + if (this->job_ids[index] != NULL) + { + free(this->job_ids[index]); + this->job_ids[index] = NULL; + } + + this->ai_qs.num_purged++; + + if ((this->ai_qs.num_purged == this->ai_qs.num_jobs) || + ((this->is_deleted() == true) && + (this->ai_qs.num_idle == 0))) + no_more_subjobs = true; + } + } + } + + return(no_more_subjobs); + } // mark_end_of_subjob() + diff --git a/src/server/job_attr_def.c b/src/server/job_attr_def.c index 5f1b6ada6c..fd175ff844 100644 --- a/src/server/job_attr_def.c +++ b/src/server/job_attr_def.c @@ -654,7 +654,7 @@ attribute_def job_attr_def[] = ATR_TYPE_LONG, PARENT_TYPE_JOB }, - /* JOB_ATR_queuerank */ + /* JOB_ATR_qrank */ { (char *)ATTR_qrank, /* "queue_rank" */ decode_l, encode_l, @@ -1318,6 +1318,45 @@ attribute_def job_attr_def[] = READ_WRITE, ATR_TYPE_LONG, PARENT_TYPE_JOB}, + + // JOB_ATR_LRequest + {(char *)ATTR_L_request, // "L_Request" + decode_str, + encode_str, + set_str, + comp_str, + free_str, + NULL_FUNC, + READ_WRITE, + ATR_TYPE_STR, + PARENT_TYPE_JOB, + }, + + // JOB_ATR_gpus_reserved + {(char *)ATTR_gpus_reserved, /* "gpus_reserved" */ + decode_str, + encode_str, + set_str, + comp_str, + free_str, + NULL_FUNC, + READ_ONLY | ATR_DFLAG_MOM | ATR_DFLAG_OPWR | ATR_DFLAG_SvWR, + ATR_TYPE_STR, + PARENT_TYPE_JOB, + }, + + // JOB_ATR_mics_reserved + {(char *)ATTR_mics_reserved, /* "mics_reserved" */ + decode_str, + encode_str, + set_str, + comp_str, + free_str, + NULL_FUNC, + READ_ONLY | ATR_DFLAG_MOM | ATR_DFLAG_OPWR | ATR_DFLAG_SvWR, + ATR_TYPE_STR, + PARENT_TYPE_JOB, + }, /* JOB_ATR_UNKN - THIS MUST BE THE LAST ENTRY */ { (char *)"_other_", diff --git a/src/server/job_container.c b/src/server/job_container.c index bbd44e31e4..6cd690a01e 100644 --- a/src/server/job_container.c +++ b/src/server/job_container.c @@ -111,14 +111,15 @@ id_map job_mapper; * * allocs the correct job name * @param jobid (I) - the jobid as passed in (NUM.SERVER_NAME) - * @return a pointer to the correct job name (alloc'd) + * @param correct (O) - the correct job id as a string object + * @return a pointer to the correct job name */ -char *get_correct_jobname( +const char *get_correct_jobname( - const char *jobid) /* I */ + const char *jobid, + std::string &correct) { - char *correct = NULL; char *dot; char *work_jobid = strdup(jobid); /* first suffix could be the server name or the alias */ @@ -161,47 +162,31 @@ char *get_correct_jobname( { if (strcmp(second_suffix,alias) == 0) { - correct = strdup(jobid); + correct = jobid; - if (correct == NULL) - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(correct); + return(correct.c_str()); } } else if (first_suffix == NULL) { /* alloc memory and sprint, add 3 for 2 '.' and NULL terminator */ len = strlen(work_jobid) + strlen(server_name) + strlen(alias) + 3; - correct = (char *)calloc(1, len); - - if (correct == NULL) - { - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(NULL); - } - - snprintf(correct,len,"%s.%s.%s", - work_jobid,server_name,alias); + correct = work_jobid; + correct += "."; + correct += server_name; + correct += "."; + correct += alias; } else { /* add 2 for null terminator and '.' */ len = strlen(alias) + 2 + strlen(work_jobid); - correct = (char *)calloc(1, len); - - if (correct == NULL) - { - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(NULL); - } - - snprintf(correct,len,"%s.%s",work_jobid,alias); + correct = work_jobid; + correct += "."; + correct += alias; } } /* END if (server_suffix && alias) */ else if (server_suffix == TRUE) @@ -210,30 +195,15 @@ char *get_correct_jobname( if (first_suffix != NULL) { - correct = strdup(work_jobid); - - if (correct == NULL) - { - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(NULL); - } + correct = work_jobid; } else { len = strlen(work_jobid) + strlen(server_name) + 2; - correct = (char *)calloc(1, len); - - if (correct == NULL) - { - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(NULL); - } - - snprintf(correct,len,"%s.%s", - work_jobid,server_name); + correct = work_jobid; + correct += "."; + correct += server_name; } } /* END if (just server_suffix) */ else if (alias != NULL) @@ -244,16 +214,9 @@ char *get_correct_jobname( { len = strlen(work_jobid) + strlen(alias) + 2; - correct = (char *)calloc(1, len); - - if (correct == NULL) - { - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(NULL); - } - - snprintf(correct,len,"%s.%s",work_jobid,alias); + correct = work_jobid; + correct += "."; + correct += alias; } else { @@ -263,18 +226,10 @@ char *get_correct_jobname( *dot = '\0'; len += strlen(work_jobid); - correct = (char *)calloc(1, len); - if (correct == NULL) - { - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(NULL); - } - - snprintf(correct,len,"%s.%s", - work_jobid, - alias); + correct = work_jobid; + correct += "."; + correct += alias; *dot = '.'; } @@ -289,16 +244,8 @@ char *get_correct_jobname( } len = strlen(work_jobid) + 1; - correct = (char *)calloc(1, len); - - if (correct == NULL) - { - log_err(-1, __func__, "ERROR: Fatal - Cannot allocate memory\n"); - free(work_jobid); - return(NULL); - } - snprintf(correct,len,"%s",work_jobid); + correct = work_jobid; if (first_suffix != NULL) *dot = '.'; @@ -306,7 +253,7 @@ char *get_correct_jobname( free(work_jobid); - return(correct); + return(correct.c_str()); } /* END get_correct_jobname() */ @@ -389,8 +336,6 @@ job *svr_find_job( { char *at; - char *comp = NULL; - int different = FALSE; char *dash = NULL; char *dot = NULL; char without_dash[PBS_MAXSVRJOBID + 1]; @@ -399,6 +344,8 @@ job *svr_find_job( job *pj = NULL; + std::string comp; + if (NULL == jobid) { log_err(-1,__func__,"jobid is null"); @@ -440,11 +387,7 @@ job *svr_find_job( if ((is_svr_attr_set(SRV_ATR_display_job_server_suffix)) || (is_svr_attr_set(SRV_ATR_job_suffix_alias))) { - comp = get_correct_jobname(jid_ptr); - different = TRUE; - - if (comp == NULL) - return(NULL); + get_correct_jobname(jid_ptr, comp); } else { @@ -456,7 +399,7 @@ job *svr_find_job( /* if we're searching for the external we want find_job_by_array to * return the parent, but if we're searching for the cray subjob then * we want find_job_by_array to return the sub job */ - pj = find_job_by_array(&alljobs, comp, (dash != NULL) ? FALSE : get_subjob, false); + pj = find_job_by_array(&alljobs, comp.c_str(), (dash != NULL) ? FALSE : get_subjob, false); } /* when remotely routing jobs, they are removed from the @@ -466,7 +409,7 @@ job *svr_find_job( if (pj == NULL) { /* see the comment on the above call to find_job_by_array() */ - pj = find_job_by_array(&array_summary, comp, (dash != NULL) ? FALSE : get_subjob, false); + pj = find_job_by_array(&array_summary, comp.c_str(), (dash != NULL) ? FALSE : get_subjob, false); } if (at) @@ -500,9 +443,6 @@ job *svr_find_job( } } - if (different) - free(comp); - return(pj); /* may be NULL */ } /* END svr_find_job() */ diff --git a/src/server/job_func.c b/src/server/job_func.c index e0ce613c9c..dbc47de458 100644 --- a/src/server/job_func.c +++ b/src/server/job_func.c @@ -142,7 +142,6 @@ #include "array.h" #include "pbs_job.h" #include "svr_func.h" /* get_svr_attr_* */ -#include "issue_request.h" /* release_req */ #include "ji_mutex.h" #include "user_info.h" #include "mutex_mgr.hpp" @@ -156,6 +155,7 @@ #define FALSE 0 #endif +#define ARRAY_ROUTING_RETRY_SECONDS 45 #define MAXLINE 1024 extern int LOGLEVEL; @@ -170,7 +170,6 @@ extern struct batch_request *setup_cpyfiles(struct batch_request *,job *,char*,c extern int job_log_open(char *, char *); extern int log_job_record(const char *buf); extern void check_job_log(struct work_task *ptask); -int issue_signal(job **, const char *, void(*)(batch_request *), void *, char *); void handle_complete_second_time(struct work_task *ptask); /* Local Private Functions */ @@ -198,8 +197,6 @@ extern char *path_jobinfo_log; extern char *log_file; extern char *job_log_file; -extern sem_t *job_clone_semaphore; - static void send_qsub_delmsg( job *pjob, /* I */ @@ -547,7 +544,7 @@ int job_abt( { svr_setjobstate(pjob, JOB_STATE_RUNNING, JOB_SUBSTATE_ABORT, FALSE); - if ((rc = issue_signal(&pjob, "SIGKILL", free_br, NULL, NULL)) != 0) + if ((rc = issue_signal(&pjob, "SIGKILL", NULL, NULL, NULL)) != 0) { if (pjob != NULL) { @@ -1148,7 +1145,10 @@ int create_and_queue_array_subjob( int newstate; int newsub; int rc = PBSE_NONE; + int old_state = PTHREAD_CANCEL_ENABLE; std::string arrayid = pa->ai_qs.parent_id; + + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state); template_job_mgr.lock(); pjobclone = job_clone(template_job, pa, index, place_hold); @@ -1157,11 +1157,13 @@ int create_and_queue_array_subjob( if (pjobclone == NULL) { log_err(-1, __func__, "unable to clone job in job_clone_wt"); + pthread_setcancelstate(old_state, NULL); return(NONFATAL_ERROR); } else if (pjobclone == (job *)1) { /* this happens if we attempted to clone an existing job */ + pthread_setcancelstate(old_state, NULL); return(PBSE_NONE); } @@ -1196,11 +1198,12 @@ int create_and_queue_array_subjob( if ((pa = get_array(arrayid.c_str())) == NULL) { - sem_wait(job_clone_semaphore); + pthread_setcancelstate(old_state, NULL); return(FATAL_ERROR); } array_mgr.mark_as_locked(); + pthread_setcancelstate(old_state, NULL); return(NONFATAL_ERROR); } @@ -1212,8 +1215,8 @@ int create_and_queue_array_subjob( /* pjobclone has been released. No mutex left to unlock */ clone_mgr.set_unlock_on_exit(false); } + pthread_setcancelstate(old_state, NULL); - sem_wait(job_clone_semaphore); return(FATAL_ERROR); } @@ -1227,7 +1230,6 @@ int create_and_queue_array_subjob( if ((pa = get_array(arrayid.c_str())) == NULL) { - sem_wait(job_clone_semaphore); return(FATAL_ERROR); } @@ -1354,9 +1356,21 @@ int perform_array_postprocessing( -#ifndef CLONE_BATCH_SIZE -#define CLONE_BATCH_SIZE 256 -#endif /* CLONE_BATCH_SIZE */ +void job_clone_task_wrapper( + + work_task *ptask) + + { + char *job_id = (char *)ptask->wt_parm1; + + if (job_id != NULL) + { + job_clone_wt(job_id); + } + + free(ptask->wt_mutex); + free(ptask); + } // END job_clone_task_wrapper() @@ -1372,10 +1386,13 @@ void *job_clone_wt( job *template_job; char *jobid; int rc; - std::string prev_job_id; + int old_state = PTHREAD_CANCEL_ENABLE; char namebuf[MAXPATHLEN]; job_array *pa; + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state); + + std::string prev_job_id; std::string adjusted_path_jobs; jobid = (char *)cloned_id; @@ -1383,15 +1400,7 @@ void *job_clone_wt( if (jobid == NULL) { log_err(ENOMEM, __func__, "Can't malloc"); - return(NULL); - } - - /* increment the job_clone_semaphore so people - know we are making jobs for this array */ - rc = sem_post(job_clone_semaphore); - if (rc) - { - log_err(-1, __func__, "failed to post job_clone_semaphore"); + pthread_setcancelstate(old_state, NULL); return(NULL); } @@ -1403,13 +1412,22 @@ void *job_clone_wt( if (template_job != NULL) unlock_ji_mutex(template_job, __func__, "1", LOGLEVEL); - sem_wait(job_clone_semaphore); + pthread_setcancelstate(old_state, NULL); return(NULL); } mutex_mgr array_mgr(pa->ai_mutex, true); mutex_mgr template_job_mgr(template_job->ji_mutex,true); + if (template_job->ji_routed == false) + { + // Do not create the array sub-jobs until the job has been routed + set_task(WORK_Timed, time(NULL) + ARRAY_ROUTING_RETRY_SECONDS, job_clone_task_wrapper, strdup(jobid), FALSE); + free(jobid); + pthread_setcancelstate(old_state, NULL); + return(NULL); + } + free(jobid); // get the adjusted path_jobs path @@ -1439,6 +1457,7 @@ void *job_clone_wt( if (rc == FATAL_ERROR) { + pthread_setcancelstate(old_state, NULL); return(NULL); } @@ -1452,8 +1471,7 @@ void *job_clone_wt( perform_array_postprocessing(pa); - unlock_ai_mutex(pa, __func__, "3", LOGLEVEL); - sem_wait(job_clone_semaphore); + pthread_setcancelstate(old_state, NULL); return(NULL); } /* END job_clone_wt */ @@ -1632,9 +1650,9 @@ void remove_checkpoint( job **pjob_ptr) /* I */ { - struct batch_request *preq = NULL; - char log_buf[LOCAL_LOG_BUF_SIZE]; - job *pjob = NULL; + batch_request *preq = NULL; + char log_buf[LOCAL_LOG_BUF_SIZE]; + job *pjob = NULL; if (pjob_ptr == NULL) { @@ -1684,7 +1702,7 @@ void remove_checkpoint( (char *)"unable to remove checkpoint file for job"); } - free_br(preq); + delete preq; } return; @@ -1939,25 +1957,12 @@ int svr_job_purge( { if (pa != NULL) { - if (pa->job_ids != NULL) - { - free(pa->job_ids[pjob->ji_wattr[JOB_ATR_job_array_id].at_val.at_long]); - pa->job_ids[pjob->ji_wattr[JOB_ATR_job_array_id].at_val.at_long] = NULL; - } - - /* if there are no more jobs in the array, - * then we can clean that up too */ - pa->ai_qs.num_purged++; - if ((pa->ai_qs.num_purged == pa->ai_qs.num_jobs) || - ((pa->is_deleted() == true) && - (pa->ai_qs.num_idle == 0))) + if ((do_delete_array = pa->mark_end_of_subjob(pjob)) == false) + array_save(pa); + else { - /* array_delete will unlock pa->ai_mutex */ - strcpy(array_id, pjob->ji_arraystructid); - do_delete_array = true; + snprintf(array_id, sizeof(array_id), "%s", pa->ai_qs.parent_id); } - else - array_save(pa); unlock_ai_mutex(pa, __func__, "1", LOGLEVEL); } @@ -1969,7 +1974,6 @@ int svr_job_purge( } } - if ((job_substate != JOB_SUBSTATE_TRANSIN) && (job_substate != JOB_SUBSTATE_TRANSICM)) { @@ -2108,6 +2112,18 @@ int svr_job_purge( +void *svr_job_purge_task( + + void *vp) + + { + job *pjob = (job *)vp; + svr_job_purge(pjob); + return(NULL); + } // END svr_job_purge_task() + + + /* * Always access the array in this way in order to avoid deadlock * @@ -2430,21 +2446,12 @@ int split_job( bool job_id_exists( - const std::string &job_id_string, - int *rcode) + const std::string &job_id_string) { - int ret; bool rc = false; - ret = alljobs.trylock(); - if (ret != 0) - { - *rcode = ret; - return(false); - } - - *rcode = ret; + alljobs.lock(); if (alljobs.find(job_id_string) != NULL) { diff --git a/src/server/job_recov.c b/src/server/job_recov.c index f811246ec6..0e13f3cdc3 100644 --- a/src/server/job_recov.c +++ b/src/server/job_recov.c @@ -1203,7 +1203,8 @@ int job_save( int mom_port) /* if 0 ignore otherwise append to end of job name. this is for multi-mom mode */ { - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 0); + int old_state = PTHREAD_CANCEL_ENABLE; + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state); char namebuf1[MAXPATHLEN]; char namebuf2[MAXPATHLEN]; @@ -1282,11 +1283,11 @@ int job_save( { log_event(PBSEVENT_ERROR | PBSEVENT_SECURITY, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, "call to saveJobToXML in job_save failed"); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return -1; } - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(PBSE_NONE); } /* END job_save() */ @@ -1499,8 +1500,8 @@ int set_array_job_ids( } else { - job_abt(&pj, "Array job missing array struct, aborting job"); snprintf(log_buf, buflen, "array struct missing for array job %s", pj->ji_qs.ji_jobid); + job_abt(&pj, "Array job missing array struct, aborting job"); return(-1); } } diff --git a/src/server/job_route.c b/src/server/job_route.c index 7e7d4cecc4..1d85ea3435 100644 --- a/src/server/job_route.c +++ b/src/server/job_route.c @@ -248,6 +248,8 @@ int default_router( /* set time to retry job */ jobp->ji_qs.ji_un.ji_routet.ji_rteretry = retry_time; + if (jobp->ji_is_array_template) + jobp->ji_routed = false; return(0); } @@ -273,6 +275,8 @@ int default_router( case ROUTE_DEFERRED: /* deferred */ + jobp->ji_routed = true; + return(0); /*NOTREACHED*/ @@ -493,6 +497,50 @@ int reroute_job( +/* + * handle_rerouting() + * + * precond: pjob is locked + * postcond: pjob is unlocked + * + */ + +int handle_rerouting( + + job *pjob, + pbs_queue *pque, + char *queue_name) + + { + int rc = PBSE_NONE; + + /* We only want to try if routing has been tried at least once - this is to let + * req_commit have the first crack at routing always. */ + if (pjob->ji_commit_done == 0) /* when req_commit is done it will set ji_commit_done to 1 */ + { + unlock_ji_mutex(pjob, __func__, "1", LOGLEVEL); + return(rc); + } + + /* queue must be unlocked when calling reroute_job */ + unlock_queue(pque, __func__, NULL, 10); + reroute_job(pjob); + unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL); + + /* need to relock queue when we go to call next_job */ + pque = find_queuebyname(queue_name); + if (pque == NULL) + { + char log_buf[LOCAL_LOG_BUF_SIZE]; + sprintf(log_buf, "Could not find queue %s", queue_name); + log_err(-1, __func__, log_buf); + rc = -1; + } + + return(rc); + } // END handle_rerouting() + + /* * queue_route - route any "ready" jobs in a specific queue @@ -557,33 +605,37 @@ void *queue_route( while ((pjob = next_job(pque->qu_jobs,iter)) != NULL) { - /* We only want to try if routing has been tried at least once - this is to let - * req_commit have the first crack at routing always. */ + if (handle_rerouting(pjob, pque, queue_name) != PBSE_NONE) + { + free(queue_name); + delete iter; + pthread_mutex_unlock(reroute_job_mutex); + return(NULL); + } + } + + delete iter; - if (pjob->ji_commit_done == 0) /* when req_commit is done it will set ji_commit_done to 1 */ + // Now try an array summary jobs that haven't been routed + pque->qu_jobs_array_sum->lock(); + iter = pque->qu_jobs_array_sum->get_iterator(); + pque->qu_jobs_array_sum->unlock(); + + while ((pjob = next_job(pque->qu_jobs_array_sum, iter)) != NULL) + { + if (pjob->ji_is_array_template == false) { - unlock_ji_mutex(pjob, __func__, "1", LOGLEVEL); + unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL); continue; } - /* queue must be unlocked when calling reroute_job */ - que_mutex.unlock(); - reroute_job(pjob); - unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL); - - /* need to relock queue when we go to call next_job */ - pque = find_queuebyname(queue_name); - if (pque == NULL) + if (handle_rerouting(pjob, pque, queue_name) != PBSE_NONE) { - sprintf(log_buf, "Could not find queue %s", queue_name); - log_err(-1, __func__, log_buf); free(queue_name); delete iter; pthread_mutex_unlock(reroute_job_mutex); return(NULL); } - - que_mutex.mark_as_locked(); } /* we come out of the while loop with the queue locked. diff --git a/src/server/mail_throttler.cpp b/src/server/mail_throttler.cpp index 47f4aa787e..b483b26500 100644 --- a/src/server/mail_throttler.cpp +++ b/src/server/mail_throttler.cpp @@ -8,12 +8,69 @@ mail_info::mail_info() : mailto(), exec_host(), jobid(), jobname(), { } +// Constructor from job - update the default values with +mail_info::mail_info( + + job *pjob) : mailto(), exec_host(), jobid(), jobname(), text(), errFile(), outFile(), mail_point(0), + queue_name(), owner(), working_directory(), resources_requested(), resources_used() + + { + if (pjob != NULL) + { + if (pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str != NULL) + this->exec_host = pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str; + + this->jobid = pjob->ji_qs.ji_jobid; + + if (pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str != NULL) + this->jobname = pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str; + + // Initialize the output files + if (pjob->ji_wattr[JOB_ATR_join].at_flags & ATR_VFLAG_SET) + { + char *join_val = pjob->ji_wattr[JOB_ATR_join].at_val.at_str; + if (!strcmp(join_val, "oe")) + { + this->errFile = pjob->ji_wattr[JOB_ATR_outpath].at_val.at_str; + this->outFile = pjob->ji_wattr[JOB_ATR_outpath].at_val.at_str; + } + else if (!strcmp(join_val, "eo")) + { + this->errFile = pjob->ji_wattr[JOB_ATR_errpath].at_val.at_str; + this->outFile = pjob->ji_wattr[JOB_ATR_errpath].at_val.at_str; + } + } + + if (this->outFile.size() == 0) + this->outFile = pjob->ji_wattr[JOB_ATR_outpath].at_val.at_str; + + if (this->errFile.size() == 0) + this->errFile = pjob->ji_wattr[JOB_ATR_errpath].at_val.at_str; + + this->queue_name = pjob->ji_qs.ji_queue; + this->owner = pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str; + + if (pjob->ji_wattr[JOB_ATR_init_work_dir].at_flags & ATR_VFLAG_SET) + this->working_directory = pjob->ji_wattr[JOB_ATR_init_work_dir].at_val.at_str; + + if (pjob->ji_wattr[JOB_ATR_resource].at_val.at_ptr != NULL) + this->resources_requested = *((std::vector *)pjob->ji_wattr[JOB_ATR_resource].at_val.at_ptr); + + if (pjob->ji_wattr[JOB_ATR_resc_used].at_val.at_ptr != NULL) + this->resources_used = *((std::vector *)pjob->ji_wattr[JOB_ATR_resc_used].at_val.at_ptr); + } + } + // Copy constructor mail_info::mail_info( const mail_info &other) : mailto(other.mailto), exec_host(other.exec_host), jobid(other.jobid), jobname(other.jobname), text(other.text), errFile(other.errFile), - outFile(other.outFile), mail_point(other.mail_point) + outFile(other.outFile), mail_point(other.mail_point), + queue_name(other.queue_name), owner(other.owner), + working_directory(other.working_directory), + resources_requested(other.resources_requested), + resources_used(other.resources_used) { } @@ -31,6 +88,11 @@ mail_info &mail_info::operator =( this->errFile = other.errFile; this->outFile = other.outFile; this->mail_point = other.mail_point; + this->queue_name = other.queue_name; + this->owner = other.owner; + this->working_directory = other.working_directory; + this->resources_requested = other.resources_requested; + this->resources_used = other.resources_used; return(*this); } diff --git a/src/server/node_func.c b/src/server/node_func.c index 8d41c3d4ca..ebd3304f4f 100644 --- a/src/server/node_func.c +++ b/src/server/node_func.c @@ -114,6 +114,7 @@ job *get_job_from_job_usage_info(job_usage_info *jui, struct pbsnode *pnode); +pbsnode *get_compute_node(const char *node_name); struct pbsnode *alps_reporter; @@ -2001,7 +2002,7 @@ void load_node_notes() (cray_enabled == true) && (isdigit(node_name.at(0)))) { - pnode = create_alps_subnode(alps_reporter, node_name.c_str()); + pnode = get_compute_node(node_name.c_str()); } if (pnode != NULL) diff --git a/src/server/node_manager.c b/src/server/node_manager.c index d79512141f..35a5860aa8 100644 --- a/src/server/node_manager.c +++ b/src/server/node_manager.c @@ -198,6 +198,9 @@ int default_gpu_mode = -1; #ifndef MAX_BM #define MAX_BM 64 #endif +#ifdef NVML_API +#include +#endif int handle_complete_first_time(job *pjob); int is_compute_node(char *node_id); @@ -572,7 +575,6 @@ int kill_job_on_mom( struct pbsnode *pnode) { - batch_request *preq; int rc = -1; int conn; int local_errno = 0; @@ -587,27 +589,20 @@ int kill_job_on_mom( if (conn >= 0) { - if ((preq = alloc_br(PBS_BATCH_SignalJob)) == NULL) - { - log_err(-1, __func__, "unable to allocate SignalJob request-trouble!"); - svr_disconnect(conn); - } - else - { - snprintf(preq->rq_ind.rq_signal.rq_jid, sizeof(preq->rq_ind.rq_signal.rq_jid), "%s", job_id); - snprintf(preq->rq_ind.rq_signal.rq_signame, sizeof(preq->rq_ind.rq_signal.rq_signame), "SIGKILL"); - preq->rq_extra = strdup(SYNC_KILL); - pnode->tmp_unlock_node(__func__, NULL, LOGLEVEL); - rc = issue_Drequest(conn, preq, true); + batch_request preq(PBS_BATCH_SignalJob); + + snprintf(preq.rq_ind.rq_signal.rq_jid, sizeof(preq.rq_ind.rq_signal.rq_jid), "%s", job_id); + snprintf(preq.rq_ind.rq_signal.rq_signame, sizeof(preq.rq_ind.rq_signal.rq_signame), "SIGKILL"); + preq.rq_extra = strdup(SYNC_KILL); + pnode->tmp_unlock_node(__func__, NULL, LOGLEVEL); + rc = issue_Drequest(conn, &preq, true); - if (preq->rq_reply.brp_code == PBSE_TIMEOUT) - update_failure_counts(node_name.c_str(), PBSE_TIMEOUT); - else - update_failure_counts(node_name.c_str(), 0); + if (preq.rq_reply.brp_code == PBSE_TIMEOUT) + update_failure_counts(node_name.c_str(), PBSE_TIMEOUT); + else + update_failure_counts(node_name.c_str(), 0); - free_br(preq); - pnode->tmp_lock_node(__func__, NULL, LOGLEVEL); - } + pnode->tmp_lock_node(__func__, NULL, LOGLEVEL); } else { @@ -1007,8 +1002,6 @@ int parse_job_information_from_legacy_format( { free(raw_job_data); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); - throw (int)PBSE_NODE_DELETED; } } @@ -1100,8 +1093,6 @@ void process_job_info_from_json( // re-lock the node if ((np = find_nodebyname(node_name.c_str())) == NULL) { - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); - throw (int)PBSE_NODE_DELETED; } @@ -1168,6 +1159,7 @@ void *sync_node_jobs( bool sync = sji->sync_jobs; std::string node_name(sji->node_name); std::string raw_job_info(sji->job_info); + int old_state = PTHREAD_CANCEL_ENABLE; delete sji; @@ -1176,7 +1168,7 @@ void *sync_node_jobs( return(NULL); } - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 0); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state); std::vector job_id_list; std::string job_list_str("jobs="); @@ -1212,7 +1204,7 @@ void *sync_node_jobs( np->unlock_node(__func__, NULL, LOGLEVEL); } - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(NULL); } } @@ -1225,7 +1217,7 @@ void *sync_node_jobs( log_err(-1, __func__, log_buf); np->unlock_node(__func__, NULL, LOGLEVEL); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(NULL); } @@ -1238,7 +1230,7 @@ void *sync_node_jobs( np->unlock_node(__func__, NULL, LOGLEVEL); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(NULL); } /* END sync_node_jobs() */ @@ -1581,8 +1573,6 @@ void *write_node_state_work( { fprintf(nstatef, fmt, np->get_name(), np->nd_state & savemask); } - - np->unlock_node(__func__, NULL, LOGLEVEL); } /* END for each node */ } else @@ -1726,8 +1716,8 @@ void write_node_power_state(void) int write_node_note(void) { - struct pbsnode *np; - all_nodes_iterator *iter = NULL; + struct pbsnode *np = NULL; + node_iterator iter; FILE *nin; if (LOGLEVEL >= 2) @@ -1747,9 +1737,11 @@ int write_node_note(void) return(-1); } + + reinitialize_node_iterator(&iter); /* for each node ... */ - while ((np = next_host(&allnodes, &iter, NULL)) != NULL) + while ((np = next_node(&allnodes, np, &iter)) != NULL) { /* write node name followed by its note string */ if (np->nd_note.size() != 0) @@ -1760,8 +1752,8 @@ int write_node_note(void) np->unlock_node(__func__, NULL, LOGLEVEL); } - if (iter != NULL) - delete iter; + if (iter.node_index != NULL) + delete iter.node_index; fflush(nin); @@ -1879,12 +1871,21 @@ static int hasppn( /* -** Count how many gpus are available for use on this node -*/ + * gpu_count() + * + * Get a gpu count for this node + * if freeonly is true, then count the free gpus; if false, then return the number + * of gpus. + * + * @param pnode - the node whose gpus we're counting + * @param freeonly - specifies if we want a total or just the available gpus + * @return the total gpus if freeonly is false, else the number of free gpus + */ + int gpu_count( - struct pbsnode *pnode, /* I */ - int freeonly) /* I */ + pbsnode *pnode, /* I */ + bool freeonly) /* I */ { int count = 0; @@ -1900,7 +1901,7 @@ int gpu_count( sprintf(log_buf, "Counted %d gpus %s on node %s that was skipped", count, - (freeonly? "free":"available"), + (freeonly? "free" : "total"), pnode->get_name()); log_ext(-1, __func__, log_buf, LOG_DEBUG); @@ -1908,49 +1909,17 @@ int gpu_count( return (count); } - if (pnode->nd_gpus_real) - { - int j; - - for (j = 0; j < pnode->nd_ngpus; j++) - { - struct gpusubn &gn = pnode->nd_gpusn[j]; - - /* always ignore unavailable gpus */ - if (gn.state == gpu_unavailable) - continue; - - if (!freeonly) - { - count++; - } - else if ((gn.state == gpu_unallocated) || - ((gn.state == gpu_shared) && - (gpu_mode_rqstd == gpu_normal))) - { - count++;; - } - } - } + if (freeonly == false) + count = pnode->nd_ngpus; else - { - /* virtual gpus */ - if (freeonly) - { - count = pnode->nd_ngpus_free; - } - else - { - count = pnode->nd_ngpus; - } - } + count = pnode->nd_ngpus_free; if (LOGLEVEL >= 7) { sprintf(log_buf, "Counted %d gpus %s on node %s", count, - (freeonly? "free":"available"), + (freeonly? "free" : "total"), pnode->get_name()); log_ext(-1, __func__, log_buf, LOG_DEBUG); @@ -2131,7 +2100,7 @@ int proplist( char name_storage[80]; char *pname; char *pequal; - int have_gpus = FALSE; + bool have_gpus = false; char log_buf[LOCAL_LOG_BUF_SIZE]; *node_req = 1; /* default to 1 processor per node */ @@ -2186,7 +2155,7 @@ int proplist( return(1); } - have_gpus = TRUE; + have_gpus = true; gpu_err_reset = FALSE; /* default to no */ // default value if no other gets specified @@ -2197,7 +2166,15 @@ int proplist( if (deflt_gpu_mode != NULL) gpu_mode_rqstd = default_gpu_mode; else + { +#if defined(NVML_API_VERSION) && (NVML_API_VERSION >= 8) + // exclusive thread mode deprecated starting in version 8 + // so use exlusive process instead + gpu_mode_rqstd = gpu_exclusive_process; +#else gpu_mode_rqstd = gpu_exclusive_thread; +#endif + } } else { @@ -2405,7 +2382,7 @@ bool node_is_spec_acceptable( return(false); if ((hasppn(pnode, ppn_req, SKIP_NONE) == FALSE) || - (gpu_count(pnode, FALSE) < gpu_req) || + (gpu_count(pnode, false) < gpu_req) || (pnode->nd_nmics < mic_req)) return(false); @@ -2414,7 +2391,7 @@ bool node_is_spec_acceptable( if (((pnode->nd_state & (INUSE_OFFLINE | INUSE_NOT_READY | INUSE_RESERVE | INUSE_JOB)) != 0)||(pnode->nd_power_state != POWER_STATE_RUNNING)) return(false); - gpu_free = gpu_count(pnode, TRUE) - pnode->nd_ngpus_to_be_used; + gpu_free = gpu_count(pnode, true) - pnode->nd_ngpus_to_be_used; np_free = pnode->nd_slots.get_number_free() - pnode->nd_np_to_be_used; mic_free = pnode->nd_nmics_free - pnode->nd_nmics_to_be_used; @@ -2422,7 +2399,8 @@ bool node_is_spec_acceptable( (gpu_req > gpu_free) || (mic_req > mic_free)) return(false); - if(job_is_exclusive) + + if (job_is_exclusive) { if(pnode->nd_slots.get_number_free() != pnode->nd_slots.get_total_execution_slots()) { @@ -2435,7 +2413,6 @@ bool node_is_spec_acceptable( - int parse_req_data( complete_spec_data &all_reqs) @@ -2561,7 +2538,7 @@ int save_node_for_adding( /* count off the number we have reserved */ pnode->nd_np_to_be_used += req.ppn; pnode->nd_ngpus_to_be_used += req.gpu; - pnode->nd_nmics_to_be_used -= req.mic; + pnode->nd_nmics_to_be_used += req.mic; return(PBSE_NONE); } /* END save_node_for_adding */ @@ -3012,7 +2989,7 @@ int select_nodes_using_hostlist( return(-2); } - if (node_is_spec_acceptable(pnode, req, ProcBMStr, eligible_nodes,job_is_exclusive) == false) + if (node_is_spec_acceptable(pnode, req, ProcBMStr, eligible_nodes, job_is_exclusive) == false) { snprintf(log_buf, sizeof(log_buf), "Requested node '%s' is not currently available", req.plist[0].name.c_str()); @@ -3712,23 +3689,17 @@ int reserve_node( int add_job_to_gpu_subnode( - struct pbsnode *pnode, - gpusubn &gn, - job *pjob) + pbsnode *pnode, + gpusubn &gn, + job *pjob) { - if (!pnode->nd_gpus_real) - { - /* update the gpu subnode */ - gn.job_internal_id = pjob->ji_internal_id; - gn.inuse = TRUE; - - /* update the main node */ - pnode->nd_ngpus_free--; - } - + gn.job_internal_id = pjob->ji_internal_id; + pnode->nd_ngpus_free--; + gn.inuse = true; gn.job_count++; - pnode->nd_ngpus_to_be_used--; + if (pnode->nd_ngpus_to_be_used > 0) + pnode->nd_ngpus_to_be_used--; return(PBSE_NONE); } /* END add_job_to_gpu_subnode() */ @@ -3851,30 +3822,8 @@ int place_gpus_in_hostlist( gpusubn &gn = pnode->nd_gpusn[j]; - if (pnode->nd_gpus_real) - { - if ((gn.state == gpu_unavailable) || - (gn.state == gpu_shared) || - (gn.state == gpu_exclusive) || - ((((int)gn.mode == gpu_normal)) && - (gpu_mode_rqstd != gpu_normal) && - (gn.state != gpu_unallocated))) - continue; - } - else - { - if ((gn.state == gpu_unavailable) || - (gn.inuse == TRUE)) - continue; - } - - if ((gn.state == gpu_unavailable) || - ((gn.state == gpu_exclusive) && pnode->nd_gpus_real) || - ((pnode->nd_gpus_real) && - ((int)gn.mode == gpu_normal) && - ((gpu_mode_rqstd != gpu_normal) && (gn.state != gpu_unallocated))) || - ((!pnode->nd_gpus_real) && - (gn.inuse == TRUE))) + if ((gn.inuse == true) || + (gn.state == gpu_unavailable)) continue; add_job_to_gpu_subnode(pnode,gn,pjob); @@ -4012,57 +3961,62 @@ int place_mics_in_hostlist( #ifdef PENABLE_LINUX_CGROUPS + /* - * save_cpus_and_memory_cpusets() - * - * Adds the cpus and mems to the job's list */ -void save_cpus_and_memory_cpusets( +void save_cgroup_string_attr( job *pjob, const char *node_name, - std::string &cpus, - std::string &mems) + std::string &value, + int index) { - if (pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str == NULL) - { - std::string formatted(node_name); - if (cpus.size() != 0) - formatted += ":" + cpus; - pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str = strdup(formatted.c_str()); - pjob->ji_wattr[JOB_ATR_cpuset_string].at_flags |= ATR_VFLAG_SET; - } - else - { - std::string all_cpus = pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str; - all_cpus += "+"; - all_cpus += node_name; - if (cpus.size() != 0) - all_cpus += ":" + cpus; - free(pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str); - pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str = strdup(all_cpus.c_str()); - } - - if (pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str == NULL) + if (pjob->ji_wattr[index].at_val.at_str == NULL) { std::string formatted(node_name); - if (mems.size() != 0) - formatted += ":" + mems; - pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str = strdup(formatted.c_str()); - pjob->ji_wattr[JOB_ATR_memset_string].at_flags |= ATR_VFLAG_SET; + + if (value.size() != 0) + formatted += ":" + value; + + pjob->ji_wattr[index].at_val.at_str = strdup(formatted.c_str()); + pjob->ji_wattr[index].at_flags |= ATR_VFLAG_SET; } else { - std::string all_mems = pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str; - all_mems += "+"; - all_mems += node_name; - if (mems.size() != 0) - all_mems += ":" + mems; - free(pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str); - pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str = strdup(all_mems.c_str()); + std::string concat = pjob->ji_wattr[index].at_val.at_str; + concat += "+"; + concat += node_name; + if (value.size() != 0) + concat += ":" + value; + + free(pjob->ji_wattr[index].at_val.at_str); + pjob->ji_wattr[index].at_val.at_str = strdup(concat.c_str()); } + } // END save_cgroup_string_attr() + +/* + * save_cpus_and_memory_cpusets() + * + * Adds the cpus and mems to the job's list + */ + +void save_cpus_and_memory_cpusets( + + job *pjob, + const char *node_name, + cgroup_info &cgi) + + { + save_cgroup_string_attr(pjob, node_name, cgi.cpu_string, JOB_ATR_cpuset_string); + save_cgroup_string_attr(pjob, node_name, cgi.mem_string, JOB_ATR_memset_string); + + if (cgi.gpu_string.size() > 0) + save_cgroup_string_attr(pjob, node_name, cgi.gpu_string, JOB_ATR_gpus_reserved); + + if (cgi.mic_string.size() > 0) + save_cgroup_string_attr(pjob, node_name, cgi.mic_string, JOB_ATR_mics_reserved); } // END save_cpus_and_memory_cpusets() @@ -4165,6 +4119,49 @@ void update_req_hostlist( } } // END update_req_hostlist() + + + +/* + * set_gpu_mode_if_needed() + * Sets the gpu mode if there's a default gpu mode, this job requests gpus, and this job + * doesn't set the gpu mode + * + * @param pjob - the job whose gpu mode we may set + */ + +void set_gpu_mode_if_needed( + + job *pjob) + + { + char *default_gpu_mode = NULL; + + if (pjob->ji_wattr[JOB_ATR_request_version].at_val.at_long > 1) + { + complete_req *cr = (complete_req *)pjob->ji_wattr[JOB_ATR_req_information].at_val.at_ptr; + + if (cr != NULL) + { + get_svr_attr_str(SRV_ATR_DefaultGpuMode, &default_gpu_mode); + + if (default_gpu_mode != NULL) + { + for (unsigned int i = 0; i < cr->get_num_reqs(); i++) + { + req &r = cr->get_req(i); + + if ((r.get_gpus() > 0) && + (r.get_gpu_mode().size() == 0)) + { + r.set_attribute(default_gpu_mode); + } + } + } + } + } + + } // END set_gpu_mode_if_needed() #endif @@ -4183,7 +4180,8 @@ int place_subnodes_in_hostlist( char *ProcBMStr) { - int rc = PBSE_NONE; + int rc = PBSE_NONE; + char log_buf[LOCAL_LOG_BUF_SIZE]; #ifdef GEOMETRY_REQUESTS if (IS_VALID_STR(ProcBMStr)) { @@ -4223,20 +4221,28 @@ int place_subnodes_in_hostlist( pnode->nd_state |= INUSE_JOB; #ifdef PENABLE_LINUX_CGROUPS - std::string cpus; - std::string mems; + cgroup_info cgi; bool legacy_vmem = false; get_svr_attr_b(SRV_ATR_LegacyVmem, &legacy_vmem); + set_gpu_mode_if_needed(pjob); + // We shouldn't be starting a job if the layout hasn't been set up yet. if (pnode->nd_layout.is_initialized() == false) return(-1); update_req_hostlist(pjob, pnode->get_name(), naji.req_index, naji.ppn_needed); - rc = pnode->nd_layout.place_job(pjob, cpus, mems, pnode->get_name(), legacy_vmem); + rc = pnode->nd_layout.place_job(pjob, cgi, pnode->get_name(), legacy_vmem); if (rc != PBSE_NONE) + { + snprintf(log_buf, sizeof(log_buf), + "Couldn't place job %s on node %s.", + pjob->ji_qs.ji_jobid, pnode->get_name()); + log_err(-1, __func__, log_buf); + return(rc); + } if ((pjob->ji_wattr[JOB_ATR_node_exclusive].at_flags & ATR_VFLAG_SET) && (pjob->ji_wattr[JOB_ATR_node_exclusive].at_val.at_long != 0) && @@ -4248,21 +4254,21 @@ int place_subnodes_in_hostlist( if (pnode->nd_layout.getTotalThreads() > 1) { sprintf(buf, "0-%d", pnode->nd_layout.getTotalThreads() - 1); - cpus = buf; + cgi.cpu_string = buf; } else - cpus = "0"; + cgi.cpu_string = "0"; if (pnode->nd_layout.getTotalChips() > 1) { sprintf(buf, "0-%d", pnode->nd_layout.getTotalChips() - 1); - mems = buf; + cgi.mem_string = buf; } else - mems = "0"; + cgi.mem_string = "0"; } - save_cpus_and_memory_cpusets(pjob, pnode->get_name(), cpus, mems); + save_cpus_and_memory_cpusets(pjob, pnode->get_name(), cgi); save_node_usage(pnode); #endif @@ -4271,6 +4277,11 @@ int place_subnodes_in_hostlist( } else { + snprintf(log_buf, sizeof(log_buf), + "Node %s doesn't have enough execution slots remaining for job %s.", + pnode->get_name(), pjob->ji_qs.ji_jobid); + log_err(-1, __func__, log_buf); + rc = -1; } @@ -4734,7 +4745,7 @@ void add_entry_to_naji_list( naji.node_id = pnode->nd_id; naji.ppn_needed = r.get_execution_slots() * tasks_placed; - naji.gpu_needed = r.getGpus() * tasks_placed; + naji.gpu_needed = r.get_gpus() * tasks_placed; naji.mic_needed = r.getMics() * tasks_placed; naji.is_external = false; @@ -5506,17 +5517,23 @@ char *get_next_exec_host( +/* + * remove_job_from_nodes_gpus() + * + * @param pnode - the node whose job is being removed + * @param pjob - the job that should be removed from the gpus + * @return PBSE_NONE + */ + int remove_job_from_nodes_gpus( - struct pbsnode *pnode, - job *pjob) + pbsnode *pnode, + job *pjob) { char *gpu_str = NULL; int i; char log_buf[LOCAL_LOG_BUF_SIZE]; - std::string tmp_str; - char num_str[6]; if (pjob->ji_wattr[JOB_ATR_exec_gpus].at_flags & ATR_VFLAG_SET) gpu_str = pjob->ji_wattr[JOB_ATR_exec_gpus].at_val.at_str; @@ -5528,56 +5545,27 @@ int remove_job_from_nodes_gpus( { gpusubn &gn = pnode->nd_gpusn[i]; - if (pnode->nd_gpus_real) + if (gn.job_internal_id == pjob->ji_internal_id) { - /* reset real gpu nodes */ - tmp_str = pnode->get_name(); - tmp_str += "-gpu/"; - sprintf (num_str, "%d", i); - tmp_str += num_str; - - /* look thru the string and see if it has this host and gpuid. - * exec_gpus string should be in format of - * -gpu/[+-gpu/...] - * - * if we are using the gpu node exclusively or if shared mode and - * this is last job assigned to this gpu then set it's state - * unallocated so its available for a new job. Takes time to get the - * gpu status report from the moms. - */ - - if (strstr(gpu_str, tmp_str.c_str()) != NULL) - { - gn.job_count--; + gn.job_internal_id = -1; + gn.inuse = false; + pnode->nd_ngpus_free++; + gn.job_count--; - if ((gn.mode == gpu_exclusive_thread) || - (gn.mode == gpu_exclusive_process) || - ((gn.mode == gpu_normal) && - (gn.job_count == 0))) - { - gn.state = gpu_unallocated; - - if (LOGLEVEL >= 7) - { - sprintf(log_buf, "freeing node %s gpu %d for job %s", - pnode->get_name(), - i, - pjob->ji_qs.ji_jobid); - - log_record(PBSEVENT_SCHED, PBS_EVENTCLASS_REQUEST, __func__, log_buf); - } - - } - } - } - else - { - if (gn.job_internal_id == pjob->ji_internal_id) + if ((gn.mode == gpu_exclusive_thread) || + (gn.mode == gpu_exclusive_process) || + ((gn.mode == gpu_normal) && + (gn.job_count == 0))) + gn.state = gpu_unallocated; + + if (LOGLEVEL >= 7) { - gn.inuse = FALSE; - gn.job_internal_id = -1; + sprintf(log_buf, "freeing node %s gpu %d for job %s", + pnode->get_name(), + i, + pjob->ji_qs.ji_jobid); - pnode->nd_ngpus_free++; + log_record(PBSEVENT_SCHED, PBS_EVENTCLASS_REQUEST, __func__, log_buf); } } } @@ -5588,11 +5576,18 @@ int remove_job_from_nodes_gpus( +/* + * remove_job_from_node() + * + * @param pnode - the node removeing the job + * @param internal_job_id - the internal job id from the job to be removed + * @return PBSE_NONE + */ int remove_job_from_node( - struct pbsnode *pnode, - int internal_job_id) + pbsnode *pnode, + int internal_job_id) { FUNCTION_TIMER @@ -5715,7 +5710,8 @@ void free_nodes( (pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN1) || (pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN2) || (pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN3) || - (pjob->ji_qs.ji_substate == JOB_SUBSTATE_QUEUED)) + (pjob->ji_qs.ji_substate == JOB_SUBSTATE_QUEUED) || + (pjob->ji_qs.ji_substate == JOB_SUBSTATE_TRNOUT)) { cr->clear_allocations(); } @@ -5741,10 +5737,9 @@ void free_nodes( - struct pbsnode *get_compute_node( - char *node_name) + const char *node_name) { struct pbsnode *ar = alps_reporter; @@ -5878,6 +5873,73 @@ int set_one_old( } /* END set_one_old() */ +/* + * Process gpu token of the form -gpu/[-] + * + * Set gpu subjob information for node. + */ + +int process_gpu_token( + + const char *gpu_token, + job *pjob) + + { + char *pc; + char *dash; + char *p; + int first; + int last; + struct pbsnode *pnode; + + // gpu_token expected to point to something like "numa3-gpu/2" + + if ((gpu_token == NULL) || (pjob == NULL)) + return(-1); + + // calculate range indices after the / + if ((pc = strchr((char *)gpu_token, (int)'/'))) + { + first = strtol(pc + 1, &dash, 10); + + *pc = '\0'; + + if (*dash == '-') + last = strtol(dash + 1, NULL, 10); + else + last = first; + } + else + { + first = 0; + last = first; + } + + // drop -gpu suffix + if ((p = strrchr((char *)gpu_token, (int)'-')) != NULL) + { + if (strcmp(p, "-gpu") == 0) + *p = '\0'; + } + + // lookup node and set gpu info on each gpu subnode + if ((pnode = find_nodebyname(gpu_token)) != NULL) + { + int i; + + for (i = first; i <= last; i++) + { + gpusubn &gn = pnode->nd_gpusn[i]; + + add_job_to_gpu_subnode(pnode, gn, pjob); + } + + // unlock node + pnode->unlock_node(__func__, NULL, LOGLEVEL); + } + + return(PBSE_NONE); + } /* * set_old_nodes - set "old" nodes as in use - called from pbsd_init() @@ -5893,6 +5955,40 @@ int set_old_nodes( char *po; int rc = PBSE_NONE; + // handle gpu info + if (pjob->ji_wattr[JOB_ATR_exec_gpus].at_flags & ATR_VFLAG_SET) + { + char *old_str; + + if ((old_str = strdup(pjob->ji_wattr[JOB_ATR_exec_gpus].at_val.at_str)) == NULL) + { + return(PBSE_SYSTEM); + } + + while ((po = strrchr(old_str, (int)'+')) != NULL) + { + // remove + + *po++ = '\0'; + + if (process_gpu_token(po, pjob) != PBSE_NONE) + { + free(old_str); + return(PBSE_SYSTEM); + } + } + + if (old_str != NULL) + { + if (process_gpu_token(old_str, pjob) != PBSE_NONE) + { + free(old_str); + return(PBSE_SYSTEM); + } + } + + free(old_str); + } + if (pjob->ji_wattr[JOB_ATR_exec_host].at_flags & ATR_VFLAG_SET) { old = strdup(pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str); diff --git a/src/server/node_power_state.c b/src/server/node_power_state.c index b58a6a4600..03f6b7a769 100644 --- a/src/server/node_power_state.c +++ b/src/server/node_power_state.c @@ -243,7 +243,7 @@ int set_node_power_state( //Can't change the power state on a node with running jobs. return PBSE_CANT_CHANGE_POWER_STATE_WITH_JOBS_RUNNING; } - struct batch_request *request = alloc_br(PBS_BATCH_ChangePowerState); + batch_request *request = new batch_request(PBS_BATCH_ChangePowerState); if (request == NULL) { return PBSE_SYSTEM; diff --git a/src/server/pbsd_init.c b/src/server/pbsd_init.c index 8727f2f46f..0233800586 100644 --- a/src/server/pbsd_init.c +++ b/src/server/pbsd_init.c @@ -137,6 +137,7 @@ #include "id_map.hpp" #include "exiting_jobs.h" #include "mom_hierarchy_handler.h" +#include "policy_values.h" /*#ifndef SIGKILL*/ @@ -1987,6 +1988,7 @@ int cleanup_recovered_arrays() char arrayid[PBS_MAXSVRJOBID+1]; all_arrays_iterator *iter = NULL; int rc = PBSE_NONE; + char log_buf[LOCAL_LOG_BUF_SIZE]; while ((pa = next_array(&iter)) != NULL) { @@ -2031,9 +2033,11 @@ int cleanup_recovered_arrays() if (job_template_exists == FALSE) { - int i; + snprintf(log_buf, sizeof(log_buf), "Cleaning up job array %s of size %d that could not be built during ini t.", + arrayid, pa->ai_qs.array_size); + log_event(PBSEVENT_DEBUG | PBSEVENT_FORCE, PBS_EVENTCLASS_JOB, __func__, log_buf); - for (i = 0; i < pa->ai_qs.array_size; i++) + for (int i = 0; i < pa->ai_qs.array_size; i++) { if (pa->job_ids[i] != NULL) { @@ -2043,13 +2047,21 @@ int cleanup_recovered_arrays() svr_job_purge(pjob); pa = get_array(arrayid); + if (pa == NULL) + { + break; + } } } } - std::string array_id(pa->ai_qs.parent_id); - unlock_ai_mutex(pa, __func__, "1", LOGLEVEL); - array_delete(array_id.c_str()); + if (pa != NULL) + { + std::string array_id(pa->ai_qs.parent_id); + unlock_ai_mutex(pa, __func__, "1", LOGLEVEL); + array_delete(array_id.c_str()); + } + continue; } else @@ -2220,8 +2232,9 @@ void setup_threadpool() void set_server_policies() { - bool cray = false; - bool recover_subjobs = false; + bool cray = false; + bool recover_subjobs = false; + char *default_gpu_str = NULL; if (get_svr_attr_b(SRV_ATR_CrayEnabled, &cray) == PBSE_NONE) cray_enabled = cray; @@ -2229,6 +2242,8 @@ void set_server_policies() if (get_svr_attr_b(SRV_ATR_GhostArrayRecovery, &recover_subjobs) == PBSE_NONE) ghost_array_recovery = recover_subjobs; + if (get_svr_attr_str(SRV_ATR_DefaultGpuMode, &default_gpu_str) == PBSE_NONE) + set_default_gpu_mode_int(default_gpu_str); } // END set_server_policies() @@ -2769,7 +2784,6 @@ void check_jobs_queue( { std::string queue_name(pjob->ji_qs.ji_queue); - std::string job_id(pjob->ji_qs.ji_jobid); unlock_ji_mutex(pjob, __func__, "1", LOGLEVEL); @@ -2794,8 +2808,9 @@ void check_jobs_queue( } unlock_queue(pque, __func__, NULL, LOGLEVEL); - - pjob = svr_find_job(job_id.c_str(), TRUE); + + // The job hasn't been queued, so we just re-lock it this way + lock_ji_mutex(pjob, __func__, "1", LOGLEVEL); } // END check_jobs_queue() diff --git a/src/server/pbsd_main.c b/src/server/pbsd_main.c index 530abd7458..b927d254a9 100644 --- a/src/server/pbsd_main.c +++ b/src/server/pbsd_main.c @@ -318,7 +318,6 @@ int MultiMomMode = 0; int allow_any_mom = FALSE; int array_259_upgrade = FALSE; -sem_t *job_clone_semaphore; /* used to track the number of job_clone_wt requests are outstanding */ acl_special limited_acls; authorized_hosts auth_hosts; @@ -1464,25 +1463,6 @@ void main_loop(void) track_save(NULL); /* save tracking data */ - /* let any array jobs that might still be cloning finish */ - int sem_val; - do - { - int rc; - - rc = sem_getvalue(job_clone_semaphore, &sem_val); - if (rc != 0) - { - sprintf(log_buf, "failed to get job_clone_semaphore value"); - log_err(-1, __func__, log_buf); - break; - } - - if (sem_val > 0) - sleep(1); - }while(sem_val > 0); - - alljobs.lock(); iter = alljobs.get_iterator(); alljobs.unlock(); @@ -1610,7 +1590,6 @@ int main( { int i; - int rc; int local_errno = 0; char lockfile[MAXPATHLEN + 1]; char EMsg[MAX_LINE]; @@ -1684,21 +1663,6 @@ int main( if (server_name_file_port != 0) pbs_server_port_dis = server_name_file_port; - /* initialize job clone semaphore so we can track the number of outstanding - job array creation requests */ - job_clone_semaphore = (sem_t *)malloc(sizeof(sem_t)); - if (job_clone_semaphore == NULL) - { - perror("failed to allocate memory for job_clone_semaphore"); - exit(1); - } - rc = sem_init(job_clone_semaphore, 1 /* share */, 0); - if (rc != 0) - { - perror("failed to initialize job clone semaphore"); - exit(1); - } - strcpy(pbs_server_name, server_name); /* The following port numbers might have been initialized in set_globals_from_environment() above. */ @@ -3084,7 +3048,7 @@ int unlock_sv_qs_mutex(pthread_mutex_t *sv_qs_mutex, const char *msg_string) // Stub out some functions for NUMA #ifdef NVIDIA_GPUS int Machine::initializeNVIDIADevices(hwloc_obj_t, hwloc_topology_t) {return(0);} -extern void PCI_Device::initializeGpu(int idx, hwloc_topology_t topology) {} +extern void PCI_Device::initializeGpu(int idx) {} #endif #ifdef MIC int Chip::initializeMICDevices(hwloc_obj_t chip_obj, hwloc_topology_t topology) {return(0);} diff --git a/src/server/process_alps_status.c b/src/server/process_alps_status.c index cc3be2a531..8415492a07 100644 --- a/src/server/process_alps_status.c +++ b/src/server/process_alps_status.c @@ -215,7 +215,6 @@ void *check_if_orphaned( char *node_name = (char *)vp; char *rsv_id = NULL; std::string job_id; - batch_request *preq; int handle = -1; int retries = 0; struct pbsnode *pnode; @@ -250,20 +249,13 @@ void *check_if_orphaned( pnode->unlock_node(__func__, NULL, LOGLEVEL); } - if ((preq = alloc_br(PBS_BATCH_DeleteReservation)) == NULL) - { - free(node_name); - alps_reservations.remove_from_orphaned_list(rsv_id); - return(NULL); - } - - preq->rq_extend = strdup(rsv_id); - if ((pnode = get_next_login_node(NULL)) != NULL) { struct in_addr hostaddr; int local_errno; pbs_net_t momaddr; + batch_request preq(PBS_BATCH_DeleteReservation); + preq.rq_extend = strdup(rsv_id); memcpy(&hostaddr, &pnode->nd_sock_addr.sin_addr, sizeof(hostaddr)); momaddr = ntohl(hostaddr.s_addr); @@ -286,9 +278,7 @@ void *check_if_orphaned( pnode->unlock_node(__func__, NULL, LOGLEVEL); if (handle >= 0) - issue_Drequest(handle, preq, true); - - free_br(preq); + issue_Drequest(handle, &preq, true); } alps_reservations.remove_from_orphaned_list(rsv_id); @@ -778,8 +768,8 @@ int process_alps_status( set_ncpus(current, parent, ncpus); #ifdef PENABLE_LINUX_CGROUPS - if (numa_nodes == 0) - numa_nodes = 1; + if (numa_nodes < sockets) + numa_nodes = sockets; if ((current->nd_layout.is_initialized() == false) || (current->nd_layout.getTotalThreads() != current->nd_slots.get_total_execution_slots())) diff --git a/src/server/process_mom_update.c b/src/server/process_mom_update.c index fb0d59dee3..962df66c54 100644 --- a/src/server/process_mom_update.c +++ b/src/server/process_mom_update.c @@ -113,16 +113,16 @@ int gpu_has_job(struct pbsnode *pnode, int gpuid); void move_past_mic_status( - unsigned int &i, + unsigned int &status_index, std::vector &status_info) { - while (i < status_info.size()) + while (status_index < status_info.size()) { - if (!strcmp(status_info[i].c_str(), END_MIC_STATUS)) + if (!strcmp(status_info[status_index].c_str(), END_MIC_STATUS)) break; - i++; + status_index++; } } /* END move_past_mic_status() */ @@ -154,7 +154,7 @@ int save_single_mic_status( int process_mic_status( struct pbsnode *pnode, - unsigned int &i, + unsigned int &status_index, std::vector &status_info) { @@ -168,13 +168,13 @@ int process_mic_status( if ((rc = decode_arst(&temp, NULL, NULL, NULL, 0)) != PBSE_NONE) { log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, __func__, "cannot initialize attribute"); - move_past_mic_status(i, status_info); + move_past_mic_status(status_index, status_info); return(rc); } - for (i++; i < status_info.size(); i++) + for (status_index++; status_index < status_info.size(); status_index++) { - const char *str = status_info[i].c_str(); + const char *str = status_info[status_index].c_str(); if (!strcmp(str, END_MIC_STATUS)) break; @@ -214,7 +214,7 @@ int process_mic_status( } } - move_past_mic_status(i, status_info); + move_past_mic_status(status_index, status_info); node_micstatus_list(&temp, pnode, ATR_ACTION_ALTER); @@ -749,13 +749,15 @@ int save_node_status( void update_layout_if_needed( pbsnode *pnode, - const std::string &layout) + const std::string &layout, + bool force) { char log_buf[LOCAL_LOG_BUF_SIZE]; std::vector valid_ids; - if (pnode->nd_layout.is_initialized() == false) + if ((pnode->nd_layout.is_initialized() == false) || + (force == true)) { for (size_t i = 0; i < pnode->nd_job_usages.size(); i++) { @@ -765,9 +767,15 @@ void update_layout_if_needed( valid_ids.push_back(id); } - pnode->nd_layout.reinitialize_from_json(layout, valid_ids); + Machine m(layout, valid_ids); + + if (pnode->nd_layout.is_initialized() == true) + m.save_allocations(pnode->nd_layout); + + pnode->nd_layout = m; } - else if ((pnode->nd_layout.getTotalThreads() != pnode->nd_slots.get_total_execution_slots()) && + else if (((pnode->nd_layout.getTotalThreads() != pnode->nd_slots.get_total_execution_slots()) || + (pnode->nd_ngpus != pnode->nd_layout.get_total_gpus())) && (pnode->nd_job_usages.size() == 0)) { int old_count = pnode->nd_layout.getTotalThreads(); @@ -824,6 +832,9 @@ int process_status_info( int rc = PBSE_NONE; bool send_hello = false; std::string temp; +#ifdef PENABLE_LINUX_CGROUPS + bool force_layout_update = false; +#endif get_svr_attr_b(SRV_ATR_MomJobSync, &mom_job_sync); get_svr_attr_b(SRV_ATR_AutoNodeNP, &auto_np); @@ -905,18 +916,26 @@ int process_status_info( else if (!strcmp(str, START_GPU_STATUS)) { is_gpustat_get(current, i, status_info); - str = status_info[i].c_str(); + continue; } else if (!strcmp(str, START_MIC_STATUS)) { process_mic_status(current, i, status_info); - str = status_info[i].c_str(); + continue; } #ifdef PENABLE_LINUX_CGROUPS + else if (!strcmp(str, "force_layout_update")) + { + force_layout_update = true; + continue; + } else if (!strncmp(str, "layout", 6)) { // Add 7 to skip "layout=" - update_layout_if_needed(current, str + 7); + update_layout_if_needed(current, str + 7, force_layout_update); + + // reset this to false in case we have a mom hierarchy in place + force_layout_update = false; continue; } @@ -947,9 +966,12 @@ int process_status_info( /* reset gpu data in case mom reconnects with changed gpus */ clear_nvidia_gpus(current); + + continue; } else { + // Save this string to our status line. if (temp.size() > 0) temp += ","; @@ -968,52 +990,53 @@ int process_status_info( } else temp += str; - } - - if (!strncmp(str, "state", 5)) - { - if (dont_change_state == FALSE) - process_state_str(current, str); - } - else if ((allow_any_mom == TRUE) && - (!strncmp(str, "uname", 5))) - { - process_uname_str(current, str); - } - else if (!strncmp(str, "me", 2)) /* shorter str compare than "message" */ - { - if ((!strncmp(str, "message=ERROR", 13)) && - (down_on_error == TRUE)) + + if (!strncmp(str, "state", 5)) { - update_node_state(current, INUSE_DOWN); - dont_change_state = TRUE; - - if (note_append_on_error == true) + if (dont_change_state == FALSE) + process_state_str(current, str); + } + else if ((allow_any_mom == TRUE) && + (!strncmp(str, "uname", 5))) + { + process_uname_str(current, str); + } + else if (!strncmp(str, "me", 2)) /* shorter str compare than "message" */ + { + if ((!strncmp(str, "message=ERROR", 13)) && + (down_on_error == TRUE)) { - set_note_error(current, str); + update_node_state(current, INUSE_DOWN); + dont_change_state = TRUE; + + if (note_append_on_error == true) + { + set_note_error(current, str); + } } } - } - else if (!strncmp(str,"macaddr=",8)) - { - update_node_mac_addr(current,str + 8); - } - else if ((mom_job_sync == true) && - (!strncmp(str, "jobdata=", 8))) - { - /* update job attributes based on what the MOM gives us */ - update_job_data(current, str + strlen("jobdata=")); - } - else if ((auto_np) && - (!(strncmp(str, "ncpus=", 6)))) + else if (!strncmp(str,"macaddr=",8)) + { + update_node_mac_addr(current,str + 8); + } + else if ((mom_job_sync == true) && + (!strncmp(str, "jobdata=", 8))) + { + /* update job attributes based on what the MOM gives us */ + update_job_data(current, str + strlen("jobdata=")); + } + else if ((auto_np) && + (!(strncmp(str, "ncpus=", 6)))) - { - handle_auto_np(current, str); - } - else if (!strncmp(str, "version=", 8)) - { - current->set_version(str + 8); + { + handle_auto_np(current, str); + } + else if (!strncmp(str, "version=", 8)) + { + current->set_version(str + 8); + } } + } /* END processing strings */ if (current != NULL) diff --git a/src/server/process_request.c b/src/server/process_request.c index 4d4f43b578..519604c2f9 100644 --- a/src/server/process_request.c +++ b/src/server/process_request.c @@ -188,9 +188,6 @@ static const int munge_on = 1; static const int munge_on = 0; #endif -static void freebr_manage(struct rq_manage *); -static void freebr_cpyfile(struct rq_cpyfile *); -static void free_rescrq(struct rq_rescq *); void close_quejob(int sfds); /* END private prototypes */ @@ -332,13 +329,22 @@ bool request_passes_acl_check( -batch_request *read_request_from_socket( +/* + * read_request_from_socket() + * + * Initializes request from the data on the socket + * @param chan - the wrapper for the socket we're reading data from + * @param request - the request we're initializing + * @return PBSE_* to indicate what happened + */ - tcp_chan *chan) +int read_request_from_socket( + + tcp_chan *chan, + batch_request &request) { int rc = PBSE_NONE; - struct batch_request *request = NULL; char log_buf[LOCAL_LOG_BUF_SIZE]; time_t time_now = time(NULL); @@ -353,7 +359,7 @@ batch_request *read_request_from_socket( if ((sfds < 0) || (sfds >= PBS_NET_MAX_CONNECTIONS)) - return(NULL); + return(-1); pthread_mutex_lock(svr_conn[sfds].cn_mutex); conn_active = svr_conn[sfds].cn_active; @@ -365,10 +371,7 @@ batch_request *read_request_from_socket( svr_conn[sfds].cn_lasttime = time_now; pthread_mutex_unlock(svr_conn[sfds].cn_mutex); - if ((request = alloc_br(0)) == NULL) - return(NULL); - - request->rq_conn = sfds; + request.rq_conn = sfds; /* * Read in the request and decode it to the internal request structure. @@ -386,14 +389,14 @@ batch_request *read_request_from_socket( } #endif /* END ENABLE_UNIX_SOCKETS */ - rc = dis_request_read(chan, request); + rc = dis_request_read(chan, &request); if ((rc == PBSE_SYSTEM) || (rc == PBSE_INTERNAL) || (rc == PBSE_SOCKET_CLOSE)) { /* read error, likely cannot send reply so just disconnect, indicate permanent * failure by setting the type to PBS_BATCH_Disconnect */ - request->rq_type = PBS_BATCH_Disconnect; - return(request); + request.rq_type = PBS_BATCH_Disconnect; + return(rc); } else if (rc > 0) { @@ -403,8 +406,8 @@ batch_request *read_request_from_socket( * request didn't decode, either garbage or unknown * request type, in either case, return reject-reply */ - request->rq_failcode = rc; - return(request); + request.rq_failcode = rc; + return(rc); } } else @@ -419,11 +422,11 @@ batch_request *read_request_from_socket( "request on invalid type of connection (%d) from %s", conn_active, netaddr_long(conn_addr, out)); - req_reject(PBSE_BADHOST, 0, request, NULL, tmpLine); - return(NULL); + req_reject(PBSE_BADHOST, 0, &request, NULL, tmpLine); + return(-1); } - if (get_connecthost(sfds, request->rq_host, PBS_MAXHOSTNAME) != 0) + if (get_connecthost(sfds, request.rq_host, PBS_MAXHOSTNAME) != 0) { char ipstr[80]; sprintf(log_buf, "%s: %s", @@ -437,17 +440,17 @@ batch_request *read_request_from_socket( "cannot determine hostname for connection from %s", log_buf); - req_reject(PBSE_BADHOST, 0, request, NULL, tmpLine); - return(NULL); + req_reject(PBSE_BADHOST, 0, &request, NULL, tmpLine); + return(-1); } if (LOGLEVEL >= 8) { sprintf(log_buf, msg_request, - reqtype_to_txt(request->rq_type), - request->rq_user, - request->rq_host, + reqtype_to_txt(request.rq_type), + request.rq_user, + request.rq_host, sfds); log_event(PBSEVENT_DEBUG2, PBS_EVENTCLASS_REQUEST, "", log_buf); @@ -455,22 +458,22 @@ batch_request *read_request_from_socket( /* is the request from a host acceptable to the server */ if (conn_socktype & PBS_SOCK_UNIX) - strcpy(request->rq_host, server_name); + strcpy(request.rq_host, server_name); - if (request_passes_acl_check(request, conn_addr) == false) + if (request_passes_acl_check(&request, conn_addr) == false) { /* See if the request is in the limited acl list */ - if (limited_acls.is_authorized(request->rq_host, request->rq_user) == false) + if (limited_acls.is_authorized(request.rq_host, request.rq_user) == false) { char tmpLine[MAXLINE]; snprintf(tmpLine, sizeof(tmpLine), "request not authorized from host %s", - request->rq_host); - req_reject(PBSE_BADHOST, 0, request, NULL, tmpLine); - return(NULL); + request.rq_host); + req_reject(PBSE_BADHOST, 0, &request, NULL, tmpLine); + return(-1); } } - return(request); + return(PBSE_NONE); } /* END read_request_from_socket() */ @@ -494,7 +497,7 @@ int process_request( { int rc = PBSE_NONE; - struct batch_request *request = NULL; + batch_request request; long state = SV_STATE_DOWN; time_t time_now = time(NULL); @@ -513,18 +516,13 @@ int process_request( svr_conn[sfds].cn_lasttime = time_now; pthread_mutex_unlock(svr_conn[sfds].cn_mutex); - request = read_request_from_socket(chan); + rc = read_request_from_socket(chan, request); - if (request == NULL) - rc = -1; - else if (request->rq_type == PBS_BATCH_Disconnect) + if (request.rq_type == PBS_BATCH_Disconnect) rc = PBSE_SOCKET_CLOSE; - else - rc = request->rq_failcode; if (rc != PBSE_NONE) { - free_br(request); return(rc); } @@ -537,9 +535,9 @@ int process_request( { /* request came from another server */ - request->rq_fromsvr = 1; + request.rq_fromsvr = 1; - request->rq_perm = + request.rq_perm = ATR_DFLAG_USRD | ATR_DFLAG_USWR | ATR_DFLAG_OPRD | ATR_DFLAG_OPWR | ATR_DFLAG_MGRD | ATR_DFLAG_MGWR | @@ -550,7 +548,7 @@ int process_request( /* request not from another server */ conn_credent[sfds].timestamp = time_now; - request->rq_fromsvr = 0; + request.rq_fromsvr = 0; /* * Client must be authenticated by an Authenticate User Request, if not, @@ -568,9 +566,9 @@ int process_request( * creds. Authorization is still granted in svr_get_privilege below */ - if (request->rq_type == PBS_BATCH_Connect) + if (request.rq_type == PBS_BATCH_Connect) { - return(req_connect(request)); + return(req_connect(&request)); } if (conn_socktype & PBS_SOCK_UNIX) @@ -586,14 +584,14 @@ int process_request( else if (munge_on) { /* If munge_on is true we will validate the connection now */ - if (request->rq_type == PBS_BATCH_AltAuthenUser) + if (request.rq_type == PBS_BATCH_AltAuthenUser) { - rc = req_altauthenuser(request); + rc = req_altauthenuser(&request); return(rc); } else { - rc = authenticate_user(request, &conn_credent[sfds], &auth_err); + rc = authenticate_user(&request, &conn_credent[sfds], &auth_err); } } #endif @@ -601,11 +599,11 @@ int process_request( /* skip checking user if we did not get an authenticated credential */ rc = PBSE_BADCRED; else - rc = authenticate_user(request, &conn_credent[sfds], &auth_err); + rc = authenticate_user(&request, &conn_credent[sfds], &auth_err); if (rc != 0) { - req_reject(rc, 0, request, NULL, auth_err); + req_reject(rc, 0, &request, NULL, auth_err); if (auth_err != NULL) free(auth_err); @@ -618,9 +616,9 @@ int process_request( * for root on the jobs execution node. */ - if (((request->rq_type == PBS_BATCH_ModifyJob) || - (request->rq_type == PBS_BATCH_ReleaseJob)) && - (strcmp(request->rq_user, PBS_DEFAULT_ADMIN) == 0)) + if (((request.rq_type == PBS_BATCH_ModifyJob) || + (request.rq_type == PBS_BATCH_ReleaseJob)) && + (strcmp(request.rq_user, PBS_DEFAULT_ADMIN) == 0)) { job *pjob; char *dptr; @@ -629,13 +627,13 @@ int process_request( /* make short host name */ - strcpy(short_host, request->rq_host); + strcpy(short_host, request.rq_host); if ((dptr = strchr(short_host, '.')) != NULL) { *dptr = '\0'; } - if ((pjob = svr_find_job(request->rq_ind.rq_modify.rq_objname, FALSE)) != NULL) + if ((pjob = svr_find_job(request.rq_ind.rq_modify.rq_objname, FALSE)) != NULL) { mutex_mgr pjob_mutex = mutex_mgr(pjob->ji_mutex, true); if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING) @@ -647,7 +645,7 @@ int process_request( (csv_find_string(pjob->ji_wattr[JOB_ATR_checkpoint].at_val.at_str, "enabled") != NULL)) && (strstr(pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str, short_host) != NULL)) { - request->rq_perm = svr_get_privilege(request->rq_user, server_host); + request.rq_perm = svr_get_privilege(request.rq_user, server_host); skip = TRUE; } } @@ -656,18 +654,18 @@ int process_request( if (!skip) { - request->rq_perm = svr_get_privilege(request->rq_user, request->rq_host); + request.rq_perm = svr_get_privilege(request.rq_user, request.rq_host); } } else { - request->rq_perm = svr_get_privilege(request->rq_user, request->rq_host); + request.rq_perm = svr_get_privilege(request.rq_user, request.rq_host); } } /* END else (conn_authen == PBS_NET_CONN_FROM_PRIVIL) */ - if (threadpool_is_too_busy(request_pool, request->rq_perm)) + if (threadpool_is_too_busy(request_pool, request.rq_perm)) { - req_reject(PBSE_SERVER_BUSY, 0, request, NULL, NULL); + req_reject(PBSE_SERVER_BUSY, 0, &request, NULL, NULL); return(PBSE_SERVER_BUSY); } @@ -676,7 +674,7 @@ int process_request( if (state > SV_STATE_RUN) { - switch (request->rq_type) + switch (request.rq_type) { case PBS_BATCH_AsyrunJob: case PBS_BATCH_JobCred: @@ -688,7 +686,7 @@ int process_request( case PBS_BATCH_jobscript: case PBS_BATCH_jobscript2: - req_reject(PBSE_SVRDOWN, 0, request, NULL, NULL); + req_reject(PBSE_SVRDOWN, 0, &request, NULL, NULL); return(PBSE_SVRDOWN); } @@ -700,7 +698,7 @@ int process_request( * the request struture. */ - rc = dispatch_request(sfds, request); + rc = dispatch_request(sfds, &request); return(rc); } /* END process_request() */ @@ -1029,40 +1027,6 @@ int dispatch_request( - -/* - * alloc_br - allocate and clear a batch_request structure - */ - -struct batch_request *alloc_br( - - int type) - - { - - struct batch_request *req = NULL; - - if ((req = (batch_request *)calloc(1, sizeof(batch_request))) == NULL) - { - log_err(errno, __func__, msg_err_malloc); - } - else - { - - req->rq_type = type; - - req->rq_conn = -1; /* indicate not connected */ - req->rq_orgconn = -1; /* indicate not connected */ - req->rq_time = time(NULL); - req->rq_reply.brp_choice = BATCH_REPLY_CHOICE_NULL; - req->rq_noreply = FALSE; /* indicate reply is needed */ - } - - return(req); - } /* END alloc_br() */ - - - /* * close_quejob() * @@ -1139,241 +1103,15 @@ void close_quejob( void free_br( - struct batch_request *preq) + batch_request *preq) { if (preq == NULL) return; - if (preq->rq_id != NULL) - { - remove_batch_request(preq->rq_id); - free(preq->rq_id); - preq->rq_id = NULL; - } - - reply_free(&preq->rq_reply); - - if (preq->rq_extend) - { - free(preq->rq_extend); - preq->rq_extend = NULL; - } - - if (preq->rq_extra) - { - free(preq->rq_extra); - preq->rq_extra = NULL; - } - - switch (preq->rq_type) - { - case PBS_BATCH_QueueJob: - case PBS_BATCH_QueueJob2: - - free_attrlist(&preq->rq_ind.rq_queuejob.rq_attr); - - break; - - case PBS_BATCH_JobCred: - - if (preq->rq_ind.rq_jobcred.rq_data) - { - free(preq->rq_ind.rq_jobcred.rq_data); - preq->rq_ind.rq_jobcred.rq_data = NULL; - } - - break; - - case PBS_BATCH_MvJobFile: - case PBS_BATCH_jobscript: - case PBS_BATCH_jobscript2: - - if (preq->rq_ind.rq_jobfile.rq_data) - { - free(preq->rq_ind.rq_jobfile.rq_data); - preq->rq_ind.rq_jobfile.rq_data = NULL; - } - break; - - case PBS_BATCH_HoldJob: - - freebr_manage(&preq->rq_ind.rq_hold.rq_orig); - - break; - - case PBS_BATCH_CheckpointJob: - - freebr_manage(&preq->rq_ind.rq_manager); - - break; - - case PBS_BATCH_MessJob: - - if (preq->rq_ind.rq_message.rq_text) - { - free(preq->rq_ind.rq_message.rq_text); - preq->rq_ind.rq_message.rq_text = NULL; - } - - break; - - case PBS_BATCH_ModifyJob: - - case PBS_BATCH_AsyModifyJob: - - freebr_manage(&preq->rq_ind.rq_modify); - - break; - - case PBS_BATCH_StatusJob: - - case PBS_BATCH_StatusQue: - - case PBS_BATCH_StatusNode: - - case PBS_BATCH_StatusSvr: - /* DIAGTODO: handle PBS_BATCH_StatusDiag */ - - free_attrlist(&preq->rq_ind.rq_status.rq_attr); - - break; - - case PBS_BATCH_JobObit: - - free_attrlist(&preq->rq_ind.rq_jobobit.rq_attr); - - break; - - case PBS_BATCH_CopyFiles: - - case PBS_BATCH_DelFiles: - - freebr_cpyfile(&preq->rq_ind.rq_cpyfile); - - break; - - case PBS_BATCH_ModifyNode: - - case PBS_BATCH_Manager: - - freebr_manage(&preq->rq_ind.rq_manager); - - break; - - case PBS_BATCH_ReleaseJob: - - freebr_manage(&preq->rq_ind.rq_release); - - break; - - case PBS_BATCH_Rescq: - - free_rescrq(&preq->rq_ind.rq_rescq); - - break; - - case PBS_BATCH_SelectJobs: - - case PBS_BATCH_SelStat: - - free_attrlist(&preq->rq_ind.rq_select); - - break; - - case PBS_BATCH_SelStatAttr: - - free_attrlist(&preq->rq_ind.rq_select); - free_attrlist(&preq->rq_ind.rq_status.rq_attr); - - break; - - case PBS_BATCH_RunJob: - case PBS_BATCH_AsyrunJob: - - if (preq->rq_ind.rq_run.rq_destin) - { - free(preq->rq_ind.rq_run.rq_destin); - preq->rq_ind.rq_run.rq_destin = NULL; - } - break; - - default: - - /* NO-OP */ - - break; - } /* END switch (preq->rq_type) */ - - free(preq); - - return; + delete preq; } /* END free_br() */ - - -static void freebr_manage( - - struct rq_manage *pmgr) - - { - free_attrlist(&pmgr->rq_attr); - - return; - } /* END freebr_manage() */ - - - - -static void freebr_cpyfile( - - struct rq_cpyfile *pcf) - - { - - struct rqfpair *ppair; - - while ((ppair = (struct rqfpair *)GET_NEXT(pcf->rq_pair)) != NULL) - { - delete_link(&ppair->fp_link); - - if (ppair->fp_local != NULL) - free(ppair->fp_local); - - if (ppair->fp_rmt != NULL) - free(ppair->fp_rmt); - - free(ppair); - } - - return; - } /* END freebr_cpyfile() */ - - - - - -static void free_rescrq( - - struct rq_rescq *pq) - - { - int i; - - i = pq->rq_num; - - while (i--) - { - if (*(pq->rq_list + i) != NULL) - free(*(pq->rq_list + i)); - } - - if (pq->rq_list != NULL) - free(pq->rq_list); - - return; - } /* END free_rescrq() */ - /* END process_requests.c */ diff --git a/src/server/receive_mom_communication.c b/src/server/receive_mom_communication.c index 69e2cc2e41..954d698ae0 100644 --- a/src/server/receive_mom_communication.c +++ b/src/server/receive_mom_communication.c @@ -432,7 +432,10 @@ void *svr_is_request( { log_err(-1, __func__, log_buf); } - + + // Reduce "spamming" from nodes not authorized + write_tcp_reply(chan, IS_PROTOCOL, IS_PROTOCOL_VER, IS_STATUS, DIS_SUCCESS); + close_conn(chan->sock, FALSE); DIS_tcp_cleanup(chan); return(NULL); diff --git a/src/server/reply_send.c b/src/server/reply_send.c index d65d0a72e2..068799efa0 100644 --- a/src/server/reply_send.c +++ b/src/server/reply_send.c @@ -95,8 +95,9 @@ static int dis_reply_write( int rc = PBSE_NONE; char log_buf[LOCAL_LOG_BUF_SIZE]; struct tcp_chan *chan = NULL; + int old_state = PTHREAD_CANCEL_ENABLE; - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 0); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state); /* setup for DIS over tcp */ if ((chan = DIS_tcp_setup(sfds)) != NULL) @@ -116,7 +117,7 @@ static int dis_reply_write( DIS_tcp_cleanup(chan); } - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return(rc); } /* END dis_reply_write() */ @@ -168,7 +169,7 @@ int reply_send_svr( { /* Otherwise, the reply is to be sent to a remote client */ - if (request->rq_noreply != TRUE) + if (request->rq_noreply != true) { rc = dis_reply_write(sfds, &request->rq_reply); @@ -183,14 +184,6 @@ int reply_send_svr( } } - if (((request->rq_type != PBS_BATCH_AsyModifyJob) && - (request->rq_type != PBS_BATCH_AsyrunJob) && - (request->rq_type != PBS_BATCH_AsySignalJob)) || - (request->rq_noreply == TRUE)) - { - free_br(request); - } - return(rc); } /* END reply_send_svr() */ #endif /* PBS_MOM */ diff --git a/src/server/req_delete.c b/src/server/req_delete.c index 7c97b152a4..d04ee08a0b 100644 --- a/src/server/req_delete.c +++ b/src/server/req_delete.c @@ -89,6 +89,7 @@ #include +#include #include #include #include @@ -117,8 +118,8 @@ #include "mutex_mgr.hpp" #include "threadpool.h" #include "req_delete.h" +#include "lib_ifl.h" #include "delete_all_tracker.hpp" -#include #define PURGE_SUCCESS 1 #define MOM_DELETE 2 @@ -139,8 +140,6 @@ extern struct server server; extern int LOGLEVEL; extern all_jobs alljobs; -extern int issue_signal(job **, const char *, void (*)(batch_request *), void *, char *); - /* Private Functions in this file */ void post_delete_route(struct work_task *); @@ -168,10 +167,11 @@ extern void set_resc_assigned(job *, enum batch_op); extern job *chk_job_request(char *, struct batch_request *); extern struct batch_request *cpy_stage(struct batch_request *, job *, enum job_atr, int); extern int svr_chk_owner(struct batch_request *, job *); +extern int svr_authorize_jobreq(struct batch_request *, job *); void chk_job_req_permissions(job **,struct batch_request *); void on_job_exit_task(struct work_task *); void remove_stagein(job **pjob_ptr); -extern void removeBeforeAnyDependencies(const char *pJobID); +extern void removeBeforeAnyDependencies(job **pjob_ptr); @@ -313,7 +313,7 @@ void remove_stagein( "unable to remove staged in files for job"); } - free_br(preq); + delete preq; } return; @@ -321,10 +321,60 @@ void remove_stagein( +/* + * perform_job_delete_array_bookkeeping() + * + * Updates the array values to account for pjob's deletion if pjob is an array subjob + * @param pjob - the job being deleted + * @param cancel_exit_code - the exit code for this job + * @return PBSE_NONE on success, PBSE_JOBNOTFOUND if the job disappears + */ + +int perform_job_delete_array_bookkeeping( + + job *pjob, + int cancel_exit_code) + + { + int rc = PBSE_NONE; + + if ((pjob->ji_arraystructid[0] != '\0') && + (pjob->ji_is_array_template == FALSE)) + { + job_array *pa = get_jobs_array(&pjob); + + if (pjob == NULL) + return(PBSE_JOBNOTFOUND); + + if (pa != NULL) + { + int old_state = pjob->ji_qs.ji_state; + std::string jobid(pjob->ji_qs.ji_jobid); + + unlock_ji_mutex(pjob, __func__, NULL, LOGLEVEL); + + pa->update_array_values(old_state, + aeTerminate, + jobid.c_str(), + cancel_exit_code); + + if ((pjob = svr_find_job((char *)jobid.c_str(),FALSE)) == NULL) + rc = PBSE_JOBNOTFOUND; + + unlock_ai_mutex(pa, __func__, "1", LOGLEVEL); + } + } + + return(rc); + } // END perform_job_delete_array_bookkeeping() + + + /* * force_purge_work() * - * Performs the work of purging an individual job (qdel -p) + * Performs the work of purging an individual job (qdel -p). The job comes in locked, + * but is never locked when this function exits. * @param pjob - the job that will be purged */ @@ -354,31 +404,14 @@ void force_purge_work( } } - depend_on_term(pjob); - - if ((pjob->ji_arraystructid[0] != '\0') && - (pjob->ji_is_array_template == false)) + // On error, the job has disappeared, which should happen once we're done with this function + // anyway. + if (perform_job_delete_array_bookkeeping(pjob, cancel_exit_code) == PBSE_NONE) { - job_array *pa = get_jobs_array(&pjob); - - if (pjob == NULL) - throw (int)PBSE_JOB_RECYCLED; - - if (pa != NULL) - { - pa->update_array_values(pjob->ji_qs.ji_state, - aeTerminate, - pjob->ji_qs.ji_jobid, - cancel_exit_code); - - unlock_ai_mutex(pa, __func__, "", LOGLEVEL); - } - } - - svr_setjobstate(pjob, JOB_STATE_COMPLETE, JOB_SUBSTATE_COMPLETE, FALSE); + depend_on_term(pjob); + + svr_setjobstate(pjob, JOB_STATE_COMPLETE, JOB_SUBSTATE_COMPLETE, FALSE); - if (pjob != NULL) - { if (is_ms_on_server(pjob)) { char log_buf[LOCAL_LOG_BUF_SIZE]; @@ -392,9 +425,10 @@ void force_purge_work( else svr_job_purge(pjob); } - } /* END force_purge_work() */ - + else + throw (int)PBSE_JOB_RECYCLED; + } /* END force_purge_work() */ @@ -436,7 +470,6 @@ void ensure_deleted( - void setup_apply_job_delete_nanny( job *pjob, /* I */ @@ -455,7 +488,63 @@ void setup_apply_job_delete_nanny( /* - * execut_job_delete() + * notify_waiting_terminal() + * + * @param pjob - the interactive job whose terminal we need to notify + */ + +void notify_waiting_terminal( + + job *pjob) + + { + unsigned int pport = pjob->ji_wattr[JOB_ATR_interactive].at_val.at_long; + char *hostname = pjob->ji_wattr[JOB_ATR_submit_host].at_val.at_str; + char log_buf[LOCAL_LOG_BUF_SIZE]; + pbs_net_t hostaddr; + int local_errno; + + if (hostname != NULL) + { + hostaddr = get_hostaddr(&local_errno, hostname); + + if (hostaddr != (pbs_net_t)0) + { + char err_msg[MAXLINE]; + int s = client_to_svr(hostaddr, pport, 0, err_msg); + + if (s >= 0) + { + // Write the letter 'C' to cancel the job + write_ac_socket(s, "C", 1); + close(s); + } + else + { + sprintf(log_buf, "Couldn't open a socket to cancel interactive job '%s' : %s", + pjob->ji_qs.ji_jobid, err_msg); + log_err(errno, __func__, log_buf); + } + } + else + { + sprintf(log_buf, "Couldn't get the address for host '%s' to cancel interactive job '%s'", + hostname, pjob->ji_qs.ji_jobid); + log_err(errno, __func__, log_buf); + } + } + else + { + sprintf(log_buf, "Couldn't find a submission host for interactive job '%s'", pjob->ji_qs.ji_jobid); + log_err(-1, __func__, log_buf); + } + + } // END notify_waiting_terminal() + + + +/* + * execute_job_delete() * * Deletes an individual job, contacting the mom if necessary * @param pjob - the job being deleted @@ -482,7 +571,7 @@ int execute_job_delete( long force_cancel = FALSE; long status_cancel_queue = FALSE; - chk_job_req_permissions(&pjob,preq); + chk_job_req_permissions(&pjob, preq); if (pjob == NULL) { @@ -490,6 +579,15 @@ int execute_job_delete( return(-1); } + removeBeforeAnyDependencies(&pjob); + + // The job disappeared unexpectedly + if (pjob == NULL) + { + reply_ack(preq); + return(PBSE_NONE); + } + mutex_mgr job_mutex(pjob->ji_mutex, true); if (LOGLEVEL >= 10) @@ -553,7 +651,7 @@ int execute_job_delete( log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,pjob->ji_qs.ji_jobid,log_buf); - pwtnew = set_task(WORK_Timed,time_now + 1,post_delete_route,preq,FALSE); + pwtnew = set_task(WORK_Timed,time_now + 1, post_delete_route, new batch_request(*preq),FALSE); if (pwtnew == NULL) { @@ -653,7 +751,7 @@ int execute_job_delete( */ get_batch_request_id(preq); - if ((rc = issue_signal(&pjob, sigt, post_delete_mom1, strdup(del), strdup(preq->rq_id)))) + if ((rc = issue_signal(&pjob, sigt, post_delete_mom1, strdup(del), strdup(preq->rq_id.c_str())))) { /* cant send to MOM */ @@ -671,57 +769,29 @@ int execute_job_delete( return(-1); } /* END if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING) */ - else if ((pjob->ji_qs.ji_state < JOB_STATE_RUNNING) && - (status_cancel_queue != 0)) + else if (pjob->ji_qs.ji_state < JOB_STATE_RUNNING) { - /* - * If a task is not running yet and is still on the queue, we - * need to set an exit status - */ - - pjob->ji_qs.ji_un.ji_exect.ji_exitstat = status_cancel_queue; - - pjob->ji_wattr[JOB_ATR_exitstat].at_val.at_long = status_cancel_queue; - pjob->ji_wattr[JOB_ATR_exitstat].at_flags |= ATR_VFLAG_SET; - } - - // Update the array book-keeping values if this is an array subjob - if ((pjob->ji_arraystructid[0] != '\0') && - (pjob->ji_is_array_template == false)) - { - job_array *pa = get_jobs_array(&pjob); - - if (pjob == NULL) + if (status_cancel_queue != 0) { - job_mutex.set_unlock_on_exit(false); - return(-1); - } - - std::string dup_job_id(pjob->ji_qs.ji_jobid); - - if (pa != NULL) - { - if (pjob != NULL) - { - if (pjob->ji_qs.ji_state != JOB_STATE_RUNNING) - { - int job_exit_status = pjob->ji_qs.ji_un.ji_exect.ji_exitstat; - int job_state = pjob->ji_qs.ji_state; - - job_mutex.unlock(); - pa->update_array_values(job_state, aeTerminate, dup_job_id.c_str(), job_exit_status); + /* + * If a task is not running yet and is still on the queue, we + * need to set an exit status + */ - if ((pjob = svr_find_job((char *)dup_job_id.c_str(),FALSE)) != NULL) - job_mutex.mark_as_locked(); - } - } + pjob->ji_qs.ji_un.ji_exect.ji_exitstat = status_cancel_queue; - unlock_ai_mutex(pa, __func__, "1", LOGLEVEL); + pjob->ji_wattr[JOB_ATR_exitstat].at_val.at_long = status_cancel_queue; + pjob->ji_wattr[JOB_ATR_exitstat].at_flags |= ATR_VFLAG_SET; } + + // Send cancel to the waiting terminal + if (pjob->ji_wattr[JOB_ATR_interactive].at_val.at_long) + notify_waiting_terminal(pjob); } - if (pjob == NULL) + if (perform_job_delete_array_bookkeeping(pjob, status_cancel_queue) != PBSE_NONE) { + // The job disappeared while updating the array job_mutex.set_unlock_on_exit(false); return -1; } @@ -771,197 +841,26 @@ int execute_job_delete( -int copy_attribute_list( - - batch_request *preq, - batch_request *preq_tmp) - - { - svrattrl *pal = (svrattrl *)GET_NEXT(preq->rq_ind.rq_manager.rq_attr); - tlist_head *phead = &preq_tmp->rq_ind.rq_manager.rq_attr; - svrattrl *newpal = NULL; - - while (pal != NULL) - { - newpal = (svrattrl *)calloc(1, pal->al_tsize + 1); - if (!newpal) - { - free_br(preq_tmp); - return(PBSE_SYSTEM); - } - - CLEAR_LINK(newpal->al_link); - - newpal->al_atopl.next = 0; - newpal->al_tsize = pal->al_tsize + 1; - newpal->al_nameln = pal->al_nameln; - newpal->al_flags = pal->al_flags; - newpal->al_atopl.name = (char *)newpal + sizeof(svrattrl); - strcpy((char *)newpal->al_atopl.name, pal->al_atopl.name); - newpal->al_nameln = pal->al_nameln; - newpal->al_atopl.resource = newpal->al_atopl.name + newpal->al_nameln; - - if (pal->al_atopl.resource != NULL) - strcpy((char *)newpal->al_atopl.resource, pal->al_atopl.resource); - - newpal->al_rescln = pal->al_rescln; - newpal->al_atopl.value = newpal->al_atopl.name + newpal->al_nameln + newpal->al_rescln; - strcpy((char *)newpal->al_atopl.value, pal->al_atopl.value); - newpal->al_valln = pal->al_valln; - newpal->al_atopl.op = pal->al_atopl.op; - - pal = (struct svrattrl *)GET_NEXT(pal->al_link); - } - - if ((phead != NULL) && - (newpal != NULL)) - append_link(phead, &newpal->al_link, newpal); - - return(PBSE_NONE); - } /* END copy_attribute_list() */ - - - - /* - * duplicate_request() - * duplicates preq and returns the duplicate request - * @param preq - the request to duplicate - * @param job_index - if desired, replace the job id with the sub job id. - * The sub-job has the index job_index and this is only performed if this - * value isn't -1 + * delete_all_work() + * + * Does the work to delete all jobs + * @param preq - the batch_request to delete all jobs. + * @return PBSE_NONE on success or the number of failed deletes. */ -batch_request *duplicate_request( - - batch_request *preq, /* I */ - int job_index) /* I - optional */ - - { - batch_request *preq_tmp = alloc_br(preq->rq_type); - char *ptr1; - char *ptr2; - char newjobname[PBS_MAXSVRJOBID+1]; - - if (preq_tmp == NULL) - return(NULL); - - preq_tmp->rq_perm = preq->rq_perm; - preq_tmp->rq_fromsvr = preq->rq_fromsvr; - preq_tmp->rq_conn = preq->rq_conn; - preq_tmp->rq_time = preq->rq_time; - preq_tmp->rq_orgconn = preq->rq_orgconn; - - memcpy(preq_tmp->rq_ind.rq_manager.rq_objname, - preq->rq_ind.rq_manager.rq_objname, PBS_MAXSVRJOBID + 1); - - strcpy(preq_tmp->rq_user, preq->rq_user); - strcpy(preq_tmp->rq_host, preq->rq_host); +int delete_all_work( - if (preq->rq_extend != NULL) - preq_tmp->rq_extend = strdup(preq->rq_extend); - - switch (preq->rq_type) - { - /* This function was created for a modify array request (PBS_BATCH_ModifyJob) - the preq->rq_ind structure was allocated in dis_request_read. If other - BATCH types are needed refer to that function to see how the rq_ind structure - was allocated and then copy it here. */ - case PBS_BATCH_DeleteJob: - case PBS_BATCH_HoldJob: - case PBS_BATCH_CheckpointJob: - case PBS_BATCH_ModifyJob: - case PBS_BATCH_AsyModifyJob: - - /* based on how decode_DIS_Manage allocates data */ - CLEAR_HEAD(preq_tmp->rq_ind.rq_manager.rq_attr); - - preq_tmp->rq_ind.rq_manager.rq_cmd = preq->rq_ind.rq_manager.rq_cmd; - preq_tmp->rq_ind.rq_manager.rq_objtype = preq->rq_ind.rq_manager.rq_objtype; - - if (job_index != -1) - { - /* If this is a job array it is possible we only have the array name - and not the individual job. We need to find out what we have and - modify the name if needed */ - ptr1 = strstr(preq->rq_ind.rq_manager.rq_objname, "[]"); - if (ptr1) - { - ptr1++; - strcpy(newjobname, preq->rq_ind.rq_manager.rq_objname); - - if ((ptr2 = strstr(newjobname, "[]")) != NULL) - { - ptr2++; - *ptr2 = 0; - } - - sprintf(preq_tmp->rq_ind.rq_manager.rq_objname,"%s%d%s", - newjobname, job_index, ptr1); - } - else - strcpy(preq_tmp->rq_ind.rq_manager.rq_objname, preq->rq_ind.rq_manager.rq_objname); - } - - /* copy the attribute list */ - if (copy_attribute_list(preq, preq_tmp) != PBSE_NONE) - return(NULL); - - break; - - case PBS_BATCH_SignalJob: - - strcpy(preq_tmp->rq_ind.rq_signal.rq_jid, preq->rq_ind.rq_signal.rq_jid); - strcpy(preq_tmp->rq_ind.rq_signal.rq_signame, preq->rq_ind.rq_signal.rq_signame); - preq_tmp->rq_extra = strdup((char *)preq->rq_extra); - - break; - - case PBS_BATCH_MessJob: - - strcpy(preq_tmp->rq_ind.rq_message.rq_jid, preq->rq_ind.rq_message.rq_jid); - preq_tmp->rq_ind.rq_message.rq_file = preq->rq_ind.rq_message.rq_file; - strcpy(preq_tmp->rq_ind.rq_message.rq_text, preq->rq_ind.rq_message.rq_text); - - break; - - case PBS_BATCH_RunJob: - case PBS_BATCH_AsyrunJob: - - if (preq->rq_ind.rq_run.rq_destin) - preq_tmp->rq_ind.rq_run.rq_destin = strdup(preq->rq_ind.rq_run.rq_destin); - - break; - - case PBS_BATCH_Rerun: - - strcpy(preq_tmp->rq_ind.rq_rerun, preq->rq_ind.rq_rerun); - break; - - default: - - break; - } - - return(preq_tmp); - } /* END duplicate_request() */ - - - -void *delete_all_work( - - void *vp) + batch_request *preq) { - batch_request *preq = (batch_request *)vp; - if (qdel_all_tracker.start_deleting_all_if_possible(preq->rq_user, preq->rq_perm) == false) { reply_ack(preq); - return(NULL); + return(PBSE_NONE); } - batch_request *preq_dup = duplicate_request(preq); + batch_request preq_dup(*preq); job *pjob; all_jobs_iterator *iter = NULL; int failed_deletes = 0; @@ -978,7 +877,8 @@ void *delete_all_work( while ((pjob = next_job(&alljobs, iter)) != NULL) { if ((pjob->ji_arraystructid[0] != '\0') && - (pjob->ji_is_array_template == false)) + (pjob->ji_is_array_template == false) && + (svr_authorize_jobreq(preq, pjob) == 0)) { // Mark this array as being deleted if it hasn't yet been marked if (marked_arrays.find(pjob->ji_arraystructid) == marked_arrays.end()) @@ -1002,7 +902,7 @@ void *delete_all_work( // use mutex manager to make sure job mutex locks are properly handled at exit mutex_mgr job_mutex(pjob->ji_mutex, true); - if ((rc = forced_jobpurge(pjob, preq_dup)) == PURGE_SUCCESS) + if ((rc = forced_jobpurge(pjob, &preq_dup)) == PURGE_SUCCESS) { job_mutex.set_unlock_on_exit(false); @@ -1013,12 +913,6 @@ void *delete_all_work( { job_mutex.unlock(); - if (rc == -1) - { - //forced_jobpurge freed preq_dup so reallocate it. - preq_dup = duplicate_request(preq); - preq_dup->rq_noreply = TRUE; - } continue; } @@ -1027,29 +921,16 @@ void *delete_all_work( /* mutex is freed below */ if (rc == PBSE_NONE) { - if ((rc = execute_job_delete(pjob, Msg, preq_dup)) == PBSE_NONE) + if ((rc = execute_job_delete(pjob, Msg, &preq_dup)) == PBSE_NONE) { // execute_job_delete() handles mutex so don't unlock on exit job_mutex.set_unlock_on_exit(false); - reply_ack(preq_dup); } - /* preq_dup has been freed at this point. Either reallocate it or set it to NULL*/ - if (rc == PURGE_SUCCESS) - { - preq_dup = duplicate_request(preq); - preq_dup->rq_noreply = TRUE; - } - else - preq_dup = NULL; } if (rc != PURGE_SUCCESS) { - /* duplicate the preq so we don't have a problem with double frees */ - preq_dup = duplicate_request(preq); - preq_dup->rq_noreply = TRUE; - if ((rc == MOM_DELETE) || (rc == ROUTE_DELETE)) failed_deletes++; @@ -1063,14 +944,7 @@ void *delete_all_work( if (failed_deletes == 0) { reply_ack(preq); - - /* PURGE SUCCESS means this was qdel -p all. In this case no reply_*() - * functions have been called */ - if (rc == PURGE_SUCCESS) - { - free_br(preq_dup); - preq_dup = NULL; - } + rc = PBSE_NONE; } else { @@ -1079,32 +953,62 @@ void *delete_all_work( total_jobs); req_reject(PBSE_SYSTEM, 0, preq, NULL, tmpLine); + rc = failed_deletes; } - /* preq_dup happens at the end of the loop, so free the extra one if - * it is there */ - if (preq_dup != NULL) - free_br(preq_dup); + return(rc); + } /* END delete_all_work() */ + + + +/* + * delete_all_task() + * + * The asynchronous work task for deleting all jobs + * @param vp - the batch_request to delete all jobs. Always deleted by the end of this function, + * unless it is NULL. + */ + +void *delete_all_task( + + void *vp) + + { + batch_request *preq = (batch_request *)vp; + + if (preq != NULL) + { + delete_all_work(preq); + + delete preq; + } return(NULL); - } /* END delete_all_work() */ + } // END delete_all_task() +/* + * handle_delete_all() + * + * Handles the logic for deleting all jobs + * @param preq - tells us what to delete and how. + * @param Msg - (optional) a message explaining why we're deleting these jobs + * @return PBSE_NONE + */ int handle_delete_all( batch_request *preq, - batch_request *preq_tmp, char *Msg) { - /* preq_tmp is not null if this is an asynchronous request */ - if (preq_tmp != NULL) + if ((preq->rq_extend != NULL) && + (!strncmp(preq->rq_extend,delasyncstr,strlen(delasyncstr)))) { - reply_ack(preq_tmp); - preq->rq_noreply = TRUE; /* set for no more replies */ - enqueue_threadpool_request(delete_all_work, preq, request_pool); + reply_ack(preq); + preq->rq_noreply = true; /* set for no more replies */ + enqueue_threadpool_request(delete_all_task, new batch_request(*preq), request_pool); } else delete_all_work(preq); @@ -1114,13 +1018,20 @@ int handle_delete_all( -void *single_delete_work( +/* + * single_delete_work() + * + * Performs the work of deleting a single job + * @param preq - the batch_request instructing us what to delete and how + * @return PBSE_* to indicate status + */ - void *vp) +int single_delete_work( + + batch_request *preq) { int rc = -1; - batch_request *preq = (batch_request *)vp; char *jobid = preq->rq_ind.rq_delete.rq_objname; job *pjob; char *Msg = preq->rq_extend; @@ -1131,7 +1042,8 @@ void *single_delete_work( if (pjob == NULL) { - req_reject(PBSE_JOBNOTFOUND, 0, preq, NULL, "job unexpectedly deleted"); + rc = PBSE_JOBNOTFOUND; + req_reject(rc, 0, preq, NULL, "job unexpectedly deleted"); } else { @@ -1141,17 +1053,57 @@ void *single_delete_work( if ((rc == PBSE_NONE) || (rc == PURGE_SUCCESS)) + { reply_ack(preq); + rc = PBSE_NONE; + } } - return(NULL); + return(rc); } /* END single_delete_work() */ + +/* + * single_delete_task() + * + * The taskpool task for performing a single job delete + * + * @param vp - a void pointer to the batch_request pointer we need. This is always deleted + * by the end of this function, unless it is NULL. + */ + +void *single_delete_task( + + void *vp) + + { + if (vp != NULL) + { + batch_request *preq = (batch_request *)vp; + + single_delete_work(preq); + + delete preq; + } + + return(NULL); + } // END single_delete_task() + + + +/* + * handle_single_delete() + * + * Handles the logic for deleting a single a job. + * @param preq - the batch_request telling us what to delete and how. + * @param Msg - (optional) a message stating why we're deleting this job. + * @return PBSE_NONE + */ + int handle_single_delete( batch_request *preq, - batch_request *preq_tmp, char *Msg) { @@ -1166,17 +1118,15 @@ int handle_single_delete( } else { - std::string jobID = pjob->ji_qs.ji_jobid; unlock_ji_mutex(pjob, __func__, NULL, LOGLEVEL); - removeBeforeAnyDependencies(jobID.c_str()); - /* send the asynchronous reply if needed */ - if (preq_tmp != NULL) + if ((preq->rq_extend != NULL) && + (!strncmp(preq->rq_extend, delasyncstr, strlen(delasyncstr)))) { - reply_ack(preq_tmp); - preq->rq_noreply = TRUE; /* set for no more replies */ - enqueue_threadpool_request(single_delete_work, preq, async_pool); + reply_ack(preq); + preq->rq_noreply = true; /* set for no more replies */ + enqueue_threadpool_request(single_delete_task, new batch_request(*preq), async_pool); } else single_delete_work(preq); @@ -1187,7 +1137,6 @@ int handle_single_delete( - /* * req_deletejob - service the Delete Job Request * @@ -1235,7 +1184,6 @@ int req_deletejob( { char *Msg = NULL; - struct batch_request *preq_tmp = NULL; char log_buf[LOCAL_LOG_BUF_SIZE]; /* check if we are getting a purgecomplete from scheduler */ @@ -1275,18 +1223,16 @@ int req_deletejob( */ snprintf(log_buf,sizeof(log_buf), "Deleting job asynchronously"); log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,preq->rq_ind.rq_delete.rq_objname,log_buf); - - preq_tmp = duplicate_request(preq); } } if (strcasecmp(preq->rq_ind.rq_delete.rq_objname,"all") == 0) { - handle_delete_all(preq, preq_tmp, Msg); + handle_delete_all(preq, Msg); } else { - handle_single_delete(preq, preq_tmp, Msg); + handle_single_delete(preq, Msg); } return(PBSE_NONE); @@ -1356,10 +1302,13 @@ void post_delete_route( struct work_task *pwt) { - batch_request *preq = get_remove_batch_request((char *)pwt->wt_parm1); + batch_request *preq = (batch_request *)pwt->wt_parm1; if (preq != NULL) + { req_deletejob(preq); + delete preq; + } free(pwt->wt_mutex); free(pwt); @@ -1401,7 +1350,6 @@ void post_delete_mom1( { preq_clt = get_remove_batch_request(preq_sig->rq_extend); } - free_br(preq_sig); /* the client request has been handled another way, nothing left to do */ if (preq_clt == NULL) @@ -1526,7 +1474,7 @@ void post_delete_mom2( if (pjob->ji_qs.ji_state == JOB_STATE_RUNNING) { - issue_signal(&pjob, sigk, free_br, NULL, NULL); + issue_signal(&pjob, sigk, NULL, NULL, NULL); if (pjob != NULL) { @@ -1709,7 +1657,7 @@ void job_delete_nanny( log_err(-1, "job nanny", log_buf); /* build up a Signal Job batch request */ - if ((newreq = alloc_br(PBS_BATCH_SignalJob)) != NULL) + if ((newreq = new batch_request(PBS_BATCH_SignalJob)) != NULL) { strcpy(newreq->rq_ind.rq_signal.rq_jid, pjob->ji_qs.ji_jobid); snprintf(newreq->rq_ind.rq_signal.rq_signame, sizeof(newreq->rq_ind.rq_signal.rq_signame), "%s", sigk); @@ -1766,7 +1714,6 @@ void post_job_delete_nanny( if (!nanny) { /* the admin disabled nanny within the last minute or so */ - free_br(preq_sig); return; } @@ -1780,8 +1727,6 @@ void post_job_delete_nanny( log_event(PBSEVENT_ERROR,PBS_EVENTCLASS_JOB,preq_sig->rq_ind.rq_signal.rq_jid,log_buf); - free_br(preq_sig); - return; } @@ -1796,9 +1741,7 @@ void post_job_delete_nanny( free_nodes(pjob); set_resc_assigned(pjob, DECR); - - free_br(preq_sig); - + job_mutex.set_unlock_on_exit(false); svr_job_purge(pjob); @@ -1806,9 +1749,6 @@ void post_job_delete_nanny( return; } - /* free task */ - free_br(preq_sig); - return; } /* END post_job_delete_nanny() */ diff --git a/src/server/req_deletearray.c b/src/server/req_deletearray.c index 6f12214bcb..a911f75373 100644 --- a/src/server/req_deletearray.c +++ b/src/server/req_deletearray.c @@ -35,7 +35,6 @@ extern int has_job_delete_nanny(struct job *); extern void setup_apply_job_delete_nanny(struct job *, time_t time_now); extern void remove_stagein(job **pjob); extern void change_restart_comment_if_needed(struct job *); -int issue_signal(job **, const char *, void(*)(batch_request *), void *, char *); extern char *msg_unkarrayid; extern char *msg_permlog; @@ -108,7 +107,7 @@ bool attempt_delete( /* need to issue a signal to the mom, but we don't want to sent an ack to the * client when the mom replies */ - issue_signal(&pjob, "SIGTERM", free_br, NULL, NULL); + issue_signal(&pjob, "SIGTERM", NULL, NULL, NULL); } if (pjob != NULL) @@ -147,6 +146,8 @@ int req_deletearray( struct work_task *ptask; char log_buf[LOCAL_LOG_BUF_SIZE]; + static const int MAX_DELETE_WAIT = 3; + int wait_tries = 0; int num_skipped = 0; char owner[PBS_MAXUSER + 1]; time_t time_now = time(NULL); @@ -159,14 +160,22 @@ int req_deletearray( pa = get_array(preq->rq_ind.rq_delete.rq_objname); - // Do not attempt to delete the array while it is still cloning + // Do not attempt to delete the array while it is still cloning. If none have been cloned, + // it's because the array was never routed. while ((pa != NULL) && ((pa->ai_qs.num_cloned < pa->ai_qs.idle_slot_limit) && - (pa->ai_qs.num_cloned != pa->ai_qs.num_jobs))) + (pa->ai_qs.num_cloned != pa->ai_qs.num_jobs)) && + (pa->ai_qs.num_cloned > 0)) { - unlock_ai_mutex(pa, __func__, NULL, 10); - sleep(1); - pa = get_array(preq->rq_ind.rq_delete.rq_objname); + if (wait_tries == MAX_DELETE_WAIT) + break; + else + { + unlock_ai_mutex(pa, __func__, NULL, 10); + sleep(1); + pa = get_array(preq->rq_ind.rq_delete.rq_objname); + wait_tries++; + } } if (pa == NULL) @@ -232,8 +241,7 @@ int req_deletearray( log_record(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); } - if (((num_skipped = delete_whole_array(pa, purge)) == NO_JOBS_IN_ARRAY) && - (purge == false)) + if ((num_skipped = delete_whole_array(pa, purge)) == NO_JOBS_IN_ARRAY) { pa_mutex.unlock(); array_delete(preq->rq_ind.rq_delete.rq_objname); diff --git a/src/server/req_holdarray.c b/src/server/req_holdarray.c index 01ec2e73d8..8358887993 100644 --- a/src/server/req_holdarray.c +++ b/src/server/req_holdarray.c @@ -61,7 +61,6 @@ void hold_job( (csv_find_string(pattr->at_val.at_str, "enabled") != NULL)))) { /* TODO */ - /* preq_tmp = alloc_br(preq->rq_type); */ } else if (old_hold != *hold_val) diff --git a/src/server/req_holdjob.c b/src/server/req_holdjob.c index 913ec14e29..ef27d868dc 100644 --- a/src/server/req_holdjob.c +++ b/src/server/req_holdjob.c @@ -177,7 +177,6 @@ int req_holdjob( pbs_attribute *pattr; batch_request *preq = (struct batch_request *)vp; char log_buf[LOCAL_LOG_BUF_SIZE]; - batch_request *dup_req = NULL; pjob = chk_job_request(preq->rq_ind.rq_hold.rq_orig.rq_objname, preq); @@ -236,15 +235,11 @@ int req_holdjob( snprintf(preq->rq_ind.rq_hold.rq_orig.rq_objname, sizeof(preq->rq_ind.rq_hold.rq_orig.rq_objname), "%s", pjob->ji_qs.ji_jobid); - if ((dup_req = duplicate_request(preq)) == NULL) - { - req_reject(rc, 0, preq, NULL, "memory allocation failure"); - } - /* The dup_req is freed in relay_to_mom (failure) - * or in issue_Drequest (success) */ - else if ((rc = relay_to_mom(&pjob, dup_req, NULL)) != PBSE_NONE) + + batch_request dup_req(*preq); + + if ((rc = relay_to_mom(&pjob, &dup_req, NULL)) != PBSE_NONE) { - free_br(dup_req); *hold_val = old_hold; /* reset to the old value */ req_reject(rc, 0, preq, NULL, "relay to mom failed"); @@ -270,7 +265,7 @@ int req_holdjob( else job_mutex.set_unlock_on_exit(false); - process_hold_reply(dup_req); + process_hold_reply(&dup_req); } } #ifdef ENABLE_BLCR @@ -326,7 +321,6 @@ void *req_checkpointjob( int rc; pbs_attribute *pattr; char log_buf[LOCAL_LOG_BUF_SIZE]; - batch_request *dup_req = NULL; if ((pjob = chk_job_request(preq->rq_ind.rq_manager.rq_objname, preq)) == NULL) { @@ -345,17 +339,11 @@ void *req_checkpointjob( { /* have MOM attempt checkpointing */ - if ((dup_req = duplicate_request(preq)) == NULL) - { - req_reject(PBSE_SYSTEM, 0, preq, NULL, "failure to allocate memory"); - } + batch_request dup_req(*preq); - /* The dup_req is freed in relay_to_mom (failure) - * or in issue_Drequest (success) */ - else if ((rc = relay_to_mom(&pjob, dup_req, NULL)) != PBSE_NONE) + if ((rc = relay_to_mom(&pjob, &dup_req, NULL)) != PBSE_NONE) { req_reject(rc, 0, preq, NULL, NULL); - free_br(dup_req); if (pjob == NULL) job_mutex.set_unlock_on_exit(false); @@ -374,13 +362,12 @@ void *req_checkpointjob( else job_mutex.set_unlock_on_exit(false); - process_checkpoint_reply(dup_req); + process_checkpoint_reply(&dup_req); } } else { - /* Job does not have checkpointing enabled, so reject the request */ - + // Job does not have checkpointing enabled, so reject the request log_event(PBSEVENT_JOB,PBS_EVENTCLASS_JOB,pjob->ji_qs.ji_jobid,log_buf); req_reject(PBSE_IVALREQ, 0, preq, NULL, "job is not checkpointable"); @@ -391,7 +378,6 @@ void *req_checkpointjob( - /* * release_job - releases the hold on job j * @param j - the job to modify @@ -473,7 +459,7 @@ int release_job( preq->rq_host); log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf); - rc = PBSE_BAD_JOB_STATE_TRANSITION; + rc = PBSE_STATE_SLOT_LIMIT; } return(rc); diff --git a/src/server/req_jobobit.c b/src/server/req_jobobit.c index cde783afb0..583a37f801 100644 --- a/src/server/req_jobobit.c +++ b/src/server/req_jobobit.c @@ -225,7 +225,7 @@ struct batch_request *setup_cpyfiles( { /* allocate and initialize the batch request struct */ - preq = alloc_br(PBS_BATCH_CopyFiles); + preq = new batch_request(PBS_BATCH_CopyFiles); if (preq == NULL) { @@ -286,7 +286,7 @@ struct batch_request *setup_cpyfiles( { /* FAILURE */ - free_br(preq); + delete preq; log_event( PBSEVENT_ERROR | PBSEVENT_JOB, @@ -383,7 +383,7 @@ struct batch_request *return_stdfile( if (preq == NULL) { - preq = alloc_br(PBS_BATCH_ReturnFiles); + preq = new batch_request(PBS_BATCH_ReturnFiles); } strcpy(preq->rq_ind.rq_returnfiles.rq_jobid, pjob->ji_qs.ji_jobid); @@ -430,7 +430,7 @@ struct batch_request *cpy_stdfile( { /* the job is interactive, don't bother to return output file */ if (preq != NULL) - free_br(preq); + delete preq; return(NULL); } @@ -461,7 +461,7 @@ struct batch_request *cpy_stdfile( log_buf); if (preq != NULL) - free_br(preq); + delete preq; return(NULL); } @@ -1078,8 +1078,7 @@ int handle_returnstd( * "faking" the immediate work task and falling through to * the next case. */ - - free_br(preq); + delete preq; } if ((pjob = svr_find_job(job_id, TRUE)) != NULL) @@ -1339,7 +1338,7 @@ int handle_stageout( preq->rq_extra = NULL; } - free_br(preq); + delete preq; preq = NULL; } /* END if preq != NULL */ @@ -1356,7 +1355,7 @@ int handle_stageout( if (preq != NULL) { - free_br(preq); + delete preq; } if (job_momname != NULL) @@ -1490,7 +1489,7 @@ int handle_stagedel( unlock_ji_mutex(pjob, __func__, "5", LOGLEVEL); } - free_br(preq); + delete preq; } if ((pjob = svr_find_job(job_id, TRUE)) != NULL) @@ -1532,7 +1531,7 @@ int handle_exited( } /* tell mom to delete the job, send final track and purge it */ - preq = alloc_br(PBS_BATCH_DeleteJob); + preq = new batch_request(PBS_BATCH_DeleteJob); if (preq != NULL) { @@ -1540,7 +1539,7 @@ int handle_exited( if ((handle = mom_comm(pjob, on_job_exit_task)) < 0) { - free_br(preq); + delete preq; return(PBSE_CONNECT); } else @@ -1559,12 +1558,12 @@ int handle_exited( } } - free_br(preq); + delete preq; + + preq = NULL; } else job_mutex.unlock(); - - preq = NULL; if ((pjob = svr_find_job(job_id, TRUE)) == NULL) return(PBSE_JOBNOTFOUND); @@ -2020,6 +2019,7 @@ void on_job_exit( job *pjob; int type = WORK_Deferred_Reply; char log_buf[LOCAL_LOG_BUF_SIZE]; + int old_state = PTHREAD_CANCEL_ENABLE; if (preq == NULL) type = WORK_Immed; @@ -2048,7 +2048,7 @@ void on_job_exit( return; } - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, 0); + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state); mutex_mgr job_mutex(pjob->ji_mutex, true); job_mutex.set_unlock_on_exit(false); @@ -2184,7 +2184,7 @@ void on_job_exit( if (job_id != NULL) free(job_id); - pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, 0); + pthread_setcancelstate(old_state, NULL); return; } /* END on_job_exit() */ @@ -2333,7 +2333,7 @@ void on_job_rerun( /* ask mom to send them back to the server */ /* mom deletes her copy if returned ok */ - if ((preq = alloc_br(PBS_BATCH_Rerun)) == NULL) + if ((preq = new batch_request(PBS_BATCH_Rerun)) == NULL) return; strcpy(preq->rq_ind.rq_rerun, pjob->ji_qs.ji_jobid); @@ -2623,7 +2623,7 @@ void on_job_rerun( case JOB_SUBSTATE_RERUN3: /* need to have MOM delete her copy of the job */ - preq = alloc_br(PBS_BATCH_DeleteJob); + preq = new batch_request(PBS_BATCH_DeleteJob); if (preq != NULL) { @@ -3308,6 +3308,18 @@ int update_substate_from_exit_status( set_job_comment(pjob, pbse_to_txt(PBSE_CGROUP_CREATE_FAIL)); // Fall through intentionally + + case JOB_EXEC_RETRY_PROLOGUE: + + if (exitstatus == JOB_EXEC_RETRY_PROLOGUE) + { + snprintf(log_buf, sizeof(log_buf), + "Job %s is being retried due to a failed prologue. See the syslog on the mother superior for more details", + pjob->ji_qs.ji_jobid); + log_err(-1, __func__, log_buf); + } + + // Fall through intentionally case JOB_EXEC_RETRY: @@ -3412,6 +3424,9 @@ bool is_job_finished( // Must be a version 6.1.0 node or higher for the mom to have cleaned up the job if (pnode->get_version() >= 610) { + // unlock the node first + node_mutex.unlock(); + /* see if job has any dependencies */ if (pjob->ji_wattr[JOB_ATR_depend].at_flags & ATR_VFLAG_SET) { @@ -3422,8 +3437,6 @@ bool is_job_finished( } } - node_mutex.unlock(); - rel_resc(pjob); svr_setjobstate(pjob, JOB_STATE_COMPLETE, JOB_SUBSTATE_COMPLETE, FALSE); handle_complete_first_time(pjob); @@ -3677,7 +3690,7 @@ int req_jobobit( "No resources used found"); } - tmppreq = alloc_br(PBS_BATCH_JobObit); + tmppreq = new batch_request(PBS_BATCH_JobObit); if (tmppreq == NULL) { diff --git a/src/server/req_manager.c b/src/server/req_manager.c index 505f515e94..18ef19f7f7 100644 --- a/src/server/req_manager.c +++ b/src/server/req_manager.c @@ -1477,19 +1477,25 @@ void mgr_queue_set( log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_QUEUE, __func__, log_buf); } - svr_queues.lock(); - iter = svr_queues.get_iterator(); - svr_queues.unlock(); - + // see if all queues or just a single queue should be set if ((*preq->rq_ind.rq_manager.rq_objname == '\0') || (*preq->rq_ind.rq_manager.rq_objname == '@')) { + // all queues + qname = all_quename; allques = TRUE; + + svr_queues.lock(); + iter = svr_queues.get_iterator(); + svr_queues.unlock(); + pque = next_queue(&svr_queues,iter); } else { + // single queue + qname = preq->rq_ind.rq_manager.rq_objname; allques = FALSE; @@ -1500,7 +1506,8 @@ void mgr_queue_set( { req_reject(PBSE_UNKQUE, 0, preq, NULL, NULL); - delete iter; + if (iter != NULL) + delete iter; return; } @@ -1531,7 +1538,8 @@ void mgr_queue_set( que_mutex.unlock(); reply_badattr(rc, bad, plist, preq); - delete iter; + if (iter != NULL) + delete iter; return; } @@ -1551,16 +1559,20 @@ void mgr_queue_set( pque = next_queue(&svr_queues,iter); } /* END while (pque != NULL) */ - /* check the appropriateness of the attributes based on queue type */ - svr_queues.lock(); + // if operating on all queues, get new iterator + if (allques == TRUE) + { + svr_queues.lock(); - delete iter; + delete iter; - iter = svr_queues.get_iterator(); - svr_queues.unlock(); + iter = svr_queues.get_iterator(); + svr_queues.unlock(); - if (allques == TRUE) pque = next_queue(&svr_queues,iter); + } + + /* check the appropriateness of the attributes based on queue type */ while (pque != NULL) { @@ -2027,29 +2039,25 @@ static bool requeue_or_delete_jobs( if(pjob != NULL) { char *dup_jobid = strdup(pjob->ji_qs.ji_jobid); - batch_request *brRerun = alloc_br(PBS_BATCH_Rerun); - batch_request *brDelete = alloc_br(PBS_BATCH_DeleteJob); - if((brRerun == NULL)||(brDelete == NULL)) - { - free_br(brRerun); - free_br(brDelete); - free(dup_jobid); - return true; //Ignoring this error. - } - strcpy(brRerun->rq_ind.rq_rerun,pjob->ji_qs.ji_jobid); - strcpy(brDelete->rq_ind.rq_delete.rq_objname,pjob->ji_qs.ji_jobid); - brRerun->rq_conn = PBS_LOCAL_CONNECTION; - brDelete->rq_conn = PBS_LOCAL_CONNECTION; - brRerun->rq_perm = preq->rq_perm; - brDelete->rq_perm = preq->rq_perm; - brDelete->rq_ind.rq_delete.rq_objtype = MGR_OBJ_JOB; - brDelete->rq_ind.rq_delete.rq_cmd = MGR_CMD_DELETE; + batch_request brRerun(PBS_BATCH_Rerun); + batch_request brDelete(PBS_BATCH_DeleteJob); + + strcpy(brRerun.rq_ind.rq_rerun,pjob->ji_qs.ji_jobid); + strcpy(brDelete.rq_ind.rq_delete.rq_objname,pjob->ji_qs.ji_jobid); + + brRerun.rq_conn = PBS_LOCAL_CONNECTION; + brDelete.rq_conn = PBS_LOCAL_CONNECTION; + brRerun.rq_perm = preq->rq_perm; + brDelete.rq_perm = preq->rq_perm; + brDelete.rq_ind.rq_delete.rq_objtype = MGR_OBJ_JOB; + brDelete.rq_ind.rq_delete.rq_cmd = MGR_CMD_DELETE; + unlock_ji_mutex(pjob,__func__,NULL,LOGLEVEL); pnode->tmp_unlock_node(__func__, NULL, LOGLEVEL); - int rc = req_rerunjob(brRerun); + int rc = req_rerunjob(&brRerun); if(rc != PBSE_NONE) { - rc = req_deletejob(brDelete); + rc = req_deletejob(&brDelete); if(rc == PBSE_NONE) { get_svr_attr_l(SRV_ATR_TimeoutForJobDelete, &delete_timeout); @@ -2064,7 +2072,6 @@ static bool requeue_or_delete_jobs( } else { - free_br(brDelete); get_svr_attr_l(SRV_ATR_TimeoutForJobRequeue, &requeue_timeout); if(!wait_for_job_state(*jid,JOB_STATE_QUEUED,requeue_timeout)) @@ -2695,6 +2702,38 @@ int extra_resc_chk( +int set_default_gpu_mode_int( + + const char *gpu_mode_str) + + { + int rc = PBSE_NONE; + + // Make sure it's one of the acceptable gpu modes + if (!strcmp(gpu_mode_str, "exclusive_thread")) + { + default_gpu_mode = gpu_exclusive_thread; + } + else if (!strcmp(gpu_mode_str, "exclusive_process")) + { + default_gpu_mode = gpu_exclusive_process; + } + else if (!strcmp(gpu_mode_str, "default")) + { + default_gpu_mode = gpu_normal; + } + else if (!strcmp(gpu_mode_str, "shared")) + { + default_gpu_mode = gpu_normal; + } + else + rc = PBSE_ATTRTYPE; + + return(rc); + } // END set_default_gpu_mode_int() + + + /* * check_default_gpu_mode_str() * @@ -2718,33 +2757,9 @@ int check_default_gpu_mode_str( // this shouldn't happen if (gpu_mode == NULL) - { - return(PBSE_ATTRTYPE); - } - - // Make sure it's one of the acceptable gpu modes - if (!strcmp(gpu_mode, "exclusive_thread")) - { - default_gpu_mode = gpu_exclusive_thread; - } - else if (!strcmp(gpu_mode, "exclusive")) - { - default_gpu_mode = gpu_exclusive; - } - else if (!strcmp(gpu_mode, "exclusive_process")) - { - default_gpu_mode = gpu_exclusive_process; - } - else if (!strcmp(gpu_mode, "default")) - { - default_gpu_mode = gpu_normal; - } - else if (!strcmp(gpu_mode, "shared")) - { - default_gpu_mode = gpu_normal; - } - else rc = PBSE_ATTRTYPE; + else + rc = set_default_gpu_mode_int(gpu_mode); break; } diff --git a/src/server/req_message.c b/src/server/req_message.c index c3142b319c..37dee30210 100644 --- a/src/server/req_message.c +++ b/src/server/req_message.c @@ -128,7 +128,6 @@ void *req_messagejob( { job *pjob; int rc; - batch_request *dup_req = NULL; if ((pjob = chk_job_request(preq->rq_ind.rq_message.rq_jid, preq)) == NULL) return(NULL); @@ -144,22 +143,15 @@ void *req_messagejob( return(NULL); } - if ((dup_req = duplicate_request(preq)) == NULL) - { - req_reject(PBSE_MEM_MALLOC, 0, preq, NULL, NULL); - } - /* pass the request on to MOM */ - /* The dup_req is freed in relay_to_mom (failure) - * or in issue_Drequest (success) */ - else if ((rc = relay_to_mom(&pjob, dup_req, NULL)) != PBSE_NONE) + batch_request dup_req(*preq); + + if ((rc = relay_to_mom(&pjob, &dup_req, NULL)) != PBSE_NONE) { req_reject(rc, 0, preq, NULL, NULL); /* unable to get to MOM */ - free_br(dup_req); } else { - post_message_req(dup_req); - free_br(preq); + post_message_req(&dup_req); } /* After MOM acts and replies to us, we pick up in post_message_req() */ diff --git a/src/server/req_modify.c b/src/server/req_modify.c index c00fa8aba8..cadf2fd3f8 100644 --- a/src/server/req_modify.c +++ b/src/server/req_modify.c @@ -143,7 +143,7 @@ extern void rel_resc(job *); extern job *chk_job_request(char *, struct batch_request *); extern struct batch_request *cpy_checkpoint(struct batch_request *, job *, enum job_atr, int); -extern char *get_correct_jobname(const char *jobid); +extern const char *get_correct_jobname(const char *jobid, std::string &correct); /* prototypes */ void post_modify_arrayreq(batch_request *preq); @@ -230,7 +230,6 @@ void mom_cleanup_checkpoint_hold( job *pjob; char *jobid; - batch_request *preq; char log_buf[LOCAL_LOG_BUF_SIZE]; time_t time_now = time(NULL); @@ -279,44 +278,30 @@ void mom_cleanup_checkpoint_hold( if (pjob->ji_qs.ji_state != JOB_STATE_RUNNING) { - if ((preq = alloc_br(PBS_BATCH_DeleteJob)) == NULL) - { - log_err(-1, __func__, "unable to allocate DeleteJob request - big trouble!"); - } - else + batch_request preq(PBS_BATCH_DeleteJob); + + strcpy(preq.rq_ind.rq_delete.rq_objname, pjob->ji_qs.ji_jobid); + + if ((rc = relay_to_mom(&pjob, &preq, NULL)) != PBSE_NONE) { - strcpy(preq->rq_ind.rq_delete.rq_objname, pjob->ji_qs.ji_jobid); - /* The preq is freed in relay_to_mom (failure) - * or in issue_Drequest (success) */ - if ((rc = relay_to_mom(&pjob, preq, NULL)) != PBSE_NONE) + if (pjob != NULL) { - if (pjob != NULL) - { - snprintf(log_buf,sizeof(log_buf), - "Unable to relay information to mom for job '%s'\n", - pjob->ji_qs.ji_jobid); - - log_err(rc, __func__, log_buf); - } - else - job_mutex.set_unlock_on_exit(false); - - free_br(preq); - - return; + snprintf(log_buf,sizeof(log_buf), + "Unable to relay information to mom for job '%s'\n", + pjob->ji_qs.ji_jobid); + + log_err(rc, __func__, log_buf); } else - free_br(preq); + job_mutex.set_unlock_on_exit(false); - if ((LOGLEVEL >= 7) && - (pjob != NULL)) - { - log_event( - PBSEVENT_JOB, - PBS_EVENTCLASS_JOB, - pjob->ji_qs.ji_jobid, - "requested mom cleanup"); - } + return; + } + + if ((LOGLEVEL >= 7) && + (pjob != NULL)) + { + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, "requested mom cleanup"); } } else @@ -357,8 +342,6 @@ void chkpt_xfr_hold( log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf); } - - free_br(preq); set_task(WORK_Immed, 0, mom_cleanup_checkpoint_hold, strdup(pjob->ji_qs.ji_jobid), FALSE); @@ -367,22 +350,6 @@ void chkpt_xfr_hold( - - -/* - * chkpt_xfr_done - Handle the clean up of the transfer of the checkpoint files. - */ - -void chkpt_xfr_done( - - batch_request *preq) - - { - free_br(preq); - } /* END chkpt_xfr_done() */ - - - /* * has_unalterable_attribute() * @@ -686,12 +653,11 @@ int modify_job( else { reply_ack(preq); - preq = NULL; } if (copy_checkpoint_files) { - struct batch_request *momreq = 0; + struct batch_request *momreq = NULL; momreq = cpy_checkpoint(momreq, pjob, JOB_ATR_checkpoint_name, CKPT_DIR_OUT); if (momreq != NULL) @@ -699,14 +665,10 @@ int modify_job( /* have files to copy */ momreq->rq_extra = strdup(pjob->ji_qs.ji_jobid); - /* The momreq is freed in relay_to_mom (failure) - * or in issue_Drequest (success) */ rc = relay_to_mom(&pjob, momreq, NULL); if (rc != PBSE_NONE) { - free_br(momreq); - if (pjob != NULL) { snprintf(log_buf,sizeof(log_buf), @@ -720,8 +682,8 @@ int modify_job( } else if (checkpoint_req == CHK_HOLD) chkpt_xfr_hold(momreq, pjob); - else - chkpt_xfr_done(momreq); + + delete momreq; } else { @@ -767,11 +729,12 @@ int modify_whole_array( else { /* NO_MOM_RELAY will prevent modify_job from calling relay_to_mom */ - batch_request *array_req = duplicate_request(preq, i); + batch_request array_req(*preq); + array_req.update_object_id(i); mutex_mgr job_mutex(pjob->ji_mutex, true); pthread_mutex_unlock(pa->ai_mutex); - array_req->rq_noreply = TRUE; - rc = modify_job((void **)&pjob, plist, array_req, checkpoint_req, NO_MOM_RELAY); + array_req.rq_noreply = true; + rc = modify_job((void **)&pjob, plist, &array_req, checkpoint_req, NO_MOM_RELAY); if (rc != PBSE_NONE) { modify_job_rc = rc; @@ -936,7 +899,7 @@ void *req_modifyarray( /* If async modify, reply now; otherwise reply is handled later */ if (preq->rq_type == PBS_BATCH_AsyModifyJob) { - preq->rq_noreply = TRUE; /* set for no more replies */ + preq->rq_noreply = true; /* set for no more replies */ reply_ack(preq); enqueue_threadpool_request(modify_array_work, preq, async_pool); @@ -949,22 +912,21 @@ void *req_modifyarray( -void *modify_job_work( +int modify_job_work( - batch_request *vp) /* I */ + batch_request *preq) /* I */ { job *pjob; svrattrl *plist; int checkpoint_req = FALSE; - batch_request *preq = (struct batch_request *)vp; pjob = svr_find_job(preq->rq_ind.rq_modify.rq_objname, FALSE); if (pjob == NULL) { req_reject(PBSE_JOBNOTFOUND, 0, preq, NULL, "Job unexpectedly deleted"); - return(NULL); + return(PBSE_JOBNOTFOUND); } mutex_mgr job_mutex(pjob->ji_mutex, true); @@ -987,12 +949,11 @@ void *modify_job_work( /* modify_job will free preq and respond to it */ modify_job((void **)&pjob, plist, preq, checkpoint_req, 0); - return(NULL); + return(PBSE_NONE); } /* END modify_job_work() */ - /* * req_modifyjob - service the Modify Job Request * @@ -1006,10 +967,9 @@ void *req_modifyjob( batch_request *preq) /* I */ { - job *pjob; - svrattrl *plist; - char log_buf[LOCAL_LOG_BUF_SIZE]; - char *correct_jobname_p; + job *pjob; + svrattrl *plist; + std::string correct_jobname; pjob = chk_job_request(preq->rq_ind.rq_modify.rq_objname, preq); @@ -1019,22 +979,17 @@ void *req_modifyjob( } // get the correct job name - correct_jobname_p = get_correct_jobname(preq->rq_ind.rq_modify.rq_objname); + get_correct_jobname(preq->rq_ind.rq_modify.rq_objname, correct_jobname); // if correct job name and one passed in don't match, adjust one passed in // so that mom will be able to match it and not reject modify request due // to job not found - if (correct_jobname_p != NULL) + if (correct_jobname != preq->rq_ind.rq_modify.rq_objname) { - if (strcmp(correct_jobname_p, preq->rq_ind.rq_modify.rq_objname) != 0) - { - // job names don't match so need to modify the requested jobname - snprintf(preq->rq_ind.rq_modify.rq_objname, - sizeof(preq->rq_ind.rq_modify.rq_objname), - "%s", correct_jobname_p); - } - - free(correct_jobname_p); + // job names don't match so need to modify the requested jobname + snprintf(preq->rq_ind.rq_modify.rq_objname, + sizeof(preq->rq_ind.rq_modify.rq_objname), + "%s", correct_jobname.c_str()); } mutex_mgr job_mutex(pjob->ji_mutex, true); @@ -1059,18 +1014,12 @@ void *req_modifyjob( /* reply_ack will free preq. We need to copy it before we call reply_ack */ batch_request *new_preq; - new_preq = duplicate_request(preq, -1); - if (new_preq == NULL) - { - sprintf(log_buf, "failed to duplicate batch request"); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); - return(NULL); - } + new_preq = new batch_request(*preq); get_batch_request_id(new_preq); reply_ack(preq); - new_preq->rq_noreply = TRUE; /* set for no more replies */ + new_preq->rq_noreply = true; /* set for no more replies */ enqueue_threadpool_request((void *(*)(void *))modify_job_work, preq, async_pool); } @@ -1082,8 +1031,6 @@ void *req_modifyjob( - - /* * modify_job_attr - modify the attributes of a job atomically * Used by req_modifyjob() to alter the job attributes and by @@ -1354,6 +1301,8 @@ int modify_job_attr( /* * post_modify_arrayreq - clean up after sending modify request to MOM + * + * FIXME: This appears to be dead code */ void post_modify_arrayreq( diff --git a/src/server/req_quejob.c b/src/server/req_quejob.c index 3f3b9742d0..e30acbb59d 100644 --- a/src/server/req_quejob.c +++ b/src/server/req_quejob.c @@ -814,32 +814,12 @@ int decode_attributes_into_job( if (rc != 0) { - if (rc == PBSE_UNKRESC) - { - /* check for RM extension */ - - /* NYI */ - - /* unknown resources not allowed in Exec queue */ - - if (pque->qu_qs.qu_type == QTYPE_Execution) - { - /* FAILURE */ - svr_job_purge(pj); - job_mutex.set_unlock_on_exit(false); - reply_badattr(rc, 1, psatl, preq); - return(rc); - } - } - else - { - /* FAILURE */ - /* any other error is fatal */ - svr_job_purge(pj); - job_mutex.set_unlock_on_exit(false); - reply_badattr(rc, 1, psatl, preq); - return(rc); - } + /* FAILURE */ + /* any other error is fatal */ + svr_job_purge(pj); + job_mutex.set_unlock_on_exit(false); + reply_badattr(rc, 1, psatl, preq); + return(rc); } /* END if (rc != 0) */ psatl = (svrattrl *)GET_NEXT(psatl->al_link); @@ -1636,12 +1616,12 @@ int perform_commit_work( if ((pattr->at_val.at_long == 0) && (nodes_avail > 0)) { /* Create a new batch request and fill it in */ - preq_run = alloc_br(PBS_BATCH_RunJob); + preq_run = new batch_request(PBS_BATCH_RunJob); preq_run->rq_perm = preq->rq_perm | ATR_DFLAG_OPWR; preq_run->rq_ind.rq_run.rq_resch = 0; preq_run->rq_ind.rq_run.rq_destin = rq_destin; preq_run->rq_fromsvr = preq->rq_fromsvr; - preq_run->rq_noreply = TRUE; /* set for no replies */ + preq_run->rq_noreply = true; /* set for no replies */ strcpy(preq_run->rq_user, preq->rq_user); strcpy(preq_run->rq_host, preq->rq_host); strcpy(preq_run->rq_ind.rq_run.rq_jid, preq->rq_ind.rq_rdytocommit); @@ -1993,76 +1973,36 @@ int req_jobcredential( /* - * req_jobscript - receive job script section + * write_job_file() * - * Each section is appended to the file + * @param pj - the job whose file we're writing + * @param append_only - we should be only appending the job file at this time + * @return PBSE_NONE on SUCCESS, false otherwise */ -int req_jobscript( +int write_job_file( batch_request *preq, - bool perform_commit) + job *pj, + std::string &errbuf, + bool append_only) { - int fds; - char namebuf[MAXPATHLEN]; - job *pj; - int filemode = 0600; - char log_buf[LOCAL_LOG_BUF_SIZE]; - int rc = PBSE_NONE; + int filemode = 0600; + int fds = -1; + char namebuf[MAXPATHLEN]; + char log_buf[LOCAL_LOG_BUF_SIZE]; + int rc = PBSE_NONE; std::string adjusted_path_jobs; - errno = 0; - - pj = locate_new_job(preq->rq_ind.rq_jobfile.rq_jobid); - - if (pj == NULL) - { - rc = PBSE_IVALREQ; - snprintf(log_buf, LOCAL_LOG_BUF_SIZE, "cannot locate new job %s (%d - %s)", - preq->rq_ind.rq_jobfile.rq_jobid, errno, strerror(errno)); - log_err(rc, __func__, log_buf); - req_reject(rc, 0, preq, NULL, log_buf); - return(rc); - } - - mutex_mgr job_mutex(pj->ji_mutex, true); - - /* what is the difference between JOB_SUBSTATE_TRANSIN and TRANSICM? */ - if (pj->ji_qs.ji_substate != JOB_SUBSTATE_TRANSIN) - { - rc = PBSE_IVALREQ; - if (errno == 0) - { - snprintf(log_buf, sizeof(log_buf), - "job %s in unexpected state '%s'", - pj->ji_qs.ji_jobid, - PJobSubState[pj->ji_qs.ji_substate]); - } - else - { - snprintf(log_buf, sizeof(log_buf), - "job %s in unexpected state '%s' (errno=%d - %s)", - pj->ji_qs.ji_jobid, - PJobSubState[pj->ji_qs.ji_substate], - errno, - strerror(errno)); - } - - log_err(rc, __func__, log_buf); - req_reject(rc, 0, preq, NULL, log_buf); - return(rc); - } - if (svr_authorize_jobreq(preq, pj) == -1) { rc = PBSE_PERM; snprintf(log_buf, LOCAL_LOG_BUF_SIZE, "cannot authorize request %s (%d-%s)", preq->rq_ind.rq_jobfile.rq_jobid, errno, strerror(errno)); - log_err(rc, __func__, log_buf); - req_reject(rc, 0, preq, NULL, log_buf); - return rc; + errbuf = log_buf; + return(rc); } // get adjusted path_jobs path @@ -2070,15 +2010,16 @@ int req_jobscript( snprintf(namebuf, sizeof(namebuf), "%s%s%s", adjusted_path_jobs.c_str(), pj->ji_qs.ji_fileprefix, JOB_SCRIPT_SUFFIX); - if (pj->ji_qs.ji_un.ji_newt.ji_scriptsz == 0) + if ((append_only == true) || + (pj->ji_qs.ji_un.ji_newt.ji_scriptsz != 0)) { - /* NOTE: fail is job script already exists */ - - fds = open(namebuf, O_WRONLY | O_CREAT | O_EXCL | O_Sync, filemode); + // Open existing file + fds = open(namebuf, O_WRONLY | O_APPEND | O_Sync, filemode); } else { - fds = open(namebuf, O_WRONLY | O_APPEND | O_Sync, filemode); + // Open a new file + fds = open(namebuf, O_WRONLY | O_CREAT | O_EXCL | O_Sync, filemode); } if (fds < 0) @@ -2089,8 +2030,7 @@ int req_jobscript( errno, strerror(errno), msg_script_open); - log_err(rc, __func__, log_buf); - req_reject(rc, 0, preq, NULL, log_buf); + errbuf = log_buf; return rc; } @@ -2105,21 +2045,113 @@ int req_jobscript( errno, strerror(errno), msg_script_write); - log_err(rc, __func__, log_buf); - req_reject(PBSE_INTERNAL, 0, preq, NULL, log_buf); + errbuf = log_buf; close(fds); return rc; } close(fds); - pj->ji_qs.ji_un.ji_newt.ji_scriptsz += preq->rq_ind.rq_jobfile.rq_size; + if (append_only == false) + pj->ji_qs.ji_un.ji_newt.ji_scriptsz += preq->rq_ind.rq_jobfile.rq_size; /* job has a script file */ - pj->ji_qs.ji_svrflags = (pj->ji_qs.ji_svrflags & ~JOB_SVFLG_CHECKPOINT_FILE) | JOB_SVFLG_SCRIPT; + return(PBSE_NONE); + } // END write_job_file() + + + +/* + * req_jobscript - receive job script section + * + * Each section is appended to the file + */ + +int req_jobscript( + + batch_request *preq, + bool perform_commit) + + { + job *pj; + char log_buf[LOCAL_LOG_BUF_SIZE]; + int rc = PBSE_NONE; + std::string errbuf; + + errno = 0; + + pj = locate_new_job(preq->rq_ind.rq_jobfile.rq_jobid); + + if (pj == NULL) + { + rc = -1; + + // Due to the condensed queuing process, sometimes jobs can be already committed by the time we + // send the next step of the job file. Check if it's in the job list and append the file if + // so. + if ((pj = svr_find_job(preq->rq_ind.rq_jobfile.rq_jobid, TRUE)) != NULL) + { + mutex_mgr job_mutex(pj->ji_mutex, true); + rc = write_job_file(preq, pj, errbuf, true); + + // Appended the job script + if (rc == PBSE_NONE) + { + reply_ack(preq); + return(rc); + } + } + + if (rc == -1) + { + rc = PBSE_IVALREQ; + snprintf(log_buf, LOCAL_LOG_BUF_SIZE, "cannot locate new job %s (%d - %s)", + preq->rq_ind.rq_jobfile.rq_jobid, errno, strerror(errno)); + log_err(rc, __func__, log_buf); + } + + req_reject(rc, 0, preq, NULL, log_buf); + return(rc); + } + + mutex_mgr job_mutex(pj->ji_mutex, true); + + /* what is the difference between JOB_SUBSTATE_TRANSIN and TRANSICM? */ + if (pj->ji_qs.ji_substate != JOB_SUBSTATE_TRANSIN) + { + rc = PBSE_IVALREQ; + if (errno == 0) + { + snprintf(log_buf, sizeof(log_buf), + "job %s in unexpected state '%s'", + pj->ji_qs.ji_jobid, + PJobSubState[pj->ji_qs.ji_substate]); + } + else + { + snprintf(log_buf, sizeof(log_buf), + "job %s in unexpected state '%s' (errno=%d - %s)", + pj->ji_qs.ji_jobid, + PJobSubState[pj->ji_qs.ji_substate], + errno, + strerror(errno)); + } + + log_err(rc, __func__, log_buf); + req_reject(rc, 0, preq, NULL, log_buf); + return(rc); + } + + if ((rc = write_job_file(preq, pj, errbuf, false)) != PBSE_NONE) + { + log_err(rc, __func__, log_buf); + req_reject(rc, 0, preq, NULL, log_buf); + return(rc); + } + /* SUCCESS */ if (perform_commit == true) { @@ -2532,12 +2564,12 @@ int req_commit2( if ((pattr->at_val.at_long == 0) && (nodes_avail > 0)) { /* Create a new batch request and fill it in */ - preq_run = alloc_br(PBS_BATCH_RunJob); + preq_run = new batch_request(PBS_BATCH_RunJob); preq_run->rq_perm = preq->rq_perm | ATR_DFLAG_OPWR; preq_run->rq_ind.rq_run.rq_resch = 0; preq_run->rq_ind.rq_run.rq_destin = rq_destin; preq_run->rq_fromsvr = preq->rq_fromsvr; - preq_run->rq_noreply = TRUE; /* set for no replies */ + preq_run->rq_noreply = true; /* set for no replies */ strcpy(preq_run->rq_user, preq->rq_user); strcpy(preq_run->rq_host, preq->rq_host); strcpy(preq_run->rq_ind.rq_run.rq_jid, preq->rq_ind.rq_rdytocommit); diff --git a/src/server/req_register.c b/src/server/req_register.c index c688475a03..f97a4e37b0 100644 --- a/src/server/req_register.c +++ b/src/server/req_register.c @@ -107,6 +107,7 @@ #include "mutex_mgr.hpp" #include "utils.h" #include "job_func.h" +#include "threadpool.h" #define SYNC_SCHED_HINT_NULL 0 @@ -122,7 +123,7 @@ extern int que_to_local_svr(struct batch_request *preq); extern long calc_job_cost(job *); -char *get_correct_jobname(const char *jobid); +const char *get_correct_jobname(const char *jobid, std::string &corrected); /* Local Private Functions */ @@ -375,13 +376,12 @@ bool job_ids_match( if ((is_svr_attr_set(SRV_ATR_display_job_server_suffix)) || (is_svr_attr_set(SRV_ATR_job_suffix_alias))) { - char *correct_parent = get_correct_jobname(parent); - char *correct_child = get_correct_jobname(child); + std::string correct_parent; + std::string correct_child; + get_correct_jobname(parent, correct_parent); + get_correct_jobname(child, correct_child); - match = strcmp(correct_parent, correct_child) == 0; - - free(correct_parent); - free(correct_child); + match = correct_parent == correct_child; } else match = strcmp(parent, child) == 0; @@ -1131,11 +1131,13 @@ bool set_array_depend_holds( job_array *pa) { - int compareNumber; + int compare_number; + int could_fulfill_dependency; bool dependency_satisfied = false; job *pjob; array_depend_job *pdj; + char log_buf[LOCAL_LOG_BUF_SIZE]; /* loop through dependencies to update holds */ for (std::list::iterator it = pa->ai_qs.deps.begin(); @@ -1143,55 +1145,39 @@ bool set_array_depend_holds( { array_depend *pdep = *it; - compareNumber = -1; + could_fulfill_dependency = pa->ai_qs.array_size; + compare_number = -1; switch (pdep->dp_type) { case JOB_DEPEND_TYPE_AFTERSTARTARRAY: - - compareNumber = pa->ai_qs.num_started; - - break; - - case JOB_DEPEND_TYPE_AFTEROKARRAY: - - compareNumber = pa->ai_qs.num_successful; - - break; - - case JOB_DEPEND_TYPE_AFTERNOTOKARRAY: - - compareNumber = pa->ai_qs.num_failed; - - break; - - case JOB_DEPEND_TYPE_AFTERANYARRAY: - - compareNumber = pa->ai_qs.jobs_done; - - break; - case JOB_DEPEND_TYPE_BEFORESTARTARRAY: - compareNumber = pa->ai_qs.num_started; + compare_number = pa->ai_qs.num_started; + could_fulfill_dependency = pa->ai_qs.num_jobs - pa->ai_qs.jobs_done; break; + case JOB_DEPEND_TYPE_AFTEROKARRAY: case JOB_DEPEND_TYPE_BEFOREOKARRAY: - compareNumber = pa->ai_qs.num_successful; + compare_number = pa->ai_qs.num_successful; + could_fulfill_dependency = pa->ai_qs.num_jobs - pa->ai_qs.num_failed; break; + case JOB_DEPEND_TYPE_AFTERNOTOKARRAY: case JOB_DEPEND_TYPE_BEFORENOTOKARRAY: - compareNumber = pa->ai_qs.num_failed; + compare_number = pa->ai_qs.num_failed; + could_fulfill_dependency = pa->ai_qs.num_jobs - pa->ai_qs.num_successful; break; + case JOB_DEPEND_TYPE_AFTERANYARRAY: case JOB_DEPEND_TYPE_BEFOREANYARRAY: - compareNumber = pa->ai_qs.jobs_done; + compare_number = pa->ai_qs.jobs_done; break; } @@ -1208,25 +1194,49 @@ bool set_array_depend_holds( { mutex_mgr job_mutex(pjob->ji_mutex, true); - if (((compareNumber < pdj->dc_num) && + if (((compare_number < pdj->dc_num) && (pdep->dp_type < JOB_DEPEND_TYPE_BEFORESTARTARRAY)) || - ((compareNumber >= pdj->dc_num) && + ((compare_number >= pdj->dc_num) && (pdep->dp_type > JOB_DEPEND_TYPE_AFTERANYARRAY))) { - /* hold */ - pjob->ji_wattr[JOB_ATR_hold].at_val.at_long |= HOLD_s; - pjob->ji_wattr[JOB_ATR_hold].at_flags |= ATR_VFLAG_SET; - - if (LOGLEVEL >= 8) + // AFTERANYARRAY can always be potentially fulfilled, and any BEFORE* dependency + // that isn't satisfied means it can never be fulfilled + if ((pdep->dp_type > JOB_DEPEND_TYPE_AFTERANYARRAY) || + ((pdep->dp_type < JOB_DEPEND_TYPE_AFTERANYARRAY) && + (could_fulfill_dependency < pdj->dc_num))) { - log_event( - PBSEVENT_JOB, - PBS_EVENTCLASS_JOB, - pjob->ji_qs.ji_jobid, - "Setting HOLD_s due to dependencies\n"); + // These are dependencies that can never be fulfilled + if ((pjob->ji_qs.ji_state < JOB_STATE_EXITING) && + (pjob->ji_qs.ji_state != JOB_STATE_RUNNING)) + { + sprintf(log_buf, + "Job %s deleted because its dependency of array %s can never be satisfied", + pjob->ji_qs.ji_jobid, pa->ai_qs.parent_id); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, pjob->ji_qs.ji_jobid, log_buf); + + /* pjob freed and set to NULL */ + job_abt(&pjob, log_buf, true); + if (pjob == NULL) + job_mutex.set_unlock_on_exit(false); + } } + else + { + // hold + pjob->ji_wattr[JOB_ATR_hold].at_val.at_long |= HOLD_s; + pjob->ji_wattr[JOB_ATR_hold].at_flags |= ATR_VFLAG_SET; - svr_setjobstate(pjob, JOB_STATE_HELD, JOB_SUBSTATE_DEPNHOLD, FALSE); + if (LOGLEVEL >= 8) + { + log_event( + PBSEVENT_JOB, + PBS_EVENTCLASS_JOB, + pjob->ji_qs.ji_jobid, + "Setting HOLD_s due to dependencies\n"); + } + + svr_setjobstate(pjob, JOB_STATE_HELD, JOB_SUBSTATE_DEPNHOLD, FALSE); + } } else { @@ -1335,8 +1345,6 @@ void post_doq( } } - free_br(preq); - return; } /* END post_doq() */ @@ -1380,7 +1388,7 @@ int alter_unreg( if ((pnewd == 0) || (find_dependjob(pnewd, oldjd->dc_child.c_str()) == 0)) { - int rc = send_depend_req(pjob, oldjd, type, JOB_DEPEND_OP_UNREG, SYNC_SCHED_HINT_NULL, free_br,false); + int rc = send_depend_req(pjob, oldjd, type, JOB_DEPEND_OP_UNREG, SYNC_SCHED_HINT_NULL, NULL, false); if (rc == PBSE_JOBNOTFOUND) return(rc); @@ -1418,31 +1426,15 @@ int depend_on_que( int rc = PBSE_NONE; int type; job *pjob = (job *)pj; - pbs_queue *pque; char job_id[PBS_MAXSVRJOBID+1]; strcpy(job_id, pjob->ji_qs.ji_jobid); - pque = get_jobs_queue(&pjob); - - if (pque == NULL) - { - if (pjob == NULL) - { - log_err(PBSE_JOBNOTFOUND, __func__, "Job lost while acquiring queue 8"); - return(PBSE_JOBNOTFOUND); - } - else - return(PBSE_NONE); - } - mutex_mgr pque_mutex = mutex_mgr(pque->qu_mutex, true); if (((mode != ATR_ACTION_ALTER) && - (mode != ATR_ACTION_NOOP)) || - (pque->qu_qs.qu_type != QTYPE_Execution)) + (mode != ATR_ACTION_NOOP))) { return(PBSE_NONE); } - pque_mutex.unlock(); if (mode == ATR_ACTION_ALTER) { @@ -1498,7 +1490,7 @@ int depend_on_que( for (unsigned int i = 0; i < pparent.size(); i++) { if ((rc = send_depend_req(pjob, pparent[i], type, JOB_DEPEND_OP_REGISTER, - SYNC_SCHED_HINT_NULL, post_doq,false)) != PBSE_NONE) + SYNC_SCHED_HINT_NULL, post_doq, false)) != PBSE_NONE) break; } @@ -1564,8 +1556,6 @@ void post_doe( } } - free_br(preq); - return; } /* END post_doe() */ @@ -1584,9 +1574,9 @@ int depend_on_exec( job *pjob) { - struct depend *pdep; - depend_job *pdj; - char jobid[PBS_MAXSVRJOBID+1]; + depend *pdep; + depend_job *pdj; + char jobid[PBS_MAXSVRJOBID+1]; strcpy(jobid, pjob->ji_qs.ji_jobid); /* If any jobs come after my start, release them */ @@ -1596,17 +1586,19 @@ int depend_on_exec( if (pdep != NULL) { + std::vector depend_jobs = pdep->dp_jobs; - for (unsigned int i = 0; i < pdep->dp_jobs.size(); i++) + for (unsigned int i = 0; i < depend_jobs.size(); i++) { - pdj = pdep->dp_jobs[i]; + pdj = depend_jobs[i]; if (send_depend_req(pjob, pdj, pdep->dp_type, JOB_DEPEND_OP_RELEASE, SYNC_SCHED_HINT_NULL, - post_doe,false) == PBSE_JOBNOTFOUND) + post_doe, + false) == PBSE_JOBNOTFOUND) { return(PBSE_JOBNOTFOUND); } @@ -1631,7 +1623,8 @@ int depend_on_exec( pdep->dp_type, JOB_DEPEND_OP_READY, SYNC_SCHED_HINT_NULL, - free_br,false) == PBSE_JOBNOTFOUND) + NULL, + false) == PBSE_JOBNOTFOUND) { return(PBSE_JOBNOTFOUND); } @@ -1660,9 +1653,6 @@ int depend_on_exec( - - - /* depend_on_term - Perform actions if job has "afterany, afterok, afternotok" * dependencies, send "register-release" or register-delete" as * appropriate. @@ -1765,7 +1755,7 @@ int depend_on_term( { pparent = pdep->dp_jobs[i]; - rc = send_depend_req(pjob, pparent, type, JOB_DEPEND_OP_DELETE, SYNC_SCHED_HINT_NULL, free_br,true); + rc = send_depend_req(pjob, pparent, type, JOB_DEPEND_OP_DELETE, SYNC_SCHED_HINT_NULL, NULL, true); if (rc == PBSE_JOBNOTFOUND) { @@ -1786,7 +1776,7 @@ int depend_on_term( pparent = pdep->dp_jobs[i]; /* "release" the job to execute */ - if ((rc = send_depend_req(pjob, pparent, type, op, SYNC_SCHED_HINT_NULL, free_br,true)) != PBSE_NONE) + if ((rc = send_depend_req(pjob, pparent, type, op, SYNC_SCHED_HINT_NULL, NULL, true)) != PBSE_NONE) { return(rc); } @@ -1847,7 +1837,7 @@ int release_cheapest( hint = SYNC_SCHED_HINT_FIRST; if ((rc = send_depend_req(pjob, cheapest, JOB_DEPEND_TYPE_SYNCWITH, - JOB_DEPEND_OP_RELEASE, hint, free_br,false)) == PBSE_NONE) + JOB_DEPEND_OP_RELEASE, hint, NULL, false)) == PBSE_NONE) { cheapest->dc_state = JOB_DEPEND_OP_RELEASE; } @@ -1884,6 +1874,7 @@ void set_depend_hold( int loop = 1; int newstate; int newsubst; + char log_buf[LOCAL_LOG_BUF_SIZE]; struct depend *pdp = NULL; depend_job *djob = NULL; @@ -1936,6 +1927,15 @@ void set_depend_hold( lock the job. It is already locked */ if (!jobids_match) djp = svr_find_job(djob->dc_child.c_str(), TRUE); + else + { + // This should never happen. A Job cannot depend on itself. + snprintf(log_buf, sizeof(log_buf), + "Job %s somehow has a dependency on itself. Purging.", pjob->ji_qs.ji_jobid); + log_err(-1, __func__, log_buf); + enqueue_threadpool_request(svr_job_purge_task, pjob, task_pool); + throw (int)PBSE_BADDEPEND; + } if (!djp || ((pdp->dp_type == JOB_DEPEND_TYPE_AFTERSTART) && @@ -2427,27 +2427,18 @@ int send_depend_req( bool bAsyncOk) { - int rc = 0; - int i; - char job_id[PBS_MAXSVRJOBID + 1]; - char br_id[MAXLINE]; + int rc = 0; + int i; + char job_id[PBS_MAXSVRJOBID + 1]; - struct batch_request *preq; - char log_buf[LOCAL_LOG_BUF_SIZE]; - - preq = alloc_br(PBS_BATCH_RegistDep); - - if (preq == NULL) - { - log_err(errno, __func__, msg_err_malloc); - return(PBSE_SYSTEM); - } + batch_request *preq = new batch_request(PBS_BATCH_RegistDep); + char log_buf[LOCAL_LOG_BUF_SIZE]; for (i = 0;i < PBS_MAXUSER;++i) { if (pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str == NULL) { - free_br(preq); + delete preq; return(PBSE_BADATVAL); } @@ -2501,9 +2492,6 @@ int send_depend_req( strcpy(job_id, pjob->ji_qs.ji_jobid); unlock_ji_mutex(pjob, __func__, "2", LOGLEVEL); - get_batch_request_id(preq); - snprintf(br_id, sizeof(br_id), "%s", preq->rq_id); - if (bAsyncOk) { snprintf(preq->rq_host, sizeof(preq->rq_host), "%s", server_name); @@ -2512,23 +2500,17 @@ int send_depend_req( } else { - rc = issue_to_svr(server_name, &preq, NULL); + rc = issue_to_svr(server_name, preq, NULL); } if (rc != PBSE_NONE) { - /* requests are conditionally freed in issue_to_svr() */ if (preq != NULL) - { - free_br(preq); - } + delete preq; sprintf(log_buf, "Unable to perform dependency with job %s\n", pparent->dc_child.c_str()); log_err(rc, __func__, log_buf); - if ((preq = get_remove_batch_request(br_id)) != NULL) - free_br(preq); - if ((pjob = svr_find_job(job_id, TRUE)) == NULL) { return(PBSE_JOBNOTFOUND); @@ -2538,9 +2520,13 @@ int send_depend_req( } /* local requests have already been processed and freed. Do not attempt to * free or reference again. */ - else if (preq != NULL) + else if ((preq != NULL) && + (postfunc != NULL)) postfunc(preq); + if (preq != NULL) + delete preq; + if ((pjob = svr_find_job(job_id, TRUE)) == NULL) { return(PBSE_JOBNOTFOUND); @@ -3239,13 +3225,16 @@ int build_depend( else /* all other dependency types */ { /* a set of job_id[\:port][@server[\:port]] */ + std::string correct_id; pdjb = new depend_job(); + get_correct_jobname(valwd, correct_id); + if (pdjb) { pdjb->dc_state = 0; pdjb->dc_cost = 0; - pdjb->dc_child = valwd; + pdjb->dc_child = correct_id; std::size_t pos = pdjb->dc_child.find("@"); @@ -3368,6 +3357,12 @@ void removeAfterAnyDependency( if (pDepJob != NULL) { del_depend_job(pDep, pDepJob); + + if (pDep->dp_jobs.size() == 0) + { + /* no more dependencies of this type */ + delete pDep; + } try { @@ -3384,39 +3379,42 @@ void removeAfterAnyDependency( /* * removeBeforeAnyDependencies() - * - * + * Forcibly removes before any dependencies from this job + * @param pjob_ptr - a pointer to the job's pointer */ void removeBeforeAnyDependencies( - const char *pJId) + job **pjob_ptr) { - job *pLockedJob = svr_find_job(pJId,FALSE); + job *pLockedJob = *pjob_ptr; + std::string jobid(pLockedJob->ji_qs.ji_jobid); - if (pLockedJob != NULL) + mutex_mgr job_mutex(pLockedJob->ji_mutex, true); + job_mutex.set_unlock_on_exit(false); + + pbs_attribute *pattr = &pLockedJob->ji_wattr[JOB_ATR_depend]; + + struct depend *pDep = find_depend(JOB_DEPEND_TYPE_BEFOREANY,pattr); + if (pDep != NULL) { - mutex_mgr job_mutex(pLockedJob->ji_mutex,true); - pbs_attribute *pattr = &pLockedJob->ji_wattr[JOB_ATR_depend]; - struct depend *pDep = find_depend(JOB_DEPEND_TYPE_BEFOREANY,pattr); - if (pDep != NULL) + unsigned int dp_jobs_size = pDep->dp_jobs.size(); + for (unsigned int i = 0; i < dp_jobs_size; i++) { - - unsigned int dp_jobs_size = pDep->dp_jobs.size(); - for (unsigned int i = 0; i < dp_jobs_size; i++) + depend_job *pDepJob = pDep->dp_jobs[i]; + std::string depID(pDepJob->dc_child); + job_mutex.unlock(); + removeAfterAnyDependency(depID.c_str(), jobid.c_str()); + pLockedJob = svr_find_job(jobid.c_str(), FALSE); + if (pLockedJob == NULL) { - depend_job *pDepJob = pDep->dp_jobs[i]; - std::string depID(pDepJob->dc_child); - job_mutex.unlock(); - removeAfterAnyDependency(depID.c_str(),pJId); - pLockedJob = svr_find_job(pJId,FALSE); - if (pLockedJob == NULL) - return; - - job_mutex.mark_as_locked(); + *pjob_ptr = NULL; + break; } + + job_mutex.mark_as_locked(); } } diff --git a/src/server/req_rerun.c b/src/server/req_rerun.c index 670ed195fd..ed3ab11909 100644 --- a/src/server/req_rerun.c +++ b/src/server/req_rerun.c @@ -118,7 +118,6 @@ extern char *msg_jobrerun; extern void rel_resc(job *); extern job *chk_job_request(char *, struct batch_request *); -extern int issue_signal(job **, const char *, void(*)(batch_request *), void *, char *); int finalize_rerunjob(struct batch_request *preq,job *pjob,int rc); @@ -160,7 +159,6 @@ void delay_and_send_sig_kill( { preq_clt = get_remove_batch_request(preq_sig->rq_extend); } - free_br(preq_sig); /* the client request has been handled another way, nothing left to do */ if (preq_clt == NULL) @@ -316,8 +314,6 @@ void post_rerun( } } - free_br(preq); - return; } /* END post_rerun() */ @@ -533,25 +529,32 @@ int req_rerunjob( (!strncasecmp(preq->rq_extend, RERUNFORCE, strlen(RERUNFORCE)))) { std::string extend = RERUNFORCE; - batch_request *dup = duplicate_request(preq, -1); + batch_request *dup = new batch_request(*preq); get_batch_request_id(dup); - rc = issue_signal(&pjob, "SIGTERM", delay_and_send_sig_kill, extra, strdup(dup->rq_id)); + rc = issue_signal(&pjob, "SIGTERM", delay_and_send_sig_kill, extra, strdup(dup->rq_id.c_str())); if (rc == PBSE_NORELYMOM) { dup->rq_reply.brp_code = PBSE_NORELYMOM; pjob_mutex.unlock(); post_rerun(dup); + pjob = svr_find_job(preq->rq_ind.rq_signal.rq_jid, FALSE); if (pjob == NULL) + { + delete dup; return(PBSE_NONE); + } + pjob_mutex.set_lock_state(true); rc = PBSE_NONE; } + + delete dup; } else { - rc = issue_signal(&pjob, "SIGTERM", delay_and_send_sig_kill, extra, strdup(preq->rq_id)); + rc = issue_signal(&pjob, "SIGTERM", delay_and_send_sig_kill, extra, strdup(preq->rq_id.c_str())); if (rc != PBSE_NONE) { /* cant send to MOM */ diff --git a/src/server/req_runjob.c b/src/server/req_runjob.c index b7d985f834..8b128b8dba 100644 --- a/src/server/req_runjob.c +++ b/src/server/req_runjob.c @@ -140,12 +140,12 @@ extern int LOGLEVEL; /* Public Functions in this file */ -int svr_startjob(job *, struct batch_request **, char *, char *); +int svr_startjob(job *, batch_request *, char *, char *); /* Private Functions local to this file */ -int svr_stagein(job **, struct batch_request **, int, int); -int svr_strtjob2(job **, struct batch_request *); +int svr_stagein(job **, batch_request *, int, int); +int svr_strtjob2(job **, batch_request *); job *chk_job_torun(struct batch_request *, int); int assign_hosts(job *, char *, int, char *, char *); @@ -267,10 +267,9 @@ int check_and_run_job_work( /* NOTE: nodes assigned to job in svr_startjob() */ - rc = svr_startjob(pjob, &preq, failhost, emsg); + rc = svr_startjob(pjob, preq, failhost, emsg); - if ((rc != 0) && - (preq != NULL)) + if (rc != 0) { free_nodes(pjob); @@ -285,10 +284,6 @@ int check_and_run_job_work( } } - if ((rc == PBSE_NONE) && - (preq != NULL)) - free_br(preq); - return(rc); } // END check_and_run_job_work() @@ -303,6 +298,8 @@ void *check_and_run_job( check_and_run_job_work(preq); + delete preq; + return(NULL); } /* END check_and_run_job() */ @@ -370,8 +367,8 @@ int req_runjob( if (preq->rq_type == PBS_BATCH_AsyrunJob) { reply_ack(preq); - preq->rq_noreply = TRUE; - enqueue_threadpool_request(check_and_run_job, preq, async_pool); + preq->rq_noreply = true; + enqueue_threadpool_request(check_and_run_job, new batch_request(*preq), async_pool); } else { @@ -421,7 +418,6 @@ void post_checkpointsend( { int code; job *pjob; - bool preq_free_done = FALSE; pbs_attribute *pwait; char log_buf[LOCAL_LOG_BUF_SIZE]; @@ -490,18 +486,13 @@ void post_checkpointsend( job_save(pjob, SAVEJOB_FULL, 0); /* continue to start job running */ - svr_strtjob2(&pjob, preq); - preq_free_done = TRUE; } if (pjob == NULL) job_mutex.set_unlock_on_exit(false); } /* END if (pjob != NULL) */ - if (!preq_free_done) - free_br(preq); /* close connection and release request */ - return; } /* END post_checkpointsend() */ @@ -514,26 +505,23 @@ void post_checkpointsend( int svr_send_checkpoint( - job **pjob_ptr, /* I */ - struct batch_request **preq, /* I */ - int state, /* I */ - int substate) /* I */ + job **pjob_ptr, /* I */ + batch_request *preq, /* I */ + int state, /* I */ + int substate) /* I */ { - - struct batch_request *momreq = 0; - int rc; - char jobid[PBS_MAXSVRJOBID + 1]; - job *pjob = *pjob_ptr; + batch_request *momreq = 0; + int rc; + char jobid[PBS_MAXSVRJOBID + 1]; + job *pjob = *pjob_ptr; momreq = cpy_checkpoint(momreq, pjob, JOB_ATR_checkpoint_name, CKPT_DIR_IN); if (momreq == NULL) { /* no files to send, go directly to sending job to mom */ - rc = svr_strtjob2(&pjob, *preq); - - *preq = NULL; + rc = svr_strtjob2(&pjob, preq); return(rc); } @@ -563,25 +551,16 @@ int svr_send_checkpoint( * checkpoint copy started ok - reply to client as copy may * take too long to wait. */ - if (*preq != NULL) - { - reply_ack(*preq); - *preq = NULL; - } - } - else - { - free_br(momreq); + reply_ack(preq); } + + delete momreq; return(rc); } /* END svr_send_checkpoint() */ - - - /* * req_stagein - service the Stage In Files for a Job Request * @@ -622,9 +601,9 @@ int req_stagein( if ((rc = svr_stagein( &pjob, - &preq, - JOB_STATE_QUEUED, - JOB_SUBSTATE_STAGEIN))) + preq, + JOB_STATE_QUEUED, + JOB_SUBSTATE_STAGEIN))) { free_nodes(pjob); @@ -713,19 +692,14 @@ void post_stagein( /* need to copy checkpoint file to mom before running */ svr_send_checkpoint( &pjob, - &preq, + preq, JOB_STATE_RUNNING, JOB_SUBSTATE_CHKPTGO); } else { /* continue to start job running */ - svr_strtjob2(&pjob, preq); - - /* svr_strjob2 would call finish_sendmom which would free preq - in its reply_send_svr, set preq now to NULL to avoid double free */ - preq = NULL; } } else @@ -740,9 +714,6 @@ void post_stagein( job_mutex.set_unlock_on_exit(false); } /* END if (pjob != NULL) */ - if (preq) - free_br(preq); /* close connection and release request */ - return; } /* END post_stagein() */ @@ -756,14 +727,14 @@ void post_stagein( int svr_stagein( - job **pjob_ptr, /* I */ - struct batch_request **preq, /* I */ - int state, /* I */ - int substate) /* I */ + job **pjob_ptr, /* I */ + batch_request *preq, /* I */ + int state, /* I */ + int substate) /* I */ { job *pjob = *pjob_ptr; - struct batch_request *momreq = 0; + struct batch_request *momreq = NULL; int rc; char jobid[PBS_MAXSVRJOBID + 1]; @@ -772,9 +743,7 @@ int svr_stagein( if (momreq == NULL) { /* no files to stage, go directly to sending job to mom */ - rc = svr_strtjob2(&pjob, *preq); - - *preq = NULL; + rc = svr_strtjob2(&pjob, preq); return(rc); } @@ -807,30 +776,13 @@ int svr_stagein( * take too long to wait. */ - if (*preq != NULL) + if (preq != NULL) { - struct batch_request *request = *preq; - int free_preq = 0; - /* Under the following circumstances, reply_send_svr called eventually - * by reply_ack/reply_send will free preq under the following cirmcustances. - * There are 60+ occurrences of reply_send, so this is the easiest way. - */ - if (((request->rq_type != PBS_BATCH_AsyModifyJob) && - (request->rq_type != PBS_BATCH_AsyrunJob) && - (request->rq_type != PBS_BATCH_AsySignalJob)) || - (request->rq_noreply == TRUE)) - free_preq = 1; - - reply_ack(*preq); - - if (free_preq) - *preq = NULL; + reply_ack(preq); } } - else - { - free_br(momreq); - } + + delete momreq; return(rc); } /* END svr_stagein() */ @@ -987,10 +939,10 @@ int verify_moms_up( int svr_startjob( - job *pjob, /* I job to run (modified) */ - struct batch_request **preq, /* I Run Job batch request (optional) */ - char *FailHost, /* O (optional,minsize=1024) */ - char *EMsg) /* O (optional,minsize=1024) */ + job *pjob, /* I job to run (modified) */ + batch_request *preq, /* I Run Job batch request (optional) */ + char *FailHost, /* O (optional,minsize=1024) */ + char *EMsg) /* O (optional,minsize=1024) */ { int f; @@ -1113,9 +1065,7 @@ int svr_startjob( { /* No stage-in or already done, start job executing */ - rc = svr_strtjob2(&pjob, *preq); - - *preq = NULL; + rc = svr_strtjob2(&pjob, preq); } return(rc); @@ -1317,12 +1267,12 @@ int handle_heterogeneous_job_launch( lock_ji_mutex(cray_clone, __func__, NULL, LOGLEVEL); /* clone the batch requests to avoid double frees */ - external_preq = duplicate_request(preq); - cray_preq = duplicate_request(preq); + external_preq = new batch_request(*preq); + cray_preq = new batch_request(*preq); /* client doesn't need a response from these */ - external_preq->rq_noreply = TRUE; - cray_preq->rq_noreply = TRUE; + external_preq->rq_noreply = true; + cray_preq->rq_noreply = true; if ((rc = send_job_to_mom(&external_clone, external_preq, pjob)) == PBSE_NONE) { @@ -1334,8 +1284,9 @@ int handle_heterogeneous_job_launch( else both_running = TRUE; } - else - free_br(cray_preq); + + delete external_preq; + delete cray_preq; if (cray_clone != NULL) unlock_ji_mutex(cray_clone, __func__, NULL, LOGLEVEL); @@ -1375,8 +1326,8 @@ int handle_heterogeneous_job_launch( int svr_strtjob2( - job **pjob_ptr, /* I */ - struct batch_request *preq) /* I (modified - report status) */ + job **pjob_ptr, /* I */ + batch_request *preq) /* I (modified - report status) */ { job *pjob = *pjob_ptr; @@ -1418,6 +1369,43 @@ int svr_strtjob2( +/* + * get_message_from_mom_err() + * + * @param mom_err (I) - the error code sent back from the mom + * @param msg (O) - the message string to populate + */ + +void get_message_from_mom_err( + + int mom_err, + std::string &msg) + + { + switch (mom_err) + { + case JOB_EXEC_CMDFAIL: + + msg = "Failed to exec the job, please retry."; + break; + + case JOB_EXEC_STDOUTFAIL: + + msg = "Failed to open the output and error files for the job."; + break; + + case JOB_EXEC_RETRY_CGROUP: + + msg = "Failed to create the cgroups for this job."; + break; + + default: + + msg = "Mom rejected the job."; + break; + } + } // get_message_from_mom_err() + void finish_sendmom( @@ -1530,7 +1518,11 @@ void finish_sendmom( if (preq != NULL) { if (mom_err != PBSE_NONE) - req_reject(mom_err, 0, preq, node_name, "connection to mom timed out"); + { + std::string mom_err_msg; + get_message_from_mom_err(mom_err, mom_err_msg); + req_reject(mom_err, 0, preq, node_name, mom_err_msg.c_str()); + } else req_reject(PBSE_MOMREJECT, 0, preq, node_name, "connection to mom timed out"); } @@ -1921,6 +1913,35 @@ int set_job_exec_info( +void set_mode_from_string( + + const char *mode_string, + std::string &mode) + + { + if (strstr(mode_string, "exclusive_thread")) + { + mode = ":exclusive_thread"; + } + else if (strstr(mode_string, "exclusive_process")) + { + mode = ":exclusive_process"; + } + else if (strstr(mode_string, "exclusive")) + { + mode = ":exclusive"; + } + else if (strstr(mode_string, "default")) + { + mode = ":default"; + } + else if (strstr(mode_string, "shared")) + { + mode = ":shared"; + } + } // END set_mode_from_string() + + char *get_correct_spec_string( @@ -1928,18 +1949,18 @@ char *get_correct_spec_string( job *pjob) { - char mode[20]; - char *mode_string; - char *request; - char *correct_spec = NULL; - char *outer_plus; - char *plus; - char *one_req; - int num_gpu_reqs; - char *gpu_req; - int len; - resource *pres = NULL; - char log_buf[LOCAL_LOG_BUF_SIZE]; + std::string mode; + char *mode_string = NULL; + char *request; + char *correct_spec = NULL; + char *outer_plus = NULL; + char *plus; + char *one_req; + int num_gpu_reqs; + char *gpu_req; + int len; + resource *pres = NULL; + char log_buf[LOCAL_LOG_BUF_SIZE]; /* check to see if there is a gpus request. If so moab * sripted the mode request if it existed. We need to @@ -1950,8 +1971,15 @@ char *get_correct_spec_string( { /* Build our host list from what is in the job attrs */ pres = find_resc_entry( - &pjob->ji_wattr[(int)JOB_ATR_resource], + &pjob->ji_wattr[JOB_ATR_resource], find_resc_def(svr_resc_def, "neednodes", svr_resc_size)); + + if (pres == NULL) + { + pres = find_resc_entry( + &pjob->ji_wattr[JOB_ATR_resource], + find_resc_def(svr_resc_def, "gres", svr_resc_size)); + } if (pres != NULL) { @@ -1970,11 +1998,17 @@ char *get_correct_spec_string( if (!(gpu_req = strstr(request, ":gpus="))) { - correct_spec = strdup(given); - return(correct_spec); + if (!(gpu_req = strstr(request, "gpus:"))) + { + correct_spec = strdup(given); + return(correct_spec); + } + else + mode_string = gpu_req + 5; // strlen(gpus:) } + else + mode_string = gpu_req + 6; // strlen(:gpus=) - mode_string = gpu_req + strlen(":gpus="); while (isdigit(*mode_string)) mode_string++; @@ -1992,43 +2026,19 @@ char *get_correct_spec_string( if ((outer_plus = strchr(mode_string, '+')) != NULL) *outer_plus = '\0'; - + /* * The neednodes original value may have non gpu things in it, so we * can not rely on the requested gpu mode being the first item in the * the string after the gpus=x:. */ - - if (strstr(mode_string, "exclusive_thread")) - { - strcpy(mode, ":exclusive_thread"); - } - else if (strstr(mode_string, "exclusive_process")) - { - strcpy(mode, ":exclusive_process"); - } - else if (strstr(mode_string, "exclusive")) - { - strcpy(mode, ":exclusive"); - } - else if (strstr(mode_string, "default")) - { - strcpy(mode, ":default"); - } - else if (strstr(mode_string, "shared")) - { - strcpy(mode, ":shared"); - } - else - { - strcpy(mode, ""); - } + set_mode_from_string(mode_string, mode); if (outer_plus != NULL) *outer_plus = '+'; /* now using the actual length of requested gpu mode */ - len = strlen(given) + 1 + (num_gpu_reqs * strlen(mode)); + len = strlen(given) + 1 + (num_gpu_reqs * mode.size()); if ((correct_spec = (char *)calloc(1, len)) != NULL) { one_req = given; @@ -2042,7 +2052,7 @@ char *get_correct_spec_string( strcat(correct_spec, one_req); if (strstr(one_req, ":gpus") != NULL) - strcat(correct_spec, mode); + strcat(correct_spec, mode.c_str()); if (plus != NULL) { @@ -2053,6 +2063,7 @@ char *get_correct_spec_string( one_req = NULL; } } + if ((LOGLEVEL >= 7) && (correct_spec != NULL) && (correct_spec[0] != '\0')) { sprintf(log_buf, "%s: job gets adjusted gpu node spec of '%s'", diff --git a/src/server/req_shutdown.c b/src/server/req_shutdown.c index 27c8f7e1bd..90afc11e82 100644 --- a/src/server/req_shutdown.c +++ b/src/server/req_shutdown.c @@ -110,13 +110,11 @@ #include "job_func.h" -int issue_signal(job **, const char *, void(*)(batch_request *), void *, char *); - /* Private Fuctions Local to this File */ /* Private Data Items */ -static struct batch_request *pshutdown_request = 0; +static batch_request *pshutdown_request = NULL; /* Global Data Items: */ @@ -252,6 +250,8 @@ void shutdown_ack(void) { reply_ack(pshutdown_request); + delete pshutdown_request; + pshutdown_request = 0; } @@ -290,7 +290,7 @@ void req_shutdown( msg_daemonname, log_buf); - pshutdown_request = preq; /* save for reply from main() when done */ + pshutdown_request = new batch_request(*preq); /* save for reply from main() when done */ svr_shutdown(preq->rq_ind.rq_shutdown); diff --git a/src/server/req_signal.c b/src/server/req_signal.c index 99fa72aab8..ee299e0bbb 100644 --- a/src/server/req_signal.c +++ b/src/server/req_signal.c @@ -136,9 +136,7 @@ int req_signaljob( job *pjob; int rc; char log_buf[LOCAL_LOG_BUF_SIZE]; - batch_request *dup_req = NULL; - /* preq free'd in error cases */ if ((pjob = chk_job_request(preq->rq_ind.rq_signal.rq_jid, preq)) == 0) { return(PBSE_NONE); @@ -202,49 +200,27 @@ int req_signaljob( /* send reply for asynchronous suspend */ if (preq->rq_type == PBS_BATCH_AsySignalJob) { - /* reply_ack will free preq. We need to copy it before we call reply_ack */ - batch_request *new_preq; - - new_preq = duplicate_request(preq, -1); - if (new_preq == NULL) - { - sprintf(log_buf, "failed to duplicate batch request"); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); - return(PBSE_MEM_MALLOC); - } - - get_batch_request_id(new_preq); - - reply_ack(new_preq); - preq->rq_noreply = TRUE; + reply_ack(preq); + preq->rq_noreply = true; } /* pass the request on to MOM */ - if ((dup_req = duplicate_request(preq)) == NULL) + batch_request dup_req(*preq); + + rc = relay_to_mom(&pjob, &dup_req, NULL); + + if (pjob != NULL) + job_mutex.unlock(); + else + job_mutex.set_unlock_on_exit(false); + + if (rc != PBSE_NONE) { - req_reject(PBSE_SYSTEM, 0, preq, NULL, "can not allocate memory"); + req_reject(rc, 0, preq, NULL, NULL); /* unable to get to MOM */ } - /* The dup_req is freed in relay_to_mom (failure) - * or in issue_Drequest (success) */ - else + else { - rc = relay_to_mom(&pjob, dup_req, NULL); - - if (pjob != NULL) - job_mutex.unlock(); - else - job_mutex.set_unlock_on_exit(false); - - if (rc != PBSE_NONE) - { - free_br(dup_req); - req_reject(rc, 0, preq, NULL, NULL); /* unable to get to MOM */ - } - else - { - post_signal_req(dup_req); - free_br(preq); - } + post_signal_req(&dup_req); } /* If successful we ack after mom replies to us, we pick up in post_signal_req() */ @@ -252,64 +228,69 @@ int req_signaljob( return(PBSE_NONE); } /* END req_signaljob() */ + + /* * issue_signal - send an internally generated signal to a running job + * + * @param pjob_ptr - a pointer to a potiner to the job + * @param signame - the name of the signal we're sending + * @param func - optional function pointer to call after sending the signal + * @param extra - optional extra info for the batch request + * @param extend - optional extend info for the batch request (basically more extra info) */ int issue_signal( job **pjob_ptr, - const char *signame, /* name of the signal to send */ + const char *signame, void (*func)(struct batch_request *), - void *extra, /* extra parameter to be stored in sig request */ - char *extend) /* Parameter to put in extended part of request */ + void *extra, + char *extend) { - int rc; - job *pjob = *pjob_ptr; - struct batch_request *newreq; - char jobid[PBS_MAXSVRJOBID + 1]; - - /* build up a Signal Job batch request */ - - if ((newreq = alloc_br(PBS_BATCH_SignalJob)) == NULL) - { - /* FAILURE */ - - return(PBSE_SYSTEM); - } + int rc; + job *pjob = *pjob_ptr; + batch_request newreq(PBS_BATCH_SignalJob); + char jobid[PBS_MAXSVRJOBID + 1]; - newreq->rq_extra = extra; - newreq->rq_extend = extend; + newreq.rq_extra = extra; + newreq.rq_extend = extend; if (extend != NULL) { - newreq->rq_extsz = strlen(extend); + newreq.rq_extsz = strlen(extend); } strcpy(jobid, pjob->ji_qs.ji_jobid); - strcpy(newreq->rq_ind.rq_signal.rq_jid, pjob->ji_qs.ji_jobid); + strcpy(newreq.rq_ind.rq_signal.rq_jid, pjob->ji_qs.ji_jobid); - snprintf(newreq->rq_ind.rq_signal.rq_signame, sizeof(newreq->rq_ind.rq_signal.rq_signame), "%s", signame); + snprintf(newreq.rq_ind.rq_signal.rq_signame, sizeof(newreq.rq_ind.rq_signal.rq_signame), + "%s", signame); /* The newreq is freed in relay_to_mom (failure) * or in issue_Drequest (success) */ - rc = relay_to_mom(&pjob, newreq, NULL); + rc = relay_to_mom(&pjob, &newreq, NULL); if ((rc == PBSE_NONE) && (pjob != NULL)) { - strcpy(jobid, pjob->ji_qs.ji_jobid); - unlock_ji_mutex(pjob, __func__, NULL, LOGLEVEL); - func(newreq); + if (func != NULL) + { + strcpy(jobid, pjob->ji_qs.ji_jobid); + + unlock_ji_mutex(pjob, __func__, NULL, LOGLEVEL); + + func(&newreq); - *pjob_ptr = svr_find_job((char *)jobid, TRUE); + *pjob_ptr = svr_find_job(jobid, TRUE); + } } else if ((extend != NULL) && (!strcmp(extend, RERUNFORCE))) { if (pjob == NULL) { - *pjob_ptr = svr_find_job((char *)jobid, TRUE); + *pjob_ptr = svr_find_job(jobid, TRUE); pjob = *pjob_ptr; } /* The job state is normally set when the obit arrives. But since the @@ -336,6 +317,7 @@ int issue_signal( pjob->ji_momhandle = -1; unlock_ji_mutex(pjob, __func__, "8", LOGLEVEL); + *pjob_ptr = NULL; return(PBSE_SYSTEM); } @@ -350,17 +332,20 @@ int issue_signal( svr_setjobstate(pjob, JOB_STATE_QUEUED, JOB_SUBSTATE_QUEUED, FALSE); unlock_ji_mutex(pjob, __func__, NULL, LOGLEVEL); - func(newreq); + + if (func != NULL) + func(&newreq); rc = PBSE_NONE; + + // We need to re-acquire the lock or set the pointer to NULL + *pjob_ptr = svr_find_job(jobid, TRUE); } else rc = PBSE_JOBNOTFOUND; } else { - free_br(newreq); - if (pjob == NULL) *pjob_ptr = NULL; } diff --git a/src/server/req_stat.c b/src/server/req_stat.c index e3282dab0b..21a608aeee 100644 --- a/src/server/req_stat.c +++ b/src/server/req_stat.c @@ -731,22 +731,36 @@ int stat_to_mom( struct stat_cntl *cntl) /* M */ { - struct batch_request *newrq; - int rc = PBSE_NONE; - unsigned long addr; - char log_buf[LOCAL_LOG_BUF_SIZE+1]; - struct pbsnode *node; - int handle = -1; - unsigned long job_momaddr = -1; - unsigned short job_momport = -1; - char *job_momname = NULL; - job *pjob = NULL; + batch_request newrq(PBS_BATCH_StatusJob); + int rc = PBSE_NONE; + unsigned long addr; + char log_buf[LOCAL_LOG_BUF_SIZE+1]; + pbsnode *node; + int handle = -1; + unsigned long job_momaddr = -1; + unsigned short job_momport = -1; + char *job_momname = NULL; + job *pjob = NULL; if ((pjob = svr_find_job(job_id, FALSE)) == NULL) return(PBSE_JOBNOTFOUND); mutex_mgr job_mutex(pjob->ji_mutex, true); + // don't continue if job no longer running + if (pjob->ji_qs.ji_state != JOB_STATE_RUNNING) + { + if (LOGLEVEL >= 6) + { + snprintf(log_buf, sizeof(log_buf), + "stat_to_mom(): job is no longer in running state. Not contacting mom."); + + log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_JOB, job_id, log_buf); + } + + return(PBSE_BADSTATE); + } + if ((pjob->ji_qs.ji_un.ji_exect.ji_momaddr == 0) || (!pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str)) { @@ -765,18 +779,12 @@ int stat_to_mom( if (job_momname == NULL) return PBSE_MEM_MALLOC; - if ((newrq = alloc_br(PBS_BATCH_StatusJob)) == NULL) - { - free(job_momname); - return PBSE_MEM_MALLOC; - } - if (cntl->sc_type == 1) - snprintf(newrq->rq_ind.rq_status.rq_id, sizeof(newrq->rq_ind.rq_status.rq_id), "%s", job_id); + snprintf(newrq.rq_ind.rq_status.rq_id, sizeof(newrq.rq_ind.rq_status.rq_id), "%s", job_id); else - newrq->rq_ind.rq_status.rq_id[0] = '\0'; /* get stat of all */ + newrq.rq_ind.rq_status.rq_id[0] = '\0'; /* get stat of all */ - CLEAR_HEAD(newrq->rq_ind.rq_status.rq_attr); + CLEAR_HEAD(newrq.rq_ind.rq_status.rq_attr); /* if MOM is down just return stale information */ addr = job_momaddr; @@ -785,7 +793,10 @@ int stat_to_mom( free(job_momname); if (node == NULL) + { return PBSE_UNKNODE; + } + if ((node->nd_state & INUSE_NOT_READY)||(node->nd_power_state != POWER_STATE_RUNNING)) { if (LOGLEVEL >= 6) @@ -798,7 +809,6 @@ int stat_to_mom( } node->unlock_node(__func__, "no rely mom", LOGLEVEL); - free_br(newrq); return PBSE_NORELYMOM; } @@ -809,9 +819,9 @@ int stat_to_mom( if (handle >= 0) { - if ((rc = issue_Drequest(handle, newrq, true)) == PBSE_NONE) + if ((rc = issue_Drequest(handle, &newrq, true)) == PBSE_NONE) { - stat_update(newrq, cntl); + stat_update(&newrq, cntl); } } else @@ -820,8 +830,6 @@ int stat_to_mom( if (rc == PBSE_SYSTEM) rc = PBSE_MEM_MALLOC; - free_br(newrq); - return(rc); } /* END stat_to_mom() */ @@ -835,8 +843,8 @@ int stat_to_mom( void stat_update( - struct batch_request *preq, - struct stat_cntl *cntl) + batch_request *preq, + struct stat_cntl *cntl) { job *pjob; @@ -916,18 +924,33 @@ void stat_update( directory is cleared, set its state to queued so job_abt doesn't think it is still running */ mutex_mgr job_mutex(pjob->ji_mutex, true); - - snprintf(log_buf, sizeof(log_buf), - "mother superior no longer recognizes %s as a valid job, aborting. Last reported time was %ld", - preq->rq_ind.rq_status.rq_id, pjob->ji_last_reported_time); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); - - svr_setjobstate(pjob, JOB_STATE_QUEUED, JOB_SUBSTATE_ABORT, FALSE); - rel_resc(pjob); - job_mutex.set_unlock_on_exit(false); - job_abt(&pjob, "Job does not exist on node"); - /* TODO, if the job is rerunnable we should set its state back to queued */ + if (pjob->ji_qs.ji_state != JOB_STATE_RUNNING) + { + // don't abort if job no longer in running state + if (LOGLEVEL >= 6) + { + snprintf(log_buf, sizeof(log_buf), + "stat_update(): job %s is no longer in running state. Ignoring unknown job id reply from mom.", + preq->rq_ind.rq_status.rq_id); + + log_event(PBSEVENT_SYSTEM, PBS_EVENTCLASS_JOB, __func__, log_buf); + } + } + else + { + snprintf(log_buf, sizeof(log_buf), + "mother superior no longer recognizes %s as a valid job, aborting. Last reported time was %ld", + preq->rq_ind.rq_status.rq_id, pjob->ji_last_reported_time); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); + + svr_setjobstate(pjob, JOB_STATE_QUEUED, JOB_SUBSTATE_ABORT, FALSE); + rel_resc(pjob); + job_mutex.set_unlock_on_exit(false); + job_abt(&pjob, "Job does not exist on node"); + + /* TODO, if the job is rerunnable we should set its state back to queued */ + } } } else diff --git a/src/server/req_track.c b/src/server/req_track.c index 8614b4b696..187b91f0c2 100644 --- a/src/server/req_track.c +++ b/src/server/req_track.c @@ -332,10 +332,9 @@ void issue_track( job *pjob) { - struct batch_request *preq; - char *pc; - char *sname; - char log_buf[LOCAL_LOG_BUF_SIZE]; + char *pc; + char *sname; + char log_buf[LOCAL_LOG_BUF_SIZE]; if ((pc = strchr(pjob->ji_qs.ji_jobid, '.')) == NULL) { @@ -359,17 +358,14 @@ void issue_track( return; } - preq = alloc_br(PBS_BATCH_TrackJob); + batch_request preq(PBS_BATCH_TrackJob); - if (preq == NULL) - return; - - preq->rq_ind.rq_track.rq_hopcount = pjob->ji_wattr[JOB_ATR_hopcount].at_val.at_long; + preq.rq_ind.rq_track.rq_hopcount = pjob->ji_wattr[JOB_ATR_hopcount].at_val.at_long; - strcpy(preq->rq_ind.rq_track.rq_jid, pjob->ji_qs.ji_jobid); - strcpy(preq->rq_ind.rq_track.rq_location, server_name); + strcpy(preq.rq_ind.rq_track.rq_jid, pjob->ji_qs.ji_jobid); + strcpy(preq.rq_ind.rq_track.rq_location, server_name); - preq->rq_ind.rq_track.rq_state[0] = pjob->ji_wattr[JOB_ATR_state].at_val.at_char; + preq.rq_ind.rq_track.rq_state[0] = pjob->ji_wattr[JOB_ATR_state].at_val.at_char; pc = pjob->ji_qs.ji_jobid; @@ -377,6 +373,4 @@ void issue_track( pc++; issue_to_svr(++pc, &preq, NULL); - if (preq != NULL) - free_br(preq); } // END issue_track() diff --git a/src/server/resc_def_all.c b/src/server/resc_def_all.c index 1621947c6d..658a327eb1 100644 --- a/src/server/resc_def_all.c +++ b/src/server/resc_def_all.c @@ -123,7 +123,7 @@ int set_tokens_nodect(struct pbs_attribute *attr, struct pbs_attribute *new_attr int set_mppnodect(resource *, pbs_attribute *, int actmode); int decode_procct(pbs_attribute *, const char *, const char *, const char *, int); int encode_procct(pbs_attribute *, tlist_head *phead, const char *atname, const char *rsname, int mode, int perm); - +int verify_as_time(resource *r, pbs_attribute *pattr, int mode); resource_def *svr_resc_def; @@ -733,7 +733,7 @@ resource_def svr_resc_def_const[] = { "loglevel", decode_str, encode_str, set_str, comp_str, free_str, NULL_FUNC, READ_WRITE, ATR_TYPE_STR }, { "minprocspeed", decode_str, encode_str, set_str, comp_str, free_str, NULL_FUNC, READ_WRITE, ATR_TYPE_STR }, { "minpreempttime", decode_str, encode_str, set_str, comp_str, free_str, NULL_FUNC, READ_WRITE, ATR_TYPE_STR }, - { "minwclimit", decode_str, encode_str, set_str, comp_str, free_str, NULL_FUNC, READ_WRITE, ATR_TYPE_STR }, + { "minwclimit", decode_str, encode_str, set_str, comp_str, free_str, verify_as_time, READ_WRITE, ATR_TYPE_STR }, { "naccesspolicy", decode_str, encode_str, set_str, comp_str, free_str, NULL_FUNC, READ_WRITE, ATR_TYPE_STR }, { "nallocpolicy", decode_str, encode_str, set_str, comp_str, free_str, NULL_FUNC, READ_WRITE, ATR_TYPE_STR }, { "nodeset", decode_str, encode_str, set_str, comp_str, free_str, NULL_FUNC, READ_WRITE, ATR_TYPE_STR }, @@ -1451,3 +1451,35 @@ int encode_procct( } + +/* + * verify_as_time() + * + * Verifies that the string set for this resource is a valid time string and can be + * converted to seconds. + * + * @param r - the resource in question (ignored) + * @param pattr - the attribute value for the resource + * @param mode - the mode of the action taken for this resource (ignored) + * @return PBSE_NONE on success, or -1 for failure. + */ + +int verify_as_time( + + resource *r, + pbs_attribute *pattr, + int mode) + + { + if (r->rs_value.at_val.at_str != NULL) + { + if (time_str_to_seconds(r->rs_value.at_val.at_str) < 0) + { + return(-1); + } + } + + return(PBSE_NONE); + } // END verify_as_time() + + diff --git a/src/server/run_sched.c b/src/server/run_sched.c index 8873fc1450..2cdecdf485 100644 --- a/src/server/run_sched.c +++ b/src/server/run_sched.c @@ -360,16 +360,18 @@ static int contact_listener( if (sock < 0) { /* FAILURE */ - - sprintf(tmpLine, "%s %d - port %d %s", - msg_listnr_nocall, - l_idx, - listener_conns[l_idx].port, - EMsg); + if (LOGLEVEL >= 6) + { + sprintf(tmpLine, "%s %d - port %d %s", + msg_listnr_nocall, + l_idx, + listener_conns[l_idx].port, + EMsg); + log_err(errno, __func__, tmpLine); + } /* we lost contact with the scheduler. reset*/ listener_conns[l_idx].first_time = 1; - log_err(errno, __func__, tmpLine); return(-1); } diff --git a/src/server/svr_attr_def.c b/src/server/svr_attr_def.c index e42cf65cf6..afba56807d 100644 --- a/src/server/svr_attr_def.c +++ b/src/server/svr_attr_def.c @@ -1651,4 +1651,17 @@ attribute_def svr_attr_def[] = PARENT_TYPE_SERVER }, + /* SRV_ATR_SendmailPath */ + {(char *)ATTR_sendmail_path, // "sendmail_path" + decode_str, + encode_str, + set_str, + comp_str, + free_str, + NULL_FUNC, + MGR_ONLY_SET, + ATR_TYPE_STR, + PARENT_TYPE_SERVER + }, + }; diff --git a/src/server/svr_format_job.c b/src/server/svr_format_job.c index 075cfff664..b78968d9b0 100644 --- a/src/server/svr_format_job.c +++ b/src/server/svr_format_job.c @@ -30,6 +30,7 @@ extern char *msg_job_stageinfail; extern char *msg_job_copychkptfail; extern char *msg_job_otherfail; +int resource_index_to_string(std::string &output, std::vector &resources, size_t index); void svr_format_job( @@ -143,7 +144,7 @@ void svr_format_job( } else if (*p == '%') /* format statement */ { - switch(p[1]) + switch (p[1]) { case 'd': /* details */ @@ -210,12 +211,72 @@ void svr_format_job( break; + case 'o': // owner + + fprintf(fh, "%s", mi->owner.c_str()); + p += 2; + + break; + + case 'q': // queue + + fprintf(fh, "%s", mi->queue_name.c_str()); + + p += 2; + + break; + + case 'R': // Resources requested + + { + std::string resource_str; + + if (mi->resources_requested.size() > 0) + fprintf(fh, "Resources Requested Summary\n"); + + for (size_t i = 0; i < mi->resources_requested.size(); i++) + { + resource_index_to_string(resource_str, mi->resources_requested, i); + fprintf(fh, "\t%s\n", resource_str.c_str()); + } + p += 2; + + break; + } + case 'r': /* reason */ if (reason != NULL) { fprintf(fh, "%s", reason); } + + p += 2; + + break; + + case 'u': // resources used summary + + { + std::string resource_str; + + if (mi->resources_used.size() > 0) + fprintf(fh, "Resources Used Summary\n"); + + for (size_t i = 0; i < mi->resources_used.size(); i++) + { + resource_index_to_string(resource_str, mi->resources_used, i); + fprintf(fh, "\t%s\n", resource_str.c_str()); + } + + p += 2; + + break; + } + + case 'w': // working directory + + fprintf(fh, "%s", mi->working_directory.c_str()); p += 2; break; diff --git a/src/server/svr_jobfunc.c b/src/server/svr_jobfunc.c index 52a9e95c94..cfdd98f609 100644 --- a/src/server/svr_jobfunc.c +++ b/src/server/svr_jobfunc.c @@ -320,7 +320,7 @@ int insert_into_alljobs_by_rank( while ((pjcur = iter->get_next_item()) != NULL) { - mutex_mgr pjcur_mgr(pjcur->ji_mutex, true); + mutex_mgr pjcur_mgr(pjcur->ji_mutex, false); if (job_qrank > pjcur->ji_wattr[JOB_ATR_qrank].at_val.at_long) { pjcur_mgr.set_unlock_on_exit(false); @@ -483,8 +483,10 @@ int svr_enquejob( } } - - if ((pque->qu_attr[QA_ATR_MaxUserJobs].at_flags & ATR_VFLAG_SET)) + // Do not perform this check for array subjobs; they've already been counted. + if ((pque->qu_attr[QA_ATR_MaxUserJobs].at_flags & ATR_VFLAG_SET) && + ((pjob->ji_arraystructid[0] == '\0') || + (pjob->ji_is_array_template))) { std::string uname(pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str); @@ -552,7 +554,7 @@ int svr_enquejob( if (!pjob->ji_is_array_template) { rc = insert_into_alljobs_by_rank(pque->qu_jobs, pjob, job_id); - + if ((rc == ALREADY_IN_LIST) || (rc == PBSE_JOBNOTFOUND)) { @@ -567,18 +569,18 @@ int svr_enquejob( /* update counts: queue and queue by state */ pque->qu_numjobs++; pque->qu_njstate[pjob->ji_qs.ji_state]++; + } - /* increment this user's job count for this queue */ - if (LOGLEVEL >= 6) - { - snprintf(log_buf, sizeof(log_buf), "jobs queued job id %s for %s", pjob->ji_qs.ji_jobid, pque->qu_qs.qu_name); - log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); - } - - increment_queued_jobs(pque->qu_uih, pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, pjob); - increment_queued_jobs(&users, pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, pjob); + /* increment this user's job count for this queue */ + if (LOGLEVEL >= 6) + { + snprintf(log_buf, sizeof(log_buf), "jobs queued job id %s for %s", pjob->ji_qs.ji_jobid, pque->qu_qs.qu_name); + log_event(PBSEVENT_JOB, PBS_EVENTCLASS_JOB, __func__, log_buf); } + increment_queued_jobs(pque->qu_uih, pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, pjob); + increment_queued_jobs(&users, pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, pjob); + if ((pjob->ji_is_array_template) || (pjob->ji_arraystructid[0] == '\0')) { @@ -653,25 +655,6 @@ int svr_enquejob( /* do anything needed doing regarding job dependencies */ que_mgr.unlock(); - if ((pjob->ji_qs.ji_state != JOB_STATE_COMPLETE) && - (pjob->ji_qs.ji_substate != JOB_SUBSTATE_COMPLETE) && - (pjob->ji_wattr[JOB_ATR_depend].at_flags & ATR_VFLAG_SET)) - { - try - { - rc = depend_on_que(&pjob->ji_wattr[JOB_ATR_depend], pjob, ATR_ACTION_NOOP); - } - catch (int pbs_errcode) - { - rc = pbs_errcode; - } - - if (rc == PBSE_JOBNOTFOUND) - return(rc); - else if (rc != PBSE_NONE) - rc = PBSE_BADDEPEND; - } - /* set eligible time */ if (((pjob->ji_wattr[JOB_ATR_etime].at_flags & ATR_VFLAG_SET) == 0) && (pjob->ji_qs.ji_state == JOB_STATE_QUEUED)) @@ -693,6 +676,37 @@ int svr_enquejob( /* start attempts to route job */ pjob->ji_qs.ji_un_type = JOB_UNION_TYPE_ROUTE; pjob->ji_qs.ji_un.ji_routet.ji_quetime = time_now; + + que_mgr.unlock(); + } + + if ((pjob->ji_qs.ji_state != JOB_STATE_COMPLETE) && + (pjob->ji_qs.ji_substate != JOB_SUBSTATE_COMPLETE) && + (pjob->ji_wattr[JOB_ATR_depend].at_flags & ATR_VFLAG_SET)) + { + try + { + rc = depend_on_que(&pjob->ji_wattr[JOB_ATR_depend], pjob, ATR_ACTION_NOOP); + } + catch (int pbs_errcode) + { + rc = pbs_errcode; + } + + if (rc == PBSE_JOBNOTFOUND) + return(rc); + else if (rc != PBSE_NONE) + { + // decrement queued job counts + + pque->qu_numjobs--; + pque->qu_njstate[pjob->ji_qs.ji_state]--; + + decrement_queued_jobs(pque->qu_uih, pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, pjob); + decrement_queued_jobs(&users, pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, pjob); + + rc = PBSE_BADDEPEND; + } } return(rc); @@ -811,15 +825,21 @@ int svr_dequejob( } /* the only reason to care about the error is if the job is gone */ - int rc2; - if ((rc2 = remove_job(pque->qu_jobs_array_sum, pjob)) == PBSE_JOBNOTFOUND) - return(rc2); - - if (rc2 == THING_NOT_FOUND && (LOGLEVEL >= 8)) + // Do not check array subjobs; they aren't in the summary + if ((pjob->ji_arraystructid[0] == '\0') || + (pjob->ji_is_array_template)) { - snprintf(log_buf,sizeof(log_buf), - "Could not remove job %s from qu_jobs_array_sum\n", jobid.c_str()); - log_ext(-1, __func__, log_buf, LOG_WARNING); + int rc2; + if ((rc2 = remove_job(pque->qu_jobs_array_sum, pjob)) == PBSE_JOBNOTFOUND) + return(rc2); + + if ((rc2 == THING_NOT_FOUND) && + (LOGLEVEL >= 8)) + { + snprintf(log_buf,sizeof(log_buf), + "Could not remove job %s from qu_jobs_array_sum\n", jobid.c_str()); + log_ext(-1, __func__, log_buf, LOG_WARNING); + } } pjob->ji_qhdr = NULL; @@ -849,9 +869,9 @@ int svr_dequejob( } /* the only error is if the job isn't present */ - if ((rc = remove_job(&alljobs, pjob)) == PBSE_NONE) + if (!pjob->ji_is_array_template) { - if (!pjob->ji_is_array_template) + if ((rc = remove_job(&alljobs, pjob)) == PBSE_NONE) { lock_sv_qs_mutex(server.sv_qs_mutex, __func__); @@ -872,18 +892,18 @@ int svr_dequejob( pthread_mutex_unlock(server.sv_jobstates_mutex); } - } - else if (rc == PBSE_JOBNOTFOUND) - { - /* calling functions know this return code means the job is gone */ - return(rc); - } - - if (rc == THING_NOT_FOUND && (LOGLEVEL >= 8)) - { - snprintf(log_buf,sizeof(log_buf), - "Could not remove job %s from alljobs\n", pjob->ji_qs.ji_jobid); - log_ext(-1, __func__, log_buf, LOG_WARNING); + else if (rc == PBSE_JOBNOTFOUND) + { + /* calling functions know this return code means the job is gone */ + return(rc); + } + if ((rc == THING_NOT_FOUND) && + (LOGLEVEL >= 8)) + { + snprintf(log_buf,sizeof(log_buf), + "Could not remove job %s from alljobs\n", pjob->ji_qs.ji_jobid); + log_ext(-1, __func__, log_buf, LOG_WARNING); + } } #ifndef NDEBUG @@ -1165,7 +1185,6 @@ int svr_setjobstate( changed = true; /* if the state is changing, also update the state counts */ - oldstate = pjob->ji_qs.ji_state; if (oldstate != newstate) @@ -1186,12 +1205,17 @@ int svr_setjobstate( if (has_queue_mutex == FALSE) { - pque = get_jobs_queue(&pjob); - if (pque == NULL) + // On start-up, we call this function even though we don't know the job's queue yet. In + // that case, we don't want this function to fail. + if (pjob->ji_qhdr != NULL) { - sprintf(log_buf, "queue not found for jobid %s", jid.c_str()); - log_err(PBSE_UNKQUE, __func__, log_buf); - return(PBSE_UNKQUE); + pque = get_jobs_queue(&pjob); + if (pque == NULL) + { + sprintf(log_buf, "queue not found for jobid %s", jid.c_str()); + log_err(PBSE_UNKQUE, __func__, log_buf); + return(PBSE_UNKQUE); + } } } else @@ -1277,7 +1301,8 @@ int svr_setjobstate( } else { - if (has_queue_mutex == FALSE) + if ((pque != NULL) && + (has_queue_mutex == FALSE)) unlock_queue(pque, __func__, NULL, LOGLEVEL); } } @@ -1762,14 +1787,6 @@ int chk_svr_resc_limit( } } } - - /* Added 6/14/2010 Ken Nielson for ability to parse the procs resource */ - else if ((r.rs_defin == procresc) && - (qtype == QTYPE_Execution)) - { - proc_ct = r.rs_value.at_val.at_long; - } - #ifdef NERSCDEV else if (r.rs_defin == mppwidthresc) { @@ -1796,6 +1813,11 @@ int chk_svr_resc_limit( (r.rs_defin != needresc)) { /* don't check neednodes */ + if ((r.rs_defin == procresc) && + (qtype == QTYPE_Execution)) + { + proc_ct = r.rs_value.at_val.at_long; + } rc = r.rs_defin->rs_comp(&cmpwith->rs_value, &r.rs_value); @@ -1991,8 +2013,8 @@ int chk_svr_resc_limit( static int count_queued_jobs( - pbs_queue *pque, /* I */ - const char *user) /* I */ + pbs_queue *pque, /* I */ + const char *user) /* I */ { int num_jobs = 0; @@ -2264,9 +2286,7 @@ static int check_queue_group_ACL( { /* check group acl against all accessible groups */ - struct group *grp; int i = 0; - int j = 0; char uname[PBS_MAXUSER + 1]; @@ -2276,26 +2296,15 @@ static int check_queue_group_ACL( pas = pque->qu_attr[QA_ATR_AclGroup].at_val.at_arst; + rc = 0; for (i = 0; pas != NULL && i < pas->as_usedptr;i++) { - char *buf = NULL; - if ((grp = getgrnam_ext(&buf, pas->as_string[i])) == NULL) - continue; - - for (j = 0;grp->gr_mem[j] != NULL;j++) + if (is_group_member(uname, pas->as_string[i]) == true) { - if (!strcmp(grp->gr_mem[j], uname)) - { rc = 1; - break; - } } - free_grname(grp, buf); - - if (rc == 1) - break; } } /* END if (rc == 0) && slpygrp && ...) */ @@ -2475,9 +2484,10 @@ int check_for_complete_req_and_limits(struct pbs_queue *const pque, job *pjob) static int check_queue_job_limit( - struct job *const pjob, - struct pbs_queue *const pque, - char *const EMsg) + struct job *const pjob, + struct pbs_queue *const pque, + char *const EMsg) + { int return_code = PBSE_NONE; /* Optimistic assumption */ int array_jobs = 0; @@ -3137,7 +3147,6 @@ int job_set_wait( - /* * default_std - make the default name for standard output or error * "job_name".[e|o]job_sequence_number @@ -3191,8 +3200,6 @@ static void default_std( - - /* * prefix_std_file - build the fully prefixed default file name for the job's * standard output or error: @@ -3271,7 +3278,6 @@ const char *prefix_std_file( - /* * add_std_filename - add the default file name for the job's * standard output or error: @@ -3344,17 +3350,69 @@ void get_jobowner( *(to + i) = '\0'; return; - } + } // END get_jobowner() + + + +/* + * is_execution_slot_resource() + * + * @param r - the resource we're checking + * @return true if this resource is an execution slot related request + */ + +inline bool is_execution_slot_resource( + + const resource &r) + + { + return((!strcmp(r.rs_defin->rs_name, "nodes")) || + (!strcmp(r.rs_defin->rs_name, "procs")) || + (!strcmp(r.rs_defin->rs_name, "size")) || + (!strcmp(r.rs_defin->rs_name, "ncpus"))); + } // is_execution_slot_resource() + + + +/* + * contains_execution_slot_request() + * + * @param jb - the resources requested by the job + */ + +bool contains_execution_slot_request( + + pbs_attribute *jb) + + { + bool execution_slot_request = false; + + std::vector *job_resc = (std::vector *)jb->at_val.at_ptr; + if (job_resc != NULL) + { + for (size_t i = 0; i < job_resc->size(); i++) + { + resource &r = job_resc->at(i); + if (is_execution_slot_resource(r)) + { + execution_slot_request = true; + break; + } + } + } + + return(execution_slot_request); + } // END contains_execution_slot_request() /** * @see set_resc_deflt() - parent * - * @param jb - * @param *dflt + * @param jb - the job's resources + * @param dflt - the specified default resources */ void set_deflt_resc( @@ -3376,10 +3434,16 @@ void set_deflt_resc( if (resc_deflt != NULL) { + bool es_spec = contains_execution_slot_request(jb); + for (size_t i = 0; i < resc_deflt->size(); i++) { resource &r = resc_deflt->at(i); + if ((es_spec == true) && + (is_execution_slot_resource(r))) + continue; + prescjb = find_resc_entry(jb, r.rs_defin); if ((prescjb == NULL) || diff --git a/src/server/svr_mail.c b/src/server/svr_mail.c index 7d297a7183..a057f2fa87 100644 --- a/src/server/svr_mail.c +++ b/src/server/svr_mail.c @@ -226,6 +226,7 @@ int get_sendmail_args( int rc = PBSE_NONE; int numargs = 0; const char *mailfrom = NULL; + const char *sendmail_path = NULL; char *mailptr; /* Who is mail from, if SRV_ATR_mailfrom not set use default */ @@ -247,7 +248,17 @@ int get_sendmail_args( } } - sendmail_args[numargs++] = (char *)SENDMAIL_CMD; + // see if sendmail path server attribute set + get_svr_attr_str(SRV_ATR_SendmailPath, (char **)&sendmail_path); + if (sendmail_path != NULL) + { + sendmail_args[numargs++] = (char *)sendmail_path; + } + else + { + sendmail_args[numargs++] = (char *)SENDMAIL_CMD; + } + sendmail_args[numargs++] = (char *)"-f"; sendmail_args[numargs++] = (char *)mailfrom; @@ -438,36 +449,6 @@ void *send_the_mail( return(NULL); } /* END send_the_mail() */ - - - -void set_output_files( - - job *pjob, - mail_info *mi) - - { - if (pjob->ji_wattr[JOB_ATR_join].at_flags & ATR_VFLAG_SET) - { - char *join_val = pjob->ji_wattr[JOB_ATR_join].at_val.at_str; - if (!strcmp(join_val, "oe")) - { - mi->errFile = pjob->ji_wattr[JOB_ATR_outpath].at_val.at_str; - mi->outFile = pjob->ji_wattr[JOB_ATR_outpath].at_val.at_str; - } - else if (!strcmp(join_val, "eo")) - { - mi->errFile = pjob->ji_wattr[JOB_ATR_errpath].at_val.at_str; - mi->outFile = pjob->ji_wattr[JOB_ATR_errpath].at_val.at_str; - } - } - - if (mi->outFile.size() == 0) - mi->outFile = pjob->ji_wattr[JOB_ATR_outpath].at_val.at_str; - - if (mi->errFile.size() == 0) - mi->errFile = pjob->ji_wattr[JOB_ATR_errpath].at_val.at_str; - } // END set_output_files() @@ -590,7 +571,7 @@ void svr_mailowner( char mailto[1024]; char *domain = NULL; int i; - mail_info mi; + mail_info mi(pjob); bool no_force = false; struct array_strings *pas; @@ -627,23 +608,6 @@ void svr_mailowner( return; } - if (LOGLEVEL >= 3) - { - char tmpBuf[LOG_BUF_SIZE]; - - snprintf(tmpBuf, LOG_BUF_SIZE, "preparing to send '%c' mail for job %s to %s (%.64s)\n", - (char)mailpoint, - pjob->ji_qs.ji_jobid, - pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, - (text != NULL) ? text : "---"); - - log_event( - PBSEVENT_ERROR | PBSEVENT_ADMIN | PBSEVENT_JOB, - PBS_EVENTCLASS_JOB, - pjob->ji_qs.ji_jobid, - tmpBuf); - } - /* * if force is true, force the mail out regardless of mailpoint * unless server no_mail_force attribute is set to true @@ -710,6 +674,23 @@ void svr_mailowner( } } + if (LOGLEVEL >= 3) + { + char tmpBuf[LOG_BUF_SIZE]; + + snprintf(tmpBuf, LOG_BUF_SIZE, "preparing to send '%c' mail for job %s to %s (%.64s)\n", + (char)mailpoint, + pjob->ji_qs.ji_jobid, + pjob->ji_wattr[JOB_ATR_job_owner].at_val.at_str, + (text != NULL) ? text : "---"); + + log_event( + PBSEVENT_ERROR | PBSEVENT_ADMIN | PBSEVENT_JOB, + PBS_EVENTCLASS_JOB, + pjob->ji_qs.ji_jobid, + tmpBuf); + } + /* Who does the mail go to? If mail-list, them; else owner */ mailto[0] = '\0'; @@ -784,17 +765,6 @@ void svr_mailowner( mi.mailto = mailto; mi.mail_point = mailpoint; - if (pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str != NULL) - mi.exec_host = pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str; - - mi.jobid = pjob->ji_qs.ji_jobid; - - if (pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str != NULL) - mi.jobname = pjob->ji_wattr[JOB_ATR_jobname].at_val.at_str; - - if (mailpoint == (int) MAIL_END) - set_output_files(pjob, &mi); - if (text) { mi.text = text; diff --git a/src/server/svr_movejob.c b/src/server/svr_movejob.c index 447d17e793..95a2f79d35 100644 --- a/src/server/svr_movejob.c +++ b/src/server/svr_movejob.c @@ -371,7 +371,8 @@ int local_move( snprintf(pjob->ji_qs.ji_queue, sizeof(pjob->ji_qs.ji_queue), "%s", destination); - pjob->ji_wattr[JOB_ATR_qrank].at_val.at_long = ++queue_rank; + if (!(pjob->ji_wattr[JOB_ATR_qrank].at_flags & ATR_VFLAG_SET)) + pjob->ji_wattr[JOB_ATR_qrank].at_val.at_long = ++queue_rank; if ((*my_err = svr_enquejob(pjob, FALSE, NULL, reservation, false)) == PBSE_JOB_RECYCLED) return(-1); @@ -758,7 +759,8 @@ int queue_job_on_mom( if ((pc = PBSD_queuejob(con, my_err, (const char *)job_id, (const char *)job_destin, pqjatr, NULL)) == NULL) { - if (*my_err == PBSE_EXPIRED) + if ((*my_err == PBSE_EXPIRED) || + (*my_err == PBSE_TIMEOUT)) { /* queue job timeout based on pbs_tcp_timeout */ timeout = true; @@ -939,6 +941,11 @@ void log_commit_error( mom_err, pbse_to_txt(mom_err), (err_text != NULL) ? err_text : "N/A"); errno2 = mom_err; } + else if (mom_err < 0) + { + // Mom failed with a JOB_EXEC_* code + errno2 = mom_err; + } else { sprintf(log_buf, "send_job commit failed, rc=%d (%s)", @@ -1117,6 +1124,11 @@ int send_job_over_network( +/* + * send_job_over_network_with_retries() + * + */ + int send_job_over_network_with_retries( char *job_id, @@ -1214,7 +1226,8 @@ int send_job_over_network_with_retries( my_err, mom_err); - if (rc == LOCUTION_SUCCESS) + if ((rc == LOCUTION_SUCCESS) || + (timeout == true)) break; } /* END for (NumRetries) */ @@ -1382,7 +1395,7 @@ int send_job_work( my_err, &mom_err); - if (Timeout == TRUE) + if (Timeout == true) { /* 10 indicates that job migrate timed out, server will mark node down * and abort the job - see post_sendmom() */ @@ -1404,10 +1417,21 @@ int send_job_work( if (type == MOVE_TYPE_Exec) { + // If the job failed to submit, we only want to call it a network error if we didn't + // get a reply from the mom, in which case mom_err will still be set to PBSE_NONE. Otherwise, + // it was not a network error and the node is responding. + int fail_count_rc = PBSE_NONE; + + if (Timeout == true) + fail_count_rc = PBSE_TIMEOUT; + else if ((rc != PBSE_NONE) && + (mom_err == PBSE_NONE)) + fail_count_rc = rc; + if (node_name != NULL) - update_failure_counts(node_name, rc); + update_failure_counts(node_name, fail_count_rc); else - update_failure_counts(job_destin, rc); + update_failure_counts(job_destin, fail_count_rc); } send_job_work_end: diff --git a/src/server/svr_process_request.h b/src/server/svr_process_request.h index 753ee3ca64..a235fea16b 100644 --- a/src/server/svr_process_request.h +++ b/src/server/svr_process_request.h @@ -12,12 +12,8 @@ int dispatch_request(int sfds, struct batch_request *request); static void svr_close_client(int sfds); -struct batch_request *alloc_br(int type); - static void close_quejob(int sfds); -void free_br(struct batch_request *preq); - static void freebr_manage(struct rq_manage *pmgr); static void freebr_cpyfile(struct rq_cpyfile *pcf); diff --git a/src/server/track_alps_reservations.cpp b/src/server/track_alps_reservations.cpp index ffbc7ae854..159cd8a287 100644 --- a/src/server/track_alps_reservations.cpp +++ b/src/server/track_alps_reservations.cpp @@ -187,15 +187,25 @@ bool reservation_holder::is_orphaned( bool orphaned = false; job *pjob; std::map::iterator it; + int internal_jid = -1; + bool res_found = false; pthread_mutex_lock(&this->rh_mutex); it = this->reservations.find(rsv_id); if (it != this->reservations.end()) { - job_id = job_mapper.get_name(it->second.internal_job_id); + internal_jid = it->second.internal_job_id; + res_found = true; + } + + pthread_mutex_unlock(&this->rh_mutex); + + if (res_found) + { + job_id = job_mapper.get_name(internal_jid); - if ((pjob = svr_find_job_by_id(it->second.internal_job_id)) != NULL) + if ((pjob = svr_find_job_by_id(internal_jid)) != NULL) { if (pjob->ji_qs.ji_state == JOB_STATE_COMPLETE) orphaned = true; @@ -215,6 +225,8 @@ bool reservation_holder::is_orphaned( if (orphaned == true) { + pthread_mutex_lock(&this->rh_mutex); + if (this->orphaned_reservations.find(rsv_id) != this->orphaned_reservations.end()) { // Only send one release reservation request at a time. @@ -225,9 +237,9 @@ bool reservation_holder::is_orphaned( // Record this so we only send one at a time this->orphaned_reservations.insert(rsv_id); } - } - pthread_mutex_unlock(&this->rh_mutex); + pthread_mutex_unlock(&this->rh_mutex); + } return(orphaned); } /* END is_orphaned() */ diff --git a/src/server/user_info.c b/src/server/user_info.c index 82cd332bd8..50fbe28cbb 100644 --- a/src/server/user_info.c +++ b/src/server/user_info.c @@ -150,7 +150,6 @@ unsigned int count_jobs_submitted( - int can_queue_new_job( char *user_name, @@ -181,7 +180,6 @@ int can_queue_new_job( - int increment_queued_jobs( user_info_holder *uih, @@ -193,7 +191,6 @@ int increment_queued_jobs( user_info *ui; unsigned bit = COUNTED_GLOBALLY; - /* If pbs_server is restarting we may get jobs in a completed state. we do not want to count these jobs as queued */ if (pjob->ji_qs.ji_state == JOB_STATE_COMPLETE) @@ -204,35 +201,40 @@ int increment_queued_jobs( // Prevent recounting a job if (pjob->ji_queue_counted & bit) - return(PBSE_NONE); + return(rc); + + // Only increment the count for array templates and normal jobs + if ((pjob->ji_is_array_template == TRUE) || + ((pjob->ji_wattr[JOB_ATR_job_array_id].at_flags & ATR_VFLAG_SET) == 0)) + { + unsigned int num_submitted = count_jobs_submitted(pjob); + std::string uname(user_name); - unsigned int num_submitted = count_jobs_submitted(pjob); - std::string uname(user_name); - - remove_server_suffix(uname); + remove_server_suffix(uname); - uih->lock(); + uih->lock(); - /* get the user if there is one */ - if ((ui = uih->find(uname)) != NULL) - { - ui->num_jobs_queued += num_submitted; - } - else - { - /* user doesn't exist, create a new one and insert */ - ui = (user_info *)calloc(1, sizeof(user_info)); - ui->user_name = strdup(uname.c_str()); - ui->num_jobs_queued = num_submitted; - - if (!uih->insert(ui,ui->user_name)) + /* get the user if there is one */ + if ((ui = uih->find(uname)) != NULL) { - rc = ENOMEM; - log_err(rc, __func__, "Can't resize the user info array"); + ui->num_jobs_queued += num_submitted; } - } + else + { + /* user doesn't exist, create a new one and insert */ + ui = (user_info *)calloc(1, sizeof(user_info)); + ui->user_name = strdup(uname.c_str()); + ui->num_jobs_queued = num_submitted; - uih->unlock(); + if (!uih->insert(ui, ui->user_name)) + { + rc = ENOMEM; + log_err(rc, __func__, "Can't resize the user info array"); + } + } + + uih->unlock(); + } // Mark this job as being counted as queued pjob->ji_queue_counted |= bit; diff --git a/src/test/Makefile.am b/src/test/Makefile.am index 730fd5628c..5824f977cf 100644 --- a/src/test/Makefile.am +++ b/src/test/Makefile.am @@ -68,7 +68,7 @@ CMDS_UT_DIRS = MXML common_cmds pbs_track pbsdsh pbsnodes pbspd pbspoe qalter qc MISC_UT_DIRS = momctl -MOM_UT_DIRS = alps_reservations catch_child checkpoint cray_energy generate_alps_status \ +MOM_UT_DIRS = alps_reservations catch_child checkpoint cray_energy cray_taskstats generate_alps_status \ mom_comm mom_inter mom_job_func mom_mach mom_main mom_process_request mom_req_quejob \ mom_server mom_start parse_config pbs_demux prolog release_reservation requests \ start_exec tmsock_recov @@ -80,6 +80,14 @@ if BUILDPMIX MOM_UT_DIRS += pmix_interface pmix_operation pmix_tracker endif +if BUILD_LINUX_CGROUPS + MOM_UT_DIRS += trq_cgroups +endif + +if NVIDIA + MOM_UT_DIRS += nvidia +endif + if INCLUDE_PAM PAM_DIRS = pam endif diff --git a/src/test/add_verify_resources/scaffolding.c b/src/test/add_verify_resources/scaffolding.c index a4b2bbdd5c..671f380a0c 100644 --- a/src/test/add_verify_resources/scaffolding.c +++ b/src/test/add_verify_resources/scaffolding.c @@ -4,7 +4,7 @@ #include "u_hash_map_structs.h" /* job_data */ -void hash_add_or_exit(job_data_container *head, const char *name, const char *value, int var_type) +void hash_priority_add_or_exit(job_data_container *head, const char *name, const char *value, int var_type) { fprintf(stderr, "The call to hash_add_or_exit needs to be mocked!!\n"); } diff --git a/src/test/allocation/scaffolding.c b/src/test/allocation/scaffolding.c index a6217771a5..31c1e252c2 100644 --- a/src/test/allocation/scaffolding.c +++ b/src/test/allocation/scaffolding.c @@ -14,6 +14,8 @@ const char *place_legacy2 = "legacy2"; const char *use_cores = "usecores"; +int tc = 1; + std::string req::getPlacementType() const { return(""); @@ -21,7 +23,7 @@ std::string req::getPlacementType() const req::req() {} -unsigned long req::getMemory() const +unsigned long long req::get_memory_per_task() const { return(1024); @@ -33,6 +35,11 @@ int req::getExecutionSlots() const return(2); } +int req::getTaskCount() const + { + return(tc); + } + std::string req::getThreadUsageString() const { @@ -45,7 +52,7 @@ int req::getMics() const return(0); } -int req::getGpus() const +int req::get_gpus() const { return(0); diff --git a/src/test/allocation/test_uut.c b/src/test/allocation/test_uut.c index a645a846e3..3a71283a0e 100644 --- a/src/test/allocation/test_uut.c +++ b/src/test/allocation/test_uut.c @@ -6,6 +6,33 @@ #include "allocation.hpp" #include "req.hpp" +extern int tc; + + +START_TEST(test_remainder_things) + { + allocation a; + + a.cpus = 9; + a.gpus = 3; + allocation b(a); + + a.adjust_for_spread(2, false); + b.adjust_for_spread(2, true); + + fail_unless(a.cpus == 4); + fail_unless(a.gpus == 1); + fail_unless(b.cpus == 1); + fail_unless(b.gpus == 1); + + a.adjust_for_remainder(b); + fail_unless(b.cpus == 0); + fail_unless(b.gpus == 0); + fail_unless(a.cpus == 5); + fail_unless(a.gpus == 2); + } +END_TEST + START_TEST(test_write_task_information) { @@ -67,6 +94,7 @@ START_TEST(test_allocation_constructors) allocation a3("1.napali"); fail_unless(a3.jobid == "1.napali"); + tc = 1; req r; allocation a4(r); fail_unless(a4.memory == 1024, "mem = %d", a.memory); @@ -251,6 +279,7 @@ Suite *allocation_suite(void) tcase_add_test(tc_core, test_allocation_constructors); tcase_add_test(tc_core, test_set_place_type); tcase_add_test(tc_core, test_fully_placed); + tcase_add_test(tc_core, test_remainder_things); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_add_allocation"); diff --git a/src/test/array_func/scaffolding.c b/src/test/array_func/scaffolding.c index 24bb20b9d7..9b32eeaeb4 100644 --- a/src/test/array_func/scaffolding.c +++ b/src/test/array_func/scaffolding.c @@ -133,9 +133,10 @@ void svr_evaljobstate(job &pjob, int &newstate, int &newsub, int forceeval) exit(1); } -char *get_correct_jobname(const char *jobid) +const char *get_correct_jobname(const char *jobid, std::string &correct) { - return(strdup(jobid)); + correct = jobid; + return(correct.c_str()); } void *get_prior(list_link pl, char *file, int line) @@ -379,3 +380,11 @@ int create_and_queue_array_subjob( { return(PBSE_NONE); } + +batch_request::batch_request(const batch_request &other) + { + } + +void batch_request::update_object_id(int job_index) {} + +batch_request::~batch_request() {} diff --git a/src/test/attr_fn_complete_req/test_uut.c b/src/test/attr_fn_complete_req/test_uut.c index 1561defa40..3899998498 100644 --- a/src/test/attr_fn_complete_req/test_uut.c +++ b/src/test/attr_fn_complete_req/test_uut.c @@ -66,12 +66,12 @@ START_TEST(test_decode_encode_complete_req) called_encode = 0; fail_unless(encode_complete_req(&pattr, NULL, NULL, NULL, 0, 0) == PBSE_NONE); - // we set 2 attributes + there are 2 that should go by default - fail_unless(called_encode == 4, "encoded %d", called_encode); + // we set 2 attributes + there are 2 that should go by default + mem_per_task + fail_unless(called_encode == 5, "encoded %d", called_encode); pattr.at_val.at_ptr = NULL; fail_unless(encode_complete_req(&pattr, NULL, NULL, NULL, 0, 0) == PBSE_NONE); // make sure we didn't encode anything new - fail_unless(called_encode == 4); + fail_unless(called_encode == 5); } END_TEST diff --git a/src/test/attr_fn_hold/test_uut.c b/src/test/attr_fn_hold/test_uut.c index f8f6681886..c432081b73 100644 --- a/src/test/attr_fn_hold/test_uut.c +++ b/src/test/attr_fn_hold/test_uut.c @@ -39,6 +39,13 @@ START_TEST(test_one) memset(&attr,0,sizeof(attr)); fail_unless(decode_hold(&attr,NULL,NULL,"banana",0) == PBSE_BADATVAL); + memset(&attr,0,sizeof(attr)); + fail_unless(decode_hold(&attr,NULL,NULL,"a",0) == 0); + fail_unless(attr.at_flags == (ATR_VFLAG_SET|ATR_VFLAG_MODIFY)); + + memset(&attr,0,sizeof(attr)); + fail_unless(decode_hold(&attr,NULL,NULL,"l",0) == 0); + fail_unless(attr.at_flags == (ATR_VFLAG_SET|ATR_VFLAG_MODIFY)); } END_TEST @@ -47,14 +54,22 @@ START_TEST(test_two) svrattrl *attrl = attrlist_create("Fred","Wilma",20); pbs_attribute f; pbs_attribute t; - - memset(&f,0,sizeof(f)); - decode_hold(&f,NULL,NULL,"so",0); - memset(&t,0,sizeof(t)); - decode_hold(&t,NULL,NULL,"nu",0); - - fail_unless(encode_hold(&f,(tlist_head *)attrl,"FALSE",NULL,0,0) == 1); - fail_unless(encode_hold(&t,(tlist_head *)attrl,"TRUE",NULL,0,0) == 1); + pbs_attribute t2; + pbs_attribute t3; + + memset(&f, 0, sizeof(f)); + decode_hold(&f, NULL, NULL, "so", 0); + memset(&t, 0, sizeof(t)); + decode_hold(&t, NULL, NULL, "nu", 0); + memset(&t2, 0, sizeof(t2)); + decode_hold(&t2, NULL, NULL, "a", 0); + memset(&t3, 0, sizeof(t3)); + decode_hold(&t3, NULL, NULL, "l", 0); + + fail_unless(encode_hold(&f, (tlist_head *)attrl, "FALSE", NULL, 0, 0) == 1); + fail_unless(encode_hold(&t, (tlist_head *)attrl, "TRUE", NULL, 0, 0) == 1); + fail_unless(encode_hold(&t2, (tlist_head *)attrl, "TRUE", NULL, 0, 0) == 1); + fail_unless(encode_hold(&t3, (tlist_head *)attrl, "TRUE", NULL, 0, 0) == 1); } END_TEST diff --git a/src/test/attr_fn_resc/test_uut.c b/src/test/attr_fn_resc/test_uut.c index 9b44358aae..687c4e3b89 100644 --- a/src/test/attr_fn_resc/test_uut.c +++ b/src/test/attr_fn_resc/test_uut.c @@ -15,7 +15,7 @@ START_TEST(test_one) fail_unless(decode_resc(NULL,NULL,NULL,NULL,0) == PBSE_INTERNAL); fail_unless(decode_resc(&attr,NULL,NULL,NULL,0) == PBSE_UNKRESC); fail_unless(decode_resc(&attr,"attrName",NULL,NULL,0) == PBSE_UNKRESC); - fail_unless(decode_resc(&attr,"attrName","arch",NULL,0) == PBSE_ATTRRO); + fail_unless(decode_resc(&attr,"attrName","arch",NULL,0) == PBSE_UNKRESC); fail_unless(decode_resc(&attr,"notherAttrName","string","Pie",0) == PBSE_ATTRRO); fail_unless(decode_resc(&attr,"notherAttrName","string","Pie",ATR_DFLAG_ACCESS) == 0); fail_unless(decode_resc(&attr,"someLong","long","314179",ATR_DFLAG_ACCESS) == 0); diff --git a/src/test/attr_fn_time/test_uut.c b/src/test/attr_fn_time/test_uut.c index 1a46d84007..2ff0cf75d3 100644 --- a/src/test/attr_fn_time/test_uut.c +++ b/src/test/attr_fn_time/test_uut.c @@ -7,6 +7,7 @@ #include "pbs_error.h" int get_time_string(char *time_string, int string_size, long timeval); +long time_str_to_seconds(const char *str); START_TEST(test_get_time_string) { @@ -25,10 +26,21 @@ START_TEST(test_get_time_string) } END_TEST -START_TEST(test_two) +START_TEST(test_time_str_to_seconds) { + const char *s1 = "10:09:08:07.654"; // DD:HH:MM:SS.mmm (.milliseconds) + const char *s2 = "18:00:00"; // HH:MM::SS + const char *s3 = "bob"; // bad + const char *s4 = "18:00:00:00:00:00"; // bad + const char *s5 = "18:00:00:walltime=18:00:00"; // bad + const char *s6 = "00:04:00"; // HH:MM:SS - + fail_unless(time_str_to_seconds(s1) == 896888); // round up for .654 + fail_unless(time_str_to_seconds(s2) == 64800); + fail_unless(time_str_to_seconds(s3) == -1); + fail_unless(time_str_to_seconds(s4) == -1); + fail_unless(time_str_to_seconds(s5) == -1); + fail_unless(time_str_to_seconds(s6) == 240); } END_TEST @@ -39,8 +51,8 @@ Suite *attr_fn_time_suite(void) tcase_add_test(tc_core, test_get_time_string); suite_add_tcase(s, tc_core); - tc_core = tcase_create("test_two"); - tcase_add_test(tc_core, test_two); + tc_core = tcase_create("test_time_str_to_seconds"); + tcase_add_test(tc_core, test_time_str_to_seconds); suite_add_tcase(s, tc_core); return s; diff --git a/src/test/attr_str_conversion/test_uut.c b/src/test/attr_str_conversion/test_uut.c index d68e36a91b..4fc6e0266a 100644 --- a/src/test/attr_str_conversion/test_uut.c +++ b/src/test/attr_str_conversion/test_uut.c @@ -6,11 +6,55 @@ #include #include "attribute.h" +#include "resource.h" #include "server.h" void appendEscapedXML(const char *xml, std::string &str); int size_to_str(struct size_value *szv, char *out, int space); int attr_to_str(std::string &output, attribute_def *at_def, pbs_attribute attr, bool xml); +int resource_index_to_string(std::string &output, std::vector &resources, size_t index); + + +START_TEST(test_resource_index_to_string) + { + std::string output; + resource_def ad[4]; + resource r[4]; + + for (int i = 0; i < 4; i++) + r[i].rs_defin = ad + i; + + ad[0].rs_name = "cput"; + ad[1].rs_name = "vmem"; + ad[2].rs_name = "mem"; + ad[3].rs_name = "stringy"; + + r[0].rs_value.at_type = ATR_TYPE_LONG; + r[0].rs_value.at_val.at_long = 200; + r[1].rs_value.at_type = ATR_TYPE_SIZE; + r[1].rs_value.at_val.at_size.atsv_num = 2048; + r[1].rs_value.at_val.at_size.atsv_shift = 10; + r[2].rs_value.at_type = ATR_TYPE_SIZE; + r[2].rs_value.at_val.at_size.atsv_num = 3050; + r[2].rs_value.at_val.at_size.atsv_shift = 10; + r[3].rs_value.at_type = ATR_TYPE_STR; + r[3].rs_value.at_val.at_str = strdup("bob"); + + std::vector resources; + for (int i = 0; i < 4; i++) + resources.push_back(r[i]); + + fail_unless(resource_index_to_string(output, resources, 0) == PBSE_NONE); + fail_unless(output == "cput=200"); + fail_unless(resource_index_to_string(output, resources, 1) == PBSE_NONE); + fail_unless(output == "vmem=2048kb"); + fail_unless(resource_index_to_string(output, resources, 2) == PBSE_NONE); + fail_unless(output == "mem=3050kb"); + fail_unless(resource_index_to_string(output, resources, 3) == PBSE_NONE); + fail_unless(output == "stringy=bob"); + fail_unless(resource_index_to_string(output, resources, 4) != PBSE_NONE); + } +END_TEST START_TEST(test_attr_to_str) @@ -107,6 +151,7 @@ Suite *attr_str_conversion_suite(void) Suite *s = suite_create("attr_str_conversion test suite methods"); TCase *tc_core = tcase_create("test_appendEscapedXML"); tcase_add_test(tc_core, test_appendEscapedXML); + tcase_add_test(tc_core, test_resource_index_to_string); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_size_to_str"); diff --git a/src/test/batch_request/scaffolding.c b/src/test/batch_request/scaffolding.c index c4730e4b73..e909f6eacd 100644 --- a/src/test/batch_request/scaffolding.c +++ b/src/test/batch_request/scaffolding.c @@ -9,3 +9,10 @@ int LOGLEVEL = 10; void log_err(int errnum, const char *routine, const char *text) {} + +void reply_free( + + struct batch_reply *prep) + + { + } diff --git a/src/test/batch_request/test_uut.c b/src/test/batch_request/test_uut.c index fe64a25829..f121aa76d7 100644 --- a/src/test/batch_request/test_uut.c +++ b/src/test/batch_request/test_uut.c @@ -7,10 +7,10 @@ START_TEST(test_one) { - batch_request *pBr1 = (batch_request *)calloc(1,sizeof(batch_request)); - batch_request *pBr2 = (batch_request *)calloc(1,sizeof(batch_request)); - batch_request *pBr3 = (batch_request *)calloc(1,sizeof(batch_request)); - batch_request *pBr4 = (batch_request *)calloc(1,sizeof(batch_request)); + batch_request *pBr1 = new (batch_request); + batch_request *pBr2 = new (batch_request); + batch_request *pBr3 = new (batch_request); + batch_request *pBr4 = new (batch_request); int rc = get_batch_request_id(pBr1); fail_unless(rc == PBSE_NONE); @@ -41,13 +41,49 @@ START_TEST(test_one) END_TEST +START_TEST(test_constructor) + { + batch_request *preq = new batch_request(PBS_BATCH_QueueJob); + fail_unless(preq->rq_type == PBS_BATCH_QueueJob); + fail_unless(preq->rq_conn == -1); + fail_unless(preq->rq_orgconn == -1); + fail_unless(preq->rq_reply.brp_choice == BATCH_REPLY_CHOICE_NULL); + fail_unless(preq->rq_noreply == FALSE); + fail_unless(preq->rq_time > 0); + delete preq; + } +END_TEST -START_TEST(test_two) + +START_TEST(test_br_copy_constructor) { + batch_request preq; + batch_request *dup; + + preq.rq_perm = 1; + strcpy(preq.rq_user, "dbeer"); + strcpy(preq.rq_host, "napali"); + preq.rq_extend = strdup("tom"); + preq.rq_type = PBS_BATCH_RunJob; + preq.rq_ind.rq_run.rq_destin = strdup("napali"); + + dup = new batch_request(preq); + fail_unless(dup != NULL); + fail_unless(!strcmp(dup->rq_extend, "tom")); + fail_unless(!strcmp(dup->rq_user, "dbeer")); + fail_unless(!strcmp(dup->rq_host, "napali")); + fail_unless(!strcmp(dup->rq_extend, "tom")); + fail_unless(!strcmp(dup->rq_ind.rq_run.rq_destin, "napali")); + + preq.rq_type = PBS_BATCH_Rerun; + const char *rerun_jobid = "4.roshar"; + strcpy(preq.rq_ind.rq_rerun, rerun_jobid); + batch_request *rerun_dep = new batch_request(preq); + fail_unless(!strcmp(rerun_dep->rq_ind.rq_rerun, rerun_jobid)); } -END_TEST +END_TEST @@ -58,10 +94,11 @@ Suite *batch_request_suite(void) TCase *tc_core = tcase_create("test_one"); tcase_add_test(tc_core, test_one); + tcase_add_test(tc_core, test_br_copy_constructor); suite_add_tcase(s, tc_core); - tc_core = tcase_create("test_two"); - tcase_add_test(tc_core, test_two); + tc_core = tcase_create("test_constructor"); + tcase_add_test(tc_core, test_constructor); suite_add_tcase(s, tc_core); return(s); diff --git a/src/test/catch_child/scaffolding.c b/src/test/catch_child/scaffolding.c index 3b4d76e644..7db1077854 100644 --- a/src/test/catch_child/scaffolding.c +++ b/src/test/catch_child/scaffolding.c @@ -25,6 +25,7 @@ int server_down; int called_open_socket = 0; int called_fork_me = 0; +bool called_epilogue = false; bool check_rur = true; char mom_alias[PBS_MAXHOSTNAME + 1]; @@ -1162,6 +1163,8 @@ char *get_local_script_path(job *pjob, char *base) int run_pelog(int which, char *specpelog, job *pjog, int pe_io_type, int deletejob) { + called_epilogue = true; + return 1; } @@ -1310,9 +1313,9 @@ void mom_job_purge(job *pjob) return; } -bool check_pwd(job *pjob) +int check_pwd(job *pjob) { - return false; + return(-1); } unsigned long gettime(resource *pres) @@ -1418,3 +1421,11 @@ int send_back_std_and_staged_files(job *pjob, int exit_value) task::~task() {} + +batch_request::batch_request(int type) : rq_type(type) + { + } + +bool get_cray_taskstats; + +void update_jobs_usage(job *pjob) {} diff --git a/src/test/catch_child/test_uut.c b/src/test/catch_child/test_uut.c index 4e967c827a..d5e9341e55 100644 --- a/src/test/catch_child/test_uut.c +++ b/src/test/catch_child/test_uut.c @@ -13,6 +13,7 @@ bool non_mother_superior_cleanup(job *pjob); bool mother_superior_cleanup(job *pjob, int limit, int *found); void *obit_reply(void *new_sock); hnodent *get_node(job *pjob, tm_node_id nodeid); +int run_epilogues(job *pjob, int i_am_ms, int deletejob); extern int termin_child; extern int server_down; @@ -21,6 +22,7 @@ extern int DIS_reply_read_count; extern int tc; extern int called_open_socket; extern int called_fork_me; +extern bool called_epilogue; extern bool eintr_test; extern std::vector exiting_job_list; @@ -200,6 +202,22 @@ START_TEST(test_mother_superior_cleanup) } END_TEST +START_TEST(test_epilogues_run_without_prologues) + { + job *pjob = (job *)calloc(1, sizeof(job)); + + pjob->ji_qs.ji_svrflags |= JOB_SVFLG_PROLOGUES_RAN; + called_epilogue = false; + run_epilogues(pjob, 0, 0); + ck_assert(called_epilogue); + + pjob->ji_qs.ji_svrflags = 0x0; + called_epilogue = false; + run_epilogues(pjob, 0, 0); + ck_assert(!called_epilogue); + } +END_TEST + Suite *catch_child_suite(void) { Suite *s = suite_create("catch_child methods"); @@ -210,6 +228,7 @@ Suite *catch_child_suite(void) tcase_add_test(tc_core, test_jobs_main_process); tcase_add_test(tc_core, test_non_mother_superior_cleanup); tcase_add_test(tc_core, test_mother_superior_cleanup); + tcase_add_test(tc_core, test_epilogues_run_without_prologues); s = suite_create("get_node_tests"); tcase_add_test(tc_core, test_get_node_1); diff --git a/src/test/cnt2server/scaffolding.c b/src/test/cnt2server/scaffolding.c index fbacf0d372..81da836858 100644 --- a/src/test/cnt2server/scaffolding.c +++ b/src/test/cnt2server/scaffolding.c @@ -7,6 +7,8 @@ int pbs_errno = 0; +extern bool global_silence; + extern "C" { int pbs_connect(char *server_name_ptr) @@ -15,6 +17,14 @@ int pbs_connect(char *server_name_ptr) exit(1); } +int pbs_connect_ext(char *server_name_ptr, bool silence) + { + // set global silence + global_silence = silence; + + return(0); + } + char *pbs_default(void) { fprintf(stderr, "The call to pbs_default needs to be mocked!!\n"); diff --git a/src/test/cnt2server/test_uut.c b/src/test/cnt2server/test_uut.c index 37c1eaaa77..e5ffdc2d48 100644 --- a/src/test/cnt2server/test_uut.c +++ b/src/test/cnt2server/test_uut.c @@ -1,5 +1,6 @@ #include "license_pbs.h" /* See here for the software license */ #include "test_cnt2server.h" +#include "cmds.h" #include #include @@ -7,6 +8,7 @@ #include "pbs_error.h" #include "pbs_ifl.h" +bool global_silence = false; START_TEST(test_cnt2server_conf) { @@ -19,10 +21,17 @@ START_TEST(test_cnt2server_conf) } END_TEST -START_TEST(test_two) +START_TEST(test_cnt2server) { + int rc; + rc = cnt2server("foo", false); + fail_unless(rc == 0); + fail_unless(global_silence == false); + rc = cnt2server("foo", true); + fail_unless(rc == 0); + fail_unless(global_silence == true); } END_TEST @@ -33,8 +42,8 @@ Suite *cnt2server_suite(void) tcase_add_test(tc_core, test_cnt2server_conf); suite_add_tcase(s, tc_core); - tc_core = tcase_create("test_two"); - tcase_add_test(tc_core, test_two); + tc_core = tcase_create("test_cnt2server"); + tcase_add_test(tc_core, test_cnt2server); suite_add_tcase(s, tc_core); return s; diff --git a/src/test/complete_req/test_uut.c b/src/test/complete_req/test_uut.c index d30b0780f6..33f4fd7b1d 100644 --- a/src/test/complete_req/test_uut.c +++ b/src/test/complete_req/test_uut.c @@ -135,7 +135,7 @@ START_TEST(test_constructor) fail_unless(list1.req_count() == 2); const req &rm1 = list1.get_req(0); - fail_unless(rm1.getMemory() == 13653, "mem is %lu", rm1.getMemory()); + fail_unless(rm1.get_total_memory() == 13653, "mem is %lu", rm1.get_total_memory()); resources.clear(); add_resource(resources, "size", NULL, 20, -1); @@ -145,7 +145,7 @@ START_TEST(test_constructor) const req &r = list2.get_req(0); fail_unless(r.getTaskCount() == 20); fail_unless(r.getExecutionSlots() == 1); - fail_unless(r.getMemory() == 40, "mem is %lu", r.getMemory()); + fail_unless(r.get_total_memory() == 40, "mem is %lu", r.get_total_memory()); resources.clear(); add_resource(resources, "ncpus", NULL, 16, -1); @@ -155,7 +155,7 @@ START_TEST(test_constructor) const req &rl = list3.get_req(0); fail_unless(rl.getTaskCount() == 1); fail_unless(rl.getExecutionSlots() == 16); - fail_unless(rl.getMemory() == 40, "mem is %lu", rl.getMemory()); + fail_unless(rl.get_total_memory() == 40, "mem is %lu", rl.get_total_memory()); resources.clear(); add_resource(resources, "nodes", "1:ppn=2", 16, -1); @@ -164,7 +164,8 @@ START_TEST(test_constructor) fail_unless(list4.req_count() == 1); const req &rl2 = list4.get_req(0); fail_unless(rl2.getTaskCount() == 1); - fail_unless(rl2.getMemory() == 160, "pmem is %lu", rl2.getMemory()); + // pmem should get multiplied because nodes=1:ppn=2 + fail_unless(rl2.get_total_memory() == 160, "pmem is %lu", rl2.get_total_memory()); resources.clear(); add_resource(resources, "nodes", "1:ppn=2", 16, -1); @@ -173,8 +174,9 @@ START_TEST(test_constructor) fail_unless(list5.req_count() == 1); const req &rl3 = list5.get_req(0); fail_unless(rl3.getTaskCount() == 1); - fail_unless(rl3.getMemory() == 0, "pvmem is %lu", rl3.getMemory()); - fail_unless(rl3.getSwap() == 160, "pvmem is %lu", rl3.getSwap()); + fail_unless(rl3.get_total_memory() == 0, "pvmem is %lu", rl3.get_total_memory()); + // pvmem should get multiplied because nodes=1:ppn=2 is one task + fail_unless(rl3.get_total_swap() == 160, "pvmem is %lu", rl3.get_total_swap()); resources.clear(); add_resource(resources, "nodes", "1:ppn=2", 16, -1); @@ -183,8 +185,8 @@ START_TEST(test_constructor) fail_unless(list6.req_count() == 1); const req &rl4 = list6.get_req(0); fail_unless(rl4.getTaskCount() == 1); - fail_unless(rl4.getMemory() == 0, "pvmem is %lu", rl4.getMemory()); - fail_unless(rl4.getSwap() == 80, "pvmem is %lu", rl4.getSwap()); + fail_unless(rl4.get_total_memory() == 0, "vmem is %lu", rl4.get_total_memory()); + fail_unless(rl4.get_total_swap() == 80, "vmem is %lu", rl4.get_total_swap()); resources.clear(); add_resource(resources, "procs", NULL, 2, -1); @@ -193,7 +195,8 @@ START_TEST(test_constructor) fail_unless(list7.req_count() == 1); const req &rl7 = list7.get_req(0); fail_unless(rl7.getTaskCount() == 2, "task count is %d", rl7.getTaskCount()); - fail_unless(rl7.getMemory() == 1024, "mem = %lu", rl7.getMemory()); + // 1024 * 2 = 2048, multiply by 2 because procs=2 + fail_unless(rl7.get_total_memory() == 2048, "mem = %lu", rl7.get_total_memory()); // Make sure that we'll set memory to the higher of pmem and mem, and set swap // as well for the same job @@ -206,8 +209,8 @@ START_TEST(test_constructor) fail_unless(list8.req_count() == 1); const req &rl8 = list8.get_req(0); fail_unless(rl8.getTaskCount() == 1); - fail_unless(rl8.getMemory() == 4096); - fail_unless(rl8.getSwap() == 8192); + fail_unless(rl8.get_total_memory() == 4096); + fail_unless(rl8.get_total_swap() == 8192); resources.clear(); add_resource(resources, "procs", NULL, 1, -1); @@ -216,7 +219,7 @@ START_TEST(test_constructor) complete_req list9(&resources, 1, false); fail_unless(list9.req_count() == 1); const req &rl9 = list9.get_req(0); - fail_unless(rl9.getMemory() == 5000); + fail_unless(rl9.get_total_memory() == 5000); } END_TEST @@ -230,7 +233,7 @@ START_TEST(test_constructor_oldstyle_req) fail_unless(list1.req_count() == 1); const req &rm1 = list1.get_req(0); - fail_unless(rm1.getMemory() == 0, "mem is %lu", rm1.getMemory()); + fail_unless(rm1.get_total_memory() == 0, "mem is %lu", rm1.get_total_memory()); } END_TEST diff --git a/src/test/completed_jobs_map/scaffolding.c b/src/test/completed_jobs_map/scaffolding.c index d1f6be84ea..b38abed049 100644 --- a/src/test/completed_jobs_map/scaffolding.c +++ b/src/test/completed_jobs_map/scaffolding.c @@ -31,8 +31,7 @@ void *handle_complete_second_time(work_task *wt) bool job_id_exists( - const std::string &job_id_string, - int *rcode) + const std::string &job_id_string) { return(job_id_exists_rc); diff --git a/src/test/cray_taskstats/Makefile.am b/src/test/cray_taskstats/Makefile.am new file mode 100644 index 0000000000..c64a3ab0cf --- /dev/null +++ b/src/test/cray_taskstats/Makefile.am @@ -0,0 +1,3 @@ +include ../Makefile_Mom.ut + +libuut_la_SOURCES = ${PROG_ROOT}/cray_taskstats.cpp diff --git a/src/test/cray_taskstats/sample_file b/src/test/cray_taskstats/sample_file new file mode 100644 index 0000000000..6ea36d3152 --- /dev/null +++ b/src/test/cray_taskstats/sample_file @@ -0,0 +1,5 @@ +2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 0, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 3, 'stime', 4, 'max_rss', 5, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0] +2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 1, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 0, 'stime', 1, 'max_rss', 2, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0] +2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 2, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 6, 'stime', 7, 'max_rss', 8, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0] +2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 0, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 10, 'stime', 20, 'max_rss', 30, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0] +2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 3, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 4, 'stime', 4, 'max_rss', 9, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0] diff --git a/src/test/cray_taskstats/scaffolding.c b/src/test/cray_taskstats/scaffolding.c new file mode 100644 index 0000000000..c9ad0bb42c --- /dev/null +++ b/src/test/cray_taskstats/scaffolding.c @@ -0,0 +1,88 @@ +#include "license_pbs.h" /* See here for the software license */ +#include +#include +#include + +#include "attribute.h" +#include "resource.h" + +resource_def *svr_resc_def; +int svr_resc_size = 0; + +int LOGLEVEL = 7; /* force logging code to be exercised as tests run */ + +void log_err(int errnum, const char *routine, const char *text) + { + } + +void log_event(int eventtype, int objclass, const char *objname, const char *text) + { + } + +void init_attr(pbs_attribute *pattr) + { + memset(pattr, 0, sizeof(pbs_attribute)); + } + +resource_def *find_resc_def( + + resource_def *rscdf, /* address of array of resource_def structs */ + const char *name, /* name of resource */ + int limit) /* number of members in resource_def array */ + + { + static resource_def mem_def; + static resource_def cput_def; + + if (!strcmp(name, "mem")) + { + mem_def.rs_name = name; + mem_def.rs_free = init_attr; + return(&mem_def); + } + else + { + cput_def.rs_name = name; + cput_def.rs_free = init_attr; + return(&cput_def); + } + } + + +resource *add_resource_entry( + + pbs_attribute *pattr, + resource_def *prdef) + + { + resource new_resource; + + if (pattr->at_val.at_ptr == NULL) + pattr->at_val.at_ptr = new std::vector(); + + std::vector *resources = (std::vector *)pattr->at_val.at_ptr; + + for (size_t i = 0; i < resources->size(); i++) + { + resource &r = resources->at(i); + + if (!strcmp(r.rs_defin->rs_name, prdef->rs_name)) + { + return(&r); + } + } + + new_resource.rs_defin = prdef; + new_resource.rs_value.at_type = prdef->rs_type; + new_resource.rs_value.at_flags = 0; + prdef->rs_free(&new_resource.rs_value); + + resources->push_back(new_resource); + pattr->at_flags |= ATR_VFLAG_SET | ATR_VFLAG_MODIFY; + + // Return the element we just added + resource &r = resources->at(resources->size() - 1); + + return(&r); + } /* END add_resource_entry() */ + diff --git a/src/test/cray_taskstats/test_uut.c b/src/test/cray_taskstats/test_uut.c new file mode 100644 index 0000000000..24833a58cc --- /dev/null +++ b/src/test/cray_taskstats/test_uut.c @@ -0,0 +1,137 @@ +#include "license_pbs.h" /* See here for the software license */ +#include +#include +#include + +#include "pbs_error.h" +#include "pbs_job.h" +#include "resource.h" + +#include + +int parse_rur_line(const std::string &line, std::set &jobs_this_iteration); +void update_jobs_usage(job *pjob); +void read_rur_stats_file(const char *basepath); + +const char *l1 = "2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 0, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 100, 'stime', 100, 'max_rss', 1984, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0]"; +const char *l2 = "2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 0, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 300, 'stime', 0, 'max_rss', 4444, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0]"; +const char *l3 = "2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 1, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 300, 'stime', 300, 'max_rss', 555555, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0]"; +const char *l4 = "2013-11-02T11:09:49.457770-05:00 c0-0c1s1n2 RUR 2417 p0-201311-1t153028 [RUR@34] uid: 12345, apid: 86989, jobid: 2, cmdname: /lus/tmp/rur01.2338/./CPU01-2338 plugin: taskstats ['utime', 400, 'stime', 300, 'max_rss', 5, 'rchar', 107480, 'wchar', 90, 'exitcode:signal', ['0:0'], 'core', 0]"; + + + +START_TEST(test_parse_rur_line) + { + job pjob; + std::set jobs_this_iteration; + + memset(&pjob, 0, sizeof(pjob)); + snprintf(pjob.ji_qs.ji_jobid, sizeof(pjob.ji_qs.ji_jobid), "0.roshar"); + + fail_unless(parse_rur_line(l1, jobs_this_iteration) == PBSE_NONE); + update_jobs_usage(&pjob); + resource_def *rd = find_resc_def(svr_resc_def, "cput", svr_resc_size); + resource *pres = add_resource_entry(pjob.ji_wattr + JOB_ATR_resc_used, rd); + fail_unless(pres != NULL); + fail_unless(pres->rs_value.at_val.at_long == 200); + + rd = find_resc_def(svr_resc_def, "mem", svr_resc_size); + pres = add_resource_entry(pjob.ji_wattr + JOB_ATR_resc_used, rd); + fail_unless(pres != NULL); + fail_unless(pres->rs_value.at_val.at_size.atsv_num == 1984, + "%llu", pres->rs_value.at_val.at_size.atsv_num); + + // Make sure we increase the value if we are using more later + jobs_this_iteration.clear(); + fail_unless(parse_rur_line(l2, jobs_this_iteration) == PBSE_NONE); + update_jobs_usage(&pjob); + rd = find_resc_def(svr_resc_def, "cput", svr_resc_size); + pres = add_resource_entry(pjob.ji_wattr + JOB_ATR_resc_used, rd); + fail_unless(pres != NULL); + fail_unless(pres->rs_value.at_val.at_long == 300); + + rd = find_resc_def(svr_resc_def, "mem", svr_resc_size); + pres = add_resource_entry(pjob.ji_wattr + JOB_ATR_resc_used, rd); + fail_unless(pres != NULL); + fail_unless(pres->rs_value.at_val.at_size.atsv_num == 4444); + + // Make sure the values don't decrease if we find less later + jobs_this_iteration.clear(); + fail_unless(parse_rur_line(l1, jobs_this_iteration) == PBSE_NONE); + update_jobs_usage(&pjob); + rd = find_resc_def(svr_resc_def, "cput", svr_resc_size); + pres = add_resource_entry(pjob.ji_wattr + JOB_ATR_resc_used, rd); + fail_unless(pres != NULL); + fail_unless(pres->rs_value.at_val.at_long == 300); + + rd = find_resc_def(svr_resc_def, "mem", svr_resc_size); + pres = add_resource_entry(pjob.ji_wattr + JOB_ATR_resc_used, rd); + fail_unless(pres != NULL); + fail_unless(pres->rs_value.at_val.at_size.atsv_num == 4444); + } +END_TEST + + +START_TEST(test_read_rur_stats_file) + { + const char *file = "sample_file"; + extern std::map usage_information; + std::map::iterator it; + + usage_information.clear(); + + read_rur_stats_file(file); + + it = usage_information.find("0"); + fail_unless(it != usage_information.end()); + fail_unless(it->second.cput == 30); + fail_unless(it->second.mem == 30); + + it = usage_information.find("1"); + fail_unless(it != usage_information.end()); + fail_unless(it->second.cput == 1); + fail_unless(it->second.mem == 2); + + it = usage_information.find("2"); + fail_unless(it != usage_information.end()); + fail_unless(it->second.cput == 13); + fail_unless(it->second.mem == 8); + + it = usage_information.find("3"); + fail_unless(it != usage_information.end()); + fail_unless(it->second.cput == 8); + fail_unless(it->second.mem == 9); + } +END_TEST + + +Suite *cray_taskstats_suite(void) + { + Suite *s = suite_create("cray_taskstats_suite methods"); + TCase *tc_core = tcase_create("test_parse_rur_line"); + tcase_add_test(tc_core, test_parse_rur_line); + suite_add_tcase(s, tc_core); + + tc_core = tcase_create("test_read_rur_stats_file"); + tcase_add_test(tc_core, test_read_rur_stats_file); + suite_add_tcase(s, tc_core); + + return s; + } + +void rundebug() + { + } + +int main(void) + { + int number_failed = 0; + SRunner *sr = NULL; + rundebug(); + sr = srunner_create(cray_taskstats_suite()); + srunner_set_log(sr, "cray_taskstats_suite.log"); + srunner_run_all(sr, CK_NORMAL); + number_failed = srunner_ntests_failed(sr); + srunner_free(sr); + return number_failed; + } diff --git a/src/test/display_alps_status/scaffolding.c b/src/test/display_alps_status/scaffolding.c index f784959185..8852ebfab8 100644 --- a/src/test/display_alps_status/scaffolding.c +++ b/src/test/display_alps_status/scaffolding.c @@ -43,3 +43,6 @@ int pbsnode::unlock_node(const char *id, const char *msg, int level) return(0); } +batch_request::batch_request() {} +batch_request::~batch_request() {} + diff --git a/src/test/geteusernam/scaffolding.c b/src/test/geteusernam/scaffolding.c index 899a3004b1..ec849ab6df 100644 --- a/src/test/geteusernam/scaffolding.c +++ b/src/test/geteusernam/scaffolding.c @@ -12,14 +12,16 @@ #include "acl_special.hpp" #include "pbs_nodes.h" -attribute_def job_attr_def[10]; -struct server server; -char *server_host; -char *msg_orighost = strdup("Couldn't find orighost"); -int LOGLEVEL; -bool exists = true; - -acl_special limited_acls; +attribute_def job_attr_def[10]; +struct server server; +char *server_host; +char *msg_orighost = strdup("Couldn't find orighost"); +int LOGLEVEL; +bool exists = true; +std::string long_name("roshar.cosmere"); +std::string short_name("threnody"); +char *var = NULL; +acl_special limited_acls; char *site_map_user(char *uname, char *host) @@ -30,7 +32,7 @@ char *site_map_user(char *uname, char *host) char *get_variable(job *pjob, const char *variable) { - return(NULL); + return(var); } struct passwd * getpwnam_ext(char **user_buf, char * user_name) @@ -188,6 +190,17 @@ int get_svr_attr_arst( bool node_exists(const char *node_name) { + if (exists == false) + { + if (long_name == node_name) + return(true); + + if (short_name == node_name) + return(true); + } + return(exists); } +job::job() {} +job::~job() {} diff --git a/src/test/geteusernam/test_uut.c b/src/test/geteusernam/test_uut.c index d2d9a2143d..f15ea8a59d 100644 --- a/src/test/geteusernam/test_uut.c +++ b/src/test/geteusernam/test_uut.c @@ -1,16 +1,42 @@ #include "license_pbs.h" /* See here for the software license */ -#include "test_geteusernam.h" #include #include #include #include "pbs_error.h" +#include "pbs_job.h" #include "attribute.h" +#include "test_geteusernam.h" int node_exception_check(pbs_attribute *pattr, void *pobject, int actmode); -void get_user_host_from_user( std::string &user_host, const std::string user); bool is_permitted_by_node_submit(const char *orighost, int logging); +bool is_user_allowed_to_submit_jobs(job *pjob, const char *luser, char *EMsg, int logging); + +extern bool exists; +extern char *var; + + +START_TEST(test_is_user_allowed_to_submit_jobs) + { + char *user = strdup("dbeer"); + job pjob; + char emsg[1024]; + + pjob.ji_wattr[JOB_ATR_job_owner].at_val.at_str = strdup("dbeer@roshar.cosmere"); + var = NULL; + exists = false; + + // get_variable() retuns NULL, so we should fail + fail_unless(is_user_allowed_to_submit_jobs(&pjob, user, emsg, 10) == false); -extern bool exists; + // roshar.cosmere will pass but roshar won't; make sure the FQDN works + var = strdup("roshar.cosmere"); + fail_unless(is_user_allowed_to_submit_jobs(&pjob, user, emsg, 10) == true); + + // threnody will pass but threnody.cosmere won't; make sure we check the short name + var = strdup("threnody.cosmere"); + fail_unless(is_user_allowed_to_submit_jobs(&pjob, user, emsg, 10) == true); + } +END_TEST START_TEST(test_is_permitted_by_node_submit) @@ -27,22 +53,6 @@ START_TEST(test_is_permitted_by_node_submit) } END_TEST - -START_TEST(test_get_user_host_from_user) - { - std::string user_host; - std::string user; - - user = "user@hostname"; - get_user_host_from_user(user_host, user); - fail_unless(strcmp(user_host.c_str(), "hostname") == 0); - - user = "user"; - get_user_host_from_user(user_host, user); - fail_unless(user_host.size() == 0); - } -END_TEST - START_TEST(test_node_exception_check) { pbs_attribute pattr; @@ -77,12 +87,12 @@ Suite *geteusernam_suite(void) { Suite *s = suite_create("geteusernam_suite methods"); TCase *tc_core = tcase_create("test_get_user_host_from_user"); - tcase_add_test(tc_core, test_get_user_host_from_user); tcase_add_test(tc_core, test_node_exception_check); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_is_permitted_by_node_submit"); tcase_add_test(tc_core, test_is_permitted_by_node_submit); + tcase_add_test(tc_core, test_is_user_allowed_to_submit_jobs); suite_add_tcase(s, tc_core); return s; diff --git a/src/test/issue_request/scaffolding.c b/src/test/issue_request/scaffolding.c index 1d79195ec6..311afbb7bc 100644 --- a/src/test/issue_request/scaffolding.c +++ b/src/test/issue_request/scaffolding.c @@ -274,40 +274,24 @@ void DIS_tcp_cleanup(struct tcp_chan *chan) { } -/* - * alloc_br - allocate and clear a batch_request structure - */ - -struct batch_request *alloc_br( +batch_request::batch_request( int type) { + this->rq_type = type; - struct batch_request *req = NULL; - - if ((req = (struct batch_request *)calloc(1, sizeof(struct batch_request))) == NULL) - { - fprintf(stderr, "failed to allocate batch request. alloc_br()\n"); - } - else - { - - req->rq_type = type; - - req->rq_conn = -1; /* indicate not connected */ - req->rq_orgconn = -1; /* indicate not connected */ - req->rq_time = time(NULL); - req->rq_reply.brp_choice = BATCH_REPLY_CHOICE_NULL; - req->rq_noreply = FALSE; /* indicate reply is needed */ - } + this->rq_conn = -1; /* indicate not connected */ + this->rq_orgconn = -1; /* indicate not connected */ + this->rq_time = time(NULL); + this->rq_reply.brp_choice = BATCH_REPLY_CHOICE_NULL; + this->rq_noreply = FALSE; /* indicate reply is needed */ - return(req); - } /* END alloc_br() */ + } batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -399,7 +383,7 @@ int copy_attribute_list( batch_request *duplicate_request(batch_request *preq, int job_index) { - batch_request *preq_tmp = alloc_br(preq->rq_type); + batch_request *preq_tmp = new batch_request(preq->rq_type); char *ptr1; char *ptr2; char newjobname[PBS_MAXSVRJOBID+1]; @@ -544,3 +528,8 @@ void set_reply_type(struct batch_reply *preply, int type) { preply->brp_choice = type; } + +batch_request::batch_request() {} +batch_request::batch_request(const batch_request &other) {} +batch_request::~batch_request() {} + diff --git a/src/test/issue_request/test_uut.c b/src/test/issue_request/test_uut.c index 04f878a172..fabd80263e 100644 --- a/src/test/issue_request/test_uut.c +++ b/src/test/issue_request/test_uut.c @@ -22,7 +22,7 @@ int issue_Drequest(int conn, batch_request *request, bool close_handle); START_TEST(queue_a_retry_task_test) { - batch_request *preq = (batch_request *)calloc(1, sizeof(batch_request)); + batch_request *preq = new batch_request(); preq->rq_id = strdup("tom"); queue_a_retry_task(preq, NULL); @@ -40,7 +40,6 @@ START_TEST(test_one) job *pTestJob; struct batch_request request; - memset(&request,0,sizeof(request)); pTestJob = &testJob; local_connect = true; @@ -59,7 +58,7 @@ START_TEST(test_send_request_to_remote_server) int rc; batch_request *preq; - preq = alloc_br(PBS_BATCH_TrackJob); + preq = new batch_request(PBS_BATCH_TrackJob); fail_unless(preq != NULL); preq->rq_ind.rq_track.rq_hopcount = 1; strcpy(preq->rq_ind.rq_track.rq_jid, "1234"); @@ -73,20 +72,20 @@ START_TEST(test_send_request_to_remote_server) free(preq); - preq = alloc_br(PBS_BATCH_DeleteJob); + preq = new batch_request(PBS_BATCH_DeleteJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_track.rq_jid, "1234"); rc = send_request_to_remote_server(10, preq, true); fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_CheckpointJob); + preq = new batch_request(PBS_BATCH_CheckpointJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_hold.rq_orig.rq_objname, "some_object"); rc = send_request_to_remote_server(10, preq, true); free(preq); - preq = alloc_br(PBS_BATCH_GpuCtrl); + preq = new batch_request(PBS_BATCH_GpuCtrl); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_gpuctrl.rq_momnode, "nodea"); strcpy(preq->rq_ind.rq_gpuctrl.rq_gpuid, "0"); @@ -98,7 +97,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_MessJob); + preq = new batch_request(PBS_BATCH_MessJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_message.rq_jid, "1234"); preq->rq_ind.rq_message.rq_file = 1; @@ -109,7 +108,7 @@ START_TEST(test_send_request_to_remote_server) free(preq); - preq = alloc_br(PBS_BATCH_Rerun); + preq = new batch_request(PBS_BATCH_Rerun); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_rerun, "1234"); @@ -117,7 +116,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_RegistDep); + preq = new batch_request(PBS_BATCH_RegistDep); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_rerun, "1234"); @@ -125,7 +124,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_AsySignalJob); + preq = new batch_request(PBS_BATCH_AsySignalJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_signal.rq_jid, "1234"); strcpy(preq->rq_ind.rq_signal.rq_signame, "SIGKILL"); @@ -135,7 +134,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_SignalJob); + preq = new batch_request(PBS_BATCH_SignalJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_signal.rq_jid, "1234"); strcpy(preq->rq_ind.rq_signal.rq_signame, "SIGKILL"); @@ -145,7 +144,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_StatusJob); + preq = new batch_request(PBS_BATCH_StatusJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); @@ -153,7 +152,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_ReturnFiles); + preq = new batch_request(PBS_BATCH_ReturnFiles); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); @@ -161,7 +160,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_CopyFiles); + preq = new batch_request(PBS_BATCH_CopyFiles); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); @@ -169,7 +168,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_DelFiles); + preq = new batch_request(PBS_BATCH_DelFiles); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); @@ -177,7 +176,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_DeleteReservation); + preq = new batch_request(PBS_BATCH_DeleteReservation); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); @@ -185,7 +184,7 @@ START_TEST(test_send_request_to_remote_server) fail_unless(rc == PBSE_NONE); free(preq); - preq = alloc_br(PBS_BATCH_GAP038); + preq = new batch_request(PBS_BATCH_GAP038); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); @@ -203,7 +202,7 @@ START_TEST(test_handle_local_request) int rc; struct batch_request *preq; - preq = alloc_br(PBS_BATCH_StatusJob); + preq = new batch_request(PBS_BATCH_StatusJob); fail_unless(preq != NULL); rc = handle_local_request(10, preq); @@ -218,7 +217,7 @@ START_TEST(test_issue_Drequest) int rc; struct batch_request *preq; - preq = alloc_br(PBS_BATCH_StatusJob); + preq = new batch_request(PBS_BATCH_StatusJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); @@ -236,37 +235,35 @@ START_TEST(test_issue_to_svr) char server_name[PBS_MAXHOSTNAME]; struct batch_request *preq; - preq = alloc_br(PBS_BATCH_StatusJob); + preq = new batch_request(PBS_BATCH_StatusJob); fail_unless(preq != NULL); strcpy(preq->rq_ind.rq_status.rq_id, "1234"); strcpy(server_name, "hosta"); return_addr = true; local_connect = true; - rc = issue_to_svr(server_name, &preq, NULL); + rc = issue_to_svr(server_name, preq, NULL); fail_unless(rc==PBSE_NONE); - // local connect should cause preq to be set to NULL - fail_unless(preq == NULL); return_addr = false; - preq = alloc_br(PBS_BATCH_StatusJob); - rc = issue_to_svr(server_name, &preq, NULL); + preq = new batch_request(PBS_BATCH_StatusJob); + rc = issue_to_svr(server_name, preq, NULL); fail_unless(rc==PBSE_NONE); return_addr = true; local_connect = false; net_rc_retry = false; - rc = issue_to_svr(server_name, &preq, NULL); + rc = issue_to_svr(server_name, preq, NULL); fail_unless(rc==PBSE_NONE); net_rc_retry = true; - rc = issue_to_svr(server_name, &preq, NULL); + rc = issue_to_svr(server_name, preq, NULL); fail_unless(rc==PBSE_NONE); local_connect = false; net_rc_retry = false; connect_error = true; - rc = issue_to_svr(server_name, &preq, NULL); + rc = issue_to_svr(server_name, preq, NULL); fail_unless(rc==PBSE_INTERNAL); } @@ -282,7 +279,7 @@ START_TEST(test_reissue_to_svr); reissue_to_svr(pwt); /* Schedule a task. Success */ - preq = alloc_br(PBS_BATCH_TrackJob); + preq = new batch_request(PBS_BATCH_TrackJob); fail_unless(preq != NULL); preq->rq_ind.rq_track.rq_hopcount = 1; strcpy(preq->rq_ind.rq_track.rq_jid, "1234"); @@ -293,7 +290,7 @@ START_TEST(test_reissue_to_svr); preq->rq_perm = ATR_DFLAG_MGRD | ATR_DFLAG_MGWR | ATR_DFLAG_SvWR; get_batch_request_id(preq); - pwt = set_task(WORK_Timed, (time(NULL) + PBS_NET_RETRY_TIME), reissue_to_svr, preq->rq_id, TRUE); + pwt = set_task(WORK_Timed, (time(NULL) + PBS_NET_RETRY_TIME), reissue_to_svr, strdup(preq->rq_id.c_str()), TRUE); fail_unless(pwt != NULL); reissue_to_svr(pwt); diff --git a/src/test/job/test_uut.c b/src/test/job/test_uut.c index 95921f525c..729c4ad6e8 100644 --- a/src/test/job/test_uut.c +++ b/src/test/job/test_uut.c @@ -62,6 +62,13 @@ START_TEST(test_plugin_things) fail_unless(acct_data.find("resources_used.breaths=10000") != std::string::npos); fail_unless(acct_data.find("resources_used.aons_drawn=20") != std::string::npos); + job pjob2; + Json::Value p2; + p2["bob"]; + p2["name"] = "tom"; + pjob2.set_plugin_resource_usage_from_json(p2); + fail_unless(pjob2.number_of_plugin_resources() == 1); + } END_TEST diff --git a/src/test/job_array/test_uut.c b/src/test/job_array/test_uut.c index 8bb5fc2ff0..9d9af011df 100644 --- a/src/test/job_array/test_uut.c +++ b/src/test/job_array/test_uut.c @@ -1,8 +1,9 @@ #include "license_pbs.h" /* See here for the software license */ #include +#include #include - +#include "pbs_job.h" #include "pbs_error.h" #include "array.h" #include @@ -72,6 +73,9 @@ START_TEST(test_set_slot_limit) request = strdup("0-99%2"); fail_unless(pa.set_slot_limit(request) == PBSE_NONE); fail_unless(pa.ai_qs.slot_limit == 2); + + // make sure request not modified + fail_unless(strcmp(request, "0-99%2") == 0); max_slot = 1; request = strdup("0-99%2"); @@ -113,7 +117,7 @@ START_TEST(test_parse_array_request) // submit a valid array array_size = 10000; - fail_unless(pa.parse_array_request("0-9") == PBSE_NONE); + fail_unless(pa.parse_array_request("0-9%1") == PBSE_NONE); fail_unless(pa.ai_qs.array_size == 10); fail_unless(pa.ai_qs.num_jobs == 10); fail_unless(pa.ai_qs.range_str == "0-9"); @@ -139,6 +143,42 @@ START_TEST(test_parse_array_request) END_TEST +START_TEST(test_mark_end_of_subjob) + { + job_array pa; + array_size = 1000; + + pa.parse_array_request("0-9"); + for (int i = 0; i < 10; i++) + pa.create_job_if_needed(); + + fail_unless(pa.ai_qs.highest_id_created == 9); + + fail_unless(pa.mark_end_of_subjob(NULL) == false); + + job pjob; + + // Make sure a bad index doesn't crash + pjob.ji_wattr[JOB_ATR_job_array_id].at_val.at_long = -1; + fail_unless(pa.mark_end_of_subjob(&pjob) == false); + + for (int i = 0; i < 10; i++) + { + pjob.ji_wattr[JOB_ATR_job_array_id].at_val.at_long = i; + + if (i == 9) + fail_unless(pa.mark_end_of_subjob(&pjob) == true); + else + fail_unless(pa.mark_end_of_subjob(&pjob) == false); + } + + // Make sure a repeated index doesn't crash + pjob.ji_wattr[JOB_ATR_job_array_id].at_val.at_long = 1; + fail_unless(pa.mark_end_of_subjob(&pjob) == false); + } +END_TEST + + START_TEST(test_initialize_uncreated_ids) { job_array pa; @@ -203,6 +243,7 @@ Suite *job_array_suite(void) tcase_add_test(tc_core, update_array_values_test); tcase_add_test(tc_core, test_set_slot_limit); tcase_add_test(tc_core, test_initialize_uncreated_ids); + tcase_add_test(tc_core, test_mark_end_of_subjob); suite_add_tcase(s, tc_core); return s; diff --git a/src/test/job_container/test_uut.c b/src/test/job_container/test_uut.c index 0b09547874..e9e9997fd4 100644 --- a/src/test/job_container/test_uut.c +++ b/src/test/job_container/test_uut.c @@ -5,17 +5,57 @@ #include "pbs_error.h" #include -char *get_correct_jobname(const char *jobid); +const char *get_correct_jobname(const char *jobid, std::string &correct); void log_err(int,const char *,const char *) {} +START_TEST(iterator_test) + { + all_jobs alljobs; + struct job *jobs[5]; + + for (int i = 0; i < 5; i++) + { + jobs[i] = job_alloc(); + sprintf(jobs[i]->ji_qs.ji_jobid, "%d.threnody", i); + insert_job(&alljobs, jobs[i]); + } + + // get a reverse iterator + all_jobs_iterator *iter = alljobs.get_iterator(true); + for (int i = 4; i >= 0; i--) + { + char buf[1024]; + job *ptr = iter->get_next_item(); + sprintf(buf, "%d.threnody", i); + fail_unless(ptr != NULL); + fail_unless(!strcmp(ptr->ji_qs.ji_jobid, buf), + "Expected %s, but got %s", buf, ptr->ji_qs.ji_jobid); + } + + // get a forwards iterator + iter = alljobs.get_iterator(); + for (int i = 0; i < 5; i++) + { + char buf[1024]; + job *ptr = iter->get_next_item(); + sprintf(buf, "%d.threnody", i); + fail_unless(ptr != NULL); + fail_unless(!strcmp(ptr->ji_qs.ji_jobid, buf), + "Expected %s, but got %s", buf, ptr->ji_qs.ji_jobid); + } + } +END_TEST + + START_TEST(get_correct_jobname_test) { // with nothing set, get_correct_jobname should just return the jobid passed in. - char *jobid = get_correct_jobname("1.napali.ac"); - fail_unless(!strcmp(jobid, "1.napali.ac")); + std::string correct; + get_correct_jobname("1.napali.ac", correct); + fail_unless(correct == "1.napali.ac"); } END_TEST @@ -295,6 +335,7 @@ Suite *job_container_suite(void) tc_core = tcase_create("find_job_by_array_with_removed_record"); tcase_add_test(tc_core, find_job_by_array_with_removed_record_test); + tcase_add_test(tc_core, iterator_test); suite_add_tcase(s, tc_core); return(s); diff --git a/src/test/job_func/scaffolding.c b/src/test/job_func/scaffolding.c index 51a4626a65..886832c64e 100644 --- a/src/test/job_func/scaffolding.c +++ b/src/test/job_func/scaffolding.c @@ -531,12 +531,11 @@ ssize_t read_ac_socket(int fd, void *buf, ssize_t count) return(0); } -struct batch_request *alloc_br(int type) +batch_request::batch_request( + + int type) : rq_type(type) + { - struct batch_request *request = NULL; - request = (batch_request *)calloc(1, sizeof(struct batch_request)); - request->rq_type = type; - return request; } void free_str(struct pbs_attribute *attr) {} @@ -715,3 +714,11 @@ bool job_array::is_deleted() const { return(this->being_deleted); } + +bool job_array::mark_end_of_subjob(job *pjob) + { + return(false); + } + +batch_request::~batch_request() {} + diff --git a/src/test/job_func/test_uut.c b/src/test/job_func/test_uut.c index 537d6067db..41595192d1 100644 --- a/src/test/job_func/test_uut.c +++ b/src/test/job_func/test_uut.c @@ -307,7 +307,7 @@ START_TEST(cpy_checkpoint_test) test_job, JOB_ATR_checkpoint_name, CKPT_DIR_IN); - struct batch_request *initial = alloc_br(/*PBS_BATCH_CheckpointJob*/0); + struct batch_request *initial = new batch_request(0); fail_unless(result == NULL, "NULL batch_request input fail"); result = cpy_checkpoint(initial, @@ -366,8 +366,9 @@ START_TEST(svr_job_purge_test) test_job = job_alloc(); test_job->ji_qs.ji_substate = JOB_SUBSTATE_QUEUED; test_job->ji_qs.ji_state = JOB_STATE_QUEUED; + memset(test_job->ji_arraystructid, 0, sizeof(test_job->ji_arraystructid)); result = svr_job_purge(test_job); - fail_unless(result == 0, "non-queued job fail", result); + fail_unless(result == 0, "non-queued job fail: %d", result); // called_remove_job once means we didn't call job_free fail_unless(called_remove_job == 1); diff --git a/src/test/job_recov/scaffolding.c b/src/test/job_recov/scaffolding.c index 27eadb63a7..b067a326c9 100644 --- a/src/test/job_recov/scaffolding.c +++ b/src/test/job_recov/scaffolding.c @@ -283,12 +283,15 @@ int set_tokens(pbs_attribute *attr, pbs_attribute *newAttr, enum batch_op op){re int comp_ll(struct pbs_attribute *attr, struct pbs_attribute *with) {return 0;} int decode_b(pbs_attribute *patr, const char *name, const char *rn, const char *val, int) {return 0;} int nextjobnum_chk(pbs_attribute *pattr, void *pobject, int actmode) {return 0;} -struct batch_request *alloc_br(int type) {return NULL;} int svr_chk_owner(struct batch_request *preq, job *pjob) {return 0;} int comp_checkpoint(pbs_attribute *attr, pbs_attribute *with) {return 0;} -batch_request *get_remove_batch_request(char *br_id) {return NULL;} long calc_job_cost(job *pjob) {return(0);} -int issue_to_svr(const char *servern, struct batch_request **preq, void (*replyfunc)(struct work_task *)) {return 0;} + +int issue_to_svr(const char *servern, batch_request *preq, void (*replyfunc)(struct work_task *)) + { + return 0; + } + int que_to_local_svr(struct batch_request *preq) {return 0;} int job_set_wait(pbs_attribute *pattr, void *pjob, int mode) {return 0;} int get_batch_request_id(batch_request *preq) {return 0;} @@ -320,9 +323,10 @@ const char *id_map::get_name(int internal_job_id) id_map job_mapper; -char *get_correct_jobname(const char *id) +const char *get_correct_jobname(const char *id, std::string &correct) { - return(strdup(id)); + correct = id; + return(correct.c_str()); } int encode_complete_req( @@ -530,7 +534,31 @@ bool job_array::is_deleted() const return(this->being_deleted); } +bool job_array::mark_end_of_subjob(job *pjob) + { + return(false); + } + int check_default_gpu_mode_str(pbs_attribute *pattr, void *pobj, int mode) { return(PBSE_NONE); } + +batch_request::~batch_request() {} + +batch_request::batch_request(int type) : rq_type(type) + { + } + +batch_request *get_remove_batch_request( + + const char *br_id) + + { + return(NULL); + } + +long time_str_to_seconds(const char *str) + { + return(0); + } diff --git a/src/test/machine/scaffolding.c b/src/test/machine/scaffolding.c index 4c41501ddb..59a07f3a32 100644 --- a/src/test/machine/scaffolding.c +++ b/src/test/machine/scaffolding.c @@ -68,6 +68,11 @@ hwloc_uint64_t Socket::getMemory() const return(sock_mem); } +hwloc_uint64_t Socket::getAvailableMemory() const + { + return(sock_mem); + } + hwloc_uint64_t Socket::get_memory_for_completely_free_chips( unsigned long diff, @@ -88,6 +93,11 @@ Socket::Socket(const Json::Value &json_layout, std::vector &valid_i json_socket++; } +Socket::Socket(const std::string &json_layout, std::vector &valid_ids) + { + json_socket++; + } + Socket::Socket() { } @@ -141,8 +151,8 @@ bool Socket::spread_place( req &r, allocation &master, - int execution_slots_per, - int &execution_slots_remainder, + allocation &remaining, + allocation &remainder, bool chips) { @@ -207,7 +217,7 @@ int Socket::getAvailableChips() const return(1); } -float Socket::how_many_tasks_fit(const req &r, int place_type) const +double Socket::how_many_tasks_fit(const req &r, int place_type) const { return(num_tasks_fit); @@ -242,11 +252,23 @@ bool Socket::is_available() const void Socket::displayAsJson(Json::Value &out, bool jobs) const {} -unsigned long req::getMemory() const +void Socket::save_allocations(const Socket &other) {} + +int Socket::get_total_gpus() const + { + return(0); + } + +unsigned long long req::get_memory_per_task() const { return(req_mem); } +int req::getTaskCount() const + { + return(1); + } + int req::getPlaceCores() const { return(my_core_count); @@ -346,7 +368,7 @@ void allocation::set_place_type(const std::string &place) this->place_type = my_placement_type; } -void allocation::place_indices_in_string(std::string &out, int which) {} +void allocation::place_indices_in_string(cgroup_info &cgi) {} int allocation::add_allocation(allocation const &a) { @@ -363,6 +385,8 @@ allocation &allocation::operator =(const allocation &other) return(*this); } +void allocation::adjust_for_spread(unsigned int quantity, bool find_remainder) {} + PCI_Device::~PCI_Device() {} PCI_Device::PCI_Device() {} PCI_Device::PCI_Device(const PCI_Device &other) {} @@ -378,7 +402,7 @@ int req::getMics() const return(0); } -int req::getGpus() const +int req::get_gpus() const { return(0); } diff --git a/src/test/machine/test_uut.c b/src/test/machine/test_uut.c index ce67b5fd99..3846bcc3a9 100644 --- a/src/test/machine/test_uut.c +++ b/src/test/machine/test_uut.c @@ -70,17 +70,18 @@ START_TEST(test_place_all_execution_slots) { Machine m; job pjob; - std::string cpu; - std::string mem; + cgroup_info cgi; complete_req cr; pjob.ji_wattr[JOB_ATR_req_information].at_val.at_ptr = &cr; + my_req_count = 1; m.addSocket(2); sockets = 0; numa_node_count = 0; placed_all = 0; exec_slots = -1; - m.place_job(&pjob, cpu, mem, "napali", false); + num_for_host = 1; + m.place_job(&pjob, cgi, "napali", false); fail_unless(placed_all == 2, "placed all: %d", placed_all); } END_TEST @@ -90,13 +91,13 @@ START_TEST(test_spread_place) { Machine m; job pjob; - std::string cpu; - std::string mem; + cgroup_info cgi; complete_req cr; pjob.ji_wattr[JOB_ATR_req_information].at_val.at_ptr = &cr; m.addSocket(2); + my_req_count = 1; sockets = 1; my_placement_type = 2; numa_node_count = 0; @@ -105,12 +106,12 @@ START_TEST(test_spread_place) spreaded = true; // Make sure we call spread place once for each successfully placed task - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_spread_place == 1, "called %d", called_spread_place); num_for_host = 3; called_spread_place = 0; - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_spread_place == 3); // Now we're multiple instead of one so it should multiply the calls @@ -119,7 +120,7 @@ START_TEST(test_spread_place) numa_node_count = 2; num_for_host = 2; called_spread_place = 0; - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_spread_place == 4, "called %d times", called_spread_place); sockets = 2; @@ -127,7 +128,7 @@ START_TEST(test_spread_place) numa_node_count = 0; num_for_host = 3; called_spread_place = 0; - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_spread_place == 6); called_spread_place = 0; @@ -138,7 +139,7 @@ START_TEST(test_spread_place) // the req is set to need 10 memory and each socket is set to only have 5, so even // though we set the req to need one socket, we should get two - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_spread_place == 2, "called %d times", called_spread_place); req_mem = 0; @@ -218,13 +219,13 @@ START_TEST(test_store_pci_device_on_appropriate_chip) // Since this is non numa it should be forced to socket 0 m.setIsNuma(false); - m.store_device_on_appropriate_chip(d); + m.store_device_on_appropriate_chip(d, true); fail_unless(called_store_pci == 1); // Since this is numa it should place on the sockets until it returns true, // which due to the scaffolding is never, so once per socket m.setIsNuma(true); - m.store_device_on_appropriate_chip(d); + m.store_device_on_appropriate_chip(d, false); fail_unless(called_store_pci == 3); } END_TEST @@ -306,8 +307,7 @@ END_TEST START_TEST(test_place_and_free_job) { - std::string cpu; - std::string mem; + cgroup_info cgi; Machine m; m.addSocket(2); job pjob; @@ -328,7 +328,7 @@ START_TEST(test_place_and_free_job) num_for_host = 4; num_tasks_fit = 4; num_placed = 4; - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_place_task == 2, "Expected 2 calls but got %d", called_place_task); std::vector job_ids; @@ -347,7 +347,7 @@ START_TEST(test_place_and_free_job) num_tasks_fit = 4; num_placed = 4; called_place_task = 0; - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_place_task == 2, "Expected 2 calls but got %d", called_place_task); num_tasks_fit = 0; @@ -358,7 +358,7 @@ START_TEST(test_place_and_free_job) partially_placed = true; called_partially_place = 0; called_fits_on_socket = 0; - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_partially_place == 1, "called %d", called_partially_place); fail_unless(called_fits_on_socket == 1); @@ -368,7 +368,7 @@ START_TEST(test_place_and_free_job) my_req_count = 1; num_for_host = 1; - m.place_job(&pjob, cpu, mem, "napali", false); + m.place_job(&pjob, cgi, "napali", false); fail_unless(called_partially_place == 2, "called %d", called_partially_place); fail_unless(called_fits_on_socket == 3, "called %d times", called_fits_on_socket); } diff --git a/src/test/mail_throttler/scaffolding.c b/src/test/mail_throttler/scaffolding.c index 8b13789179..e7b3961f9f 100644 --- a/src/test/mail_throttler/scaffolding.c +++ b/src/test/mail_throttler/scaffolding.c @@ -1 +1,10 @@ +#include + +#include "pbs_job.h" + +int pthread_mutex_lock(pthread_mutex_t *mutex) throw() + { return(0); } + +job::job() {} +job::~job() {} diff --git a/src/test/mail_throttler/test_uut.c b/src/test/mail_throttler/test_uut.c index 3761a4e8e0..fc597827ac 100644 --- a/src/test/mail_throttler/test_uut.c +++ b/src/test/mail_throttler/test_uut.c @@ -1,11 +1,53 @@ #include #include -#include #include "mail_throttler.hpp" #include "pbs_error.h" +#include "resource.h" + +#include +START_TEST(test_job_constructor) + { + job pjob; + const char *jname = "highstorm_predictor"; + const char *owner = "dbeer@roshar"; + const char *queue = "batch"; + const char *jid = "1.nalthis"; + const char *hlist = "scadrial:ppn=16"; + const char *join = "oe"; + const char *outpath = "/home/dbeer/highstorms/"; + + std::vector resources; + + pjob.ji_wattr[JOB_ATR_jobname].at_val.at_str = strdup(jname); + pjob.ji_wattr[JOB_ATR_job_owner].at_val.at_str = strdup(owner); + pjob.ji_wattr[JOB_ATR_exec_host].at_val.at_str = strdup(hlist); + pjob.ji_wattr[JOB_ATR_join].at_flags = ATR_VFLAG_SET; + pjob.ji_wattr[JOB_ATR_join].at_val.at_str = strdup(join); + pjob.ji_wattr[JOB_ATR_outpath].at_val.at_str = strdup(outpath); + pjob.ji_wattr[JOB_ATR_init_work_dir].at_flags = ATR_VFLAG_SET; + pjob.ji_wattr[JOB_ATR_init_work_dir].at_val.at_str = strdup(outpath); + pjob.ji_wattr[JOB_ATR_resource].at_val.at_ptr = &resources; + pjob.ji_wattr[JOB_ATR_resc_used].at_val.at_ptr = &resources; + + strcpy(pjob.ji_qs.ji_queue, queue); + strcpy(pjob.ji_qs.ji_jobid, jid); + + mail_info mi(&pjob); + + fail_unless(mi.errFile == outpath); + fail_unless(mi.outFile == outpath); + fail_unless(mi.jobid == jid); + fail_unless(mi.exec_host == hlist); + fail_unless(mi.jobname == jname); + fail_unless(mi.queue_name == queue); + fail_unless(mi.owner == owner); + fail_unless(mi.working_directory == outpath, "Got '%s', expected '%s'", mi.working_directory.c_str(), outpath); + } +END_TEST + START_TEST(test_adding) { @@ -52,12 +94,6 @@ START_TEST(test_adding) END_TEST -START_TEST(test_two) - { - } -END_TEST - - Suite *mail_throttler_suite(void) { Suite *s = suite_create("mail_throttler test suite methods"); @@ -65,8 +101,8 @@ Suite *mail_throttler_suite(void) tcase_add_test(tc_core, test_adding); suite_add_tcase(s, tc_core); - tc_core = tcase_create("test_two"); - tcase_add_test(tc_core, test_two); + tc_core = tcase_create("test_job_constructor"); + tcase_add_test(tc_core, test_job_constructor); suite_add_tcase(s, tc_core); return(s); diff --git a/src/test/mom_comm/scaffolding.c b/src/test/mom_comm/scaffolding.c index 09cf622d7d..023150a209 100644 --- a/src/test/mom_comm/scaffolding.c +++ b/src/test/mom_comm/scaffolding.c @@ -114,14 +114,15 @@ int exec_job_on_ms(job *pjob) } struct passwd *check_pwd_return; -bool check_pwd(job *pjob) + +int check_pwd(job *pjob) { - bool good = false; - if(check_pwd_return == NULL) + int rc = -1; + if (check_pwd_return == NULL) { - good = true; + rc = PBSE_NONE; } - return(good); + return(rc); } int mom_do_poll(job *pjob) @@ -676,3 +677,5 @@ void authorized_hosts::list_authorized_hosts(std::string &output) {} authorized_hosts::authorized_hosts() {} authorized_hosts auth_hosts; +void mom_deljob(job *pjob) {} + diff --git a/src/test/mom_comm/test_uut.c b/src/test/mom_comm/test_uut.c index 958121783d..31d8632cf1 100644 --- a/src/test/mom_comm/test_uut.c +++ b/src/test/mom_comm/test_uut.c @@ -41,6 +41,8 @@ int handle_im_poll_job_response(struct tcp_chan *chan, job &pjob, int nodeidx, h received_node *get_received_node_entry(char *str); bool is_nodeid_on_this_host(job *pjob, tm_node_id nodeid); task *find_task_by_pid(job *pjob, int pid); +int readit(int, int); +bool is_mother_superior(hnodent *np); #ifdef PENABLE_LINUX_CGROUPS int get_req_and_task_index_from_local_rank(job *pjob, int local_rank, unsigned int &req_index, unsigned int &task_index); @@ -49,18 +51,6 @@ extern bool per_task; #endif -START_TEST(test_get_reply_stream) - { - job pjob; - pjob.ji_hosts = NULL; - - // Make sure we don't segfault - fail_unless(get_reply_stream(NULL) == -1); - fail_unless(get_reply_stream(&pjob) == -1); - } -END_TEST - - #ifdef PENABLE_LINUX_CGROUPS START_TEST(test_get_req_and_task_index_from_local_rank) { @@ -87,6 +77,30 @@ END_TEST #endif +START_TEST(test_is_mother_superior) + { + hnodent hn; + + hn.hn_node = 1; + fail_unless(is_mother_superior(&hn) == false); + hn.hn_node = 0; + fail_unless(is_mother_superior(&hn) == true); + } +END_TEST + + +START_TEST(test_get_reply_stream) + { + job pjob; + pjob.ji_hosts = NULL; + + // Make sure we don't segfault + fail_unless(get_reply_stream(NULL) == -1); + fail_unless(get_reply_stream(&pjob) == -1); + } +END_TEST + + START_TEST(test_find_task_by_pid) { job pjob; @@ -580,6 +594,13 @@ START_TEST(get_stat_update_interval_test) } END_TEST +START_TEST(test_readit) + { + // readset uninitialized so expect failure + fail_unless(readit(0, 0) == -2); + } +END_TEST + Suite *mom_comm_suite(void) { Suite *s = suite_create("mom_comm_suite methods"); @@ -638,6 +659,11 @@ Suite *mom_comm_suite(void) #endif suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_readit"); + tcase_add_test(tc_core, test_readit); + tcase_add_test(tc_core, test_is_mother_superior); + suite_add_tcase(s, tc_core); + return(s); } diff --git a/src/test/mom_inter/scaffolding.c b/src/test/mom_inter/scaffolding.c index 9a3a295b0e..a4cc3d5dde 100644 --- a/src/test/mom_inter/scaffolding.c +++ b/src/test/mom_inter/scaffolding.c @@ -18,7 +18,7 @@ ssize_t read_blocking_socket(int fd, void *buf, ssize_t count) return(0); } -void port_forwarder( struct pfwdsock *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, char *EMsg) +void port_forwarder(std::vector *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, char *EMsg) { fprintf(stderr, "The call to port_forwarder needs to be mocked!!\n"); exit(1); diff --git a/src/test/mom_mach/scaffolding.c b/src/test/mom_mach/scaffolding.c index 1869108e87..e8ff8aa547 100644 --- a/src/test/mom_mach/scaffolding.c +++ b/src/test/mom_mach/scaffolding.c @@ -26,6 +26,7 @@ extern std::string cg_memory_path; char mom_alias[PBS_MAXHOSTNAME + 1]; +int ServerStatUpdateInterval = 45; Machine this_node; std::string cg_cpuacct_path; std::list alljobs_list; @@ -57,6 +58,8 @@ int numa_index; #endif int job_oom_score_adjust = 0; /* no oom score adjust by default */ int mom_oom_immunize = 0; /* make pbs_mom processes immune? no by default */ +unsigned long max_memory = 0; +unsigned long max_swap = 0; void *get_next_return_value = NULL; diff --git a/src/test/mom_mach/test_uut.c b/src/test/mom_mach/test_uut.c index 618fc15261..ac09220bf7 100644 --- a/src/test/mom_mach/test_uut.c +++ b/src/test/mom_mach/test_uut.c @@ -10,6 +10,8 @@ #include "test_mom_mach.h" #include "pbs_job.h" #include "pbs_error.h" +#include "mom_memory.h" +#include "resmon.h" std::string cg_memory_path; @@ -20,6 +22,10 @@ int overmem_proc(job*, unsigned long long); int overcpu_proc(job*, unsigned long); unsigned long long resi_sum(job*); unsigned long long mem_sum(job*); +proc_mem_t *get_proc_mem(); +const char *physmem (struct rm_attribute *); +const char *totmem (struct rm_attribute *); + double cputfactor; @@ -33,6 +39,50 @@ extern proc_stat_t *proc_array; extern void *get_next_return_value; +extern unsigned long max_memory; +extern unsigned long max_swap; + + +START_TEST(memory_tests) + { + proc_mem_t *pm = get_proc_mem(); + const char *mem_str; + char buf[1024]; + rm_attribute ra; + + if (pm != NULL) + { + // Unset these parameters + max_memory = 0; + max_swap = 0; + + mem_str = physmem(&ra); + fail_unless(mem_str != NULL); + sprintf(buf, "%llukb", pm->mem_total >> 10); + fail_unless(!strcmp(buf, mem_str)); + + mem_str = totmem(&ra); + fail_unless(mem_str != NULL); + sprintf(buf, "%llukb", (pm->mem_total >> 10) + (pm->swap_total >> 10)); + fail_unless(!strcmp(buf, mem_str)); + + // Make max_memory and max_swap half of the machine's memory and swap + max_memory = pm->mem_total >> 11; + max_swap = pm->swap_total >> 11; + mem_str = physmem(&ra); + fail_unless(mem_str != NULL); + sprintf(buf, "%lukb", max_memory); + fail_unless(!strcmp(buf, mem_str)); + + mem_str = totmem(&ra); + fail_unless(mem_str != NULL); + sprintf(buf, "%lukb", max_memory + max_swap); + fail_unless(!strcmp(buf, mem_str)); + } + } +END_TEST + + START_TEST(test_get_job_sid_from_pid) { int index = 0; @@ -482,6 +532,7 @@ Suite *mom_mach_suite(void) tc_core = tcase_create("test_mem_sum"); tcase_add_test(tc_core, test_mem_sum); + tcase_add_test(tc_core, memory_tests); suite_add_tcase(s, tc_core); return s; diff --git a/src/test/mom_main/scaffolding.c b/src/test/mom_main/scaffolding.c index 46b6c37542..0f784c92bc 100644 --- a/src/test/mom_main/scaffolding.c +++ b/src/test/mom_main/scaffolding.c @@ -89,6 +89,7 @@ struct config *config_array = NULL; char **maskclient = NULL; /* wildcard connections */ char PBSNodeCheckPath[MAXLINE]; int CheckPollTime; +int varattr_tv; char rcp_args[MAXPATHLEN]; long log_file_max_size; char mom_host[PBS_MAXHOSTNAME + 1]; @@ -1136,3 +1137,17 @@ void authorized_hosts::add_authorized_address(unsigned long addr, unsigned short authorized_hosts::authorized_hosts() {} authorized_hosts auth_hosts; + +int read_all_devices() + { + return(PBSE_NONE); + } + +bool get_cray_taskstats; + +void read_rur_stats_file(const char *basepath) {} + +int get_local_address(struct sockaddr_in &new_sockaddr) + { + return PBSE_NONE; + } diff --git a/src/test/mom_main/test_uut.c b/src/test/mom_main/test_uut.c index 08ec2061e7..8ce2651af4 100644 --- a/src/test/mom_main/test_uut.c +++ b/src/test/mom_main/test_uut.c @@ -22,16 +22,19 @@ extern bool ForceServerUpdate; extern int MOMCudaVisibleDevices; extern bool daemonize_mom; +int PBSNodeCheckProlog; +int PBSNodeCheckEpilog; time_t pbs_tcp_timeout; unsigned long setcudavisibledevices(const char *); void set_report_mom_cuda_visible_devices(std::stringstream &output, char *curr); void read_mom_hierarchy(); int parse_integer_range(const char *range_str, int &start, int &end); -time_t calculate_select_timeout(); +time_t calculate_poll_timeout(); int process_layout_request(tcp_chan *chan); bool should_resend_obit(job *pjob, int diff); void check_job_in_mom_wait(job *pjob); void evaluate_job_in_prerun(job *pjob); +void add_diag_header(std::stringstream &output); extern attribute_def job_attr_def[]; extern int exiting_tasks; @@ -45,6 +48,10 @@ extern int flush_ret; extern int job_bailed; extern bool am_i_ms; +extern char mom_host[PBS_MAXHOSTNAME + 1]; +extern char mom_short_name[PBS_MAXHOSTNAME + 1]; +extern char mom_ipaddr[INET_ADDRSTRLEN]; + bool are_we_forking() { @@ -342,7 +349,7 @@ END_TEST /** - * time_t calculate_select_timeout(void) + * time_t calculate_poll_timeout(void) * Input: * (G) time_t time_now - periodically updated current time. * (G) time_t wait_time - constant systme-dependent value. @@ -351,7 +358,7 @@ END_TEST * (G) time_t last_poll_time - last poll timestamp. * (G) int CheckPollTime - poll interval. */ -START_TEST(calculate_select_timeout_test) +START_TEST(calculate_poll_timeout_test) { ForceServerUpdate = false; @@ -362,7 +369,7 @@ START_TEST(calculate_select_timeout_test) ServerStatUpdateInterval = 20; /* 10 seconds till the next status update */ last_poll_time = 90; CheckPollTime = 30; /* 10 seconds till the next poll */ - fail_unless(calculate_select_timeout() == 9); + fail_unless(calculate_poll_timeout() == 9); /* status update is minimum */ time_now = 110; @@ -371,7 +378,7 @@ START_TEST(calculate_select_timeout_test) ServerStatUpdateInterval = 19; /* 9 seconds till the next status update */ last_poll_time = 90; CheckPollTime = 30; /* 10 seconds till the next poll */ - fail_unless(calculate_select_timeout() == 9); + fail_unless(calculate_poll_timeout() == 9); /* poll is minimum */ time_now = 110; @@ -380,7 +387,7 @@ START_TEST(calculate_select_timeout_test) ServerStatUpdateInterval = 20; /* 10 seconds till the next status update */ last_poll_time = 90; CheckPollTime = 29; /* 9 seconds till the next poll */ - fail_unless(calculate_select_timeout() == 9); + fail_unless(calculate_poll_timeout() == 9); /* LastServerUpdateTime is zero */ time_now = 110; @@ -389,7 +396,7 @@ START_TEST(calculate_select_timeout_test) ServerStatUpdateInterval = 20; last_poll_time = 90; CheckPollTime = 30; /* 10 seconds till the next poll */ - fail_unless(calculate_select_timeout() == 1); + fail_unless(calculate_poll_timeout() == 1); /* status update is minimum and was needed to be sent some time ago */ time_now = 110; @@ -398,7 +405,7 @@ START_TEST(calculate_select_timeout_test) ServerStatUpdateInterval = 20; /* -1 seconds till the next status update */ last_poll_time = 90; CheckPollTime = 30; /* 10 seconds till the next poll */ - fail_unless(calculate_select_timeout() == 1); + fail_unless(calculate_poll_timeout() == 1); /* poll is minimum and was needed to be sent some time ago */ time_now = 110; @@ -407,7 +414,7 @@ START_TEST(calculate_select_timeout_test) ServerStatUpdateInterval = 20; /* 10 seconds till the next status update */ last_poll_time = 79; CheckPollTime = 30; /* -1 seconds till the next poll */ - fail_unless(calculate_select_timeout() == 1); + fail_unless(calculate_poll_timeout() == 1); } END_TEST @@ -448,6 +455,24 @@ START_TEST(test_parse_command_line3) END_TEST +START_TEST(test_add_diag_header) + { + std::stringstream output; + std::string output_str; + + mom_ipaddr[0] = '\0'; + mom_short_name[0] = '\0'; + mom_host[0] = '\0'; + + add_diag_header(output); + + output_str = output.str(); + + fail_unless(output_str.find("IP address") != std::string::npos); + } +END_TEST + + Suite *mom_main_suite(void) { Suite *s = suite_create("mom_main_suite methods"); @@ -462,8 +487,8 @@ Suite *mom_main_suite(void) tcase_add_test(tc_core, test_check_job_in_mom_wait); suite_add_tcase(s, tc_core); - tc_core = tcase_create("calculate_select_timeout_test"); - tcase_add_test(tc_core, calculate_select_timeout_test); + tc_core = tcase_create("calculate_poll_timeout_test"); + tcase_add_test(tc_core, calculate_poll_timeout_test); tcase_add_test(tc_core, test_evaluate_job_in_prerun); suite_add_tcase(s, tc_core); @@ -484,6 +509,10 @@ Suite *mom_main_suite(void) tcase_add_test(tc_core, test_parse_command_line3); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_add_diag_header"); + tcase_add_test(tc_core, test_add_diag_header); + suite_add_tcase(s, tc_core); + return s; } diff --git a/src/test/mom_req_quejob/scaffolding.c b/src/test/mom_req_quejob/scaffolding.c index d59f57ca9b..900766a5d0 100644 --- a/src/test/mom_req_quejob/scaffolding.c +++ b/src/test/mom_req_quejob/scaffolding.c @@ -56,10 +56,9 @@ void mom_job_purge(job *pjob) exit(1); } -bool check_pwd(job *pjob) +int check_pwd(job *pjob) { - fprintf(stderr, "The call to check_pwd needs to be mocked!!\n"); - exit(1); + return(0); } int reply_send_mom(struct batch_request *request_mom) @@ -229,3 +228,6 @@ void free_pwnam(struct passwd *pwdp, char *buf) void send_update_soon() {} + +batch_request::batch_request() {} +batch_request::~batch_request() {} diff --git a/src/test/mom_req_quejob/test_uut.c b/src/test/mom_req_quejob/test_uut.c index e7e04e382b..22866c9b7b 100644 --- a/src/test/mom_req_quejob/test_uut.c +++ b/src/test/mom_req_quejob/test_uut.c @@ -15,7 +15,6 @@ START_TEST(test_mom_req_quejob) { batch_request preq; - memset(&preq, 0, sizeof(preq)); preq.rq_fromsvr = TRUE; strcpy(preq.rq_ind.rq_queuejob.rq_jid, "1.napali"); CLEAR_HEAD(preq.rq_ind.rq_queuejob.rq_attr); diff --git a/src/test/mom_server/scaffolding.c b/src/test/mom_server/scaffolding.c index 17a306cf64..5892d42262 100644 --- a/src/test/mom_server/scaffolding.c +++ b/src/test/mom_server/scaffolding.c @@ -26,6 +26,7 @@ #define MAXLINE 1024 +bool force_layout_update = false; std::list alljobs_list; char log_buffer[LOG_BUF_SIZE]; char *apbasil_protocol = NULL; diff --git a/src/test/mom_server/test_uut.c b/src/test/mom_server/test_uut.c index 7e54052eb3..fe2d2c1b2f 100644 --- a/src/test/mom_server/test_uut.c +++ b/src/test/mom_server/test_uut.c @@ -20,6 +20,7 @@ int mom_server_update_stat(mom_server *pms, std::vector &strings); char PBSNodeMsgBuf[MAXLINE]; char PBSNodeCheckPath[MAXLINE]; int PBSNodeCheckInterval = 2; +u_long pbsclient; extern bool no_error; extern bool no_event; @@ -33,7 +34,7 @@ extern int is_reporter_mom; extern mom_server mom_servers[PBS_MAXSERVER]; bool is_for_this_host(std::string gpu_spec, const char *suffix); -void get_device_indices(const char *gpu_str, std::vector &gpu_indices, const char *suffix); +void get_device_indices(const char *gpu_str, std::vector &gpu_indices, const char *suffix); START_TEST(test_sort_paths) { @@ -208,11 +209,13 @@ START_TEST(test_mom_server_all_update_stat_clear_force) is_reporter_mom = false; + mom_servers[0].MOMLastSendToServerTime = 0; LastServerUpdateTime = time(NULL) - 100; ForceServerUpdate = true; mom_server_all_update_stat(); fail_unless(ForceServerUpdate == false); + mom_servers[0].MOMLastSendToServerTime = 0; LastServerUpdateTime = time(NULL) - 100; ForceServerUpdate = true; mom_server_all_update_stat(); @@ -248,7 +251,7 @@ END_TEST START_TEST(test_get_device_indices) { std::string spec; - std::vector gpu_indices; + std::vector gpu_indices; char suffix[10]; extern char mom_alias[]; diff --git a/src/test/net_client/scaffolding.c b/src/test/net_client/scaffolding.c index 741f38a6da..f6c7ddf638 100644 --- a/src/test/net_client/scaffolding.c +++ b/src/test/net_client/scaffolding.c @@ -1,5 +1,15 @@ #include "license_pbs.h" /* See here for the software license */ #include +#include void log_err(int errnum, const char *func_id, const char *msg) {} +int global_poll_rc = 0; +short global_poll_revents = 0; + +int poll(struct pollfd *fds, nfds_t nfds, int timeout) + { + fds->revents = global_poll_revents; + + return(global_poll_rc); + } diff --git a/src/test/net_client/test_uut.c b/src/test/net_client/test_uut.c index e4714543b5..39ce006851 100644 --- a/src/test/net_client/test_uut.c +++ b/src/test/net_client/test_uut.c @@ -3,14 +3,32 @@ #include "test_net_client.h" #include #include +#include #include "pbs_error.h" -START_TEST(test_one) +bool wait_for_write_ready(int, int); + +extern int global_poll_rc; +extern short global_poll_revents; + +START_TEST(test_wait_for_write_ready) { + global_poll_rc = 0; + fail_unless(wait_for_write_ready(0, 0) == false); + + global_poll_rc = -1; + fail_unless(wait_for_write_ready(0, 0) == false); + + global_poll_rc = 1; + global_poll_revents = 0; + fail_unless(wait_for_write_ready(0, 0) == false); + global_poll_rc = 1; + global_poll_revents = POLLOUT; + fail_unless(wait_for_write_ready(0, 0) == true); } END_TEST @@ -24,8 +42,8 @@ END_TEST Suite *net_client_suite(void) { Suite *s = suite_create("net_client_suite methods"); - TCase *tc_core = tcase_create("test_one"); - tcase_add_test(tc_core, test_one); + TCase *tc_core = tcase_create("test_wait_for_write_ready"); + tcase_add_test(tc_core, test_wait_for_write_ready); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_two"); diff --git a/src/test/net_common/scaffolding.c b/src/test/net_common/scaffolding.c index a10f1b835f..e8dce86c96 100644 --- a/src/test/net_common/scaffolding.c +++ b/src/test/net_common/scaffolding.c @@ -1,11 +1,19 @@ #include "license_pbs.h" /* See here for the software license */ #include #include +#include +#include +#include bool socket_success = true; bool close_success = true; bool connect_success = true; +int global_poll_rc = 0; +short global_poll_revents = 0; +int global_getsockopt_rc = 0; +int global_sock_errno = 0; + struct addrinfo * insert_addr_name_info(struct addrinfo *pAddrInfo,const char *host) { @@ -76,4 +84,19 @@ int connect( return(-1); } +int ppoll(struct pollfd *fds, nfds_t nfds, const struct timespec *timeout, const sigset_t *sigmask) + { + fds->revents = global_poll_revents; + + return(global_poll_rc); + } + +int getsockopt(int sockfd, int level, int optname, void *optval, socklen_t *optlen) throw() + { + int *iptr; + + iptr = (int *)optval; + *iptr = global_sock_errno; + return(global_getsockopt_rc); + } diff --git a/src/test/net_common/test_uut.c b/src/test/net_common/test_uut.c index c97749fa1f..0cb7686269 100644 --- a/src/test/net_common/test_uut.c +++ b/src/test/net_common/test_uut.c @@ -4,6 +4,10 @@ #include #include #include +#include +#include +#include +#include #include "pbs_error.h" #include "net_cache.h" @@ -15,6 +19,12 @@ bool connect_success; int get_random_reserved_port(); int process_and_save_socket_error(int socket_errno); +int socket_wait_for_write(int); + +extern int global_poll_rc; +extern short global_poll_revents; +extern int global_getsockopt_rc; +extern int global_sock_errno; START_TEST(test_get_random_reserved_port) { @@ -105,6 +115,57 @@ START_TEST(test_socket_connect_unix) } END_TEST +START_TEST(test_socket_wait_for_write) + { + global_poll_rc = 0; + fail_unless(socket_wait_for_write(0) == PERMANENT_SOCKET_FAIL); + + global_poll_rc = -1; + fail_unless(socket_wait_for_write(0) == PERMANENT_SOCKET_FAIL); + + global_poll_rc = 1; + global_poll_revents = 0; + fail_unless(socket_wait_for_write(0) == PERMANENT_SOCKET_FAIL); + + global_poll_rc = 1; + global_poll_revents = POLLOUT; + global_getsockopt_rc = 1; + global_sock_errno = EALREADY; + fail_unless(socket_wait_for_write(0) == TRANSIENT_SOCKET_FAIL); + + global_poll_rc = 1; + global_poll_revents = POLLOUT; + global_getsockopt_rc = 0; + global_sock_errno = EINPROGRESS; + fail_unless(socket_wait_for_write(0) == TRANSIENT_SOCKET_FAIL); + + global_poll_rc = 1; + global_poll_revents = POLLOUT; + global_getsockopt_rc = 0; + global_sock_errno = 0; + fail_unless(socket_wait_for_write(0) == PBSE_NONE); + } +END_TEST + +START_TEST(test_get_local_address) + { + int rc; + struct sockaddr_in new_sockaddr; + char hostname[NI_MAXHOST]; + char hbuf[NI_MAXHOST]; + + gethostname(hostname, sizeof(hostname)); + + rc = get_local_address(new_sockaddr); + fail_unless(rc == PBSE_NONE); + + rc = getnameinfo((struct sockaddr*)&new_sockaddr, sizeof(new_sockaddr), hbuf, sizeof(hbuf), NULL, 0, 0); + + fail_unless(rc == 0); + fail_unless(strncmp(hostname, hbuf, NI_MAXHOST) == 0); + } +END_TEST + Suite *net_common_suite(void) { Suite *s = suite_create("net_common_suite methods"); @@ -128,6 +189,14 @@ Suite *net_common_suite(void) tcase_add_test(tc_core, test_socket_connect_unix); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_socket_wait_for_write"); + tcase_add_test(tc_core, test_socket_wait_for_write); + suite_add_tcase(s, tc_core); + + tc_core = tcase_create("test_get_local_address"); + tcase_add_test(tc_core, test_get_local_address); + suite_add_tcase(s, tc_core); + return s; } diff --git a/src/test/net_server/scaffolding.c b/src/test/net_server/scaffolding.c index b4e1c18b71..e5611732f4 100644 --- a/src/test/net_server/scaffolding.c +++ b/src/test/net_server/scaffolding.c @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include "pbs_error.h" #include "net_connect.h" @@ -14,6 +16,9 @@ bool socket_write_success = true; bool socket_read_success = true; bool socket_read_code = true; +nfds_t global_poll_nfds = 0; +int global_poll_timeout_sec = 0; + const char *msg_daemonname = "unset"; void disiui_() {} @@ -109,7 +114,6 @@ void log_event(int eventtype, int objclass, const char *objname, const char *tex void initialize_connections_table() { - fprintf(stderr, "The call to initialize_connections_table needs to be mocked!!\n"); return; } @@ -119,25 +123,13 @@ char *PAddrToString(pbs_net_t *Addr) return(0); } -int get_max_num_descriptors(void) - { - fprintf(stderr, "The call to get_max_num_descriptors needs to be mocked!!\n"); - return(0); - } - -int get_fdset_size(void) - { - fprintf(stderr, "The call to get_fdset_size needs to be mocked!!\n"); - return(0); - } - void log_err(int errnum, const char *routine, const char *text) {} void log_record(int eventtype, int objclass, const char *objname, const char *text) {} int pbs_getaddrinfo(const char *pNode,struct addrinfo *pHints,struct addrinfo **ppAddrInfoOut) { - return(0); + return(1); } char *get_cached_nameinfo(const struct sockaddr_in *sai) @@ -149,3 +141,20 @@ bool log_available(int eventtype) { return true; } + +char *netaddr_long(long ap, char *out) + { + return(NULL); + } + +int get_max_num_descriptors(void) + { + return(getdtablesize()); + } + +int ppoll(struct pollfd *fds, nfds_t nfds, const struct timespec *timeout, const sigset_t *sigmask) + { + global_poll_nfds = nfds; + global_poll_timeout_sec = timeout->tv_sec; + return(0); + } diff --git a/src/test/net_server/test_uut.c b/src/test/net_server/test_uut.c index 16414af743..a7ce05787d 100644 --- a/src/test/net_server/test_uut.c +++ b/src/test/net_server/test_uut.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "server_limits.h" @@ -21,9 +22,16 @@ extern bool socket_write_success; extern bool socket_read_success; extern bool socket_read_code; +extern nfds_t global_poll_nfds; +extern int global_poll_timeout_sec; + int add_connection(int sock, enum conn_type type, pbs_net_t addr, unsigned int port, unsigned int socktype, void *(*func)(void *), int add_wait_request); void *accept_conn(void *new_conn); +int init_network(unsigned int port, void *(*readfunc)(void *)); +void globalset_add_sock(int, u_long, u_long); +void globalset_del_sock(int); + START_TEST(netaddr_pbs_net_t_test_one) { @@ -145,6 +153,22 @@ START_TEST(test_check_trqauthd_unix_domain_port) } END_TEST +START_TEST(test_init_network) + { + int rc; + int timeout_sec = 10; + nfds_t MaxNumDescriptors = get_max_num_descriptors(); + + rc = init_network(0, NULL); + fail_unless(rc == PBSE_NONE); + + rc = wait_request(timeout_sec, NULL); + + // examine the values passed to poll() to see if they were expected + fail_unless(global_poll_nfds == MaxNumDescriptors); + fail_unless(global_poll_timeout_sec == timeout_sec); + } +END_TEST Suite *net_server_suite(void) { @@ -169,6 +193,10 @@ Suite *net_server_suite(void) tcase_add_test(tc_core, test_check_trqauthd_unix_domain_port); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_init_network"); + tcase_add_test(tc_core, test_init_network); + suite_add_tcase(s, tc_core); + return s; } diff --git a/src/test/node_func/scaffolding.c b/src/test/node_func/scaffolding.c index e1cf4600f2..a6206b7ef5 100644 --- a/src/test/node_func/scaffolding.c +++ b/src/test/node_func/scaffolding.c @@ -598,3 +598,11 @@ bool authorized_hosts::remove_address(unsigned long addr, unsigned short port) authorized_hosts::authorized_hosts() {} authorized_hosts auth_hosts; + +struct pbsnode *get_compute_node( + + const char *node_name) + + { + return(NULL); + } diff --git a/src/test/node_manager/scaffolding.c b/src/test/node_manager/scaffolding.c index 508eb765d7..f76232afc9 100644 --- a/src/test/node_manager/scaffolding.c +++ b/src/test/node_manager/scaffolding.c @@ -57,6 +57,7 @@ bool job_mode = false; int can_place = 0; pbsnode napali_node; +int create_a_gpusubnode(struct pbsnode*); struct batch_request *alloc_br(int type) { @@ -134,11 +135,20 @@ struct pbsnode *find_nodebyname(const char *nodename) { static struct pbsnode bob; static struct pbsnode other; + static struct pbsnode gpunode; static int called = 0; if (called == 0) { other.change_name("lihue"); + + create_a_gpusubnode(&gpunode); + create_a_gpusubnode(&gpunode); + create_a_gpusubnode(&gpunode); + create_a_gpusubnode(&gpunode); + create_a_gpusubnode(&gpunode); + create_a_gpusubnode(&gpunode); + called++; } @@ -153,9 +163,9 @@ struct pbsnode *find_nodebyname(const char *nodename) else if (!strcmp(nodename, "3")) return(&bob); else if (!strcmp(nodename, "lihue")) - { return(&other); - } + else if (!strcmp(nodename, "gpunode")) + return(&gpunode); else return(NULL); } @@ -537,6 +547,12 @@ pbs_net_t get_hostaddr( int create_a_gpusubnode(struct pbsnode *np) { + gpusubn tmp(np->nd_gpusn.size()); + + np->nd_gpusn.push_back(tmp); + np->nd_ngpus++; + np->nd_ngpus_free++; + return(0); } @@ -981,11 +997,10 @@ void Machine::displayAsJson(stringstream &out, bool include_jobs) const {} int Machine::place_job( - job *pjob, - string &cpu_string, - string &mem_string, - const char *hostname, - bool legacy_vmem) + job *pjob, + cgroup_info &cgi, + const char *hostname, + bool legacy_vmem) { return(0); @@ -1092,3 +1107,7 @@ pbsnode *authorized_hosts::get_authorized_node(unsigned long addr, unsigned shor authorized_hosts::authorized_hosts() {} authorized_hosts auth_hosts; + +batch_request::batch_request(int type) {} +batch_request::~batch_request() {} + diff --git a/src/test/node_manager/test_uut.c b/src/test/node_manager/test_uut.c index 71dc19c128..5d224b7f5c 100644 --- a/src/test/node_manager/test_uut.c +++ b/src/test/node_manager/test_uut.c @@ -4,14 +4,17 @@ #include #include #include +#include #include #include "license_pbs.h" /* See here for the software license */ #include "node_manager.h" #include "test_uut.h" #include "pbs_error.h" -#include "server.h" /* server */ +#include "server.h" #include "json/json.h" +#include "complete_req.hpp" +#include "pbs_nodes.h" const char *exec_hosts = "napali/0+napali/1+napali/2+napali/50+napali/4+l11/0+l11/1+l11/2+l11/3"; char buf[4096]; @@ -19,6 +22,7 @@ const char *napali = "napali"; const char *l11 = "l11"; struct server server; +void free_nodes(job *pjob, const char *spec); int kill_job_on_mom(const char *job_id, struct pbsnode *pnode); int remove_job_from_node(struct pbsnode *pnode, int internal_job_id); bool node_in_exechostlist(const char *, char *, const char *); @@ -41,6 +45,9 @@ int add_job_to_mic(struct pbsnode *pnode, int index, job *pjob); int remove_job_from_nodes_mics(struct pbsnode *pnode, job *pjob); void update_failure_counts(const char *node_name, int rc); void check_node_jobs_existence(struct work_task *pwt); +int add_job_to_gpu_subnode(pbsnode *pnode, gpusubn &gn, job *pjob); +int proplist(char **str, std::vector &plist, int *node_req, int *gpu_req, int *mic_req); +int process_gpu_token(const char*, job*); @@ -52,6 +59,61 @@ extern int decode_resc_count; extern bool conn_success; extern bool alloc_br_success; extern bool cray_enabled; +extern int gpu_mode_rqstd; + + +START_TEST(free_nodes_test) + { + job pjob; + +#ifdef PENABLE_LINUX_CGROUPS + complete_req cr; + req r; + allocation a; + pjob.ji_wattr[JOB_ATR_exec_host].at_val.at_str = NULL; + pjob.ji_wattr[JOB_ATR_login_node_id].at_val.at_str = NULL; + pjob.ji_wattr[JOB_ATR_req_information].at_flags = ATR_VFLAG_SET; + pjob.ji_wattr[JOB_ATR_req_information].at_val.at_ptr = &cr; + pjob.ji_wattr[JOB_ATR_cpuset_string].at_val.at_str = strdup("roshar:0-31"); + pjob.ji_wattr[JOB_ATR_cpuset_string].at_flags = ATR_VFLAG_SET; + pjob.ji_wattr[JOB_ATR_memset_string].at_val.at_str = strdup("roshar:0-1"); + pjob.ji_wattr[JOB_ATR_memset_string].at_flags = ATR_VFLAG_SET; + r.record_allocation(a); + cr.add_req(r); + + // We shouldn't free allocations for completed jobs + pjob.ji_qs.ji_substate = JOB_SUBSTATE_COMPLETE; + free_nodes(&pjob, "roshar:ppn=32"); + req &ref1 = cr.get_req(0); + fail_unless(ref1.get_req_allocation_count() == 1); + fail_unless(pjob.ji_wattr[JOB_ATR_cpuset_string].at_val.at_str == NULL); + fail_unless(pjob.ji_wattr[JOB_ATR_memset_string].at_val.at_str == NULL); + + // We should free allocations for jobs that failed to start + pjob.ji_qs.ji_substate = JOB_SUBSTATE_TRNOUT; + free_nodes(&pjob, "roshar:ppn=32"); + req &ref2 = cr.get_req(0); + fail_unless(ref2.get_req_allocation_count() == 0); +#endif + } +END_TEST + + +START_TEST(add_job_to_gpu_subnode_test) + { + gpusubn gn; + job pjob; + pbsnode pnode; + + pnode.nd_ngpus_to_be_used = 1; + pjob.ji_internal_id = 10; + + fail_unless(add_job_to_gpu_subnode(&pnode, gn, &pjob) == PBSE_NONE); + fail_unless(gn.inuse == true); + fail_unless(pnode.nd_ngpus_to_be_used == 0); + fail_unless(gn.job_internal_id == pjob.ji_internal_id); + } +END_TEST START_TEST(check_node_jobs_exitence_test) @@ -83,24 +145,29 @@ START_TEST(check_node_jobs_exitence_test) END_TEST #ifdef PENABLE_LINUX_CGROUPS -void save_cpus_and_memory_cpusets(job *pjob, const char *host, std::string &cpus, std::string &mems); +void save_cpus_and_memory_cpusets(job *pjob, const char *host, cgroup_info &cgi); START_TEST(test_save_cpus_and_memory_cpusets) { job *pjob = new job(); - std::string cpus("0-3"); - std::string mems("0"); + cgroup_info cgi; + cgi.cpu_string = "0-3"; + cgi.mem_string = "0"; - save_cpus_and_memory_cpusets(pjob, "napali", cpus, mems); + save_cpus_and_memory_cpusets(pjob, "napali", cgi); fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str, "napali:0-3")); fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str, "napali:0")); - save_cpus_and_memory_cpusets(pjob, "wailua", cpus, mems); + save_cpus_and_memory_cpusets(pjob, "wailua", cgi); fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str, "napali:0-3+wailua:0-3")); fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str, "napali:0+wailua:0")); - - save_cpus_and_memory_cpusets(pjob, "waimea", cpus, mems); + + cgi.gpu_string = "0-1"; + cgi.mic_string = "4"; + save_cpus_and_memory_cpusets(pjob, "waimea", cgi); fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_cpuset_string].at_val.at_str, "napali:0-3+wailua:0-3+waimea:0-3")); fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_memset_string].at_val.at_str, "napali:0+wailua:0+waimea:0")); + fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_gpus_reserved].at_val.at_str, "waimea:0-1")); + fail_unless(!strcmp(pjob->ji_wattr[JOB_ATR_mics_reserved].at_val.at_str, "waimea:4")); } END_TEST @@ -226,11 +293,7 @@ START_TEST(test_kill_job_on_mom) rc = kill_job_on_mom(job_id, &pnode); fail_unless(rc == PBSE_NONE); - alloc_br_success = false; - rc = kill_job_on_mom(job_id, &pnode); - fail_unless(rc == -1); - - alloc_br_success = true;; + alloc_br_success = true; conn_success = false; rc = kill_job_on_mom(job_id, &pnode); fail_unless(rc == -1); @@ -892,6 +955,38 @@ START_TEST(place_subnodes_in_hostlist_job_exclusive_test) } END_TEST +START_TEST(test_process_gpu_token) + { + job *pjob; + char *s; + struct pbsnode *pnode; + + pjob = (job *)calloc(1, sizeof(job)); + s = strdup("gpunode/5"); + + fail_unless(process_gpu_token(NULL, pjob) != PBSE_NONE); + fail_unless(process_gpu_token(s, NULL) != PBSE_NONE); + fail_unless(process_gpu_token(NULL, NULL) != PBSE_NONE); + + pjob->ji_internal_id = 10; + fail_unless(process_gpu_token(s, pjob) == PBSE_NONE); + + fail_unless((pnode = find_nodebyname("gpunode")) != NULL); + + fail_unless(pnode->nd_gpusn[5].job_internal_id == 10); + fail_unless(pnode->nd_gpusn[5].inuse == true); + fail_unless(pnode->nd_gpusn[5].job_count == 1); + + s = strdup("gpunode/0-2"); + fail_unless(process_gpu_token(s, pjob) == PBSE_NONE); + fail_unless(pnode->nd_gpusn[0].inuse == true); + fail_unless(pnode->nd_gpusn[0].job_count == 1); + fail_unless(pnode->nd_gpusn[1].inuse == true); + fail_unless(pnode->nd_gpusn[1].job_count == 1); + fail_unless(pnode->nd_gpusn[2].inuse == true); + fail_unless(pnode->nd_gpusn[2].job_count == 1); + } +END_TEST Suite *node_manager_suite(void) { @@ -935,6 +1030,7 @@ Suite *node_manager_suite(void) tc_core = tcase_create("place_subnodes_in_hostlist_job_exclusive_test"); tcase_add_test(tc_core, place_subnodes_in_hostlist_job_exclusive_test); + tcase_add_test(tc_core, add_job_to_gpu_subnode_test); suite_add_tcase(s, tc_core); tc_core = tcase_create("record_external_node_test"); @@ -945,6 +1041,7 @@ Suite *node_manager_suite(void) tc_core = tcase_create("more tests"); tcase_add_test(tc_core, translate_job_reservation_info_to_string_test); tcase_add_test(tc_core, test_initialize_alps_req_data); + tcase_add_test(tc_core, free_nodes_test); suite_add_tcase(s, tc_core); tc_core = tcase_create("even more tests"); @@ -953,6 +1050,10 @@ Suite *node_manager_suite(void) tcase_add_test(tc_core, check_node_jobs_exitence_test); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_process_gpu_token"); + tcase_add_test(tc_core, test_process_gpu_token); + suite_add_tcase(s, tc_core); + return(s); } diff --git a/src/test/numa_chip/scaffolding.c b/src/test/numa_chip/scaffolding.c index 0ce3f894c0..e621557af8 100644 --- a/src/test/numa_chip/scaffolding.c +++ b/src/test/numa_chip/scaffolding.c @@ -137,11 +137,16 @@ int get_machine_total_memory(hwloc_topology_t topology, unsigned long *memory) return(PBSE_NONE); } -req::req() : mem(0), cores(0), threads(0), gpus(0), mics(0) {} +req::req() : mem_per_task(0), cores(0), threads(0), gpus(0), mics(0), task_count(1) {} -unsigned long req::getMemory() const +unsigned long long req::get_memory_per_task() const { - return(this->mem); + return(this->mem_per_task); + } + +int req::getTaskCount() const + { + return(this->task_count); } int req::getPlaceCores() const @@ -175,11 +180,18 @@ int req::set_value(const char *name, const char *value, bool is_default) if (!strcmp(name, "lprocs")) this->execution_slots = atoi(value); else if (!strcmp(name, "memory")) - this->mem = atoi(value); + this->mem_per_task = strtol(value, NULL, 10); + else if (!strcmp(name, "total_memory")) + { + this->total_mem = strtol(value, NULL, 10); + this->mem_per_task = this->total_mem / this->task_count; + } else if (!strcmp(name, GPUS)) this->gpus = atoi(value); else if (!strcmp(name, MICS)) this->mics = atoi(value); + else if (!strcmp(name, "task_count")) + this->task_count = atoi(value); return(0); } @@ -189,7 +201,7 @@ int req::getMics() const return(this->mics); } -int req::getGpus() const +int req::get_gpus() const { return(this->gpus); } @@ -266,6 +278,7 @@ void setAllocJson(int test_num) alloc_json[ALLOCATIONS][0][ALLOCATION][CPUS] = "0,4,8,12"; alloc_json[ALLOCATIONS][0][ALLOCATION][EXCLUSIVE] = 3; alloc_json[ALLOCATIONS][0][ALLOCATION][CORES_ONLY] = 1; + alloc_json[GPUS] = "0-1"; return; } }//end test_num <5 diff --git a/src/test/numa_chip/test_uut.c b/src/test/numa_chip/test_uut.c index 98cd19229a..9166401b6c 100644 --- a/src/test/numa_chip/test_uut.c +++ b/src/test/numa_chip/test_uut.c @@ -16,6 +16,65 @@ extern std::string thread_type; extern Json::Value alloc_json; extern void setAllocJson(int); + +START_TEST(test_get_contiguous_thread_vector) + { + Chip c; + std::vector list; + + // Make sure a memory only node doesn't crash + fail_unless(c.getContiguousThreadVector(list, 34) == false); + + c.setId(0); + c.setThreads(32); + c.setCores(16); + c.setMemory(6); + c.setChipAvailable(true); + for (int i = 0; i < 16; i++) + c.make_core(i); + + fail_unless(c.getContiguousThreadVector(list, 34) == true); + fail_unless(list.size() == 32); + } +END_TEST + + +START_TEST(test_get_contiguous_core_vector) + { + Chip c; + std::vector list; + + // Make sure a memory only node doesn't crash + fail_unless(c.getContiguousCoreVector(list, 18) == false); + + c.setId(0); + c.setThreads(32); + c.setCores(16); + c.setMemory(6); + c.setChipAvailable(true); + for (int i = 0; i < 16; i++) + c.make_core(i); + + fail_unless(c.getContiguousCoreVector(list, 18) == true); + fail_unless(list.size() == 16); + } +END_TEST + + +START_TEST(test_initialize_cores_from_strings) + { + Chip c; + std::string cores; + std::string threads; + + // Make sure this doesn't segfault for a memory only node + c.initialize_cores_from_strings(cores, threads); + fail_unless(c.getTotalCores() == 0); + fail_unless(c.getTotalThreads() == 0); + } +END_TEST + + START_TEST(test_place_tasks_execution_slots) { const char *jobid = "1.napali"; @@ -420,22 +479,23 @@ START_TEST(test_spread_place) Json::Value out; allocation a(jobid); + allocation remaining; + allocation remainder; Chip c; c.setId(0); c.setThreads(32); c.setCores(16); c.setMemory(6); c.setChipAvailable(true); + c.set_gpus(2); for (int i = 0; i < 16; i++) c.make_core(i); - int remaining = 0; a.place_type = exclusive_chip; // Make sure we get 4 evenly spread cores - fail_unless(c.spread_place(r, a, 4, remaining) == true); - - + remaining.cpus = 4; + fail_unless(c.spread_place(r, a, remaining, remainder) == true); setAllocJson(4);//initialize alloc_json to this tests configuration c.displayAsJson(out, true); @@ -446,14 +506,19 @@ START_TEST(test_spread_place) // Make sure we do not place another task on this chip before freeing the old one fail_unless(c.reserve_core(0, a) == false); - remaining = 1; - fail_unless(c.spread_place(r, a, 4, remaining) == false); + remaining.cpus = 4; + remainder.cpus = 1; + remaining.gpus = 1; + remainder.gpus = 1; + fail_unless(c.spread_place(r, a, remaining, remainder) == false); fail_unless(c.has_socket_exclusive_allocation() == false); c.free_task(jobid); // Check that we place 5 cores correctly - fail_unless(c.spread_place(r, a, 4, remaining) == true); - fail_unless(remaining == 0, "remaining = %d", remaining); + fail_unless(c.spread_place(r, a, remaining, remainder) == true); + fail_unless(remainder.cpus == 0, "remainder = %d", remainder.cpus); + fail_unless(remainder.gpus == 0, "remainder = %d", remainder.gpus); + fail_unless(remaining.gpus == 0, "remainder = %d", remaining.gpus); fail_unless(c.reserve_core(0, a) == false); fail_unless(c.reserve_core(3, a) == false); fail_unless(c.reserve_core(6, a) == false); @@ -462,15 +527,17 @@ START_TEST(test_spread_place) c.free_task(jobid); // Make sure we get core 0 if we request 0 + 1 in the remainder - remaining = 1; - fail_unless(c.spread_place(r, a, 0, remaining) == true); + remaining.cpus = 0; + remainder.cpus = 1; + fail_unless(c.spread_place(r, a, remaining, remainder) == true); fail_unless(c.reserve_core(0, a) == false); - fail_unless(remaining == 0); + fail_unless(remainder.cpus == 0); // Check what happens we we place an empty set on the chip c.free_task(jobid); a.place_type = exclusive_socket; - fail_unless(c.spread_place(r, a, 0, remaining) == true); + remaining.cpus = 0; + fail_unless(c.spread_place(r, a, remaining, remainder) == true); fail_unless(c.getAvailableCores() == 0); fail_unless(c.getAvailableThreads() == 0); fail_unless(c.has_socket_exclusive_allocation() == true); @@ -478,9 +545,10 @@ START_TEST(test_spread_place) c.free_task(jobid); allocation a2(jobid2); a2.place_type = exclusive_chip; - remaining = 0; - fail_unless(c.spread_place(r, a2, 40, remaining) == false); - fail_unless(c.spread_place(r, a2, 18, remaining) == true); + remaining.cpus = 40; + fail_unless(c.spread_place(r, a2, remaining, remainder) == false); + remaining.cpus = 18; + fail_unless(c.spread_place(r, a2, remaining, remainder) == true); fail_unless(c.getAvailableCores() == 0); fail_unless(c.getAvailableThreads() == 0); c.free_task(jobid2); @@ -493,7 +561,8 @@ START_TEST(test_spread_place) int tasks = c.place_task(r2, a3, 1, host); fail_unless(tasks == 1); // We shouldn't place anything in spread place unless we're completely free - fail_unless(c.spread_place(r, a2, 18, remaining) == false); + remaining.cpus = 18; + fail_unless(c.spread_place(r, a2, remaining, remainder) == false); } END_TEST @@ -741,6 +810,18 @@ START_TEST(test_how_many_tasks_fit) c.setMemory(5); tasks = c.how_many_tasks_fit(r, 0); fail_unless(tasks == 5, "%d tasks fit, expected 5", tasks); + + // Make sure we handle memory per task + req rpt; + rpt.set_value("lprocs", "1", false); + rpt.set_value("total_memory", "5kb", false); + tasks = c.how_many_tasks_fit(rpt, 0); + fail_unless(tasks == 1, "%d tasks fit, expected 1", tasks); + + rpt.set_value("task_count", "5", false); + rpt.set_value("total_memory", "5kb", false); + tasks = c.how_many_tasks_fit(rpt, 0); + fail_unless(tasks == 5, "%d tasks fit, expected 5", tasks); thread_type = use_cores; // Cores are currently 0 @@ -774,6 +855,19 @@ START_TEST(test_how_many_tasks_fit) fail_unless(c.how_many_tasks_fit(r2, 0) == 5); r2.set_value("mics", "1", false); fail_unless(c.how_many_tasks_fit(r2, 0) == 0); + + Chip large_mem; + large_mem.setThreads(8); + large_mem.setMemory(128849018880); // 256 GB + large_mem.setCores(8); + large_mem.setChipAvailable(true); + for (int i = 0; i < 8; i++) + large_mem.make_core(i); + req r3; + r3.set_value("lprocs", "8", false); + r3.set_value("memory", "128849018880kb", false); // 256 gb + float fitting_tasks = large_mem.how_many_tasks_fit(r3, exclusive_none); + fail_unless(fitting_tasks == 1.0); } END_TEST @@ -1198,6 +1292,7 @@ Suite *numa_socket_suite(void) tcase_add_test(tc_core, test_spread_place_threads); tcase_add_test(tc_core, test_spread_place_cores); tcase_add_test(tc_core, test_spread_place); + tcase_add_test(tc_core, test_initialize_cores_from_strings); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_displayAsString"); @@ -1210,6 +1305,11 @@ Suite *numa_socket_suite(void) tcase_add_test(tc_core, test_initialize_allocation); tcase_add_test(tc_core, test_place_tasks_execution_slots); suite_add_tcase(s, tc_core); + + tc_core = tcase_create("test_get_contiguous_core_vector"); + tcase_add_test(tc_core, test_get_contiguous_core_vector); + tcase_add_test(tc_core, test_get_contiguous_thread_vector); + suite_add_tcase(s, tc_core); return(s); } diff --git a/src/test/numa_core/test_uut.c b/src/test/numa_core/test_uut.c index 66a60950bc..877e7da4b6 100644 --- a/src/test/numa_core/test_uut.c +++ b/src/test/numa_core/test_uut.c @@ -34,7 +34,7 @@ END_TEST START_TEST(test_displayAsString) { Core c; - c.add_processing_unit(CORE, 0); + c.add_processing_unit(CORE_INT, 0); std::stringstream out; c.displayAsString(out); @@ -105,19 +105,19 @@ START_TEST(test_add_processing_unit) fail_unless(c.getNumberOfProcessingUnits() == 0, "%d processing units", c.getNumberOfProcessingUnits()); fail_unless(c.get_id() == -1); - c.add_processing_unit(CORE, 0); - c.add_processing_unit(THREAD, 1); + c.add_processing_unit(CORE_INT, 0); + c.add_processing_unit(THREAD_INT, 1); fail_unless(c.getNumberOfProcessingUnits() == 2); fail_unless(c.get_id() == 0); - fail_unless(c.add_processing_unit(CORE, 2) != 0); + fail_unless(c.add_processing_unit(CORE_INT, 2) != 0); fail_unless(c.getNumberOfProcessingUnits() == 2); fail_unless(c.get_id() == 0); Core c2; - c2.add_processing_unit(CORE, 4); - c2.add_processing_unit(THREAD, 5); - c2.add_processing_unit(THREAD, 6); - c2.add_processing_unit(THREAD, 7); + c2.add_processing_unit(CORE_INT, 4); + c2.add_processing_unit(THREAD_INT, 5); + c2.add_processing_unit(THREAD_INT, 6); + c2.add_processing_unit(THREAD_INT, 7); fail_unless(c2.getNumberOfProcessingUnits() == 4, "%d", c2.getNumberOfProcessingUnits()); fail_unless(c2.get_id() == 4); } diff --git a/src/test/numa_pci_device/scaffolding.c b/src/test/numa_pci_device/scaffolding.c index f3bbcf198f..719276333c 100644 --- a/src/test/numa_pci_device/scaffolding.c +++ b/src/test/numa_pci_device/scaffolding.c @@ -38,7 +38,7 @@ void log_nvml_error(nvmlReturn_t rc, char* gpuid, const char* id) } #endif -void Machine::store_device_on_appropriate_chip( PCI_Device &device) +void Machine::store_device_on_appropriate_chip(PCI_Device &device, bool no_info) { return; } diff --git a/src/test/numa_pci_device/test_uut.c b/src/test/numa_pci_device/test_uut.c index f5acaeb3f8..95555c8a7f 100644 --- a/src/test/numa_pci_device/test_uut.c +++ b/src/test/numa_pci_device/test_uut.c @@ -8,12 +8,13 @@ START_TEST(test_displayAsString) { PCI_Device p; - p.setName("gpu"); + p.setName("k80"); + p.set_type(GPU); p.setId(0); std::stringstream out; p.displayAsString(out); - fail_unless(out.str() == " pci 0 gpu\n", out.str().c_str()); + fail_unless(out.str() == " GPU 0 k80\n", out.str().c_str()); } END_TEST diff --git a/src/test/numa_socket/scaffolding.c b/src/test/numa_socket/scaffolding.c index 1a06b9532e..625578a163 100644 --- a/src/test/numa_socket/scaffolding.c +++ b/src/test/numa_socket/scaffolding.c @@ -109,6 +109,10 @@ Chip::Chip(const Json::Value &json_layout, std::vector &valid_ids) json_chip++; } +Chip::Chip(const std::string &json_layout, std::vector &valid_ids) + { + } + Chip::Chip(const Chip &other) { } @@ -128,6 +132,8 @@ Chip::~Chip() void Chip::aggregate_allocations(std::vector &master_list) {} +void Chip::save_allocations(const Chip &other) {} + void Chip::displayAsJson( Json::Value &out, @@ -136,13 +142,18 @@ void Chip::displayAsJson( { } +int Chip::get_total_gpus() const + { + return(0); + } + bool Chip::spread_place( req &r, allocation &master, - int execution_slots_per, - int &remaining) + allocation &remaining, + allocation &remainder) { called_spread_place++; @@ -251,7 +262,7 @@ int get_machine_total_memory(hwloc_topology_t topology, unsigned long *memory) return(PBSE_NONE); } -float Chip::how_many_tasks_fit(req const &r, int place_type) const +double Chip::how_many_tasks_fit(req const &r, int place_type) const { return(tasks); } @@ -355,6 +366,12 @@ void allocation::set_place_type( else this->place_type = exclusive_none; } // END set_place_type() + +void allocation::adjust_for_spread(unsigned int quantity, bool find_remainder) {} + +void allocation::adjust_for_remainder(allocation &remainder) {} + +void allocation::clear() {} bool Chip::spread_place_threads( diff --git a/src/test/numa_socket/test_uut.c b/src/test/numa_socket/test_uut.c index 4d43332948..0c7be6ebf3 100644 --- a/src/test/numa_socket/test_uut.c +++ b/src/test/numa_socket/test_uut.c @@ -66,25 +66,29 @@ START_TEST(test_spread_place) Socket s; req r; allocation a; - int remaining = 0; + allocation remaining; + allocation remainder; s.addChip(); s.addChip(); completely_free = true; called_spread_place = 0; - fail_unless(s.spread_place(r, a, 5, remaining, false) == true); + remaining.cpus = 5; + fail_unless(s.spread_place(r, a, remaining, remainder, false) == true); fail_unless(called_spread_place == 2); completely_free = false; - fail_unless(s.spread_place(r, a, 5, remaining, false) == false); + remaining.cpus = 5; + fail_unless(s.spread_place(r, a, remaining, remainder, false) == false); fail_unless(called_spread_place == 2); completely_free = true; oscillate = false; s.free_task("1.napali"); - fail_unless(s.spread_place(r, a, 5, remaining, true) == true); + remaining.cpus = 5; + fail_unless(s.spread_place(r, a, remaining, remainder, true) == true); fail_unless(called_spread_place == 3); - fail_unless(s.spread_place(r, a, 5, remaining, true) == true); + fail_unless(s.spread_place(r, a, remaining, remainder, true) == true); fail_unless(called_spread_place == 4); } diff --git a/src/test/nvidia/Makefile.am b/src/test/nvidia/Makefile.am new file mode 100644 index 0000000000..1827b7bc0a --- /dev/null +++ b/src/test/nvidia/Makefile.am @@ -0,0 +1,4 @@ + +include ../Makefile_Mom.ut + +libuut_la_SOURCES = ${PROG_ROOT}/nvidia.c diff --git a/src/test/nvidia/scaffolding.c b/src/test/nvidia/scaffolding.c new file mode 100644 index 0000000000..e20adbf5ef --- /dev/null +++ b/src/test/nvidia/scaffolding.c @@ -0,0 +1,180 @@ +#include "license_pbs.h" /* See here for the software license */ +#include +#include /* fprintf */ +#include /* winsize */ +#include /* termios */ +#include +#include "log.h" +#include "mcom.h" +#include +#include +#include + +#include "attribute.h" +#include "resource.h" +#include "complete_req.hpp" + +char log_buffer[LOG_BUF_SIZE]; +nvmlReturn_t global_nvmlDeviceGetFanSpeed_rc = NVML_SUCCESS; +nvmlDevice_t global_device; +unsigned int global_device_minor_number = 0; +std::string global_string = "foo"; + +int MXMLFromString( + + mxml_t **EP, /* O (populate or create) */ + char *XMLString, /* I */ + char **Tail, /* O (optional) */ + char *EMsg, /* O (optional) */ + int emsg_size) /* I */ + + { + return(0); + } + +int MXMLDestroyE(mxml_t **EP) {return(0); } + +void log_ext(int errnum, const char *routine, const char *text, int severity) { } + +void log_event(int eventtype, int objclass, const char *objname, const char *text) {} + +void send_update_soon() { return; } + +void get_device_indices(const char *device_str, std::vector &device_indices, const char *suffix) {} + +nvmlReturn_t nvmlDeviceGetFanSpeed(nvmlDevice_t device_hndl, unsigned int *i) + { + return(global_nvmlDeviceGetFanSpeed_rc); + } + +nvmlReturn_t nvmlDeviceGetHandleByIndex(unsigned int index, nvmlDevice_t *device_hndl) + { + *device_hndl = global_device; + + return(NVML_SUCCESS); + } + +nvmlReturn_t nvmlDeviceGetMinorNumber(nvmlDevice_t device_hndl, unsigned int *minor) + { + *minor = global_device_minor_number++; + + return(NVML_SUCCESS); + } + +void log_err( + int errnum, /* I (errno or PBSErrno) */ + const char *routine, /* I */ + const char *text) /* I */ + + { + snprintf(log_buffer, sizeof(log_buffer), "%s %s", routine, text); + } + + +int is_whitespace( + + char c) + + { + if ((c == ' ') || + (c == '\n') || + (c == '\t') || + (c == '\r') || + (c == '\f')) + return(TRUE); + else + return(FALSE); + } /* END is_whitespace */ + +int find_range_in_cpuset_string( + + std::string &source, + std::string &output) + + { + output = global_string; + return(0); + } + +void translate_range_string_to_vector( + + const char *range_string, + std::vector &indices) + + { + char *str = strdup(range_string); + char *ptr = str; + int prev; + int curr; + + while (*ptr != '\0') + { + prev = strtol(ptr, &ptr, 10); + + if (*ptr == '-') + { + ptr++; + curr = strtol(ptr, &ptr, 10); + + while (prev <= curr) + { + indices.push_back(prev); + + prev++; + } + + if ((*ptr == ',') || + (is_whitespace(*ptr))) + ptr++; + } + else + { + indices.push_back(prev); + + if ((*ptr == ',') || + (is_whitespace(*ptr))) + ptr++; + } + } + + free(str); + } /* END translate_range_string_to_vector() */ + +bool have_incompatible_dash_l_resource( + + pbs_attribute *pattr) + + { + return(false); + } + +unsigned int complete_req::get_num_reqs() + { + return(1); + } + +req &complete_req::get_req(int i) + { + static req r; + + return(r); + } + +int req::get_gpus() const + + { + return(0); + } + +req::req() {} +req::req(const req &other) {} +req &req::operator =(const req &other) + { + return(*this); + } +std::string req::get_gpu_mode() const + + { + return(this->gpu_mode); + } + diff --git a/src/test/nvidia/test_nvidia.h b/src/test/nvidia/test_nvidia.h new file mode 100644 index 0000000000..f9b8d20f3c --- /dev/null +++ b/src/test/nvidia/test_nvidia.h @@ -0,0 +1,9 @@ +#include "license_pbs.h" /* See here for the software license */ +#ifndef _NVIDIA_CT_H +#define _NVIDIA_CT_H +#include + +#define NVIDIA_SUITE 1 +Suite *nvidia_suite(); + +#endif /* _NVIDIA_CT_H */ diff --git a/src/test/nvidia/test_uut.c b/src/test/nvidia/test_uut.c new file mode 100644 index 0000000000..9d39ddfee8 --- /dev/null +++ b/src/test/nvidia/test_uut.c @@ -0,0 +1,124 @@ +#include "license_pbs.h" /* See here for the software license */ +#include "mom_mach.h" +#include "test_nvidia.h" +#include +#include +#include +#include +#include + +#include "pbs_error.h" +#include "pbs_job.h" +#include "req.hpp" + +int MOMNvidiaDriverVersion = 0; +int LOGLEVEL = 0; +int use_nvidia_gpu = TRUE; +int global_gpu_count = 0; +int time_now = 0; +char mom_host[PBS_MAXHOSTNAME + 1]; + +extern nvmlReturn_t global_nvmlDeviceGetFanSpeed_rc; +extern std::map gpu_minor_to_gpu_index; +extern nvmlDevice_t global_device; +extern unsigned int global_device_minor_number; + +int get_nvml_version(); +void generate_server_gpustatus_nvml(std::vector &gpu_status); +int build_gpu_minor_to_gpu_index_map(unsigned int); +int get_gpu_handle_by_minor(unsigned int, nvmlDevice_t*); + +START_TEST(test_get_nvml_version) + { + int version; + + nvmlInit(); + + version = get_nvml_version(); + fail_unless(version == NVML_API_VERSION); + + nvmlShutdown(); + } +END_TEST + +START_TEST(test_generate_server_gpustatus_nvml) + { + std::vector gpu_status; + + // force nvmlDeviceGetFanSpeed() to return error + global_nvmlDeviceGetFanSpeed_rc = NVML_ERROR_UNKNOWN; + LOGLEVEL = 2; + + generate_server_gpustatus_nvml(gpu_status); + + // expect log_buffer to hold something like + // "nvmlDeviceGetFanSpeed() called from generate_server_gpustatus_nvml (idx=0) Unknown error" + fail_unless((strcasestr(log_buffer, "nvmlDeviceGetFanSpeed() called from generate_server_gpustatus_nvml") != NULL) && + (strcasestr(log_buffer, "unknown error") != NULL)); + } +END_TEST + +START_TEST(test_build_gpu_minor_to_gpu_index_map) + { + nvmlDevice_t *handle; + + global_device_minor_number = 0; + build_gpu_minor_to_gpu_index_map(2); + fail_unless(global_device_minor_number == 3); + fail_unless(gpu_minor_to_gpu_index[0] == 0); + fail_unless(gpu_minor_to_gpu_index[1] == 1); + + get_gpu_handle_by_minor(0, handle); + fail_unless(handle == &global_device); + get_gpu_handle_by_minor(1, handle); + fail_unless(handle == &global_device); + } +END_TEST + +START_TEST(test_two) + { + } +END_TEST + +Suite *nvidia_suite(void) + { + Suite *s = suite_create("nvidia_suite methods"); + TCase *tc_core = tcase_create("test_get_nvml_version"); + + // give this test a bit of extra time since nvmlInit() can take awhile + tcase_set_timeout(tc_core, 15); + + tcase_add_test(tc_core, test_get_nvml_version); + suite_add_tcase(s, tc_core); + + tc_core = tcase_create("test_generate_server_gpustatus_nvml"); + tcase_add_test(tc_core, test_generate_server_gpustatus_nvml); + suite_add_tcase(s, tc_core); + + tc_core = tcase_create("test_build_gpu_minor_to_gpu_index_map"); + tcase_add_test(tc_core, test_build_gpu_minor_to_gpu_index_map); + suite_add_tcase(s, tc_core); + + tc_core = tcase_create("test_two"); + tcase_add_test(tc_core, test_two); + suite_add_tcase(s, tc_core); + + return s; + } + +void rundebug() + { + } + +int main(void) + { + int number_failed = 0; + SRunner *sr = NULL; + rundebug(); + sr = srunner_create(nvidia_suite()); + srunner_set_log(sr, "nvidia_suite.log"); + srunner_run_all(sr, CK_NORMAL); + number_failed = srunner_ntests_failed(sr); + srunner_free(sr); + return number_failed; + } diff --git a/src/test/parse_config/scaffolding.c b/src/test/parse_config/scaffolding.c index 375dffffe7..34963fa3b0 100644 --- a/src/test/parse_config/scaffolding.c +++ b/src/test/parse_config/scaffolding.c @@ -226,3 +226,13 @@ void authorized_hosts::add_authorized_address(unsigned long addr, unsigned short authorized_hosts::authorized_hosts() {} authorized_hosts auth_hosts; +int read_mem_value( + + const char *value, + unsigned long &parsed) + + { + parsed = 10; + return(0); + } + diff --git a/src/test/pbsD_submit/scaffolding.c b/src/test/pbsD_submit/scaffolding.c index 74b793db1a..99cc856c4d 100644 --- a/src/test/pbsD_submit/scaffolding.c +++ b/src/test/pbsD_submit/scaffolding.c @@ -18,6 +18,12 @@ int PBSD_commit(int connect, char *jobid) exit(1); } +int PBSD_commit2(int connect, char *jobid) + { + fprintf(stderr, "The call to PBSD_commit needs to be mocked!!\n"); + exit(1); + } + int PBSD_rdytocmt(int connect, char *jobid) { fprintf(stderr, "The call to PBSD_rdytocmt needs to be mocked!!\n"); diff --git a/src/test/pbsD_submit_hash/scaffolding.c b/src/test/pbsD_submit_hash/scaffolding.c index 8fe49af129..6f03541434 100644 --- a/src/test/pbsD_submit_hash/scaffolding.c +++ b/src/test/pbsD_submit_hash/scaffolding.c @@ -21,6 +21,12 @@ int PBSD_commit(int connect, char *jobid) exit(1); } +int PBSD_commit2(int connect, char *jobid) + { + fprintf(stderr, "The call to PBSD_commit needs to be mocked!!\n"); + exit(1); + } + int PBSD_rdytocmt(int connect, char *jobid) { fprintf(stderr, "The call to PBSD_rdytocmt needs to be mocked!!\n"); diff --git a/src/test/pbs_demux/test_uut.c b/src/test/pbs_demux/test_uut.c index a7f8fa44b1..380d052ebd 100644 --- a/src/test/pbs_demux/test_uut.c +++ b/src/test/pbs_demux/test_uut.c @@ -6,10 +6,14 @@ #include "pbs_error.h" -START_TEST(test_one) - { - +void readit(int, struct routem*); +START_TEST(test_readit) + { + // this is a void function but expect + // immediate return with no segfault + // since readset not initialized + readit(0, NULL); } END_TEST @@ -23,8 +27,8 @@ END_TEST Suite *pbs_demux_suite(void) { Suite *s = suite_create("pbs_demux_suite methods"); - TCase *tc_core = tcase_create("test_one"); - tcase_add_test(tc_core, test_one); + TCase *tc_core = tcase_create("test_readit"); + tcase_add_test(tc_core, test_readit); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_two"); diff --git a/src/test/pbsd_init/scaffolding.c b/src/test/pbsd_init/scaffolding.c index ddfc79affa..24ee10ff4e 100644 --- a/src/test/pbsd_init/scaffolding.c +++ b/src/test/pbsd_init/scaffolding.c @@ -117,6 +117,8 @@ int evaluated; int aborted; int freed_job_allocation; int job_state; +int locked_job = 0; +int unlocked_job = 0; bool dont_find_job; bool dont_find_node; @@ -577,11 +579,13 @@ int insert_addr_name_info( int unlock_ji_mutex(job *pjob, const char *id, const char *msg, int logging) { + unlocked_job++; return(0); } int lock_ji_mutex(job *pjob, const char *id, const char *msg, int logging) { + locked_job++; return(0); } @@ -986,3 +990,17 @@ void job_array::update_array_values( { } + +int set_default_gpu_mode_int(const char *default_gpu_str) + { + return(0); + } + +int get_svr_attr_str( + + int attr_index, + char **str) + + { + return(0); + } diff --git a/src/test/pbsd_init/test_uut.c b/src/test/pbsd_init/test_uut.c index 772cb59e2a..f2335a4e69 100644 --- a/src/test/pbsd_init/test_uut.c +++ b/src/test/pbsd_init/test_uut.c @@ -21,6 +21,8 @@ extern int evaluated; extern int aborted; extern int freed_job_allocation; extern int job_state; +extern int unlocked_job; +extern int locked_job; extern bool dont_find_job; extern bool dont_find_node; extern pbs_queue *allocd_queue; @@ -62,12 +64,19 @@ START_TEST(test_check_jobs_queue) sprintf(pjob.ji_qs.ji_jobid, "1.napali"); sprintf(pjob.ji_qs.ji_queue, "lost"); + unlocked_job = 0; + locked_job = 0; + check_jobs_queue(&pjob); fail_unless(allocd_queue != NULL); fail_unless(!strcmp(allocd_queue->qu_qs.qu_name, pjob.ji_qs.ji_queue)); fail_unless(allocd_queue->qu_qs.qu_type == QTYPE_Execution); fail_unless(allocd_queue->qu_attr[QA_ATR_GhostQueue].at_val.at_long == 1); fail_unless(!strcmp(allocd_queue->qu_attr[QA_ATR_QType].at_val.at_str, "Execution")); + + // check_jobs_queue should unlock and re-lock the job once each + fail_unless(unlocked_job == 1); + fail_unless(locked_job == 1); } END_TEST diff --git a/src/test/pbsnodes/scaffolding.c b/src/test/pbsnodes/scaffolding.c index 60f052f140..e7830388e1 100644 --- a/src/test/pbsnodes/scaffolding.c +++ b/src/test/pbsnodes/scaffolding.c @@ -41,7 +41,7 @@ int MXMLToXString(mxml_t *E, char **Buf, int *BufSize, int MaxBufSize, char **Ta extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/process_alps_status/scaffolding.c b/src/test/process_alps_status/scaffolding.c index 55cea6d1d9..dd8ab2c577 100644 --- a/src/test/process_alps_status/scaffolding.c +++ b/src/test/process_alps_status/scaffolding.c @@ -1914,3 +1914,7 @@ void Machine::setMemory(long long mem) } #endif +batch_request::batch_request(int type) {} +batch_request::batch_request() {} +batch_request::~batch_request() {} + diff --git a/src/test/process_mom_update/scaffolding.c b/src/test/process_mom_update/scaffolding.c index 4569395967..37a1a48872 100644 --- a/src/test/process_mom_update/scaffolding.c +++ b/src/test/process_mom_update/scaffolding.c @@ -14,7 +14,7 @@ #include "u_tree.h" #include "id_map.hpp" -#include "id_map.hpp" +#include "machine.hpp" char server_name[PBS_MAXSERVERNAME + 1]; /* host_name[:service|port] */ int allow_any_mom; @@ -650,6 +650,8 @@ Chip::~Chip() {} Core::Core() {} Core::~Core() {} +void Machine::save_allocations(const Machine &other) {} + bool Machine::is_initialized() const { return(this->initialized); @@ -686,5 +688,10 @@ Machine::Machine( { } +int Machine::get_total_gpus() const + { + return(0); + } + #endif diff --git a/src/test/process_mom_update/test_uut.c b/src/test/process_mom_update/test_uut.c index aef55b28d5..ded0e03d1b 100644 --- a/src/test/process_mom_update/test_uut.c +++ b/src/test/process_mom_update/test_uut.c @@ -12,7 +12,7 @@ int set_note_error(struct pbsnode *np, const char *str); int restore_note(struct pbsnode *np); #ifdef PENABLE_LINUX_CGROUPS -void update_layout_if_needed(pbsnode *pnode, const std::string &layout); +void update_layout_if_needed(pbsnode *pnode, const std::string &layout, bool force); extern int event_logged; @@ -26,7 +26,7 @@ START_TEST(test_update_layout_if_needed) fail_unless(pnode.nd_layout.is_initialized() == false); event_logged = 0; - update_layout_if_needed(&pnode, sans_threads); + update_layout_if_needed(&pnode, sans_threads, false); fail_unless(event_logged == 0); fail_unless(pnode.nd_layout.is_initialized() != false); @@ -35,10 +35,10 @@ START_TEST(test_update_layout_if_needed) pnode.nd_slots.add_execution_slot(); // Calling again without changing the number of slots should do nothing - update_layout_if_needed(&pnode, sans_threads); - update_layout_if_needed(&pnode, sans_threads); - update_layout_if_needed(&pnode, sans_threads); - update_layout_if_needed(&pnode, sans_threads); + update_layout_if_needed(&pnode, sans_threads, false); + update_layout_if_needed(&pnode, sans_threads, false); + update_layout_if_needed(&pnode, sans_threads, false); + update_layout_if_needed(&pnode, sans_threads, false); fail_unless(event_logged == 0); // Simulate that threads were turned on in this machine, therefore increasing the number of @@ -46,16 +46,21 @@ START_TEST(test_update_layout_if_needed) for (int i = 0; i < 16; i++) pnode.nd_slots.add_execution_slot(); - update_layout_if_needed(&pnode, with_threads); + update_layout_if_needed(&pnode, with_threads, false); fail_unless(event_logged == 1); // Calling again without changing the number of slots should do nothing - update_layout_if_needed(&pnode, with_threads); - update_layout_if_needed(&pnode, with_threads); - update_layout_if_needed(&pnode, with_threads); - update_layout_if_needed(&pnode, with_threads); + update_layout_if_needed(&pnode, with_threads, false); + update_layout_if_needed(&pnode, with_threads, false); + update_layout_if_needed(&pnode, with_threads, false); + update_layout_if_needed(&pnode, with_threads, false); fail_unless(event_logged == 1, "event logged: %d", event_logged); + + // Make sure that changing the number of gpus on the node triggers an event + pnode.nd_ngpus = 1; + update_layout_if_needed(&pnode, with_threads, false); + fail_unless(event_logged == 2, "event logged: %d", event_logged); } END_TEST #endif diff --git a/src/test/process_request/scaffolding.c b/src/test/process_request/scaffolding.c index 8750c4bf79..66618f8900 100644 --- a/src/test/process_request/scaffolding.c +++ b/src/test/process_request/scaffolding.c @@ -553,12 +553,16 @@ int req_job_cleanup_done(batch_request *preq) } int req_quejob2(batch_request *preq) - { - return(0); - } + { + return(0); + } int req_commit2(batch_request *preq) - { - return(0); - } + { + return(0); + } + +batch_request::batch_request() {} +batch_request::~batch_request() {} + diff --git a/src/test/process_request/test_uut.c b/src/test/process_request/test_uut.c index d6c6331ddf..704ef6de61 100644 --- a/src/test/process_request/test_uut.c +++ b/src/test/process_request/test_uut.c @@ -12,8 +12,7 @@ extern struct connection svr_conn[]; int process_request(struct tcp_chan *chan); bool request_passes_acl_check(batch_request *request, unsigned long conn_addr); -batch_request *alloc_br(int type); -batch_request *read_request_from_socket(tcp_chan *chan); +int read_request_from_socket(tcp_chan *chan, batch_request &preq); extern bool check_acl; extern bool find_node; @@ -46,43 +45,43 @@ void set_connection_type( START_TEST(test_read_request_from_socket) { - tcp_chan chan; + tcp_chan chan; + batch_request preq; memset(&chan, 0, sizeof(chan)); chan.sock = -1; - fail_unless(read_request_from_socket(&chan) == NULL, "invalid socket (-1) should return NULL"); + fail_unless(read_request_from_socket(&chan, preq) == -1, "invalid socket (-1) should return NULL"); chan.sock = PBS_NET_MAX_CONNECTIONS; - fail_unless(read_request_from_socket(&chan) == NULL, "invalid socket (PBS_NET_MAX_CONNECTIONS) should return NULL"); + fail_unless(read_request_from_socket(&chan, preq) == -1, "invalid socket (PBS_NET_MAX_CONNECTIONS) should return NULL"); chan.sock = 1; initialize_svr_conn(chan.sock); set_connection_type(chan.sock, Idle); - fail_unless(read_request_from_socket(&chan) == NULL, "Idle connection type should fail"); + fail_unless(read_request_from_socket(&chan, preq) == -1, "Idle connection type should fail"); strcpy(server_name, "napali"); set_connection_type(chan.sock, ToServerDIS); find_node = false; check_acl = true; fail_check = false; - fail_unless(read_request_from_socket(&chan) == NULL, "Node not found should fail"); + fail_unless(read_request_from_socket(&chan, preq) == -1, "Node not found should fail"); fail_check = true; find_node = true; check_acl = false; - fail_unless(read_request_from_socket(&chan) != NULL, "should pass"); + fail_unless(read_request_from_socket(&chan, preq) == PBSE_NONE, "should pass"); fail_get_connecthost = true; - fail_unless(read_request_from_socket(&chan) == NULL, "fail if can't get connect host"); + fail_unless(read_request_from_socket(&chan, preq) == -1, "fail if can't get connect host"); dis_req_read_rc = PBSE_SYSTEM; - batch_request *preq; - fail_unless((preq = read_request_from_socket(&chan)) != NULL, "should get bad request"); - fail_unless(preq->rq_type == PBS_BATCH_Disconnect); + fail_unless((read_request_from_socket(&chan, preq)) == PBSE_SYSTEM, "should get bad request: %d"); + fail_unless(preq.rq_type == PBS_BATCH_Disconnect); dis_req_read_rc = 5; - fail_unless((preq = read_request_from_socket(&chan)) != NULL, "should get bad request"); - fail_unless(preq->rq_failcode == dis_req_read_rc); + fail_unless((read_request_from_socket(&chan, preq)) == dis_req_read_rc, "should get bad request"); + fail_unless(preq.rq_failcode == dis_req_read_rc); } END_TEST @@ -116,22 +115,6 @@ START_TEST(test_request_passes_acl_check) } END_TEST -START_TEST(test_alloc_br) - { - batch_request *preq = alloc_br(PBS_BATCH_QueueJob); - - fail_unless(preq->rq_type == PBS_BATCH_QueueJob); - fail_unless(preq->rq_conn == -1); - fail_unless(preq->rq_orgconn == -1); - fail_unless(preq->rq_reply.brp_choice == BATCH_REPLY_CHOICE_NULL); - fail_unless(preq->rq_noreply == FALSE); - fail_unless(preq->rq_time > 0); - - free_br(preq); - fail_unless(free_attrlist_called > 0); - } -END_TEST - START_TEST(test_process_request_bad_host_err) { struct tcp_chan chan; @@ -161,7 +144,7 @@ Suite *process_request_suite(void) tc_core = tcase_create("test_request_passes_acl_check"); tcase_add_test(tc_core, test_request_passes_acl_check); - tcase_add_test(tc_core, test_alloc_br); + suite_add_tcase(s, tc_core); tc_core = tcase_create("test_process_request_bad_host_err"); tcase_add_test(tc_core, test_process_request_bad_host_err); diff --git a/src/test/prolog/scaffolding.c b/src/test/prolog/scaffolding.c index 1bc810c9fe..3675af9c67 100644 --- a/src/test/prolog/scaffolding.c +++ b/src/test/prolog/scaffolding.c @@ -67,7 +67,8 @@ void encode_used(job *pjob, int perm, Json::Value *list, tlist_head *phead) int become_the_user( - job *pjob) + job *pjob, + bool want_effective) { return(0); diff --git a/src/test/qalter/scaffolding.c b/src/test/qalter/scaffolding.c index 434c6b2f8a..cb0f7d7ff5 100644 --- a/src/test/qalter/scaffolding.c +++ b/src/test/qalter/scaffolding.c @@ -37,7 +37,7 @@ int locate_job(char *job_id, char *parent_server, char *located_server) extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qchkpt/scaffolding.c b/src/test/qchkpt/scaffolding.c index cc73683957..ab4546b636 100644 --- a/src/test/qchkpt/scaffolding.c +++ b/src/test/qchkpt/scaffolding.c @@ -15,7 +15,7 @@ int pbs_disconnect(int connect) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qdel/scaffolding.c b/src/test/qdel/scaffolding.c index 31689eafd8..de9d19b2cc 100644 --- a/src/test/qdel/scaffolding.c +++ b/src/test/qdel/scaffolding.c @@ -34,7 +34,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { return 10; } diff --git a/src/test/qdisable/scaffolding.c b/src/test/qdisable/scaffolding.c index 2f2afc7f61..920999796c 100644 --- a/src/test/qdisable/scaffolding.c +++ b/src/test/qdisable/scaffolding.c @@ -26,7 +26,7 @@ int parse_destination_id(char *destination_in, char **queue_name_out, char **ser extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qenable/scaffolding.c b/src/test/qenable/scaffolding.c index 6e8f90af03..bad698c8b1 100644 --- a/src/test/qenable/scaffolding.c +++ b/src/test/qenable/scaffolding.c @@ -26,7 +26,7 @@ int parse_destination_id(char *destination_in, char **queue_name_out, char **ser extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qgpumode/scaffolding.c b/src/test/qgpumode/scaffolding.c index 03d21b1d39..baffbf500d 100644 --- a/src/test/qgpumode/scaffolding.c +++ b/src/test/qgpumode/scaffolding.c @@ -24,7 +24,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qgpureset/scaffolding.c b/src/test/qgpureset/scaffolding.c index 88f1c933b0..3c71b41b2e 100644 --- a/src/test/qgpureset/scaffolding.c +++ b/src/test/qgpureset/scaffolding.c @@ -24,7 +24,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qhold/scaffolding.c b/src/test/qhold/scaffolding.c index f3b60e3aff..e95d23d71e 100644 --- a/src/test/qhold/scaffolding.c +++ b/src/test/qhold/scaffolding.c @@ -21,7 +21,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qmgr/scaffolding.c b/src/test/qmgr/scaffolding.c index 7da9381b8c..c3dc507392 100644 --- a/src/test/qmgr/scaffolding.c +++ b/src/test/qmgr/scaffolding.c @@ -32,7 +32,7 @@ int pbs_disconnect(int connect) extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qmove/scaffolding.c b/src/test/qmove/scaffolding.c index 4a3418f769..7f75057a86 100644 --- a/src/test/qmove/scaffolding.c +++ b/src/test/qmove/scaffolding.c @@ -21,7 +21,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qmsg/scaffolding.c b/src/test/qmsg/scaffolding.c index 22fa1d5480..dc39370ae0 100644 --- a/src/test/qmsg/scaffolding.c +++ b/src/test/qmsg/scaffolding.c @@ -21,7 +21,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qorder/scaffolding.c b/src/test/qorder/scaffolding.c index 83dbe44059..2d814bc6c0 100644 --- a/src/test/qorder/scaffolding.c +++ b/src/test/qorder/scaffolding.c @@ -31,7 +31,7 @@ char *pbs_default(void) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qrerun/scaffolding.c b/src/test/qrerun/scaffolding.c index bb839b3a97..a6e901fb7a 100644 --- a/src/test/qrerun/scaffolding.c +++ b/src/test/qrerun/scaffolding.c @@ -21,7 +21,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qrls/scaffolding.c b/src/test/qrls/scaffolding.c index 57e67b4334..746229ed3c 100644 --- a/src/test/qrls/scaffolding.c +++ b/src/test/qrls/scaffolding.c @@ -21,7 +21,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qrun/scaffolding.c b/src/test/qrun/scaffolding.c index 6672cba1b6..f209bcfeb0 100644 --- a/src/test/qrun/scaffolding.c +++ b/src/test/qrun/scaffolding.c @@ -33,7 +33,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qselect/scaffolding.c b/src/test/qselect/scaffolding.c index 4035d0c88c..7f1dfc1b51 100644 --- a/src/test/qselect/scaffolding.c +++ b/src/test/qselect/scaffolding.c @@ -27,7 +27,7 @@ int parse_destination_id(char *destination_in, char **queue_name_out, char **ser extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qsig/scaffolding.c b/src/test/qsig/scaffolding.c index c260a56bce..63506a8a27 100644 --- a/src/test/qsig/scaffolding.c +++ b/src/test/qsig/scaffolding.c @@ -21,7 +21,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qstart/scaffolding.c b/src/test/qstart/scaffolding.c index 877cc3ccd1..77af2ab827 100644 --- a/src/test/qstart/scaffolding.c +++ b/src/test/qstart/scaffolding.c @@ -33,7 +33,7 @@ int locate_job(char *job_id, char *parent_server, char *located_server) extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qstat/scaffolding.c b/src/test/qstat/scaffolding.c index 68f7440ecf..8158095b6c 100644 --- a/src/test/qstat/scaffolding.c +++ b/src/test/qstat/scaffolding.c @@ -157,7 +157,7 @@ int MXMLToXString(mxml_t *E, char **Buf, int *BufSize, int MaxBufSize, char **Ta extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { if (connect_success == false) return(-1); diff --git a/src/test/qstop/scaffolding.c b/src/test/qstop/scaffolding.c index c8c4073ad7..0b9d631c57 100644 --- a/src/test/qstop/scaffolding.c +++ b/src/test/qstop/scaffolding.c @@ -32,7 +32,7 @@ int locate_job(char *job_id, char *parent_server, char *located_server) extern "C" { -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); diff --git a/src/test/qsub_functions/scaffolding.c b/src/test/qsub_functions/scaffolding.c index 4e21c50f7f..ff10fccab3 100644 --- a/src/test/qsub_functions/scaffolding.c +++ b/src/test/qsub_functions/scaffolding.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "u_hash_map_structs.h" #include "port_forwarding.h" #include "req.hpp" @@ -28,7 +29,13 @@ bool find_mpp = false; bool find_nodes = false; bool find_size = false; bool validate_path = false; +bool mem_fail = false; int req_val = 0; +bool init_work_dir = false; + +int global_poll_rc = 0; +short global_poll_revents = 0; +int global_poll_errno = 0; std::string added_value; std::string added_name; @@ -44,6 +51,7 @@ char *pbs_geterrmsg(int connect) int hash_find(job_data_container *head, const char *name, job_data **env_var) { + static job_data ev("a", "b", 0, 0); if ((find_mpp == true) && (!strcmp(name, "mppwidth"))) return(1); @@ -56,6 +64,19 @@ int hash_find(job_data_container *head, const char *name, job_data **env_var) else if ((validate_path == true) && (!strcmp(name, "validate_path"))) return(1); + else if ((init_work_dir == true) && + (!strcmp(name, "init_work_dir"))) + { + // set env_var to point to a legit directory, ".." + *env_var = new job_data(strdup("foo"), strdup(".."), 0, 0); + return(1); + } + else if (mem_fail == true) + { + ev.value = "1"; + *env_var = &ev; + return(1); + } for (unsigned int i = 0; i < in_hash.size(); i++) { @@ -102,7 +123,7 @@ int locate_job(char *job_id, char *parent_server, char *located_server) exit(1); } -void port_forwarder( struct pfwdsock *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, char *EMsg) +void port_forwarder(std::vector *socks, int (*connfunc)(char *, long, char *), char *phost, int pport, char *EMsg) { fprintf(stderr, "The call to port_forwarder to be mocked!!\n"); exit(1); @@ -273,7 +294,7 @@ int pbs_disconnect(int connect) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server to be mocked!!\n"); exit(1); @@ -480,4 +501,27 @@ struct group *getgrnam_ext( return(grp); } /* END getgrnam_ext() */ +int ppoll(struct pollfd *fds, nfds_t nfds, const struct timespec *timeout, const sigset_t *sigmask) + { + fds->revents = global_poll_revents; + errno = global_poll_errno; + return(global_poll_rc); + } + +int read_mem_value(const char *value, unsigned long &parsed) + { + static int count = 0; + + if (mem_fail) + { + if (count++ % 2 == 0) + parsed = 1; + else + parsed = 2; + } + else + parsed = 0; + + return(0); + } diff --git a/src/test/qsub_functions/test_uut.c b/src/test/qsub_functions/test_uut.c index ab5a60c539..cc206a8b3c 100644 --- a/src/test/qsub_functions/test_uut.c +++ b/src/test/qsub_functions/test_uut.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "qsub_functions.h" #include "test_qsub_functions.h" @@ -20,17 +21,30 @@ int process_opt_k(job_info *ji, const char *cmd_arg, int data_type); int process_opt_K(job_info *ji, const char *cmd_arg, int data_type); int process_opt_m(job_info *ji, const char *cmd_arg, int data_type); int process_opt_p(job_info *ji, const char *cmd_arg, int data_type); +int wait_for_read_ready(int, int); +bool is_memory_request_valid(job_info *ji, std::string &err_msg); +int validate_pbs_o_workdir(const job_info *); +unsigned int get_port_in_range(); extern complete_req cr; +extern bool mem_fail; extern bool submission_string_fail; extern bool added_req; extern bool find_nodes; extern bool find_mpp; extern bool find_size; extern bool validate_path; +extern bool init_work_dir; extern std::string added_value; extern std::string added_name; +extern int global_poll_rc; +extern short global_poll_revents; +extern int global_poll_errno; + +extern int interactive_port_min; +extern int interactive_port_max; + bool are_we_forking() { @@ -44,6 +58,37 @@ bool are_we_forking() } +START_TEST(test_get_port_in_range) + { + interactive_port_min = 30000; + interactive_port_max = 30100; + + for (int i = 0; i <= 100; i++) + { + unsigned int port = get_port_in_range(); + fail_unless(port >= interactive_port_min); + fail_unless(port <= interactive_port_max); + } + } +END_TEST + + +START_TEST(test_is_memory_request_valid) + { +#ifdef PENABLE_LINUX_CGROUPS + job_info ji; + std::string err; + + mem_fail = false; + fail_unless(is_memory_request_valid(&ji, err) == true); + mem_fail = true; + fail_unless(is_memory_request_valid(&ji, err) == false); +#endif + + } +END_TEST + + START_TEST(test_process_opt_K) { job_info ji; @@ -378,6 +423,58 @@ START_TEST(test_make_argv) } END_TEST +START_TEST(test_wait_for_read_ready) + { + int rc; + int some_fd = 0; // arbitrary value + int some_timeout_sec = 1; // arbitrary value > 0 + + // emulate timeout + global_poll_rc = 0; + global_poll_revents = 0; + rc = wait_for_read_ready(some_fd, some_timeout_sec); + fail_unless(rc == 0); + + // emulate ready to read + global_poll_rc = 1; + global_poll_revents = POLLIN; + rc = wait_for_read_ready(some_fd, some_timeout_sec); + fail_unless(rc == 1); + + // emulate not ready to read + global_poll_rc = 1; + global_poll_revents = 0; + rc = wait_for_read_ready(some_fd, some_timeout_sec); + fail_unless(rc == 0); + + // emulate failure + global_poll_rc = -1; + global_poll_errno = EFAULT; + rc = wait_for_read_ready(some_fd, some_timeout_sec); + fail_unless(rc == -1); + + // emulate recoverable failure + global_poll_rc = -1; + global_poll_errno = EINTR; + rc = wait_for_read_ready(some_fd, some_timeout_sec); + fail_unless(rc == 0); + } +END_TEST + + +START_TEST(test_validate_pbs_o_workdir) + { + job_info ji; + + validate_path = true; + + init_work_dir = true; + validate_pbs_o_workdir(&ji); + fail_unless(strcmp(added_name.c_str(), "..") != 0); + } +END_TEST + + Suite *qsub_functions_suite(void) { Suite *s = suite_create("qsub_functions methods"); @@ -390,6 +487,7 @@ Suite *qsub_functions_suite(void) tcase_add_test(tc_core, test_process_opt_m); tcase_add_test(tc_core, test_process_opt_p); tcase_add_test(tc_core, test_retry_submit_error); + tcase_add_test(tc_core, test_validate_pbs_o_workdir); suite_add_tcase(s, tc_core); tc_core = tcase_create("test isWindowsFormat"); @@ -410,6 +508,12 @@ Suite *qsub_functions_suite(void) tc_core = tcase_create("test_make_argv"); tcase_add_test(tc_core, test_make_argv); tcase_add_test(tc_core, test_is_resource_request_valid); + tcase_add_test(tc_core, test_get_port_in_range); + suite_add_tcase(s, tc_core); + + tc_core = tcase_create("test_wait_for_read_ready"); + tcase_add_test(tc_core, test_wait_for_read_ready); + tcase_add_test(tc_core, test_is_memory_request_valid); suite_add_tcase(s, tc_core); return s; diff --git a/src/test/qterm/scaffolding.c b/src/test/qterm/scaffolding.c index f08197d897..670f1ff398 100644 --- a/src/test/qterm/scaffolding.c +++ b/src/test/qterm/scaffolding.c @@ -1,6 +1,8 @@ #include "license_pbs.h" /* See here for the software license */ #include #include /* fprintf */ +#include "pbs_error.h" +#include "net_connect.h" int pbs_errno = 0; @@ -24,7 +26,7 @@ char *pbs_strerror(int err) exit(1); } -int cnt2server(const char *SpecServer) +int cnt2server(const char *SpecServer, bool silence) { fprintf(stderr, "The call to cnt2server needs to be mocked!!\n"); exit(1); @@ -44,3 +46,33 @@ int pbs_terminate_err(int c, int manner, char *extend, int *local_errno) } void initialize_network_info() {} + +int get_fullhostname( + + char *shortname, /* I */ + char *namebuf, /* O */ + int bufsize, /* I */ + char *EMsg) /* O (optional,minsize=MAXLINE - 1024) */ + + { + return(PBSE_NONE); + } + +pbs_net_t get_hostaddr( + + int *local_errno, /* O */ + const char *hostname) /* I */ + + { + return(0); + } + +int get_active_pbs_server( + + char **active_server, + int *port) + + { + return(0); + } + diff --git a/src/test/receive_mom_communication/Makefile.am b/src/test/receive_mom_communication/Makefile.am index 1584d00804..7794457d47 100644 --- a/src/test/receive_mom_communication/Makefile.am +++ b/src/test/receive_mom_communication/Makefile.am @@ -2,5 +2,8 @@ include ../Makefile_Server.ut libuut_la_SOURCES = ${PROG_ROOT}/process_mom_update.c ${PROG_ROOT}/receive_mom_communication.c \ - ${PROG_ROOT}/../lib/Libutils/machine.cpp ${PROG_ROOT}/../lib/Libutils/numa_pci_device.cpp \ - ${PROG_ROOT}/../lib/Libutils/numa_socket.cpp ${PROG_ROOT}/../lib/Libutils/numa_chip.cpp + ${PROG_ROOT}/../lib/Libutils/machine.cpp \ + ${PROG_ROOT}/../lib/Libutils/numa_pci_device.cpp \ + ${PROG_ROOT}/../lib/Libutils/numa_socket.cpp \ + ${PROG_ROOT}/../lib/Libutils/numa_chip.cpp \ + ${PROG_ROOT}/../lib/Libutils/jsoncpp.cpp diff --git a/src/test/receive_mom_communication/scaffolding.c b/src/test/receive_mom_communication/scaffolding.c index 0dcd691b81..a19648cdb9 100644 --- a/src/test/receive_mom_communication/scaffolding.c +++ b/src/test/receive_mom_communication/scaffolding.c @@ -17,6 +17,7 @@ #include "node_manager.h" #include "pbs_ifl.h" #include "authorized_hosts.hpp" +#include "machine.hpp" id_map job_mapper; @@ -934,3 +935,29 @@ pbsnode *authorized_hosts::get_authorized_node(unsigned long addr, unsigned shor authorized_hosts::authorized_hosts() {} authorized_hosts auth_hosts; + +const char *LPROCS = "lprocs"; +const char *NODE = "node"; +const char *NODES = "nodes"; +const char *SOCKET = "socket"; +const char *SOCKETS = "sockets"; +const char *NUMA_NODE = "numanode"; +const char *NUMA_NODES = "numanodes"; +const char *CORE = "core"; +const char *CORES = "cores"; +const char *THREAD = "thread"; +const char *THREADS = "threads"; +const char *MEMORY = "memory"; +const char *SWAP = "swap"; +const char *DISK = "disk"; + +const char *ALLOCATION = "allocation"; +const char *ALLOCATIONS = "allocations"; +const char *MEM = "mem"; +const char *GPUS = "gpus"; +const char *MICS = "mics"; +const char *CORES_ONLY = "cores_only"; +const char *JOBID = "jobid"; +const char *CPUS = "cpus"; +const char *EXCLUSIVE = "exclusive"; +const char *OS_INDEX = "os_index"; diff --git a/src/test/req/test_uut.c b/src/test/req/test_uut.c index bd910ba5d5..f891eb0f5a 100644 --- a/src/test/req/test_uut.c +++ b/src/test/req/test_uut.c @@ -69,7 +69,7 @@ START_TEST(test_get_gpu_mode) r.set_value("gpus", "1", false); r.set_value("gpu_mode", "exclusive_thread", false); - fail_unless(r.getGpus() == 1); + fail_unless(r.get_gpus() == 1); gpu_mode = r.get_gpu_mode(); fail_unless(gpu_mode.compare("exclusive_thread") == 0); } @@ -136,7 +136,7 @@ START_TEST(test_string_constructor) req r3(strdup("2:ppn=8:fast:gpus=2")); fail_unless(r3.getTaskCount() == 2); fail_unless(r3.getExecutionSlots() == 8); - fail_unless(r3.getGpus() == 2); + fail_unless(r3.get_gpus() == 2); } END_TEST @@ -145,13 +145,13 @@ START_TEST(test_get_set_values) req r; r.set_value("index", "0", false); + r.set_value("task_count", "5", false); r.set_value("lprocs", "all", false); r.set_value("memory", "1024mb", false); r.set_value("swap", "1024kb", false); r.set_value("disk", "10000000kb", false); r.set_value("socket", "1", false); r.set_value("gpus", "2", false); - r.set_value("task_count", "5", false); r.set_value("gpu_mode", "exclusive_thread", false); r.set_value("mics", "1", false); r.set_value("thread_usage_policy", "use threads", false); @@ -172,39 +172,43 @@ START_TEST(test_get_set_values) fail_unless(names[0] == "task_count.0"); fail_unless(names[1] == "lprocs.0"); - fail_unless(names[2] == "memory.0"); - fail_unless(names[3] == "swap.0"); - fail_unless(names[4] == "disk.0"); - fail_unless(names[5] == "socket.0"); - fail_unless(names[6] == "gpus.0"); - fail_unless(names[7] == "gpu_mode.0"); - fail_unless(names[8] == "mics.0"); - fail_unless(names[9] == "thread_usage_policy.0", names[9].c_str()); - fail_unless(names[10] == "reqattr.0"); - fail_unless(names[11] == "gres.0"); - fail_unless(names[12] == "opsys.0"); - fail_unless(names[13] == "arch.0"); - fail_unless(names[14] == "features.0", names[14].c_str()); - fail_unless(names[15] == "single_job_access.0"); - fail_unless(names[16] == "hostlist.0", names[16].c_str()); + fail_unless(names[2] == "total_memory.0"); + fail_unless(names[3] == "memory.0"); + fail_unless(names[4] == "total_swap.0"); + fail_unless(names[5] == "swap.0"); + fail_unless(names[6] == "disk.0"); + fail_unless(names[7] == "socket.0"); + fail_unless(names[8] == "gpus.0"); + fail_unless(names[9] == "gpu_mode.0"); + fail_unless(names[10] == "mics.0"); + fail_unless(names[11] == "thread_usage_policy.0", names[9].c_str()); + fail_unless(names[12] == "reqattr.0"); + fail_unless(names[13] == "gres.0"); + fail_unless(names[14] == "opsys.0"); + fail_unless(names[15] == "arch.0"); + fail_unless(names[16] == "features.0", names[14].c_str()); + fail_unless(names[17] == "single_job_access.0"); + fail_unless(names[18] == "hostlist.0", names[16].c_str()); fail_unless(values[0] == "5"); fail_unless(values[1] == "all"); - fail_unless(values[2] == "1048576kb", "value: %s", values[2].c_str()); - fail_unless(values[3] == "1024kb"); - fail_unless(values[4] == "10000000kb"); - fail_unless(values[5] == "1"); - fail_unless(values[6] == "2"); - fail_unless(values[7] == "exclusive_thread"); - fail_unless(values[8] == "1"); - fail_unless(values[9] == "use threads"); - fail_unless(values[10] == "matlab>7"); - fail_unless(values[11] == "gresA"); - fail_unless(values[12] == "ubuntu"); - fail_unless(values[13] == "64bit"); - fail_unless(values[14] == "fast"); - fail_unless(values[15] == "true"); - fail_unless(values[16] == "napali:ppn=32"); + fail_unless(values[2] == "5242880kb", "value: %s", values[2].c_str()); + fail_unless(values[3] == "1048576kb", "value: %s", values[3].c_str()); + fail_unless(values[4] == "5120kb", "value: %s", values[4].c_str()); + fail_unless(values[5] == "1024kb"); + fail_unless(values[6] == "10000000kb"); + fail_unless(values[7] == "1"); + fail_unless(values[8] == "2"); + fail_unless(values[9] == "exclusive_thread"); + fail_unless(values[10] == "1"); + fail_unless(values[11] == "use threads"); + fail_unless(values[12] == "matlab>7"); + fail_unless(values[13] == "gresA"); + fail_unless(values[14] == "ubuntu"); + fail_unless(values[15] == "64bit"); + fail_unless(values[16] == "fast"); + fail_unless(values[17] == "true"); + fail_unless(values[18] == "napali:ppn=32"); req r2; r2.set_value("lprocs", "2", false); @@ -354,8 +358,8 @@ START_TEST(test_constructors) fail_unless(r.getOS().size() == 0); fail_unless(r.getGres().size() == 0); fail_unless(r.getDisk() == 0); - fail_unless(r.getSwap() == 0); - fail_unless(r.getMemory() == 0); + fail_unless(r.get_total_swap() == 0); + fail_unless(r.get_total_memory() == 0); fail_unless(r.getExecutionSlots() == 1, "slots: %d", r.getExecutionSlots()); std::string req2("5:lprocs=4:memory=12gb:place=socket=2:usecores:pack:gpus=2:mics=1:gres=matlab=1:feature=fast"); @@ -366,8 +370,9 @@ START_TEST(test_constructors) fail_unless(r2.getOS().size() == 0); fail_unless(r2.getGres() == "matlab=1"); fail_unless(r2.getDisk() == 0); - fail_unless(r2.getSwap() == 0); - fail_unless(r2.getMemory() == 12 * 1024 * 1024); + fail_unless(r2.get_total_swap() == 0); + fail_unless(r2.get_total_memory() == 12 * 1024 * 1024 * 5); + fail_unless(r2.get_memory_per_task() == 12 * 1024 * 1024); fail_unless(r2.getExecutionSlots() == 4); fail_unless(r2.getFeatures() == "fast", "features '%s'", r2.getFeatures().c_str()); @@ -378,8 +383,9 @@ START_TEST(test_constructors) fail_unless(copy_r2.getOS().size() == 0); fail_unless(copy_r2.getGres() == "matlab=1"); fail_unless(copy_r2.getDisk() == 0); - fail_unless(copy_r2.getSwap() == 0); - fail_unless(copy_r2.getMemory() == 12 * 1024 * 1024); + fail_unless(copy_r2.get_total_swap() == 0); + fail_unless(copy_r2.get_total_memory() == 12 * 1024 * 1024 * 5); + fail_unless(copy_r2.get_memory_per_task() == 12 * 1024 * 1024); fail_unless(copy_r2.getExecutionSlots() == 4); fail_unless(copy_r2.getFeatures() == "fast", "features '%s'", copy_r2.getFeatures().c_str()); @@ -463,9 +469,9 @@ START_TEST(test_equals_operator) r2 = r; fail_unless(r2.getExecutionSlots() == ALL_EXECUTION_SLOTS); - fail_unless(r2.getMemory() == 1024 * 1024 * 1024); + fail_unless(r2.get_total_memory() == 1024 * 1024 * 1024); fail_unless(r2.getMaxtpn() == 4); - fail_unless(r2.getGpuMode() == "exclusive_thread"); + fail_unless(r2.get_gpu_mode() == "exclusive_thread"); fail_unless(r2.getReqAttr() == "matlab>=7", "reqattr: '%s'", r2.getReqAttr().c_str()); // make sure this doesn't segfault @@ -519,8 +525,9 @@ START_TEST(test_set_from_string) fail_unless(r.getThreadUsageString() == "usecores", r.getThreadUsageString().c_str()); fail_unless(r.getFeatures() == "fast"); fail_unless(r.getExecutionSlots() == ALL_EXECUTION_SLOTS); - fail_unless(r.getMemory() == 10000); - fail_unless(r.getSwap() == 1024); + fail_unless(r.get_total_memory() == 100000); + fail_unless(r.get_memory_per_task() == 10000, "%d per task", r.get_memory_per_task()); + fail_unless(r.get_total_swap() == 10240); fail_unless(r.getDisk() == 10000000); fail_unless(r.getTaskCount() == 10); std::vector l; @@ -577,19 +584,31 @@ END_TEST START_TEST(test_get_memory_for_host) { req r; + req r2; std::string host = "napali"; unsigned long mem; r.set_value("index", "0", false); + r.set_value("task_count", "5", false); r.set_value("lprocs", "all", false); r.set_value("memory", "1024kb", false); r.set_value("swap", "1024kb", false); - r.set_value("task_count", "5", false); r.set_value("thread_usage_policy", "use threads", false); r.set_value("hostlist", "napali:ppn=32", false); + + r2.set_value("index", "0", false); + r2.set_value("lprocs", "all", false); + r2.set_value("memory", "1024kb", false); + r2.set_value("swap", "1024kb", false); + r2.set_value("task_count", "1", false); + r2.set_value("thread_usage_policy", "use threads", false); + r2.set_value("hostlist", "napali:ppn=32", false); mem = r.get_memory_for_host(host); - fail_unless(mem != 0); + fail_unless(mem == 1024); + + mem = r2.get_memory_for_host(host); + fail_unless(mem == 1024); host = "right said fred"; mem = r.get_memory_for_host(host); diff --git a/src/test/req_delete/scaffolding.c b/src/test/req_delete/scaffolding.c index d6bf832b7a..eb6f284e4c 100644 --- a/src/test/req_delete/scaffolding.c +++ b/src/test/req_delete/scaffolding.c @@ -37,48 +37,47 @@ int nanny = 1; bool br_freed; int alloc_work = 1; int depend_term_called; +int updated_array_values = 0; +bool find_job_fail = false; batch_request *alloc_br(int type) { if (alloc_work) - return((batch_request *)calloc(1, sizeof(batch_request))); + return(new batch_request()); else return(NULL); } job_array *get_jobs_array(job **pjob) { - return(NULL); + if (pjob == NULL) + return(NULL); + + return((job_array *)calloc(1, sizeof(job_array))); } void account_record(int acctype, job *pjob, const char *text) { - fprintf(stderr, "The call to acctype needs to be mocked!!\n"); - exit(1); } int job_save(job *pjob, int updatetype, int mom_port) { - fprintf(stderr, "The call to job_save needs to be mocked!!\n"); - exit(1); + return(0); } int svr_job_purge(job *pjob, int leaveSpoolFiles) { pjob->ji_qs.ji_state = JOB_STATE_COMPLETE; + pthread_mutex_unlock(pjob->ji_mutex); return(0); } void chk_job_req_permissions(job **pjob_ptr, struct batch_request *preq) { - fprintf(stderr, "The call to chk_job_req_permissions needs to be mocked!!\n"); - exit(1); } void svr_mailowner(job *pjob, int mailpoint, int force, const char *text) { - fprintf(stderr, "The call to svr_mailowner needs to be mocked!!\n"); - exit(1); } long attr_ifelse_long(pbs_attribute *attr1, pbs_attribute *attr2, long deflong) @@ -113,7 +112,6 @@ void free_nodes(job *pjob, const char *spec) void free_br(struct batch_request *preq) { - br_freed = true; } struct work_task *set_task(enum work_type type, long event_id, void (*func)(struct work_task *), void *parm, int get_lock) @@ -163,7 +161,7 @@ char *pbse_to_txt(int err) batch_request *cpy_stage(batch_request *preq, job *pjob, enum job_atr ati, int direction) { - return((batch_request *)calloc(1, sizeof(batch_request))); + return(new batch_request()); } int svr_setjobstate(job *pjob, int newstate, int newsubstate, int has_queue_mute) @@ -175,7 +173,9 @@ int svr_setjobstate(job *pjob, int newstate, int newsubstate, int has_queue_mut job *svr_find_job(const char *jobid, int get_subjob) { - if (strcmp(jobid, "1.napali") == 0) + if ((strcmp(jobid, "1.napali") == 0) || + ((strstr(jobid, "roshar") != NULL) && + (find_job_fail == false))) { job *pjob = new job(); strcpy(pjob->ji_qs.ji_jobid, jobid); @@ -228,7 +228,7 @@ int get_svr_attr_b(int index, bool *b) batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -262,7 +262,7 @@ int relay_to_mom(job **pjob_ptr, batch_request *request, void (*func)(struct w return(bad_relay); } -void removeBeforeAnyDependencies(const char *) {} +void removeBeforeAnyDependencies(job **pjob_ptr) {} /* @@ -359,40 +359,6 @@ int lock_ji_mutex( { int rc = PBSE_NONE; - char *err_msg = NULL; - char stub_msg[] = "no pos"; - - if (logging >= 10) - { - err_msg = (char *)calloc(1, MSG_LEN_LONG); - if (msg == NULL) - msg = stub_msg; - snprintf(err_msg, MSG_LEN_LONG, "locking %s in method %s-%s", pjob->ji_qs.ji_jobid, id, msg); - log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); - } - - if (pjob->ji_mutex != NULL) - { - if (pthread_mutex_lock(pjob->ji_mutex) != 0) - { - if (logging >= 20) - { - snprintf(err_msg, MSG_LEN_LONG, "ALERT: cannot lock job %s mutex in method %s", - pjob->ji_qs.ji_jobid, id); - log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); - } - rc = PBSE_MUTEX; - } - } - else - { - rc = -1; - log_err(rc, __func__, "Uninitialized mutex pass to pthread_mutex_lock!"); - } - - if (err_msg != NULL) - free(err_msg); - return rc; } @@ -406,42 +372,8 @@ int unlock_ji_mutex( { int rc = PBSE_NONE; - char *err_msg = NULL; - char stub_msg[] = "no pos"; - - if (logging >= 10) - { - err_msg = (char *)calloc(1, MSG_LEN_LONG); - if (msg == NULL) - msg = stub_msg; - snprintf(err_msg, MSG_LEN_LONG, "unlocking %s in method %s-%s", pjob->ji_qs.ji_jobid, id, msg); - log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); - } - - if (pjob->ji_mutex != NULL) - { - if (pthread_mutex_unlock(pjob->ji_mutex) != 0) - { - if (logging >= 20) - { - snprintf(err_msg, MSG_LEN_LONG, "ALERT: cannot unlock job %s mutex in method %s", - pjob->ji_qs.ji_jobid, id); - log_record(PBSEVENT_DEBUG, PBS_EVENTCLASS_NODE, id, err_msg); - } - rc = PBSE_MUTEX; - } - } - else - { - rc = -1; - log_err(rc, __func__, "Uninitialized mutex pass to pthread_mutex_unlock!"); - } - - if (err_msg != NULL) - free(err_msg); - - return rc; - } + return rc; + } void log_record( @@ -579,6 +511,8 @@ int depend_on_term( job::job() : ji_has_delete_nanny(false) { memset(this->ji_wattr, 0, sizeof(this->ji_wattr)); + this->ji_mutex = (pthread_mutex_t *)calloc(1, sizeof(pthread_mutex_t)); + pthread_mutex_init(this->ji_mutex, NULL); } job::~job() {} @@ -591,8 +525,49 @@ void job_array::update_array_values( int job_exit_status) { + updated_array_values++; } void job_array::mark_deleted() {} +batch_request::batch_request(const batch_request &other) + { + if (other.rq_extend == NULL) + this->rq_extend = NULL; + else + this->rq_extend = strdup(other.rq_extend); + } + +batch_request::~batch_request() + + { + br_freed = true; + } + +batch_request::batch_request() {} +batch_request::batch_request(int type) : rq_type(type) {} + +int client_to_svr( + + pbs_net_t hostaddr, /* I - internet addr of host */ + unsigned int port, /* I - port to which to connect */ + int local_port, /* I - BOOLEAN: not 0 to use local reserved port */ + char *EMsg) /* O (optional,minsize=1024) */ + + { + return(1); + } + +pbs_net_t get_hostaddr( + int *local_errno, /* O */ + const char *hostname) /* I */ + + { + return(0); + } + +int svr_authorize_jobreq(batch_request *brp, job *pjob) + { + return(-1); + } diff --git a/src/test/req_delete/test_uut.c b/src/test/req_delete/test_uut.c index 6c2f3c5a46..4dd24e3df1 100644 --- a/src/test/req_delete/test_uut.c +++ b/src/test/req_delete/test_uut.c @@ -21,15 +21,16 @@ int delete_inactive_job(job **, const char *); void force_purge_work(job *pjob); -void *delete_all_work(void *vp); +int delete_all_work(batch_request *); void ensure_deleted(struct work_task *ptask); int apply_job_delete_nanny(job *pjob, int delay); void job_delete_nanny(struct work_task *ptask); void post_job_delete_nanny(batch_request *preq_sig); int forced_jobpurge(job *pjob, batch_request *preq); void post_delete_mom2(struct work_task *pwt); -int handle_delete_all(batch_request *preq, batch_request *preq_tmp, char *Msg); -int handle_single_delete(batch_request *preq, batch_request *preq_tmp, char *Msg); +int handle_delete_all(batch_request *preq, char *Msg); +int handle_single_delete(batch_request *preq, char *Msg); +int perform_job_delete_array_bookkeeping(job *pjob, int cancel_exit_code); bool exit_called; extern int depend_term_called; extern long keep_seconds; @@ -39,6 +40,8 @@ extern int signal_issued; extern int nanny; extern bool br_freed; extern int alloc_work; +extern int updated_array_values; +extern bool find_job_fail; struct server server; extern const char *delpurgestr; @@ -75,60 +78,48 @@ int set_pbs_server_name() return 0; } +START_TEST(test_perform_job_delete_array_bookkeeping) + { + job *pjob = (job *)calloc(1, sizeof(job)); + sprintf(pjob->ji_arraystructid, "1[].roshar"); + sprintf(pjob->ji_qs.ji_jobid, "1[0].roshar"); + + find_job_fail = false; + updated_array_values = 0; + fail_unless(perform_job_delete_array_bookkeeping(pjob, 1) == PBSE_NONE); + fail_unless(updated_array_values == 1); + + find_job_fail = true; + fail_unless(perform_job_delete_array_bookkeeping(pjob, 1) != PBSE_NONE); + fail_unless(updated_array_values == 2); + } +END_TEST + START_TEST(test_handle_single_delete) { - batch_request *preq = (batch_request *)calloc(1,sizeof(batch_request)); + batch_request *preq = new batch_request(); strcpy(preq->rq_ind.rq_delete.rq_objname, "2.napali"); - fail_unless(handle_single_delete(preq, preq, NULL) == PBSE_NONE); + fail_unless(handle_single_delete(preq, NULL) == PBSE_NONE); fail_unless(preq->rq_noreply == FALSE); + // Now an asynchronous delete + preq->rq_extend = strdup(DELASYNC); strcpy(preq->rq_ind.rq_delete.rq_objname, "1.napali"); - fail_unless(handle_single_delete(preq, preq, NULL) == PBSE_NONE); - fail_unless(preq->rq_noreply == TRUE); + fail_unless(handle_single_delete(preq, NULL) == PBSE_NONE); + fail_unless(preq->rq_noreply == true); } END_TEST START_TEST(test_handle_delete_all) { batch_request preq; - memset(&preq, 0, sizeof(preq)); + preq.rq_extend = strdup(DELASYNC); - fail_unless(handle_delete_all(&preq, &preq, NULL) == PBSE_NONE); - fail_unless(preq.rq_noreply == TRUE); + fail_unless(handle_delete_all(&preq, NULL) == PBSE_NONE); + fail_unless(preq.rq_noreply == true); } END_TEST -START_TEST(test_duplicate_request) - { - batch_request *preq = (batch_request *)calloc(1, sizeof(batch_request)); - batch_request *dup; - alloc_work = 0; - fail_unless(duplicate_request(preq) == NULL); - - alloc_work = 1; - preq->rq_perm = 1; - strcpy(preq->rq_user, "dbeer"); - strcpy(preq->rq_host, "napali"); - preq->rq_extend = strdup("tom"); - preq->rq_type = PBS_BATCH_RunJob; - preq->rq_ind.rq_run.rq_destin = strdup("napali"); - - dup = duplicate_request(preq); - fail_unless(dup != NULL); - fail_unless(!strcmp(dup->rq_extend, "tom")); - fail_unless(!strcmp(dup->rq_user, "dbeer")); - fail_unless(!strcmp(dup->rq_host, "napali")); - fail_unless(!strcmp(dup->rq_extend, "tom")); - fail_unless(!strcmp(dup->rq_ind.rq_run.rq_destin, "napali")); - - preq->rq_type = PBS_BATCH_Rerun; - const char *rerun_jobid = "4.roshar"; - strcpy(preq->rq_ind.rq_rerun, rerun_jobid); - batch_request *rerun_dep = duplicate_request(preq, -1); - fail_unless(!strcmp(rerun_dep->rq_ind.rq_rerun, rerun_jobid)); - } -END_TEST - START_TEST(test_post_delete_mom2) { struct work_task *ptask; @@ -154,7 +145,7 @@ START_TEST(test_forced_jobpurge) batch_request *preq; pjob = new job(); - preq = (batch_request *)calloc(1, sizeof(batch_request)); + preq = new batch_request(); strcpy(pjob->ji_qs.ji_jobid, "1.napali"); memset(pjob->ji_arraystructid, 0, sizeof(pjob->ji_arraystructid)); @@ -175,14 +166,8 @@ END_TEST START_TEST(test_delete_all_work) { //struct all_jobs alljobs; - job *pjob; - batch_request *preq; - - pjob = new job(); - pjob->ji_mutex = (pthread_mutex_t *)calloc(1, sizeof(pthread_mutex_t)); - pthread_mutex_init(pjob->ji_mutex,NULL); - - preq = (batch_request *)calloc(1, sizeof(batch_request)); + job *pjob = new job(); + batch_request *preq = new batch_request(); preq->rq_extend = strdup(delpurgestr); nanny = 0; @@ -190,7 +175,8 @@ START_TEST(test_delete_all_work) // no lock should remain on job after delete_all_work() // so test by making sure we can set lock - fail_unless(delete_all_work((void *) preq) == NULL && pthread_mutex_trylock(pjob->ji_mutex) == 0); + fail_unless(delete_all_work(preq) == PBSE_NONE); + fail_unless(pthread_mutex_trylock(pjob->ji_mutex) == 0); nanny = 1; } @@ -200,37 +186,22 @@ START_TEST(test_post_job_delete_nanny) { batch_request *preq_sig; - br_freed = FALSE; - post_job_delete_nanny(NULL); - fail_unless(br_freed == FALSE); - - preq_sig = (batch_request *)calloc(1, sizeof(batch_request)); - br_freed = FALSE; + preq_sig = new batch_request(); nanny = 0; post_job_delete_nanny(preq_sig); - fail_unless(br_freed == TRUE); - preq_sig = (batch_request *)calloc(1, sizeof(batch_request)); - br_freed = FALSE; nanny = 1; strcpy(preq_sig->rq_ind.rq_signal.rq_jid, "2.napali"); post_job_delete_nanny(preq_sig); - fail_unless(br_freed == TRUE); - preq_sig = (batch_request *)calloc(1, sizeof(batch_request)); - br_freed = FALSE; nanny = 1; strcpy(preq_sig->rq_ind.rq_signal.rq_jid, "1.napali"); post_job_delete_nanny(preq_sig); - fail_unless(br_freed == TRUE); - preq_sig = (batch_request *)calloc(1, sizeof(batch_request)); - br_freed = FALSE; nanny = 1; strcpy(preq_sig->rq_ind.rq_signal.rq_jid, "1.napali"); preq_sig->rq_reply.brp_code = PBSE_UNKJOBID; post_job_delete_nanny(preq_sig); - fail_unless(br_freed == TRUE); } END_TEST @@ -321,6 +292,7 @@ START_TEST(test_force_purge_work) { job *pjob = new job(); + sprintf(pjob->ji_qs.ji_jobid, "17.roshar"); pjob->ji_wattr[JOB_ATR_exec_host].at_val.at_str = strdup("bob"); depend_term_called = 0; force_purge_work(pjob); @@ -416,9 +388,9 @@ Suite *req_delete_suite(void) suite_add_tcase(s, tc_core); tc_core = tcase_create("more"); - tcase_add_test(tc_core, test_duplicate_request); tcase_add_test(tc_core, test_handle_delete_all); tcase_add_test(tc_core, test_handle_single_delete); + tcase_add_test(tc_core, test_perform_job_delete_array_bookkeeping); suite_add_tcase(s, tc_core); return s; diff --git a/src/test/req_deletearray/scaffolding.c b/src/test/req_deletearray/scaffolding.c index 116a11bdef..00f6a39c77 100644 --- a/src/test/req_deletearray/scaffolding.c +++ b/src/test/req_deletearray/scaffolding.c @@ -153,7 +153,7 @@ int array_delete( batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); diff --git a/src/test/req_getcred/scaffolding.c b/src/test/req_getcred/scaffolding.c index 3938c2ba31..664524508f 100644 --- a/src/test/req_getcred/scaffolding.c +++ b/src/test/req_getcred/scaffolding.c @@ -70,3 +70,12 @@ unsigned long get_connectaddr(int sock, int mutex) void log_err(int errnum, const char *routine, const char *text) {} void log_record(int eventtype, int objclass, const char *objname, const char *text) {} void log_event(int eventtype, int objclass, const char *objname, const char *text) {} + +batch_request::batch_request() + { + } + +batch_request::~batch_request() + + { + } diff --git a/src/test/req_getcred/test_uut.c b/src/test/req_getcred/test_uut.c index bda5966f11..6b4242783b 100644 --- a/src/test/req_getcred/test_uut.c +++ b/src/test/req_getcred/test_uut.c @@ -15,7 +15,6 @@ START_TEST(test_one) batch_request req; - memset(&req,0,sizeof(batch_request)); memset(svr_conn,0,sizeof(svr_conn)); req.rq_ind.rq_authen.rq_port = 42; strcpy(req.rq_user,"Tron"); diff --git a/src/test/req_gpuctrl/scaffolding.c b/src/test/req_gpuctrl/scaffolding.c index d92c16dade..a51192b866 100644 --- a/src/test/req_gpuctrl/scaffolding.c +++ b/src/test/req_gpuctrl/scaffolding.c @@ -15,7 +15,7 @@ void req_reject(int code, int aux, struct batch_request *preq, const char *HostN batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); diff --git a/src/test/req_holdjob/scaffolding.c b/src/test/req_holdjob/scaffolding.c index 3de92296c5..70c3b83c42 100644 --- a/src/test/req_holdjob/scaffolding.c +++ b/src/test/req_holdjob/scaffolding.c @@ -119,7 +119,7 @@ int copy_batchrequest(struct batch_request **newreq, struct batch_request *preq, batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -161,3 +161,16 @@ int check_array_slot_limits(job *pjob, job_array *pa) { return(0); } + +batch_request::batch_request() + { + } + +batch_request::batch_request(const batch_request &other) + { + } + +batch_request::~batch_request() + + { + } diff --git a/src/test/req_jobobit/scaffolding.c b/src/test/req_jobobit/scaffolding.c index c564206560..0be4e1f413 100644 --- a/src/test/req_jobobit/scaffolding.c +++ b/src/test/req_jobobit/scaffolding.c @@ -63,20 +63,10 @@ bool purged = false; bool completed = false; bool exited = false; long disable_requeue = 0; +std::string log_err_buf; completed_jobs_map_class completed_jobs_map; -struct batch_request *alloc_br(int type) - { - batch_request *preq; - if (alloc_br_null) - return(NULL); - - preq = (batch_request *)calloc(1, sizeof(batch_request)); - preq->rq_type = type; - - return(preq); - } char *parse_servername(const char *name, unsigned int *service) { @@ -209,7 +199,9 @@ job *svr_find_job(const char *jobid, int get_subjob) if (bad_job == 0) { - pjob = (job *)calloc(1, sizeof(job)); + pjob = new job(); + + memset(pjob->ji_wattr, 0, sizeof(pjob->ji_wattr)); strcpy(pjob->ji_qs.ji_jobid, jobid); pjob->ji_wattr[JOB_ATR_reported].at_flags = ATR_VFLAG_SET; @@ -267,7 +259,7 @@ int safe_strncat(char *str, const char *to_append, size_t space_remaining) batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -338,7 +330,10 @@ void log_event(int eventtype, int objclass, const char *objname, const char *tex { } -void log_err(int error, const char *func_id, const char *msg) {} +void log_err(int error, const char *func_id, const char *msg) + { + log_err_buf = msg; + } void log_record(int eventtype, int objclass, const char *objname, const char *text) {} @@ -516,3 +511,15 @@ void job_array::update_array_values( { } +batch_request::batch_request(int type) : rq_type(type) + { + } + +batch_request::batch_request() + { + } + +batch_request::~batch_request() + + { + } diff --git a/src/test/req_jobobit/test_uut.c b/src/test/req_jobobit/test_uut.c index 233d4b0d30..1ddc1ac215 100644 --- a/src/test/req_jobobit/test_uut.c +++ b/src/test/req_jobobit/test_uut.c @@ -61,6 +61,7 @@ extern long disable_requeue; extern int attr_count; extern int next_count; extern int called_account_jobend; +extern std::string log_err_buf; void init_server() @@ -206,7 +207,6 @@ START_TEST(return_stdfile_test) batch_request *p1 = &preq; batch_request *p2; - memset(&preq, 0, sizeof(preq)); pjob.ji_wattr[JOB_ATR_checkpoint_name].at_flags = ATR_VFLAG_SET; fail_unless(return_stdfile(&preq, &pjob, JOB_ATR_outpath) != NULL); @@ -278,8 +278,6 @@ START_TEST(setup_cpyfiles_test) { job *pjob = new job(); batch_request *preq; - alloc_br_null = 1; - fail_unless(setup_cpyfiles(NULL, pjob, strdup("from"), strdup("to"), 1, 1) == NULL); alloc_br_null = 0; strcpy(pjob->ji_qs.ji_jobid, "1.napali"); @@ -303,7 +301,7 @@ START_TEST(handle_returnstd_test) job *pjob; pjob = new job(); - preq = (batch_request *)calloc(1, sizeof(batch_request)); + preq = new batch_request(); strcpy(pjob->ji_qs.ji_jobid, "1.napali"); strcpy(pjob->ji_qs.ji_fileprefix, "1.napali"); @@ -673,7 +671,13 @@ START_TEST(update_substate_from_exit_status_test) fail_unless(update_substate_from_exit_status(pjob, &alreadymailed,"Some random message") == PBSE_NONE); fail_unless(pjob->ji_qs.ji_substate == JOB_SUBSTATE_RUNNING, "Shouldn't update substate when job is being deleted"); - + + pjob->ji_being_deleted = false; + pjob->ji_qs.ji_substate = JOB_SUBSTATE_RUNNING; + pjob->ji_qs.ji_un.ji_exect.ji_exitstat = JOB_EXEC_RETRY_PROLOGUE; + fail_unless(update_substate_from_exit_status(pjob, &alreadymailed,"Some random message") == PBSE_NONE); + fail_unless(pjob->ji_qs.ji_substate == JOB_SUBSTATE_RERUN); + fail_unless(log_err_buf.find("prologue") != std::string::npos); } END_TEST diff --git a/src/test/req_manager/scaffolding.c b/src/test/req_manager/scaffolding.c index 36595e05e6..1cd12c1ead 100644 --- a/src/test/req_manager/scaffolding.c +++ b/src/test/req_manager/scaffolding.c @@ -512,3 +512,8 @@ acl_special::acl_special() {} acl_special limited_acls; +batch_request::batch_request(int type) : rq_type(type) + { + } + +batch_request::~batch_request() {} diff --git a/src/test/req_manager/test_uut.c b/src/test/req_manager/test_uut.c index 7a44842be4..966ee1579b 100644 --- a/src/test/req_manager/test_uut.c +++ b/src/test/req_manager/test_uut.c @@ -19,15 +19,15 @@ START_TEST(test_check_default_gpu_mode_str) pattr.at_val.at_str = strdup("exclusive_process"); fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_NONE); fail_unless(default_gpu_mode == gpu_exclusive_process); + pattr.at_val.at_str = strdup("exclusive_thread"); fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_NONE); fail_unless(default_gpu_mode == gpu_exclusive_thread); - pattr.at_val.at_str = strdup("exclusive"); - fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_NONE); - fail_unless(default_gpu_mode == gpu_exclusive); + pattr.at_val.at_str = strdup("default"); fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_NONE); fail_unless(default_gpu_mode == gpu_normal); + pattr.at_val.at_str = strdup("shared"); fail_unless(check_default_gpu_mode_str(&pattr, NULL, ATR_ACTION_ALTER) == PBSE_NONE); fail_unless(default_gpu_mode == gpu_normal); diff --git a/src/test/req_message/scaffolding.c b/src/test/req_message/scaffolding.c index 6bbca748fb..a18fbeb40c 100644 --- a/src/test/req_message/scaffolding.c +++ b/src/test/req_message/scaffolding.c @@ -42,7 +42,7 @@ void free_br(struct batch_request *preq) {} batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -68,7 +68,15 @@ int relay_to_mom(job **pjob_ptr, batch_request *request, void (*func)(struct w return(0); } -batch_request *duplicate_request(batch_request *preq, int type) +batch_request::batch_request(const batch_request &other) + { + } + +batch_request::batch_request() + { + } + +batch_request::~batch_request() + { - return(NULL); } diff --git a/src/test/req_modify/scaffolding.c b/src/test/req_modify/scaffolding.c index 49903af258..fe7a15080e 100644 --- a/src/test/req_modify/scaffolding.c +++ b/src/test/req_modify/scaffolding.c @@ -27,7 +27,7 @@ int LOGLEVEL = 7; /* force logging code to be exercised as tests run */ static int acl_check_n = 0; -char *get_correct_jobname_return; +std::string get_correct_jobname_return; struct batch_request *alloc_br(int type) { @@ -218,7 +218,7 @@ int get_svr_attr_b(int index, bool *b) batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -270,12 +270,16 @@ batch_request *duplicate_request(batch_request *preq, int type) return(NULL); } -char *get_correct_jobname(char const *p) +const char *get_correct_jobname(const char *jobid, std::string &correct) { - if (get_correct_jobname_return == NULL) - return(strdup(p)); + if (get_correct_jobname_return.size() == 0) + { + correct = jobid; + } + else + correct = get_correct_jobname_return; - return(get_correct_jobname_return); + return(correct.c_str()); } job *chk_job_request(char *p, batch_request *b) @@ -330,3 +334,22 @@ void job::set_plugin_resource_usage( } void update_slot_held_jobs(job_array *pa, int num_to_release) {} + +batch_request::batch_request(const batch_request &other) + { + } + +batch_request::batch_request(int type) + { + } + +void batch_request::update_object_id(int index) {} + +batch_request::batch_request() + { + } + +batch_request::~batch_request() + + { + } diff --git a/src/test/req_modify/test_uut.c b/src/test/req_modify/test_uut.c index f57ee83ddd..9a229fbc2e 100644 --- a/src/test/req_modify/test_uut.c +++ b/src/test/req_modify/test_uut.c @@ -7,14 +7,14 @@ batch_request *preq; -extern char *get_correct_jobname_return; +extern std::string get_correct_jobname_return; bool allowed_gres_modifier(const char*, const char *); START_TEST(test_req_modifyjob) { // initialize - get_correct_jobname_return = NULL; + get_correct_jobname_return.clear(); // create space for the req preq = (batch_request *)calloc(1, sizeof(batch_request)); @@ -32,7 +32,7 @@ START_TEST(test_req_modifyjob) sizeof(preq->rq_ind.rq_modify.rq_objname))); // set up get_correct_jobname() to return a different value - get_correct_jobname_return = strdup("123"); + get_correct_jobname_return = "123"; // call the function to unit test req_modifyjob(preq); diff --git a/src/test/req_quejob/scaffolding.c b/src/test/req_quejob/scaffolding.c index 0aa3869681..324b2c34d9 100644 --- a/src/test/req_quejob/scaffolding.c +++ b/src/test/req_quejob/scaffolding.c @@ -526,6 +526,13 @@ int req_runjob(batch_request *preq) return(0); } +batch_request::batch_request() {} +batch_request::batch_request(int type) : rq_type(type) {} + +batch_request::~batch_request() + + { + } diff --git a/src/test/req_quejob/test_uut.c b/src/test/req_quejob/test_uut.c index fef4327be6..6e2f810084 100644 --- a/src/test/req_quejob/test_uut.c +++ b/src/test/req_quejob/test_uut.c @@ -103,12 +103,11 @@ END_TEST START_TEST(test_one) { - struct batch_request req; + batch_request req; job j; char cmd[500]; memset(&j,0,sizeof(j)); - memset(&req,0,sizeof(req)); fail_unless(req_jobcredential(&req) == PBSE_IVALREQ); @@ -120,17 +119,16 @@ START_TEST(test_one) fail_unless(req_jobcredential(&req) == PBSE_NONE); - memset(&req,0,sizeof(req)); - - strcpy(req.rq_ind.rq_jobfile.rq_jobid,"NotThisJob"); - fail_unless(req_jobscript(&req, false) == PBSE_IVALREQ); + batch_request req2; + strcpy(req2.rq_ind.rq_jobfile.rq_jobid,"NotThisJob"); + fail_unless(req_jobscript(&req2, false) == PBSE_IVALREQ); path_jobs = getcwd(path_to_jobs,sizeof(path_to_jobs)); strcat(path_jobs,"/"); sprintf(cmd,"rm -f %s*.SC",path_jobs); system(cmd); - strcpy(req.rq_ind.rq_jobfile.rq_jobid,"1.napali"); - fail_unless(req_jobscript(&req, false) == PBSE_NONE); + strcpy(req2.rq_ind.rq_jobfile.rq_jobid,"1.napali"); + fail_unless(req_jobscript(&req2, false) == PBSE_NONE); system(cmd); } END_TEST @@ -143,8 +141,6 @@ START_TEST(test_get_job_id) int created_here = 0; batch_request preq; - memset(&preq, 0, sizeof(preq)); - preq.rq_fromsvr = 1; strcpy(preq.rq_ind.rq_queuejob.rq_jid, "1.napali"); fail_unless(get_job_id(&preq, resc_access_perm, created_here, job_id) == PBSE_NONE); @@ -193,9 +189,7 @@ END_TEST START_TEST(test_req_commit) { - struct batch_request *preq; - - fail_unless((preq = (struct batch_request *)calloc(1, sizeof(struct batch_request))) != NULL); + struct batch_request *preq = new batch_request(); strcpy(preq->rq_ind.rq_commit, "1.napali"); default_queue = false; diff --git a/src/test/req_register/scaffolding.c b/src/test/req_register/scaffolding.c index 01c1d1fdae..27333d1efe 100644 --- a/src/test/req_register/scaffolding.c +++ b/src/test/req_register/scaffolding.c @@ -11,6 +11,7 @@ #include "array.h" /* job_array */ #include "work_task.h" /* work_task */ #include "queue.h" +#include "threadpool.h" const int DEFAULT_IDLE_SLOT_LIMIT = 300; const char *msg_illregister = "Illegal op in register request received for job %s"; @@ -25,7 +26,7 @@ const char *msg_registerrel = "Dependency on job %s released."; int i = 2; int svr = 2; int is_attr_set; - +int job_aborted; struct batch_request *alloc_br(int type) { @@ -121,6 +122,7 @@ int svr_chk_owner(struct batch_request *preq, job *pjob) int job_abt(struct job **pjobp, const char *text, bool b=false) { *pjobp = NULL; + job_aborted++; return(0); } @@ -167,7 +169,7 @@ void append_link(tlist_head *head, list_link *new_link, void *pobj) new_link->ll_prior->ll_next = new_link; /* now visible to forward iteration */ } -int issue_to_svr(const char *servern, struct batch_request **preq, void (*replyfunc)(struct work_task *)) +int issue_to_svr(const char *servern, batch_request *preq, void (*replyfunc)(struct work_task *)) { fprintf(stderr, "The call to issue_to_svr to be mocked!!\n"); exit(1); @@ -308,7 +310,7 @@ int unlock_queue(struct pbs_queue *the_queue, const char *id, const char *msg, i batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -357,7 +359,7 @@ job *job_alloc(void) return(pj); } -char *get_correct_jobname(const char *jobid) +const char *get_correct_jobname(const char *jobid, std::string &correct) { char *rv = strdup(jobid); @@ -366,7 +368,10 @@ char *get_correct_jobname(const char *jobid) if ((dot = strchr(rv, '.')) != NULL) *dot = '\0'; - return(rv); + correct = rv; + free(rv); + + return(correct.c_str()); } int is_svr_attr_set(int index) @@ -377,6 +382,7 @@ int is_svr_attr_set(int index) job::job() : ji_rejectdest() { + this->ji_wattr[JOB_ATR_hold].at_flags = 0; } job::~job() {} @@ -396,3 +402,37 @@ job_array::job_array() : job_ids(NULL), jobs_recovered(0), ai_ghost_recovered(fa { } + +batch_request::~batch_request() + + { + } + +batch_request::batch_request() + + { + } + +batch_request::batch_request(int type) : rq_type(type) + + { + } + +int enqueue_threadpool_request( + + void *(*func)(void *), + void *arg, + threadpool_t *tp) + + { + return(0); + } + +threadpool_t *task_pool; + +void *svr_job_purge_task(void *vp) + { + return(NULL); + } + + diff --git a/src/test/req_register/test_uut.c b/src/test/req_register/test_uut.c index 92c27341e6..762f8af280 100644 --- a/src/test/req_register/test_uut.c +++ b/src/test/req_register/test_uut.c @@ -39,12 +39,14 @@ int req_register(batch_request *preq); bool remove_array_dependency_job_from_job(struct array_depend *pdep, job *pjob, char *job_array_id); void removeAfterAnyDependency(const char *pJobID, const char *targetJob); bool job_ids_match(const char *parent, const char *child); +int depend_on_que(pbs_attribute *pattr, void *pj, int mode); -extern char server_name[]; -extern int i; -extern int svr; -extern int is_attr_set; +extern char server_name[]; +extern int i; +extern int svr; +extern int is_attr_set; +extern int job_aborted; char *job1 = (char *)"1.napali"; char *job2 = (char *)"2.napali"; @@ -62,6 +64,26 @@ void initialize_depend_attr( } /* END initialize_depend_attr() */ +START_TEST(test_depend_on_que) + { + job *pjob = (job *)calloc(1, sizeof(job)); + depend *pdep; + initialize_depend_attr(pjob->ji_wattr + JOB_ATR_depend); + pdep = make_depend(JOB_DEPEND_TYPE_AFTERNOTOK, pjob->ji_wattr + JOB_ATR_depend); + make_dependjob(pdep, job2); + + // These three modes should do nothing + fail_unless(depend_on_que(pjob->ji_wattr + JOB_ATR_depend, pjob, ATR_ACTION_NEW) == PBSE_NONE); + fail_unless(depend_on_que(pjob->ji_wattr + JOB_ATR_depend, pjob, ATR_ACTION_FREE) == PBSE_NONE); + fail_unless(depend_on_que(pjob->ji_wattr + JOB_ATR_depend, pjob, ATR_ACTION_RECOV) == PBSE_NONE); + + fail_unless(depend_on_que(pjob->ji_wattr + JOB_ATR_depend, pjob, ATR_ACTION_ALTER) == PBSE_NONE); + fail_unless((pjob->ji_wattr[JOB_ATR_hold].at_flags & ATR_VFLAG_SET) != 0); + fail_unless(pjob->ji_qs.ji_state == JOB_STATE_HELD); + } +END_TEST + + START_TEST(test_job_ids_match) { is_attr_set = 1; @@ -92,7 +114,7 @@ END_TEST START_TEST(set_array_depend_holds_test) { - batch_request *preq = (batch_request *)calloc(1, sizeof(batch_request)); + batch_request *preq = new batch_request(); job_array *pa = new job_array(); strcpy(preq->rq_ind.rq_register.rq_child, job1); @@ -101,6 +123,43 @@ START_TEST(set_array_depend_holds_test) fail_unless(register_array_depend(pa, preq, JOB_DEPEND_TYPE_AFTEROKARRAY, 10) == PBSE_NONE); pa->ai_qs.num_successful = 12; fail_unless(set_array_depend_holds(pa) == true); + + // Make sure we abort the job when the dependency can't be fulfilled + job_aborted = 0; + pa->ai_qs.deps.clear(); + fail_unless(register_array_depend(pa, preq, JOB_DEPEND_TYPE_AFTEROKARRAY, 1) == PBSE_NONE); + pa->ai_qs.num_successful = 0; + pa->ai_qs.num_failed = 10; + pa->ai_qs.jobs_done = 10; + pa->ai_qs.num_jobs = 10; + fail_unless(set_array_depend_holds(pa) == false); + fail_unless(job_aborted == 1); + + // Make sure we don't abort a job just because the dependency isn't fulfilled + pa->ai_qs.deps.clear(); + fail_unless(register_array_depend(pa, preq, JOB_DEPEND_TYPE_AFTERNOTOKARRAY, 1) == PBSE_NONE); + pa->ai_qs.num_successful = 10; + pa->ai_qs.num_failed = 0; + pa->ai_qs.jobs_done = 10; + pa->ai_qs.num_jobs = 20; + fail_unless(set_array_depend_holds(pa) == false); + fail_unless(job_aborted == 1); // Still 1 + + // Now abort it + pa->ai_qs.num_successful = 20; + pa->ai_qs.jobs_done = 20; + fail_unless(set_array_depend_holds(pa) == false); + fail_unless(job_aborted == 2); + + // Abort afterstart as well + pa->ai_qs.deps.clear(); + fail_unless(register_array_depend(pa, preq, JOB_DEPEND_TYPE_AFTERSTARTARRAY, 1) == PBSE_NONE); + pa->ai_qs.num_successful = 0; + pa->ai_qs.num_failed = 10; + pa->ai_qs.jobs_done = 10; + pa->ai_qs.num_jobs = 10; + fail_unless(set_array_depend_holds(pa) == false); + fail_unless(job_aborted == 3); } END_TEST @@ -111,7 +170,7 @@ START_TEST(check_dependency_job_test) batch_request preq; char *jobid = strdup("1.napali"); - memset(&preq, 0, sizeof(preq)); + preq.rq_fromsvr = 0; fail_unless(check_dependency_job(NULL, NULL, NULL) == PBSE_BAD_PARAMETER, "passed bad params?"); fail_unless(check_dependency_job(jobid, &preq, &pjob) == PBSE_IVALREQ, "accepted non-server request?"); @@ -127,7 +186,8 @@ START_TEST(check_dependency_job_test) fail_unless(check_dependency_job((char *)"bob", &preq, &pjob) == PBSE_JOBNOTFOUND, "wrong rc"); fail_unless(pjob == NULL, "didn't set job to NULL"); svr = 2; - + + preq.rq_ind.rq_register.rq_dependtype = JOB_DEPEND_TYPE_AFTERANY; fail_unless(check_dependency_job((char *)"2.napali", &preq, &pjob) == PBSE_BADSTATE, "wrong rc"); } END_TEST @@ -280,7 +340,6 @@ START_TEST(register_dep_test) int made = 0; initialize_depend_attr(&pattr); - memset(&preq, 0, sizeof(preq)); strcpy(preq.rq_ind.rq_register.rq_svr, host); strcpy(preq.rq_ind.rq_register.rq_child, job1); @@ -300,7 +359,6 @@ START_TEST(unregister_dep_test) struct depend *pdep; batch_request preq; - memset(&preq, 0, sizeof(preq)); strcpy(preq.rq_ind.rq_register.rq_svr, host); strcpy(preq.rq_ind.rq_register.rq_child, job1); preq.rq_ind.rq_register.rq_dependtype = 1; @@ -428,7 +486,7 @@ END_TEST START_TEST(req_register_test) { - batch_request *preq = (batch_request *)calloc(1, sizeof(batch_request)); + batch_request *preq = new batch_request(); strcpy(preq->rq_ind.rq_register.rq_parent, job1); preq->rq_fromsvr = 1; @@ -557,7 +615,6 @@ START_TEST(register_before_dep_test) int rc = 0; char buf[1000]; - memset(&preq, 0, sizeof(preq)); memset(&pjob, 0, sizeof(pjob)); strcpy(preq.rq_ind.rq_register.rq_owner, "dbeer"); @@ -587,7 +644,6 @@ START_TEST(register_dependency_test) job pjob; pbs_attribute *pattr; - memset(&preq, 0, sizeof(preq)); memset(&pjob, 0, sizeof(pjob)); strcpy(preq.rq_ind.rq_register.rq_owner, "dbeer"); @@ -614,7 +670,6 @@ START_TEST(release_before_dependency_test) pbs_attribute *pattr; struct depend *pdep; - memset(&preq, 0, sizeof(preq)); memset(&pjob, 0, sizeof(pjob)); strcpy(preq.rq_ind.rq_register.rq_owner, "dbeer"); @@ -640,7 +695,6 @@ START_TEST(release_syncwith_dependency_test) struct depend *pdep; batch_request preq; - memset(&preq, 0, sizeof(preq)); memset(&pjob, 0, sizeof(pjob)); pattr = &pjob.ji_wattr[JOB_ATR_depend]; initialize_depend_attr(pattr); @@ -664,7 +718,7 @@ START_TEST(set_depend_hold_test) struct depend *pdep; strcpy(pjob.ji_qs.ji_jobid, job1); - strcpy(pjob2.ji_qs.ji_jobid, job3); + strcpy(pjob2.ji_qs.ji_jobid, job2); pattr = &pjob.ji_wattr[JOB_ATR_depend]; initialize_depend_attr(pattr); // Job 2 will be complete with an exit status of 0 @@ -682,9 +736,17 @@ START_TEST(set_depend_hold_test) // Job 2 will be complete with an exit status of 0 make_dependjob(pdep, job2); - set_depend_hold(&pjob2, pattr, NULL); - fail_unless((pjob2.ji_wattr[JOB_ATR_hold].at_flags & ATR_VFLAG_SET) == 0); - fail_unless(pjob2.ji_qs.ji_state != JOB_STATE_HELD); + int saved_err = 0; + try + { + set_depend_hold(&pjob2, pattr, NULL); + } + catch (int err) + { + saved_err = err; + } + + fail_unless(saved_err == PBSE_BADDEPEND); memset(&pjob, 0, sizeof(pjob)); pattr = &pjob.ji_wattr[JOB_ATR_depend]; @@ -706,7 +768,6 @@ START_TEST(delete_dependency_job_test) job *pjob = job_alloc(); batch_request preq; - memset(&preq, 0, sizeof(preq)); strcpy(preq.rq_ind.rq_register.rq_parent, job1); strcpy(preq.rq_ind.rq_register.rq_child, job1); @@ -773,10 +834,7 @@ START_TEST(remove_after_any_test) pattr = &pJob->ji_wattr[JOB_ATR_depend]; pdep = find_depend(JOB_DEPEND_TYPE_AFTERANY,pattr); - pj = find_dependjob(pdep,pTJob->ji_qs.ji_jobid); - fail_unless((pj == NULL),"Dependency not deleted."); - pj = find_dependjob(pdep,pOJob->ji_qs.ji_jobid); - fail_unless((pj == NULL),"Dependency not deleted."); + fail_unless(pdep == NULL, "Dependency not deleted"); } END_TEST @@ -808,6 +866,7 @@ Suite *req_register_suite(void) tc_core = tcase_create("cat_jobsvr_test"); tcase_add_test(tc_core, cat_jobsvr_test); tcase_add_test(tc_core, fast_strcat_test); + tcase_add_test(tc_core, test_depend_on_que); tc_core = tcase_create("set_depend_test"); tcase_add_test(tc_core, set_depend_test); diff --git a/src/test/req_rerun/scaffolding.c b/src/test/req_rerun/scaffolding.c index 262007e1e7..5e90a95cc7 100644 --- a/src/test/req_rerun/scaffolding.c +++ b/src/test/req_rerun/scaffolding.c @@ -96,7 +96,7 @@ job *chk_job_request(char *jobid, struct batch_request *preq) batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -182,3 +182,17 @@ batch_request *duplicate_request( { return(NULL); } + +batch_request::batch_request(const batch_request &other) + { + } + +batch_request::batch_request() : rq_perm(0), rq_fromsvr(0) + { + } + +batch_request::~batch_request() + + { + } + diff --git a/src/test/req_rerun/test_uut.c b/src/test/req_rerun/test_uut.c index 8451848ae1..7fe37d78eb 100644 --- a/src/test/req_rerun/test_uut.c +++ b/src/test/req_rerun/test_uut.c @@ -35,8 +35,6 @@ START_TEST(test_handle_requeue_all) { batch_request preq; - memset(&preq, 0, sizeof(preq)); - // fail due to lack of permissions fail_unless(handle_requeue_all(&preq) == PBSE_PERM); diff --git a/src/test/req_runjob/scaffolding.c b/src/test/req_runjob/scaffolding.c index e8b056e00c..b0c1824243 100644 --- a/src/test/req_runjob/scaffolding.c +++ b/src/test/req_runjob/scaffolding.c @@ -302,7 +302,7 @@ int unlock_queue(struct pbs_queue *the_queue, const char *id, const char *msg, i batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -403,3 +403,16 @@ void job_array::update_array_values( { } +batch_request::~batch_request() + + { + } + +batch_request::batch_request(const batch_request &other) + + { + } + +batch_request::batch_request() + { + } diff --git a/src/test/req_runjob/test_uut.c b/src/test/req_runjob/test_uut.c index c6434ade0c..af94cfd829 100644 --- a/src/test/req_runjob/test_uut.c +++ b/src/test/req_runjob/test_uut.c @@ -38,7 +38,6 @@ START_TEST(test_get_mail_text) const char *txt = "tom is bob"; char *ret_txt; - memset(&preq, 0, sizeof(preq)); preq.rq_reply.brp_un.brp_txt.brp_txtlen = 999000001; preq.rq_reply.brp_un.brp_txt.brp_str = strdup(txt); fail_unless(get_mail_text(&preq, "1.napali") == NULL); @@ -54,7 +53,6 @@ START_TEST(test_two) { struct batch_request request; job myjob; - memset(&request, 0, sizeof(struct batch_request)); memset(&myjob, 0, sizeof(job)); myjob.ji_qs.ji_state = JOB_STATE_RUNNING; myjob.ji_qs.ji_un.ji_exect.ji_momaddr = 167838724; diff --git a/src/test/req_shutdown/scaffolding.c b/src/test/req_shutdown/scaffolding.c index 9af49c6f57..30056aa97f 100644 --- a/src/test/req_shutdown/scaffolding.c +++ b/src/test/req_shutdown/scaffolding.c @@ -116,7 +116,7 @@ int unlock_queue(struct pbs_queue *the_queue, const char *id, const char *msg, i batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -158,3 +158,7 @@ AvlTree AVL_clear_tree(AvlTree a) fprintf(stderr, "The call to AVL_clear_tree needs to be mocked!!\n"); exit(1); } + +batch_request::batch_request(const batch_request &other) {} +batch_request::~batch_request() {} + diff --git a/src/test/req_signal/scaffolding.c b/src/test/req_signal/scaffolding.c index 41f8b053b1..70cefb9d1a 100644 --- a/src/test/req_signal/scaffolding.c +++ b/src/test/req_signal/scaffolding.c @@ -7,16 +7,21 @@ #include "node_func.h" /* node_info */ #include "attribute.h" +bool find_job = true; +bool nullify_pointer = false; +int unlocked_job = 0; +int found_job = 0; +int relay_rc = 0; attribute_def job_attr_def[10]; int LOGLEVEL = 7; /* force logging code to be exercised as tests run */ char scaff_buffer[1024]; -struct batch_request *alloc_br(int type) +batch_request *alloc_br(int type) { - fprintf(stderr, "The call to alloc_br to be mocked!!\n"); - exit(1); + static batch_request preq; + return(&preq); } void set_old_nodes(job *pjob) @@ -63,10 +68,20 @@ char *pbse_to_txt(int err) job *svr_find_job(const char *jobid, int get_subjob) { - fprintf(stderr, "The call to find_job to be mocked!!\n"); - exit(1); + static job pjob; + + if (find_job) + { + found_job++; + strcpy(pjob.ji_qs.ji_jobid, jobid); + return(&pjob); + } + + return(NULL); } + + job *chk_job_request(char *jobid, struct batch_request *preq) { //We're mocking so that the following field has the address of a job struct. @@ -82,7 +97,7 @@ void free_br(struct batch_request *b) {} batch_request *get_remove_batch_request( - char *br_id) + const char *br_id) { return(NULL); @@ -98,6 +113,8 @@ int get_batch_request_id( int unlock_ji_mutex(job *pjob, const char *id, const char *msg, int logging) { + unlocked_job++; + return(0); } @@ -111,7 +128,10 @@ void log_record(int eventtype, int objclass, const char *objname, const char *te int relay_to_mom(job **pjob_ptr, batch_request *request, void (*func)(struct work_task *)) { - return(0); + if (nullify_pointer) + *pjob_ptr = NULL; + + return(relay_rc); } batch_request *duplicate_request(batch_request *preq, int type) @@ -134,7 +154,11 @@ char * netaddr_long(long ap, char *out) return(NULL); } -job::job() {} +job::job() + { + memset(&this->ji_qs, 0, sizeof(this->ji_qs)); + } + job::~job() {} int svr_setjobstate( @@ -163,4 +187,20 @@ void rel_resc( return; } +batch_request::~batch_request() + { + } + +batch_request::batch_request(const batch_request &other) + + { + } + +batch_request::batch_request() + { + } + +batch_request::batch_request(int type) : rq_type(type) + { + } diff --git a/src/test/req_signal/test_uut.c b/src/test/req_signal/test_uut.c index de60063f75..d201ea4895 100644 --- a/src/test/req_signal/test_uut.c +++ b/src/test/req_signal/test_uut.c @@ -4,13 +4,22 @@ #include #include #include "pbs_error.h" +#include "batch_request.h" +#include "attribute.h" + +int issue_signal(job **, const char *, void (*func)(batch_request *), void *extra, char *extend); + extern char scaff_buffer[]; +extern int unlocked_job; +extern int found_job; +extern int relay_rc; +extern bool nullify_pointer; +extern bool find_job; START_TEST(test_req_signaljob_relaying_msg) { struct batch_request request; job myjob; - memset(&request, 0, sizeof(struct batch_request)); memset(&myjob, 0, sizeof(job)); myjob.ji_qs.ji_state = JOB_STATE_RUNNING; myjob.ji_qs.ji_un.ji_exect.ji_momaddr = 167838724; @@ -22,13 +31,74 @@ START_TEST(test_req_signaljob_relaying_msg) } END_TEST -START_TEST(test_two) +void free_null(pbs_attribute *attr); +void dummy_func(batch_request *preq) {} + +void init_attr_def() + + { + extern attribute_def job_attr_def[]; + + for (int i = 0; i < 10; i++) + job_attr_def[i].at_free = free_null; + } + + +START_TEST(test_issue_signal) { + job pjob; + memset(&pjob, 0, sizeof(job)); + strcpy(pjob.ji_qs.ji_jobid, "1.nalthis"); + job *ptr = &pjob; + + found_job = 0; + unlocked_job = 0; + relay_rc = 1; + + // Make sure that if we have a non-NULL job pointer the mutex is locked + // found_job is set each time it's 'locked' in svr_find_job and unlocked_job is set each + // time it's 'unlocked' in unlock_ji_mutex() + fail_unless(issue_signal(&ptr, "SIGKILL", dummy_func, NULL, strdup("force")) == PBSE_NONE); + fail_unless(ptr != NULL); + fail_unless(found_job == unlocked_job); + + relay_rc = 0; + fail_unless(issue_signal(&ptr, "SIGKILL", dummy_func, NULL, strdup("force")) == PBSE_NONE); + fail_unless(ptr != NULL); + fail_unless(found_job == unlocked_job); + + nullify_pointer = true; + relay_rc = 1; + fail_unless(issue_signal(&ptr, "SIGKILL", dummy_func, NULL, strdup("force")) == PBSE_NONE); + fail_unless(ptr != NULL); + // On this code path we should have locked twice and unlocked only once, because we 'unlocked' + // in relay_to_mom() + fail_unless(found_job == unlocked_job + 1); + + found_job = 0; + unlocked_job = 0; + nullify_pointer = true; + relay_rc = 1; + fail_unless(issue_signal(&ptr, "SIGKILL", dummy_func, NULL, NULL) != PBSE_NONE); + fail_unless(ptr == NULL); + // We come back NULL from relay_to_mom(), so we shouldn't change these variables + fail_unless(unlocked_job == 0); + fail_unless(found_job == 0); + + ptr = &pjob; + nullify_pointer = false; + relay_rc = 1; + fail_unless(issue_signal(&ptr, "SIGKILL", dummy_func, NULL, NULL) != PBSE_NONE); + fail_unless(ptr != NULL); + fail_unless(unlocked_job == 0); + fail_unless(found_job == 0); + } END_TEST + Suite *req_signal_suite(void) { Suite *s = suite_create("req_signal_suite methods"); @@ -36,8 +106,8 @@ Suite *req_signal_suite(void) tcase_add_test(tc_core, test_req_signaljob_relaying_msg); suite_add_tcase(s, tc_core); - tc_core = tcase_create("test_two"); - tcase_add_test(tc_core, test_two); + tc_core = tcase_create("test_issue_signal"); + tcase_add_test(tc_core, test_issue_signal); suite_add_tcase(s, tc_core); return s; @@ -51,6 +121,9 @@ int main(void) { int number_failed = 0; SRunner *sr = NULL; + + init_attr_def(); + rundebug(); sr = srunner_create(req_signal_suite()); srunner_set_log(sr, "req_signal_suite.log"); diff --git a/src/test/req_stat/scaffolding.c b/src/test/req_stat/scaffolding.c index 889041764b..cbba4d809a 100644 --- a/src/test/req_stat/scaffolding.c +++ b/src/test/req_stat/scaffolding.c @@ -29,6 +29,7 @@ int svr_totnodes = 0; int LOGLEVEL = 7; /* force logging code to be exercised as tests run */ bool exit_called = false; int abort_called; +bool svr_find_job_returns_null = false; struct batch_request *alloc_br(int type) { @@ -247,6 +248,9 @@ int svr_setjobstate(job *pjob, int newstate, int newsubstate, int has_queue_mut job *svr_find_job(const char *jobid, int get_subjob) { job *pjob = (job *)calloc(1, sizeof(job)); + if (svr_find_job_returns_null) + return(NULL); + strcpy(pjob->ji_qs.ji_jobid, jobid); return(pjob); } @@ -368,3 +372,21 @@ bool pbsnode::hasprop(std::vector *needed) const { preply->brp_choice = type; } + +batch_request::~batch_request() + + { + } + +batch_request::batch_request(const batch_request &other) + + { + } + +batch_request::batch_request() + { + } + +batch_request::batch_request(int type) : rq_type(type) + { + } diff --git a/src/test/req_stat/test_uut.c b/src/test/req_stat/test_uut.c index 9c062ff0ed..02bfe26bd3 100644 --- a/src/test/req_stat/test_uut.c +++ b/src/test/req_stat/test_uut.c @@ -7,8 +7,11 @@ #include "array.h" bool in_execution_queue(job *pjob, job_array *pa); +//int stat_to_mom(const char*, struct stat_cntl*); + job *get_next_status_job(struct stat_cntl *cntl, int &job_array_index, job_array *pa, all_jobs_iterator *iter); extern int abort_called; +extern bool svr_find_job_returns_null; enum TJobStatTypeEnum { @@ -47,7 +50,6 @@ START_TEST(test_stat_update) batch_request preq; stat_cntl cntl; - memset(&preq, 0, sizeof(preq)); memset(&cntl, 0, sizeof(cntl)); abort_called = 0; @@ -107,6 +109,21 @@ START_TEST(test_get_next_status_job) END_TEST +START_TEST(test_stat_to_mom) + { + struct stat_cntl *scp; + + scp = (struct stat_cntl *)calloc(1, sizeof(struct stat_cntl)); + + svr_find_job_returns_null = true; + fail_unless(stat_to_mom(strdup("12345"), scp) == PBSE_JOBNOTFOUND); + + svr_find_job_returns_null = false; + fail_unless(stat_to_mom(strdup("12345"), scp) == PBSE_BADSTATE); + } +END_TEST + + Suite *req_stat_suite(void) { Suite *s = suite_create("req_stat_suite methods"); @@ -119,6 +136,10 @@ Suite *req_stat_suite(void) tcase_add_test(tc_core, test_get_next_status_job); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_stat_to_mom"); + tcase_add_test(tc_core, test_stat_to_mom); + suite_add_tcase(s, tc_core); + return s; } diff --git a/src/test/req_track/scaffolding.c b/src/test/req_track/scaffolding.c index 86ac86018e..4d1f7d06b8 100644 --- a/src/test/req_track/scaffolding.c +++ b/src/test/req_track/scaffolding.c @@ -52,10 +52,9 @@ void release_req(struct work_task *pwt) exit(1); } -int issue_to_svr(const char *servern, struct batch_request **preq, void (*replyfunc)(struct work_task *)) +int issue_to_svr(const char *servern, batch_request *preq, void (*replyfunc)(struct work_task *)) { - fprintf(stderr, "The call to issue_to_svr to be mocked!!\n"); - exit(1); + return(0); } void free_br(batch_request *preq) {} @@ -87,3 +86,21 @@ mutex_mgr::~mutex_mgr() { } +batch_request::~batch_request() + + { + } + +batch_request::batch_request(const batch_request &other) + + { + } + +batch_request::batch_request() + { + } + +batch_request::batch_request(int type) : rq_type(type) + { + } + diff --git a/src/test/requests/scaffolding.c b/src/test/requests/scaffolding.c index 0344ef28a7..ef277f485d 100644 --- a/src/test/requests/scaffolding.c +++ b/src/test/requests/scaffolding.c @@ -60,7 +60,7 @@ const char *wdir_ret; void log_event(int event, int event_class, char *func_name, char *buf) {} -int become_the_user(job *pjob) +int become_the_user(job *pjob, bool want_effective) { return(0); } @@ -536,3 +536,5 @@ int send_job_obit(job *pjob, int exit_status) { return(PBSE_NONE); } + +batch_request::batch_request(int type) : rq_type(type) {} diff --git a/src/test/requests/test_uut.c b/src/test/requests/test_uut.c index b30211fd6f..ec544a5f86 100644 --- a/src/test/requests/test_uut.c +++ b/src/test/requests/test_uut.c @@ -3,12 +3,15 @@ #include #include +#include "pbs_config.h" +#include "batch_request.h" #include "pbs_job.h" #include "pbs_error.h" #include "test_requests.h" void string_replchar(const char*, char, char); void determine_spooldir(std::string &spooldir, job *pjob); +batch_request *get_std_file_info(job*); extern char *TNoSpoolDirList[]; extern char *path_spool; @@ -21,6 +24,9 @@ START_TEST(test_determine_spooldir) std::string spooldir; const char *val_ptr; + memset(&pjob, 0, sizeof(job)); + pjob.ji_grpcache = (struct grpcache *)calloc(1, sizeof(struct grpcache)); + // If the spool directories don't match, we should get the default TNoSpoolDirList[0] = strdup("/home/bob/jobs/"); wdir_ret = "PBS_O_WORKDIR=/home/dbeer/jobs/"; @@ -61,6 +67,24 @@ START_TEST(test_string_replchar) } END_TEST +START_TEST(test_get_std_file_info) + { + job myjob; + batch_request *brp; + + memset(&myjob, 0, sizeof(job)); + + // both stdout and stderr are to be kept so don't copy + myjob.ji_wattr[JOB_ATR_keep].at_val.at_str = strdup("eo"); + + brp = get_std_file_info(&myjob); + fail_unless(brp != NULL); + + // now check for empty list (expect no copy requests) + fail_unless(brp->rq_ind.rq_cpyfile.rq_pair.ll_next == brp->rq_ind.rq_cpyfile.rq_pair.ll_prior); + } +END_TEST + Suite *requests_suite(void) { @@ -73,6 +97,10 @@ Suite *requests_suite(void) tcase_add_test(tc_core, test_determine_spooldir); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_get_std_file_info"); + tcase_add_test(tc_core, test_get_std_file_info); + suite_add_tcase(s, tc_core); + return s; } diff --git a/src/test/resc_def_all/scaffolding.c b/src/test/resc_def_all/scaffolding.c index 66062f4712..d30bf6b85a 100644 --- a/src/test/resc_def_all/scaffolding.c +++ b/src/test/resc_def_all/scaffolding.c @@ -282,3 +282,7 @@ char *threadsafe_tokenizer( return(start); } /* END threadsafe_tokenizer() */ +long time_str_to_seconds(const char *str) + { + return(0); + } diff --git a/src/test/rm/scaffolding.c b/src/test/rm/scaffolding.c index 6fc4ff0970..8a40b03112 100644 --- a/src/test/rm/scaffolding.c +++ b/src/test/rm/scaffolding.c @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include "dis.h" #include "tcp.h" @@ -87,14 +89,7 @@ int connect(int sock,const sockaddr *addr,socklen_t addrLen) int get_max_num_descriptors() { - fprintf(stderr, "The call to get_max_num_descriptors needs to be mocked!!\n"); - exit(1); - } - -int get_fdset_size() - { - fprintf(stderr, "The call to get_fdset_size needs to be mocked!!\n"); - exit(1); + return(getdtablesize()); } char *pbs_strerror(int err) @@ -110,3 +105,8 @@ int pbs_getaddrinfo(const char *pNode,struct addrinfo *pHints,struct addrinfo ** (*ppAddrInfoOut)->ai_addr = (struct sockaddr *)calloc(1,sizeof(struct sockaddr_in)); return(0); } + +int poll(struct pollfd *fds, nfds_t nfds, int timeout) + { + return(0); + } diff --git a/src/test/rm/test_uut.c b/src/test/rm/test_uut.c index f05770c000..f8e6ddfb93 100644 --- a/src/test/rm/test_uut.c +++ b/src/test/rm/test_uut.c @@ -11,7 +11,6 @@ extern int debug_read(int sock,char **bf,long long *len); extern int debug_write(int sock,char *bf,long long len); - START_TEST(test_addreq) { @@ -123,6 +122,15 @@ START_TEST(test_getreq) } END_TEST +START_TEST(test_activereq) + { + int rc; + + rc = activereq(); + fail_unless(rc == -2); + } +END_TEST + Suite *rm_suite(void) { Suite *s = suite_create("rm_suite methods"); @@ -150,6 +158,10 @@ Suite *rm_suite(void) tcase_add_test(tc_core, test_getreq); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_activereq"); + tcase_add_test(tc_core, test_activereq); + suite_add_tcase(s, tc_core); + return s; } diff --git a/src/test/start_exec/scaffolding.c b/src/test/start_exec/scaffolding.c index a9ad650668..be08b26105 100644 --- a/src/test/start_exec/scaffolding.c +++ b/src/test/start_exec/scaffolding.c @@ -11,6 +11,7 @@ #include /* std::string */ #include #include +#include #include "attribute.h" /* attribute_def, pbs_attribute, svrattrl */ #include "resource.h" /* resource_def */ @@ -104,6 +105,9 @@ int job_saved; int task_saved; std::string presetup_prologue; +int global_poll_fd = -1; +int global_poll_timeout_sec = -1; + #ifdef NUMA_SUPPORT nodeboard node_boards[MAX_NODE_BOARDS]; int num_node_boards = 10; @@ -695,7 +699,10 @@ proc_stat_t *get_proc_stat(int pid) int setuid_ext(uid_t uid, int set_euid) { - return(0); + if (set_euid == TRUE) + return(seteuid(uid)); + + return(setuid(uid)); } int destroy_alps_reservation(char *reservation_id, char *apbasil_path, char *apbasil_protocol, int retries) @@ -1098,3 +1105,30 @@ void register_jobs_nspace(job *pjob, pjobexec_t *TJE) {} int setup_gpus_for_job(job *pjob) {return(0);} + +int ppoll(struct pollfd *fds, nfds_t nfds, const struct timespec *timeout, const sigset_t *sigmask) + { + if (nfds > 0) + { + global_poll_fd = fds[0].fd; + global_poll_timeout_sec = timeout->tv_sec; + } + + return(0); + } + +int get_local_address(struct sockaddr_in &new_sockaddr) + { + // return 127.0.0.1 + new_sockaddr.sin_addr.s_addr = (uint32_t)2130706433; + return(0); + } + +bool islocalhostaddr(struct sockaddr_in *saip) + { + if (saip == NULL) + return(false); + + // is 127.0.0.1? + return((saip->sin_addr.s_addr == (uint32_t)((127 << 24) + 1))); +} diff --git a/src/test/start_exec/test_uut.c b/src/test/start_exec/test_uut.c index 9daa38ae24..a6b3324f21 100644 --- a/src/test/start_exec/test_uut.c +++ b/src/test/start_exec/test_uut.c @@ -7,8 +7,12 @@ #include #include #include +#include #include #include +#include +#include +#include #include "pbs_error.h" #include "pbs_nodes.h" @@ -19,6 +23,7 @@ int get_indices_from_exec_str(const char *exec_str, char *buf, int buf_size); int remove_leading_hostname(char **jobpath); int get_num_nodes_ppn(const char*, int*, int*); int setup_process_launch_pipes(int &kid_read, int &kid_write, int &parent_read, int &parent_write); +int update_path_attribute(job*, job_atr); #ifdef NUMA_SUPPORT extern nodeboard node_boards[]; @@ -31,6 +36,7 @@ extern char mom_alias[]; char *penv[MAX_TEST_ENVP]; /* max number of pointers bld_env_variables will create */ char *envBuffer = NULL; /* points to the max block that bld_env_variables would ever need in this test suite */ +int spoolasfinalname = FALSE; extern int logged_event; extern int num_contacted; extern int send_sisters_called; @@ -41,12 +47,23 @@ extern bool fail_site_grp_check; extern bool am_ms; extern bool addr_fail; +extern int global_poll_fd; +extern int global_poll_timeout_sec; + void create_command(std::string &cmd, char **argv); void no_hang(int sig); void exec_bail(job *pjob, int code, std::set *sisters_contacted); int read_launcher_child_status(struct startjob_rtn *sjr, const char *job_id, int parent_read, int parent_write); int process_launcher_child_status(struct startjob_rtn *sjr, const char *job_id, const char *application_name); void update_task_and_job_states_after_launch(task *ptask, job *pjob, const char *application_name); +void escape_spaces(const char *str, std::string &escaped); +int become_the_user(job*, bool); +int TMomCheckJobChild(pjobexec_t*, int, int*, int*); + +#if NO_SPOOL_OUTPUT == 1 +int save_supplementary_group_list(int*, gid_t**); +int restore_supplementary_group_list(int, gid_t*); +#endif #ifdef PENABLE_LINUX_CGROUPS unsigned long long get_memory_limit_from_resource_list(job *pjob); @@ -139,6 +156,34 @@ START_TEST(test_read_launcher_child_status) fail_unless(read_launcher_child_status(&srj, jobid, parent_read, parent_write) != PBSE_NONE); } END_TEST + + +START_TEST(test_escape_spaces) + { + std::string escaped; + + const char *ws1 = "one two"; + const char *ws2 = "one two three"; + const char *ws3 = "one two three four"; + + escape_spaces(ws1, escaped); + // should have added a slash + fail_unless(escaped.size() == strlen(ws1) + 1); + fail_unless(escaped == "one\\ two"); + + escaped.clear(); + escape_spaces(ws2, escaped); + // should have added two slashes + fail_unless(escaped.size() == strlen(ws2) + 2); + fail_unless(escaped == "one\\ two\\ three"); + + escaped.clear(); + escape_spaces(ws3, escaped); + // should have added three slashes + fail_unless(escaped.size() == strlen(ws3) + 3); + fail_unless(escaped == "one\\ two\\ three\\ four"); + } +END_TEST #ifdef PENABLE_LINUX_CGROUPS @@ -486,34 +531,34 @@ END_TEST START_TEST(test_check_pwd_euser) { job *pjob = (job *)calloc(1, sizeof(job)); - bool pwd = false; + int pwd = -1; pwd = check_pwd(pjob); - fail_unless(pwd == false, "check_pwd succeeded with an empty job"); + fail_unless(pwd != PBSE_NONE, "check_pwd succeeded with an empty job"); bad_pwd = true; decode_str(&pjob->ji_wattr[JOB_ATR_euser], "euser", NULL, "rightsaidfred", 0); pwd = check_pwd(pjob); - fail_unless(pwd == false, "bad pwd fail"); + fail_unless(pwd != PBSE_NONE, "bad pwd fail"); bad_pwd = false; fail_init_groups = true; decode_str(&pjob->ji_wattr[JOB_ATR_euser], "euser", NULL, "dbeer", 0); pwd = check_pwd(pjob); - fail_unless(pwd == false, "bad grp fail"); + fail_unless(pwd != PBSE_NONE, "bad grp fail"); pjob->ji_grpcache = NULL; fail_init_groups = false; fail_site_grp_check = true; decode_str(&pjob->ji_wattr[JOB_ATR_euser], "euser", NULL, "dbeer", 0); pwd = check_pwd(pjob); - fail_unless(pwd == false, "bad site fail"); + fail_unless(pwd != PBSE_NONE, "bad site fail"); pjob->ji_grpcache = NULL; fail_site_grp_check = false; decode_str(&pjob->ji_wattr[JOB_ATR_euser], "euser", NULL, "dbeer", 0); pwd = check_pwd(pjob); - fail_unless(pwd == true); + fail_unless(pwd == PBSE_NONE); } END_TEST @@ -595,6 +640,162 @@ START_TEST(test_get_num_nodes_ppn) } END_TEST + +#if NO_SPOOL_OUTPUT == 1 +START_TEST(test_save_supplementary_group_list) + { + int ngroups; + gid_t *group_list; + int rc; + + rc = save_supplementary_group_list(&ngroups, &group_list); + fail_unless(rc == 0); + fail_unless(ngroups > 0); + fail_unless((rc == 0) && (group_list != NULL)); + + if (rc == 0) + free(group_list); + } +END_TEST + +START_TEST(restore_supplementary_group_list_test) + { + int ngroups; + gid_t *group_list; + int rc; + + rc = restore_supplementary_group_list(0, NULL); + fail_unless(rc < 0); + + + // must be root to run the following since they call setgroups() + if (getuid() == 0) + { + int i; + int ngroups_alt; + gid_t *group_list_alt; + + // save the supplementary groups + rc = save_supplementary_group_list(&ngroups, &group_list); + fail_unless(rc == 0); + + // make a back up copy of the array + group_list_alt = (gid_t *)malloc(ngroups * sizeof(gid_t)); + fail_unless(group_list_alt != NULL); + memcpy(group_list_alt, group_list, ngroups * sizeof(gid_t)); + + // set to a smaller set + rc = setgroups(1, group_list); + fail_unless(rc == 0); + + // restore the full set + rc = restore_supplementary_group_list(ngroups, group_list); + fail_unless(rc == 0); + + // check to make sure full set restored + ngroups = getgroups(0, NULL); + group_list = (gid_t *)malloc(ngroups * sizeof(gid_t)); + fail_unless(group_list != NULL); + rc = getgroups(ngroups, group_list); + + for (i = 0; i < ngroups; i++) + fail_unless(group_list[i] == group_list_alt[i]); + + free(group_list_alt); + } + } +END_TEST +#endif + +START_TEST(test_become_the_user) + { + int rc; + job *pjob; + int uid; + int gid; + + // must be root to run this test + if (getuid() != 0) + return; + + pjob = (job *)calloc(1, sizeof(job)); + fail_unless(pjob != NULL); + + pjob->ji_grpcache = (struct grpcache *)calloc(1, sizeof(struct grpcache)); + fail_unless(pjob->ji_grpcache != NULL); + + pjob->ji_qs.ji_un.ji_momt.ji_exuid = 500; + pjob->ji_qs.ji_un.ji_momt.ji_exgid = 500; + + pjob->ji_grpcache->gc_ngroup = 1; + pjob->ji_grpcache->gc_groups[0] = 500; + + // fork so we can test the setxid/setexid calls in the child + rc = fork(); + fail_unless(rc != -1); + + if (rc > 0) + { + int status; + + // parent + wait(&status); + return; + } + + // child + + rc = become_the_user(pjob, true); + fail_unless(rc == PBSE_NONE); + + // check the group list, uid, gid + uid = geteuid(); + gid = getegid(); + fail_unless(uid != 500); + fail_unless(gid != 500); + + // put things back in place + fail_unless(seteuid(0) == 0); + fail_unless(setegid(0) == 0); + + rc = become_the_user(pjob, false); + fail_unless(rc == PBSE_NONE); + + // check the uid, gid + uid = getuid(); + gid = getgid(); + fail_unless(uid != 500); + fail_unless(gid != 500); + } +END_TEST + +START_TEST(test_TMomCheckJobChild) + { + pjobexec_t TJE; + int timeout_sec = 39; + int count; + int rc; + int fd = 99; + + TJE.jsmpipe[0] = fd; + TMomCheckJobChild(&TJE, timeout_sec, &count, &rc); + fail_unless(global_poll_fd == fd); + fail_unless(global_poll_timeout_sec == timeout_sec); + } +END_TEST + +START_TEST(test_update_path_attribute) + { + job *pjob = NULL; + + fail_unless(update_path_attribute(pjob, JOB_ATR_outpath) != 0); + pjob = (job *)calloc(1, sizeof(job)); + fail_unless(update_path_attribute(pjob, JOB_ATR_LAST) != 0); + fail_unless(update_path_attribute(pjob, JOB_ATR_outpath) == 0); + fail_unless(update_path_attribute(pjob, JOB_ATR_errpath) == 0); + } +END_TEST + Suite *start_exec_suite(void) { Suite *s = suite_create("start_exec_suite methods"); @@ -643,11 +844,31 @@ Suite *start_exec_suite(void) tcase_add_test(tc_core, test_get_num_nodes_ppn); tcase_add_test(tc_core, test_setup_process_launch_pipes); tcase_add_test(tc_core, test_read_launcher_child_status); + tcase_add_test(tc_core, test_escape_spaces); #ifdef PENABLE_LINUX_CGROUPS tcase_add_test(tc_core, get_memory_limit_from_resource_list_test); #endif suite_add_tcase(s, tc_core); +#if NO_SPOOL_OUTPUT == 1 + tc_core = tcase_create("test_save_supplementary_group_list"); + tcase_add_test(tc_core, test_save_supplementary_group_list); + tcase_add_test(tc_core, restore_supplementary_group_list_test); + suite_add_tcase(s, tc_core); +#endif + + tc_core = tcase_create("test_become_the_user"); + tcase_add_test(tc_core, test_become_the_user); + suite_add_tcase(s, tc_core); + + tc_core = tcase_create("test_TMomCheckJobChild"); + tcase_add_test(tc_core, test_TMomCheckJobChild); + suite_add_tcase(s, tc_core); + + tc_core = tcase_create("test_update_path_attribute"); + tcase_add_test(tc_core, test_update_path_attribute); + suite_add_tcase(s, tc_core); + return s; } diff --git a/src/test/svr_format_job/scaffolding.c b/src/test/svr_format_job/scaffolding.c index 684206f8c6..46bedc8f0f 100644 --- a/src/test/svr_format_job/scaffolding.c +++ b/src/test/svr_format_job/scaffolding.c @@ -2,6 +2,12 @@ #include #include /* fprintf */ +#include +#include + +#include "resource.h" +#include "mail_throttler.hpp" + const char *msg_job_stageinfail = "File stage in failed, see below.\nJob will be retried later, please investigate and correct problem."; const char *msg_job_start = "Begun execution"; const char *msg_job_end = "Execution terminated"; @@ -14,8 +20,56 @@ extern "C" { char *pbse_to_txt(int err) { - fprintf(stderr, "The call to pbse_to_txt to be mocked!!\n"); - exit(1); + return(strdup("Error")); } } +int resource_index_to_string( + + std::string &output, + std::vector &resources, + size_t index) + + { + static int rcount = 0; + + switch (rcount) + { + case 0: + + output = "walltime=100"; + break; + + case 1: + + output = "nodes=2:ppn=8"; + break; + + case 2: + + output="cput=800"; + break; + + case 3: + + output="vmem=8024mb"; + break; + + case 4: + + output="mem=7000mb"; + break; + + default: + break; + } + + rcount++; + + return(0); + } + +mail_info::mail_info() + { + } + diff --git a/src/test/svr_format_job/test_uut.c b/src/test/svr_format_job/test_uut.c index 8f67fda60a..bc8f15aabd 100644 --- a/src/test/svr_format_job/test_uut.c +++ b/src/test/svr_format_job/test_uut.c @@ -1,11 +1,60 @@ #include "license_pbs.h" /* See here for the software license */ -#include "test_svr_format_job.h" #include #include #include "pbs_error.h" -START_TEST(test_one) +#include "mail_throttler.hpp" +#include "test_svr_format_job.h" + +void svr_format_job(FILE *fh, mail_info *mi, const char *fmt); + +START_TEST(test_new_formats) { + const char *path = "./new_formats.txt"; + const char *format = "My job is owned by %o in queue %q and it executed in working directory %w\n%R%u"; + const char *result = "My job is owned by dbeer@sel in queue batch and it executed in working \ +directory /home/dbeer\nResources Requested Summary\n\twalltime=100\n\ +\tnodes=2:ppn=8\nResources Used Summary\n\tcput=800\n\tvmem=8024mb\n\ +\tmem=7000mb\n"; + mail_info mi; + mi.owner = "dbeer@sel"; + mi.queue_name = "batch"; + mi.working_directory = "/home/dbeer"; + + // These vectors have to be the correct size, but the actual information is put in through + // the scaffolding + std::vector requested; + std::vector used; + resource r; + for (int i = 0; i < 2; i++) + requested.push_back(r); + for (int i = 0; i < 3; i++) + used.push_back(r); + + mi.resources_requested = requested; + mi.resources_used = used; + + FILE *test_file = fopen(path, "w+"); + + if (test_file != NULL) + { + char buf[4096]; + svr_format_job(test_file, &mi, format); + fclose(test_file); + + test_file = fopen(path, "r"); + + if (test_file != NULL) + { + char *ptr = buf; + memset(ptr, 0, sizeof(buf)); + if (fread(ptr, 1, sizeof(buf) - 1, test_file) > 0) + { + fail_unless((!strcmp(buf, result)), "Expected:\n'%s'\nGot:\n'%s'", result, buf); + } + } + unlink(path); + } } END_TEST @@ -20,8 +69,8 @@ END_TEST Suite *svr_format_job_suite(void) { Suite *s = suite_create("svr_format_job_suite methods"); - TCase *tc_core = tcase_create("test_one"); - tcase_add_test(tc_core, test_one); + TCase *tc_core = tcase_create("test_new_formats"); + tcase_add_test(tc_core, test_new_formats); suite_add_tcase(s, tc_core); tc_core = tcase_create("test_two"); diff --git a/src/test/svr_jobfunc/scaffolding.c b/src/test/svr_jobfunc/scaffolding.c index 80c1c1d932..2f106a0e36 100644 --- a/src/test/svr_jobfunc/scaffolding.c +++ b/src/test/svr_jobfunc/scaffolding.c @@ -742,3 +742,8 @@ job::~job() {} #include "../../lib/Libattr/req.cpp" #include "../../lib/Libattr/complete_req.cpp" #include "../../lib/Libattr/attr_req_info.cpp" + +long time_str_to_seconds(const char *str) + { + return(0); + } diff --git a/src/test/svr_jobfunc/test_uut.c b/src/test/svr_jobfunc/test_uut.c index 67b45392c6..8592768f8b 100644 --- a/src/test/svr_jobfunc/test_uut.c +++ b/src/test/svr_jobfunc/test_uut.c @@ -23,6 +23,7 @@ int chk_mppnodect(resource *mppnodect, pbs_queue *pque, long nppn, long mpp_widt void job_wait_over(struct work_task *); bool is_valid_state_transition(job &pjob, int newstate, int newsubstate); bool has_conflicting_resource_requests(job *pjob, pbs_queue *pque); +bool contains_execution_slot_request(pbs_attribute *jb); extern int decrement_count; extern job napali_job; @@ -33,7 +34,7 @@ extern bool possible; extern bool get_jobs_queue_force_null; extern std::string global_log_buf; -void add_resc_attribute(pbs_attribute *pattr, resource_def *prdef, const char *value) +void add_resc_attribute(pbs_attribute *pattr, resource_def *prdef, const char *value, long v) { if (pattr->at_val.at_ptr == NULL) { @@ -44,13 +45,59 @@ void add_resc_attribute(pbs_attribute *pattr, resource_def *prdef, const char *v resource rsc; rsc.rs_defin = prdef; rsc.rs_value.at_type = prdef->rs_type; - rsc.rs_value.at_val.at_str = strdup(value); + + if (value != NULL) + rsc.rs_value.at_val.at_str = strdup(value); + else + rsc.rs_value.at_val.at_long = v; + resources->push_back(rsc); pattr->at_flags = ATR_VFLAG_SET; pattr->at_val.at_ptr = resources; } +START_TEST(contains_execution_slot_request_test) + { + pbs_attribute pattr; + + resource_def rdef[5]; + rdef[0].rs_name = strdup("nodes"); + rdef[1].rs_name = strdup("walltime"); + rdef[2].rs_name = strdup("ncpus"); + rdef[3].rs_name = strdup("procs"); + rdef[4].rs_name = strdup("size"); + memset(&pattr, 0, sizeof(pattr)); + + // Add walltime to the job request + add_resc_attribute(&pattr, rdef + 1, NULL, 60); + fail_unless(contains_execution_slot_request(&pattr) == false); + + // Now add nodes + add_resc_attribute(&pattr, rdef, "2:ppn=10", 60); + fail_unless(contains_execution_slot_request(&pattr) == true); + + // Make a new request with ncpus + pbs_attribute ncpus_attr; + memset(&ncpus_attr, 0, sizeof(ncpus_attr)); + add_resc_attribute(&ncpus_attr, rdef + 2, NULL, 6); + fail_unless(contains_execution_slot_request(&ncpus_attr) == true); + + // Make a new request with procs + pbs_attribute procs_attr; + memset(&procs_attr, 0, sizeof(procs_attr)); + add_resc_attribute(&procs_attr, rdef + 3, NULL, 4); + fail_unless(contains_execution_slot_request(&procs_attr) == true); + + // Finally, make one with size + pbs_attribute size_attr; + memset(&size_attr, 0, sizeof(size_attr)); + add_resc_attribute(&size_attr, rdef + 4, NULL, 64); + fail_unless(contains_execution_slot_request(&size_attr) == true); + } +END_TEST + + START_TEST(test_numa_task_exceeds_resources) { job *pjob = (job *)calloc(1, sizeof(job)); @@ -277,6 +324,8 @@ START_TEST(svr_setjobstate_test) fail_unless(svr_setjobstate(&test_job, JOB_STATE_QUEUED, JOB_SUBSTATE_QUEUED, FALSE) == PBSE_NONE); fail_unless(test_job.ji_wattr[JOB_ATR_exec_host].at_val.at_str != NULL, "exec_host list got removed when it shouldn't have..."); + // We don't use this, but we need a dummy due to protections + test_job.ji_qhdr = (pbs_queue *)0x1; decrement_count = 0; fail_unless(svr_setjobstate(&test_job, JOB_STATE_COMPLETE, JOB_SUBSTATE_COMPLETE, FALSE) == PBSE_NONE); fail_unless(decrement_count == 2); @@ -429,14 +478,15 @@ START_TEST(chk_resc_min_limits_test) memset(&test_attribute, 0, sizeof(test_attribute)); memset(&test_queue, 0, sizeof(test_queue)); - add_resc_attribute(&test_queue.qu_attr[QA_ATR_ResourceMin], &nodes_resource_def, "3"); - add_resc_attribute(&test_attribute, &nodes_resource_def, "2:ppn=1+1:ppn=4"); + add_resc_attribute(&test_queue.qu_attr[QA_ATR_ResourceMin], &nodes_resource_def, "3", 1); + add_resc_attribute(&test_attribute, &nodes_resource_def, "2:ppn=1+1:ppn=4", 1); result = chk_resc_limits(&test_attribute, &test_queue, message); fail_unless(result == PBSE_NONE, "Fail to approve queue minimum resource"); } END_TEST + START_TEST(svr_chkque_test) { struct job test_job; @@ -733,6 +783,7 @@ Suite *svr_jobfunc_suite(void) tc_core = tcase_create("set_chkpt_deflt_test"); tcase_add_test(tc_core, set_chkpt_deflt_test); + tcase_add_test(tc_core, contains_execution_slot_request_test); suite_add_tcase(s, tc_core); tc_core = tcase_create("set_statechar_test"); diff --git a/src/test/svr_mail/scaffolding.c b/src/test/svr_mail/scaffolding.c index b9202e03ca..0a4eb390fb 100644 --- a/src/test/svr_mail/scaffolding.c +++ b/src/test/svr_mail/scaffolding.c @@ -26,6 +26,7 @@ pthread_mutex_t *listener_command_mutex; int listening_socket; threadpool_t *task_pool; int called; +bool get_svr_attr_str_return_svr_sm_attr = false; int LOGLEVEL = 7; /* force logging code to be exercised as tests run */ @@ -128,7 +129,12 @@ void log_err(int errnum, const char *routine, const char *text) {} void log_record(int eventtype, int objclass, const char *objname, const char *text) {} void log_event(int eventtype, int objclass, const char *objname, const char *text) {} -job::job() {} +job::job() + { + memset(&this->ji_qs, 0, sizeof(this->ji_qs)); + memset(this->ji_wattr, 0, sizeof(this->ji_wattr)); + } + job::~job() {} bool empty_body = false; @@ -141,6 +147,12 @@ int get_svr_attr_str( { static char *bodyfmt = strdup("Zaphod"); + if (get_svr_attr_str_return_svr_sm_attr) + { + *str = server.sv_attr[SRV_ATR_SendmailPath].at_val.at_str; + return(0); + } + if ((index == SRV_ATR_MailBodyFmt) && (empty_body == true)) { @@ -150,3 +162,13 @@ int get_svr_attr_str( return(0); } +int resource_index_to_string( + + std::string &output, + std::vector &resources, + size_t index) + + { + return(0); + } + diff --git a/src/test/svr_mail/test_uut.c b/src/test/svr_mail/test_uut.c index 7d2ff51e79..8b10d6aae5 100644 --- a/src/test/svr_mail/test_uut.c +++ b/src/test/svr_mail/test_uut.c @@ -10,6 +10,7 @@ #include "test_svr_mail.h" #include "mail_throttler.hpp" #include "work_task.h" +#include "server.h" @@ -27,8 +28,10 @@ extern int called; void send_email_batch(struct work_task *pwt); +int get_sendmail_args(char **, mail_info *, char **); extern mail_throttler pending_emails; extern bool empty_body; +extern bool get_svr_attr_str_return_svr_sm_attr; void init_server() { @@ -332,7 +335,10 @@ START_TEST(mail_point_p) job pjob; char p[]= "p"; - pjob.ji_wattr[JOB_ATR_mailpnts].at_val.at_str = p; + pjob.ji_wattr[JOB_ATR_mailpnts].at_val.at_str = p; + pjob.ji_wattr[JOB_ATR_outpath].at_val.at_str = strdup("/home/dbeer/forgery"); + pjob.ji_wattr[JOB_ATR_errpath].at_val.at_str = strdup("/home/dbeer/forgery"); + pjob.ji_wattr[JOB_ATR_job_owner].at_val.at_str = strdup("dbeer@sel"); svr_mailowner(&pjob, 1, 1, p); fail_unless((called == 0),"one"); @@ -345,6 +351,27 @@ START_TEST(mail_point_p) } END_TEST +START_TEST(test_get_sendmail_args) + { + char *sendmail_args[100]; + mail_info mi; + char *p; + + get_sendmail_args(sendmail_args, &mi, (char **)&p); + + // expect value compiled in + fail_unless(strcmp(sendmail_args[0], UT_SENDMAIL_CMD) == 0); + + // now set the server attribute + server.sv_attr[SRV_ATR_SendmailPath].at_val.at_str = (char *)"./foo"; + + // expect to match server attribute (not one compiled in) + get_svr_attr_str_return_svr_sm_attr = true; + get_sendmail_args(sendmail_args, &mi, (char **)&p); + fail_unless(strcmp(sendmail_args[0], "./foo") == 0); + } +END_TEST + Suite *svr_mail_suite(void) { Suite *s = suite_create("svr_mail_suite methods"); @@ -365,6 +392,10 @@ Suite *svr_mail_suite(void) tcase_add_test(tc_core, mail_point_p); suite_add_tcase(s, tc_core); + tc_core = tcase_create("test_get_sendmail_args"); + tcase_add_test(tc_core, test_get_sendmail_args); + suite_add_tcase(s, tc_core); + return s; } diff --git a/src/test/trq_auth/scaffolding.c b/src/test/trq_auth/scaffolding.c index ca59e8e5e9..df8fb69a58 100644 --- a/src/test/trq_auth/scaffolding.c +++ b/src/test/trq_auth/scaffolding.c @@ -657,3 +657,20 @@ struct passwd *get_password_entry_by_uid(char **user_buf, uid_t uid) void free_pwnam(struct passwd *pwdp, char *buf) { } + +int get_local_address(struct sockaddr_in &new_sockaddr) + { + // return 127.0.0.1 + new_sockaddr.sin_addr.s_addr = (uint32_t) ((127 << 24) + 1); + return(0); + } + +bool islocalhostaddr(struct sockaddr_in *saip) + { + if (saip == NULL) + return(false); + + // is 127.0.0.1? + return((saip->sin_addr.s_addr == (uint32_t)((127 << 24) + 1))); +} + diff --git a/src/test/trq_auth/test_uut.c b/src/test/trq_auth/test_uut.c index 5bfac6f8bd..37cf40a31f 100644 --- a/src/test/trq_auth/test_uut.c +++ b/src/test/trq_auth/test_uut.c @@ -159,9 +159,10 @@ START_TEST(test_trq_simple_disconnect) } END_TEST + START_TEST(test_validate_server) { - char active_server_name[PBS_MAXHOSTNAME+1]; + std::string active_server("localhost"); char *ssh_key = NULL; char *sign_key = NULL; int rc; @@ -180,13 +181,12 @@ START_TEST(test_validate_server) socket_connect_success = true; DIS_success = true; - strcpy(active_server_name, "localhost"); - rc = validate_server(active_server_name, port, ssh_key, &sign_key); + rc = validate_server(active_server, port, ssh_key, &sign_key); fail_unless(rc == PBSE_NONE, "validate_server success case failed", rc); - } END_TEST + START_TEST(test_set_active_pbs_server) { char new_server_name[PBS_MAXHOSTNAME + 1]; @@ -195,10 +195,10 @@ START_TEST(test_set_active_pbs_server) strcpy(new_server_name, "localhost"); rc = set_active_pbs_server(new_server_name, 15001); fail_unless(rc == PBSE_NONE, "set_active_pbs_server failed", rc); - } END_TEST + START_TEST(test_validate_active_pbs_server) { int rc; diff --git a/src/test/trq_cgroups/Makefile.am b/src/test/trq_cgroups/Makefile.am new file mode 100644 index 0000000000..3d364a3e5a --- /dev/null +++ b/src/test/trq_cgroups/Makefile.am @@ -0,0 +1,4 @@ + +include ../Makefile_Mom.ut + +libuut_la_SOURCES = ${PROG_ROOT}/trq_cgroups.c diff --git a/src/test/trq_cgroups/scaffolding.c b/src/test/trq_cgroups/scaffolding.c new file mode 100644 index 0000000000..8308bb93eb --- /dev/null +++ b/src/test/trq_cgroups/scaffolding.c @@ -0,0 +1,170 @@ +#include "license_pbs.h" /* See here for the software license */ +#include +#include /* fprintf */ +#include /* winsize */ +#include /* termios */ + +#include "pbs_ifl.h" /* PBS_MAXHOSTNAME */ +#include "log.h" /* LOG_BUF_SIZE */ +#include "pbs_job.h" /* job, task */ +#include "complete_req.hpp" +#include "machine.hpp" +#include "numa_node.hpp" + +bool global_trq_cg_signal_tasks_ok = false; + +char mom_alias[1]; + +char log_buffer[LOG_BUF_SIZE]; + +bool per_task; + +int is_login_node; + +int LOGLEVEL = 0; + +Machine::Machine() {} +Machine::~Machine() {} + +PCI_Device::~PCI_Device() {} + +Socket::~Socket() {} + +Chip::~Chip() {} + +Core::~Core() {} + +Machine this_node; + +req::req() {} + +bool req::is_per_task() const + + { + return(per_task); + } + +void req::get_task_host_name( + + std::string &task_host, + unsigned int task_index) + + {} + +int Machine::getTotalChips() const + { + return(0); + } + +int Machine::getTotalThreads() const + { + return(0); + } + +bool task_hosts_match(const char *one, const char *two) + { + return(true); + } + +allocation::allocation( + + const allocation &alloc) : cpu_indices(alloc.cpu_indices), memory(alloc.memory), cpus(alloc.cpus) + + { + this->jobid = alloc.jobid; + } + +allocation::allocation() : cpu_indices(), memory(0), cpus(0) + + { + this->jobid[0] = '\0'; + } + +allocation &allocation::operator =(const allocation &other) + { + return(*this); + } + +void log_err(int errnum, const char *routine, const char *text) {} + +int rmdir_ext(const char *dirname, int retry_limit) + { + return(0); + } + +void get_device_indices( + + const char *device_str, + std::vector &device_indices, + const char *suffix) + + {} + +int complete_req::req_count() const + { + return(0); + } + +bool have_incompatible_dash_l_resource( + + pbs_attribute *pattr) + + { + return(false); + } + +int req::get_task_allocation( + + unsigned int index, + allocation &task_allocation) const + + { + return(PBSE_NONE); + } + +int req::get_req_allocation_count() + + { + return(0); + } + +unsigned int complete_req::get_num_reqs() + { + return(1); + } + +req &complete_req::get_req(int i) + { + static req r; + + return(r); + } + +int req::getTaskCount() const + + { + return(0); + } + +void log_event( + + int eventtype, + int objclass, + const char *objname, + const char *text) + + { + if ((objname != NULL) && (strcmp(objname, "trq_cg_signal_tasks") == 0)) + { + if ((text != NULL) && (strcmp(text, "sending signal 15 to pid 9999 in cpuset .") == 0)) + global_trq_cg_signal_tasks_ok = true; + } + } + +int translate_range_string_to_vector( + + const char *range_string, + std::vector &indices) +{ +return(0); +} diff --git a/src/test/trq_cgroups/tasks b/src/test/trq_cgroups/tasks new file mode 100644 index 0000000000..3c2df076c2 --- /dev/null +++ b/src/test/trq_cgroups/tasks @@ -0,0 +1 @@ +9999 diff --git a/src/test/trq_cgroups/test_trq_cgroups.h b/src/test/trq_cgroups/test_trq_cgroups.h new file mode 100644 index 0000000000..7fcfe357c9 --- /dev/null +++ b/src/test/trq_cgroups/test_trq_cgroups.h @@ -0,0 +1,9 @@ +#include "license_pbs.h" /* See here for the software license */ +#ifndef _TRQ_CGROUPS_CT_H +#define _TRQ_CGROUPS_CT_H +#include + +#define TRQ_CGROUPS_SUITE 1 +Suite *trq_cgroups_suite(); + +#endif /* _TRQ_CGROUPS_CT_H */ diff --git a/src/test/trq_cgroups/test_uut.c b/src/test/trq_cgroups/test_uut.c new file mode 100644 index 0000000000..13d85e206b --- /dev/null +++ b/src/test/trq_cgroups/test_uut.c @@ -0,0 +1,58 @@ +#include "license_pbs.h" /* See here for the software license */ +#include "mom_mach.h" +#include "test_trq_cgroups.h" +#include +#include +#include +#include + +#include "pbs_error.h" +#include "pbs_job.h" +#include "trq_cgroups.h" + +extern int LOGLEVEL; +extern bool global_trq_cg_signal_tasks_ok; + +START_TEST(test_trq_cg_signal_tasks) + { + LOGLEVEL = 9; + trq_cg_signal_tasks(".", SIGTERM); + fail_unless(global_trq_cg_signal_tasks_ok, "test_trq_cg_signal_tasks failed"); + } +END_TEST + +START_TEST(test_two) + { + } +END_TEST + +Suite *trq_cgroups_suite(void) + { + Suite *s = suite_create("trq_cgroups_suite methods"); + TCase *tc_core = tcase_create("test_trq_cg_signal_tasks"); + tcase_add_test(tc_core, test_trq_cg_signal_tasks); + suite_add_tcase(s, tc_core); + + tc_core = tcase_create("test_two"); + tcase_add_test(tc_core, test_two); + suite_add_tcase(s, tc_core); + + return s; + } + +void rundebug() + { + } + +int main(void) + { + int number_failed = 0; + SRunner *sr = NULL; + rundebug(); + sr = srunner_create(trq_cgroups_suite()); + srunner_set_log(sr, "trq_cgroups_suite.log"); + srunner_run_all(sr, CK_NORMAL); + number_failed = srunner_ntests_failed(sr); + srunner_free(sr); + return number_failed; + } diff --git a/src/test/u_hash_map_structs/test_uut.c b/src/test/u_hash_map_structs/test_uut.c index 6acc507dbf..873877dc16 100644 --- a/src/test/u_hash_map_structs/test_uut.c +++ b/src/test/u_hash_map_structs/test_uut.c @@ -25,6 +25,31 @@ void callocVal(char **dest, const char *src) strcpy(*dest, src); } + +START_TEST(test_hash_priority_add_or_exit) + { + job_data_container map; + + hash_priority_add_or_exit(&map, "planet", "sel", CMDLINE_DATA); + hash_priority_add_or_exit(&map, "planet", "nalthis", SCRIPT_DATA); + + job_data *stored; + fail_unless(hash_find(&map, "planet", &stored) == TRUE); + fail_unless(stored->value == "sel"); // command line is higher priority than script + + hash_priority_add_or_exit(&map, "planet", "roshar", ENV_DATA); + fail_unless(hash_find(&map, "planet", &stored) == TRUE); + fail_unless(stored->value == "roshar"); // environment data is higher priority than command line + + // insert a duplicate item + char *planet_name = strdup("mars"); + hash_priority_add_or_exit(&map, "planet", planet_name, ENV_DATA); + fail_unless(hash_find(&map, "planet", &stored) == TRUE); + fail_unless(stored->value == planet_name); // should retrieve lastest entry added + } +END_TEST + + START_TEST(test_hash_add_item_new_count_clear) { int rc = FALSE; @@ -196,6 +221,7 @@ Suite *u_hash_map_structs_suite(void) tcase_add_test(tc_core, test_hash_print); /*tcase_add_exit_test(tc_core, test_add_or_exit, 1);*/ tcase_add_test(tc_core, test_hash_add_hash); + tcase_add_test(tc_core, test_hash_priority_add_or_exit); suite_add_tcase(s, tc_core); return s; diff --git a/src/test/u_misc/test_uut.c b/src/test/u_misc/test_uut.c index b1fec8bf9c..5efcb1336c 100644 --- a/src/test/u_misc/test_uut.c +++ b/src/test/u_misc/test_uut.c @@ -113,6 +113,18 @@ START_TEST(test_translate_range_string_to_vector) fail_unless(indices.size() == 2); fail_unless(indices[0] == 6142); fail_unless(indices[1] == 6143); + + indices.clear(); + fail_unless(translate_range_string_to_vector("0--1", indices) != PBSE_NONE); + + indices.clear(); + fail_unless(translate_range_string_to_vector("3-1", indices) != PBSE_NONE); + + indices.clear(); + fail_unless(translate_range_string_to_vector("10,10", indices) != PBSE_NONE); + + indices.clear(); + fail_unless(translate_range_string_to_vector("5-5", indices) != PBSE_NONE); } END_TEST diff --git a/src/test/user_info/test_uut.c b/src/test/user_info/test_uut.c index 9903135e9e..c4194160e0 100644 --- a/src/test/user_info/test_uut.c +++ b/src/test/user_info/test_uut.c @@ -146,12 +146,24 @@ START_TEST(increment_queued_jobs_test) // Enqueue the job without resetting to make sure the count doesn't change fail_unless(increment_queued_jobs(&users, strdup("tom@napali"), &pjob) == 0); fail_unless(get_num_queued(&users, "tom") == 3, "Shouldn't have incremented with the bit set"); + + // Make sure array subjobs are a no-op + pjob.ji_queue_counted = 0; + pjob.ji_is_array_template = FALSE; + pjob.ji_wattr[JOB_ATR_job_array_id].at_flags = ATR_VFLAG_SET; + fail_unless(increment_queued_jobs(&users, strdup("tom"), &pjob) == 0); + fail_unless(get_num_queued(&users, "tom") == 3, "Shouldn't have incremented for an array sub-job"); + fail_unless(pjob.ji_queue_counted != 0); + + pjob.ji_queue_counted = 0; + pjob.ji_wattr[JOB_ATR_job_array_id].at_flags = 0; + fail_unless(increment_queued_jobs(&users, strdup("tom"), &pjob) == 0); + fail_unless(get_num_queued(&users, "tom") == 4, "Should have incremented for non array sub-job"); } END_TEST - START_TEST(decrement_queued_jobs_test) { users.lock(); diff --git a/torque.setup b/torque.setup deleted file mode 100755 index d7ac900b90..0000000000 --- a/torque.setup +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/sh -# torque.setup - -# USAGE: torque.setup [] - -if [ "$1" = "" ] ; then - echo "USAGE: torque.setup []" - exit 1 - fi - -export PATH=/usr/local/sbin:/usr/local/bin:$PATH - -unset PBSDEBUG -TRQ_CHECK=`pgrep trqauthd` -TRQ_CHECK_RES=$? -if [ "$TRQ_CHECK_RES" = "1" ]; then - TRQ_START=`trqauthd` - TRQ_START_RES=$? - if [ "$TRQ_START_RES" -ne "0" ]; then - echo "trqauthd failed to start!!! exiting setup" - exit 1 - else - echo "trqauthd successfully started" - fi -fi - -if [ "$2" = "" ] ; then - # obtain name obtained from reverse lookup of ip address - HOSTNAME=`hostname` -else - HOSTNAME=$2 -fi - -# enable operator privileges - -USER=$1@$HOSTNAME - -echo "initializing TORQUE (admin: $USER)" - -ps -ef | grep -v grep | grep pbs_server - -if [ "$?" -eq "0" ] ; then - echo "ERROR: pbs_server already running... run 'qterm' to stop pbs_server and rerun" - exit 1; -fi - -pbs_server -t create -# Starting in TORQUE 3.1 the server is multi-threaded. -# We need to pause a second to allow the server to finish coming -# up. If we go to qmgr right away it will fail. -sleep 2 - -ps -ef | grep -v grep | grep pbs_server - -if [ "$?" -ne "0" ] ; then - echo "ERROR: pbs_server failed to start, check syslog and server logs for more information" - exit 1; -fi - -echo set server operators += $USER | qmgr - -if [ "$?" -ne "0" ] ; then - echo "ERROR: cannot set TORQUE admins" - qterm - exit 1; -fi - -echo set server managers += $USER | qmgr - - -qmgr -c 'set server scheduling = true' -qmgr -c 'set server keep_completed = 300' -qmgr -c 'set server mom_job_sync = true' - -# create default queue - -qmgr -c 'create queue batch' -qmgr -c 'set queue batch queue_type = execution' -qmgr -c 'set queue batch started = true' -qmgr -c 'set queue batch enabled = true' -qmgr -c 'set queue batch resources_default.walltime = 1:00:00' -qmgr -c 'set queue batch resources_default.nodes = 1' - -qmgr -c 'set server default_queue = batch' - diff --git a/torque.setup.in b/torque.setup.in new file mode 100644 index 0000000000..15716c9b97 --- /dev/null +++ b/torque.setup.in @@ -0,0 +1,302 @@ +#!/bin/sh +# +# torque.setup - create pbs_server database and default queue +# + +# print out basic usage info + +function usage + { + echo "USAGE: torque.setup [-h|--help] [-d config_path|--config-path=config_path] + [--bin_path=bin_path] [--sbin_path=sbin_path] [--syslog] + [] + + -h|--help Show command help. + + -d config_path|--config-path=CONFIG_PATH Specifies the path of the directory + which is home to the server\'s configuration + files. Default is [$PBS_HOME] + + --prefix=PATH Specifies the path to executables in PATH/bin + and PATH/sbin. Default is [$PREFIX] + + --syslog=true|false Log errors to syslog. Default is false. +" >&2 + } + +# +# logit - write a message to a file descriptor and syslog if desired +# +# usage: logit msg [fd [pri]] +# msg - Message. +# fd - File descriptor to write msg to. Default: 2. +# pri - Syslog priority. Default: user.error. See -p argument listed in logger man page. +# +# Note: message is written to syslog only if WANTSYSLOG is true. +# + +function logit + { + # need a message + [ $# -eq 0 ] && return 1 + + # get the message + msg=$1 + shift + + # get the fd for non-syslog output + if [ $# -gt 0 ]; then + fd=$1 + shift + else + # default + fd=2 + fi + + # write to syslog? + if [ $WANTSYSLOG = true ]; then + if [ $# -gt 0 ]; then + pri=$1 + shift + else + # default + pri=user.error + fi + + # write to syslog + logger -t $PROG -p $pri $msg + rc=$? + + if [ $rc -ne 0 ]; then + echo "$PROG: logger failed with $rc" >&2 + exit $rc + fi + fi + + # write to fd + echo "$msg" >&$fd + + return 0 + } + + +# return list of ip addresses (one from each interface on this host) +function get_ip_addrs + { + ip addr show | awk '/inet [0-9]+/ {split($2, f, "/"); print f[1]}' + } + +# return hostname of given ip addr +function ip2hostname + { + if [ ! -z "$1" ]; then + getent hosts $1 | awk '$1 == "'$1'" {print $2; exit}' + fi + } + +# return list containing the hostname of each interface on this machine +function get_hostnames + { + for ip in `get_ip_addrs`; do + hostname=`ip2hostname $ip` + if [ ! -z "$hostname" ]; then + # add hostname to the list + echo "$hostname" + fi + done | sort -u + } + +# remove localhost from hostname list if it is not the hostname of this host +function remove_localhost + { + hostname_list="$1" + this_hostname=`hostname` + + for hostname in $hostname_list; do + echo "$hostname" | egrep \^localhost &>/dev/null + if [ $? -eq 0 ]; then + # do not add localhost if it is not the name of this host + if [ "$hostname" != "$this_hostname" ]; then + continue + fi + fi + + # add hostname to the list + echo "$hostname" + done + } + + +PROG=`basename $0` + +PREFIX=@exec_prefix@ +BIN_PATH=$PREFIX/bin +SBIN_PATH=$PREFIX/sbin +PBS_HOME=@PBS_HOME@ +WANTSYSLOG=false + +#export POSIXLY_CORRECT=1 +OPTS=`getopt -o hd: -l config-path: -l prefix: -l syslog: -l help -- "$@"` + +if [ $? != 0 ]; then + logit "getop failed" + usage + exit 1 +fi + +eval set -- "$OPTS" + +while true; do + case "$1" in + -d|--config-path) + + PBS_HOME="$2" + shift 2 + ;; + + --prefix) + + PREFIX="$2" + BIN_PATH=$PREFIX/bin + SBIN_PATH=$PREFIX/sbin + shift 2 + ;; + + --syslog) + + WANTSYSLOG=`echo "$2" | tr '[:upper:]' '[:lower:]'` + shift 2 + ;; + + -h|--help) + + usage + exit 0 + ;; + + --) + shift + break + ;; + + *) + break + ;; + + esac +done + +# now check options + +if [ $# -lt 1 -o $# -gt 2 ]; then + usage + exit 1 +fi + +if [ ! -d "$BIN_PATH" ]; then + logit "$BIN_PATH does not exist." + exit 1 +fi + +if [ ! -d "$SBIN_PATH" ]; then + logit "$SBIN_PATH does not exist." + exit 1 +fi + +if [ "$WANTSYSLOG" != true -a "$WANTSYSLOG" != false ]; then + usage + exit 1 +fi + +# make sure logger command present if syslog desired +if [ $WANTSYSLOG = true ]; then + logger -V &>/dev/null + if [ $? -ne 0 ]; then + WANTSYSLOG=false + logit "cannot find logger command." + exit 1 + fi +fi + +export PATH=$SBIN_PATH:$BIN_PATH:$PATH + +unset PBSDEBUG +pgrep trqauthd >/dev/null + +if [ $? -ne 0 ]; then + trqauthd + if [ $? -ne 0 ]; then + logit "trqauthd failed to start!!! exiting setup" + exit 1 + else + logit "trqauthd successfully started" 1 user.info + fi +fi + +# user and optional hostname +USER=$1 +HOSTNAME=$2 + +logit "initializing TORQUE (admin: $USER)" 1 user.info + +pgrep pbs_server >/dev/null + +if [ $? -eq 0 ]; then + logit "pbs_server already running... run 'qterm' to stop pbs_server and rerun" + exit 1 +fi + +pbs_server -t create -d $PBS_HOME +# Starting in TORQUE 3.1 the server is multi-threaded. +# We need to pause a second to allow the server to finish coming +# up. If we go to qmgr right away it will fail. +sleep 2 + +pgrep pbs_server >/dev/null + +if [ $? -ne 0 ]; then + logit "ERROR: pbs_server failed to start, check syslog and server logs for more information" + exit 1 +fi + +# setup admins + +if [ ! -z "$HOSTNAME" ]; then + # supplied to script by caller + hostname_list=$HOSTNAME +else + # look up hostnames for all interfaces on this host + hostname_list_all=`get_hostnames` + + # remove localhost if necessary + hostname_list=`remove_localhost "$hostname_list_all"` +fi + +for hostname in $hostname_list; do + for admin_type in operators managers; do + qmgr -c "s s $admin_type += $USER@$hostname" + + if [ $? -ne 0 ] ; then + logit "ERROR: cannot set $USER@$hostname in $admin_type list" + qterm + exit 1 + fi + done +done + +# setup basic parameters + +qmgr -c 'set server scheduling = true' +qmgr -c 'set server keep_completed = 300' +qmgr -c 'set server mom_job_sync = true' + +# create default queue + +qmgr -c 'create queue batch' +qmgr -c 'set queue batch queue_type = execution' +qmgr -c 'set queue batch started = true' +qmgr -c 'set queue batch enabled = true' +qmgr -c 'set queue batch resources_default.walltime = 1:00:00' +qmgr -c 'set queue batch resources_default.nodes = 1' + +qmgr -c 'set server default_queue = batch'