Merge branch 'v2.x/staging' into feature/v2/template-yamls

.dependency/zwe_doc_generation/md-content.js

@@ -46,7 +46,7 @@ function generateDocumentationForNode(curNode, assembledParentNode) {
             docContent += createDocContent(assembledDocNode[docType.fileName].content, docType);
             const parentDocContent = createDocContent(assembledDocNode[docType.fileName].parentContent, docType);
             if (parentDocContent) {
-                docContent += SUB_SECTION_HEADER_PREFIX + 'Inherited from parent command' + SEPARATOR + parentDocContent;
+                docContent += SEPARATOR + SUB_SECTION_HEADER_PREFIX + 'Inherited from parent command' + SEPARATOR + parentDocContent;
             }
         }
 

.github/workflows/cicd-test.yml

@@ -63,8 +63,8 @@ env:
   DEFAULT_ZOWE_EXT_ARTIFACTORY_PATTERN: libs-snapshot-local/org/zowe/{ext-name}/*/{ext-name}-*.pax
 
   # below block can be overwritten, adjusted by DevOps only
-  # ZOS_NODE_VERSION more to choose from: v14.21.3, v16.19.1
-  ZOS_NODE_VERSION: v14.21.3
+  # ZOS_NODE_VERSION more to choose from: v14.21.3.1, v16.20.1
+  ZOS_NODE_VERSION: v14.21.3.1
   CLIENT_NODE_VERSION: v12.18.3
   INSTALL_TEST_DEBUG_INFORMATION: zowe-install-test:*
   SANITY_TEST_DEBUG_INFORMATION: zowe-sanity-test:*

.pax/pre-packaging.sh

@@ -147,7 +147,7 @@ SCRIPT_NAME=$(basename "$0")  # $0=./pre-packaging.sh
 BASE_DIR=$(cd $(dirname "$0"); pwd)      # <something>/.pax
 
 # use node v14 to build
-export NODE_HOME=/ZOWE/node/node-v14.21.3-os390-s390x
+export NODE_HOME=/ZOWE/node/node-v14.21.3.1-os390-s390x
 
 ZOWE_ROOT_DIR="${BASE_DIR}/content"
 

CHANGELOG.md

@@ -3,6 +3,15 @@
 All notable changes to the Zowe Installer will be documented in this file.
 <!--Add the PR or issue number to the entry if available.-->
 
+## `2.11.0`
+
+### New features and enhancements
+- Enhancement: Duplicate log messages into syslog according to "zowe.sysMessages" array [#93](https://github.com/zowe/launcher/pull/93)
+
+#### Minor enhancements/defect fixes
+- Bugfix: `zowe-yaml-schema.json` validates unix directory path as `server-common#zowePath`
+- Bugfix: Fixed that Zowe would allow newer Java versions to generate PKCS12 keystores that was not compatible with some components. Newer versions of Java by default create PKCS12 keystores that aren't compatible with GSK / SystemSSL which components such as ZSS use, but include flags to restore a compatibility mode, which Zowe now uses. (#3507)
+
 ## `2.10.0`
 
 ### New features and enhancements

bin/commands/components/search/.parameters

@@ -1,5 +1,5 @@
 component-name,component|o|string|||||Component name to search for.
 component-id,id|d|string|||||Component id to search for.
-registry|r|string|||||Specifies the registry to searh within instead of the default. The registry must be compatible with the manager used.
+registry|r|string|||||Specifies the registry to search within instead of the default. The registry must be compatible with the manager used.
 handler||string|||||Specifies the registry handler name used with the package registry, instead of the default. The handler must be compatible with the registry used.
 

bin/commands/components/uninstall/.parameters

@@ -1,4 +1,4 @@
 component-name,component|o|string|required||||The name of an installed component.
-registry|r|string|||||Specifies the registry to searh within instead of the default. The registry must be compatible with the manager used.
+registry|r|string|||||Specifies the registry to search within instead of the default. The registry must be compatible with the manager used.
 handler||string|||||Specifies the registry handler name used with the package registry, instead of the default. The handler must be compatible with the registry used.
 dry-run|d|boolean|||||Whether or not to perform the upgrade versus just checking if an upgrade is available

bin/commands/components/upgrade/.parameters

@@ -1,4 +1,4 @@
 component-file,component|o|string|required||||Either a path or component name. The path must be to a component package or directory. If a name is specified instead, install checks the zowe package registry.
-registry|r|string|||||Specifies the registry to searh within instead of the default. The registry must be compatible with the manager used.
+registry|r|string|||||Specifies the registry to search within instead of the default. The registry must be compatible with the manager used.
 handler||string|||||Specifies the registry handler name used with the package registry, instead of the default. The handler must be compatible with the registry used.
 dry-run|d|boolean|||||Whether or not to perform the upgrade versus just checking if an upgrade is available

bin/commands/config/validate/.examples

@@ -1,2 +1,3 @@
 zwe config validate -c /path/to/zowe.yaml
-zwe config validate -c FILE(/customizations/zowe.yaml):FILE(/defaults/zowe.yaml) --all
+zwe config validate -c FILE(/customizations/zowe.yaml):FILE(/defaults/zowe.yaml) --all
+zwe config validate -c 'FILE(/path/to/zowe.yaml):PARMLIB(ZOWE.PARMLIB(YAML))'

bin/libs/certificate.sh

@@ -199,6 +199,8 @@ pkcs12_create_certificate_authority() {
   password="${3}"
   common_name=${4:-${ZWE_PRIVATE_DEFAULT_CERTIFICATE_CA_COMMON_NAME}}
 
+  flags=$(get_java_pkcs12_keystore_flag)
+
   print_message ">>>> Generate PKCS12 format local CA with alias ${alias}:"
   mkdir -p "${keystore_dir}/${alias}"
   result=$(pkeytool -genkeypair -v \
@@ -210,6 +212,7 @@ pkcs12_create_certificate_authority() {
             -storepass "${password}" \
             -storetype "PKCS12" \
             -validity "${ZWE_PRIVATE_CERTIFICATE_CA_VALIDITY:-${ZWE_PRIVATE_DEFAULT_CERTIFICATE_CA_VALIDITY}}" \
+            ${flags} \
             -ext KeyUsage="keyCertSign" \
             -ext BasicConstraints:"critical=ca:true")
   if [ $? -ne 0 ]; then
@@ -231,9 +234,14 @@ pkcs12_create_certificate_and_sign() {
   ca_alias=${7}
   ca_password=${8}
 
+
   print_message ">>>> Generate certificate \"${alias}\" in the keystore ${keystore_name}:"
+
+  flags=$(get_java_pkcs12_keystore_flag)
+
   mkdir -p "${keystore_dir}/${keystore_name}"
   result=$(pkeytool -genkeypair -v \
+            ${flags} \
             -alias "${alias}" \
             -keyalg RSA -keysize 2048 \
             -keystore "${keystore_dir}/${keystore_name}/${keystore_name}.keystore.p12" \
@@ -251,7 +259,8 @@ pkcs12_create_certificate_and_sign() {
   fi
 
   print_message ">>>> Generate CSR for the certificate \"${alias}\" in the keystore \"${keystore_name}\":"
-  result=$(pkeytool -certreq -v \
+  result=$(pkeytool ${flags} \
+            -certreq -v \
             -alias "${alias}" \
             -keystore "${keystore_dir}/${keystore_name}/${keystore_name}.keystore.p12" \
             -storepass "${password}" \
@@ -279,7 +288,8 @@ pkcs12_create_certificate_and_sign() {
   san="${san}dns:localhost.localdomain,dns:localhost,ip:127.0.0.1"
 
   print_message ">>>> Sign the CSR using the Certificate Authority \"${ca_alias}\":"
-  result=$(pkeytool -gencert -v \
+  result=$(pkeytool ${flags} \
+            -gencert -v \
             -infile "${keystore_dir}/${keystore_name}/${alias}.csr" \
             -outfile "${keystore_dir}/${keystore_name}/${alias}.signed.cer" \
             -keystore "${keystore_dir}/${ca_alias}/${ca_alias}.keystore.p12" \
@@ -315,7 +325,8 @@ pkcs12_create_certificate_and_sign() {
     >/dev/null 2>/dev/null
   if [ "$?" != "0" ]; then
     print_message ">>>> Import the Certificate Authority \"${ca_alias}\" to the keystore \"${keystore_name}\":"
-    result=$(pkeytool -importcert -v \
+    result=$(pkeytool ${flags} \
+              -importcert -v \
               -trustcacerts -noprompt \
               -file "${ca_cert_file}" \
               -alias "${ca_alias}" \
@@ -333,7 +344,8 @@ pkcs12_create_certificate_and_sign() {
     >/dev/null 2>/dev/null
   if [ "$?" != "0" ]; then
     print_message ">>>> Import the Certificate Authority \"${ca_alias}\" to the truststore \"${keystore_name}\":"
-    result=$(pkeytool -importcert -v \
+    result=$(pkeytool ${flags} \
+              -importcert -v \
               -trustcacerts -noprompt \
               -file "${ca_cert_file}" \
               -alias "${ca_alias}" \
@@ -348,7 +360,8 @@ pkcs12_create_certificate_and_sign() {
   fi
 
   print_message ">>>> Import the signed CSR to the keystore \"${keystore_name}\":"
-  result=$(pkeytool -importcert -v \
+  result=$(pkeytool ${flags} \
+            -importcert -v \
             -trustcacerts -noprompt \
             -file "${keystore_dir}/${keystore_name}/${alias}.signed.cer" \
             -alias "${alias}" \
@@ -464,7 +477,10 @@ pkcs12_import_pkcs12_keystore() {
     return 1
   fi
 
-  result=$(pkeytool -importkeystore -v \
+  flags=$(get_java_pkcs12_keystore_flag)
+
+  result=$(pkeytool ${flags} \
+            -importkeystore -v \
             -noprompt \
             -deststoretype "PKCS12" \
             -destkeystore "${dest_keystore}" \
@@ -492,12 +508,15 @@ pkcs12_import_certificates() {
   ca_files="${3}"
   alias="${4:-extca}"
 
+  flags=$(get_java_pkcs12_keystore_flag)
+
   ca_index=1
   while read -r ca_file; do
     ca_file=$(echo "${ca_file}" | trim)
     if [ -n "${ca_file}" ]; then
       print_message ">>>> Import \"${ca_file}\" to the keystore \"${dest_keystore}\":"
-      result=$(pkeytool -importcert -v \
+      result=$(pkeytool ${flags} \
+                -importcert -v \
                 -trustcacerts -noprompt \
                 -file "${ca_file}" \
                 -alias "${alias}${ca_index}" \
@@ -549,6 +568,8 @@ pkcs12_trust_service() {
     return 1
   fi
 
+  flags=$(get_java_pkcs12_keystore_flag)
+
   # parse keytool output into separate files
   csplit -s -k -f "${keystore_dir}/${keystore_name}/${service_alias}" "${tmp_file}" /-----END\ CERTIFICATE-----/1 \
     {$(expr `grep -c -e '-----END CERTIFICATE-----' "${tmp_file}"` - 1)}
@@ -557,7 +578,8 @@ pkcs12_trust_service() {
     cert_file=$(basename "${cert}")
     cert_alias=${cert_file%.cer}
     echo ">>>> Import a certificate \"${cert_alias}\" to the truststore:"
-    result=$(pkeytool -importcert -v \
+    result=$(pkeytool ${flags} \
+              -importcert -v \
               -trustcacerts \
               -noprompt \
               -file "${cert}" \
@@ -883,9 +905,11 @@ EOF
     racf_connect2="s/^ \+LABEL[(]'certlabel'[)].*\$/            LABEL('${connect_label}') +/"
   fi
 
-  # used by ACF2
-  # TODO: how to pass this?
-  stc_group=
+  # used by ACF2  
+  stc_group=$(read_yaml "${ZWE_CLI_PARAMETER_CONFIG}" ".zowe.setup.security.groups.stc")
+  if [ -z "${stc_group}" ]; then
+    stc_group=${ZWE_PRIVATE_DEFAULT_ADMIN_GROUP}
+  fi
 
   ###############################
   # prepare ZWEKRING JCL
@@ -1010,8 +1034,10 @@ keyring_run_zwenokyr_jcl() {
   security_product=${7:-RACF}
 
   # used by ACF2
-  # TODO: how to pass this?
-  stc_group=
+  stc_group=$(read_yaml "${ZWE_CLI_PARAMETER_CONFIG}" ".zowe.setup.security.groups.stc")
+  if [ -z "${stc_group}" ]; then
+    stc_group=${ZWE_PRIVATE_DEFAULT_ADMIN_GROUP}
+  fi
 
   ###############################
   # prepare ZWENOKYR JCL
@@ -1161,10 +1187,13 @@ keyring_export_to_pkcs12() {
 
   print_debug ">>>> Export certificate \"${label}\" from safkeyring:////${keyring_owner}/${keyring_name} to PKCS#12 keystore ${keystore_file}"
 
+ flags=$(get_java_pkcs12_keystore_flag)
+
   # create keystore if it doesn't exist
   if [ -f "${keystore_file}" ]; then
     print_debug "- Create keystore with dummy certificate ${dummy_cert}"
-    result=$(pkeytool -genkeypair \
+    result=$(pkeytool ${flags} \
+            -genkeypair \
             -alias "${dummy_cert}" \
             -dname "CN=Zowe Dummy Cert, OU=ZWELS, O=Zowe, C=US" \
             -keystore "${keystore_file}" \
@@ -1200,7 +1229,8 @@ keyring_export_to_pkcs12() {
   if [ "${cert_only}" = "true" ]; then
     # use keytool to import certificate
     print_debug "- Import certificate into keystore as \"${label}\""
-    result=$(pkeytool -import -v \
+    result=$(pkeytool ${flags} \
+            -import -v \
             -trustcacerts -noprompt \
             -alias "${label}" \
             -file "${uss_temp_target}.cer" \

bin/libs/config.ts

@@ -95,7 +95,7 @@ export function generateInstanceEnvFromYamlConfig(haInstance: string) {
   }
 
   // delete old files to avoid potential issues
-  common.printFormattedTrace( "ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `deleting old files under ${zwePrivateWorkspaceEnvDir}`);
+  common.printFormattedTrace( "ZWELS", "bin/libs/config.ts,generate_instance_env_from_yaml_config", `deleting old files under ${zwePrivateWorkspaceEnvDir}`);
   let foundFiles = fs.getFilesInDirectory(zwePrivateWorkspaceEnvDir);
   if (foundFiles) {
     foundFiles.forEach((file:string)=> {
@@ -111,12 +111,12 @@ export function generateInstanceEnvFromYamlConfig(haInstance: string) {
   //TODO use configmgr to write json and ha json, and components json
 
   // prepare .zowe.json and .zowe-<ha-id>.json
-  common.printFormattedTrace("ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `config-converter yaml convert --ha ${haInstance} ${cliParameterConfig}`);
+  common.printFormattedTrace("ZWELS", "bin/libs/config.ts,generate_instance_env_from_yaml_config", `config-converter yaml convert --ha ${haInstance} ${cliParameterConfig}`);
   let result = shell.execOutSync('node', `${runtimeDirectory}/bin/utils/config-converter/src/cli.js`, `yaml`, `convert`, `--wd`, zwePrivateWorkspaceEnvDir, `--ha`, haInstance, cliParameterConfig, `--verbose`);
 
-  common.printFormattedTrace("ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `- Exit code: ${result.rc}: ${result.out}`);
+  common.printFormattedTrace("ZWELS", "bin/libs/config.ts,generate_instance_env_from_yaml_config", `- Exit code: ${result.rc}: ${result.out}`);
   if ( !fs.fileExists(`${zwePrivateWorkspaceEnvDir}/.zowe.json`)) {
-    common.printFormattedError( "ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `ZWEL0140E: Failed to translate Zowe configuration (${cliParameterConfig}).`);
+    common.printFormattedError( "ZWELS", "bin/libs/config.ts,generate_instance_env_from_yaml_config", `ZWEL0140E: Failed to translate Zowe configuration (${cliParameterConfig}).`);
     std.exit(140);
   }
 
@@ -126,9 +126,9 @@ export function generateInstanceEnvFromYamlConfig(haInstance: string) {
 
 
   // convert YAML configurations to backward compatible .instance-<ha-id>.env files
-  common.printFormattedTrace("ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `config-converter yaml env --ha ${haInstance}`);
+  common.printFormattedTrace("ZWELS", "bin/libs/config.ts,generate_instance_env_from_yaml_config", `config-converter yaml env --ha ${haInstance}`);
   const envs = configmgr.getZoweConfigEnv(haInstance);
-  common.printFormattedTrace("ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `- Output: ${JSON.stringify(envs, null, 2)}`);
+  common.printFormattedTrace("ZWELS", "bin/libs/config.ts,generate_instance_env_from_yaml_config", `- Output: ${JSON.stringify(envs, null, 2)}`);
   const envKeys = Object.keys(envs);
   let envFileArray=[];
 
@@ -145,7 +145,7 @@ export function generateInstanceEnvFromYamlConfig(haInstance: string) {
     let rc = fs.mkdirp(folderName, 0o700);
     if (rc) {
       //TODO error code
-      common.printFormattedError("ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `Failed to make env var folder for component=${currentComponent}`);
+      common.printFormattedError("ZWELS", "bin/libs/config.ts,generate_instance_env_from_yaml_config", `Failed to make env var folder for component=${currentComponent}`);
     }
     let componentFileArray = [];
     componentFileArray.push('#!/bin/sh');
@@ -194,7 +194,7 @@ export function generateInstanceEnvFromYamlConfig(haInstance: string) {
     const componentFileContent = componentFileArray.join('\n');
     rc = xplatform.storeFileUTF8(`${folderName}/.instance-${haInstance}.env`, xplatform.AUTO_DETECT, componentFileContent);
     if (rc) { 
-      common.printFormattedError("ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `ZWEL0140E: Failed to translate Zowe configuration (${cliParameterConfig}).`);
+      common.printFormattedError("ZWELS", "bin/libs/config.ts,generate_instance_env_from_yaml_config", `ZWEL0140E: Failed to translate Zowe configuration (${cliParameterConfig}).`);
       std.exit(140);
       return;
     }
@@ -203,7 +203,7 @@ export function generateInstanceEnvFromYamlConfig(haInstance: string) {
   let envFileContent = envFileArray.join('\n');
   let rc = xplatform.storeFileUTF8(`${zwePrivateWorkspaceEnvDir}/.instance-${haInstance}.env`, xplatform.AUTO_DETECT, envFileContent);
   if (rc) {
-    common.printFormattedError("ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `ZWEL0140E: Failed to translate Zowe configuration (${cliParameterConfig}).`);
+    common.printFormattedError("ZWELS", "bin/libs/config.ts,generate_instance_env_from_yaml_config", `ZWEL0140E: Failed to translate Zowe configuration (${cliParameterConfig}).`);
     std.exit(140);
     return;
   }

bin/libs/java.sh

@@ -132,3 +132,32 @@ validate_java_home() {
 
   print_debug "Java check is successful."
 }
+
+get_java_pkcs12_keystore_flag() {
+  java_version=$("${JAVA_HOME}/bin/java" -version 2>&1) # Capture stderr to stdout, so we can print below if error
+
+
+  # As we know the java -version command works then strip out the line we need
+  java_version_short=$(echo "${java_version}" | grep ^"java version" | sed -e "s/java version //g"| sed -e "s/\"//g")
+  if [[ $java_version_short == "" ]]; then
+    java_version_short=$(echo "${java_version}" | grep ^"openjdk version" | sed -e "s/openjdk version //g"| sed -e "s/\"//g")
+  fi
+  java_major_version=$(echo "${java_version_short}" | cut -d '.' -f 1)
+  java_minor_version=$(echo "${java_version_short}" | cut -d '.' -f 2)
+  java_fix_version=$(echo "${java_version_short}" | cut -d '_' -f 2)
+
+  if [ ${java_major_version} -eq 1 -a ${java_minor_version} -eq 8 ]; then
+    if [ ${java_fix_version} -lt 341 ]; then
+      printf " "
+    elif [ ${java_fix_version} -lt 361 ]; then
+      printf " -J-Dkeystore.pkcs12.certProtectionAlgorithm=PBEWithSHAAnd40BitRC2 -J-Dkeystore.pkcs12.certPbeIterationCount=50000 -J-Dkeystore.pkcs12.keyProtectionAlgorithm=PBEWithSHAAnd3KeyTripleDES -J-Dkeystore.pkcs12.keyPbeIterationCount=50000 "
+    else
+      printf " -J-Dkeystore.pkcs12.legacy "
+    fi
+  elif [ ${java_major_version} -eq 1 -a ${java_minor_version} -gt 8 ]; then
+    printf " -J-Dkeystore.pkcs12.legacy "
+  else
+    printf " "  
+  fi
+}
+