From ff1082d1ef384ead9307d18193e8fc727e42b0f9 Mon Sep 17 00:00:00 2001 From: 1000TurquoisePogs Date: Wed, 6 Jul 2022 15:13:34 -0400 Subject: [PATCH] zwe ts edition, with configmgr (#2885) * Update schema files to match reality of code and also configmgr parsing Signed-off-by: 1000TurquoisePogs * Checkpoint on configmgr-ts Signed-off-by: 1000TurquoisePogs * WIP: getting to commands section Signed-off-by: 1000TurquoisePogs * More wip, some unfinished files. Some left behind node code Signed-off-by: 1000TurquoisePogs * More lib updates Signed-off-by: 1000TurquoisePogs * Added caching of env vars, optimized manifest reading Signed-off-by: 1000TurquoisePogs * ObjUtils for merge, flatten, deepcopy, etc Signed-off-by: Joe * Build files Signed-off-by: 1000TurquoisePogs * Temp changes Signed-off-by: 1000TurquoisePogs * Fixes after getting pretty far in 'prepare' code. up to global validate area, stuck on java. Signed-off-by: 1000TurquoisePogs * Adding d.ts for all QJS stuff, introducing path library and fanout of all tsc compilation issues Signed-off-by: Joe * WIP on prepare, up to component stage but disabled static def setup and app-server hangs Signed-off-by: 1000TurquoisePogs * Some fixes Signed-off-by: 1000TurquoisePogs * Commit where internal start prepare phase works almost 100%, just missing one or two env vars Signed-off-by: 1000TurquoisePogs * Multi-file is working Signed-off-by: 1000TurquoisePogs * Cleanup build system Signed-off-by: 1000TurquoisePogs * Adding integration into build Signed-off-by: 1000TurquoisePogs * Cleanup of console logs and adding license headers to some files Signed-off-by: 1000TurquoisePogs * Fix mkdirp missing Signed-off-by: 1000TurquoisePogs * Test configmgr extract Signed-off-by: 1000TurquoisePogs * Fix typo Signed-off-by: 1000TurquoisePogs * Fix fact that find is returning absolute path Signed-off-by: 1000TurquoisePogs * Cleanup configmgr pax after extract Signed-off-by: 1000TurquoisePogs * Update index.sh Signed-off-by: 1000TurquoisePogs * Update index.sh Signed-off-by: 1000TurquoisePogs * Update example-zowe.yaml The enum says this must be lowercase Signed-off-by: 1000TurquoisePogs * Attempt to build zwe ts Signed-off-by: 1000TurquoisePogs * Sync ts with latest sh changes Signed-off-by: 1000TurquoisePogs * Add build folder, use, the remove Signed-off-by: 1000TurquoisePogs * Tracing Signed-off-by: 1000TurquoisePogs * Tracing Signed-off-by: 1000TurquoisePogs * Tracing Signed-off-by: 1000TurquoisePogs * Tracing Signed-off-by: 1000TurquoisePogs * Tracing Signed-off-by: 1000TurquoisePogs * Fix a typo making build fail Signed-off-by: 1000TurquoisePogs * build file cleanup Signed-off-by: 1000TurquoisePogs * Revert template in example-zowe.yaml because it is not processable without configmgr Signed-off-by: 1000TurquoisePogs * Set runopts to get tagging Signed-off-by: 1000TurquoisePogs * Move tsc build step to prepare-workspace time to get ebcdic conversion right Signed-off-by: 1000TurquoisePogs * Update manifest.json.template Point at branches with schemas Signed-off-by: 1000TurquoisePogs * Update manifest.json.template Fix jobs/files api not found Signed-off-by: 1000TurquoisePogs * Fixed a bug where the shell command args were an array of strings rather than a string with spaces which caused validate.sh and preconfigure.sh not to run. now fixed and validate.sh of apiml components appear to work as expected Signed-off-by: 1000TurquoisePogs * Update manifest.json.template Update versions to snapshots of mainline Signed-off-by: 1000TurquoisePogs * Removed commented out lines Removed commented out lines Signed-off-by: 1000TurquoisePogs Co-authored-by: Joe Co-authored-by: Tom Zhang <14881754+FlappiTomic@users.noreply.github.com> Co-authored-by: Joe Winchester --- .github/workflows/build-packaging.yml | 2 +- .gitignore | 14 +- .pax/pre-packaging.sh | 16 +- .pax/prepare-workspace.sh | 12 + bin/commands/internal/start/index.sh | 1 + bin/commands/internal/start/index.ts | 42 ++ bin/commands/internal/start/prepare/cli.ts | 13 + bin/commands/internal/start/prepare/index.sh | 7 + bin/commands/internal/start/prepare/index.ts | 460 ++++++++++++ bin/commands/start/index.ts | 66 ++ bin/libs/common.ts | 355 +++++++++ bin/libs/component.ts | 733 +++++++++++++++++++ bin/libs/config.ts | 257 +++++++ bin/libs/configmgr.ts | 174 +++++ bin/libs/container.ts | 120 +++ bin/libs/fs.ts | 297 ++++++++ bin/libs/index.ts | 19 + bin/libs/java.ts | 135 ++++ bin/libs/json.ts | 207 ++++++ bin/libs/logging.ts | 35 + bin/libs/network.ts | 170 +++++ bin/libs/node.ts | 134 ++++ bin/libs/pathoid.ts | 177 +++++ bin/libs/shell.ts | 178 +++++ bin/libs/strftime.ts | 87 +++ bin/libs/string.ts | 335 +++++++++ bin/libs/sys.ts | 179 +++++ bin/libs/var.ts | 176 +++++ bin/libs/zos-fs.ts | 110 +++ bin/libs/zosmf.ts | 60 ++ bin/utils/ObjUtils.js | 247 +++++++ bin/utils/ObjUtils.ts | 274 +++++++ build/zwe/package-lock.json | 37 + build/zwe/package.json | 27 + build/zwe/tsconfig.dev.json | 37 + build/zwe/tsconfig.prod.json | 37 + build/zwe/types/@qjstypes/Configuration.d.ts | 26 + build/zwe/types/@qjstypes/os.d.ts | 63 ++ build/zwe/types/@qjstypes/posix.d.ts | 11 + build/zwe/types/@qjstypes/std.d.ts | 70 ++ build/zwe/types/@qjstypes/xplatform.d.ts | 32 + build/zwe/types/@qjstypes/zos.d.ts | 28 + example-zowe.yaml | 5 +- manifest.json.template | 18 +- schemas/manifest-schema.json | 218 ++++++ schemas/server-common.json | 5 +- schemas/trivial-component-schema.json | 21 + schemas/zowe-yaml-schema.json | 2 +- 48 files changed, 5713 insertions(+), 16 deletions(-) create mode 100644 bin/commands/internal/start/index.ts create mode 100644 bin/commands/internal/start/prepare/cli.ts create mode 100644 bin/commands/internal/start/prepare/index.ts create mode 100644 bin/commands/start/index.ts create mode 100644 bin/libs/common.ts create mode 100644 bin/libs/component.ts create mode 100644 bin/libs/config.ts create mode 100644 bin/libs/configmgr.ts create mode 100644 bin/libs/container.ts create mode 100644 bin/libs/fs.ts create mode 100644 bin/libs/index.ts create mode 100644 bin/libs/java.ts create mode 100644 bin/libs/json.ts create mode 100644 bin/libs/logging.ts create mode 100644 bin/libs/network.ts create mode 100644 bin/libs/node.ts create mode 100644 bin/libs/pathoid.ts create mode 100644 bin/libs/shell.ts create mode 100644 bin/libs/strftime.ts create mode 100644 bin/libs/string.ts create mode 100644 bin/libs/sys.ts create mode 100644 bin/libs/var.ts create mode 100644 bin/libs/zos-fs.ts create mode 100644 bin/libs/zosmf.ts create mode 100644 bin/utils/ObjUtils.js create mode 100644 bin/utils/ObjUtils.ts create mode 100644 build/zwe/package-lock.json create mode 100644 build/zwe/package.json create mode 100644 build/zwe/tsconfig.dev.json create mode 100644 build/zwe/tsconfig.prod.json create mode 100644 build/zwe/types/@qjstypes/Configuration.d.ts create mode 100644 build/zwe/types/@qjstypes/os.d.ts create mode 100644 build/zwe/types/@qjstypes/posix.d.ts create mode 100644 build/zwe/types/@qjstypes/std.d.ts create mode 100644 build/zwe/types/@qjstypes/xplatform.d.ts create mode 100644 build/zwe/types/@qjstypes/zos.d.ts create mode 100644 schemas/manifest-schema.json create mode 100644 schemas/trivial-component-schema.json diff --git a/.github/workflows/build-packaging.yml b/.github/workflows/build-packaging.yml index 3db0c30e22..38663f6e59 100644 --- a/.github/workflows/build-packaging.yml +++ b/.github/workflows/build-packaging.yml @@ -171,7 +171,7 @@ jobs: with: manifest-file-path: ${{ github.workspace }}/manifest.json default-target-path: .pax/binaryDependencies/ - expected-count: 26 + expected-count: 27 # this step is not doing a publish, we are just utilizing this actions to get the PUBLISH_TARGET_PATH, # and it will be used in the next step: [Download 3] Download SMPE build log diff --git a/.gitignore b/.gitignore index 04d0ffe161..0f955ef035 100644 --- a/.gitignore +++ b/.gitignore @@ -4,18 +4,30 @@ tmp/ .pytest_cache/ *.pyc +# emacs save files +*~ + # logs *.log -# zip files +# archive files # *.pax *.zip *.tar *.gz +*.xz +*.bz2 # compile java binary *.class +# nodejs +**/node_modules + +# Compiled TS files +bin/libs/*.js +build/zwe/out + # Mac files .DS_Store diff --git a/.pax/pre-packaging.sh b/.pax/pre-packaging.sh index 969a1ea594..4076537882 100755 --- a/.pax/pre-packaging.sh +++ b/.pax/pre-packaging.sh @@ -149,8 +149,12 @@ BASE_DIR=$(cd $(dirname "$0"); pwd) # /.pax # use node v12 to build export NODE_HOME=/ZOWE/node/node-v12.18.4-os390-s390x -cd "${BASE_DIR}" ZOWE_ROOT_DIR="${BASE_DIR}/content" + +cd "${BASE_DIR}" +# Done with build, remove build folder +rm -rf "${ZOWE_ROOT_DIR}/build" + ZOWE_VERSION=$(cat ${ZOWE_ROOT_DIR}/manifest.json | grep version | head -1 | awk -F: '{ print $2 }' | sed 's/[",]//g' | tr -d '[[:space:]]') # add zwe command to PATH export PATH=${ZOWE_ROOT_DIR}/bin:${PATH} @@ -200,6 +204,16 @@ echo "[$SCRIPT_NAME] templates is not part of zowe.pax, moving it out ..." mv ./content/templates . chmod +x templates/*.rex + +mkdir -p "${ZOWE_ROOT_DIR}/bin/utils" +configmgr=$(find "${ZOWE_ROOT_DIR}/files" -type f \( -name "configmgr*.pax" \) | head -n 1) +echo "[$SCRIPT_NAME] extract configmgr $configmgr" +cd "${ZOWE_ROOT_DIR}/bin/utils" +pax -ppx -rf "${configmgr}" +rm "${configmgr}" +cd "${BASE_DIR}" + + echo "[$SCRIPT_NAME] create dummy zowe.yaml for install" cat <> "${BASE_DIR}/zowe.yaml" zowe: diff --git a/.pax/prepare-workspace.sh b/.pax/prepare-workspace.sh index 5ea6aa41e1..1101d902a8 100755 --- a/.pax/prepare-workspace.sh +++ b/.pax/prepare-workspace.sh @@ -111,10 +111,17 @@ echo "[${SCRIPT_NAME}] preparing folders ..." rm -fr "${ASCII_DIR}" && mkdir -p "${ASCII_DIR}" rm -fr "${CONTENT_DIR}" && mkdir -p "${CONTENT_DIR}/bin" mkdir -p "${CONTENT_DIR}/files" +mkdir -p "${CONTENT_DIR}/schemas" + # FIXME: remove these debug code # rm -fr "${PAX_WORKSPACE_DIR}/binaryDependencies" && mkdir -p "${PAX_WORKSPACE_DIR}/binaryDependencies" # cp -r "${PAX_WORKSPACE_DIR}/bak/binaryDependencies/" "${PAX_WORKSPACE_DIR}/binaryDependencies" +# Building TS files +cd "${ROOT_DIR}/build/zwe" +npm ci && npm run prod + + # copy from current github source echo "[${SCRIPT_NAME}] copying files ..." cd "${ROOT_DIR}" @@ -124,6 +131,11 @@ cp ZOWE.md "${CONTENT_DIR}/README.md" cp DEVELOPERS.md "${CONTENT_DIR}/DEVELOPERS.md" cp -R bin/* "${CONTENT_DIR}/bin" cp -R files/* "${CONTENT_DIR}/files" +cp -R schemas/* "${CONTENT_DIR}/schemas" + + +# build dir should not end up in release, will be removed after build in pre-packaging phase +#cp -R build "${CONTENT_DIR}/" # move licenses mkdir -p "${CONTENT_DIR}/licenses" diff --git a/bin/commands/internal/start/index.sh b/bin/commands/internal/start/index.sh index faf530f66a..a8f943a3fa 100644 --- a/bin/commands/internal/start/index.sh +++ b/bin/commands/internal/start/index.sh @@ -11,6 +11,7 @@ # Copyright Contributors to the Zowe Project. ####################################################################### + ############################### # validation require_zowe_yaml diff --git a/bin/commands/internal/start/index.ts b/bin/commands/internal/start/index.ts new file mode 100644 index 0000000000..d39ee12903 --- /dev/null +++ b/bin/commands/internal/start/index.ts @@ -0,0 +1,42 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +// @ts-ignore +import * as std from 'std'; +// @ts-ignore +import * as os from 'os'; +// @ts-ignore +import * as zos from 'zos'; +import * as common from '../../../libs/common'; +//import * as stringlib from '../../../libs/string'; +//import * as shell from '../../../libs/shell'; +//import * as config from '../../../libs/config'; +import * as internalStartPrepare from './prepare/index'; +//import * as internalStartComponent from './component/index'; + +// Validation +common.requireZoweYaml(); + +// Read job name and validate +//const zoweConfig = config.getZoweConfig(); + +// prepare instance/.env and instance/workspace directories +internalStartPrepare.execute(); + +if (std.getenv('ZWE_PRIVATE_CONTAINER_COMPONENT_ID')) { +// internalStartComponent.execute(std.getenv("ZWE_PRIVATE_CONTAINER_COMPONENT_ID"), false); + //TODO ensure this waits +} else { + const launchComponents = std.getenv("ZWE_LAUNCH_COMPONENTS").split(','); + launchComponents.forEach(function(componentName:string) { +// internalStartComponent.execute(componentName, true); + }); +} diff --git a/bin/commands/internal/start/prepare/cli.ts b/bin/commands/internal/start/prepare/cli.ts new file mode 100644 index 0000000000..2001e54af7 --- /dev/null +++ b/bin/commands/internal/start/prepare/cli.ts @@ -0,0 +1,13 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as index from './index'; +index.execute(); diff --git a/bin/commands/internal/start/prepare/index.sh b/bin/commands/internal/start/prepare/index.sh index 5f5898f255..529c8ff425 100644 --- a/bin/commands/internal/start/prepare/index.sh +++ b/bin/commands/internal/start/prepare/index.sh @@ -15,6 +15,11 @@ # This command prepares everything needed to start Zowe. ################################################################################ +if [ "${ZWE_CLI_CONFIGMGR}" = "true" ]; then + _CEE_RUNOPTS="XPLINK(ON),HEAPPOOLS(OFF)" ${ZWE_zowe_runtimeDirectory}/bin/utils/configmgr -script "${ZWE_zowe_runtimeDirectory}/bin/commands/internal/start/prepare/cli.js" +else + + ################################################################################ # FUNCTIONS @@ -385,3 +390,5 @@ configure_components ############################### # display instance prepared info print_formatted_info "ZWELS" "zwe-internal-start-prepare:${LINENO}" "Zowe runtime environment prepared" + +fi diff --git a/bin/commands/internal/start/prepare/index.ts b/bin/commands/internal/start/prepare/index.ts new file mode 100644 index 0000000000..6b86444ff0 --- /dev/null +++ b/bin/commands/internal/start/prepare/index.ts @@ -0,0 +1,460 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + + +import * as std from 'std'; +import * as os from 'os'; +import * as zos from 'zos'; +import * as xplatform from 'xplatform'; + +import * as fs from '../../../../libs/fs'; +import * as common from '../../../../libs/common'; +import * as stringlib from '../../../../libs/string'; +import * as shell from '../../../../libs/shell'; +import * as sys from '../../../../libs/sys'; +import * as config from '../../../../libs/config'; +import * as component from '../../../../libs/component'; +import * as varlib from '../../../../libs/var'; +import * as java from '../../../../libs/java'; +import * as node from '../../../../libs/node'; +import * as zosmf from '../../../../libs/zosmf'; + +//# This command prepares everything needed to start Zowe. +const cliParameterConfig = std.getenv('ZWE_CLI_PARAMETER_CONFIG'); +const runInContainer = std.getenv('ZWE_RUN_IN_CONTAINER'); +const containerComponentId = std.getenv('ZWE_PRIVATE_CONTAINER_COMPONENT_ID'); +//const installedComponentsEnv=std.getenv('ZWE_INSTALLED_COMPONENTS'); +//const installedComponents = installedComponentsEnv ? installedComponentsEnv.split(',') : null; + +const zosmfHost = std.getenv('ZOSMF_HOST'); +const zosmfPort = Number(std.getenv('ZOSMF_PORT')); + + +const user = std.getenv('USER'); + +const ZOWE_CONFIG=config.getZoweConfig(); + +// Extra preparations for running in container +// - link component runtime under zowe /components +// - `commands.configureInstance` is deprecated in v2 +function prepareRunningInContainer() { + // gracefully shutdown all processes + common.printFormattedDebug("ZWELS", "zwe-internal-start-prepare,prepare_running_in_container", "Register SIGTERM handler for graceful shutdown."); + os.signal(os.SIGTERM, sys.gracefullyShutdown); + + // read ZWE_PRIVATE_CONTAINER_COMPONENT_ID from component manifest + // /component is hardcoded path we asked for in conformance + const manifest = component.getManifest('/component'); + if (!manifest) { + std.exit(1); //TODO error code + } + let componentId = std.getenv('ZWE_PRIVATE_CONTAINER_COMPONENT_ID'); + if (componentId) { + componentId = manifest.name; + std.setenv('ZWE_PRIVATE_CONTAINER_COMPONENT_ID', componentId); + } + + common.printFormattedTrace("ZWELS", "zwe-internal-start-prepare,prepare_running_in_container:", `Prepare /components/${componentId} directory.`); + if (fs.directoryExists(`${runtimeDirectory}/components/${componentId}`)) { + shell.execSync('rm', `-rf`, `"${runtimeDirectory}/components/${componentId}"`); + } + os.symlink(`${runtimeDirectory}/components/${componentId}`, '/component'); +} + +// Prepare log directory +function prepareLogDirectory() { + const logDir = std.getenv('ZWE_zowe_logDirectory'); + if (logDir) { + os.mkdir(logDir, 0o750); + if (!fs.isDirectoryWritable(logDir)) { + common.printFormattedError("ZWELS", "zwe-internal-start-prepare,prepare_log_directory", `ZWEL0141E: User $(get_user_id) does not have write permission on ${logDir}.`); + std.exit(141); + } + } +} + +// Prepare workspace directory +function prepareWorkspaceDirectory() { + const zwePrivateWorkspaceEnvDir=`${workspaceDirectory}/.env`; + std.setenv('ZWE_PRIVATE_WORKSPACE_ENV_DIR', zwePrivateWorkspaceEnvDir); + const zweStaticDefinitionsDir=`${workspaceDirectory}/api-mediation/api-defs`; + std.setenv('ZWE_STATIC_DEFINITIONS_DIR', zweStaticDefinitionsDir); + const zweGatewaySharedLibs=`${workspaceDirectory}/gateway/sharedLibs/`; + std.setenv('ZWE_GATEWAY_SHARED_LIBS', zweGatewaySharedLibs); + const zweDiscoverySharedLibs=`${workspaceDirectory}/discovery/sharedLibs/`; + std.setenv('ZWE_DISCOVERY_SHARED_LIBS', zweDiscoverySharedLibs); + + let rc = fs.mkdirp(workspaceDirectory, 0o770); + if (rc != 0) { + common.printFormattedError("ZWELS", "zwe-internal-start-prepare,prepare_workspace_directory", `WARNING: Failed to set permission of some existing files or directories in ${workspaceDirectory}:`); + common.printFormattedError("ZWELS", "zwe-internal-start-prepare,prepare_workspace_directory" , ''+rc); + } + + // Create apiml dirs + fs.mkdirp(zweStaticDefinitionsDir, 0o770); + fs.mkdirp(zweGatewaySharedLibs, 0o770); + fs.mkdirp(zweDiscoverySharedLibs, 0o770); + + shell.execSync('cp', `${runtimeDirectory}/manifest.json`, workspaceDirectory); + + fs.mkdirp(zwePrivateWorkspaceEnvDir, 0o700); + + const zweCliParameterHaInstance = std.getenv('ZWE_CLI_PARAMETER_HA_INSTANCE'); + common.printFormattedDebug("ZWELS", "zwe-internal-start-prepare,prepare_workspace_directory", `initialize .instance-${zweCliParameterHaInstance}.env(s)`); + // TODO delete this or get this from configmgr instead. + config.generateInstanceEnvFromYamlConfig(zweCliParameterHaInstance); +} + +// Global validations +function globalValidate(enabledComponents:string[]): void { + common.printFormattedInfo("ZWELS", "zwe-internal-start-prepare,global_validate", "process global validations ..."); + + // validate_runtime_user + if (user == "IZUSVR") { + common.printFormattedWarn("ZWELS", "zwe-internal-start-prepare,global_validate", "ZWEL0302W: You are running the Zowe process under user id IZUSVR. This is not recommended and may impact your z/OS MF server negatively."); + } + + // reset error counter + let privateErrors = 0; + std.setenv('ZWE_PRIVATE_ERRORS_FOUND','0'); + + let writable = fs.isDirectoryWritable(workspaceDirectory); + if (!writable) { + privateErrors++; + common.printFormattedError('ZWELS', "zwe-internal-start-prepare,global_validate", `Workspace directory ${workspaceDirectory} is not writable`); + } + + if (runInContainer != 'true') { + // only do these check when it's not running in container + + // currently node is always required + let nodeOk = node.validateNodeHome(); + if (!nodeOk) { + privateErrors++; + common.printFormattedError('ZWELS', "zwe-internal-start-prepare,global_validate", `Could not validate node home`); + } + + // validate java for some core components + //TODO this should be a manifest parameter that you require java, not a hardcoded list. What if extensions require it? + if (enabledComponents.includes('gateway') || enabledComponents.includes('discovery') || enabledComponents.includes('api-catalog') || enabledComponents.includes('caching-service') || enabledComponents.includes('metrics-service') || enabledComponents.includes('files-api') || enabledComponents.includes('jobs-api')) { + let javaOk = java.validateJavaHome(); + if (!javaOk) { + privateErrors++; + common.printFormattedError('ZWELS', "zwe-internal-start-prepare,global_validate", `Could not validate java home`); + } + } + } else { + if (!containerComponentId) { + let isSet = varlib.isVariableSet("ZWE_PRIVATE_CONTAINER_COMPONENT_ID", "Cannot find name from the component image manifest file"); + if (!isSet) { + privateErrors++; + common.printFormattedError('ZWELS', "zwe-internal-start-prepare,global_validate", "Cannot find name from the component image manifest file"); + } + } + } + + // validate z/OSMF for some core components + if (zosmfHost && zosmfPort) { + if (enabledComponents.includes('discovery') || enabledComponents.includes('files-api') || enabledComponents.includes('jobs-api')) { + let zosmfOk = zosmf.validateZosmfHostAndPort(zosmfHost, zosmfPort); + if (!zosmfOk) { + privateErrors++; + common.printFormattedError('ZWELS', "zwe-internal-start-prepare,global_validate", "Zosmf validation failed"); + } + } else if (std.getenv('ZWE_components_gateway_apiml_security_auth_provider') == "zosmf") { + let zosmfOk = zosmf.validateZosmfAsAuthProvider(zosmfHost, zosmfPort, 'zosmf'); + if (!zosmfOk) { + privateErrors++; + common.printFormattedError('ZWELS', "zwe-internal-start-prepare,global_validate", "Zosmf validation failed"); + } + } + } + + std.setenv('ZWE_PRIVATE_ERRORS_FOUND',''+privateErrors); + varlib.checkRuntimeValidationResult("zwe-internal-start-prepare,global_validate"); + + common.printFormattedInfo("ZWELS", "zwe-internal-start-prepare,global_validate", "global validations are successful"); +} + + + + +// Validate component properties if script exists +function validateComponents(enabledComponents:string[]): any { + common.printFormattedInfo("ZWELS", "zwe-internal-start-prepare,validate_components", "process component validations ..."); + + const componentEnvironments = {}; + + // reset error counter + let privateErrors = 0; + std.setenv('ZWE_PRIVATE_ERRORS_FOUND','0'); + + enabledComponents.forEach((componentId: string)=> { + common.printFormattedTrace("ZWELS", "zwe-internal-start-prepare,validate_components", `- checking ${componentId}`); + const componentDir = component.findComponentDirectory(componentId); + common.printFormattedTrace("ZWELS", "zwe-internal-start-prepare,validate_components", `- in directory ${componentDir}`); + if (componentDir) { + const manifest = component.getManifest(componentDir); + + // check validate script + const validateScript = manifest.commands ? manifest.commands.validate : undefined; + common.printFormattedTrace("ZWELS", "zwe-internal-start-prepare,validate_components", `- commands.validate is ${validateScript}`); + if (validateScript) { + let fullPath = `${componentDir}/${validateScript}`; + if (fs.fileExists(fullPath)) { + common.printFormattedDebug("ZWELS", "zwe-internal-start-prepare,validate_components", `- process ${componentId} validate command ...`); + const prevErrors = privateErrors; + privateErrors = 0; + //TODO verify that this returns things that we want, currently it just uses setenv + const envVars = config.loadEnvironmentVariables(componentId); + componentEnvironments[manifest.name] = envVars; + let result = shell.execSync('sh', '-c', `. ${runtimeDirectory}/bin/libs/index.sh && . ${fullPath}`); + privateErrors=prevErrors; + if (result.rc) { + privateErrors++; + common.printFormattedError(`ZWELS`, "zwe-internal-start-prepare,validate_components", `Error: Component ${componentId} ended with nonzero rc=${result.rc}`); + } + } else { + common.printFormattedError("ZWELS", "zwe-internal-start-prepare,validate_components", `Error ZWEL0172E: Component ${componentId} has commands.validate defined but the file is missing.`); + } + } + + // check platform dependencies + if (os.platform != 'zos') { + const zosDeps = manifest.dependencies ? manifest.dependencies.zos : undefined; + if (zosDeps) { + common.printFormattedWarn("ZWELS", "zwe-internal-start-prepare,validate_components", `- ${componentId} depends on z/OS service(s). This dependency may require additional setup, please refer to the component documentation`); + } + } + } + }); + + std.setenv('ZWE_PRIVATE_ERRORS_FOUND', ''+privateErrors); + varlib.checkRuntimeValidationResult("zwe-internal-start-prepare,validate_components"); + + common.printFormattedDebug("ZWELS", "zwe-internal-start-prepare,validate_components", "component validations are successful"); + return componentEnvironments; +} + + +// Run setup/configure on components if script exists +function configureComponents(componentEnvironments?: any, enabledComponents?:string[]) { + common.printFormattedInfo("ZWELS", "zwe-internal-start-prepare,configure_components", "process component configurations ..."); + + const zwePrivateWorkspaceEnvDir = std.getenv('ZWE_PRIVATE_WORKSPACE_ENV_DIR'); + const zweCliParameterHaInstance = std.getenv('ZWE_CLI_PARAMETER_HA_INSTANCE'); + + + enabledComponents.forEach((componentId: string)=> { + common.printFormattedTrace("ZWELS", "zwe-internal-start-prepare,configure_components", `- checking ${componentId}`); + const componentDir = component.findComponentDirectory(componentId); + common.printFormattedTrace("ZWELS", "zwe-internal-start-prepare,validate_components", `- in directory ${componentDir}`); + if (componentDir) { + const manifestPath = component.getManifestPath(componentDir); + const manifest = component.getManifest(componentDir); + + // prepare component workspace + const componentName=manifest.name; + const privateWorkspaceEnvDir=`${zwePrivateWorkspaceEnvDir}/${componentName}`; + fs.mkdirp(privateWorkspaceEnvDir, 0o700); + + // copy manifest to workspace + shell.execSync('cp', manifestPath, `${privateWorkspaceEnvDir}/`); + + common.printFormattedDebug("ZWELS", "zwe-internal-start-prepare,configure_components", `- configure ${componentId}`); + + // check configure script + // TODO if this is to force 1 component to configure before another, we should really just make a manifest declaration that 1 component needs to run before another. It's fine to run a simple dependency chain to determine order of execution without getting into the advanced realm of full blown package manager dependency management tier checks + const preconfigureScript=manifest.commands ? manifest.commands.preConfigure : undefined; + common.printFormattedTrace("ZWELS", "zwe-internal-start-prepare,configure_components", `- commands.preConfigure is ${preconfigureScript}`); + if (preconfigureScript) { + const preconfigurePath=`${componentDir}/${preconfigureScript}`; + if (fs.fileExists(preconfigurePath)) { + common.printFormattedDebug("ZWELS", "zwe-internal-start-prepare,configure_components", `* process ${componentId} pre-configure command ...`); + // execute preconfigure step. preconfigure does NOT export env vars. + if (componentEnvironments) { + config.applyEnviron(componentEnvironments[componentName]); + } else { + config.loadEnvironmentVariables(componentId); + } + const result = shell.execOutErrSync('sh', '-c', `. ${runtimeDirectory}/bin/libs/index.sh && . ${preconfigurePath}`); + common.printFormattedDebug("ZWELS", "zwe-internal-start-prepare,configure_components", result.rc ? result.err : result.out); + } else { + common.printFormattedError("ZWELS", "zwe-internal-start-prepare,configure_components", `Error ZWEL0172E: Component ${componentId} has commands.preConfigure defined but the file is missing.`); + } + } + + // default build-in behaviors + // - apiml static definitions + let success=component.processComponentApimlStaticDefinitions(componentDir); + if (success) { + common.printFormattedDebug("ZWELS", "zwe-internal-start-prepare,configure_components", `${componentName} processComponentApimlStaticDefinitions success`); + } else { + common.printFormattedError("ZWELS", "zwe-internal-start-prepare,configure_components", `${componentName} processComponentApimlStaticDefinitions failure`); + } + // - generic app framework plugin + success=component.processComponentAppfwPlugin(componentDir); + if (success) { + common.printFormattedDebug("ZWELS", "zwe-internal-start-prepare,configure_components", `${componentName} processComponentAppfwPlugin success`); + } else { + common.printFormattedError("ZWELS", "zwe-internal-start-prepare,configure_components", `${componentName} processComponentAppfwPlugin failure`); + } + + // - gateway shared lib + success=component.processComponentGatewaySharedLibs(componentDir); + if (success) { + common.printFormattedDebug("ZWELS", "zwe-internal-start-prepare,configure_components", `${componentName} processComponentGatewaySharedLibs success`); + } else { + common.printFormattedError("ZWELS", "zwe-internal-start-prepare,configure_components", `${componentName} processComponentGatewaySharedLibs failure`); + } + + // - discovery shared lib + success=component.processComponentDiscoverySharedLibs(componentDir); + if (success) { + common.printFormattedDebug("ZWELS", "zwe-internal-start-prepare,configure_components", `${componentName} processComponentDiscoverySharedLibs success`); + } else { + common.printFormattedError("ZWELS", "zwe-internal-start-prepare,configure_components", `${componentName} processComponentDiscoverySharedLibs failure`); + } + + // check configure script + const configureScript = manifest.commands ? manifest.commands.configure : undefined; + common.printFormattedTrace("ZWELS", "zwe-internal-start-prepare,configure_components", `- commands.configure is ${configureScript}`); + if (configureScript) { + const fullPath = `${componentDir}/${configureScript}`; + if (fs.fileExists(fullPath)) { + common.printFormattedDebug("ZWELS", "zwe-internal-start-prepare,configure_components", `* process ${componentId} configure command ...`); + // execute configure step and generate environment snapshot + // NOTE: env var list is not updated because it should not have changed between preconfigure step and now + const result = shell.execOutSync('sh', '-c', `. ${runtimeDirectory}/bin/libs/index.sh && . ${fullPath} ; export rc=$? ; export -p`); + + common.printFormattedDebug("ZWELS", "zwe-internal-start-prepare,configure_components", `${componentName} configure ended with rc=${result.rc}`); + + if (result.rc==0) { + const exportContent = varlib.getEnvironmentExports(result.out); + if (exportContent) { + const rc = xplatform.storeFileUTF8(`${zwePrivateWorkspaceEnvDir}/${componentName}/.${zweCliParameterHaInstance}.env`, xplatform.AUTO_DETECT, exportContent); + if (!rc) { + + } else { + // set permission for the component environment snapshot + shell.execSync('chmod', `700`, `"${zwePrivateWorkspaceEnvDir}/${componentName}/.${zweCliParameterHaInstance}.env"`); + } + } + } + + if (result.rc == 0) { + common.printFormattedDebug("ZWELS", "zwe-internal-start-prepare,configure_components", result.out); + } + } else { + common.printFormattedError("ZWELS", "zwe-internal-start-prepare,configure_components", `Error ZWEL0172E: Component ${componentId} has commands.configure defined but the file is missing.`); + } + } + } + }); + + common.printFormattedDebug("ZWELS", "zwe-internal-start-prepare,configure_components", "component configurations are successful"); +} + + +// Few early steps even before initialization + +// init ZWE_RUN_IN_CONTAINER variable +const runtimeDirectory=ZOWE_CONFIG.zowe.runtimeDirectory; +std.setenv('ZWE_zowe_runtimeDirectory', runtimeDirectory); + +const extensionDirectory=ZOWE_CONFIG.zowe.extensionDirectory; +std.setenv('ZWE_zowe_extensionDirectory', extensionDirectory); + +const workspaceDirectory=ZOWE_CONFIG.zowe.workspaceDirectory; +if (!workspaceDirectory) { + common.printErrorAndExit("Error ZWEL0157E: Zowe workspace directory (zowe.workspaceDirectory) is not defined in Zowe YAML configuration file.", undefined, 157); +} +std.setenv('ZWE_zowe_workspaceDirectory', workspaceDirectory); + + +if (fs.fileExists(`${workspaceDirectory}/.init-for-container`)) { + std.setenv('ZWE_RUN_IN_CONTAINER', 'true'); +} + +// Fix node.js piles up in IPC message queue +// run this before any node command we start +if (os.platform == 'zos') { + common.printFormattedTrace("ZWELS", "zwe-internal-start-prepare", "Clean up IPC message queue before using node.js."); + shell.execSync('sh', `${runtimeDirectory}/bin/utils/cleanup-ipc-mq.sh`); +} + + +// display starting information +let manifestReturn = shell.execOutSync('cat', `${runtimeDirectory}/manifest.json`); + +const runtimeManifest = manifestReturn.rc == 0 ? JSON.parse(manifestReturn.out) : undefined; +const zoweVersion = runtimeManifest ? runtimeManifest.version : undefined; +if (zoweVersion) { + std.setenv('ZWE_VERSION', ''+zoweVersion); +} + +export function execute() { + common.printFormattedInfo("ZWELS", "zwe-internal-start-prepare", `Zowe version: v${zoweVersion}`); + common.printFormattedInfo("ZWELS", "zwe-internal-start-prepare", `build and hash: ${runtimeManifest.build.branch}#${runtimeManifest.build.number} (${runtimeManifest.build.commitHash})`); + + + // validation + if (stringlib.itemInList(std.getenv('ZWE_PRIVATE_CORE_COMPONENTS_REQUIRE_JAVA'), std.getenv('ZWE_CLI_PARAMETER_COMPONENT'))) { + // other extensions need to specify `require_java` in their validate.sh + java.requireJava(); + } + node.requireNode(); + common.requireZoweYaml(); + + // overwrite ZWE_PRIVATE_LOG_LEVEL_ZWELS with zowe.launchScript.logLevel config in YAML + if (ZOWE_CONFIG.zowe.launchScript) { + std.setenv('ZWE_PRIVATE_LOG_LEVEL_ZWELS', ZOWE_CONFIG.zowe.launchScript.logLevel.toUpperCase()); + }; + + // check and sanitize ZWE_CLI_PARAMETER_HA_INSTANCE + config.sanitizeHaInstanceId(); + common.printFormattedInfo("ZWELS", "zwe-internal-start-prepare", `starting Zowe instance ${std.getenv('ZWE_CLI_PARAMETER_HA_INSTANCE')} with ${cliParameterConfig} ...`); + + // extra preparations for running in container + // this is running in containers + if (runInContainer == 'true') { + prepareRunningInContainer(); + } + + // init log directory + prepareLogDirectory(); + + // init workspace directory and generate environment variables from YAML + prepareWorkspaceDirectory(); + + // now we can load all variables + // TODO really this should use configmgr + config.loadEnvironmentVariables(); + common.printFormattedTrace("ZWELS", "zwe-internal-start-prepare", ">>> all environment variables"); + common.printFormattedTrace("ZWELS", "zwe-internal-start-prepare", JSON.stringify(std.getenviron())); + common.printFormattedTrace("ZWELS", "zwe-internal-start-prepare", "<<<"); + + const enabledComponents=component.getEnabledComponents(); + + // main lifecycle + // global validations + // no validation for running in container + globalValidate(enabledComponents); + // no validation for running in container + let environments; + if (runInContainer != 'true') { + environments = validateComponents(enabledComponents); + } + configureComponents(environments, enabledComponents); + + // display instance prepared info + common.printFormattedInfo("ZWELS", "zwe-internal-start-prepare", "Zowe runtime environment prepared"); +} diff --git a/bin/commands/start/index.ts b/bin/commands/start/index.ts new file mode 100644 index 0000000000..e9964e7c66 --- /dev/null +++ b/bin/commands/start/index.ts @@ -0,0 +1,66 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; +import * as zos from 'zos'; +import * as common from '../../libs/common'; +import * as stringlib from '../../libs/string'; +import * as shell from '../../libs/shell'; +import * as config from '../../libs/config'; + + +common.printLevel0Message('Starting zowe'); + +// Validation +common.requireZoweYaml(); + +// Read job name and validate +const zoweConfig = config.getZoweConfig(); +const jobname = zoweConfig.zowe.job.name; +let securityStcsZowe = zoweConfig.zowe.setup.security.stcs.zowe; +if (!securityStcsZowe) { + //TODO defaults should be stored in default yaml, not out of thin air + securityStcsZowe=std.getenv('ZWE_PRIVATE_DEFAULT_ZOWE_STC'); +} +let routeSysname; + +sanitizeHaInstanceId(); +const haInstance=std.getenv('ZWE_CLI_PARAMETER_HA_INSTANCE'); +if (haInstance) { + routeSysname=zoweConfig.haInstances[haInstance].sysname; +} + +// Start job +let cmd=`S ${securityStcsZowe}`; +if (haInstance) { + cmd+=`,HAINST=${haInstance}`; +} +if (jobname) { + cmd+=`,JOBNAME=${jobname}`; +} +if (routeSysname) { + cmd=`RO ${routeSysname},${cmd}`; +} + +const shellReturn = operatorCommand(cmd); +if (shellReturn.rc) { + common.printErrorAndExit(`Error ZWEL0165E: Failed to start ${securityStcsZowe}: exit code ${shellReturn.rc}.`, undefined, 165); +} else { + //TODO handle awk and set patterns here + let errorMessage;//stringlib.trim(shellReturn.out | awk "/-S ${security_stcs_zowe}/{x=NR+1;next}(NR<=x){print}" | sed "s/^\([^ ]\+\) \+\([^ ]\+\) \+\([^ ]\+\) \+\(.\+\)\$/\4/"); + if (errorMessage) { + common.printErrorAndExit(`Error ZWEL0165E: Failed to start ${securityStcsZowe}: ${errorMessage}.`, undefined, 165); + } +} + +// Exit message +common.printLevel1Message(`Job ${jobname?jobname:securityStcsZowe} is started successfully.`); diff --git a/bin/libs/common.ts b/bin/libs/common.ts new file mode 100644 index 0000000000..547c7df333 --- /dev/null +++ b/bin/libs/common.ts @@ -0,0 +1,355 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; +import * as xplatform from 'xplatform'; + +import * as fs from './fs'; +//import * as stringlib from './string'; +import * as shell from './shell'; +import * as strftime from './strftime'; + +declare namespace console { + function log(...args:string[]): void; +}; + + +// these are shell environments we want to enforce in all cases +std.setenv('_CEE_RUNOPTS', "FILETAG(AUTOCVT,AUTOTAG) POSIX(ON)"); +std.setenv('_TAG_REDIR_IN', 'txt'); +std.setenv('_TAG_REDIR_OUT', 'txt'); +std.setenv('_TAG_REDIR_ERR', 'txt'); +std.setenv('_BPXK_AUTOCVT', "ON"); +std.setenv('_EDC_ADD_ERRNO2', '1'); //show details on error +std.unsetenv('ENV'); // just in case, as it can cause unexpected output + +enum LOG_LEVEL { + ERROR = 0, + WARN, + INFO, + DEBUG, + TRACE +}; + +function getLogLevel(name:string, defaultLevel:LOG_LEVEL):LOG_LEVEL { + switch (name.toUpperCase()){ + case "ERROR": return LOG_LEVEL.ERROR; + case "WARN": return LOG_LEVEL.WARN; + case "INFO": return LOG_LEVEL.INFO; + case "DEBUG": return LOG_LEVEL.DEBUG; + case "TRACE": return LOG_LEVEL.TRACE; + default: return defaultLevel; + } +} + + +export function requireZoweYaml() { + const configFiles = std.getenv('ZWE_CLI_PARAMETER_CONFIG'); + if (!configFiles) { + printErrorAndExit(`Error ZWEL0108E: Zowe YAML config file is required.`); + } else { + configFiles.split(',').forEach(function(file: string) { + //TODO parmlib + if (!fs.fileExists(file)) { + printErrorAndExit(`Error ZWEL0109E: The Zowe YAML config file ${file} does not exist.`, undefined, 109); + } + }); + } +} + +export function getUserId(): string|undefined { + //moved from sys to simplify dependency + let user = std.getenv('USER'); + if (!user) { + user = std.getenv('USERNAME'); + } + if (!user) { + user = std.getenv('LOGNAME'); + } + if (!user) { + let out; + let handler = (data:string)=> { + out=data; + } + const rc = os.exec(['whoami'], + {block: true, usePath: true, out: handler}); + if (!rc) { + return out; + } + } + return user; +} + + +export function date(...args: string[]): string|undefined { + if (!args) { + return strftime.strftime('%a %b %e %T %Z %Y'); + } else { + let arg = args.length == 1 ? args[0] : args[args.length-1]; + if (arg.startsWith("'+")) { + arg=arg.substring(2,arg.length-1); + } + return strftime.strftime(arg); + } +} + + +let logExists = false; +let logFile:std.File|null = null; + +function writeLog(message: string): boolean { + if (!logExists) { + const filename = std.getenv('ZWE_PRIVATE_LOG_FILE'); + if (filename) { + logExists = fs.fileExists(filename); + if (!logExists) { + fs.createFile(filename, 0o640, message); + logExists = fs.fileExists(filename); + } + if (logExists) { + let errObj = {errno:undefined}; + logFile = std.open(filename, 'w', errObj); + if (errObj.errno) { + printError(`Error opening file ${filename}, errno=${errObj.errno}`); + logFile=null; + logExists=false; + return false; + } + } + } + } + if (logFile===undefined || logFile===null) { + return false; + } else { + //TODO this does utf8. should we flip it to 1047 on zos? + logFile.puts(message); + return true; + } +} + + +export function printRawMessage(message: string, isError: boolean, writeTo:string[]=['console','log']): boolean { + if (writeTo.includes('console')) { + if (isError) { + //TODO this prints junk + //std.err.printf(stringlib.asciiToEbcdic(message+'\n')); + console.log('ERROR: '+message); + } else if (std.getenv('ZWE_CLI_PARAMETER_SILENT') != 'true') { + + console.log(message); + } + } + if (writeTo.includes('log')) { + writeLog(message+'\n'); + } + return true; +} + +export function printMessage(message: string, writeTo?:string[]): boolean { + return printRawMessage(message, false, writeTo); +} + +// errors are written to STDERR +export function printError(message: string, writeTo?:string[]): boolean { + return printRawMessage(message, true, writeTo); +} + +export function printDebug(message: string, writeTo?:string[]): boolean { + const level = std.getenv('ZWE_PRIVATE_LOG_LEVEL_ZWELS'); + if (level == 'DEBUG' || level == 'TRACE') { + return printRawMessage(message, false, writeTo); + } + return false; +} + +export function printTrace(message: string, writeTo?:string[]): boolean { + const level = std.getenv('ZWE_PRIVATE_LOG_LEVEL_ZWELS'); + if (level == 'TRACE') { + return printRawMessage(message, false, writeTo); + } + return false; +} + +export function printErrorAndExit(message: string, writeTo:string[]=['console','log'], exitCode:number=1): void { + printError(message, writeTo); + std.exit(exitCode); +} + +export function printEmptyLine(writeTo?:string[]): boolean { + return printMessage("", writeTo); +} + +export function printLevel0Message(message?: string, writeTo?:string[]): boolean { + printMessage("===============================================================================", writeTo); + if (message) { + printMessage(`>> ${message.toUpperCase()}`, writeTo); + } + return printEmptyLine(writeTo); +} + +export function printLevel1Message(message?: string, writeTo?:string[]): boolean { + printMessage("-------------------------------------------------------------------------------", writeTo); + if (message) { + printMessage(`>> ${message}`, writeTo); + } + return printEmptyLine(writeTo); +} + +export function printLevel2Message(message?: string, writeTo?:string[]): boolean { + printEmptyLine(writeTo); + if (message) { + printMessage(`>> ${message}`, writeTo); + } + return printEmptyLine(writeTo); +} + +export function printLevel0Debug(message?: string, writeTo?:string[]): boolean { + printDebug("===============================================================================", writeTo); + if (message) { + printDebug(`>> ${message.toUpperCase()}`, writeTo); + } + return printDebug('', writeTo); +} + +export function printLevel1Debug(message?: string, writeTo?:string[]): boolean { + printDebug("-------------------------------------------------------------------------------", writeTo); + if (message) { + printDebug(`>> ${message}`, writeTo); + } + return printDebug('', writeTo); +} + +export function printLevel2Debug(message?: string, writeTo?:string[]): boolean { + printDebug('', writeTo); + if (message) { + printDebug(`>> ${message}`, writeTo); + } + return printDebug('', writeTo); +} + +export function printLevel0Trace(message?: string, writeTo?:string[]): boolean { + printTrace("===============================================================================", writeTo); + if (message) { + printTrace(`>> ${message.toUpperCase()}`, writeTo); + } + return printTrace('', writeTo); +} + +export function printLevel1Trace(message?: string, writeTo?:string[]): boolean { + printTrace("-------------------------------------------------------------------------------", writeTo); + if (message) { + printTrace(`>> ${message}`, writeTo); + } + return printTrace('', writeTo); +} + +//TODO is message ever missing or should we just enforce it +export function printLevel2Trace(message?: string, writeTo?:string[]): boolean { + printTrace('', writeTo); + if (message) { + printTrace(`>> ${message}`, writeTo); + } + return printTrace('', writeTo); +} + + +const FORMATTING_TEST = /^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}/; +// runtime logging functions, follow zowe service logging standard +export function printFormattedMessage(service: string, logger: string, levelName: string, message: string): boolean { + if (message == '-') { + //TODO readmessage + if (!message) { + return false; + } + } + + const level:LOG_LEVEL = getLogLevel(levelName,LOG_LEVEL.INFO); + const canonicalLevelName = LOG_LEVEL[level]; + const envLogValue = std.getenv(`ZWE_PRIVATE_LOG_LEVEL_${service}`); + const expectedLogLevelVal:LOG_LEVEL = envLogValue ? getLogLevel(envLogValue,LOG_LEVEL.INFO) : LOG_LEVEL.INFO; + let displayLog = expectedLogLevelVal >= level; + if (!displayLog) { + return false; + } + + const logLinePrefix=`${date("-u", "'+%Y-%m-%d %T'")} <${service}:${xplatform.getpid()}> ${getUserId()} ${canonicalLevelName} (${logger})`; + let lines = message.split('\n'); + lines.forEach((line: string)=> { + if (!FORMATTING_TEST.test(line)) { + line = `${logLinePrefix} ${line}`; + } + if (level == LOG_LEVEL.ERROR) { + return printError(line, ['console']); + } + return printMessage(line, ['console']); + }); + return true; +} + +export function printFormattedTrace(service: string, logger: string, message: string): boolean { + return printFormattedMessage(service, logger, "TRACE", message); +} + +export function printFormattedDebug(service: string, logger: string, message: string): boolean { + return printFormattedMessage(service, logger, "DEBUG", message); +} + +export function printFormattedInfo(service: string, logger: string, message: string): boolean { + return printFormattedMessage(service, logger, "INFO", message); +} + +export function printFormattedWarn(service: string, logger: string, message: string): boolean { + return printFormattedMessage(service, logger, "WARN", message); +} + +export function printFormattedError(service: string, logger: string, message: string): boolean { + return printFormattedMessage(service, logger, "ERROR", message); +} + + +let runtimeManifest:any; +export function getZoweRuntimeManifest(): any|undefined { + if (!runtimeManifest) { + const manifestFileName = `${std.getenv('ZWE_zowe_runtimeDirectory')}/manifest.json`; + const result = xplatform.loadFileUTF8(manifestFileName,xplatform.AUTO_DETECT); + if (result){ + printError('Could not read runtime manifest in '+manifestFileName); + } else { + runtimeManifest=JSON.parse(result); + } + } + return runtimeManifest; +} + +export function getZoweVersion(): string|undefined { + if (!std.getenv('ZWE_VERSION')) { + let manifest = getZoweRuntimeManifest(); + if (manifest) { + std.setenv('ZWE_VERSION', manifest.version); + } + } + return std.getenv('ZWE_VERSION'); +} + + +//From 'index.sh' +std.setenv('ZWE_PRIVATE_DS_SZWESAMP', 'SZWESAMP'); +std.setenv('ZWE_PRIVATE_DS_SZWEEXEC', 'SZWEEXEC'); +std.setenv('ZWE_PRIVATE_DEFAULT_ADMIN_GROUP', 'ZWEADMIN'); +std.setenv('ZWE_PRIVATE_DEFAULT_ZOWE_USER', 'ZWESVUSR'); +std.setenv('ZWE_PRIVATE_DEFAULT_ZIS_USER', 'ZWESIUSR'); +std.setenv('ZWE_PRIVATE_DEFAULT_ZOWE_STC', 'ZWESLSTC'); +std.setenv('ZWE_PRIVATE_DEFAULT_ZIS_STC', 'ZWESISTC'); +std.setenv('ZWE_PRIVATE_DEFAULT_AUX_STC', 'ZWESASTC'); +std.setenv('ZWE_PRIVATE_CORE_COMPONENTS_REQUIRE_JAVA', 'gateway,discovery,api-catalog,caching-service,metrics-service,files-api,jobs-api'); + +std.setenv('ZWE_PRIVATE_CLI_LIBRARY_LOADED', 'true'); diff --git a/bin/libs/component.ts b/bin/libs/component.ts new file mode 100644 index 0000000000..71d7a0a356 --- /dev/null +++ b/bin/libs/component.ts @@ -0,0 +1,733 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; +import * as zos from 'zos'; +import * as xplatform from 'xplatform'; +import { ConfigManager } from 'Configuration'; + +import * as common from './common'; +import * as fs from './fs'; +import * as zosfs from './zos-fs'; +import * as stringlib from './string'; +import * as shell from './shell'; +import * as configmgr from './configmgr'; + +const CONFIG_MGR=configmgr.CONFIG_MGR; +const ZOWE_CONFIG=configmgr.ZOWE_CONFIG; +const runtimeDirectory=ZOWE_CONFIG.zowe.runtimeDirectory; +const extensionDirectory=ZOWE_CONFIG.zowe.extensionDirectory; +const workspaceDirectory=ZOWE_CONFIG.zowe.workspaceDirectory; + +//key: name of config, value: boolean on if it is cached already +const configLoadedList:any = {}; + + +//TODO this file is full of printErrorAndExit. unreasonable? + +const COMMON_SCHEMA = `${runtimeDirectory}/schemas/server-common.json`; +const MANIFEST_SCHEMA_ID = 'https://zowe.org/schemas/v2/server-component-manifest'; +const MANIFEST_SCHEMAS = `${runtimeDirectory}/schemas/manifest-schema.json:${COMMON_SCHEMA}`; +const PLUGIN_DEF_SCHEMA_ID = "https://zowe.org/schemas/v2/appfw-plugin-definition"; +const PLUGIN_DEF_SCHEMAS = `${runtimeDirectory}/components/app-server/schemas/plugindefinition-schema.json`; + + +export function getEnabledComponents(): string[] { + let components = Object.keys(ZOWE_CONFIG.components); + let enabled:string[] = []; + components.forEach((key:string) => { + if (ZOWE_CONFIG.components[key].enabled == true) { + enabled.push(key); + } + }); + return enabled; +} + +export function getManifestPath(componentDir: string): string|undefined { + if (fs.fileExists(`${componentDir}/manifest.yaml`)) { + return `${componentDir}/manifest.yaml`; + } else if (fs.fileExists(`${componentDir}/manifest.yml`)) { + return `${componentDir}/manifest.yml`; + } else if (fs.fileExists(`${componentDir}/manifest.yaml`)) { + return `${componentDir}/manifest.json`; + } + return undefined; +} + +export function findComponentDirectory(componentId: string): string|undefined { + if (fs.directoryExists(`${runtimeDirectory}/components/${componentId}`)) { + return `${runtimeDirectory}/components/${componentId}`; + } else if (extensionDirectory && fs.directoryExists(`${extensionDirectory}/${componentId}`)) { + return `${extensionDirectory}/${componentId}`; + } + return undefined; +} + +const pluginPointerDirectory = `${workspaceDirectory}/app-server/plugins`; +export function registerPlugin(path:string, pluginDefinition:any){ + const filePath = `${pluginPointerDirectory}/${pluginDefinition.identifier}.json`; + if (fs.fileExists(filePath)) { + return true; + } else { + let location, relativeTo; + const index = path.indexOf(runtimeDirectory); + if (index != -1) { + relativeTo = "$ZWE_zowe_runtimeDirectory"; + location = filePath.substring(index); + return fs.createFile(filePath, 0o770, JSON.stringify({ + "identifier": pluginDefinition.identifier, + "pluginLocation": location, + "relativeTo": relativeTo + }, null, 2)); + } else { + return fs.createFile(filePath, 0o770, JSON.stringify({ + "identifier": pluginDefinition.identifier, + "pluginLocation": filePath + }, null, 2)); + } + } +} + +function showExceptions(e: any,depth: number): void { + let blanks = " "; + let subs = e.subExceptions; + common.printError(blanks.substring(0,depth*2)+e.message); + if (subs){ + for (const sub of subs){ + showExceptions(sub,depth+1); + } + } +} + +export function getPluginDefinition(pluginRootPath:string) { + const pluginDefinitionPath = `${pluginRootPath}/pluginDefinition.json`; + + if (fs.fileExists(pluginDefinitionPath)) { + let status; + if ((status = CONFIG_MGR.addConfig(pluginRootPath))) { + common.printErrorAndExit(`Could not add config for ${pluginRootPath}, status=${status}`); + return null; + } + + if ((status = CONFIG_MGR.loadSchemas(pluginRootPath, PLUGIN_DEF_SCHEMAS))) { + common.printErrorAndExit(`Could not load schemas ${PLUGIN_DEF_SCHEMAS} for plugin ${pluginRootPath}, status=${status}`); + return null; + } + + + if ((status = CONFIG_MGR.setConfigPath(pluginRootPath, `FILE(${pluginDefinitionPath})`))) { + common.printErrorAndExit(`Could not set config path for ${pluginDefinitionPath}, status=${status}`); + return null; + } + if ((status = CONFIG_MGR.loadConfiguration(pluginRootPath))) { + common.printErrorAndExit(`Could not load config for ${pluginDefinitionPath}, status=${status}`); + return null; + } + + let validation = CONFIG_MGR.validate(pluginRootPath); + if (validation.ok){ + if (validation.exceptionTree){ + common.printError(`Validation of ${pluginDefinitionPath} against schema ${PLUGIN_DEF_SCHEMA_ID} found invalid JSON Schema data`); + showExceptions(validation.exceptionTree, 0); + std.exit(1); + return null; + } else { + return CONFIG_MGR.getConfigData(pluginRootPath); + } + } else { + common.printErrorAndExit(`Error occurred on validation of ${pluginDefinitionPath} against schema ${PLUGIN_DEF_SCHEMA_ID} `); + return null; + } + } else { + common.printErrorAndExit(`Plugin at ${pluginRootPath} has no pluginDefinition.json`); + return null; + } +} + + +export function getManifest(componentDirectory: string): any { + let manifestPath = getManifestPath(componentDirectory); + + if (manifestPath) { + let status; + + let manifestId = componentDirectory; + if (configLoadedList[manifestId] === true) { + return CONFIG_MGR.getConfigData(manifestId); + } + + if ((status = CONFIG_MGR.addConfig(manifestId))) { + common.printErrorAndExit(`Could not add config for ${manifestPath}, status=${status}`); + return null; + } + + if ((status = CONFIG_MGR.loadSchemas(manifestId, MANIFEST_SCHEMAS))) { + common.printErrorAndExit(`Could not load schemas ${MANIFEST_SCHEMAS} for manifest ${manifestPath}, status=${status}`); + return null; + } + + if ((status = CONFIG_MGR.setConfigPath(manifestId, `FILE(${manifestPath})`))) { + common.printErrorAndExit(`Could not set config path for ${manifestPath}, status=${status}`); + return null; + } + + if ((status = CONFIG_MGR.loadConfiguration(manifestId))) { + common.printErrorAndExit(`Could not load config for ${manifestPath}, status=${status}`); + return null; + } + + let validation = CONFIG_MGR.validate(manifestId); + if (validation.ok){ + if (validation.exceptionTree){ + common.printError(`Validation of ${manifestPath} against schema ${MANIFEST_SCHEMA_ID} found invalid JSON Schema data`); + showExceptions(validation.exceptionTree, 0); + std.exit(1); + return null; + } else { + configLoadedList[manifestId] = true; + return CONFIG_MGR.getConfigData(manifestId); + } + } else { + common.printErrorAndExit(`Error occurred on validation of ${manifestPath} against schema ${MANIFEST_SCHEMA_ID} `); + return null; + } + } else { + common.printErrorAndExit(`Component at ${componentDirectory} has no manifest`); + return null; + } +} + +export function detectComponentManifestEncoding(componentDir: string): number|undefined { + const manifestPath = getManifestPath(componentDir); + if (!manifestPath) { + return undefined; + } + const encoding = zosfs.detectFileEncoding(manifestPath, 'name'); + return encoding!==-1 ? encoding : undefined; +} + +export function detectIfComponentTagged(componentDir: string): boolean { + const manifestPath = getManifestPath(componentDir); + if (!manifestPath) { + return false; + } + const encoding = zosfs.getFileEncoding(manifestPath); + if (encoding===undefined) { + return false; + } + return encoding!==0; +} + +export function findAllInstalledComponents(): string { + let components=''; + let subDirectories = fs.getSubdirectories(`${runtimeDirectory}/components`); + if (subDirectories) { + subDirectories.forEach((component:string)=> { + if (getManifestPath(`${runtimeDirectory}/components/${component}`)) { + components=`${components},${component}`; + } + }); + } + + if (extensionDirectory && fs.directoryExists(extensionDirectory)) { + subDirectories = fs.getSubdirectories(extensionDirectory); + if (subDirectories) { + subDirectories.forEach((component: string)=> { + if (getManifestPath(`${extensionDirectory}/${component}`)) { + components=`${components},${component}`; + } + }); + } + } + return components.length > 1 ? components.substring(1) : components; +} + +export function findAllInstalledComponents2(): string[] { + let components:string[] = []; + let subDirectories = fs.getSubdirectories(`${runtimeDirectory}/components`); + if (subDirectories) { + subDirectories.forEach((component:string)=> { + if (getManifestPath(`${runtimeDirectory}/components/${component}`)) { + components.push(component); + } + }); + } + + if (extensionDirectory && fs.directoryExists(extensionDirectory)) { + subDirectories = fs.getSubdirectories(extensionDirectory); + if (subDirectories) { + subDirectories.forEach((component: string)=> { + if (getManifestPath(`${extensionDirectory}/${component}`)) { + components.push(component); + } + }); + } + } + return components; +} + +export function findAllEnabledComponents(): string { + return findAllEnabledComponents2().join(','); +} + +export function findAllEnabledComponents2(): string[] { + let installedComponentsEnv=std.getenv('ZWE_INSTALLED_COMPONENTS'); + let installedComponents = installedComponentsEnv ? installedComponentsEnv.split(',') : null; + if (!installedComponents) { + installedComponents = findAllInstalledComponents2(); + } + return installedComponents.filter(function(component: string) { + let componentNameAsEnv=stringlib.sanitizeAlphanum(component); + return std.getenv(`ZWE_components_${componentNameAsEnv}_enabled`) == 'true'; + }); +} + +export function findAllLaunchComponents(): string { + return findAllLaunchComponents2().join(','); +} + +export function findAllLaunchComponents2(): string[] { + let enabledComponentsEnv=std.getenv('ZWE_ENABLED_COMPONENTS'); + let enabledComponents = enabledComponentsEnv ? enabledComponentsEnv.split(',') : null; + if (!enabledComponents) { + enabledComponents = findAllEnabledComponents2(); + } + + return enabledComponents.filter(function(component: string) { + const componentDir = findComponentDirectory(component); + if (componentDir) { + const manifest = getManifest(componentDir); + if (manifest && manifest.commands && manifest.commands.start) { + return fs.fileExists(`${componentDir}/${manifest.commands.start}`); + } + } + return false; + }); +} + +/* +${var#Pattern}, ${var##Pattern} + + ${var#Pattern} Remove from $var the shortest part of $Pattern that matches the front end of $var. + + ${var##Pattern} Remove from $var the longest part of $Pattern that matches the front end of $var. + +*/ + +function resolveShellVariable(previousKey: string, currentKey: string, currentValue: string|undefined, modifier: string): string|undefined { + switch (modifier) { + //${parameter-default}, ${parameter:-default} + // If parameter not set, use default. + case '-': { + return std.getenv(currentKey); + } + //${parameter+alt_value}, ${parameter:+alt_value} + // If parameter set, use alt_value, else use null string. + case '+': { + return std.getenv(previousKey) ? std.getenv(currentKey) : 'null'; + } + //${parameter?err_msg}, ${parameter:?err_msg} + // If parameter set, use it, else print err_msg and abort the script with an exit status of 1. + case '?': { + let prev; + if ((prev=std.getenv(previousKey))) { + return prev; + } else { + common.printError(currentKey); + return currentValue; + } + } + //${parameter=default}, ${parameter:=default} + // If parameter not set, set it to default. + case '=': { + if (!std.getenv(previousKey)) { + std.setenv(previousKey, std.getenv(currentKey)); + } + return std.getenv(previousKey); + } + default: + return undefined; + } +} + + +export function resolveShellTemplate(content: string): string { + let position = 0; + let output = ''; + + while (position != -1 && position < content.length) { + let index = content.indexOf('$', position); + if (index == -1) { + output+=content.substring(position); + return output; + } else { + output+=content.substring(position, index); + if (content[index+1] === '{') { + let endIndex = content.indexOf('}', index+2); + if (endIndex == -1) { + output+=content.substring(position); + return output; + } + + //${#var} + // String length (number of characters in $var). For an array, ${#array} is the length of the first element in the array. + if (content[index+2] === '#') { + let value = std.getenv(content.substring(index+3, endIndex)); + if (value!==undefined) { + output+=value.length; + position=endIndex; + continue; + } + } + + let accumIndex = index+2; + let currentIndex = index+2; + let envValue; + let firstKey; + let previousKey=null; + let currentKey; + let previousModifier; + while ((currentIndex 0x40) || (charCode < 0x7b && charCode > 0x60) || (charCode > 0x2f && charCode < 0x40) || (charCode == 0x5f)) { + keyIndex++; + } + let val = std.getenv(content.substring(index+1, keyIndex)); + if (val!==undefined) { + output+=val; + } + position=keyIndex; + } + } + } + return output; +} + + +export function processComponentApimlStaticDefinitions(componentDir: string): boolean { + const STATIC_DEF_DIR=std.getenv('ZWE_STATIC_DEFINITIONS_DIR'); + if (!STATIC_DEF_DIR) { + common.printError("Error: ZWE_STATIC_DEFINITIONS_DIR is required to process component definitions for API Mediation Layer."); + return false; + } + + const manifest = getManifest(componentDir); + if (!manifest) { + common.printError(`Error: manifest read or validation fail for ${componentDir}`); + return false; + } + + let allSucceed=true; + const componentName = manifest.name; + if (manifest.apimlServices && manifest.apimlServices.static) { + let staticDefs = manifest.apimlServices.static; + for (let i = 0; i < staticDefs.length; i++) { + const staticDef = staticDefs[i]; + const file=staticDef.file; + const path = `${componentDir}/${file}` + if (!fs.fileExists(path)){ + common.printError("Error: static definition file ${file} of ${componentName} is not accessible"); + allSucceed=false; + break; + } else { + common.printDebug(`Process ${componentName} service static definition file ${file}`); + const sanitizedDefName=stringlib.sanitizeAlphanum(file); + + const contents = xplatform.loadFileUTF8(path,xplatform.AUTO_DETECT); + if (contents) { + const resolvedContents = resolveShellTemplate(contents); + + const zweCliParameterHaInstance=std.getenv("ZWE_CLI_PARAMETER_HA_INSTANCE"); + const outPath=`${STATIC_DEF_DIR}/${componentName}.${sanitizedDefName}.${zweCliParameterHaInstance}.yaml`; + + common.printDebug(`- writing ${outPath}`); + + + xplatform.storeFileUTF8(outPath, xplatform.AUTO_DETECT, resolvedContents); + shell.execSync(`chmod`, `770`, outPath); + } + } + } + } + return allSucceed; +} + +/* + Parse and process manifest App Framework Plugin (appfwPlugins) definitions + + The supported manifest entry is ".appfwPlugins". All plugins + defined will be passed to install-app.sh for proper installation. +*/ +export function testOrSetPcBit(path: string): boolean { + if (!hasPCBit(path)) { + common.printError("Plugin ZSS API not program controlled. Attempting to add PC bit."); + zos.changeExtAttr(path, zos.EXTATTR_PROGCTL, true); + const success = hasPCBit(path); + if (!success) { + common.printErrorAndExit("PC bit not set. This must be set such as by executing 'extattr +p $COMPONENT_HOME/lib/sys.so' as a user with sufficient privilege."); + } + return success; + } else { + return true; + } +} + +export function hasPCBit(path: string): boolean { + const returnArray = zos.zstat(path); + if (!returnArray[1]) { //no error + return returnArray[0].extattrs == zos.EXTATTR_PROGCTL + } else { + if (returnArray[1] != std.Error.ENOENT) { + common.printError(`hasPCBit path=${path}, err=${returnArray[1]}`); + } + return false; + } +} + + +export function checkZssPcBit(appfwPluginPath: string): void { + const pluginDefinition = getPluginDefinition(appfwPluginPath); + if (pluginDefinition) { + if (pluginDefinition.dataServices) { + common.printDebug(`Checking ZSS services in plugin path=${appfwPluginPath}`); + pluginDefinition.dataServices.forEach(function(service: any){ + if (service.type == 'service') { + if (service.libraryName31) { + testOrSetPcBit(`${appfwPluginPath}/lib/${service.libraryName31}`); + } + if (service.libraryName64) { + testOrSetPcBit(`${appfwPluginPath}/lib/${service.libraryName64}`); + } + if (service.libraryName) { + testOrSetPcBit(`${appfwPluginPath}/lib/${service.libraryName}`); + } + } + }); + } + } else { + common.printErrorAndExit(`Skipping ZSS PC bit check of plugin at ${appfwPluginPath} due to pluginDefinition missing or invalid`); + } +} + +export function processZssPluginInstall(componentDir: string): void { + if (os.platform == 'zos') { + common.printDebug(`- Checking for zss plugins and verifying them`); + const manifest = getManifest(componentDir); + if (manifest && manifest.appfwPlugins) { + manifest.appfwPlugins.forEach(function(appfwPlugin: any) { + const path = appfwPlugin.path; + checkZssPcBit(`${componentDir}/${path}`); + }); + } + } +} + + + +export function processComponentAppfwPlugin(componentDir: string): boolean { + const manifest = getManifest(componentDir); + if (manifest && manifest.appfwPlugins) { + for (let i = 0; i < manifest.appfwPlugins.length; i++) { + const appfwPlugin = manifest.appfwPlugins[i]; + const fullPath = `${componentDir}/${appfwPlugin.path}`; + if (!fs.fileExists(`${fullPath}/pluginDefinition.json`)) { + common.printError(`App Framework plugin directory ${fullPath} does not have pluginDefinition.json`); + return false; + } + + if (os.platform != 'zos') { + const pluginDefinition = getPluginDefinition(fullPath); + if (pluginDefinition && pluginDefinition.identifier) { + const pluginDirsPath=`${workspaceDirectory}/app-server/pluginDirs`; + let rc = fs.mkdirp(`${pluginDirsPath}/${pluginDefinition.identifier}`, 0o770); + if (rc) { + common.printError(`Plugin registration failed because cannot make directory = ${pluginDirsPath}/${pluginDefinition.identifier}`); + } + fs.cpr(`${fullPath}/.`, `${pluginDirsPath}/${pluginDefinition.identifier}`); + + return registerPlugin(fullPath, pluginDefinition); + } else { + common.printError(`Cannot read identifier from App Framework plugin ${fullPath}/pluginDefinition.json`); + return false; + } + } + } + } + return true; +} + +/* + Parse and process manifest Gateway Shared Libs (gatewaySharedLibs) definitions + + The supported manifest entry is ".gatewaySharedLibs". All shared libs + defined will be passed to install-app.sh for proper installation. +*/ +export function processComponentGatewaySharedLibs(componentDir: string): boolean { + const gatewaySharedLibs = std.getenv('ZWE_GATEWAY_SHARED_LIBS'); + fs.mkdirp(gatewaySharedLibs, 0o770); + + const manifest = getManifest(componentDir); + let pluginName; + let gatewaySharedLibsWorkspacePath:string|undefined; + + if (manifest && manifest.gatewaySharedLibs) { + for (let i = 0; i < manifest.gatewaySharedLibs.length; i++) { + const gatewaySharedLibsDef = manifest.gatewaySharedLibs[i]; + const fileOrDir=`${componentDir}/${gatewaySharedLibsDef}`; + if (!pluginName) { + pluginName = manifest.name; + if (!pluginName) { + common.printError(`Cannot read name from the plugin ${componentDir}`); + return false; + } + gatewaySharedLibsWorkspacePath = `${gatewaySharedLibs}/${pluginName}`; + fs.mkdirp(gatewaySharedLibsWorkspacePath, 0o770); + } + + if (!gatewaySharedLibsWorkspacePath){ + common.printError("Unexpected error: did not find gatewaySharedLibsWorkspacePath"); + return false; + } + + const manifestPath = getManifestPath(componentDir); + if (manifestPath){ + fs.cp(manifestPath, gatewaySharedLibsWorkspacePath); + } + + if (fs.fileExists(fileOrDir)) { + fs.cp(fileOrDir, gatewaySharedLibsWorkspacePath); + } else if (fs.directoryExists(fileOrDir)) { + fs.cp(`${fileOrDir}/\*`, gatewaySharedLibsWorkspacePath); + } else { + common.printError(`Gateway shared libs directory ${fileOrDir} is not accessible`); + return false; + } + } + } + return true; +} + + +/* + Parse and process manifest Discovery Shared Libs (discoverySharedLibs) definitions + + The supported manifest entry is ".discoverySharedLibs". All shared libs + defined will be passed to install-app.sh for proper installation. +*/ +export function processComponentDiscoverySharedLibs(componentDir: string): boolean { + const discoverySharedLibs = std.getenv('ZWE_DISCOVERY_SHARED_LIBS'); + fs.mkdirp(discoverySharedLibs, 0o770); + + const manifest = getManifest(componentDir); + let pluginName; + let discoverySharedLibsWorkspacePath; + + if (manifest && manifest.discoverySharedLibs) { + for (let i = 0; i < manifest.discoverySharedLibs.length; i++) { + const discoverySharedLibsDef = manifest.discoverySharedLibs[i]; + const fileOrDir=`${componentDir}/${discoverySharedLibsDef}`; + if (!pluginName) { + pluginName = manifest.name; + if (!pluginName) { + common.printError(`Cannot read name from the plugin ${componentDir}`); + return false; + } + discoverySharedLibsWorkspacePath = `${discoverySharedLibs}/${pluginName}`; + fs.mkdirp(discoverySharedLibsWorkspacePath, 0o770); + } + + if (!discoverySharedLibsWorkspacePath){ + common.printError('Unexpected error: did not find discoverySharedLibsWorkspacePath'); + return false; + } + + const manifestPath = getManifestPath(componentDir); + if (manifestPath) { + fs.cp(manifestPath, discoverySharedLibsWorkspacePath); + } + + if (fs.fileExists(fileOrDir)) { + fs.cp(fileOrDir, discoverySharedLibsWorkspacePath); + } else if (fs.directoryExists(fileOrDir)) { + fs.cp(`${fileOrDir}/\*`, discoverySharedLibsWorkspacePath); + } else { + common.printError(`Discovery shared libs directory ${fileOrDir} is not accessible`); + return false; + } + } + } + return true; +} +/* +const gatewayHost = std.getenv('ZWE_GATEWAY_HOST'); +const haInstanceHostname = std.getenv('ZWE_haInstance_hostname'); +const catalogPort = Number(std.getenv('ZWE_components_api_catalog_port')); +const zoweCertificatePemKey = std.getenv('ZWE_zowe_certificate_pem_key'); +const zoweCertificatePemCertificate = std.getenv('ZWE_zowe_certificate_pem_certificate'); +const zoweCertificatePemCertificateAuthorities = std.getenv('ZWE_zowe_certificate_pem_certificateAuthorities'); +//TODO implement refreshStaticRegistration + +export function refreshStaticRegistration(apimlcatalogHost: string=gatewayHost, apimlcatalogPort: number= catalogPort, + authKey: string=zoweCertificatePemKey, authCert: string=zoweCertificatePemCertificate, + caCert: string=zoweCertificatePemCertificateAuthorities): number{ + if (!apimlcatalogHost) { + if (haInstanceHostname) { + apimlcatalogHost = haInstanceHostname; + } else { + apimlcatalogHost = 'localhost'; + } + } +} +*/ diff --git a/bin/libs/config.ts b/bin/libs/config.ts new file mode 100644 index 0000000000..566a5ca129 --- /dev/null +++ b/bin/libs/config.ts @@ -0,0 +1,257 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; +import * as zos from 'zos'; +import * as xplatform from 'xplatform'; + +import * as common from './common'; +import * as fs from './fs'; +import * as stringlib from './string'; +import * as shell from './shell'; +import * as varlib from './var'; +import * as configmgr from './configmgr'; +import * as component from './component'; +import * as zosfs from './zos-fs'; +import * as sys from './sys'; +import * as container from './container'; +import * as node from './node'; + +const cliParameterConfig:string = function() { + let value = std.getenv('ZWE_CLI_PARAMETER_CONFIG'); + if (!value){ + std.out.printf("No ZWE_CLI_PARAMETER_CONFIG env var, exiting"); + std.exit(0); + } + return (value as string); +}(); + +const runtimeDirectory=configmgr.ZOWE_CONFIG.zowe.runtimeDirectory; +//const extensionDirectory=ZOWE_CONFIG.zowe.extensionDirectory; +const workspaceDirectory=configmgr.ZOWE_CONFIG.zowe.workspaceDirectory; + +export function getZoweConfig(): any { + return configmgr.ZOWE_CONFIG; +} + +// Convert instance.env to zowe.yaml file +export function convertInstanceEnvToYaml(instanceEnv: string, zoweYaml?: string) { + // we need node for following commands + node.ensureNodeIsOnPath(); + + if (!zoweYaml) { + shell.execSync('node', `${std.getenv('ROOT_DIR')}/bin/utils/config-converter/src/cli.js`, `env`, `yaml`, instanceEnv); + } else { + shell.execSync('node', `${std.getenv('ROOT_DIR')}/bin/utils/config-converter/src/cli.js`, `env`, `yaml`, instanceEnv, `-o`, zoweYaml); + + zosfs.ensureFileEncoding(zoweYaml, "zowe:", 1047); + + shell.execSync('chmod', `640`, zoweYaml); + } +} + +////////////////////////////////////////////////////////////// +// Check encoding of a file and convert to IBM-1047 if needed. +// +// Note: usually this is required if the file is supposed to be shell script, +// which requires to be IBM-1047 encoding. +// +export function zosConvertEnvDirFileEncoding(file: string) { + const encoding=zosfs.getFileEncoding(file); + if (encoding && encoding != 0 && encoding != 1047) { + const tmpfile=`${std.getenv('ZWE_PRIVATE_WORKSPACE_ENV_DIR')}/t`; + os.remove(tmpfile); + shell.execSync(`iconv`, `-f`, ""+encoding, `-t`, `IBM-1047`, file, `>`, tmpfile); + os.rename(tmpfile, file); + shell.execSync('chmod', `640`, file); + common.printTrace(`>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>AFTER ${file}`); + const lsReturn = shell.execOutSync('ls', `-laT`, file); + common.printTrace(lsReturn.out || ""); + } +} + +// Prepare configuration for current HA instance, and generate backward +// compatible instance.env files from zowe.yaml. +// +export function generateInstanceEnvFromYamlConfig(haInstance: string) { + let zwePrivateWorkspaceEnvDir = std.getenv('ZWE_PRIVATE_WORKSPACE_ENV_DIR'); + if (!zwePrivateWorkspaceEnvDir) { + zwePrivateWorkspaceEnvDir=`${workspaceDirectory}/.env` + std.setenv('zwePrivateWorkspaceEnvDir', zwePrivateWorkspaceEnvDir); + } + + // delete old files to avoid potential issues + common.printFormattedTrace( "ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `deleting old files under ${zwePrivateWorkspaceEnvDir}`); + let foundFiles = fs.getFilesInDirectory(zwePrivateWorkspaceEnvDir); + if (foundFiles) { + foundFiles.forEach((file:string)=> { + if (file.endsWith(".zowe.json") + || file.endsWith(`-${haInstance}.env`) + || file.endsWith(`-${haInstance}.json`)) { + os.remove(zwePrivateWorkspaceEnvDir+'/'+file); + } + }); + } + + const components = component.findAllInstalledComponents2(); + //TODO use configmgr to write json and ha json, and components json + + // prepare .zowe.json and .zowe-.json + common.printFormattedTrace("ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `config-converter yaml convert --ha ${haInstance} ${cliParameterConfig}`); + let result = shell.execOutSync('node', `${runtimeDirectory}/bin/utils/config-converter/src/cli.js`, `yaml`, `convert`, `--wd`, zwePrivateWorkspaceEnvDir, `--ha`, haInstance, cliParameterConfig, `--verbose`); + + common.printFormattedTrace("ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `- Exit code: ${result.rc}: ${result.out}`); + if ( !fs.fileExists(`${zwePrivateWorkspaceEnvDir}/.zowe.json`)) { + common.printFormattedError( "ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `ZWEL0140E: Failed to translate Zowe configuration (${cliParameterConfig}).`); + std.exit(140); + } + + // convert YAML configurations to backward compatible .instance-.env files + common.printFormattedTrace("ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `config-converter yaml env --ha ${haInstance}`); + const envs = configmgr.getZoweConfigEnv(); + common.printFormattedTrace("ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `- Output: ${JSON.stringify(envs, null, 2)}`); + const envKeys = Object.keys(envs); + let envFileArray=[]; + + envFileArray.push('#!/bin/sh'); + envKeys.forEach((key:string)=> { + envFileArray.push(`${key}=${envs[key]}`); + }); + + let envFileContent = envFileArray.join('\n'); + let rc = xplatform.storeFileUTF8(`${zwePrivateWorkspaceEnvDir}/.instance-${haInstance}.env`, xplatform.AUTO_DETECT, envFileContent); + if (rc) { + common.printFormattedError("ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `ZWEL0140E: Failed to translate Zowe configuration (${cliParameterConfig}).`); + std.exit(140); + return; + } + + + components.forEach((component:string)=> { + const componentAlpha = stringlib.sanitizeAlpha(component); + const folderName = `${zwePrivateWorkspaceEnvDir}/${component}`; + let rc = fs.mkdirp(folderName, 0o700); + if (rc) { + //TODO error code + common.printFormattedError("ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `Failed to make env var folder for component=${component}`); + } + let componentFileArray = []; + componentFileArray.push('#!/bin/sh'); + + envKeys.forEach((key:string)=> { + componentFileArray.push(`${key}=${envs[key]}`); + if (key.startsWith(`ZWE_components_${componentAlpha}_`)) { + let keyPrefixLength=`ZWE_components_${componentAlpha}_`.length; + componentFileArray.push(`ZWE_configs_${key.substring(keyPrefixLength)}=${envs[key]}`); + } + }); + + const componentFileContent = componentFileArray.join('\n'); + rc = xplatform.storeFileUTF8(`${folderName}/.instance-${haInstance}.env`, xplatform.AUTO_DETECT, componentFileContent); + if (rc) { + common.printFormattedError("ZWELS", "bin/libs/config.sh,generate_instance_env_from_yaml_config", `ZWEL0140E: Failed to translate Zowe configuration (${cliParameterConfig}).`); + std.exit(140); + return; + } + }); +} + + +// check and sanitize ZWE_CLI_PARAMETER_HA_INSTANCE +export function sanitizeHaInstanceId() { + // ignore default value passed from ZWESLSTC + let zweCliParameterHaInstance = std.getenv('ZWE_CLI_PARAMETER_HA_INSTANCE'); + if (zweCliParameterHaInstance == "{{ha_instance_id}}" || zweCliParameterHaInstance == "__ha_instance_id__") { + std.unsetenv('ZWE_CLI_PARAMETER_HA_INSTANCE'); + zweCliParameterHaInstance=undefined; + } + if (!zweCliParameterHaInstance) { + zweCliParameterHaInstance=sys.getSysname(); + } + // sanitize instance id + if (zweCliParameterHaInstance){ + zweCliParameterHaInstance=stringlib.sanitizeAlphanum(zweCliParameterHaInstance.toLowerCase()); + std.setenv('ZWE_CLI_PARAMETER_HA_INSTANCE', zweCliParameterHaInstance ); + } +} + +export function applyEnviron(environ: any): void { + let keys = Object.keys(environ); + keys.forEach(function(key:string) { + common.printMessage(`applyEnviron setting ${key}=${environ[key]}`); + std.setenv(key, environ[key]); + }); +} + +////////////////////////////////////////////////////////////// +// Load environment variables used by components +// +// NOTE: all environment variables used/defined by Zowe should be ensured in this function. +// "zwe internal start prepare" is the only special case where we may need to define some variables before calling +// this function. The reason is to properly prepare the directories, logging, etc. +export function loadEnvironmentVariables(componentId?: string) { + + // check and sanitize zweCliParameterHaInstance + sanitizeHaInstanceId(); + std.setenv('ZWE_zowe_workspaceDirectory',workspaceDirectory); + + if (!std.getenv('ZWE_VERSION')) { +// display starting information + let manifestReturn = xplatform.loadFileUTF8(`${runtimeDirectory}/manifest.json`,xplatform.AUTO_DETECT); + + const runtimeManifest = manifestReturn ? JSON.parse(manifestReturn) : undefined; + const zoweVersion = runtimeManifest ? runtimeManifest.version : undefined; + std.setenv('ZWE_VERSION', zoweVersion); + } + + // we must have $workspaceDirectory at this point + if (fs.fileExists(`${workspaceDirectory}/.init-for-container`)) { + std.setenv('ZWE_RUN_IN_CONTAINER','true'); + } + + // these are already set in prepare stage, re-ensure for start + let zwePrivateWorkspaceEnvDir=`${workspaceDirectory}/.env`; + std.setenv('ZWE_PRIVATE_WORKSPACE_ENV_DIR', zwePrivateWorkspaceEnvDir); + std.setenv('ZWE_STATIC_DEFINITIONS_DIR', `${workspaceDirectory}/api-mediation/api-defs`); + std.setenv('ZWE_GATEWAY_SHARED_LIBS', `${workspaceDirectory}/gateway/sharedLibs/`); + std.setenv('ZWE_DISCOVERY_SHARED_LIBS', `${workspaceDirectory}/discovery/sharedLibs/`); + + // now we can load all variables + let zweCliParameterHaInstance=std.getenv('ZWE_CLI_PARAMETER_HA_INSTANCE'); + + if (componentId && fs.fileExists(`${workspaceDirectory}/.env/${componentId}/.instance-${zweCliParameterHaInstance}.env`)) { + varlib.sourceEnv(`${zwePrivateWorkspaceEnvDir}/${componentId}/.instance-${zweCliParameterHaInstance}.env`); + } else if (fs.fileExists(`${zwePrivateWorkspaceEnvDir}/.instance-${zweCliParameterHaInstance}.env`)) { + varlib.sourceEnv(`${zwePrivateWorkspaceEnvDir}/.instance-${zweCliParameterHaInstance}.env`); + } else { + common.printErrorAndExit( "Error ZWEL0112E: Zowe runtime environment must be prepared first with \"zwe internal start prepare\" command.", undefined, 112); + } + + // ZWE_DISCOVERY_SERVICES_LIST should have been prepared in zowe-install-packaging-tools and had been sourced. + + // overwrite ZWE_PRIVATE_LOG_LEVEL_ZWELS with zowe.launchScript.logLevel config in YAML + let logLevel = configmgr.ZOWE_CONFIG.zowe.launchScript.logLevel; + if (logLevel) { + std.setenv('ZWE_PRIVATE_LOG_LEVEL_ZWELS', logLevel.toUpperCase()); + } + // generate other variables + std.setenv('ZWE_INSTALLED_COMPONENTS', component.findAllInstalledComponents()); + std.setenv('ZWE_ENABLED_COMPONENTS', component.findAllEnabledComponents()); + std.setenv('ZWE_LAUNCH_COMPONENTS', component.findAllLaunchComponents()); + + // ZWE_DISCOVERY_SERVICES_LIST should have been prepared in zowe-install-packaging-tools + + if (std.getenv('ZWE_RUN_IN_CONTAINER') == "true") { + container.prepareContainerRuntimeEnvironments(); + } + return std.getenviron(); +} diff --git a/bin/libs/configmgr.ts b/bin/libs/configmgr.ts new file mode 100644 index 0000000000..e9a63c4068 --- /dev/null +++ b/bin/libs/configmgr.ts @@ -0,0 +1,174 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; +import * as xplatform from 'xplatform'; +import { ConfigManager } from 'Configuration'; + +import * as objUtils from '../utils/ObjUtils'; + +declare namespace console { + function log(...args:string[]): void; +}; + +export const CONFIG_MGR = new ConfigManager(); +CONFIG_MGR.setTraceLevel(0); + +//these show the list of files used for zowe config prior to merging into a unified one. +// ZWE_CLI_PARAMETER_CONFIG gets updated to point to the unified one once written. +const parameterConfig = std.getenv('ZWE_CLI_PARAMETER_CONFIG'); +const configPath = (parameterConfig && !parameterConfig.startsWith('FILE(')) ? `FILE(${parameterConfig})` : parameterConfig; +let configLoaded = false; + +const COMMON_SCHEMA = `${std.getenv('ZWE_zowe_runtimeDirectory')}/schemas/server-common.json`; +const ZOWE_SCHEMA = `${std.getenv('ZWE_zowe_runtimeDirectory')}/schemas/zowe-yaml-schema.json`; +const ZOWE_SCHEMA_ID = 'https://zowe.org/schemas/v2/server-base'; +const ZOWE_SCHEMA_SET=`${ZOWE_SCHEMA}:${COMMON_SCHEMA}`; + +export const ZOWE_CONFIG=getZoweConfig(); + +function mkdirp(path:string, mode?: number): number { + const parts = path.split('/'); + let dir = '/'; + for (let i = 0; i < parts.length; i++) { + dir+=parts[i]+'/'; + let rc = os.mkdir(dir, mode ? mode : 0o777); + if (rc && (rc!=(0-std.Error.EEXIST))) { + return rc; + } + } + return 0; +} + +function writeMergedConfig(config: any): number { + const workspace = config.zowe.workspaceDirectory; + let zwePrivateWorkspaceEnvDir = std.getenv('ZWE_PRIVATE_WORKSPACE_ENV_DIR'); + if (!zwePrivateWorkspaceEnvDir) { + zwePrivateWorkspaceEnvDir=`${workspace}/.env`; + std.setenv('ZWE_PRIVATE_WORKSPACE_ENV_DIR', zwePrivateWorkspaceEnvDir); + } + const mkdirrc = mkdirp(zwePrivateWorkspaceEnvDir); + if (mkdirrc) { return mkdirrc; } + const destination = `${zwePrivateWorkspaceEnvDir}/.zowe-merged.yaml`; + //const yamlReturn = CONFIG_MGR.writeYAML('zowe-server-base', destination); + let [ yamlStatus, textOrNull ] = CONFIG_MGR.writeYAML('zowe-server-base'); + if (yamlStatus == 0){ + const rc = xplatform.storeFileUTF8(destination, xplatform.AUTO_DETECT, textOrNull); + if (!rc) { + std.setenv('ZWE_CLI_PARAMETER_CONFIG', destination); + } + return rc; + } + return yamlStatus; +} + +function showExceptions(e: any,depth: number): void { + let blanks = " "; + let subs = e.subExceptions; + console.log(blanks.substring(0,depth*2)+e.message); + if (subs){ + for (const sub of subs){ + showExceptions(sub,depth+1); + } + } +} + +export function getZoweConfig(): any { + if (configLoaded) { + return CONFIG_MGR.getConfigData('zowe-server-base'); + } + + if (configPath) { + let status; + + if ((status = CONFIG_MGR.addConfig('zowe-server-base'))) { + console.log(`Error: Could not add config for ${configPath}, status=${status}`); + std.exit(1); + } + + if ((status = CONFIG_MGR.loadSchemas('zowe-server-base', ZOWE_SCHEMA_SET))) { + console.log(`Error: Could not load schemas ${ZOWE_SCHEMA_SET} for configs ${configPath}, status=${status}`); + std.exit(1); + } + + if ((status = CONFIG_MGR.setConfigPath('zowe-server-base', configPath))) { + console.log(`Error: Could not set config path for ${configPath}, status=${status}`); + std.exit(1); + } + + if ((status = CONFIG_MGR.loadConfiguration('zowe-server-base'))) { + console.log(`Error: Could not load config for ${configPath}, status=${status}`); + std.exit(1); + } + + let validation = CONFIG_MGR.validate('zowe-server-base'); + if (validation.ok){ + if (validation.exceptionTree){ + console.log(`Error: Validation of ${configPath} against schema ${ZOWE_SCHEMA_ID} found invalid JSON Schema data`); + showExceptions(validation.exceptionTree, 0); + std.exit(1); + } else { + configLoaded = true; + + const config = CONFIG_MGR.getConfigData('zowe-server-base'); + const writeResult = writeMergedConfig(config); + return config; + } + } else { + console.log(`Error: Error occurred on validation of ${configPath} against schema ${ZOWE_SCHEMA_ID}<`); + std.exit(1); + } + } else { + console.log(`Error: Server config path not given`); + std.exit(1); + } +} + +const SPECIAL_ENV_MAPS = { + ZWE_node_home: 'NODE_HOME', + ZWE_java_home:'JAVA_HOME', + ZWE_zOSMF_host: 'ZOSMF_HOST', + ZWE_zOSMF_port: 'ZOSMF_PORT', + ZWE_zOSMF_applId: 'ZOSMF_APPLID' +}; + +// TODO haInstance values should be overriding the base values +const keyNameRegex = /[^a-zA-Z0-9]/g; +export function getZoweConfigEnv(haInstance?: string): any { + let config = getZoweConfig(); + let flattener = new objUtils.Flattener(); + flattener.setSeparator('_'); + flattener.setPrefix('ZWE_'); + let envs = flattener.flatten(config); + + //env var key name sanitization + let keys = Object.keys(envs); + keys.forEach((key:string)=> { + const newKey = key.replace(keyNameRegex, '_'); + if (key != newKey) { + envs[newKey]=envs[key]; + delete envs[key]; + } + }); + + let specialKeys = Object.keys(SPECIAL_ENV_MAPS); + specialKeys.forEach((key:string)=> { + envs[SPECIAL_ENV_MAPS[key]] = envs[key]; + }); + + + + //special things to keep as-is + envs['ZWE_DISCOVERY_SERVICES_LIST'] = std.getenv('ZWE_DISCOVERY_SERVICES_LIST'); + + return envs; +} diff --git a/bin/libs/container.ts b/bin/libs/container.ts new file mode 100644 index 0000000000..a6489e87a9 --- /dev/null +++ b/bin/libs/container.ts @@ -0,0 +1,120 @@ +/* +// This program and the accompanying materials are made available +// under the terms of the Eclipse Public License v2.0 which +// accompanies this distribution, and is available at +// https://www.eclipse.org/legal/epl-v20.html +// +// SPDX-License-Identifier: EPL-2.0 +// +// Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; +import * as fs from './fs'; +import * as shell from './shell'; +import * as sys from './sys'; +import * as component from './component'; +import * as config from './config'; +import * as network from './network'; + +std.setenv('ZWE_PRIVATE_CONTAINER_HOME_DIRECTORY', '/home/zowe'); +std.setenv('ZWE_PRIVATE_CONTAINER_RUNTIME_DIRECTORY', '/home/zowe/runtime'); +std.setenv('ZWE_PRIVATE_CONTAINER_COMPONENT_RUNTIME_DIRECTORY', '/component'); +std.setenv('ZWE_PRIVATE_CONTAINER_WORKSPACE_DIRECTORY', '/home/zowe/instance/workspace'); +std.setenv('ZWE_PRIVATE_CONTAINER_LOG_DIRECTORY', '/home/zowe/instance/logs'); +std.setenv('ZWE_PRIVATE_CONTAINER_KEYSTORE_DIRECTORY', '/home/zowe/keystore'); + +// prepare all environment variables used in containerization +// these variables shouldn't be modified +export function prepareContainerRuntimeEnvironments():void { + // to fix issues: + // - https://github.com/zowe/api-layer/issues/1768 + // - https://github.com/spring-cloud/spring-cloud-netflix/issues/3941 + // if you run Gateway on mainframe or locally, it will register with eureka health status - that is UP or DOWN, but + // in kubernetes, Gateway creates 2 additional health indicators and they are causing this OUT_OF_SERVICE, but this + // status is coming from spring, not from eureka + std.setenv('MANAGEMENT_ENDPOINT_HEALTH_PROBES_ENABLED', "false"); + + let podNamespace:string|null = null; + if (!"${ZWE_POD_NAMESPACE}" && fs.fileExists('/var/run/secrets/kubernetes.io/serviceaccount/namespace')) { + // try to detect ZWE_POD_NAMESPACE, this requires automountServiceAccountToken to be true + podNamespace=std.loadFile('/var/run/secrets/kubernetes.io/serviceaccount/namespace'); + } + if (!podNamespace) { + // fall back to default value + podNamespace='zowe'; + } + std.setenv('ZWE_POD_NAMESPACE',podNamespace); + + let podClustername = std.getenv('ZWE_POD_CLUSTERNAME'); + if (!podClustername) { + // fall back to default value + podClustername = 'cluster.local' + std.setenv('ZWE_POD_CLUSTERNAME',podClustername); + } + + // read ZWE_PRIVATE_CONTAINER_COMPONENT_ID from component manifest + // /component is hardcoded path we asked for in conformance + let zwePrivateContainerComponentId = std.getenv('ZWE_PRIVATE_CONTAINER_COMPONENT_ID'); + if (!zwePrivateContainerComponentId) { + zwePrivateContainerComponentId = component.getManifest('/component').name; + if (zwePrivateContainerComponentId){ + std.setenv('ZWE_PRIVATE_CONTAINER_COMPONENT_ID', zwePrivateContainerComponentId); + } else { + std.printf("*** severe *** - could not get zwePrivateContainerComponentId\n"); + return; + } + } + + // in kubernetes, replace ZWE_haInstance_hostname with pod dns name + const hostName=sys.getSysname() + if (!hostName){ + std.printf("*** severe *** could not get hostName\n"); + return; + } + const hostIp=network.getIpAddress(hostName); + std.setenv('ZWE_haInstance_hostname', `$(echo "${hostIp}" | sed -e 's#\.#-#g').${podNamespace}.pod.${podClustername}`); + + // kubernetes gateway service internal dns name + std.setenv('GATEWAY_HOST', `gateway-service.${podNamespace}.svc.${podClustername}`); + + // overwrite ZWE_DISCOVERY_SERVICES_LIST from ZWE_DISCOVERY_SERVICES_REPLICAS + let discoveryServiceReplicas = std.getenv('ZWE_DISCOVERY_SERVICES_REPLICAS'); + if (!discoveryServiceReplicas){ + std.printf("*** SEVERE *** could not get discoveryServiceReplicas\n"); + return; + } + let echoVal=shell.execOutSync('sh', `echo`, discoveryServiceReplicas, `|`, `tr`, `-cd`, `'[[:digit:]]'`, `|`, `tr`, `-d`, `'[[:space:]]'`); + let zweDiscoveryServiceReplicas=Number(echoVal.out); + if (isNaN(zweDiscoveryServiceReplicas)) { + zweDiscoveryServiceReplicas=1; + std.setenv('ZWE_DISCOVERY_SERVICES_REPLICAS', ""+zweDiscoveryServiceReplicas); + } + + let discoveryIndex=0; + let zweDiscoveryServiceList; + const zoweConfig=config.getZoweConfig(); + const discoveryPort=zoweConfig.components.discovery ? zoweConfig.components.discovery.port : undefined; + while (discoveryIndex < zweDiscoveryServiceReplicas) { + if (zweDiscoveryServiceList) { + zweDiscoveryServiceList=`${zweDiscoveryServiceList},` + } + zweDiscoveryServiceList=`${zweDiscoveryServiceList}https://discovery-${discoveryIndex}.discovery-service.${podNamespace}.svc.${podClustername}:${discoveryPort}/eureka/`; + discoveryIndex++; + } + if (zweDiscoveryServiceList){ + std.setenv('ZWE_DISCOVERY_SERVICES_LIST',zweDiscoveryServiceList); + } else { + std.printf("*** SEVERE *** could not get discoveryServiceList"); + } + // overwrite component list variables + std.setenv('ZWE_INSTALLED_COMPONENTS', zwePrivateContainerComponentId); + std.setenv('ZWE_ENABLED_COMPONENTS', zwePrivateContainerComponentId); + std.setenv('ZWE_LAUNCH_COMPONENTS', zwePrivateContainerComponentId); + + // FIXME: below variables are different from HA configuration, we should consolidate and make them consistent + // in HA setup, this is used to point where is gateway accessible from internal + // export EUREKA_INSTANCE_HOMEPAGEURL=https://${GATEWAY_HOST}:${GATEWAY_PORT}/ + std.unsetenv('EUREKA_INSTANCE_HOMEPAGEURL'); +} diff --git a/bin/libs/fs.ts b/bin/libs/fs.ts new file mode 100644 index 0000000000..dc7493f3e2 --- /dev/null +++ b/bin/libs/fs.ts @@ -0,0 +1,297 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; +import * as zos from 'zos'; + +import * as common from './common'; +import * as shell from './shell'; +import { PathAPI as pathoid } from './pathoid'; + +/* + Below are not found in os module, but can be used against os.stat mode return values. + They were pulled from a system header which I hope is portable on all platforms +*/ +//const S_ISUID = 0x0800; +//const S_ISGID = 0x0400; +//const S_ISVTX = 0x0200; +/* User permissions, rwx */ +//const S_IRUSR = 0x0100; +const S_IWUSR = 0x0080; +//const S_IXUSR = 0x0040; +//const S_IRWXU = 0x01C0; +/* group permissions, rwx */ +//const S_IRGRP = 0x0020; +//const S_IWGRP = 0x0010; +//const S_IXGRP = 0x0008; +//const S_IRWXG = 0x0038; +/* other permissions, rwx */ +//const S_IROTH = 0x0004; +//const S_IWOTH = 0x0002; +//const S_IXOTH = 0x0001; +//const S_IRWXO = 0x0007; + + +export function resolvePath(...parts:string[]): string { + let separator=os.platform == 'win32' ? '\\' : '/'; +// let badSeparator = os.platform == 'win32' ? '/' : '\\'; + let path=''; + parts.forEach((part:string)=> { +// part=part.replaceAll(badSeparator, separator); + if (part.startsWith('"') && part.endsWith('"')) { + part=part.substring(1,part.length-1); + } else if (part.startsWith("'") && part.endsWith("'")) { + part=part.substring(1,part.length-1); + } + + if (part.startsWith(separator)) { + part=part.substring(1); + } + if (part.endsWith(separator)) { + part = part.substring(0,part.length-1); + } + path+=separator+part; + }); + return path; +} + +export function mkdirp(path:string, mode?: number): number { + const parts = path.split('/'); + let dir = '/'; + for (let i = 0; i < parts.length; i++) { + dir+=parts[i]+'/'; + let rc = os.mkdir(dir, mode ? mode : 0o777); + if (rc && (rc!=(0-std.Error.EEXIST))) { + return rc; + } + } + return 0; +} + +export function cp(from: string, to: string): void { + shell.execSync('cp', `-r`, from, to); +} + + +export function cpr(from: string, to: string): void { + shell.execSync('cp', `-r`, from, to); +} + +export function appendToFile(path:string, content:string):void { + //TODO + throw 'Implement me!'; +} + +export function createFile(path: string, mode: number, content?: string): boolean { + let errObj = {errno:undefined}; + const file:std.File|null = std.open(path, 'w', errObj); + if (errObj.errno) { + common.printError(`File create failed for ${path}, error=${errObj.errno}`); + return false; + } else if (file == null){ + common.printError(`File create failed for ${path} unexpectedly`); + return false; + } +/* + if (content) { + let offset = 0; + + const buff = true ? stringlib.stringTo1047Buffer(content) : stringlib.stringToBuffer(content); + + os.write(fd, buff.buffer, offset, buff.byteLength); + } +*/ + if (content) { + file.puts(content); + } + file.close(); + return true; +} + +export function createFileFromBuffer(path: string, mode: number, buff?: Uint8Array) { + let errObj = {errno:undefined}; + const file = std.open(path, 'w', errObj); + if (errObj.errno) { + common.printError(`File create failed for ${path}, error=${errObj.errno}`); + return false; + } else if (file == null){ + common.printError(`File create failed for ${path} unexpectedly`); + return false; + } +/* + if (content) { + let offset = 0; + + const buff = true ? stringlib.stringTo1047Buffer(content) : stringlib.stringToBuffer(content); + + os.write(fd, buff.buffer, offset, buff.byteLength); + } +*/ + if (buff) { + file.write(buff.buffer, 0, buff.byteLength); + } + file.close(); + return true; +} + +export function getFilesInDirectory(path: string): string[]|undefined { + let returnArray = os.readdir(path); + let files:string[] = []; + if (!returnArray[1]) { //no error + returnArray[0].forEach((file:string)=> { + if (fileExists(pathoid.join(path, file))) { + files.push(file); + } + }); + return files; + } else { + common.printError(`getFilesInDirectory path=${path}, err=`+returnArray[1]); + return undefined; + } +} + +export function getSubdirectories(path: string): string[]|undefined { + let returnArray = os.readdir(path); + let subdirs:string[] = []; + if (!returnArray[1]) { //no error + returnArray[0].forEach((dir:string)=> { + if (directoryExists(pathoid.join(path,dir))) { + subdirs.push(dir); + } + }); + return subdirs; + } else { + common.printError(`getSubdirectories path=${path}, err=`+returnArray[1]); + return undefined; + } +} + +export function directoryExists(path: string, silenceNotFound?: boolean): boolean { + let returnArray = os.stat(path); + if (!returnArray[1]) { //no error + return ((returnArray[0].mode & os.S_IFDIR) == os.S_IFDIR) + } else { + if ((returnArray[1] != std.Error.ENOENT) && !silenceNotFound) { + common.printError(`directoryExists path=${path}, err=`+returnArray[1]); + } + return false; + } +} + +export function fileExists(path: string, silenceNotFound?: boolean): boolean { + let returnArray = os.stat(path); + if (!returnArray[1]) { //no error + return ((returnArray[0].mode & os.S_IFREG) == os.S_IFREG) + } else { + if ((returnArray[1] != std.Error.ENOENT) && !silenceNotFound) { + common.printError(`fileExists path=${path}, err=${returnArray[1]}`); + } + return false; + } +} + +export function pathExists(path: string): boolean { + let returnArray = os.stat(path); + if (!returnArray[1]) { //no error + return true; + } else { + return returnArray[1] != std.Error.ENOENT; + } + +} + +export function pathHasPermissions(path: string, mode: number): boolean { + let returnArray = os.stat(path); + if (!returnArray[1]) { //no error + return (returnArray[0].mode & mode) === mode; + } else { + return false; + } +} + +export function convertToAbsolutePath(file: string): string|undefined { + const result = os.realpath(file); + if (!result[1]) { + return result[0]; + } else { + common.printError(`Could not convert ${file} to absolute path, err=${result[1]}`); + } + return undefined; +} + +export function getTmpDir(): string { + let tmp = std.getenv('TMPDIR'); + if (!tmp) { + tmp = std.getenv('TMP'); + } + if (!tmp) { + tmp = '/tmp'; + } + return tmp; +} + +/* + NOTE: Contrary to function name, this does not create a temp file. It just checks if it exists (safe to create) +*/ +export function createTmpFile(prefix: string = 'zwe', tmpdir?: string): string|undefined { + if (!tmpdir) { + tmpdir = getTmpDir(); + } + common.printTrace(` > create_tmp_file on ${tmpdir}`); + while (true) { + let file = `${tmpdir}/${prefix}-${std.getenv('random')}`; + common.printTrace(` - test ${file}`); + if (!pathExists(file)) { + common.printTrace(` - good`); + return file; + } + } +} + +export function isFileAccessible(file: string): boolean { + return fileExists(file); +} + +export function isDirectoryAccessible(directory: string): boolean { + return directoryExists(directory); +} + +export function countMissingDirectories(directories: string[]): number { + let missing = 0; + for (let i = 0; i < directories.length; i++) { + if (!directoryExists(directories[i])) { + missing++; + } + } + return missing; +} + +/* + NOTE: Original shell function did a file existence check, not a dir existence check. Fixed a bug? Causing a bug? +*/ +export function areDirectoriesAccessible(directories: string): number { + return countMissingDirectories(directories.split(',')); +} + +export function isDirectoryWritable(directory: string): boolean { + return pathHasPermissions(directory, os.S_IFDIR | S_IWUSR); +} + +export function isFileWritable(file: string): boolean { + return pathHasPermissions(file, os.S_IFREG | S_IWUSR); +} + +export function areDirectoriesSame(dir1: string, dir2: string): boolean { + let abs1 = convertToAbsolutePath(dir1); + let abs2 = convertToAbsolutePath(dir2); + return (abs1 === abs2) && abs1 !== undefined; +} diff --git a/bin/libs/index.ts b/bin/libs/index.ts new file mode 100644 index 0000000000..946af9236a --- /dev/null +++ b/bin/libs/index.ts @@ -0,0 +1,19 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; + +if (!std.getenv("ZWE_zowe_runtimeDirectory")) { + std.err.printf("Error ZWEL0101E: ZWE_zowe_runtimeDirectory is not defined.\n"); + std.exit(101); +} + diff --git a/bin/libs/java.ts b/bin/libs/java.ts new file mode 100644 index 0000000000..20d6f8b30b --- /dev/null +++ b/bin/libs/java.ts @@ -0,0 +1,135 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; +import * as fs from './fs'; +import * as common from './common'; +import * as shell from './shell'; +import * as config from './config'; + +const JAVA_MIN_VERSION=8; + +export function ensureJavaIsOnPath(): void { + let path=std.getenv('PATH'); + let javaHome=std.getenv('JAVA_HOME'); + if (path && !path.includes(`:${javaHome}/bin:`)) { + std.setenv('PATH', `${javaHome}/bin:${path}`); + } +} + +export function shellReadYamlJavaHome(configList?: string, skipValidate?: boolean): string { + const zoweConfig = config.getZoweConfig(); + if (zoweConfig && zoweConfig.java && zoweConfig.java.home) { + if (!skipValidate) { + if (validateJavaHome(zoweConfig.java.home)) { + return ''; + } + } + return zoweConfig.java.home; + } + return ''; +} + +export function detectJavaHome(): string|undefined { + let javaBinHome = shell.which(`java`); + if (javaBinHome) { + let returnVal = os.realpath(`${javaBinHome}/../..`); + if (!returnVal[1]) { + return returnVal[0]; + } + } + + if (!javaBinHome && fs.fileExists('/usr/lpp/java/J8.0_64/bin/java')) { + return '/usr/lpp/java/J8.0_64'; + } + return undefined; +} + +export function requireJava() { + if (std.getenv('ZWE_CLI_PARAMETER_CONFIG')) { + const customJavaHome = shellReadYamlJavaHome(); + if (customJavaHome) { + std.setenv('JAVA_HOME', customJavaHome); + } + } + if (!std.getenv('JAVA_HOME')) { + let detectedHome = detectJavaHome(); + if (detectedHome){ + std.setenv('JAVA_HOME', detectedHome); + } + } + if (!std.getenv('JAVA_HOME')) { + common.printErrorAndExit("Error ZWEL0122E: Cannot find java. Please define JAVA_HOME environment variable.", undefined, 122); + } + + ensureJavaIsOnPath(); +} + +export function validateJavaHome(javaHome:string|undefined=std.getenv("JAVA_HOME")): boolean { + if (!javaHome) { + common.printError("Cannot find java. Please define JAVA_HOME environment variable."); + return false; + } + if (!fs.fileExists(fs.resolvePath(javaHome,`/bin/java`))) { + common.printError(`JAVA_HOME: ${javaHome}/bin does not point to a valid install of Java.`); + return false; + } + + let execReturn = shell.execErrSync(fs.resolvePath(javaHome,`/bin/java`), `-version`); + const version = execReturn.err; + if (execReturn.rc != 0) { + common.printError(`Java version check failed with return code: ${execReturn.rc}: ${version}`); + return false; + } + + try { + let index = 0; + let javaVersionShort; + let versionLines = (version as string).split('\n'); // valid because of above rc check + for (let i = 0; i < versionLines.length; i++) { + if ((index = versionLines[i].indexOf('java version')) != -1) { + //format of: java version "1.8.0_321" + javaVersionShort=versionLines[i].substring(index+('java version'.length)+2, versionLines[i].length-1); + break; + } else if ((index = versionLines[i].indexOf('openjdk version')) != -1) { + javaVersionShort=versionLines[i].substring(index+('openjdk version'.length)+2, versionLines[i].length-1); + break; + } + } + if (!javaVersionShort){ + common.printError("could not find java version"); + return false; + } + let versionParts = javaVersionShort.split('.'); + const javaMajorVersion=versionParts[0]; + const javaMinorVersion=versionParts[1]; + + let tooLow=false; + if (javaMajorVersion != '1') { + tooLow=true; + } + if (javaMajorVersion != '1' && Number(javaMinorVersion) < JAVA_MIN_VERSION) { + tooLow=true; + } + + if (tooLow) { + common.printError(`Java ${javaVersionShort} is less than the minimum level required of Java ${JAVA_MIN_VERSION} (1.${JAVA_MIN_VERSION}.0).`); + return false; + } + + common.printDebug(`Java ${javaVersionShort} is supported.`); + common.printDebug(`Java check is successful.`); + return true; + } catch (e) { + return false; + } +} diff --git a/bin/libs/json.ts b/bin/libs/json.ts new file mode 100644 index 0000000000..5f3fb32f05 --- /dev/null +++ b/bin/libs/json.ts @@ -0,0 +1,207 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; +import * as zos from 'zos'; + + +// Read JSON configuration from shell script +// +// Note: this is not a reliable way to read JSON file. The JSON file must be +// properly formatted, each key/value pair takes one line. +// +// FIXME: we should have a language neutral JSON reading tool, not using shell script. +// +// @param string JSON file name +// @param string parent key to read after +// @param string which key to read +// @param string if this variable is required. If this is true and we cannot +// find the value of the key, an error will be displayed. +export function shellReadJsonConfig(jsonFile: string, parentKey: string, key: string, required: boolean): any { + + json_file="${1}" + parent_key="${2}" + key="${3}" + required="${4}" + + val=$(cat "${json_file}" | awk "/\"${parent_key}\":/{x=NR+200}(NR<=x){print}" | grep "\"${key}\":" | head -n 1 | awk -F: '{print $2;}' | tr -d '[[:space:]]' | sed -e 's/,$//' | sed -e 's/^"//' -e 's/"$//') + if [ -z "${val}" ]; then + if [ "${required}" = "true" ]; then + print_error_and_exit "Error ZWEL0131E: Cannot find key ${parent_key}.${key} defined in file ${json_file}." "" 131 + fi + else + printf "${val}" + fi +} + + +// Read YAML configuration from shell script +// +// Note: this is not a reliable way to read YAML file, but we need this to find +// out ROOT_DIR to execute further functions. +// +// FIXME: we should have a language neutral YAML reading tool, not using shell script. +// +// @param string YAML file name +// @param string parent key to read after +// @param string which key to read +// @param string if this variable is required. If this is true and we cannot +// find the value of the key, an error will be displayed. +shell_read_yaml_config() { + yaml_file="${1}" + parent_key="${2}" + key="${3}" + required="${4}" + + val=$(cat "${yaml_file}" | awk "/^ *${parent_key}:/{x=NR+2000;next}(NR<=x){print}" | grep -e "^ \+${key}:" | head -n 1 | awk -F: '{print $2;}' | tr -d '[[:space:]]' | sed -e 's/^"//' -e 's/"$//') + if [ -z "${val}" ]; then + if [ "${required}" = "true" ]; then + print_error_and_exit "Error ZWEL0131E: Cannot find key ${parent_key}.${key} defined in file ${yaml_file}." "" 131 + fi + else + printf "${val}" + fi +} + +read_yaml() { + file="${1}" + key="${2}" + ignore_null="${3:-true}" + + utils_dir="${ZWE_zowe_runtimeDirectory}/bin/utils" + fconv="${utils_dir}/fconv/src/index.js" + jq="${utils_dir}/njq/src/index.js" + + print_trace "- read_yaml load content from ${file}" + ZWE_PRIVATE_YAML_CACHE=$(node "${fconv}" --input-format=yaml "${file}" 2>&1) + code=$? + print_trace " * Exit code: ${code}" + if [ ${code} -ne 0 ]; then + print_error " * Output:" + print_error "$(padding_left "${ZWE_PRIVATE_YAML_CACHE}" " ")" + return ${code} + fi + + print_trace "- read_yaml ${key} from yaml content" + result=$(echo "${ZWE_PRIVATE_YAML_CACHE}" | node "${jq}" -r "${key}" 2>&1) + code=$? + print_trace " * Exit code: ${code}" + print_trace " * Output:" + if [ -n "${result}" ]; then + print_trace "$(padding_left "${result}" " ")" + fi + + if [ ${code} -eq 0 ]; then + if [ "${ignore_null}" = "true" ]; then + if [ "${result}" = "null" -o "${result}" = "undefined" ]; then + result= + fi + fi + printf "${result}" + fi + + return ${code} +} + +read_json() { + file="${1}" + key="${2}" + ignore_null="${3:-true}" + + utils_dir="${ZWE_zowe_runtimeDirectory}/bin/utils" + jq="${utils_dir}/njq/src/index.js" + + print_trace "- read_json ${key} from ${file}" + result=$(cat "${file}" | node "${jq}" -r "${key}" 2>&1) + code=$? + print_trace " * Exit code: ${code}" + print_trace " * Output:" + if [ -n "${result}" ]; then + print_trace "$(padding_left "${result}" " ")" + fi + + if [ ${code} -eq 0 ]; then + if [ "${ignore_null}" = "true" -a "${result}" = "null" ]; then + result= + fi + printf "${result}" + fi + + return ${code} +} + +update_yaml() { + file="${1}" + key="${2}" + val="${3}" + expected_sample="${4}" + + utils_dir="${ZWE_zowe_runtimeDirectory}/bin/utils" + config_converter="${utils_dir}/config-converter/src/cli.js" + + print_message "- update \"${key}\" with value: ${val}" + result=$(node "${config_converter}" yaml update "${file}" "${key}" "${val}") + code=$? + if [ ${code} -eq 0 ]; then + print_trace " * Exit code: ${code}" + print_trace " * Output:" + if [ -n "${result}" ]; then + print_trace "$(padding_left "${result}" " ")" + fi + else + print_error " * Exit code: ${code}" + print_error " * Output:" + if [ -n "${result}" ]; then + print_error "$(padding_left "${result}" " ")" + fi + print_error_and_exit "Error ZWEL0138E: Failed to update key ${key} of file ${file}." "" 138 + fi + + ensure_file_encoding "${file}" "${expected_sample}" +} + +update_zowe_yaml() { + update_yaml "${1}" "${2}" "${3}" "zowe:" +} + +delete_yaml() { + file="${1}" + key="${2}" + expected_sample="${3}" + + utils_dir="${ZWE_zowe_runtimeDirectory}/bin/utils" + config_converter="${utils_dir}/config-converter/src/cli.js" + + print_message "- delete \"${key}\"" + result=$(node "${config_converter}" yaml delete "${file}" "${key}") + code=$? + if [ ${code} -eq 0 ]; then + print_trace " * Exit code: ${code}" + print_trace " * Output:" + if [ -n "${result}" ]; then + print_trace "$(padding_left "${result}" " ")" + fi + else + print_error " * Exit code: ${code}" + print_error " * Output:" + if [ -n "${result}" ]; then + print_error "$(padding_left "${result}" " ")" + fi + print_error_and_exit "Error ZWEL0138E: Failed to delete key ${key} of file ${file}." "" 138 + fi + + ensure_file_encoding "${file}" "${expected_sample}" +} + +delete_zowe_yaml() { + delete_yaml "${1}" "${2}" "zowe:" +} diff --git a/bin/libs/logging.ts b/bin/libs/logging.ts new file mode 100644 index 0000000000..6a1f2d3309 --- /dev/null +++ b/bin/libs/logging.ts @@ -0,0 +1,35 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; + +import * as common from './common'; +import * as fs from './fs'; +import * as stringlib from './string'; + + +std.unsetenv('ZWE_PRIVATE_LOG_FILE'); + + +export function prepareLogFile(logDir: string, logFilePrefix: string): void { + logDir=stringlib.removeTrailingSlash(fs.convertToAbsolutePath(logDir) as string); + + const logFile=`${logDir}/${logFilePrefix}-${common.date('+%Y%m%dT%H%M%S')}.log`; + if (!fs.fileExists(logFile)) { + fs.mkdirp(logDir, 0o770); + if (!fs.isDirectoryWritable(logDir)) { + common.printErrorAndExit("Error ZWEL0110E: Doesn't have write permission on ${1} directory.", undefined, 110); + } + let success = fs.createFile(logFile, 0o666); + common.printDebug(`Log file created: ${logFile}`, ["console"]); + } +} diff --git a/bin/libs/network.ts b/bin/libs/network.ts new file mode 100644 index 0000000000..1f8763c91d --- /dev/null +++ b/bin/libs/network.ts @@ -0,0 +1,170 @@ +/* +// This program and the accompanying materials are made available +// under the terms of the Eclipse Public License v2.0 which +// accompanies this distribution, and is available at +// https://www.eclipse.org/legal/epl-v20.html +// +// SPDX-License-Identifier: EPL-2.0 +// +// Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; + +import * as common from './common'; +import * as shell from './shell'; +import * as fs from './fs'; +import * as stringlib from './string'; +//if on zos +import * as zosfs from './zos-fs'; + +// get ping command, could be empty +export function getPing(): string|undefined { + let ping = shell.which('ping'); + + // z/OS + if (!ping) { + ping = shell.which('oping'); + } + return ping; +} + +// get netstat command, could be empty +export function getNetstat(): string|undefined { + let netstat = shell.which('netstat'); + + // z/OS + if (!netstat) { + netstat = shell.which('onetstat'); + } + return netstat; +} + +// should not be bound to a port currently +export function isPortAvailable(port: number): boolean { + const netstat=getNetstat(); + + if (!netstat) { + common.printError("No netstat tool found.") + return false; + } + + // QUESTION: should we ignore netstat command stderr? + let retVal; + let lines; + switch (os.platform) { + case 'zos': + retVal=shell.execOutErrSync(netstat, `-c`, `SERVER`, `-P`, `${port}`); + if (retVal.rc != 0) { + common.printError(`Netstat test fail with exit code ${retVal.rc} (${retVal.err})`); + return false; + } + if ((retVal.out as string).includes('Listen')) { + common.printError(`Port ${port} is already in use by process (${retVal.out})`); + return false; + } + break; + case "darwin": + retVal=shell.execOutErrSync(netstat, `-an`, `-p`, `tcp`); + if (retVal.rc != 0) { + common.printError(`Netstat test fail with exit code ${retVal.rc} (${retVal.err})`); + return false + } + lines = (retVal.out as string).split('\n'); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (line.includes('LISTEN') && line.includes(''+port)) { + common.printError(`Port ${port} is already in use by process (${retVal.out})`); + return false; + } + } + break; + default: + // linux-ish + retVal=shell.execOutErrSync(netstat, `-nlt`); + if (retVal.rc != 0) { + common.printError(`Netstat test fail with exit code ${retVal.rc} (${retVal.err})`); + return false; + } + lines = (retVal.out as string).split('\n'); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (line.includes('LISTEN') && line.includes(''+port)) { + common.printError(`Port ${port} is already in use by process (${retVal.out})`); + return false; + } + } + } + return true; +} + +// get current IP address +const ipv4Regexp = /[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/; +export function getIpAddress(hostname: string): string|undefined { + let ip; + + // dig is preferred than ping + const digResult=shell.execOutErrSync('sh', `dig`, `-4`, `+short`, hostname, `||`, `dig`, `+short`, hostname); + if (digResult.out) { + const digLines = digResult.out.split('\n'); + for (let i = 0; i < digLines.length; i++) { + let matchResult = digLines[0].match(ipv4Regexp); // Type error was here + if (matchResult){ + ip = matchResult[0]; + break; + } + } + } + + // try ping + if (!ip) { + let ping=getPing(); + const timeout=2 + if (ping) { + // try to get IPv4 address only + // - Mac: ipv4-only not supported + let pingResult; + if (os.platform == 'zos') { + pingResult=shell.execOutErrSync('sh', ping, `-c`, `1`, `-A`, `ipv4`, `-t`, ""+timeout, hostname); + } else if (os.platform == 'darwin') { + pingResult=shell.execOutErrSync('sh', ping, `-c`, `1`, `-t`, ""+timeout, hostname); + } else { //linux + pingResult=shell.execOutErrSync('sh', ping, `-c`, `1`, `-4`, `-W`, ""+timeout, hostname); + } + if (pingResult.rc==0) { + let pingOut:string = pingResult.out as string; + let index = pingOut.indexOf('('); + if (index != -1) { + let index2 = pingOut.indexOf(')',index); + if (index2 != -1) { + ip=pingOut.substring(index+1,index2); + } + } + } + } + } + + // we don't have dig and ping, let's check /etc/hosts + if (!ip && fs.fileExists('/etc/hosts')) { + // const hosts = std.loadFile('/etc/hosts'); + let hosts = std.loadFile('/etc/hosts'); + if (hosts) { + if (os.platform=='zos') { + let encoding = zosfs.detectFileEncoding('/etc/hosts',hostname,1047); + if (encoding==1047) { + hosts = stringlib.ebcdicToAscii(hosts); + } + } + const lines = hosts.split('\n'); + for (let i = 0; i < lines.length; i++) { + let cols = lines[i].split(' '); + if (cols.includes(hostname)) { + return cols[0]; + } + } + } + } + + return ip; +} diff --git a/bin/libs/node.ts b/bin/libs/node.ts new file mode 100644 index 0000000000..cf66813817 --- /dev/null +++ b/bin/libs/node.ts @@ -0,0 +1,134 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; + +import * as fs from './fs'; +import * as common from './common'; +import * as shell from './shell'; +import * as config from './config'; +import { PathAPI as pathoid } from './pathoid'; + +const NODE_MIN_VERSION=12; + +// enforce encoding of stdio/stdout/stderr +// sometimes /dev/tty* ($SSH_TTY) are not configured properly, for example tagged as binary or wrong encoding +std.setenv('NODE_STDOUT_CCSID','1047'); +std.setenv('NODE_STDERR_CCSID','1047'); +std.setenv('NODE_STDIN_CCSID','1047'); + +// Workaround Fix for node 8.16.1 that requires compatibility mode for untagged files +std.setenv('__UNTAGGED_READ_MODE','V6'); + + +export function ensureNodeIsOnPath(): void { + let path=std.getenv('PATH'); + let nodeHome=std.getenv('NODE_HOME'); + if (path && !path.includes(`:${nodeHome}/bin:`)) { + std.setenv('PATH', `${nodeHome}/bin:${path}`); + } +} + +export function shellReadYamlNodeHome(configList?: string, skipValidate?: boolean): string { + const zoweConfig = config.getZoweConfig(); + if (zoweConfig && zoweConfig.node && zoweConfig.node.home) { + if (!skipValidate) { + if (validateNodeHome(zoweConfig.node.home)) { + return ''; + } + } + return zoweConfig.node.home; + } + return ''; +} + +export function detectNodeHome(): string|undefined { + let nodeBinHome = shell.which(`node`); + if (nodeBinHome) { + let returnVal = pathoid.normalize(`${nodeBinHome}/../..`); + return returnVal; + } + return undefined; +} + +export function requireNode() { + if (std.getenv('ZWE_CLI_PARAMETER_CONFIG')) { + const customNodeHome = shellReadYamlNodeHome(); + if (customNodeHome) { + std.setenv('NODE_HOME', customNodeHome); + } + } + if (!std.getenv('NODE_HOME')) { + let discoveredHome = detectNodeHome(); + if (discoveredHome){ + std.setenv('NODE_HOME', discoveredHome); + } + } + if (!std.getenv('NODE_HOME')) { + common.printErrorAndExit("Error ZWEL0121E: Cannot find node. Please define NODE_HOME environment variable.", undefined, 121); + } + + ensureNodeIsOnPath(); +} + +export function validateNodeHome(nodeHome:string|undefined=std.getenv("NODE_HOME")): boolean { + if (!nodeHome) { + common.printError("Cannot find node. Please define NODE_HOME environment variable."); + return false; + } + if (!fs.fileExists(fs.resolvePath(nodeHome,`/bin/node`))) { + common.printError(`NODE_HOME: ${nodeHome}/bin does not point to a valid install of Node.`); + return false; + } + + let shellReturn = shell.execOutSync(fs.resolvePath(nodeHome,`/bin/node`), `--version`); + const version = shellReturn.out; + if (shellReturn.rc != 0) { + common.printError(`Node version check failed with return code: ${shellReturn.rc}: ${version}`); + return false; + } + + try { + if ((version as string).startsWith('v')) { // valid because rc check + let parts = (version as string).split('.'); + const nodeMajorVersion = Number(parts[0].substring(1)); + //const nodeMinorVersion = Number(parts[1]); + //const nodePatchVersion = Number(parts[2]); + + if (version == 'v14.17.2') { + common.printError(`Node ${version} specifically is not compatible with Zowe. Please use a different version. See https://docs.zowe.org/stable/troubleshoot/app-framework/app-known-issues.html#desktop-apps-fail-to-load for more details.`); + return false; + } + if (nodeMajorVersion < NODE_MIN_VERSION) { + common.printError(`Node ${version} is less than the minimum level required of v${NODE_MIN_VERSION}.`); + return false; + } + common.printDebug(`Node ${version} is supported.`) + + shellReturn = shell.execOutSync(fs.resolvePath(nodeHome,`/bin/node`), `-e`, "const process = require('process'); console.log('ok'); process.exit(0);"); + const ok = shellReturn.out; + if (ok != 'ok' || shellReturn.rc != 0) { + common.printError(`${nodeHome}/bin/node is not functioning correctly (exit code ${shellReturn.rc}): '${ok}', len=${ok.length}`); + return false; + } + + common.printDebug(`Node check is successful.`); + + return true; + } else { + common.printError(`Cannot validate node version '${version}'. Unexpected format`); + return false; + } + } catch (e) { + return false; + } +} diff --git a/bin/libs/pathoid.ts b/bin/libs/pathoid.ts new file mode 100644 index 0000000000..38e0061625 --- /dev/null +++ b/bin/libs/pathoid.ts @@ -0,0 +1,177 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as os from 'os'; +import * as xplatform from 'xplatform'; + +declare namespace console { + function log(...args:string[]): void; +}; + +export class PathObject { + +} + +const isWindows:boolean = os.platform == "win32"; +const winDriveRoot = /^[A-Za-z]\:\\/; + +export class PathAPI { + static sep: string; + static delimiter: string; + + static pathRoot(path:string):string|null{ + if (isWindows){ + if (path.startsWith("\\\\")){ // UNC network path + return path.substring(0,2); + } else if (path.startsWith("\\")){ // root of current drive (ugh!) + return path.substring(0,1); + } else { + let match = winDriveRoot.exec(path); + if (match){ + return match[0]; + } else { + return null; + } + } + } else { + if ((path.length > 0) && (path.charAt(0) == "/")){ + return "/"; + } else { + return null; + } + } + } + + static resolveDots(path:string):string { + let root:string|null = PathAPI.pathRoot(path); + let isRooted:boolean = (root != null) && (root.length > 0); + let sep = isWindows ? "\\" : "/"; + let unrooted = (root && isRooted) ? path.substring(root.length) : path; + let tokens = unrooted.split(sep); + /* + console.log("root = "+root); + console.log("tokens.length="+tokens.length); + */ + let result = []; + let upCount = 0; + for (let i=tokens.length-1; i>=0; i--){ + let token = tokens[i]; + //console.log("token '"+token+"'"); + if (token == "."){ + // do nothing + } else if (token == ".."){ + upCount++; + } else { + if (upCount > 0){ + upCount--; + } else { + result.push(token); + } + } + } + /* + console.log("backwards "+result); + */ + if (isRooted){ + let rootedPath = result.reverse().join(sep); + return root+rootedPath; + } else { + for (let u=0; u): string{ + throw "Unimplemented"; + } + + static parse(path: string): PathObject{ + throw "Unimplemented"; + } + + +} diff --git a/bin/libs/shell.ts b/bin/libs/shell.ts new file mode 100644 index 0000000000..8561e791fe --- /dev/null +++ b/bin/libs/shell.ts @@ -0,0 +1,178 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; +import * as xplatform from 'xplatform'; + +import * as fs from './fs'; +import * as stringlib from './string'; + +declare namespace console { + function log(...args:string[]): void; +}; + + +const BUFFER_SIZE=4096; + +export type ExecReturn = { + rc: number, + out?: string, + err?: string +}; + +export type path = string; + +const hexDigits = [ "0", "1", "2", "3", + "4", "5", "6", "7", + "8", "9", "A", "B", + "C", "D", "E", "F" ]; + +class ExpandableBuffer { + size:number; + buffer:ArrayBuffer; + bytes:Uint8Array; + pos:number; + + constructor(size:number){ + this.size = size; + this.buffer = new ArrayBuffer(size); + this.bytes = new Uint8Array(this.buffer); + this.pos = 0; + } + + append(data:Uint8Array, offset:number, length:number):void { + if (this.pos+length > this.size){ + let newSize = this.size * 2; + let newBuffer = new ArrayBuffer(newSize); + let newBytes = new Uint8Array(newBuffer); + newBytes.set(this.bytes,0); + this.bytes = newBytes; + this.buffer = newBuffer; + this.size = newSize; + } + this.bytes.set(data.slice(offset,length), this.pos); + this.pos += length; + } + + dump(lim:number):string { + let chunks = []; + for (let i=0; i> 4)&0xf; + let l = b&0xf; + chunks.push(hexDigits[h]+hexDigits[l]+""); + if ((i % 4) == 3){ chunks.push(" "); } + if ((i % 16) == 15){ chunks.push("\n"); } + } + return chunks.join(""); + } + + getString():string{ + return xplatform.stringFromBytes(this.buffer, 0, this.pos, xplatform.AUTO_DETECT); + } +} + + +export function execSync(command: string, ...args: string[]): ExecReturn { + const rc = os.exec([command, ...args], + {block: true, usePath: true}); + return { + rc + }; +} + +function readStreamFully(fd:number):string{ + let readBuffer = new Uint8Array(BUFFER_SIZE); + let fileBuffer = new ExpandableBuffer(BUFFER_SIZE); + + let bytesRead = 0; + do { + bytesRead = os.read(fd, readBuffer.buffer, 0, BUFFER_SIZE); + fileBuffer.append(readBuffer,0,bytesRead); + } while (bytesRead == BUFFER_SIZE); + // let hex = fileBuffer.dump(fileBuffer.pos); + // console.log("out "+hex); + let result = fileBuffer.getString(); + if (result.endsWith('\n')) { + return result.substring(0,result.length-1); + } else { + return result; + } +} + +export function execOutSync(command: string, ...args: string[]): ExecReturn { + let pipeArray = os.pipe(); + if (!pipeArray){ + return { rc: -1 }; + } + const rc = os.exec([command, ...args], { block: true, usePath: true, stdout: pipeArray[1]}); + + let out = readStreamFully(pipeArray[0]); + os.close(pipeArray[0]); + os.close(pipeArray[1]); + + return { + rc, out + }; +} + +export function execErrSync(command: string, ...args: string[]): ExecReturn { + let pipeArray = os.pipe(); + if (!pipeArray){ + return { rc: -1 }; + } + const rc = os.exec([command, ...args], { block: true, usePath: true, stderr: pipeArray[1]}); + + let err = readStreamFully(pipeArray[0]); + os.close(pipeArray[0]); + os.close(pipeArray[1]); + + return { + rc, err + }; +} + + +export function execOutErrSync(command: string, ...args: string[]): ExecReturn { + let pipeArray = os.pipe(); + if (!pipeArray){ + return { rc: -1 }; + } + let errArray = os.pipe(); + if (!errArray){ + return { rc: -1 }; + } + const rc = os.exec([command, ...args], { block: true, usePath: true, stdout: pipeArray[1], stderr: errArray[1]}); + + let out = readStreamFully(pipeArray[0]); + os.close(pipeArray[0]); + os.close(pipeArray[1]); + + let err = readStreamFully(errArray[0]); + os.close(errArray[0]); + os.close(errArray[1]); + return { + rc, err + }; +} + +export function which(command: string): path|undefined { + //TODO not windows + let pathParts = (std.getenv('PATH') || "").split(':'); + for (let i = 0; i < pathParts.length; i++) { + let files:string[] = fs.getFilesInDirectory(pathParts[i]) || []; + if (files.indexOf(command) != -1) { + return `${pathParts[i]}/${command}`; + } + } + return undefined; +} diff --git a/bin/libs/strftime.ts b/bin/libs/strftime.ts new file mode 100644 index 0000000000..417f3ebce4 --- /dev/null +++ b/bin/libs/strftime.ts @@ -0,0 +1,87 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +/* +Derived from strftime() by T. H. Doan (https://thdoan.github.io/strftime/) +Licensed: +MIT License + +Copyright (c) 2016 Tom Doan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ +export function strftime(sFormat:string, dateArg?:Date): string { + let date:Date = (dateArg instanceof Date) ? dateArg : new Date(); + const nDay = date.getDay(); + const nDate = date.getDate(); + const nMonth = date.getMonth(); + const nYear = date.getFullYear(); + const nHour = date.getHours(); + const aDays = ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday']; + const aMonths = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']; + const aDayCount = [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334]; + + let isLeapYear = function() { + return (nYear%4===0 && nYear%100!==0) || nYear%400===0; + }; + let zeroPad = function(nNum:number, nPad:number) { + return ((Math.pow(10, nPad) + nNum) + '').slice(1); + }; + + return sFormat.replace(/%[a-z]/gi, function(sMatch) { + return (({ + '%a': aDays[nDay].slice(0,3), + '%A': aDays[nDay], + '%b': aMonths[nMonth].slice(0,3), + '%B': aMonths[nMonth], + '%c': date.toUTCString(), + '%C': Math.floor(nYear/100), + '%d': zeroPad(nDate, 2), + '%e': nDate, + '%H': zeroPad(nHour, 2), + '%I': zeroPad((nHour+11)%12 + 1, 2), + '%j': zeroPad(aDayCount[nMonth] + nDate + ((nMonth>1 && isLeapYear()) ? 1 : 0), 3), + '%l': (nHour+11)%12 + 1, + '%m': zeroPad(nMonth + 1, 2), + '%n': nMonth + 1, + '%M': zeroPad(date.getMinutes(), 2), + '%p': (nHour<12) ? 'AM' : 'PM', + '%P': (nHour<12) ? 'am' : 'pm', + '%s': Math.round(date.getTime()/1000), + '%S': zeroPad(date.getSeconds(), 2), + '%T': `${zeroPad(nHour, 2)}:${zeroPad(date.getMinutes(), 2)}:${zeroPad(date.getSeconds(), 2)}`, + '%u': nDay || 7, + '%w': nDay, + '%x': date.toLocaleDateString(), + '%X': date.toLocaleTimeString(), + '%y': (nYear + '').slice(2), + '%Y': nYear, + '%z': date.toTimeString().replace(/.+GMT([+-]\d+).+/, '$1'), + '%Z': date.toTimeString().replace(/.+\((.+?)\)$/, '$1') + }[sMatch] || '') + '') || sMatch; + }); +} diff --git a/bin/libs/string.ts b/bin/libs/string.ts new file mode 100644 index 0000000000..572596c65a --- /dev/null +++ b/bin/libs/string.ts @@ -0,0 +1,335 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; + +export const ENCODING_NAME_TO_CCSID:any = { + 'IBM-037': 37, + 'IBM-273': 273, + 'IBM-277': 277, + 'IBM-278': 278, + 'IBM-280': 280, + 'IBM-284': 284, + 'IBM-285': 285, + 'IBM-297': 297, + 'US-ASCII': 367, + 'IBM-420': 420, + 'IBM-423': 423, + 'IBM-424': 424, + 'IBM-437': 437, + 'IBM-500': 500, + 'IBM-808': 808, + 'ISO-8859-7': 813, + 'ISO-8859-1': 819, + 'IBM-Thai': 838, + 'IBM-850': 850, + 'IBM-852': 852, + 'IBM-855': 855, + 'IBM-857': 857, + 'IBM-858': 858, + 'IBM-862': 862, + 'IBM-863': 863, + 'IBM-864': 864, + 'IBM-866': 866, + 'IBM-867': 867, + 'IBM-869': 869, + 'IBM-870': 870, + 'IBM-871': 871, + 'IBM-872': 872, + 'TIS-620': 874, + 'KOI8-R': 878, + 'ISO-8859-13': 901, + 'IBM-902': 902, + 'IBM-904': 904, + 'ISO-8859-2': 912, + 'ISO-8859-5': 915, + 'ISO-8859-8-I': 916, + 'ISO-8859-9': 920, + 'IBM-921': 921, + 'IBM-922': 922, + 'ISO-8859-15': 923, + 'IBM-924': 924, + 'Shift_JIS': 932, + 'Windows-31J': 943, + 'EUC-KR': 949, + 'Big5': 950, + 'EUC-JP': 954, + 'EUC-TW': 964, + 'Microsoft-Publish': 1004, + 'IBM-1026': 1026, + 'IBM-1043': 1043, + 'IBM-1047': 1047, + 'hp-roman8': 1051, + 'ISO-8859-6': 1089, + 'VISCII': 1129, + 'IBM-1140': 1140, + 'IBM-1141': 1141, + 'IBM-1142': 1142, + 'IBM-1143': 1143, + 'IBM-1144': 1144, + 'IBM-1145': 1145, + 'IBM-1146': 1146, + 'IBM-1147': 1147, + 'IBM-1148': 1148, + 'IBM-1149': 1149, + 'IBM-1153': 1153, + 'IBM-1155': 1155, + 'KOI8-U': 1168, + 'UTF-16BE': 1200, + 'UTF-16LE': 1202, + 'UTF-16': 1204, + 'UTF-8': 1208, + 'UTF-32BE': 1232, + 'UTF-32LE': 1234, + 'UTF-32': 1236, + 'windows-1250': 1250, + 'windows-1251': 1251, + 'windows-1252': 1252, + 'windows-1253': 1253, + 'windows-1254': 1254, + 'windows-1255': 1255, + 'windows-1256': 1256, + 'windows-1257': 1257, + 'windows-1258': 1258, + 'MACINTOSH': 1275, + 'KSC_5601': 1363, + 'GBK': 1386, + 'GB18030': 1392 +}; + +const MAP_1047_TO_819 = [ +/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ +/* 0 */ 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, +/* 1 */ 0x10, 0x11, 0x12, 0x13, 0x9D, 0x0A, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, +/* 2 */ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, +/* 3 */ 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A, +/* 4 */ 0x20, 0xA0, 0xE2, 0xE4, 0xE0, 0xE1, 0xE3, 0xE5, 0xE7, 0xF1, 0xA2, 0x2E, 0x3C, 0x28, 0x2B, 0x7C, +/* 5 */ 0x26, 0xE9, 0xEA, 0xEB, 0xE8, 0xED, 0xEE, 0xEF, 0xEC, 0xDF, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0x5E, +/* 6 */ 0x2D, 0x2F, 0xC2, 0xC4, 0xC0, 0xC1, 0xC3, 0xC5, 0xC7, 0xD1, 0xA6, 0x2C, 0x25, 0x5F, 0x3E, 0x3F, +/* 7 */ 0xF8, 0xC9, 0xCA, 0xCB, 0xC8, 0xCD, 0xCE, 0xCF, 0xCC, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22, +/* 8 */ 0xD8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xAB, 0xBB, 0xF0, 0xFD, 0xFE, 0xB1, +/* 9 */ 0xB0, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xAA, 0xBA, 0xE6, 0xB8, 0xC6, 0xA4, +/* A */ 0xB5, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xA1, 0xBF, 0xD0, 0x5B, 0xDE, 0xAE, +/* B */ 0xAC, 0xA3, 0xA5, 0xB7, 0xA9, 0xA7, 0xB6, 0xBC, 0xBD, 0xBE, 0xDD, 0xA8, 0xAF, 0x5D, 0xB4, 0xD7, +/* C */ 0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xAD, 0xF4, 0xF6, 0xF2, 0xF3, 0xF5, +/* D */ 0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xB9, 0xFB, 0xFC, 0xF9, 0xFA, 0xFF, +/* E */ 0x5C, 0xF7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xB2, 0xD4, 0xD6, 0xD2, 0xD3, 0xD5, +/* F */ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xB3, 0xDB, 0xDC, 0xD9, 0xDA, 0x9F +]; + +const MAP_819_TO_1047 = [ +/* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ +/* 0 */ 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, +/* 1 */ 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, +/* 2 */ 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, +/* 3 */ 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F, +/* 4 */ 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, +/* 5 */ 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xAD, 0xE0, 0xBD, 0x5F, 0x6D, +/* 6 */ 0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, +/* 7 */ 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07, +/* 8 */ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x06, 0x17, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B, +/* 9 */ 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0xFF, +/* A */ 0x41, 0xAA, 0x4A, 0xB1, 0x9F, 0xB2, 0x6A, 0xB5, 0xBB, 0xB4, 0x9A, 0x8A, 0xB0, 0xCA, 0xAF, 0xBC, +/* B */ 0x90, 0x8F, 0xEA, 0xFA, 0xBE, 0xA0, 0xB6, 0xB3, 0x9D, 0xDA, 0x9B, 0x8B, 0xB7, 0xB8, 0xB9, 0xAB, +/* C */ 0x64, 0x65, 0x62, 0x66, 0x63, 0x67, 0x9E, 0x68, 0x74, 0x71, 0x72, 0x73, 0x78, 0x75, 0x76, 0x77, +/* D */ 0xAC, 0x69, 0xED, 0xEE, 0xEB, 0xEF, 0xEC, 0xBF, 0x80, 0xFD, 0xFE, 0xFB, 0xFC, 0xBA, 0xAE, 0x59, +/* E */ 0x44, 0x45, 0x42, 0x46, 0x43, 0x47, 0x9C, 0x48, 0x54, 0x51, 0x52, 0x53, 0x58, 0x55, 0x56, 0x57, +/* F */ 0x8C, 0x49, 0xCD, 0xCE, 0xCB, 0xCF, 0xCC, 0xE1, 0x70, 0xDD, 0xDE, 0xDB, 0xDC, 0x8D, 0x8E, 0xDF +]; + +// Note: Will error if string contains characters not present in 1047 +export function stringTo1047Buffer(input: string) { + const ebcdic = new Array(input.length); + for (let i = 0; i < input.length; i++){ + ebcdic[i] = MAP_819_TO_1047[input.charCodeAt(i)]; + } + return Uint8Array.from(ebcdic); +} + +export function ebcdicToAscii(input: string): string { + let ascii = []; + for (let i = 0; i < input.length; i++){ + ascii.push(MAP_1047_TO_819[input.charCodeAt(i)]); + } + return String.fromCharCode.apply(null, ascii); +} + +export function asciiToEbcdic(input: string): string { + let ebcdic = []; + for (let i = 0; i < input.length; i++){ + ebcdic.push(MAP_819_TO_1047[input.charCodeAt(i)]); + } + return String.fromCharCode.apply(null, ebcdic); +} + + +// via https://gist.github.com/joni/3760795 +export function stringToBuffer(input: string) { + const utf8 = []; + for (let i=0; i < input.length; i++) { + let charcode = input.charCodeAt(i); + if (charcode < 0x80) utf8.push(charcode); + else if (charcode < 0x800) { + utf8.push(0xc0 | (charcode >> 6), + 0x80 | (charcode & 0x3f)); + } + else if (charcode < 0xd800 || charcode >= 0xe000) { + utf8.push(0xe0 | (charcode >> 12), + 0x80 | ((charcode>>6) & 0x3f), + 0x80 | (charcode & 0x3f)); + } + // surrogate pair + else { + i++; + // UTF-16 encodes 0x10000-0x10FFFF by + // subtracting 0x10000 and splitting the + // 20 bits of 0x0-0xFFFFF into two halves + charcode = 0x10000 + (((charcode & 0x3ff)<<10) + | (input.charCodeAt(i) & 0x3ff)) + utf8.push(0xf0 | (charcode >>18), + 0x80 | ((charcode>>12) & 0x3f), + 0x80 | ((charcode>>6) & 0x3f), + 0x80 | (charcode & 0x3f)); + } + } + return Uint8Array.from(utf8); +} + +export function trim(input: string): string { + return input.trim(); +} + +//TODO this is a way to not have lossy output https://github.com/zowe/zlux-server-framework/blob/v2.x/staging/utils/argumentParser.js +export function sanitizeAlphanum(input: string): string { + const regex = /[^a-zA-Z0-9]/g; + return input.replace(regex, '_'); +} + +export function sanitizeAlpha(input: string): string { + const regex = /[^a-zA-Z]/g; + return input.replace(regex, '_'); +} + +export function sanitizeNum(input: string): string { + const regex = /[^0-9]/g; + return input.replace(regex, '_'); +} + +export function lowerCase(input: string): string { + return input.toLowerCase(); +} + +export function upperCase(input: string): string { + return input.toUpperCase(); +} + + +// Padding string on every lines of a multiple-line string +export function paddingLeft(str: string, pad: string): string { + return str.split('\n') + .map(function(line:string) { + return pad+line;}) + .join('\n'); +} + +/* +############################### +# Padding string on every lines of a file +# +# @param string optional string +file_padding_left() { + file="${1}" + pad="${2}" + + cat "${file}" | sed "s/^/${pad}/" +} + +############################### +# Padding string on every lines of multiple files separated by comma +# +# @param string optional string +files_padding_left() { + files="${1}" + pad="${2}" + separator="${3:-,}" + + OLDIFS=$IFS + IFS="${separator}" + for file in ${files}; do + file=$(trim "${file}") + if [ -n "${file}" -a -f "${file}" ]; then + cat "${file}" | sed "s/^/${pad}/" + fi + done + IFS=$OLDIFS +} +*/ + +export function removeTrailingSlash(input: string): string { + if (input.endsWith('/')) { + return input.substring(0, input.length-1); + } else { + return input; + } +} + +const binToB64 =[0x41,0x42,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4A,0x4B,0x4C,0x4D,0x4E,0x4F,0x50, + 0x51,0x52,0x53,0x54,0x55,0x56,0x57,0x58,0x59,0x5A,0x61,0x62,0x63,0x64,0x65,0x66, + 0x67,0x68,0x69,0x6A,0x6B,0x6C,0x6D,0x6E,0x6F,0x70,0x71,0x72,0x73,0x74,0x75,0x76, + 0x77,0x78,0x79,0x7A,0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0x2B,0x2F]; + +//TODO may be more complex than this, we have more thorough functions elsewhere in zlux +export function base64Encode(input: string): string { + let out = []; + + const inputLen = input.length; + const numFullGroups = Math.floor(inputLen / 3); + const numBytesInPartialGroup = inputLen - 3 * numFullGroups; + let inCursor = 0; + + // Translate all full groups from byte array elements to Base64 + for (let i = 0; i < numFullGroups; i++) { + let byte0 = input.charCodeAt(inCursor++) & 0xff; + let byte1 = input.charCodeAt(inCursor++) & 0xff; + let byte2 = input.charCodeAt(inCursor++) & 0xff; + out.push(binToB64[byte0 >> 2]); + out.push(binToB64[(byte0 << 4) & 0x3f | (byte1 >> 4)]); + out.push(binToB64[(byte1 << 2) & 0x3f | (byte2 >> 6)]); + out.push(binToB64[byte2 & 0x3f]); + } + + // Translate partial group if present + if (numBytesInPartialGroup != 0) { + let byte0 = input.charCodeAt(inCursor++) & 0xff; + out.push(binToB64[byte0 >> 2]); + if (numBytesInPartialGroup == 1) { + out.push(binToB64[(byte0 << 4) & 0x3f]); + out.push(0x3d); + out.push(0x3d); + } + else { + let byte1 = input.charCodeAt(inCursor++) & 0xff; + out.push(binToB64[(byte0 << 4) & 0x3f | (byte1 >> 4)]); + out.push(binToB64[(byte1 << 2) & 0x3f]); + out.push(0x3d); + } + } + + return String.fromCharCode.apply(null, out); +} + +export function itemInList(stringList: string, stringToFind?: string, separator: string=','): boolean { + if (!stringToFind) { + return false; + } + return stringList.split(separator).includes(stringToFind); +} diff --git a/bin/libs/sys.ts b/bin/libs/sys.ts new file mode 100644 index 0000000000..0e85065b9b --- /dev/null +++ b/bin/libs/sys.ts @@ -0,0 +1,179 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; + +import * as common from './common'; +import * as shell from './shell'; +import * as stringlib from './string'; + +std.setenv('ZWE_RUN_ON_ZOS', ""+(os.platform == 'zos')); +const pwd = os.getcwd(); +std.setenv('ZWE_PWD', pwd[0] ? pwd[0] : '/'); + +// Return system name in lower case +export function getSysname(): string|undefined { + let shellReturn = shell.execOutSync('sysvar', 'SYSNAME'); + if (!shellReturn.out) { + // works for z/OS and most Linux with hostname command + shellReturn = shell.execOutSync('hostname', '-s'); + } + if (!shellReturn.out) { + shellReturn = shell.execOutSync('uname', '-n'); + } + if (shellReturn.out) { + return shellReturn.out.toLowerCase(); + } + return undefined; +} + +export function getUserId(): string|undefined { + //moved to simplify dependency + return common.getUserId(); +} + +export function requireZos(): void { + if (os.platform != 'zos') { + common.printErrorAndExit("Error ZWEL0120E: This command must run on a z/OS system.", undefined, 120); + } +} + +// List direct children PIDs of a process +export function findDirectChildProcesses(parent: number, tree?: string[]): number[] { + let pids:number[] = []; + + if (!tree) { + let shellReturn = shell.execOutSync('ps', '-o', `pid,ppid,comm`, '-A'); + if (shellReturn.out) { + tree=shellReturn.out.split('\n').slice(2); + } + } + if (tree) { + tree.forEach((line: string)=> { + let parts = line.split(' '); + let pid = parts[0]; + let ppid = parts[1]; + let comm = parts[2]; + if (ppid == parent+'' && comm != 'ps' && comm != '/bin/ps') { + pids.push(Number(pid)); + } + }); + } + return pids; +} + +// List all children PIDs of a process +export function findAllChildProcesses(parent: number, tree?: string[]): number[] { + let pids:number[] = []; + + if (!tree) { + let shellReturn = shell.execOutSync('ps', '-o', `pid,ppid,comm`, '-A'); + if (shellReturn.out) { + tree=shellReturn.out.split('\n').slice(2); + } + } + if (tree) { + let children = findDirectChildProcesses(parent, tree); + children.forEach((child:number)=> { + pids.push(child); + pids.concat(findAllChildProcesses(child, tree)); + }); + } + return pids; +} + +// Wait until a single process exits +export function waitForProcessExit(pid: number): boolean { + common.printFormattedDebug("ZWELS", "sys-utils.ts,waitForProcessExit", `waiting for process ${pid} to exit`); + + let iteratorIndex=0; + const maxIteratorIndex=30; + + let shellReturn=shell.execOutSync('ps', `-p`, `${pid}`, `-o`, `pid`); + while (shellReturn.rc == 0) { //0 when there are processes in the list + os.sleep(1000); + iteratorIndex++; + if (iteratorIndex > maxIteratorIndex) { + break; + } + shellReturn=shell.execOutSync('ps', `-p`, `${pid}`, `-o`, `pid`); + } + + if (shellReturn.rc == 0) { + common.printFormattedDebug("ZWELS", "sys-utils.sh,wait_for_process_exit:", `process ${pid} does NOT exit before timeout`); + return false; + } else { + common.printFormattedDebug("ZWELS", "sys-utils.sh,wait_for_process_exit:", `process ${pid} no longer exists`); + return true; + } +} + +export function gracefullyShutdown(pid?: number): boolean { + if (pid === undefined || pid < 1) { + let pidString = std.getenv("ZWE_GRACEFULLY_SHUTDOWN_PID"); + if (pidString){ + pid = parseInt(pidString); + } + if ((pid===undefined || pid < 1) && os.platform == 'linux') { + //container case + pid = 1; + } else { + //dont try to shut down pid 1, its unlikely a container + return false; + } + } + if (pid >= 1) { + let children = findAllChildProcesses(pid); + common.printFormattedDebug("ZWELS", "sys-utils.sh,gracefully_shutdown", "SIGTERM signal received, shutting down process ${pid} and all child processes"); + + // send SIGTERM to all children + os.kill(pid, 15); + children.forEach((pid:number)=>{ + waitForProcessExit(pid); + }); + common.printFormattedDebug("ZWELS", "sys-utils.sh,gracefully_shutdown", "all child processes exited"); + return true; + } + return false; +} + +export function executeCommand(...args:string[]) { + common.printDebug(`- ${args}`); + let out; + let handler = (data:string)=> { + out=data; + } + let err; + let errHandler = (data:string)=> { + err=data; + } + const rc = os.exec(args, + {block: true, usePath: true, out: handler, err: errHandler}); + if (!rc) { + common.printDebug(` * Succeeded`); + common.printTrace(` * Exit code: ${rc}`); + common.printTrace(` * Output:`); + if (out) { + common.printTrace(stringlib.paddingLeft(out,' ')); + } + } else { + common.printDebug(` * Failed`); + common.printError(` * Exit code: ${rc}`); + common.printError(` * Output:`); + if (err) { + common.printError(stringlib.paddingLeft(err,' ')); + } + } + return { + rc, out, err + }; +} diff --git a/bin/libs/var.ts b/bin/libs/var.ts new file mode 100644 index 0000000000..cbb8a2b414 --- /dev/null +++ b/bin/libs/var.ts @@ -0,0 +1,176 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; +import * as xplatform from 'xplatform'; + +import * as common from './common'; +import * as shell from './shell'; +import * as stringlib from './string'; + +// internal errors counter for validations +std.setenv('ZWE_PRIVATE_ERRORS_FOUND', '0'); + +// Check if a shell function is defined +export function functionExists(fn: string): boolean { + let lcAll=std.getenv('LC_ALL'); + std.setenv('LC_ALL', 'C'); + let shellReturn = shell.execOutSync('type', fn);// 2>&1 | grep 'function') + if (lcAll){ + std.setenv('LC_ALL', lcAll); + } else { + std.unsetenv('LC_ALL'); + } + + if (shellReturn.out) { + return shellReturn.out.split('\n')[0].endsWith('function'); + } + return false; +} + + +// if a string has any env variables, replace them with values +export function parseStringVars(key: string): string|undefined { + return std.getenv(key); +} + +// return value of the variable +export function getVarValue(key: string): string { + return `\$\{${std.getenv(key)}`; +} + +// get all environment variable exports line by line +const exportFilter=/^export (run_zowe_start_component_id=|ZWELS_START_COMPONENT_ID|ZWE_LAUNCH_COMPONENTS|env_file=|key=|line=|service=|logger=|level=|expected_log_level_val=|expected_log_level_var=|display_log=|message=|utils_dir=|print_formatted_function_available=|LINENO=|ENV|opt|OPTARG|OPTIND|LOGNAME=|USER=|SSH_|SHELL=|PWD=|OLDPWD=|PS1=|ENV=|LS_COLORS=|_=)/ +const declareFilter=/^declare -x (run_zowe_start_component_id=|ZWELS_START_COMPONENT_ID|ZWE_LAUNCH_COMPONENTS|env_file=|key=|line=|service=|logger=|level=|expected_log_level_val=|expected_log_level_var=|display_log=|message=|utils_dir=|print_formatted_function_available=|LINENO=|ENV|opt|OPTARG|OPTIND|LOGNAME=|USER=|SSH_|SHELL=|PWD=|OLDPWD|PS1=|ENV=|LS_COLORS=|_=)/ + +const keyFilter=/^(run_zowe_start_component_id|ZWELS_START_COMPONENT_ID|ZWE_LAUNCH_COMPONENTS|env_file|key|line|service|logger|level|expected_log_level_val|expected_log_level_var|display_log|message|utils_dir|print_formatted_function_available|LINENO|ENV|opt|OPTARG|OPTIND|LOGNAME|USER|SSH_|SHELL|PWD|OLDPWD|PS1|ENV|LS_COLORS|_)$/ + +export function getEnvironmentExports(input?:string): string { + let exports:string[]=[]; + if (!input) { + let envvars = std.getenviron(); + let keys = Object.keys(envvars); + keys.forEach((key: string)=> { + if (!keyFilter.test(key)) { + exports.push(`export ${key}=${envvars[key]}`); + } + }); + } else { + const lines = input.split('\n'); + lines.forEach((line:string)=> { + if ((line.startsWith('export ') || line.startsWith('declare -x ')) && (!exportFilter.test(line) && !declareFilter.test(line))) { + exports.push(line); + } + }); + } + return exports.join('\n'); +} + +// get all environment variable exports line by line +export function getEnvironments(): string { + let envvars = std.getenviron(); + let keys = Object.keys(envvars); + let exports:string[]=[]; + keys.forEach((key: string)=> { + if (keyFilter.test(key)) { + exports.push(`${key}=${envvars[key]}`); + } + }); + return exports.join('\n'); +} + +// Shell sourcing an environment env file +// +// All variables defined in env file will be exported. +export function sourceEnv(envFile: string): boolean { + //TODO i hope encoding is correct here + + let fileContents = xplatform.loadFileUTF8(envFile,xplatform.AUTO_DETECT); + let fileLines = fileContents.split('\n'); + let index; + fileLines.forEach((line: string)=> { + if ((index = line.indexOf('=')) != -1) { + let key = line.substring(0, index); + if ((line[index+1] == "'" && line.endsWith("'")) || (line[index+1] == '"' && line.endsWith('"'))) { + let val = line.substring(index + 2, line.length-1); + std.setenv(key, val); + common.printTrace(`Set env var ${key} to ${val}`); + } else { + let val = line.substring(index + 1); + std.setenv(key, val); + common.printTrace(`Set env var ${key} to ${val}`); + } + } + }); + return true; +} + +// Takes in a single parameter - the name of the variable +export function isVariableSet(variableName: string, message?: string): boolean { + if (!message) { + message=`${variableName} is not defined or empty.` + } + const value = std.getenv(variableName); + if (value === undefined) { + common.printError(message); + return false; + } + return true; +} + +// Takes in a list of space separated names of the variables +export function areVariablesSet(variables: string[]): number { + let invalid=0 + + variables.forEach((variable: string) => { + if (!isVariableSet(variable)) { + invalid++; + } + }); + + return invalid; +} + +export function validateThis(cmd: string, origin: string): number { + common.printFormattedTrace("ZWELS", origin, `Validate: ${cmd}`); + let shellReturn = shell.execOutErrSync('sh', `eval`, `\"${cmd}\"`); + if (!shellReturn.rc) { + if (shellReturn.out) { + common.printFormattedDebug("ZWELS", origin, stringlib.paddingLeft(shellReturn.out, '- ')); + } else { + common.printFormattedTrace("ZWELS", origin, "- Passed."); + } + } else { + common.printFormattedTrace("ZWELS", origin, `- Failed with exit code ${shellReturn.rc}`); + if (shellReturn.err) { + common.printFormattedError("ZWELS", origin, stringlib.paddingLeft(shellReturn.err, '- ')); + } + } + + let prevErr = std.getenv("ZWE_PRIVATE_ERRORS_FOUND"); + let prevErrCount = !prevErr ? 0 : Number(prevErr); + if (shellReturn.rc) { + prevErrCount++; + std.setenv("ZWE_PRIVATE_ERRORS_FOUND",''+prevErrCount); + } + return shellReturn.rc; +} + +export function checkRuntimeValidationResult(origin: string) { + // Summary errors check, exit if errors found + let prevErr = std.getenv("ZWE_PRIVATE_ERRORS_FOUND"); + let prevErrCount = !prevErr ? 0 : Number(prevErr); + if ( prevErrCount > 0) { + common.printFormattedWarn("ZWELS", origin, `${prevErrCount} errors were found during validation, please check the message, correct any properties required in ${std.getenv('ZWE_CLI_PARAMETER_CONFIG')} and re-launch Zowe.`); + std.exit(prevErrCount ); + } +} diff --git a/bin/libs/zos-fs.ts b/bin/libs/zos-fs.ts new file mode 100644 index 0000000000..6848eb3e35 --- /dev/null +++ b/bin/libs/zos-fs.ts @@ -0,0 +1,110 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; +import * as zos from 'zos'; + +import * as common from './common'; +import * as stringlib from './string'; +import * as shell from './shell'; + +// Get file encoding from z/OS USS tagging +export function getFileEncoding(filePath: string): number|undefined { + //zos.changeTag(file, id) + let returnArray = os.stat(filePath); + if (!returnArray[1] && ((returnArray[0].mode & os.S_IFREG) == os.S_IFREG)) { //no error, and is file + return returnArray[0].ccsid; + } else { + common.printError(`getFileEncoding path=${filePath}, err=${returnArray[1]}`); + } + return undefined; +} + +// TODO logic rewritten, needs more testing +// Detect and verify file encoding +// +// This function will try to verify file encoding by reading sample string. +// +// Note: Return depends on the cases, the output is +// confirmed encoding to stdout. +// - file is already tagged: the output will be the encoding tag, +// - file is not tagged: +// - expected encoding is auto: the output will be one of IBM-1047, +// ISO8859-1, IBM-850 based on the guess. Output is -1 if none +// of those encodings are correct. +// - expected encoding is not auto: the output will be same as expected +// encoding if it's correct. otherwise output will be -1. +// +// Example: +// - detect manifest encoding by checking result "name" +// detectFileEncoding "/path/to/zowe/components/my-component/manifest.yaml" "name" +export function detectFileEncoding(fileName: string, expectedSample: string, expectedEncoding?: number|string): number { + let autoEncoding = expectedEncoding == 'auto' || expectedEncoding == 'AUTO'; + let expectedEncodingNumber= typeof expectedEncoding == 'string' ? stringlib.ENCODING_NAME_TO_CCSID[expectedEncoding.toUpperCase()] : expectedEncoding; + + let currentTag = getFileEncoding(fileName); + if (currentTag) { + return currentTag; + } else { + //loadfile does not convert untagged ebcdic + let fileContents = std.loadFile(fileName); + if (fileContents) { + if ((!expectedEncoding || + expectedEncodingNumber == 1047 || + autoEncoding) && !fileContents.includes(expectedSample)) { + return 1047; + } else if (expectedEncodingNumber) { + let execReturn = shell.execOutSync('sh', `iconv`, `-f`, `${expectedEncodingNumber}`, `-t`, `1047`, `${fileName}`, `|`, `grep`, `${expectedSample}`); + if (execReturn.rc == 0 && execReturn.out) { + return expectedEncodingNumber; + } + } else { + //Common encodings, 8859-1 and ascii 850 + const commonEncodings = [819, 850]; + for (let i = 0; i < commonEncodings.length; i++) { + const encoding = commonEncodings[i]; + let execReturn = shell.execOutSync('sh', `iconv`, `-f`, `${encoding}`, `-t`, `1047`, `${fileName}`, `|`, `grep`, `${expectedSample}`); + if (execReturn.rc == 0 && execReturn.out) { + return encoding; + } + } + } + } + } + return -1; +} + +// On z/OS, some file generated could be in ISO8859-1 encoding, but we need it to be IBM-1047 +export function ensureFileEncoding(file: string, expectedSample: string, expectedEncoding?: number) { + if (os.platform != 'zos') { + return; + } + if (!expectedEncoding) { + expectedEncoding=1047; + } + + let fileEncoding=detectFileEncoding(file, expectedSample); + if (fileEncoding) { + // TODO any cases we cannot find encoding? + if (fileEncoding != expectedEncoding) { + common.printTrace(`- Convert encoding of ${file} from ${fileEncoding} to ${expectedEncoding}.`); + let shellReturn = shell.execSync('sh', `iconv`, `-f`, `${fileEncoding}`, `-t`, `${expectedEncoding}`, `${file}`, `>`, `${file}.tmp`); + if (!shellReturn.rc) { + os.rename(`${file}.tmp`, file); + } + } + common.printTrace(`- Remove encoding tag of ${file}.`); + zos.changeTag(file, 0); + } else { + common.printTrace(`- Failed to detect encoding of ${file}.`); + } +} diff --git a/bin/libs/zosmf.ts b/bin/libs/zosmf.ts new file mode 100644 index 0000000000..283197013b --- /dev/null +++ b/bin/libs/zosmf.ts @@ -0,0 +1,60 @@ +/* + This program and the accompanying materials are made available + under the terms of the Eclipse Public License v2.0 which + accompanies this distribution, and is available at + https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +import * as std from 'std'; +import * as os from 'os'; +import * as zos from 'zos'; + +import * as common from './common'; +import * as shell from './shell'; + +export function validateZosmfHostAndPort(zosmfHost: string, zosmfPort: number): boolean { + if (!zosmfHost) { + common.printError('z/OSMF host is not set.'); + return false; + } + if (!zosmfPort) { + common.printError('z/OSMF port is not set.'); + return false; + } + let zosmfCheckPassed=true; + + if (!std.getenv('NODE_HOME')) { + common.printError(`Warning: Could not validate if z/OS MF is available on 'https://${zosmfHost}:${zosmfPort}/zosmf/info'. NODE_HOME is not defined.`); + zosmfCheckPassed=false; + } else { + let execReturn = shell.execOutSync(`${std.getenv('NODE_HOME')}/bin/node`, `${std.getenv('ZWE_zowe_runtimeDirectory')}/bin/utils/curl.js`, `"https://${zosmfHost}:${zosmfPort}/zosmf/info"`, `-k`, `-H`, `"X-CSRF-ZOSMF-HEADER: true"`, `--response-type`, `status`); + if (execReturn.rc || !execReturn.out) { + common.printError(`Warning: Could not validate if z/OS MF is available on 'https://${zosmfHost}:${zosmfPort}/zosmf/info'. No response code from z/OSMF server.`); + zosmfCheckPassed=false + } else if (execReturn.out != '200') { + common.printError(`Could not contact z/OS MF on 'https://${zosmfHost}:${zosmfPort}/zosmf/info' - ${execReturn.out}`); + zosmfCheckPassed=false + return false; + } + } + + if (zosmfCheckPassed) { + common.printMessage(`Successfully checked z/OS MF is available on 'https://${zosmfHost}:${zosmfPort}/zosmf/info'`) + } + return zosmfCheckPassed; +} + +//TODO isnt this completely backwards? +export function validateZosmfAsAuthProvider(zosmfHost: string, zosmfPort: number, authProvider: string): boolean { + if (zosmfHost && zosmfPort) { + if (authProvider == 'zosmf') { + common.printError("z/OSMF is not configured. Using z/OSMF as authentication provider is not supported."); + return true; + } + } + return false; +} diff --git a/bin/utils/ObjUtils.js b/bin/utils/ObjUtils.js new file mode 100644 index 0000000000..bf4fb55c40 --- /dev/null +++ b/bin/utils/ObjUtils.js @@ -0,0 +1,247 @@ +/* + Some very common JS object manipulation utilities to + avoid the need for loads of NPM effluvia. + +*/ +; +export class Objutils { + static isObject(x) { + let type = typeof x; + return (x != null) && (type === 'object'); + } + static isAtom(x) { + return !Objutils.isObject(x) && !Array.isArray(x); + } + static arrayAll(a, predicate) { + for (const elt of a) { + if (!predicate(a)) { + return false; + } + } + return true; + } + static propertyValuesAll(obj, predicate) { + for (const key in obj) { + let value = obj[key]; + if (!predicate(value)) { + return false; + } + } + return true; + } +} +export class Flattener { + constructor() { + this.prefix = ""; + this.separator = "."; + } + flatten1(json, first, keyPrefix, result) { + if (Array.isArray(json)) { + for (let i = 0; i < json.length; i++) { + this.flatten1(json[i], false, keyPrefix + (first ? "" : this.separator) + i, result); + } + return result; + } + else if (Objutils.isObject(json)) { + const keys = Object.keys(json); + for (const key of keys) { + this.flatten1(json[key], false, keyPrefix + (first ? "" : this.separator) + key, result); + } + } + else { + result[keyPrefix] = json; + } + } + setSeparator(separator) { + this.separator = separator; + } + setPrefix(prefix) { + this.prefix = prefix; + } + flatten(x) { + let result = {}; + this.flatten1(x, true, this.prefix, result); + return result; + } +} +export class Copier { + constructor() { + } + copy(json) { + if (Array.isArray(json)) { + let result = []; + for (let i = 0; i < json.length; i++) { + result[i] = this.copy(json[i]); + } + return result; + } + else if (Objutils.isObject(json)) { + let result = {}; + const keys = Object.keys(json); + for (const key of keys) { + result[key] = this.copy(json[key]); + } + return result; + } + else { + return json; // string, boolean, number, Symbol, null, ... + } + } +} +export class Merger extends Copier { + constructor() { + super(); + this.mergeArrays = true; + } + merge(overrides, base) { + if (Array.isArray(base)) { + if (Array.isArray(overrides)) { + if (this.mergeArrays) { + return base.concat(overrides); + } + else { + let empty = []; + return empty.concat(overrides); + } + } + else { + return this.copy(overrides); + } + } + else if (Objutils.isObject(base)) { + if (Objutils.isObject(overrides)) { + /* + console.log("merging "+JSON.stringify(base)); + console.log("and "+JSON.stringify(overrides)); + */ + let result = {}; + const keys = Object.keys(base); + for (const key of keys) { + let baseValue = base[key]; + let overrideValue = overrides[key]; + if (overrideValue === undefined) { + result[key] = this.copy(baseValue); + } + else { + result[key] = this.merge(overrideValue, baseValue); + } + } + const overrideKeys = Object.keys(overrides); + for (const key of overrideKeys) { + if (base[key] === undefined) { + result[key] = this.copy(overrides[key]); + } + } + return result; + } + else { + return this.copy(overrides); + } + } + else { + return this.copy(overrides); + } + } +} +const MAX_SHORT_ARRAY = 4; +const MAX_SMALL_OBJECT = 3; +export class PrettyPrinter { + constructor() { + this.chunks = []; + } + newline() { + this.chunks.push("\n"); + return this; + } + indent(depth) { + let count = depth * 4; + this.chunks.push("".padEnd(count, " ")); + } + out(x) { + this.chunks.push(x.toString()); + return this; + } + keyOut(x) { + this.chunks.push(x.toString()); + this.chunks.push(": "); + return this; + } + valueOut(x) { + if (typeof x === "string") { + this.chunks.push('"'); + this.chunks.push(x); + this.chunks.push('"'); + } + else { + this.chunks.push(x.toString()); + } + return this; + } + prettyPrint1(x, depth, pendingKey, lastInParent) { + if (Array.isArray(x)) { + let len = x.length; + if (false) { // (len < MAX_SHORT_ARRAY) && Objutils.arrayAll(x, Objutils.isAtom)){ + } + else { + this.indent(depth); + if (pendingKey) { + this.keyOut(pendingKey); + } + this.out("["); + this.newline(); + for (let i = 0; i < x.length; i++) { + let value = x[i]; + let isLast = (i + 1 == x.length); + this.prettyPrint1(value, depth + 1, null, isLast); + } + this.indent(depth); + this.out("]"); + if (!lastInParent) { + this.out(","); + } + this.newline(); + } + } + else if (Objutils.isObject(x)) { + const keys = Object.keys(x); + if (false) { + } + else { + this.indent(depth); + if (pendingKey) { + this.keyOut(pendingKey); + } + this.out("{"); + this.newline(); + for (let i = 0; i < keys.length; i++) { + let key = keys[i]; + let value = x[key]; + let isLast = (i + 1 == keys.length); + this.prettyPrint1(value, depth + 1, key, isLast); + } + this.indent(depth); + this.out("}"); + if (!lastInParent) { + this.out(","); + } + this.newline(); + } + } + else { + this.indent(depth); + if (pendingKey) { + this.keyOut(pendingKey); + } + this.valueOut(x); + if (!lastInParent) { + this.out(","); + } + this.newline(); + } + } + print(x) { + this.chunks = []; + this.prettyPrint1(x, 0, null, true); + console.log("" + this.chunks.join("")); + } +} diff --git a/bin/utils/ObjUtils.ts b/bin/utils/ObjUtils.ts new file mode 100644 index 0000000000..b53f8c0d22 --- /dev/null +++ b/bin/utils/ObjUtils.ts @@ -0,0 +1,274 @@ +/* + Some very common JS object manipulation utilities to + avoid the need for loads of NPM effluvia. + +*/ + +declare namespace console { + function log(...args:string[]): void; +}; + +export class Objutils { + static isObject(x:any){ + let type = typeof x; + return (x != null) && (type === 'object') + } + + static isAtom(x:any){ + return !Objutils.isObject(x) && !Array.isArray(x); + } + + static arrayAll(a:any, predicate: (x:any) => boolean){ + for (const elt of a){ + if (!predicate(a)){ + return false; + } + } + return true; + } + + static propertyValuesAll(obj:any, predicate: (x:any) => boolean){ + for (const key in obj){ + let value = obj[key]; + if (!predicate(value)){ + return false; + } + } + return true; + } + +} + +export class Flattener { + prefix:string = ""; + separator:string = "."; + + constructor(){ + + } + + flatten1(json:any, first:boolean, keyPrefix:string, result:any):void { + if (Array.isArray(json)){ + for (let i=0; i=4.2.0" + } + } + }, + "dependencies": { + "typescript": { + "version": "4.6.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.6.4.tgz", + "integrity": "sha512-9ia/jWHIEbo49HfjrLGfKbZSuWo9iTMwXO+Ca3pRsSpbsMbc7/IU8NKdCZVRRBafVPGnoJeFL76ZOAA84I9fEg==", + "dev": true + } + } +} diff --git a/build/zwe/package.json b/build/zwe/package.json new file mode 100644 index 0000000000..4d7aac5a0f --- /dev/null +++ b/build/zwe/package.json @@ -0,0 +1,27 @@ +{ + "name": "zowe-zwe-cli", + "version": "2.0.0", + "description": "A CLI and set of libraries that constitute the Zowe server infrastructure", + "scripts": { + "clean": "tsc --build tsconfig.json --clean", + "dev": "tsc --build tsconfig.dev.json --force", + "prod": "tsc --build tsconfig.prod.json --force", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/zowe/zowe-install-packaging.git" + }, + "keywords": [ + "zowe" + ], + "author": "Zowe", + "license": "EPL-2.0", + "bugs": { + "url": "https://github.com/zowe/zowe-install-packaging/issues" + }, + "homepage": "https://github.com/zowe/zowe-install-packaging#readme", + "devDependencies": { + "typescript": "4.6.4" + } +} diff --git a/build/zwe/tsconfig.dev.json b/build/zwe/tsconfig.dev.json new file mode 100644 index 0000000000..0f17b3f737 --- /dev/null +++ b/build/zwe/tsconfig.dev.json @@ -0,0 +1,37 @@ +{ + "compilerOptions": { + "lib": [ "ES2020" ], + "module": "ES2020", + "target": "ES2020", + "moduleResolution": "node", + "removeComments": false, + "preserveConstEnums": true, + "sourceMap": false, + "noUnusedLocals": false, + "noUnusedParameters": false, + "pretty": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": false, + "allowUnreachableCode": false, + "allowJs": false, + "listEmittedFiles": true, + "strict": false, + "baseUrl": "../../bin", + "outDir": "../../build/zwe/out", + "paths": { + // A series of entries which re-map imports to lookup locations relative to the baseUrl, + // there is a larger coverage of paths in the handbook. + // See: https://www.typescriptlang.org/tsconfig + "*": [ "../build/zwe/types/@qjstypes/*" ], + } + }, + "include": [ + "../../bin/libs/*", + "../../bin/commands/**/*", + "../../bin/utils/*" + ], + "exclude": [ + "../../bin/libs/json.ts", + "../../bin/commands/start/index.ts" + ] +} diff --git a/build/zwe/tsconfig.prod.json b/build/zwe/tsconfig.prod.json new file mode 100644 index 0000000000..e34ce6cdcc --- /dev/null +++ b/build/zwe/tsconfig.prod.json @@ -0,0 +1,37 @@ +{ + "compilerOptions": { + "lib": [ "ES2020" ], + "module": "ES2020", + "target": "ES2020", + "moduleResolution": "node", + "removeComments": false, + "preserveConstEnums": true, + "sourceMap": false, + "noUnusedLocals": false, + "noUnusedParameters": false, + "pretty": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": false, + "allowUnreachableCode": false, + "allowJs": false, + "listEmittedFiles": true, + "strict": false, + "baseUrl": "../../bin", + "outDir": "../../bin", + "paths": { + // A series of entries which re-map imports to lookup locations relative to the baseUrl, + // there is a larger coverage of paths in the handbook. + // See: https://www.typescriptlang.org/tsconfig + "*": [ "../build/zwe/types/@qjstypes/*" ], + } + }, + "include": [ + "../../bin/libs/*", + "../../bin/commands/**/*", + "../../bin/utils/*" + ], + "exclude": [ + "../../bin/libs/json.ts", + "../../bin/commands/start/index.ts" + ] +} diff --git a/build/zwe/types/@qjstypes/Configuration.d.ts b/build/zwe/types/@qjstypes/Configuration.d.ts new file mode 100644 index 0000000000..eed7b7e428 --- /dev/null +++ b/build/zwe/types/@qjstypes/Configuration.d.ts @@ -0,0 +1,26 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +export interface ConfigManager { + setTraceLevel(level:number):void; + getTraceLevel():number; + addConfig(name:string):number; + setConfigPath(configName:string,configPath:string):number; + loadSchemas(configName:string,schemaList:string):number; + getConfigData(configName:string):any; + loadConfiguration(configName:string):number; + validate(configName:string):any; // should give this a type + writeYAML(configName:string):[ number, string|null]; // 0 means status is good , string present if 0 +} + +declare var ConfigManager: { + prototype: ConfigManager; + new(): ConfigManager; +} diff --git a/build/zwe/types/@qjstypes/os.d.ts b/build/zwe/types/@qjstypes/os.d.ts new file mode 100644 index 0000000000..14bf73ef77 --- /dev/null +++ b/build/zwe/types/@qjstypes/os.d.ts @@ -0,0 +1,63 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +export type path = string; + +export function exec(a:string[], options?:any):number; + +export var O_CREAT:number; +export var O_WRONLY:number; +export var O_RDONLY:number; +export var S_IFMT:number; +export var S_IFIFO:number; +export var S_IFCHR:number; +export var S_IFDIR:number; +export var S_IFBLK:number; +export var S_IFREG:number; +export var S_IFSOCK:number; +export var S_IFLNK:number; +export var S_IFGID:number; +export var S_IFUID:number; +export var ENOENT:number; + +export var SIGINT: number; +export var SIGABRT: number; +export var SIGFPE: number; +export var SIGILL: number; +export var SIGSEGV: number; +export var SIGTERM: number; + +export function open(filename:string, flags:number, mode:number):number; +export function close(fd:number):number; +export function read(fd:number, buffer:ArrayBuffer, offset:number, length:number):number; +export function write(fd:number, buffer:ArrayBuffer, offset:number, length:number):number; + +export function remove(filename: string):number; +export function rename(oldname: string, newname :string):number; + +export function stat(path:string):[any,number]; +export function lstat(path:string):[any,number]; + +export function signal(signal:number, fun:() => void):void; +export function kill(pid:number, signal:number):void; +export function waitpid(pid:number, options:any):[number, number]; + +export function readdir(path:string):[string[],number]; +export function realpath(path:string):[string,number] +export function getcwd():[string,number]; +export function chdir(path:string):number; + +export function symlink(target: string, linkpath: string):number; +export function mkdir(path:string, mode?:number):number; +export function dup2(oldfd:number, newfd:number):void; +export function sleep(millis:number):void; +export function pipe():[number,number]|null; + +export var platform:string; diff --git a/build/zwe/types/@qjstypes/posix.d.ts b/build/zwe/types/@qjstypes/posix.d.ts new file mode 100644 index 0000000000..3cfe7d53d7 --- /dev/null +++ b/build/zwe/types/@qjstypes/posix.d.ts @@ -0,0 +1,11 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +export function chmod(path:string, mode:number):number; diff --git a/build/zwe/types/@qjstypes/std.d.ts b/build/zwe/types/@qjstypes/std.d.ts new file mode 100644 index 0000000000..08c6bf50b1 --- /dev/null +++ b/build/zwe/types/@qjstypes/std.d.ts @@ -0,0 +1,70 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +export function getenv(varName:string):string|undefined; + +export function setenv(varName:string, value:string):void; + +export function unsetenv(varName:string):void; + +/** + really key-value pairs, but no strong type for return yet +*/ +export function getenviron():any; + +export function exit(status:number):void; + +export function loadFile(filename:string):string|null; // returns a string, or NULL if IO error + + +export interface File { + printf(formatString:string, ...args:any[]):void; + puts(s:string):void; + close():number; + tell():number; + eof():boolean; + error():boolean; + clearerr():void; + read(buffer:ArrayBuffer, position:number, length:number):number; + write(buffer:ArrayBuffer, position:number, length:number):number; +} + +export function fdopen(fd:number, fopenMode:string, errorObj?:any):File|null; +export function open(command:string, fopenMode:string, errorObj?:any):File|null; +export function popen(command:string, fopenMode:string, errorObj?:any):File|null; + +/* STDOUT convenience functions */ +export function puts(s:string):void; +export function printf(formatString:string, ...args:any[]):void; + +/* builds a new string */ +export function sprintf(formatString:string, ...args:any[]):string; + +export function parseExtJSON(s:string):any; + +export var out:File; +export var err:File; +export var frog:File; // JOE "in" is a reserved word + +export type ErrorEnumType = { + EINVAL: number, + EIO:number, + EACCES:number, + EEXIST:number, + ENOSPC:number, + ENOSYS:number, + EBUSY:number, + ENOENT:number, + EPERM:number, + EPIPE:number +}; + +export var Error:ErrorEnumType; + diff --git a/build/zwe/types/@qjstypes/xplatform.d.ts b/build/zwe/types/@qjstypes/xplatform.d.ts new file mode 100644 index 0000000000..f139e1ac00 --- /dev/null +++ b/build/zwe/types/@qjstypes/xplatform.d.ts @@ -0,0 +1,32 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +export function fileCopy(source:string, destination:string):[number,number,number]; +export function fileCopyConverted(source:string, sourceCCSID:number, + destination:string, destinationCCSID:number):[number,number,number]; +export function dirname(path:string):[string,number]; + +export function getpid():number; + +/** + sourceCCSID == -1, means apply default charset conversion if necessary. + + sourceCCSID == 0, mean don't translate bytes, trust that they are UTF8, even if they aren't! +*/ +export function stringFromBytes(data:ArrayBuffer, offset:number, length:number, sourceCCSID:number):string; + +/** + sourceCCSID as above +*/ +export function loadFileUTF8(path:string, sourceCCSID:number):string; +export function storeFileUTF8(path:string, targetCCSID:number, content:string):number; + +export var AUTO_DETECT:number; +export var NO_CONVERT:number; diff --git a/build/zwe/types/@qjstypes/zos.d.ts b/build/zwe/types/@qjstypes/zos.d.ts new file mode 100644 index 0000000000..ec41ae6571 --- /dev/null +++ b/build/zwe/types/@qjstypes/zos.d.ts @@ -0,0 +1,28 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +export type ZStat = { + dev: number; + ino: number; + uid: number; + gid: number; + atime: number; + mtime: number; + ctime: number; + extattrs: number; + isText: boolean; + ccsid: number; +}; + +export function changeTag(path:string, ccsid:number):number; +export function changeExtAttr(path: string, extattr:number, onOff:boolean):number; +export function zstat(path:string):[ZStat, number]; +export var EXTATTR_SHARELIB:number; +export var EXTATTR_PROGCTL:number; diff --git a/example-zowe.yaml b/example-zowe.yaml index ace629e09f..9c35217d2f 100644 --- a/example-zowe.yaml +++ b/example-zowe.yaml @@ -50,8 +50,7 @@ zowe: # JCL library where Zowe will store temporary JCLs during initialization jcllib: IBMUSER.ZWEV2.CUST.JCLLIB # APF authorized LOADLIB for Zowe - # Optional. If it's empty, .SZWEAUTH will be APF authorized. - authLoadlib: "" + authLoadlib: IBMUSER.ZWEV2.SZWEAUTH # **COMMONLY_CUSTOMIZED** # APF authorized LOADLIB for Zowe ZIS Plugins authPluginLib: IBMUSER.ZWEV2.CUST.ZWESAPL @@ -331,7 +330,7 @@ zowe: # Enable debug mode for zowe launch scripts launchScript: # set to "debug" or "trace" to display extra debug information - logLevel: "" + logLevel: "info" # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> # Default Zowe certificate diff --git a/manifest.json.template b/manifest.json.template index f2f1d4527d..492fa103ee 100644 --- a/manifest.json.template +++ b/manifest.json.template @@ -44,23 +44,23 @@ "artifact": "*.pax" }, "org.zowe.explorer.jobs.jobs-api-package": { - "version": "^2.0.0-SNAPSHOT", + "version": "^2.0.7-SNAPSHOT", "artifact": "jobs-api-package-*.zip", "exclusions": ["*PR*.zip","*BRANCH*"] }, "org.zowe.explorer.files.files-api-package": { - "version": "^2.0.0-SNAPSHOT", + "version": "^2.0.7-SNAPSHOT", "artifact": "files-api-package-*.zip", "exclusions": ["*PR*.zip","*BRANCH*"] }, "org.zowe.explorer-jes": { - "version": "~2.0.0-SNAPSHOT" + "version": "~2.0.1-SNAPSHOT" }, "org.zowe.explorer-mvs": { - "version": "~2.0.0-SNAPSHOT" + "version": "~2.0.1-SNAPSHOT" }, "org.zowe.explorer-uss": { - "version": "~2.0.0-SNAPSHOT" + "version": "~2.0.1-SNAPSHOT" }, "org.zowe.explorer-ip": { "version": "~1.0.0-SNAPSHOT", @@ -97,7 +97,7 @@ "exclusions": ["*PR*.zip"] }, "org.zowe.apiml.sdk.common-java-lib-package": { - "version": "~1.21.3-SNAPSHOT", + "version": "~2.0.0-SNAPSHOT", "artifact": "common-java-lib-*.zip", "exclusions": ["*PR*.zip"] }, @@ -106,8 +106,12 @@ "artifact": "apiml-sample-extension-*.zip", "exclusions": ["*PR*.zip"] }, + "org.zowe.configmgr": { + "version": "~0.3.0-feature_configmgr-integration", + "artifact": "*.pax" + }, "org.zowe.launcher": { - "version": "~2.0.0-SNAPSHOT" + "version": "~2.0.4-SNAPSHOT" }, "org.zowe.keyring-utilities": { "version": "1.0.4", diff --git a/schemas/manifest-schema.json b/schemas/manifest-schema.json new file mode 100644 index 0000000000..6fb931e3c8 --- /dev/null +++ b/schemas/manifest-schema.json @@ -0,0 +1,218 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "$id": "https://zowe.org/schemas/v2/server-component-manifest", + "title": "Zowe server component manifest file", + "description": "Manifest file spec for declaring properties of a zowe server component", + "type": "object", + "additionalProperties": true, + "required": [ "name", "id", "schemas" ], + "properties": { + "name": { + "type": "string", + "description": "Defines a short, computer-readable name of the component. This component name is used as directory name after it is installed. The allowed characters in the name are alphabets, numbers, hyphen (-) and underscore (_). For example, 'explorer-jes' is a valid extension name." + }, + "id": { + "$ref": "/schemas/v2/server-common#zoweReverseDomainNotation", + "description": "Defines a long, computer-readable identifier of the component. If the component is hosted as one of the projects in Open Mainframe Project, the identifier also matches the component path in the Zowe Artifactory. For example, 'org.zowe.explorer-jes' is a valid identifier. You can locate the component's official releases by looking into the 'libs-release-local/org/zowe/explorer-jes/' directory in the Zowe Artifactory (https://zowe.jfrog.io/ui/repos/tree/General/libs-release-local%2Forg%2Fzowe%2Fexplorer-jes)" + }, + "version": { + "$ref": "/schemas/v2/server-common#zoweSemverVersion", + "description": "This is the current version of the component without the prefix of v. For example, 2.0.0 is a valid version value." + }, + "title": { + "type": "string", + "description": "Defines a short human-readable name for this component. This value will also be used as the default title for API Catalog tile, or App Framework plug-in title. For example, 'JES Explorer' is a valid title for the 'explorer-jes' component." + }, + "description": { + "type": "string", + "description": "Defines a long human-readable description of this component. There is no restriction on what you can put in the field." + }, + "license": { + "type": "string", + "description": "Defines the license code of the component. For example, Zowe core components have EPL-2.0 value in this field." + }, + "schemas": { + "type": "object", + "description": "Defines the location of json schema files that are compatible with certain portions of Zowe as denoted by each child property.", + "additionalProperties": false, + "properties": { + "configs": { + "type": "string", + "description": "Defines the location of the json schema file which extends the Zowe Component base schema." + } + } + }, + "autoEncoding": { + "type": "array", + "description": "An array of paths to recursively search and automatically tag according to file extension, via execution of the tag-files script 'https://github.com/zowe/zowe-install-packaging/blob/v2.x/master/bin/utils/tag-files.sh'" + }, + "homepage": { + "type": "string", + "description": "A URL pointing to the homepage of the group that wrote this plugin" + }, + "keywords": { + "type": "array", + "description": "A list of terms that describe this component for easy search and indexing", + "items": { + "type": "string" + } + }, + "repository": { + "type": "object", + "description": "The type and location of the component source code repository, if applicable", + "properties": { + "type": { + "type": "string", + "enum": [ "accurev", "arch", "azure", "bazaar", "bitkeeper", "clearcase", "coop", "cvs", "darcs", "dat", "dimensions", "endevor", "fossil", "git", "icmanage", "integrity", "mercurial", "monotone", "perforce", "plastic", "pvcs", "rtc", "rcs", "scmanywhere", "sccs", "sclm", "starteam", "svn", "surround", "synergy", "vault", "vesta", "vss" ], + "description": "The type of version control system used" + }, + "url": { + "type": "string", + "description": "The URL where your source code repository is located" + } + } + }, + "build": { + "type": "object", + "description": "Defines the build information of the current package, including git commit hash, and so on. When Zowe core components define manifest file, these fields are left as template variables. The template will be updated when a publishable package is created.", + "additionalProperties": false, + "properties": { + "branch": { + "type": "string", + "description": "It indicates which branch this package is built from." + }, + "number": { + "type": "string", + "description": "You may create multiple packages in the same branch. This is the sequential number of the current package." + }, + "commitHash": { + "type": "string", + "description": "This is the commit hash of the package that can be used to match the exact source code in the repository. Zowe core components usually use 'git rev-parse --verify HEAD' to retrieve the commit hash." + }, + "timestamp": { + "type": "integer", + "description": "This is the UNIX timestamp when the package is created." + } + } + }, + "commands": { + "type": "object", + "description": "This defines actions that should be taken when the component is installed, configured, started, or tested. You must issue this command with one or more subfields as listed below. For example, 'commands.install'. All subfields should point to a USS command or script.", + "additionalProperties": false, + "properties": { + "install": { + "type": "string", + "description": "This defines extra steps when installing this component. It will be automatically executed if you install your component with the 'zwe components install' server command." + }, + "validate": { + "type": "string", + "description": "This defines extra validations that the component requires other than global validations. It is for runtime purpose, and will be automatically executed each time Zowe is started." + }, + "configure": { + "type": "string", + "description": "This defines extra configuration steps before starting the component. It is for runtime purpose, and will be automatically executed each time Zowe is started." + }, + "start": { + "type": "string", + "description": "This tells the Zowe launch script how to start the component. It is for runtime purpose, and will be automatically executed each time Zowe is started." + } + } + }, + "apimlServices": { + "type": "object", + "description": "This section defines how the component will be registered to the API Mediation Layer Discovery Service.", + "additionalProperties": false, + "properties": { + "dynamic": { + "type": "array", + "description": "This information will tell Zowe and users what services you will register under the Discovery service.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ "serviceId" ], + "properties": { + "serviceId": { + "type": "string", + "description": "This defines the service ID registered to the Discovery service." + } + } + } + }, + "static": { + "type": "array", + "description": "When the component is statically registered under the Discovery service, this tells Zowe where to find these static definitions. This information is for the Zowe runtime. When Zowe is starting, the launch script will check this field and put the parse static definition file into the directory defined as 'ZWE_STATIC_DEFINITIONS_DIR' in the Zowe instance.", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "file": { + "type": "string", + "description": "Defines the path to the static definition file. This file is supposed to be a template." + }, + "basePackage": { + "type": "string", + "description": "Defines the base package name of the extension. It is used to notify the extended service of the location for component scan." + } + } + } + } + } + }, + "appfwPlugins": { + "type": "array", + "description": "This section defines how the component will be registered to the App Framework plug-in.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ "path" ], + "properties": { + "path": { + "type": "string", + "description": "This points to the directory where App Framework pluginDefinition.json file is located. When Zowe is starting, the launch script will check this field and register the plug-in to Zowe App Framework Server." + } + } + } + }, + "gatewaySharedLibs": { + "type": "array", + "description": "This section defines the API ML extension(s) attributes which will get installed and used by API ML.", + "items": { + "type": "string", + "description": "This points to the directory where the JAR files are housed for an extension and later on copied into the API ML extensions workspace directory. If there is more than 1 extension to a single manifest (say for a product family of multiple extensions), then multiple path variables can be contained within the manifest denoted by individual folders, for example 'path/to/yourextension1/'. Alternatively, 'path' can be the JAR file path rather than a directory path." + } + }, + "discoverySharedLibs": { + "type": "array", + "description": "This section defines the API ML extension(s) attributes which will get installed and used by API ML.", + "items": { + "type": "string", + "description": "This points to the directory where the JAR files are housed for an extension and later on copied into the API ML extensions workspace directory. If there is more than 1 extension to a single manifest (say for a product family of multiple extensions), then multiple path variables can be contained within the manifest denoted by individual folders, for example 'path/to/yourextension1/'. Alternatively, 'path' can be the JAR file path rather than a directory path." + } + }, + "zisPlugins": { + "type": "array", + "description": "This section defines the ZIS plugin(s) attributes necessary for ZIS plugin installation and automation.", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ "id", "path" ], + "properties": { + "id": { + "type": "string", + "description": "This is the unique plugin ID of the ZIS plugin." + }, + "path": { + "type": "string", + "description": "This points to the directory where the load modules are housed for a plugin, for example '/zisServer'. If there is more than 1 plugin to a single manifest (say for a product family of multiple plugins), then multiple path variables can be contained within the manifest denoted by individual folders, for example 'yourplugin1/zisServer'. The parameters for the Zowe parmlib are assumed to be in '/samplib'. The names of the plugin executables are assumed to be in '/loadlib'." + } + } + } + }, + "configs": { + "type": "object", + "additionalProperties": true, + "description": "Component can define it's own configuration in this section in desired hierarchy. This is the brief guidance for component user to learn what are the configurations and what are the default values. Any configurations defined here can be placed into 'zowe.yaml' 'components.' section for customization. You can choose to put configurations into 'components.myextension' or 'haInstance..components.myextension' of 'zowe.yaml'. Component can use auto-generate environment variables in lifecycle scripts to learn how the component is configured for current HA instance." + } + } +} + diff --git a/schemas/server-common.json b/schemas/server-common.json index 715f307d8c..8b24afbb01 100644 --- a/schemas/server-common.json +++ b/schemas/server-common.json @@ -8,7 +8,7 @@ "$anchor": "zoweSemverVersion", "type": "string", "description": "A semantic version, see https://semver.org/", - "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$" + "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(-*[a-zA-Z][0-9a-zA-Z\\-\\.]*)?(\\+[0-9a-zA-Z\\-\\.]*)?$" }, "semverRange": { "$anchor": "zoweSemverRange", @@ -34,7 +34,7 @@ "reverseDomainNotation": { "$anchor": "zoweReverseDomainNotation", "type": "string", - "pattern": "^[A-Za-z]{2,6}((?!-)\\.[A-Za-z0-9-]{1,63}(?