diff --git a/.github/workflows/build-configmgr.yml b/.github/workflows/build-configmgr.yml index 548169160..1f94fb246 100644 --- a/.github/workflows/build-configmgr.yml +++ b/.github/workflows/build-configmgr.yml @@ -126,7 +126,7 @@ jobs: - name: '[Dep 2] Quickjs' uses: actions/checkout@v3 with: - repository: joenemo/quickjs-portable + repository: JoeNemo/quickjs-portable path: deps/configmgr/quickjs ref: 'main' diff --git a/CHANGELOG.md b/CHANGELOG.md index 30906c5ed..6c9a23b07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,23 @@ # Zowe Common C Changelog +## `2.17.0` +- Fixed `xplatform.loadFileUTF8` when trying to open nonexistent file (#454) +- Bugfix: fix an incorrect check in the recovery router code which might lead to + the state cell-pool being released prematurely (#446) +- Allocating SLH for http server with configurable value 'httpRequestHeapMaxBlocks' in yaml (#447). +- Return error when last config file is non existent or has some error (#460). + +## `2.16.0` +- No yaml value converted to null (#442) +- Added `zos.getZosVersion()` and `zos.getEsm()` calls for configmgr QJS (#429) +- For correct base64 encoding scheme the buffer size is made to be divisble by 3 (#431). +- Take into account leap seconds in xmem log messages' timestamps (#432, #433) +- Using a temporary buffer pointer to avoid pointer corruption during file write (#437). + +## `2.15.0` +- Remove obsolete building script build_configmgr.sh (#410). (#423) +- Add flags to avoid linkage-stack queries in the recovery facility (#404, #412) ## `2.13.0` - Added support for using "zowe.network" and "components.zss.zowe.network" to set TLS version properties. (#411) diff --git a/build/build_configmgr.sh b/build/build_configmgr.sh deleted file mode 100755 index 3a3e8ca40..000000000 --- a/build/build_configmgr.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/sh - -# This program and the accompanying materials are -# made available under the terms of the Eclipse Public License v2.0 which accompanies -# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html -# -# SPDX-License-Identifier: EPL-2.0 -# -# Copyright Contributors to the Zowe Project. - -WORKING_DIR=$(dirname "$0") - - -echo "********************************************************************************" -echo "Building configmgr..." - -rm -f "${COMMON}/bin/configmgr" - -mkdir -p "${WORKING_DIR}/tmp-configmgr" && cd "$_" -COMMON="../.." - -LIBYAML="libyaml" -MAJOR=0 -MINOR=2 -PATCH=5 -VERSION="\"${MAJOR}.${MINOR}.${PATCH}\"" - -if [ ! -d "${LIBYAML}" ]; then - git clone git@github.com:yaml/libyaml.git -fi - -export _C89_ACCEPTABLE_RC=0 - -if ! c89 \ - -c \ - -Wc,dll,expo,langlvl\(extc99\),gonum,goff,hgpr,roconst,ASM,asmlib\('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN'\) \ - -Wc,agg,list\(\),so\(\),off,xref \ - -Wc,ascii,noxplink \ - -D_ENHANCED_ASCII_EXT=0xFFFFFFFF \ - -DYAML_VERSION_MAJOR="${MAJOR}" \ - -DYAML_VERSION_MINOR="${MINOR}" \ - -DYAML_VERSION_PATCH="${PATCH}" \ - -DYAML_VERSION_STRING="${VERSION}" \ - -I "${LIBYAML}/include" \ - ${LIBYAML}/src/api.c \ - ${LIBYAML}/src/reader.c \ - ${LIBYAML}/src/scanner.c \ - ${LIBYAML}/src/parser.c \ - ${LIBYAML}/src/loader.c \ - ${LIBYAML}/src/writer.c \ - ${LIBYAML}/src/emitter.c \ - ${LIBYAML}/src/dumper.c -then - echo "Build failed" - exit 8 -fi - -if c89 \ - -D_XOPEN_SOURCE=600 \ - -DNOIBMHTTP=1 \ - -D_OPEN_THREADS=1 \ - -Wc,dll,expo,langlvl\(extc99\),gonum,goff,hgpr,roconst,ASM,asmlib\('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN'\) \ - -Wc,agg,exp,list\(\),so\(\),off,xref \ - -Wl,dll \ - -I "${COMMON}/h" \ - -I "${COMMON}/platform/posix" \ - -I "${LIBYAML}/include" \ - -o "${COMMON}/bin/configmgr" \ - api.o \ - reader.o \ - scanner.o \ - parser.o \ - loader.o \ - writer.o \ - emitter.o \ - dumper.o \ - ${COMMON}/c/alloc.c \ - ${COMMON}/c/bpxskt.c \ - ${COMMON}/c/charsets.c \ - ${COMMON}/c/collections.c \ - ${COMMON}/c/configmgr.c \ - ${COMMON}/c/json.c \ - ${COMMON}/c/jsonschema.c \ - ${COMMON}/c/le.c \ - ${COMMON}/c/logging.c \ - ${COMMON}/platform/posix/psxregex.c \ - ${COMMON}/c/recovery.c \ - ${COMMON}/c/scheduling.c \ - ${COMMON}/c/timeutls.c \ - ${COMMON}/c/utils.c \ - ${COMMON}/c/xlate.c \ - ${COMMON}/c/yaml2json.c \ - ${COMMON}/c/zos.c \ - ${COMMON}/c/zosfile.c -then - echo "Build successful" - exit 0 -else - # remove configmgr in case the linker had RC=4 and produced the binary - rm -f "${COMMON}/bin/configmgr" - echo "Build failed" - exit 8 -fi - - -# This program and the accompanying materials are -# made available under the terms of the Eclipse Public License v2.0 which accompanies -# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html -# -# SPDX-License-Identifier: EPL-2.0 -# -# Copyright Contributors to the Zowe Project. \ No newline at end of file diff --git a/build/configmgr.proj.env b/build/configmgr.proj.env index 4050346f9..d1b325746 100644 --- a/build/configmgr.proj.env +++ b/build/configmgr.proj.env @@ -3,7 +3,7 @@ VERSION=3.0.0 DEPS="QUICKJS LIBYAML" QUICKJS="quickjs" -QUICKJS_SOURCE="git@github.com:joenemo/quickjs-portable.git" +QUICKJS_SOURCE="git@github.com:JoeNemo/quickjs-portable.git" QUICKJS_BRANCH="main" LIBYAML="libyaml" diff --git a/c/configmgr.c b/c/configmgr.c index 512ec30fa..460ff60c8 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -773,7 +773,7 @@ int cfgSetParmlibMemberName(ConfigManager *mgr, const char *configName, const ch CFGConfig *config = getConfig(mgr,configName); if (config){ int len = strlen(parmlibMemberName); - if (len < 3 || len > PARMLIB_MEMBER_MAX){ + if (len < 1 || len > PARMLIB_MEMBER_MAX){ return ZCFG_BAD_PARMLIB_MEMBER_NAME; } else { config->parmlibMemberName = substring(mgr,(char*)parmlibMemberName,0,len); @@ -827,6 +827,9 @@ static int overloadConfiguration(ConfigManager *mgr, trace(mgr, DEBUG2, "at end of config path\n"); bool dontCare = false; config->configData = readJson(mgr,config,pathElement,&dontCare); + if ((config->configData == NULL) && !dontCare){ + return ZCFG_MISSING_CONFIG_SOURCE; + } trace(mgr, DEBUG2, "mgr->config = 0x%p\n", config); return 0; /* success */ } else { diff --git a/c/crossmemory.c b/c/crossmemory.c index f5f47c673..d7f132f17 100644 --- a/c/crossmemory.c +++ b/c/crossmemory.c @@ -510,19 +510,27 @@ static void getSTCK(uint64 *stckValue) { __asm(" STCK 0(%0)" : : "r"(stckValue)); } -int64 getLocalTimeOffset() { +static int64 getLocalTimeOffset(void) { CVT * __ptr32 cvt = *(void * __ptr32 * __ptr32)0x10; void * __ptr32 cvtext2 = cvt->cvtext2; int64 *cvtldto = (int64 * __ptr32)(cvtext2 + 0x38); return *cvtldto; } +static int64 getLeapSecondsOffset(void) { + CVT * __ptr32 cvt = *(void * __ptr32 * __ptr32)0x10; + void * __ptr32 cvtext2 = cvt->cvtext2; + int64 *cvtlso = (int64 * __ptr32)(cvtext2 + 0x50); + return *cvtlso; +} + static void getCurrentLogTimestamp(LogTimestamp *timestamp) { uint64 stck = 0; getSTCK(&stck); stck += getLocalTimeOffset(); + stck -= getLeapSecondsOffset(); stckToLogTimestamp(stck, timestamp); diff --git a/c/embeddedjs.c b/c/embeddedjs.c index aad75a32d..e2697fc0d 100644 --- a/c/embeddedjs.c +++ b/c/embeddedjs.c @@ -612,7 +612,6 @@ uint8_t *js_load_file(JSContext *ctx, size_t *pbuf_len, const char *filename); /* (filename, ccsid) ccsid -1 implies guess best for platform, 0 implies don't translate */ static JSValue xplatformLoadFileUTF8(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv){ - size_t size; size_t length; char *buf; @@ -626,7 +625,9 @@ static JSValue xplatformLoadFileUTF8(JSContext *ctx, JSValueConst this_val, JS_ToInt32(ctx, &sourceCCSID, argv[1]); buf = (char*)js_load_file(ctx, &length, filename); - + if (!buf) { + return JS_EXCEPTION; + } if (sourceCCSID < 0){ char *nativeBuffer = safeMalloc(length+1,"xplatformStringFromBytes"); memcpy(nativeBuffer,buf,length); @@ -678,7 +679,6 @@ static int appendToFile(char *filename, const char *data, int length) { /* (filename, ccsid) ccsid -1 implies guess best for platform, 0 implies don't translate */ static JSValue xplatformAppendFileUTF8(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv){ - size_t size; size_t length; NATIVE_STR(filename,nativeFilename,0); @@ -718,7 +718,6 @@ static JSValue xplatformAppendFileUTF8(JSContext *ctx, JSValueConst this_val, /* (filename, ccsid) ccsid -1 implies guess best for platform, 0 implies don't translate */ static JSValue xplatformStoreFileUTF8(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv){ - size_t size; size_t length; NATIVE_STR(filename,nativeFilename,0); diff --git a/c/httpfileservice.c b/c/httpfileservice.c index b383b9f59..45d35baa4 100644 --- a/c/httpfileservice.c +++ b/c/httpfileservice.c @@ -935,11 +935,12 @@ int writeAsciiDataFromBase64(UnixFile *file, char *fileContents, int contentLeng &reasonCode); if (status == 0) { int writtenLength = 0; + char *currentBufferStart = dataToWrite; while (dataSize != 0) { - writtenLength = fileWrite(file, dataToWrite, dataSize, &returnCode, &reasonCode); + writtenLength = fileWrite(file, currentBufferStart, dataSize, &returnCode, &reasonCode); if (writtenLength >= 0) { dataSize -= writtenLength; - dataToWrite -= writtenLength; + currentBufferStart += writtenLength; } else { zowelog(NULL, LOG_COMP_RESTFILE, ZOWE_LOG_DEBUG, "Error writing to file: return: %d, rsn: %d.\n", returnCode, reasonCode); diff --git a/c/httpserver.c b/c/httpserver.c index 109666951..c3cf7d9c5 100644 --- a/c/httpserver.c +++ b/c/httpserver.c @@ -1564,6 +1564,7 @@ HttpServer *makeHttpServerInner(STCBase *base, memcpy(&server->config->sessionTokenKey[0], &sessionTokenKey, sizeof (sessionTokenKey)); server->config->authTokenType = SERVICE_AUTH_TOKEN_TYPE_LEGACY; + server->config->httpRequestHeapMaxBlocks = HTTP_REQUEST_HEAP_DEFAULT_BLOCKS; return server; } @@ -1656,6 +1657,8 @@ static HttpServer *makeSecureHttpServerInner(STCBase *base, int port, int64 now = getFineGrainedTime(); server->config->sessionTokenKeySize = sizeof (now); memcpy(&server->config->sessionTokenKey[0], &now, sizeof (now)); + server->config->httpRequestHeapMaxBlocks = HTTP_REQUEST_HEAP_DEFAULT_BLOCKS; + return server; } #endif @@ -3210,10 +3213,10 @@ static int serviceAuthNativeWithSessionToken(HttpService *service, HttpRequest * request->flags = HTTP_REQUEST_NO_PASSWORD; authDataFound = TRUE; } else { - zowelog(NULL, LOG_COMP_HTTPSERVER, ZOWE_LOG_INFO, "No user was found for client certificate. (rc = 0x%x racfRC = 0x%x racfRSN = 0x%x)\n", safReturnCode, racfReturnCode, racfReasonCode); + zowelog(NULL, LOG_COMP_HTTPSERVER, ZOWE_LOG_DEBUG, "No user was found for client certificate. (rc = 0x%x racfRC = 0x%x racfRSN = 0x%x)\n", safReturnCode, racfReturnCode, racfReasonCode); } } else { - zowelog(NULL, LOG_COMP_HTTPSERVER, ZOWE_LOG_INFO, "Client certificate was attached to request, but credentials are also attached. Server won't attempt to map the client certificate.\n"); + zowelog(NULL, LOG_COMP_HTTPSERVER, ZOWE_LOG_DEBUG, "Client certificate was attached to request, but credentials are also attached. Server won't attempt to map the client certificate.\n"); } } @@ -4637,8 +4640,9 @@ static int streamBinaryForFile2(HttpResponse *response, Socket *socket, UnixFile if (encoding == ENCODING_CHUNKED) { stream = makeChunkedOutputStreamInternal(response); } - - int bufferSize = FILE_STREAM_BUFFER_SIZE; + + // To make bufferSize divisble by 3 for correct base64 encoding. + int bufferSize = FILE_STREAM_BUFFER_SIZE - (FILE_STREAM_BUFFER_SIZE % 3); char *buffer = safeMalloc(bufferSize+4, "streamBinaryBuffer"); int encodedLength; @@ -4710,7 +4714,8 @@ static int streamTextForFile2(HttpResponse *response, Socket *socket, UnixFile * stream = makeChunkedOutputStreamInternal(response); /* fallthrough */ case ENCODING_SIMPLE: { - int bufferSize = FILE_STREAM_BUFFER_SIZE; + // To make bufferSize divisble by 3 for correct base64 encoding. + int bufferSize = FILE_STREAM_BUFFER_SIZE - (FILE_STREAM_BUFFER_SIZE % 3); char *buffer = safeMalloc(bufferSize+4, "streamTextBuffer"); char *translation = safeMalloc((2*bufferSize)+4, "streamTextConvertBuffer"); /* UTF inflation tolerance */ int encodedLength; @@ -5819,7 +5824,9 @@ static int httpHandleTCP(STCBase *base, } } #endif // USE_ZOWE_TLS - ShortLivedHeap *slh = makeShortLivedHeap(READ_BUFFER_SIZE,100); + HttpServer *server = (HttpServer*) module->data; + unsigned int maxBlocks = server->config->httpRequestHeapMaxBlocks; + ShortLivedHeap *slh = makeShortLivedHeap(READ_BUFFER_SIZE, maxBlocks); #ifndef __ZOWE_OS_WINDOWS int writeBufferSize = 0x40000; setSocketWriteBufferSize(peerSocket,0x40000, &returnCode, &reasonCode); diff --git a/c/qjszos.c b/c/qjszos.c index d62529013..f2cf738d8 100644 --- a/c/qjszos.c +++ b/c/qjszos.c @@ -154,6 +154,41 @@ static JSValue zosChangeStreamCCSID(JSContext *ctx, JSValueConst this_val, return JS_NewInt64(ctx,(int64_t)status); } +static JSValue getZosVersion(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv){ + +#ifdef __ZOWE_OS_ZOS + ECVT *ecvt = getECVT(); + int version = ecvt->ecvtpseq; +#else + int version = -1; +#endif + return JS_NewInt64(ctx,(int64_t)version); +} + + +static JSValue getEsm(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv){ + +#ifdef __ZOWE_OS_ZOS + switch(getExternalSecurityManager()) { + case ZOS_ESM_RTSS: + return newJSStringFromNative(ctx, "TSS", 3); + case ZOS_ESM_RACF: + return newJSStringFromNative(ctx, "RACF", 4); + case ZOS_ESM_ACF2: + return newJSStringFromNative(ctx, "ACF2", 4); + case ZOS_ESM_NONE: + return newJSStringFromNative(ctx, "NONE", 4); + default: + return JS_NewString(ctx, NULL); + } +#else + return JS_NewString(ctx, NULL); +#endif + +} + /* return [obj, errcode] */ static JSValue zosStat(JSContext *ctx, JSValueConst this_val, @@ -377,6 +412,10 @@ static const char changeStreamCCSIDASCII[18] ={ 0x63, 0x68, 0x61, 0x6e, 0x67, 0x 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x43, 0x53, 0x49, 0x44, 0x00}; static const char zstatASCII[6] ={ 0x7A, 0x73, 0x74, 0x61, 0x74, 0}; +static const char getZosVersionASCII[14] ={ 0x67, 0x65, 0x74, 0x5A, 0x6F, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6F, 0x6E, 0x00}; + +static const char getEsmASCII[7] = { 0x67, 0x65, 0x74, 0x45, 0x73, 0x6D, 0x00 }; + static const char EXTATTR_SHARELIB_ASCII[17] = { 0x45, 0x58, 0x54, 0x41, 0x54, 0x54, 0x52, 0x5f, 0x53, 0x48, 0x41, 0x52, 0x45, 0x4c, 0x49, 0x42, 0x0}; @@ -398,6 +437,8 @@ static const JSCFunctionListEntry zosFunctions[] = { JS_CFUNC_DEF(changeExtAttrASCII, 3, zosChangeExtAttr), JS_CFUNC_DEF(changeStreamCCSIDASCII, 2, zosChangeStreamCCSID), JS_CFUNC_DEF(zstatASCII, 1, zosStat), + JS_CFUNC_DEF(getZosVersionASCII, 0, getZosVersion), + JS_CFUNC_DEF(getEsmASCII, 0, getEsm), JS_CFUNC_DEF(dslistASCII, 1, zosDatasetInfo), JS_CFUNC_DEF(resolveSymbolASCII, 1, zosResolveSymbol), JS_PROP_INT32_DEF(EXTATTR_SHARELIB_ASCII, EXTATTR_SHARELIB, JS_PROP_CONFIGURABLE ), diff --git a/c/recovery.c b/c/recovery.c index fd1ff5611..2c5da2a0a 100644 --- a/c/recovery.c +++ b/c/recovery.c @@ -497,7 +497,7 @@ static void * __ptr32 getRecoveryRouterAddress() { " LA 1,RCVXINF LOAD ROUTER SERVICE INFO \n" " BRAS 14,RCVSIFLB RECORD IT, REMOVE CONTEXT, PERCOLATE \n" " TM RCXFLAG1,R@CF1USP USER STATE POOL? \n" - " BZ RCVFRL04 NO, DO NOT FREE IT \n" + " BNZ RCVFRL04 NO, DO NOT FREE IT \n" " LT 2,RCXSCPID CELL POOL ZERO? \n" " BZ RCVFRL04 YES, DO NOT FREE IT \n" #ifdef _LP64 @@ -505,6 +505,8 @@ static void * __ptr32 getRecoveryRouterAddress() { " SYSSTATE AMODE64=NO \n" #endif " CPOOL DELETE,CPID=(2) FREE THE STATE CELL POOL \n" + " LGFI 2,X'7FFFFBA3' MAKE AN OBVIOUSLY BAD ADDRESS \n" + " ST 2,RCXSCPID MARK THE CPID FOR DEBUGGING PURPOSES \n" #ifdef _LP64 " SAM64 \n" " SYSSTATE AMODE64=YES \n" @@ -1291,6 +1293,8 @@ RecoveryStatePool *recoveryMakeStatePool(unsigned int stateCount) { void recoveryRemoveStatePool(RecoveryStatePool *statePool) { removeRecoveryStatePool(statePool->cellPool); + // put a bad address for debugging (in case the statePool storage survives) + statePool->cellPool = 0x7FFFFBA1; storageRelease(statePool, sizeof(RecoveryStatePool)); } diff --git a/c/yaml2json.c b/c/yaml2json.c index aabc08210..9af3e5239 100644 --- a/c/yaml2json.c +++ b/c/yaml2json.c @@ -396,7 +396,9 @@ static bool isSyntacticallyBool(yaml_char_t *data, int length){ } static bool isSyntacticallyNull(yaml_char_t *data, int length){ - if ((length == 1) && + if (length == 0 ) { + return true; + } else if ((length == 1) && !memcmp(data,"~",1)){ return true; } else if ((length == 4) && diff --git a/h/httpserver.h b/h/httpserver.h index 09f0d2de0..2cbbd6610 100644 --- a/h/httpserver.h +++ b/h/httpserver.h @@ -64,6 +64,10 @@ #define HTTP_SERVER_PRIVILEGED_SERVER_PROPERTY "zisServerName" +#define HTTP_REQUEST_HEAP_DEFAULT_BLOCKS 1024 +#define HTTP_REQUEST_HEAP_MIN_BLOCKS 100 +#define HTTP_REQUEST_HEAP_MAX_BLOCKS 4096 + typedef struct BigBuffer_tag{ ShortLivedHeap *slh; /* can be null */ char *data; @@ -220,6 +224,7 @@ typedef struct HTTPServerConfig_tag { hashtable *userTimeouts; hashtable *groupTimeouts; int defaultTimeout; + unsigned int httpRequestHeapMaxBlocks; /* The config manager is optional, but zss and other servers need a near-global way to get configuration data. */