diff --git a/bin/.gitignore b/bin/.gitignore new file mode 100644 index 000000000..16a3e70e1 --- /dev/null +++ b/bin/.gitignore @@ -0,0 +1,18 @@ +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. + +* +!.gitignore + +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. \ No newline at end of file diff --git a/build/.gitignore b/build/.gitignore new file mode 100644 index 000000000..1f13f011a --- /dev/null +++ b/build/.gitignore @@ -0,0 +1,17 @@ +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. + +tmp* + +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. \ No newline at end of file diff --git a/build/build_cmgr_xlclang.sh b/build/build_cmgr_xlclang.sh new file mode 100755 index 000000000..b8f7f4d1f --- /dev/null +++ b/build/build_cmgr_xlclang.sh @@ -0,0 +1,143 @@ +#!/bin/sh + +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. + +WORKING_DIR=$(dirname "$0") + +# set -v + +echo "********************************************************************************" +echo "Building configmgr..." + +# These paths assume that the build is run from /zss/deps/zowe-common-c/builds + + + +mkdir -p "${WORKING_DIR}/tmp-configmgr" && cd "$_" + +COMMON="../.." +QUICKJS="../../../../../quickjs" +LIBYAML="../../../../../libyaml" + +rm -f "${COMMON}/bin/configmgr" + +MAJOR=0 +MINOR=2 +PATCH=5 +VERSION="\"${MAJOR}.${MINOR}.${PATCH}\"" + +#if [ ! -d "${LIBYAML}" ]; then +# git clone git@github.com:yaml/libyaml.git +#fi + +# export _C89_ACCEPTABLE_RC=4 + +xlclang \ + -c \ + -q64 \ + -qascii \ + "-Wc,float(ieee),longname,langlvl(extc99),gonum,goff,ASM,asmlib('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN')" \ + -DYAML_VERSION_MAJOR=${MAJOR} \ + -DYAML_VERSION_MINOR=${MINOR} \ + -DYAML_VERSION_PATCH=${PATCH} \ + -DYAML_VERSION_STRING="${VERSION}" \ + -DYAML_DECLARE_STATIC=1 \ + -D_OPEN_SYS_FILE_EXT=1 \ + -D_XOPEN_SOURCE=600 \ + -D_OPEN_THREADS=1 \ + -DCONFIG_VERSION=\"2021-03-27\" \ + -I "${LIBYAML}/include" \ + -I "${QUICKJS}" \ + ${LIBYAML}/src/api.c \ + ${LIBYAML}/src/reader.c \ + ${LIBYAML}/src/scanner.c \ + ${LIBYAML}/src/parser.c \ + ${LIBYAML}/src/loader.c \ + ${LIBYAML}/src/writer.c \ + ${LIBYAML}/src/emitter.c \ + ${LIBYAML}/src/dumper.c \ + ${QUICKJS}/cutils.c \ + ${QUICKJS}/quickjs.c \ + ${QUICKJS}/quickjs-libc.c \ + ${QUICKJS}/libunicode.c \ + ${QUICKJS}/libregexp.c \ + ${QUICKJS}/porting/polyfill.c +#then +# echo "Done with qascii-compiled open-source parts" +#else +# echo "Build failed" +# exit 8 +#fi + +xlclang \ + -q64 \ + "-Wc,float(ieee),longname,langlvl(extc99),gonum,goff,ASM,asmlib('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN')" \ + -D_OPEN_SYS_FILE_EXT=1 \ + -D_XOPEN_SOURCE=600 \ + -D_OPEN_THREADS=1 \ + -DNOIBMHTTP=1 \ + -I "${COMMON}/h" \ + -I "${COMMON}/platform/posix" \ + -I "${LIBYAML}/include" \ + -I "${QUICKJS}" \ + -o "${COMMON}/bin/configmgr" \ + api.o \ + reader.o \ + scanner.o \ + parser.o \ + loader.o \ + writer.o \ + emitter.o \ + dumper.o \ + cutils.o \ + quickjs.o \ + quickjs-libc.o \ + libunicode.o \ + libregexp.o \ + polyfill.o \ + ${COMMON}/c/alloc.c \ + ${COMMON}/c/bpxskt.c \ + ${COMMON}/c/charsets.c \ + ${COMMON}/c/collections.c \ + ${COMMON}/c/configmgr.c \ + ${COMMON}/c/embeddedjs.c \ + ${COMMON}/c/json.c \ + ${COMMON}/c/jsonschema.c \ + ${COMMON}/c/le.c \ + ${COMMON}/c/logging.c \ + ${COMMON}/c/microjq.c \ + ${COMMON}/c/parsetools.c \ + ${COMMON}/platform/posix/psxregex.c \ + ${COMMON}/c/recovery.c \ + ${COMMON}/c/scheduling.c \ + ${COMMON}/c/timeutls.c \ + ${COMMON}/c/utils.c \ + ${COMMON}/c/xlate.c \ + ${COMMON}/c/yaml2json.c \ + ${COMMON}/c/zos.c \ + ${COMMON}/c/zosfile.c +#then +# echo "Build successful" +# ls -l "${COMMON}/bin" +# exit 0 +#else +# # remove configmgr in case the linker had RC=4 and produced the binary +# rm -f "${COMMON}/bin/configmgr" +# echo "Build failed" +# exit 8 +#fi + + +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. diff --git a/build/build_configmgr.sh b/build/build_configmgr.sh new file mode 100755 index 000000000..3a3e8ca40 --- /dev/null +++ b/build/build_configmgr.sh @@ -0,0 +1,112 @@ +#!/bin/sh + +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. + +WORKING_DIR=$(dirname "$0") + + +echo "********************************************************************************" +echo "Building configmgr..." + +rm -f "${COMMON}/bin/configmgr" + +mkdir -p "${WORKING_DIR}/tmp-configmgr" && cd "$_" +COMMON="../.." + +LIBYAML="libyaml" +MAJOR=0 +MINOR=2 +PATCH=5 +VERSION="\"${MAJOR}.${MINOR}.${PATCH}\"" + +if [ ! -d "${LIBYAML}" ]; then + git clone git@github.com:yaml/libyaml.git +fi + +export _C89_ACCEPTABLE_RC=0 + +if ! c89 \ + -c \ + -Wc,dll,expo,langlvl\(extc99\),gonum,goff,hgpr,roconst,ASM,asmlib\('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN'\) \ + -Wc,agg,list\(\),so\(\),off,xref \ + -Wc,ascii,noxplink \ + -D_ENHANCED_ASCII_EXT=0xFFFFFFFF \ + -DYAML_VERSION_MAJOR="${MAJOR}" \ + -DYAML_VERSION_MINOR="${MINOR}" \ + -DYAML_VERSION_PATCH="${PATCH}" \ + -DYAML_VERSION_STRING="${VERSION}" \ + -I "${LIBYAML}/include" \ + ${LIBYAML}/src/api.c \ + ${LIBYAML}/src/reader.c \ + ${LIBYAML}/src/scanner.c \ + ${LIBYAML}/src/parser.c \ + ${LIBYAML}/src/loader.c \ + ${LIBYAML}/src/writer.c \ + ${LIBYAML}/src/emitter.c \ + ${LIBYAML}/src/dumper.c +then + echo "Build failed" + exit 8 +fi + +if c89 \ + -D_XOPEN_SOURCE=600 \ + -DNOIBMHTTP=1 \ + -D_OPEN_THREADS=1 \ + -Wc,dll,expo,langlvl\(extc99\),gonum,goff,hgpr,roconst,ASM,asmlib\('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN'\) \ + -Wc,agg,exp,list\(\),so\(\),off,xref \ + -Wl,dll \ + -I "${COMMON}/h" \ + -I "${COMMON}/platform/posix" \ + -I "${LIBYAML}/include" \ + -o "${COMMON}/bin/configmgr" \ + api.o \ + reader.o \ + scanner.o \ + parser.o \ + loader.o \ + writer.o \ + emitter.o \ + dumper.o \ + ${COMMON}/c/alloc.c \ + ${COMMON}/c/bpxskt.c \ + ${COMMON}/c/charsets.c \ + ${COMMON}/c/collections.c \ + ${COMMON}/c/configmgr.c \ + ${COMMON}/c/json.c \ + ${COMMON}/c/jsonschema.c \ + ${COMMON}/c/le.c \ + ${COMMON}/c/logging.c \ + ${COMMON}/platform/posix/psxregex.c \ + ${COMMON}/c/recovery.c \ + ${COMMON}/c/scheduling.c \ + ${COMMON}/c/timeutls.c \ + ${COMMON}/c/utils.c \ + ${COMMON}/c/xlate.c \ + ${COMMON}/c/yaml2json.c \ + ${COMMON}/c/zos.c \ + ${COMMON}/c/zosfile.c +then + echo "Build successful" + exit 0 +else + # remove configmgr in case the linker had RC=4 and produced the binary + rm -f "${COMMON}/bin/configmgr" + echo "Build failed" + exit 8 +fi + + +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. \ No newline at end of file diff --git a/c/alloc.c b/c/alloc.c index f6f1f567c..33742901a 100644 --- a/c/alloc.c +++ b/c/alloc.c @@ -518,8 +518,10 @@ static char *safeMalloc64Internal(int size, char *site, long long token){ } #if defined(METTLE) && defined(_LP64) res = getmain64((long long)sizeInMegabytes,NULL,NULL); -#elif defined(_LP64) /* LE case */ +#elif defined(_LP64) /* LE case */ res = malloc(size); /* According to Redpaper http://www.redbooks.ibm.com/redpapers/pdfs/redp9110.pdf allocated above bar */ +#elif defined(_MSC_VER) && defined(_M_X64) /* Windows 64 case */ + res = malloc(size); #else res = NULL; /* Invalid if not compiled _LP64 */ #endif diff --git a/c/as.c b/c/as.c index bc1585c25..3ab42e77f 100644 --- a/c/as.c +++ b/c/as.c @@ -260,9 +260,9 @@ int addressSpaceCreateWithTerm(const ASParmString *startParmString, void * __ptr32 termCallbackParm, int *reasonCode) { - __packed union { + _Packed union { ASUserToken tokenValue; - __packed struct { + _Packed struct { ASCRETermCallback * __ptr32 termCallback; void * __ptr32 termCallbackParm; }; diff --git a/c/bpxskt.c b/c/bpxskt.c index fa8d74d8a..2893ac6bd 100644 --- a/c/bpxskt.c +++ b/c/bpxskt.c @@ -107,6 +107,28 @@ #endif +/* xlclang and clang what prototypes, these are incomplete, but quiet the compiler */ +int BPXSOC(); +int BPXCON(); +int BPXGHN(); +int BPXSLP(); +int BPXCHR(); +int BPXBND(); +int BPXLSN(); +int BPXACP(); +int BPXSEL(); +int BPXOPT(); +int BPXGNM(); +int BPXSTO(); +int BPXRFM(); +int BPXHST(); +int BPXIOC(); +int BPXRED(); +int BPXWRT(); +int BPXFCT(); +int BPXCLO(); + + #define SOCK_SO_REUSEADDR 0x00000004 #define SOCK_SO_SNDBUF 0x00001001 #define SOCK_SO_RCVBUF 0x00001002 @@ -151,6 +173,7 @@ void bpxSleep(int seconds) int socketInit(char *uniqueName){ /* do nothing for now */ + return 0; } SocketAddress *makeSocketAddr(InetAddr *addr, @@ -172,7 +195,7 @@ SocketAddress *makeSocketAddrIPv6(InetAddr *addr, unsigned short port){ SocketAddress *address = (SocketAddress*)safeMalloc31(sizeof(SocketAddress),"BPX SocketAddress"); memset(address,0,sizeof(SocketAddress)); if (socketTrace){ - printf("socket address at 0x%x\n",address); + printf("socket address at 0x%p\n",address); } address->length = 26; address->family = AF_INET6; @@ -181,7 +204,7 @@ SocketAddress *makeSocketAddrIPv6(InetAddr *addr, unsigned short port){ address->data6 = addr->data.data6; } if (socketTrace){ - printf("about to return socket address at 0x%x\n",address); + printf("about to return socket address at 0x%p\n",address); } return address; } @@ -424,6 +447,9 @@ Socket *udpPeer(SocketAddress *socketAddress, } } +/* Forward decl */ +static int setSocketReuseAddr(int sd, int *returnCode, int *reasonCode); + Socket *tcpServer2(InetAddr *addr, int port, int tlsFlags, @@ -498,7 +524,7 @@ Socket *tcpServer2(InetAddr *addr, dumpbuffer((char*)socketAddress,sizeof(SocketAddress)); dumpbuffer((char*)addr,sizeof(InetAddr)); } - setSocketReuseAddr(&sd, + setSocketReuseAddr(sd, returnCode, reasonCodePtr); if (returnValue != 0){ @@ -744,7 +770,7 @@ int getV4HostByName(char *string){ /* dumpbuffer((char*)hostent,20); */ for (i=0; ilength; i++){ if (socketTrace){ - printf(" addr[%d] = %x\n",i,hostent->addrList[i]); + printf(" addr[%d] = 0x%p\n",i,hostent->addrList[i]); } if (hostent->addrList[i]){ numericAddress = *(hostent->addrList[i]); @@ -1207,10 +1233,9 @@ int getSocketOption(Socket *socket, int optionName, int *optionDataLength, char } } -int setSocketOption(Socket *socket, int level, int optionName, int optionDataLength, char *optionData, - int *returnCode, int *reasonCode){ +static int setSDOption(int sd, int level, int optionName, int optionDataLength, char *optionData, + int *returnCode, int *reasonCode){ int status = 0; - int sd = socket->sd; int returnValue = 0; *returnCode = *reasonCode = 0; int *reasonCodePtr; @@ -1243,6 +1268,12 @@ int setSocketOption(Socket *socket, int level, int optionName, int optionDataLen } } +int setSocketOption(Socket *socket, int level, int optionName, int optionDataLength, char *optionData, + int *returnCode, int *reasonCode){ + return setSDOption(socket->sd,level,optionName,optionDataLength,optionData,returnCode,reasonCode); +} + + int setSocketNoDelay(Socket *socket, int noDelay, int *returnCode, int *reasonCode){ return setSocketOption(socket,IPPROTO_TCP,TCP_NODELAY,sizeof(int),(char*)&noDelay,returnCode,reasonCode); } @@ -1255,9 +1286,9 @@ int setSocketReadBufferSize(Socket *socket, int bufferSize, int *returnCode, int return setSocketOption(socket,SOL_SOCKET,SOCK_SO_RCVBUF,sizeof(int),(char*)&bufferSize,returnCode,reasonCode); } -int setSocketReuseAddr(Socket *socket, int *returnCode, int *reasonCode){ +static int setSocketReuseAddr(int sd, int *returnCode, int *reasonCode){ int on = 1; - return setSocketOption(socket,SOL_SOCKET,SOCK_SO_REUSEADDR,sizeof(int),(char*)&on,returnCode,reasonCode); + return setSDOption(sd,SOL_SOCKET,SOCK_SO_REUSEADDR,sizeof(int),(char*)&on,returnCode,reasonCode); } int udpReceiveFrom(Socket *socket, @@ -1276,7 +1307,7 @@ int udpReceiveFrom(Socket *socket, int socketAddressSize = SOCKET_ADDRESS_SIZE_IPV4; if (socketTrace > 2){ - printf("receiveFrom into buffer=0x%x bufLen=%d retVal=%d retCode=%d reasonCode=%d\n", + printf("receiveFrom into buffer=0x%p bufLen=%d retVal=%d retCode=%d reasonCode=%d\n", buffer,bufferLength,returnValue,*returnCode,*reasonCode); } @@ -1300,7 +1331,7 @@ int udpReceiveFrom(Socket *socket, return -1; } else { if (socketTrace > 2){ - printf("recvFrom into buffer=0x%x %d bytes\n",buffer,returnValue);fflush(stdout); + printf("recvFrom into buffer=0x%p %d bytes\n",buffer,returnValue);fflush(stdout); } *returnCode = 0; *reasonCode = 0; @@ -1680,7 +1711,7 @@ int not_main(int argc, char **argv){ sleep(1); } } - + return 0; } diff --git a/c/charsets.c b/c/charsets.c index a9d1947fc..6db31041b 100644 --- a/c/charsets.c +++ b/c/charsets.c @@ -35,6 +35,10 @@ #ifdef __ZOWE_OS_WINDOWS +/* JOE 1/20/22 */ +#include + + /* Windows doc @@ -114,7 +118,7 @@ int convertCharset(char *input, return 0; } -#elif defined(__ZOWE_OS_ZOS) +#elif defined(__ZOWE_OS_ZOS) && !defined(__ZOWE_COMP_XLCLANG) /* @@ -163,7 +167,11 @@ static const union { }; +#ifdef __ZOWE_COMP_XLCLANG +#include "CUNHC.h" +#else #include "//'SYS1.SCUNHF(CUNHC)'" +#endif int convertCharset(char *input, int inputLength, @@ -219,6 +227,7 @@ int convertCharset(char *input, case CHARSET_OUTPUT_USE_SLH: outputBuffer = SLHAlloc(slh,outputAllocLength); outputLength = outputAllocLength; + *output = outputBuffer; break; } parms.Targ_Buf_Ptr = outputBuffer; @@ -249,11 +258,15 @@ int convertCharset(char *input, #endif *conversionOutputLength = (((char*)parms.Targ_Buf_Ptr) - outputBuffer); + printf("inputLen=%d reasonCode = %d src=%d targ=%d\n",inputLength,parms.Reason_Code,parms.Src_CCSID,parms.Targ_CCSID); + fflush(stdout); + if (parms.Return_Code){ if (outputMode == CHARSET_OUTPUT_SAFE_MALLOC){ safeFree(parms.Targ_Buf_Ptr,outputAllocLength); } *reasonCode = parms.Return_Code; + return CHARSET_CONVERSION_ROUTINE_FAILURE; } else { if (outputMode == CHARSET_OUTPUT_SAFE_MALLOC){ @@ -306,7 +319,9 @@ int getCharsetCode(const char *charsetName) { } } -#elif defined(__ZOWE_OS_LINUX) || defined(__ZOWE_OS_AIX) /* end of ZOWE_OS_ZOS */ +/* End of Traditional METAL and XLC cases, since linkage(OS64_NOSTACK) doesn't work in xlclang and clang + some C code goes through here, too. We should short circuit easy special cases here some day */ +#elif defined(__ZOWE_OS_LINUX) || defined(__ZOWE_OS_AIX) || (defined(__ZOWE_OS_ZOS) && defined(__ZOWE_COMP_XLCLANG)) #include #include @@ -387,6 +402,7 @@ int convertCharset(char *input, case CHARSET_OUTPUT_USE_SLH: outputBuffer = SLHAlloc(slh,outputAllocLength); outputSize = outputAllocLength; + *output = outputBuffer; break; default: return CHARSET_INTERNAL_ERROR; diff --git a/c/cmutils.c b/c/cmutils.c index 48c68a6b6..dd3b5b46c 100644 --- a/c/cmutils.c +++ b/c/cmutils.c @@ -26,6 +26,7 @@ #include "cmutils.h" #include "zos.h" + static int getCallersKey() { int key = 0; @@ -44,9 +45,9 @@ static int getCallersKey() { return key; } -#define IS_LE64 (!defined(METTLE) && defined(_LP64)) +/* #define IS_LE64 (!defined(METTLE) && defined(_LP64)) */ -#if !IS_LE64 +#if !(!defined(METTLE) && defined(_LP64)) /* Disable these for LE 64-bit for now, the compile complains about GPR 4 */ diff --git a/c/collections.c b/c/collections.c index 34d4e155b..ce9a5fb97 100644 --- a/c/collections.c +++ b/c/collections.c @@ -516,13 +516,13 @@ void htDump(hashtable *ht){ for (i=0; ibackboneSize; i++){ hashentry *entry = ht->backbone[i]; if (entry != NULL){ - printf("in slot %d: entry=%x\n",i,entry); + printf("in slot %d: entry=0x%p\n",i,entry); fflush(stdout); while (entry != NULL){ if (isString){ - printf(" key=%x '%s' value: %x\n",entry->key,entry->key,entry->value); + printf(" key=0x%p '%s' value: 0x%p\n",entry->key,(char*)entry->key,entry->value); } else{ - printf(" key=%x value: %x\n",entry->key,entry->value); + printf(" key=0x%p value: 0x%p\n",entry->key,entry->value); } fflush(stdout); entry = entry->next; @@ -793,7 +793,7 @@ void lhtMap(LongHashtable *ht, void (*visitor)(void *, int64, void *), void *use static void lruHashVisitor(void *key, void *value){ LRUElement *element = (LRUElement*)value; - printf(" %16.16s: 0x%x containing 0x%x\n",key,value,element->data); + printf(" %16.16s: 0x%p containing 0x%p\n",(char*)key,value,element->data); } LRUCache *makeLRUCache(int size){ @@ -822,13 +822,13 @@ void lruDump(LRUCache *cache){ LRUElement *element = cache->newest; printf(" Newest To Oldest:\n"); while (element){ - printf(" Elt: %16.16s -> 0x%x\n",element->digest,element->data); + printf(" Elt: %16.16s -> 0x%p\n",(char*)element->digest,element->data); element = element->older; } element = cache->oldest; printf(" Oldest To Newest:\n"); while (element){ - printf(" Elt: %16.16s -> 0x%x\n",element->digest,element->data); + printf(" Elt: %16.16s -> 0x%p\n",(char*)element->digest,element->data); element = element->newer; } printf(" In Hash Order:\n"); @@ -838,7 +838,7 @@ void lruDump(LRUCache *cache){ /* can change recency */ void *lruGet(LRUCache *cache, char *digest){ if (cache->trace){ - printf("LRU Get digest: 0x%x\n",digest); + printf("LRU Get digest: 0x%p\n",digest); dumpbuffer(digest,16); } LRUElement *element = (LRUElement*)htGet(cache->ht,digest); @@ -868,14 +868,14 @@ static LRUElement *allocLRUElement(LRUCache *cache){ */ void *lruStore(LRUCache *cache, char *digest, void *thing){ if (cache->trace){ - printf("LRU Store digest: 0x%x\n",digest); + printf("LRU Store digest: 0x%p\n",digest); dumpbuffer(digest,16); } hashtable *ht = cache->ht; LRUElement *existingElement = htGet(ht,digest); if (cache->trace){ - printf("lruStore existing = 0x%x\n",existingElement); + printf("lruStore existing = 0x%p\n",existingElement); } if (existingElement){ if (existingElement == cache->newest){ @@ -886,7 +886,7 @@ void *lruStore(LRUCache *cache, char *digest, void *thing){ } else if (existingElement == cache->oldest){ LRUElement *newOldest = cache->oldest->newer; if (cache->trace){ - printf("recaching oldest: rotate to front, newOldest = 0x%x\n",newOldest); + printf("recaching oldest: rotate to front, newOldest = 0x%p\n",newOldest); } existingElement->newer = NULL; @@ -903,7 +903,7 @@ void *lruStore(LRUCache *cache, char *digest, void *thing){ LRUElement *prev = existingElement->older; LRUElement *next = existingElement->newer; if (cache->trace){ - printf("recaching middle, prun between older=0x%x and newer 0x%x\n", + printf("recaching middle, prun between older=0x%p and newer 0x%p\n", prev->data,next->data); } @@ -924,14 +924,14 @@ void *lruStore(LRUCache *cache, char *digest, void *thing){ if (cache->oldest && cache->newest){ LRUElement *oldNewest = cache->newest; if (cache->trace){ - printf(">1 room case newest->data=0x%x\n",oldNewest->data); + printf(">1 room case newest->data=0x%p\n",oldNewest->data); } newElement->older = cache->newest; cache->newest = newElement; oldNewest->newer = newElement; } else{ if (cache->trace){ - printf("== 0 case, go for it, newElement=0x%x\n",newElement); + printf("== 0 case, go for it, newElement=0x%p\n",newElement); } newElement->older = NULL; cache->newest = newElement; @@ -946,7 +946,7 @@ void *lruStore(LRUCache *cache, char *digest, void *thing){ LRUElement *secondOldest = oldOldest->newer; LRUElement *recycledElement = (LRUElement*)htGet(ht,oldOldest->digest); if (cache->trace){ - printf("recycle case oldOldest=0x%x recycled = 0x%x\n",oldOldest,recycledElement); + printf("recycle case oldOldest=0x%p recycled = 0x%p\n",oldOldest,recycledElement); } void *decached = recycledElement->data; htRemove(ht,oldOldest->digest); @@ -965,7 +965,7 @@ void *lruStore(LRUCache *cache, char *digest, void *thing){ LRUElement *recycledElement = cache->oldest; if (cache->trace){ - printf("recycle case (size=1) recycled = 0x%x\n",recycledElement); + printf("recycle case (size=1) recycled = 0x%p\n",recycledElement); } void *decached = recycledElement->data; htRemove(ht,recycledElement->digest); @@ -1101,7 +1101,7 @@ static int compareAndLoad(long *oldCounter, long *counterAddress, long *sourceAd : "r"(counterAddress), "r"(sourceAddress), "r"(resultAddress), "i"(sizeof(oldCounter)) : - "r0 r1 r14 r15"); + "r0","r1","r14","r15"); return status; } @@ -1133,7 +1133,7 @@ static int compareAndSwapTriple(long *oldCounter, long newCounter, long *counter : "r"(counterAddress), "r"(parms), "r"(newCounter), "i"(sizeof(oldCounter)) : - "r0 r1 r14 r15"); + "r0","r1","r14","r15"); return status; } @@ -1142,7 +1142,7 @@ void qEnqueue(Queue *q, QueueElement *newElement) { union { long long alignit; CSTSTParms parms; - }; + } vars; newElement->next = NULL; @@ -1197,7 +1197,7 @@ void qEnqueue(Queue *q, QueueElement *newElement) { } else { - memset(&parms,0,sizeof(CSTSTParms)); + memset(&vars.parms,0,sizeof(CSTSTParms)); /* Insert a queue element at the tail of a queue using PLO. @@ -1216,17 +1216,17 @@ void qEnqueue(Queue *q, QueueElement *newElement) { void *desiredQHead = (q->head ? q->head : newElement); /* HEAD */ - parms.thing1 = (long)desiredQHead; - parms.thing1Addr = &(q->head); + vars.parms.thing1 = (long)desiredQHead; + vars.parms.thing1Addr = &(q->head); /* TAIL */ - parms.thing2 = (long)newElement; - parms.thing2Addr = &(q->tail); + vars.parms.thing2 = (long)newElement; + vars.parms.thing2Addr = &(q->tail); /* PENULTIMATE */ - parms.thing3 = (long)newElement; + vars.parms.thing3 = (long)newElement; QueueElement *last = q->tail; - parms.thing3Addr = (q->tail ? &(last->next) : &(q->tail)); + vars.parms.thing3Addr = (q->tail ? &(last->next) : &(q->tail)); - if (compareAndSwapTriple(&lockCounter,newCounter,&q->counter,&parms)){ + if (compareAndSwapTriple(&lockCounter,newCounter,&q->counter,&vars.parms)){ break; } } @@ -1249,7 +1249,7 @@ QueueElement *qDequeue(Queue *q) { union { long long alignit; CSTSTParms parms; - }; + } vars; QueueElement *currentHead; @@ -1312,7 +1312,7 @@ QueueElement *qDequeue(Queue *q) { } else { - memset(&parms,0,sizeof(CSTSTParms)); + memset(&vars.parms,0,sizeof(CSTSTParms)); /* Remove a queue element from the head of a queue using PLO. @@ -1335,18 +1335,18 @@ QueueElement *qDequeue(Queue *q) { void *desiredQTail = (desiredQHead ? q->tail : NULL); /* HEAD */ - parms.thing1 = (long)desiredQHead; - parms.thing1Addr = &(q->head); + vars.parms.thing1 = (long)desiredQHead; + vars.parms.thing1Addr = &(q->head); /* TAIL */ - parms.thing2 = (long)desiredQTail; - parms.thing2Addr = &(q->tail); + vars.parms.thing2 = (long)desiredQTail; + vars.parms.thing2Addr = &(q->tail); /* CURRENT_HEAD */ - /* parms.thing3 = NULL -- was set to NULL when parmData was initialized */ + /* vars.parms.thing3 = NULL -- was set to NULL when parmData was initialized */ QueueElement *last = NULL; - parms.thing3Addr = &(currentHead->next); + vars.parms.thing3Addr = &(currentHead->next); - if (compareAndSwapTriple(&lockCounter,newCounter,&q->counter,&parms)) + if (compareAndSwapTriple(&lockCounter,newCounter,&q->counter,&vars.parms)) break; } } @@ -1417,6 +1417,84 @@ void *qRemove(Queue *q){ #endif /* END OF OS-VARIANT Queue stuff */ +/* The Array List (flexible Array thing that smells like Java and javascript) */ + +static void *arrayListAlloc(ArrayList *list, uint32_t size, char *location){ + if (list->slh){ + return (void*)SLHAlloc(list->slh,size); + } else{ + return safeMalloc(size,location); + } +} + +ArrayList *makeArrayList(void){ + ArrayList *list = (ArrayList*)safeMalloc(sizeof(ArrayList),"ArrayList"); + list->capacity = 8; + list->size = 0; + list->array = (void**)safeMalloc(list->capacity*sizeof(void*),"ArrayListArray"); + list->slh = NULL; + return list; +} + +void arrayListFree(ArrayList *list){ + if (list->slh == NULL){ + safeFree((char*)list->array,list->capacity*sizeof(void*)); + safeFree((char*)list,sizeof(ArrayList)); + } +} + +void initEmbeddedArrayList(ArrayList *list, + ShortLivedHeap *slh){ /* can be null for use safeMalloc - standard heap alloc */ + list->capacity = 8; + list->size = 0; + list->slh = slh; + list->array = (void**)arrayListAlloc(list,list->capacity*sizeof(void*),"ArrayListArray"); +} + +void arrayListAdd(ArrayList *list, void *thing){ + if (list->size == list->capacity){ + int newCapacity = 2*list->capacity; + void** newArray = (void**)arrayListAlloc(list,newCapacity*sizeof(void*),"ArrayListExtend"); + memcpy(newArray,list->array,list->capacity*sizeof(void*)); + if (list->slh == NULL){ + safeFree((char*)list->array,list->capacity*sizeof(void*)); + } + list->array = newArray; + list->capacity = newCapacity; + } + list->array[list->size++] = thing; +} + +void arrayListSort(ArrayList *list, int (*comparator)(const void *a, const void *b)){ + qsort(list->array,list->size,sizeof(void*),comparator); +} + +bool arrayListContains(ArrayList *list, void *element){ + for (int i=0; isize; i++){ + if (list->array[i] == element){ + return true; + } + } + return false; +} + +void *arrayListElement(ArrayList *list, int i){ + if (isize){ + return list->array[i]; + } else{ + return NULL; + } +} + +void *arrayListShallowCopy(ArrayList *source, ArrayList *target){ + target->capacity = source->capacity; + target->size = source->size; + target->slh = source->slh; + target->array = (void**)arrayListAlloc(target,target->capacity*sizeof(void*),"ArrayListArray"); + memcpy(target->array,source->array,target->size*sizeof(void**)); + return target; +} + /* This program and the accompanying materials are diff --git a/c/configmgr.c b/c/configmgr.c new file mode 100644 index 000000000..0e2956a1c --- /dev/null +++ b/c/configmgr.c @@ -0,0 +1,1132 @@ + + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#ifdef METTLE + +#include +#include +#include +#include +#include +#include +#include +#include +#include "metalio.h" +#include "qsam.h" + +#else + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 600 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#endif + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "openprims.h" +#include "bpxnet.h" +#include "unixfile.h" +#include "charsets.h" +#include "collections.h" +#include "json.h" +#include "jsonschema.h" +#include "yaml.h" +#include "yaml2json.h" +#include "embeddedjs.h" +#include "parsetools.h" +#include "microjq.h" +#include "configmgr.h" + +#ifdef __ZOWE_OS_WINDOWS +typedef int64_t ssize_t; +#include "winregex.h" +#else +#include "psxregex.h" /* works on ZOS and Linux */ +#endif + + +/* A configuration manager is a facility providing access to a set of configuration source + (most likely textual) in many formats in one or more locations whose data definition is + provided by a set of JSON Schema Definions. + + Formats + YAML + JSON + PARMLIB + Environment Variables + + Paths + Configs and Schemas Live on paths of higher-to-lower precedence directories and libraries + (PDS/PDSE's). + + + -- References ---------- + + http://json-schema.org + + https://datatracker.ietf.org/doc/html/draft-bhutton-json-schema-validation) + + + -- Compilation ---------- + + set QJS=c:\repos\quickjs-portable ## wherever you git cloned quickjs-portable + + set YAML=c:\repos\libyaml ## Wherever you git clone'd libyaml + + clang++ -c ../platform/windows/cppregex.cpp ../platform/windows/winregex.cpp + + clang -I %QJS%\porting -I%YAML%/include -I %QJS% -I./src -I../h -I ../platform/windows -DCMGRTEST=1 -DCONFIG_VERSION=\"2021-03-27\" -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 --rtlib=compiler-rt -o configmgr.exe configmgr.c embeddedjs.c %QJS%\quickjs.c %QJS%\cutils.c %QJS%\quickjs-libc.c %QJS%\libbf.c %QJS%\libregexp.c %QJS%\libunicode.c %QJS%\porting\winpthread.c %QJS%\porting\wintime.c %QJS%\porting\windirent.c %QJS%\porting\winunistd.c %YAML%/src/api.c %YAML%/src/reader.c %YAML%/src/scanner.c %YAML%/src/parser.c %YAML%/src/loader.c %YAML%/src/writer.c %YAML%/src/emitter.c %YAML%/src/dumper.c ../c/yaml2json.c ../c/microjq.c ../c/parsetools.c ../c/jsonschema.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c cppregex.o winregex.o + + configmgr "../tests/schemadata" "" "LIBRARY(FOO):DIR(BAR)" yak + + configmgr -s "../tests/schemadata" -p "LIBRARY(FOO):DIR(BAR)" extract "/zowe/setup/" + + configmgr -s "../tests/schemadata" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowebase.yaml)" validate + + configmgr -t 2 -s "../tests/schemadata/zoweyaml.schema" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowebase.yaml)" validate + + configmgr -t 1 -s "../tests/schemadata/zoweappserver.json:../tests/schemadata/zowebase.json:../tests/schemadata/zowecommon.json" -p "FILE(../tests/schemadata/bundle1.json)" validate + + configmgr -s "../tests/schemadata/zoweappserver.json:../tests/schemadata/zowebase.json:../tests/schemadata/zowecommon.json" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowedefault.yaml)" extract "/zowe/setup/mvs/proclib" + + -- Compilation with XLCLang on ZOS ------------------------------------- + + export YAML="/u/zossteam/jdevlin/git2022/libyaml" + export QJS="/u/zossteam/jdevlin/git2022/quickjs" + + NOTE!! charsets.c requires a path that is NOT unix path. xlclang and clang will not support this + + Easy, maybe temporary workaround is to copy this H file somewhere + + cp "//'SYS1.SCUNHF(CUNHC)'" ../h/CUNHC.h + + xlclang -q64 -qascii -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 -DCONFIG_VERSION=\"2021-03-27\" -D_OPEN_SYS_FILE_EXT=1 -D_XOPEN_SOURCE=600 -D_OPEN_THREADS=1 -DSUBPOOL=132 "-Wc,float(ieee),longname,langlvl(extc99),gonum,goff,ASM,asmlib('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN')" -I ../h -I ../platform/posix -I ${YAML}/include -I ${QJS} -Wbitwise-op-parentheses -o configmgr configmgr.c yaml2json.c embeddedjs.c jsonschema.c json.c xlate.c charsets.c zosfile.c logging.c recovery.c scheduling.c zos.c le.c collections.c timeutls.c utils.c alloc.c ../platform/posix/psxregex.c ${QJS}/quickjs.c ${QJS}/cutils.c ${QJS}/quickjs-libc.c ${QJS}/libregexp.c ${QJS}/libunicode.c ${QJS}/porting/polyfill.c ${YAML}/src/api.c ${YAML}/src/reader.c ${YAML}/src/scanner.c ${YAML}/src/parser.c ${YAML}/src/loader.c ${YAML}/src/writer.c ${YAML}/src/emitter.c ${YAML}/src/dumper.c + + + ------- + + Supporting a subset of jq syntax for extraction: + + JQ available at + + https://stedolan.github.io/jq/ + + Examples: + + jq ".properties.zowe.type" ../tests/schemadata/zoweyaml.schema + + Good examples in https://stedolan.github.io/jq/tutorial/ + + curl https://api.github.com/repos/stedolan/jq/commits?per_page=5 > commits.json + + --- + 1) top level call + 2) The "zowe" global, eg. zowe.validate(json,schema) -> null | [ ] + 3) configmgr - make schema arg explicit and path + 4) build file references "jdevlin" + 5) semantic validation + 6) $ {{ zowe.runtimeDirectory }} + +*/ + +#define CONFIG_PATH_OMVS_FILE 0x0001 +#define CONFIG_PATH_OMVS_DIR 0x0002 +#define CONFIG_PATH_OMVS_LIBRARY 0x0004 + +#ifdef __ZOWE_OS_WINDOWS +static int stdoutFD(void){ + return _fileno(stdout); +} + +static int stderrFD(void){ + return _fileno(stdout); +} +#else +static int stdoutFD(void){ +#ifdef STDOUT_FILENO + return STDOUT_FILENO; +#else + return 1; +#endif +} + +static int stderrFD(void){ +#ifdef STDERR_FILENO + return STDERR_FILENO; +#else + return 2; +#endif +} +#endif + + +#define INFO 0 +#define DEBUG 1 +#define DEBUG2 2 + +static void trace(ConfigManager *mgr, int level, char *formatString, ...){ + if (mgr && mgr->traceLevel >= level){ + va_list argPointer; + va_start(argPointer,formatString); + vfprintf(mgr->traceOut,formatString,argPointer); + va_end(argPointer); + } +} + +/* configlet semantcs *IS* javascript semantics (and-or-not scalars) + Configs live in product read-only directories + + configrunner handles all the yaml/json/parmlib that the ZWE scripts need + + */ + +typedef struct ConfigletParm_tag { + int type; // int,string + char *name; +} ConfigletParm; + +typedef struct ConfigVersion_tag { + int major; + int minor; + int patch; +} ConfigVersion; + + + +#define CONFIG_ACTION_MKDIR 0x0001 +#define CONFIG_ACTION_CHMOD 0x0002 +#define CONFIG_ACTION_RMDIR 0x0003 +#define CONFIG_ACTION_RM 0x0004 + +#define CONFIG_ACTION_SCRIPT_STEP 0x0005 + +#define CONFIG_ACTION_SET_LOGICAL_DIR 0x0100 + +#define CONFIG_SERIAL_COMPOUND_ACTION +#define CONFIG_PARALLEL_COMPOUND_ACTION + +#define CONFIG_ACTION_UNPAX 0x0200 + +/* An action is an expression that + 1) returns status information + 2) logs the fact that it ran and status with args + + + Only Actions change the status of the system. + + Expressions are exactly what they sound like + 1) Functional, stateless + 2) Arith, logic, + +*/ + + +typedef struct ConfigAction_tag { + int type; + ArrayList parameters; /* ConfigletParms's. Formal Parameters, not values */ +} ConfigAction; + +typedef struct ConfigPrerequisite_tag { + char *name; + ConfigVersion *minVersion; +} ConfigPrerequisite; + +typedef struct Configlet_tag { + char *fullyQualifiedName; // dotted namespace, like org.zowe... + char *displayName; + char *schemaFilename; + ConfigVersion *version; + ArrayList prerequisites; +} Configlet; + + + +/* + zwe install doesn't set up to clone well HA + + targets: + zowe_early + zowe 1.X_to_2.0 migrate + zss_install + + Scenario 1 + + Optionally Get Pax into install (unpax) dir + + build instance dir if not there + copy files into instance directory + build instance yaml + + Saturday + What do these do + Zwe install carrying stuff into the right places (node pre-req-no-code) + Zwe init need these post-install or post-SMPE + + Data first, for config + + QuickJS + - Native Packed Decimal + +*/ + +static char *substring(ConfigManager *mgr, char *s, int start, int end){ + int len = end-start; + char *sub = (mgr ? SLHAlloc(mgr->slh,len+1) : safeMalloc(len+1,"ConfigMgr.substring")); + memcpy(sub,s+start,len); + sub[len] = 0; + return sub; +} + +static char *makeFullPath(ConfigManager *mgr, char *path, char *simpleFilename, bool useSLH){ + int pathLen = strlen(path); + int fileLen = strlen(simpleFilename); + int len = pathLen + fileLen + 4; + bool needsSlash = path[pathLen-1] != '/'; + char *result = (useSLH ? SLHAlloc(mgr->slh, len) : malloc(len)); + snprintf(result,len,"%s%s%s", + path,(needsSlash ? "/" : ""),simpleFilename); + return result; +} + +static char *extractMatch(ConfigManager *mgr, char *s, regmatch_t *match){ + return substring(mgr,s,(int)match->rm_so,(int)match->rm_eo); +} + +ConfigPathElement *makePathElement(ConfigManager *mgr, int flags, char *name, char *data){ + ConfigPathElement *element = (ConfigPathElement*)SLHAlloc(mgr->slh,sizeof(ConfigPathElement)); + memset(element,0,sizeof(ConfigPathElement)); + element->flags = flags; + element->name = name; + element->data = data; + if (mgr->configPath == NULL){ + mgr->configPath = element; + } else { + ConfigPathElement *tail = mgr->configPath; + while (tail->next){ + tail = tail->next; + } + tail->next = element; + } + return element; +} + +static bool addPathElement(ConfigManager *mgr, char *pathElementArg){ + ConfigPathElement *element = (ConfigPathElement*)SLHAlloc(mgr->slh,sizeof(ConfigPathElement)); + memset(element,0,sizeof(ConfigPathElement)); + + regmatch_t matches[10]; + regex_t *argPattern = regexAlloc(); + /* Nice Regex test site */ + char *pat = "^(LIBRARY|DIR|FILE)\\(([^)]+)\\)$"; + int compStatus = regexComp(argPattern,pat,REG_EXTENDED); + if (compStatus != 0){ + trace(mgr,INFO,"Internal error, pattern compilation failed\n"); + return false; + } + if (!strcmp(pathElementArg,"PARMLIBS")){ + /* need ZOS impl */ + printf("implement me\n"); + return false; + } else if (regexExec(argPattern,pathElementArg,10,matches,0) == 0){ + char *elementTypeName = extractMatch(mgr,pathElementArg,&matches[1]); + char *elementData = extractMatch(mgr,pathElementArg,&matches[2]); + int flags = 0; + if (!strcmp(elementTypeName,"LIBRARY")){ + flags = CONFIG_PATH_OMVS_LIBRARY; + } else if (!strcmp(elementTypeName,"DIR")){ + flags = CONFIG_PATH_OMVS_DIR; + } else if (!strcmp(elementTypeName,"FILE")){ + flags = CONFIG_PATH_OMVS_FILE; + + } else { + return false; /* internal logic error */ + } + makePathElement(mgr,flags,pathElementArg,elementData); + return true; + } else { + trace(mgr,DEBUG,"unhandled path element '%s'\n",pathElementArg); + return false; + } + +} + +static int buildConfigPath(ConfigManager *mgr, char *configPathArg){ + int pos = 0; + int len = strlen(configPathArg); + while (pos < len){ + int nextColon = indexOf(configPathArg,len,':',pos); + int nextPos; + int pathElementLength = 0; + trace(mgr,DEBUG2,"nextColon = %d\n",nextColon); + + if (nextColon == -1){ + nextPos = len; + pathElementLength = len - pos; + } else { + nextPos = nextColon+1; + pathElementLength = nextColon - pos; + } + + trace(mgr,DEBUG2,"ckpt.2 nextPos=%d\n",nextPos); + + char *pathElementArg = substring(mgr,configPathArg,pos,pos+pathElementLength); + + trace(mgr,DEBUG2,"ckpt.3\n"); + + bool status = addPathElement(mgr,pathElementArg); + + if (status == false){ + trace(mgr,INFO,"path building failed\n"); + return 12; + } + pos = nextPos; + } + return 0; +} + +static void printConfigPath(ConfigManager *mgr){ + /* Thurs AM */ + trace(mgr,INFO,"Path Elements: mgr=0x%p\n",mgr); + ConfigPathElement *element = mgr->configPath; + while (element){ + trace(mgr,INFO," %04X %s\n",element->flags,element->name); + element = element->next; + } +} + +static void jsonPrettyPrint(ConfigManager *mgr, Json *json){ + jsonPrinter *p = makeJsonPrinter(mgr->traceOut == stdout ? stdoutFD() : stderrFD()); + jsonEnablePrettyPrint(p); + jsonPrint(p,json); + trace(mgr,INFO,"\n"); + +} + +/* #define ZOWE_SCHEMA_FILE "zoweyaml.schema" */ + +void freeConfigManager(ConfigManager *mgr); + +static JsonSchema *loadOneSchema(ConfigManager *mgr, char *schemaFilePath){ + int returnCode = 0; + int reasonCode = 0; + FileInfo yamlFileInfo; + trace(mgr,DEBUG,"before file info\n"); + int infoStatus = fileInfo(schemaFilePath,&yamlFileInfo,&returnCode,&reasonCode); + if (infoStatus){ + trace(mgr,INFO,"failed to get fileInfo of '%s', infoStatus=%d\n",schemaFilePath,infoStatus); + freeConfigManager(mgr); + return NULL; + } + char errorBuffer[1024]; + trace(mgr,DEBUG,"before jsonParseFile info\n"); + Json *jsonWithSchema = jsonParseFile2(mgr->slh,schemaFilePath,errorBuffer,1024); + if (jsonWithSchema == NULL){ + trace(mgr,INFO,"failed to read JSON with base schema: %s\n",errorBuffer); + freeConfigManager(mgr); + return NULL; + } + if (mgr->traceLevel >= 1){ + jsonPrettyPrint(mgr,jsonWithSchema); + + } + JsonSchemaBuilder *builder = makeJsonSchemaBuilder(DEFAULT_JSON_SCHEMA_VERSION); + if (mgr->traceLevel >= 1){ + printf("about to build schema for '%s'\n",schemaFilePath);fflush(stdout); + } + JsonSchema *schema = jsonBuildSchema(builder,jsonWithSchema); + if (schema == NULL){ + printf("Schema Build for '%s' Failed: %s\n",schemaFilePath,builder->errorMessage); + freeJsonSchemaBuilder(builder); + freeConfigManager(mgr); + return NULL; + } else { + trace(mgr,DEBUG,"JSON Schema built successfully\n"); + } + freeJsonSchemaBuilder(builder); + return schema; +} + +#define MAX_SCHEMAS 100 + +ConfigManager *makeConfigManager(char *configPathArg, char *schemaPath, + int traceLevel, FILE *traceOut){ + fprintf(traceOut,"makeConfigMgr\n");fflush(traceOut); + ConfigManager *mgr = (ConfigManager*)safeMalloc(sizeof(ConfigManager),"ConfigManager"); + + memset(mgr,0,sizeof(ConfigManager)); + mgr->traceLevel = traceLevel; + mgr->traceOut = traceOut; + mgr->slh = makeShortLivedHeap(0x10000,0x100); + EmbeddedJS *ejs = makeEmbeddedJS(NULL); + mgr->ejs = ejs; + trace(mgr,DEBUG,"before build config path\n"); + if (buildConfigPath(mgr,configPathArg)){ + printf("built config path failed\n");fflush(stdout); + safeFree((char*)mgr,sizeof(ConfigManager)); + return NULL; + } + trace(mgr,DEBUG,"built config path\n"); + + mgr->otherSchemas = (JsonSchema**)SLHAlloc(mgr->slh,MAX_SCHEMAS*sizeof(JsonSchema)); + mgr->rootSchemaDirectory = schemaPath; /* rootSchemaDirectory; */ + + int pos = 0; + int len = strlen(schemaPath); + int schemaCount = 0; + while (pos < len){ + int nextColon = indexOf(schemaPath,len,':',pos); + int nextPos; + int pathElementLength = 0; + + if (nextColon == -1){ + nextPos = len; + pathElementLength = len - pos; + } else { + nextPos = nextColon+1; + pathElementLength = nextColon - pos; + } + + char *schemaFilePath = substring(mgr,schemaPath,pos,pos+pathElementLength); + + /* char *schemaFilePath = makeFullPath(mgr,rootSchemaDirectory,ZOWE_SCHEMA_FILE,true); */ + + if (schemaCount == 0){ + mgr->topSchema = loadOneSchema(mgr,schemaFilePath); + } else { + mgr->otherSchemas[schemaCount-1] = loadOneSchema(mgr,schemaFilePath); + } + + pos = nextPos; + schemaCount++; + } + mgr->otherSchemasCount = schemaCount-1; + return mgr; +} + +#define YAML_ERROR_MAX 1024 + +static Json *readJson(ConfigManager *mgr, ConfigPathElement *pathElement){ + char errorBuffer[YAML_ERROR_MAX]; + if (pathElement->flags & CONFIG_PATH_OMVS_FILE){ + trace(mgr,DEBUG,"before read YAML file=%s\n",pathElement->data); + yaml_document_t *doc = readYAML(pathElement->data,errorBuffer,YAML_ERROR_MAX); + trace(mgr,DEBUG,"yaml doc at 0x%p\n",doc); + if (doc){ + if (mgr->traceLevel >= 1){ + pprintYAML(doc); + } + return yaml2JSON(doc,mgr->slh); + } else { + trace(mgr,INFO,"WARNING, yaml read failed\n"); + return NULL; + } + } else { + trace(mgr,INFO,"WARNING, only simple file case yet implemented\n"); + return NULL; + } +} + +void freeConfigManager(ConfigManager *mgr){ + SLHFree(mgr->slh); + safeFree((char*)mgr,sizeof(ConfigManager)); +} + +#define JSON_POINTER_TOO_DEEP 101 +#define JSON_POINTER_ARRAY_INDEX_NOT_INTEGER 102 +#define JSON_POINTER_ARRAY_INDEX_OUT_OF_BOUNDS 103 + +#define ZCFG_ALLOC_HEAP 1 +#define ZCFG_ALLOC_SLH 2 +#define ZCFG_ALLOC_MEMCPY 3 + +/* These eventually need ZWE unique messages */ +#define ZCFG_SUCCESS 0 +#define ZCFG_TYPE_MISMATCH 1 +#define ZCFG_EVAL_FAILURE 2 +#define ZCFG_POINTER_TOO_DEEP JSON_POINTER_TOO_DEEP +#define ZCFG_POINTER_ARRAY_INDEX_NOT_INTEGER JSON_POINTER_ARRAY_INDEX_NOT_INTEGER +#define ZCFG_POINTER_ARRAY_INDEX_OUT_OF_BOUNDS JSON_POINTER_ARRAY_INDEX_OUT_OF_BOUNDS + + +/* need to collect violations as this goes */ +static int overloadConfiguration(ConfigManager *mgr, + ConfigPathElement *pathElement, + ConfigPathElement *pathTail){ + if (pathTail == NULL){ + trace(mgr, DEBUG2, "at end of config path\n"); + mgr->config = readJson(mgr,pathElement); + trace(mgr, DEBUG2, "mgr->config = 0x%p\n", mgr->config); + return 0; /* success */ + } else { + Json *overlay = readJson(mgr,pathElement); + int rhsStatus = overloadConfiguration(mgr,pathTail,pathTail->next); + trace(mgr, DEBUG2, "read the overlay with json=0x%p and status=%d\n",overlay,rhsStatus); + if (rhsStatus){ + return rhsStatus; /* don't merge if we couldn't load what's to the right in the list */ + } + int mergeStatus = 0; + mgr->config = jsonMerge(mgr->slh,overlay,mgr->config, + JSON_MERGE_FLAG_CONCATENATE_ARRAYS, + &mergeStatus); + return mergeStatus; + } +} + +int loadConfigurations(ConfigManager *mgr){ + ConfigPathElement *pathElement = mgr->configPath; + int overloadStatus = overloadConfiguration(mgr,pathElement,pathElement->next); + if (overloadStatus){ + return overloadStatus; + } else { + if (mgr->traceLevel >= 1){ + printf("config before template eval:\n"); + jsonPrettyPrint(mgr, mgr->config); + } + Json *evaluatedConfig = evaluateJsonTemplates(mgr->ejs,mgr->slh,mgr->config); + if (evaluatedConfig){ + mgr->config = evaluatedConfig; + return ZCFG_SUCCESS; + } else { + return ZCFG_EVAL_FAILURE; + } + } +} + + +/* Merging notes + defaulting and merging requires the same data shape at all levels and sources +*/ + +/* all calls return status */ + +/* zowe.setup.mvs.proclib */ + +int cfgGetInt(ConfigManager *mgr, int *value, ...){ /* path elements which are strings and ints for array indexing */ + return 0; +} + +int cfgGetInt64(ConfigManager *mgr, int64 *value, ...){ + return 0; +} + + +static Json *jsonPointerDereference(Json *json, JsonPointer *jsonPointer, int *errorReason, int traceLevel){ + ArrayList *elements = &(jsonPointer->elements); + Json *value = json; + for (int i=0; isize; i++){ + JsonPointerElement *element = (JsonPointerElement*)arrayListElement(elements,i); + if (traceLevel >= 1){ + printf("deref elt=0x%p, i=%d value=0x%p\n",element,i,value);fflush(stdout); + } + if (jsonIsArray(value)){ + JsonArray *array = jsonAsArray(value); + if (element->type == JSON_POINTER_INTEGER){ + int index = atoi(element->string); + int arraySize = jsonArrayGetCount(array); + if (index >= arraySize){ + *errorReason = JSON_POINTER_ARRAY_INDEX_OUT_OF_BOUNDS; + return NULL; + } + value = jsonArrayGetItem(array,index); + } else { + *errorReason = JSON_POINTER_ARRAY_INDEX_NOT_INTEGER; + return NULL; + } + } else if (jsonIsObject(value)){ + JsonObject *object = jsonAsObject(value); + value = jsonObjectGetPropertyValue(object,element->string); + if (value == NULL){ + *errorReason = JSON_POINTER_TOO_DEEP; + return NULL; + } + } else { + if (traceLevel >= 1){ + printf("cannot dereference scalar\n"); + fflush(stdout); + } + return NULL; + } + } + return value; +} + +int cfgGetStringJ(ConfigManager *mgr, Json **result, JsonPointer *jp){ + int errorReason = 0; + Json *value = jsonPointerDereference(mgr->config,jp,&errorReason,mgr->traceLevel); + if (value){ + if (jsonIsString(value)){ + *result = value; + return ZCFG_SUCCESS; + } else { + *result = NULL; + return ZCFG_TYPE_MISMATCH; + } + } else { + *result = NULL; + return errorReason; + } +} + +int vcfgGetStringC(ConfigManager *mgr, char **value, int allocOptions, void *mem, va_list args){ + return ZCFG_SUCCESS; +} + +int cfgGetStringC(ConfigManager *mgr, char **value, int allocOptions, void *mem, ...){ + va_list argPointer; + va_start(argPointer,mem); + vcfgGetStringC(NULL,value,allocOptions,mem,argPointer); + va_end(argPointer); + + return ZCFG_SUCCESS; +} + +int cfgGetInstanceType(ConfigManager *mgr, int *type, int *subType, ...){ + return 0; +} + +/* like get string, but doesn't care about data types */ + +int cfgGetAnyJ(ConfigManager *mgr, Json **result, JsonPointer *jp){ + int errorReason = 0; + Json *value = jsonPointerDereference(mgr->config,jp,&errorReason,mgr->traceLevel); + if (mgr->traceLevel >= 1){ + printf("cfgGetAny: value=0x%p error=%d\n",value,errorReason); + } + if (value){ + *result = value; + return ZCFG_SUCCESS; + } else { + *result = NULL; + return errorReason; + } +} + +int cfgGetAny(ConfigManager *mgr, char *value, int allocOptions, void *mem, ...){ + return 0; +} + +static void extractText(ConfigManager *mgr, JsonPointer *jp, FILE *out){ + Json *value = NULL; + int status = cfgGetAnyJ(mgr,&value,jp); + if (status){ + fprintf(out,"error not found, reason=%d",status); + } else { + if (jsonIsObject(value) || + jsonIsArray(value)){ + fprintf(out,"error: cannot access whole objects or arrays"); + } else if (jsonIsString(value)){ + fprintf(out,"%s",jsonAsString(value)); + } else if (jsonIsInt64(value)){ + fprintf(out,"%lld",INT64_LL(jsonAsInt64(value))); + } else if (jsonIsDouble(value)){ + fprintf(out,"%f",jsonAsDouble(value)); + } else if (jsonIsBoolean(value)){ + fprintf(out,"%s",jsonAsBoolean(value) ? "true" : "false"); + } else if (jsonIsNull(value)){ + fprintf(out,"null"); + } else { + fprintf(out,"error: unhandled type"); + } + } +} + +#define MAX_PATH_NAME 1024 +#define DATA_BUFFER_SIZE 2048 + +static void showDirectory(const char *dirname){ + int returnCode = 0; + int reasonCode = 0; + int stemLen = strlen(dirname); + int needsSlash = dirname[stemLen-1] != '/'; + char path[MAX_PATH_NAME]; + char directoryDataBuffer[DATA_BUFFER_SIZE]; + bool includeDotted = false; + UnixFile *directory = NULL; + printf("in show directory\n"); + fflush(stdout); + if ((directory = directoryOpen(dirname,&returnCode,&reasonCode)) == NULL){ + printf("directory open (%s) failure rc=%d reason=0x%x\n",dirname,returnCode,reasonCode); + } else { + + int entriesRead = 0; + while ((entriesRead = directoryRead(directory,directoryDataBuffer,DATA_BUFFER_SIZE,&returnCode,&reasonCode)) > 0){ + char *entryStart = directoryDataBuffer; + for (int e = 0; e \n"); + fprintf(out," options\n"); + fprintf(out," -h : show help\n"); + fprintf(out," -t : enable tracing with level from 1-3\n"); + fprintf(out," -o : OUT|ERR , ERR is default\n"); + fprintf(out," -s : (: : workspace directory\n"); + fprintf(out," -c : compact output for jq and extract commands\n"); + fprintf(out," -r : raw string output for jq and extract commands\n"); + fprintf(out," -p : list of colon-separated configPathElements - see below\n"); + fprintf(out," commands:\n"); + fprintf(out," extract : prints value to stdout\n"); + fprintf(out," validate : just loads and validates merged configuration\n"); + fprintf(out," env : prints merged configuration to a file as a list of environment vars\n"); + fprintf(out," configPathElement: \n"); + fprintf(out," LIB(datasetName) - a library that can contain config data\n"); + fprintf(out," FILE(filename) - the name of a file containing Yaml\n"); + fprintf(out," PARMLIBS - all PARMLIBS that are defined to this running Program in ZOS, nothing if not on ZOS\n"); +} + +#define ZWE_PREFIX "ZWE" +#define ZOWE_ENV_PATH ZWE_PREFIX "_zowe_environments" + +static void convertJsonToEnv(FILE *fp, const char *path, Json *value); +static void convertJsonObjectToEnv(FILE *fp, const char *path, JsonObject *object); +static void convertJsonArrayToEnv(FILE *fp, const char *path, JsonArray *array); + +static int numberLen(int number) { + if (number == 0) { + return 1; + } + if (number < 0) { + return 1 + numberLen(-number); + } + return (int)(log10((double)number) + 1); +} + +static size_t escapeForEnv(const char *s, bool isKey, char *buffer, size_t bufferSize) { + size_t pos = 0; + int len = strlen(s); + memset(buffer, 0, bufferSize); + for (int i = 0; i < len; i++) { + char c = s[i]; + if (c == '\"' || c == '\"') { + pos += snprintf (buffer + pos, bufferSize - pos, "\\"); + } + if (isKey && c == '-') { + c = '_'; + } + pos += snprintf (buffer + pos, bufferSize - pos, "%c", c); + } + return pos; +} + +static void outputEnvKey(FILE * out, const char *str) { + size_t escapedSize = strlen(str) * 2 + 1; + char escaped[escapedSize]; + escapeForEnv(str, true, escaped, escapedSize); + fprintf(out, "%s=", escaped); +} + +static void outputEnvString(FILE * out, const char *str) { + size_t escapedSize = strlen(str) * 2 + 1; + char escaped[escapedSize]; + escapeForEnv(str, false, escaped, escapedSize); + fprintf(out, "\"%s\"\n", escaped); +} + +static void outputEnvInt64(FILE * out, int64_t num) { + fprintf(out, "%lld\n", INT64_LL(num)); +} + +static void outputEnvDouble(FILE * out, double num) { + fprintf(out, "%f\n", num); +} + +static void outputEnvBoolean(FILE * out, bool b) { + fprintf(out, "%s\n", b ? "true": "false"); +} + +static void convertJsonToEnv(FILE *out, const char *path, Json *value) { + if (jsonIsObject(value)) { + convertJsonObjectToEnv(out, path, jsonAsObject(value)); + } else if (jsonIsArray(value)) { + convertJsonArrayToEnv(out, path, jsonAsArray(value)); + } else { + if (jsonIsString(value)) { + outputEnvKey(out, path); + outputEnvString(out, jsonAsString(value)); + } else if (jsonIsInt64(value)) { + outputEnvKey(out, path); + outputEnvInt64(out, jsonAsInt64(value)); + } else if (jsonIsDouble(value)) { + outputEnvKey(out, path); + outputEnvDouble(out, jsonAsDouble(value)); + } else if (jsonIsBoolean(value)) { + outputEnvKey(out, path); + outputEnvBoolean(out, jsonAsBoolean(value)); + } else if (jsonIsNull(value)) { + /* omit null */ + } + } +} + + +static void convertJsonObjectToEnv(FILE *out, const char *path, JsonObject *object) { + bool isEnvVar = (0 == strcmp(path, ZOWE_ENV_PATH)); + for (JsonProperty *prop = jsonObjectGetFirstProperty(object); prop != NULL; prop = jsonObjectGetNextProperty(prop)) { + const char *key = jsonPropertyGetKey(prop); + size_t pathSize = strlen(path) + strlen(key) + 2; + char currentPath[pathSize]; + snprintf (currentPath, pathSize, "%s_%s", path, key); + Json *value = jsonPropertyGetValue(prop); + convertJsonToEnv(out, isEnvVar ? key : currentPath, value); + } +} + +static void convertJsonArrayToEnv(FILE *out, const char *path, JsonArray *array) { + int count = jsonArrayGetCount(array); + for (int i = 0; i < count; i ++) { + int len = numberLen(i); + size_t pathSize = strlen(path) + len + 2; + char currentPath[pathSize]; + snprintf (currentPath, pathSize, "%s_%d", path, i); + Json *value = jsonArrayGetItem(array, i); + convertJsonToEnv(out, currentPath, value); + } +} + +#ifdef CMGRTEST +int main(int argc, char **argv){ +#else +static int not_main(int argc, char **argv){ +#endif + char *schemaPath = NULL; + char *zoweWorkspaceHome = NULL; // Read-write is there always a zowe.yaml in there + char *configPath = NULL; + char *command = NULL; + char *traceArg = NULL; + int argx = 1; + int traceLevel = 0; + FILE *traceOut = stderr; + bool jqCompact = false; + bool jqRaw = false; + if (argc == 1){ + showHelp(traceOut); + return 0; + } + while (argx < argc){ + char *optionValue = NULL; + if (getStringOption(argc,argv,&argx,"-h")){ + showHelp(traceOut); + return 0; + } else if ((optionValue = getStringOption(argc,argv,&argx,"-s")) != NULL){ + schemaPath = optionValue; + } else if ((optionValue = getStringOption(argc,argv,&argx,"-t")) != NULL){ + traceArg = optionValue; + } else if ((optionValue = getStringOption(argc,argv,&argx,"-o")) != NULL){ + if (!strcmp(optionValue,"OUT")){ + traceOut = stdout; + } + } else if ((optionValue = getStringOption(argc,argv,&argx,"-w")) != NULL){ + zoweWorkspaceHome = optionValue; + } else if ((optionValue = getStringOption(argc,argv,&argx,"-p")) != NULL){ + configPath = optionValue; + } else if ((optionValue = getStringOption(argc,argv,&argx,"-c")) != NULL){ + jqCompact = true; + } else if ((optionValue = getStringOption(argc,argv,&argx,"-r")) != NULL){ + jqRaw = true; + } else { + char *nextArg = argv[argx]; + if (strlen(nextArg) && nextArg[0] == '-'){ + fprintf(traceOut,"\n *** unknown option *** '%s'\n",nextArg); + showHelp(traceOut); + return 0; + } else { + break; + } + } + } + + if (traceArg){ + traceLevel = atoi(traceArg); + if (traceLevel >= 1){ + printf("ConfigMgr tracelevel set to %d\n",traceLevel); + } + } + + if (schemaPath == NULL){ + fprintf(traceOut,"Must specify schema path with at least one schema"); + showHelp(traceOut); + return 0; + } + if (configPath == NULL){ + fprintf(traceOut,"Must specify config path\n"); + showHelp(traceOut); + return 0; + } + + ConfigManager *mgr = makeConfigManager(configPath,schemaPath,traceLevel,traceOut); + if (mgr == NULL){ + trace(mgr,INFO,"Failed to build configmgr\n"); + return 0; + } + trace(mgr,DEBUG,"ConfigMgr built at 0x%p\n",mgr); + + + if (argx >= argc){ + printf("\n *** No Command Seen ***\n\n"); + showHelp(traceOut); + return 0; + } + command = argv[argx++]; + trace(mgr,DEBUG,"command = %s\n",command); + if (mgr->traceLevel >= 1){ + printConfigPath(mgr); + } + int loadStatus = loadConfigurations(mgr); + if (loadStatus){ + trace(mgr,INFO,"Failed to load configuration, element may be bad, or less likey a bad merge\n"); + } + trace(mgr,DEBUG,"configuration parms are loaded\n"); + if (!strcmp(command,"validate")){ /* just a testing mode */ + trace(mgr,INFO,"about to validate merged yamls as\n"); + jsonPrettyPrint(mgr,mgr->config); + JsonValidator *validator = makeJsonValidator(); + validator->traceLevel = mgr->traceLevel; + trace(mgr,DEBUG,"Before Validate\n"); + int validateStatus = jsonValidateSchema(validator,mgr->config, + mgr->topSchema,mgr->otherSchemas,mgr->otherSchemasCount); + trace(mgr,INFO,"validate status = %d\n",validateStatus); + switch (validateStatus){ + case JSON_VALIDATOR_NO_EXCEPTIONS: + trace(mgr,INFO,"No validity Exceptions\n"); + break; + case JSON_VALIDATOR_HAS_EXCEPTIONS: + { + trace(mgr,INFO,"Validity Exceptions:\n"); + ValidityException *e = validator->firstValidityException; + while (e){ + trace(mgr,INFO," %s\n",e->message); + e = e->next; + } + } + break; + case JSON_VALIDATOR_INTERNAL_FAILURE: + trace(mgr,INFO,"validation internal failure"); + break; + } + freeJsonValidator(validator); + } else if (!strcmp(command,"jq")){ + if (argx >= argc){ + trace(mgr,INFO,"jq requires at least one filter argument"); + } else { + JQTokenizer jqt; + memset(&jqt,0,sizeof(JQTokenizer)); + jqt.data = argv[argx++]; + jqt.length = strlen(jqt.data); + jqt.ccsid = 1208; + + Json *jqTree = parseJQ(&jqt,mgr->slh,0); + if (jqTree){ + int flags = ((jqCompact ? 0 : JQ_FLAG_PRINT_PRETTY)| + (jqRaw ? JQ_FLAG_RAW_STRINGS : 0)); + int evalStatus = evalJQ(mgr->config,jqTree,stdout,flags,mgr->traceLevel); + if (evalStatus != 0){ + trace(mgr, INFO,"micro jq eval problem %d\n",evalStatus); + } + } else { + trace(mgr, INFO, "Failed to parse jq expression\n"); + } + } + } else if (!strcmp(command,"extract")){ + if (argx >= argc){ + trace(mgr,INFO,"extract command requires a json path\n"); + } else { + char *path = argv[argx++]; + JsonPointer *jp = parseJsonPointer(path); + if (jp == NULL){ + trace(mgr,INFO,"Could not parse JSON pointer '%s'\n",path); + return 0; + } + if (mgr->traceLevel >= 1){ + printJsonPointer(mgr->traceOut,jp); + fflush(mgr->traceOut); + } + extractText(mgr,jp,stdout); + printf("\n"); + fflush(stdout); + } + } else if (!strcmp(command, "env")) { + if (argx >= argc){ + trace(mgr, INFO, "env command requires an env file path\n"); + } else { + char *outputPath = argv[argx++]; + Json *config = mgr->config; + FILE *out = fopen(outputPath, "w"); + if (out) { + convertJsonToEnv(out, ZWE_PREFIX, config); + fclose(out); + } else { + trace (mgr, INFO, "failed to open output file '%s' - %s\n", outputPath, strerror(errno)); + } + } + } + return 0; +} + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/crossmemory.c b/c/crossmemory.c index 9ef6bc4fa..bd15f1ca3 100644 --- a/c/crossmemory.c +++ b/c/crossmemory.c @@ -602,7 +602,7 @@ static void initLogMessagePrefix(LogMessagePrefix *prefix) { ASCB *ascb = getASCB(); char *jobName = getASCBJobname(ascb); TCB *tcb = getTCB(); - snprintf(prefix->text, sizeof(prefix->text), "%22.22s %8.8s %08X(%04X) %08X ", currentTime.text, jobName, ascb, ascb->ascbasid, tcb); + snprintf(prefix->text, sizeof(prefix->text), "%22.22s %8.8s %8p(%04X) %8p ", currentTime.text, jobName, ascb, ascb->ascbasid, tcb); prefix->text[sizeof(prefix->text) - 1] = ' '; } @@ -632,7 +632,7 @@ static void printWithPrefix(LoggingContext *context, LoggingComponent *component if (lineIdx == 0) { initLogMessagePrefix(&prefix); } - printf("%.*s%.*s\n", sizeof(prefix.text), prefix.text, nextLineLength, nextLine); + printf("%.*s%.*s\n", (int)sizeof(prefix.text), prefix.text, (int)nextLineLength, nextLine); nextLine += (nextLineLength + 1); } diff --git a/c/discovery.c b/c/discovery.c index 2f2306837..a8c04834e 100644 --- a/c/discovery.c +++ b/c/discovery.c @@ -259,7 +259,7 @@ static ASCB *getASCBByASID(int asid){ CVT *cvt = getCVT(); ASVT *asvt = (ASVT*)cvt->cvtasvt; - ASCB *ascb = (ASCB*)asvt->asvtenty; + ASCB *ascb = (ASCB*)INT2PTR(asvt->asvtenty); while (ascb){ if (ascb->ascbasid == asid){ return ascb; @@ -334,6 +334,7 @@ int walkTCBs(DiscoveryContext *context, TCB *firstTCB = (TCB*)getStructCopy(context,ascb,0,asxb->asxbftcb,sizeof(TCB)); walkTCBs1(context,ascb,firstTCB,(TCB*)ANY_TCB,0,visitor,visitorContext,NULL); } + return 0; } @@ -344,7 +345,7 @@ static void visitSSCTEntry(DiscoveryContext *context, if (context->ssctTraceLevel >= 1){ dumpbuffer((char*)ssctChain,sizeof(SSCT)); } - void *usr1 = (void*)ssctChain->ssctsuse; + void *usr1 = (void*)INT2PTR(ssctChain->ssctsuse); zowelog(NULL, LOG_COMP_DISCOVERY, ZOWE_LOG_DEBUG, "user pointer at 0x%x COMMON?=%s\n",usr1,isPointerCommon(gda,usr1) ? "YES" : "NO"); if (isPointerCommon(gda,usr1)){ if (context->ssctTraceLevel >= 1){ @@ -449,7 +450,7 @@ static void ispfAnchorVisitor(DiscoveryContext *context, */ if (tcbfsa[10] == *((int*)"ISPF")){ /* printf("WOO HOO\n"); */ - int *tldHandle = (int*)tcbfsa[6]; + int *tldHandle = (int*)INT2PTR(tcbfsa[6]); int *tldPtr = (int*)((int*)getStructCopy(context,ascb,0,tldHandle,4))[0]; /* printf("tldPtr=0x%x\n",tldPtr); */ if (tldPtr){ @@ -738,6 +739,7 @@ int findSessions(DiscoveryContext *context, break; /* if we ever get a good result, leave the while loop */ } } + return 0; } /************************** ZOS Model Maintenance ********************************/ @@ -759,7 +761,7 @@ static void gatherStartedTasks(DiscoveryContext *context, ZOSModel *model){ StartedTaskVisitor *userVisitor = model ? model->startedTaskVisitor : NULL; void* userVisitorData = model ? model->visitorsData : NULL; - ASCB *ascb = (ASCB*)asvt->asvtenty; + ASCB *ascb = (ASCB*)INT2PTR(asvt->asvtenty); while (ascb){ char *jobname = getASCBJobname(ascb); char *jobnameCopy = SLHAlloc(context->slh,12); @@ -818,13 +820,13 @@ static void scanTSBs(ZOSModel *model){ zowedump(NULL, LOG_COMP_DISCOVERY, ZOWE_LOG_DEBUG2,(char*)tsbxCopy,sizeof(IKTTSBX)); } TSBInfo *tsbInfo = (TSBInfo*)SLHAlloc(model->tsbScanSLH,sizeof(TSBInfo)); - tsbInfo->ascb = (ASCB*)(tsbCopy->tcbstatAndASCB & 0x00FFFFFF); + tsbInfo->ascb = (ASCB*)INT2PTR(tsbCopy->tcbstatAndASCB & 0x00FFFFFF); tsbInfo->tsb = tsbCopy; tsbInfo->tsbx = tsbxCopy; char *luname = &(tsbCopy->tsbtrmid[0]); htPut(tsbTable,luname,tsbInfo); - tsb = (IKJTSB*)(tsbxCopy->flagAndFwdPointer&0xFFFFFF); + tsb = (IKJTSB*)INT2PTR(tsbxCopy->flagAndFwdPointer&0xFFFFFF); } else{ tsb = NULL; } diff --git a/c/embeddedjs.c b/c/embeddedjs.c new file mode 100644 index 000000000..b5ef4490a --- /dev/null +++ b/c/embeddedjs.c @@ -0,0 +1,774 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#ifdef METTLE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "metalio.h" +#include "qsam.h" + +#else + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#endif + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "openprims.h" +#include "bpxnet.h" +#include "unixfile.h" +#include "json.h" +#include "charsets.h" +#include "embeddedjs.h" + +#include "cutils.h" +#include "quickjs-libc.h" + +#ifdef __ZOWE_OS_WINDOWS +typedef int64_t ssize_t; +#endif + +#ifdef __ZOWE_OS_ZOS + +int __atoe_l(char *bufferptr, int leng); +int __etoa_l(char *bufferptr, int leng); + +#endif + +static int convertToNative(char *buf, size_t size) { +#ifdef __ZOWE_OS_ZOS + return __atoe_l(buf, size); +#endif + return 0; +} + +static int convertFromNative(char *buf, size_t size) { +#ifdef __ZOWE_OS_ZOS + return __etoa_l(buf, size); +#endif + return 0; +} + +struct trace_malloc_data { + uint8_t *base; +}; + +#define FILE_LOAD_AUTODETECT -1 +#define FILE_LOAD_GLOBAL 0 /* not module */ +#define FILE_LOAD_MODULE 1 + +struct EmbeddedJS_tag { + JSRuntime *rt; + JSContext *ctx; + struct trace_malloc_data trace_data; /* = { NULL }; */ + int optind; + char *expr; /* = NULL; */ + int interactivel; /* = 0; */ + int dump_memory; /* = 0; */ + int trace_memory; /* = 0;*/ + int empty_run; /* = 0; */ + int loadMode; /* FILE_LOAD_AUTODETECT */ + int load_std; /* = 0; */ + int dump_unhandled_promise_rejection; /* = 0; */ + size_t memory_limit; /* = 0; */ + char *include_list[32]; +#ifdef CONFIG_BIGNUM + int load_jscalc; +#endif + size_t stack_size; /* = 0; */ +}; + +struct JSValueBox_tag { + JSValue value; +}; + +static JSValue ejsEvalBuffer1(EmbeddedJS *ejs, + const void *buffer, int bufferLength, + const char *filename, int eval_flags, + int *statusPtr){ + JSContext *ctx = ejs->ctx; + JSValue val; + int ret; + + if ((eval_flags & JS_EVAL_TYPE_MASK) == JS_EVAL_TYPE_MODULE) { + /* for the modules, we compile then run to be able to set + import.meta */ + val = JS_Eval(ctx, buffer, bufferLength, filename, + eval_flags | JS_EVAL_FLAG_COMPILE_ONLY); + if (!JS_IsException(val)) { + js_module_set_import_meta(ctx, val, TRUE, TRUE); + val = JS_EvalFunction(ctx, val); + } + } else { + val = JS_Eval(ctx, buffer, bufferLength, filename, eval_flags); + } + if (JS_IsException(val)) { + js_std_dump_error(ctx); + ret = -1; + } else { + ret = 0; + } + *statusPtr = ret; + return val; +} + +JSValueBox ejsEvalBuffer(EmbeddedJS *ejs, + const void *buffer, int bufferLength, + const char *filename, int eval_flags, + int *statusPtr){ + JSValue value = ejsEvalBuffer1(ejs,buffer,bufferLength,filename,eval_flags,statusPtr); + return *(JSValueBox*)(&value); +} + +void ejsFreeJSValue(EmbeddedJS *ejs, JSValueBox valueBox){ + JS_FreeValue(ejs->ctx, *((JSValue*)&valueBox)); +} + +static int ejsSetGlobalProperty_internal(EmbeddedJS *ejs, const char *propertyName, JSValue value){ + JSContext *ctx = ejs->ctx; + JSValue theGlobal = JS_GetGlobalObject(ctx); + + return JS_SetPropertyStr(ctx,theGlobal,propertyName,value); +} + +int ejsSetGlobalProperty(EmbeddedJS *ejs, const char *propertyName, JSValueBox valueBox){ + return ejsSetGlobalProperty_internal(ejs,propertyName,valueBox.value); +} + +int ejsEvalFile(EmbeddedJS *ejs, const char *filename, int loadMode){ + uint8_t *buf; + int ret, eval_flags; + size_t bufferLength; + + buf = js_load_file(ejs->ctx, &bufferLength, filename); + if (!buf) { + perror(filename); + exit(1); + } + + /* Beware of overly-expedient 3-valued logic here! */ + if (loadMode < 0) { + loadMode = (has_suffix(filename, ".mjs") || + JS_DetectModule((const char *)buf, bufferLength)); + } + if (loadMode) + eval_flags = JS_EVAL_TYPE_MODULE; + else + eval_flags = JS_EVAL_TYPE_GLOBAL; + JSValue evalResult = ejsEvalBuffer1(ejs, buf, bufferLength, filename, eval_flags, &ret); + js_free(ejs->ctx, buf); + JS_FreeValue(ejs->ctx, evalResult); + return ret; +} + +static JSClassID js_experiment_thingy_class_id; + +static void js_experiment_thingy_finalizer(JSRuntime *rt, JSValue val){ + /* JSSTDFile *s = JS_GetOpaque(val, js_std_file_class_id); */ + printf("Experiment Thingy Finalizer running\n"); +} + + +static JSClassDef js_experiment_thingy_class = { + "THINGY", + .finalizer = js_experiment_thingy_finalizer, +}; + +/* + #define JS_CFUNC_DEF(name, length, func1) + + name is JS name + + #define JS_CFUNC_MAGIC_DEF(name, length, func1, magic) + +*/ + +JSValue js_experiment_thingy_three(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv){ + return JS_NewInt32(ctx, 3); +} + +static const JSCFunctionListEntry js_experiment_thingy_proto_funcs[] = { + JS_CFUNC_DEF("three", 0, js_experiment_thingy_three ), +}; + +static JSValue js_experiment_boop(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + printf("boop boop boop!!\n"); + return JS_UNDEFINED; +} + +static const JSCFunctionListEntry js_experiment_funcs[] = { + JS_CFUNC_DEF("boop", 0, js_experiment_boop ), +}; + + + +static int js_experiment_init(JSContext *ctx, JSModuleDef *m) +{ + JSValue proto; + + /* FILE class */ + /* the class ID is created once */ + JS_NewClassID(&js_experiment_thingy_class_id); + /* the class is created once per runtime */ + JS_NewClass(JS_GetRuntime(ctx), js_experiment_thingy_class_id, &js_experiment_thingy_class); + proto = JS_NewObject(ctx); + JS_SetPropertyFunctionList(ctx, + proto, + js_experiment_thingy_proto_funcs, + countof(js_experiment_thingy_proto_funcs)); + JS_SetClassProto(ctx, + js_experiment_thingy_class_id, + proto); + + JS_SetModuleExportList(ctx, m, js_experiment_funcs, + countof(js_experiment_funcs)); + return 0; +} + + +JSModuleDef *js_init_module_experiment(JSContext *ctx, const char *module_name) +{ + JSModuleDef *m; + m = JS_NewCModule(ctx, module_name, js_experiment_init); + if (!m){ + return NULL; + } + JS_AddModuleExportList(ctx, m, js_experiment_funcs, countof(js_experiment_funcs)); + return m; +} + +/* also used to initialize the worker context */ +static JSContext *makeEmbeddedJSContext(JSRuntime *rt) +{ + JSContext *ctx; + ctx = JS_NewContext(rt); + if (!ctx) + return NULL; +#ifdef CONFIG_BIGNUM + if (bignum_ext) { + JS_AddIntrinsicBigFloat(ctx); + JS_AddIntrinsicBigDecimal(ctx); + JS_AddIntrinsicOperators(ctx); + JS_EnableBignumExt(ctx, TRUE); + } +#endif + /* system modules */ + js_init_module_std(ctx, "std"); + js_init_module_os(ctx, "os"); + js_init_module_experiment(ctx, "experiment"); + return ctx; +} + +/* + + How to build JSValues + + many are static inlines in quickjs.h + + int's + static js_force_inline JSValue JS_NewInt64(JSContext *ctx, int64_t val) + + floats + static js_force_inline JSValue JS_NewFloat64(JSContext *ctx, double d) + + bool + static js_force_inline JSValue JS_NewBool(JSContext *ctx, JS_BOOL val) + + string + JSValue JS_NewStringLen(JSContext *ctx, const char *str1, size_t len1); + JSValue JS_NewString(JSContext *ctx, const char *str); + + Array + JSValue JS_NewArray(JSContext *ctx); + + setting values + int JS_SetPropertyUint32(JSContext *ctx, + JSValueConst this_obj, <--- JSValue of array ok here + uint32_t idx, JSValue val); + + + JS_SetPropertyUint32( + + Object + JSValue JS_NewObject(JSContext *ctx); + + setting props + int JS_SetPropertyStr(JSContext *ctx, JSValueConst this_obj, + const char *prop, JSValue val); + + testing + static inline JS_BOOL JS_IsNumber(JSValueConst v) + static inline JS_BOOL JS_IsBool(JSValueConst v) + static inline JS_BOOL JS_IsNull(JSValueConst v) + static inline JS_BOOL JS_IsUndefined(JSValueConst v) + static inline JS_BOOL JS_IsString(JSValueConst v) + static inline JS_BOOL JS_IsSymbol(JSValueConst v) + static inline JS_BOOL JS_IsObject(JSValueConst v) + int JS_IsArray(JSContext *ctx, JSValueConst val); + + Accessors: + Object: + JSValue JS_GetPropertyStr(JSContext *ctx, JSValueConst this_obj, + const char *prop); + Array: + JSValue JS_GetPropertyUint32(JSContext *ctx, JSValueConst this_obj, + uint32_t idx); + + Here + Convert with deletions/filtering + ejsEvalBuffer needs to return value and not free the JSValue + JSTest can show values + simple print + optional pretty print + expression can be as-is + function( -the 2nd dimensions) { <-- if object + return + + */ + +static Json *jsToJson1(EmbeddedJS *ejs, + JsonBuilder *b, Json *parent, char *parentKey, + JSValue value, + int depth){ + int buildStatus = 0; + int32_t tag = JS_VALUE_GET_TAG(value); + JSContext *ctx = ejs->ctx; + /* printf("jsToJSON1 tag=%d\n",tag);*/ + if (JS_IsUndefined(value)){ + fprintf(stderr,"*** Panic *** Undefined not handled\n"); + fflush(stderr); + return NULL; + } else if (JS_IsSymbol(value)){ + fprintf(stderr,"*** Panic *** Symbol not handled\n"); + fflush(stderr); + return NULL; + } else if (JS_IsNumber(value)){ + /* printf("number case\n"); */ + // Int64 vs, 32 vs float + if (JS_TAG_IS_FLOAT64(tag)){ + return jsonBuildDouble(b,parent,parentKey,JS_VALUE_GET_FLOAT64(value),&buildStatus); + } else { + int intValue = 0; + int intStatus = JS_ToInt32(ctx,&intValue,value); + /* printf("non float %d, status=%d\n",intValue,intStatus); */ + return jsonBuildInt64(b,parent,parentKey,(int64_t)intValue,&buildStatus); + } + } else if (JS_IsBool(value)){ + return jsonBuildBool(b,parent,parentKey,(bool)JS_VALUE_GET_BOOL(value),&buildStatus); + } else if (JS_IsString(value)){ + const char *str = JS_ToCString(ctx, value); + if (str == NULL){ + fprintf(stderr,"*** Panic *** Could not extract CString from Value\n"); + fflush(stderr); + return NULL; + } else { + size_t strLen = strlen(str); + char nativeStr[strLen+1]; + snprintf (nativeStr, strLen + 1, "%.*s", (int)strLen, str); + convertToNative(nativeStr, strLen); + Json *jsonString = jsonBuildString(b,parent,parentKey,(char*)nativeStr,strlen(nativeStr),&buildStatus); + JS_FreeCString(ctx, str); + return jsonString; + } + } else if (JS_IsNull(value)){ + return jsonBuildNull(b,parent,parentKey,&buildStatus); + } else if (JS_IsObject(value)){ + Json *jsonObject = jsonBuildObject(b,parent,parentKey,&buildStatus); + /* iterate properties */ + JSPropertyEnum *properties = NULL; /* I assume this will get fresh storage */ + uint32_t propertyCount = 0; + if (JS_GetOwnPropertyNames(ctx,&properties,&propertyCount,value,JS_GPN_STRING_MASK /* flags */)){ + printf("getOwnProperties failed\n"); + return NULL; + } else { + for (uint32_t i=0; iatom); + JSPropertyDescriptor descriptor; + /* BEWARE!!! + JS_GetOwnProperty has *NASTY* 3-valued logic. < 0 error, 0 not found, 1 found */ + int valueStatus = JS_GetOwnProperty(ctx,&descriptor,value,property->atom); + const char *cPropertyName = JS_ToCString(ctx,propertyName); + /* printf("jsToJson property i=%d %s valStatus=%d\n",i,cPropertyName,valueStatus); */ + if (valueStatus > 0){ + jsToJson1(ejs,b,jsonObject,(char*)cPropertyName,descriptor.value,depth+1); + } else { + printf("*** WARNING *** could not get value for property '%s', status=%d\n",cPropertyName,valueStatus); + } + + } + } + + return jsonObject; + } else if (JS_IsArray(ctx,value)){ + Json *jsonArray = jsonBuildArray(b,parent,parentKey,&buildStatus); + JSValue aLenJS = JS_GetPropertyStr(ctx,value,"length"); + int aLen = 0; + buildStatus = JS_ToInt32(ctx,&aLen,aLenJS); + printf("JS array len = %d\n",aLen); + for (int i=0; ictx; + switch (json->type) { + case JSON_TYPE_NUMBER: + case JSON_TYPE_INT64: + return JS_NewInt64(ctx,(int64_t)jsonAsInt64(json)); + case JSON_TYPE_DOUBLE: + return JS_NewFloat64(ctx,jsonAsDouble(json)); + case JSON_TYPE_STRING: + { + char *str = jsonAsString(json); + size_t strLen = strlen(str); + char convertedStr[strLen+1]; + snprintf (convertedStr, strLen + 1, "%.*s", (int)strLen, str); + /* printf ("about to convert string '%s'\n", convertedStr); */ + convertFromNative(convertedStr, strLen); + return JS_NewString(ctx, convertedStr); + } + case JSON_TYPE_BOOLEAN: + return JS_NewBool(ctx,jsonAsBoolean(json)); + case JSON_TYPE_NULL: + return JS_NULL; + case JSON_TYPE_OBJECT: + { + if (hideUnevaluated && isZoweUnevaluated(json)){ + return JS_UNDEFINED; + } else { + JsonObject *jsonObject = jsonAsObject(json); + JSValue object = JS_NewObject(ctx); + JsonProperty *property; + + for (property = jsonObjectGetFirstProperty(jsonObject); + property != NULL; + property = jsonObjectGetNextProperty(property)) { + { + char *key = jsonPropertyGetKey(property); + size_t keyLen = strlen(key); + char convertedKey[keyLen+1]; + snprintf (convertedKey, keyLen + 1, "%.*s", (int)keyLen, key); + /* printf ("about to convert key '%s'\n", convertedKey); */ + convertFromNative(convertedKey, keyLen); + JS_SetPropertyStr(ctx, + object, + convertedKey, + jsonToJS1(ejs,jsonPropertyGetValue(property),hideUnevaluated)); + } + } + return object; + } + } + case JSON_TYPE_ARRAY: + { + JsonArray *jsonArray = jsonAsArray(json); + JSValue array = JS_NewArray(ctx); + int count = jsonArrayGetCount(jsonArray); + + for (uint32_t i = 0; i < count; i++) { + JS_SetPropertyUint32(ctx, + array, + i, + jsonToJS1(ejs,jsonArrayGetItem(jsonArray,i),hideUnevaluated)); + } + return array; + } + default: + printf("*** PANIC *** unknown JSON type %d\n",json->type); + /* The next line is cheesy, but ... */ + return JS_NULL; + } +} + + + + +static JSValue ejsJsonToJS_internal(EmbeddedJS *ejs, Json *json){ + return jsonToJS1(ejs,json,false); +} + +JSValueBox ejsJsonToJS(EmbeddedJS *ejs, Json *json){ + JSValue value = ejsJsonToJS_internal(ejs,json); + return *((JSValueBox*)&value); +} + +char *json2JS(Json *json){ + JsonBuffer *buffer = makeJsonBuffer(); + /* jsonPrinter *p = makeBufferJsonPrinter(sourceCCSID,buffer); */ +#ifdef __ZOWE_OS_WINDOWS + int stdoutFD = _fileno(stdout); +#elif defined(STDOUT_FILENO) + int stdoutFD = STDOUT_FILENO; +#else + int stdoutFD = 1; /* this looks hacky, but it's been true for about 50 years */ +#endif + jsonPrinter *p = makeJsonPrinter(stdoutFD); + jsonEnablePrettyPrint(p); + + return NULL; +} + +static void visitJSON(Json *json, Json *parent, char *key, int index, + bool (*visitor)(void *context, Json *value, Json *parent, char *key, int index), + void *context){ + switch (json->type) { + case JSON_TYPE_NUMBER: + case JSON_TYPE_INT64: + case JSON_TYPE_DOUBLE: + case JSON_TYPE_STRING: + case JSON_TYPE_BOOLEAN: + case JSON_TYPE_NULL: + visitor(context,json,parent,key,index); + break; + case JSON_TYPE_OBJECT: + { + JsonObject *jsonObject = jsonAsObject(json); + JsonProperty *property; + for (property = jsonObjectGetFirstProperty(jsonObject); + property != NULL; + property = jsonObjectGetNextProperty(property)) { + char *key = jsonPropertyGetKey(property); + Json *value = jsonPropertyGetValue(property); + bool shouldVisitChildren = visitor(context,value,json,key,-1); + if (shouldVisitChildren){ + visitJSON(value,json,key,-1,visitor,context); + } + } + } + break; + case JSON_TYPE_ARRAY: + { + JsonArray *jsonArray = jsonAsArray(json); + int count = jsonArrayGetCount(jsonArray); + for (uint32_t i = 0; i < count; i++) { + Json *element = jsonArrayGetItem(jsonArray,i); + bool shouldVisitChildren = visitor(context,element,json,NULL,i); + if (shouldVisitChildren){ + visitJSON(element,json,NULL,i,visitor,context); + } + } + } + break; + default: + printf("*** PANIC *** unknown JSON type %d\n",json->type); + /* The next line is cheesy, but ... */ + break; + } +} + +static void setJsonProperty(JsonObject *object, char *key, Json *value){ + JsonProperty *property; + for (property = jsonObjectGetFirstProperty(object); + property != NULL; + property = jsonObjectGetNextProperty(property)) { + char *propertyKey = jsonPropertyGetKey(property); + if (!strcmp(propertyKey,key)){ + property->value = value; + break; + } + } +} + +static void setJsonArrayElement(JsonArray *array, int index, Json *value){ + array->elements[index] = value; +} + +typedef struct TemplateEvaluationContext_tag { + EmbeddedJS *ejs; + Json *topJson; + ShortLivedHeap *slh; +} TemplateEvaluationContext; + +static bool evaluationVisitor(void *context, Json *json, Json *parent, char *keyInParent, int indexInParent){ + if (isZoweUnevaluated(json)){ + JsonObject *object = jsonAsObject(json); + JsonProperty *property; + TemplateEvaluationContext *evalContext = (TemplateEvaluationContext*)context; + EmbeddedJS *ejs = evalContext->ejs; + JsonObject *topObject = jsonAsObject(evalContext->topJson); + for (property = jsonObjectGetFirstProperty(topObject); + property != NULL; + property = jsonObjectGetNextProperty(property)) { + char *key = jsonPropertyGetKey(property); + Json *value = jsonPropertyGetValue(property); + /* printf ("global object key '%s'\n", key); */ + size_t keyLen = strlen(key); + char convertedKey[keyLen+1]; + snprintf (convertedKey, keyLen + 1, "%.*s", (int)keyLen, key); + convertFromNative(convertedKey, keyLen); + ejsSetGlobalProperty_internal(ejs,convertedKey,ejsJsonToJS_internal(ejs,value)); + } + Json *sourceValue = jsonObjectGetPropertyValue(object,"source"); + if (sourceValue){ + char *source = jsonAsString(sourceValue); + size_t sourceLen = strlen(source); + char asciiSource[sourceLen + 1]; + snprintf (asciiSource, sourceLen + 1, "%.*s", (int)sourceLen, source); + convertFromNative(asciiSource, sourceLen); + /* printf("should evaluate: %s\n",source); */ + int evalStatus = 0; + char embedded[] = ""; + convertFromNative(embedded, sizeof(embedded)); + JSValue output = ejsEvalBuffer1(ejs,asciiSource,strlen(asciiSource),embedded,0,&evalStatus); + if (evalStatus){ + printf("failed to evaluate '%s', status=%d\n",source,evalStatus); + } else { + /* + printf("evaluation succeeded\n"); + dumpbuffer((char*)&output,sizeof(JSValue)); + */ + Json *evaluationResult = ejsJSToJson_internal(ejs,output,evalContext->slh); + if (evaluationResult){ + if (keyInParent){ + setJsonProperty(jsonAsObject(parent),keyInParent,evaluationResult); + } else { + setJsonArrayElement(jsonAsArray(parent),indexInParent,evaluationResult); + } + } else { + printf("Warning EJS failed to translate eval result JSValue to Json\n"); + } + } + } + return false; + } else { + return true; + } +} + +Json *evaluateJsonTemplates(EmbeddedJS *ejs, ShortLivedHeap *slh, Json *json){ + /* jsonToJS1(ejs,json,false); */ + TemplateEvaluationContext evalContext; + if (jsonIsObject(json)){ + JsonObject *topObject = jsonAsObject(json); + + evalContext.ejs = ejs; + evalContext.slh = slh; + evalContext.topJson = json; + visitJSON(json,NULL,NULL,-1,evaluationVisitor,&evalContext); + return json; + } else { + printf("top json is not an object\n"); + return NULL; + } +} + + + + +EmbeddedJS *makeEmbeddedJS(EmbeddedJS *sharedRuntimeEJS){ /* can be NULL */ + EmbeddedJS *embeddedJS = (EmbeddedJS*)safeMalloc(sizeof(EmbeddedJS),"EmbeddedJS"); + memset(embeddedJS,0,sizeof(EmbeddedJS)); + if (sharedRuntimeEJS){ + embeddedJS->rt = sharedRuntimeEJS->rt; + } else { + JSRuntime *rt = JS_NewRuntime(); + embeddedJS->rt = rt; + } + /* + JS_SetMemoryLimit(rt, memory_limit); + JS_SetMaxStackSize(rt, stack_size); + */ + js_std_set_worker_new_context_func(makeEmbeddedJSContext); + js_std_init_handlers(embeddedJS->rt); + JSContext *ctx = makeEmbeddedJSContext(embeddedJS->rt); + embeddedJS->ctx = ctx; + if (!ctx) { + fprintf(stderr, "qjs: cannot allocate JS context\n"); + exit(2); + } + + + /* loader for ES6 modules */ + JS_SetModuleLoaderFunc(embeddedJS->rt, NULL, js_module_loader, NULL); + + if (false){ /* dump_unhandled_promise_rejection) { - do this later */ + JS_SetHostPromiseRejectionTracker(embeddedJS->rt, js_std_promise_rejection_tracker, + NULL); + } + js_std_add_helpers(ctx, 0, NULL); /* argc - optind, argv + optind); */ + + int evalStatus = 0; + /* make 'std' and 'os' visible to non module code */ + if (true){ /* load_std) {*/ + const char *str = "import * as std from 'std';\n" + "import * as os from 'os';\n" + "import * as experiment from 'experiment';\n" + "globalThis.std = std;\n" + "globalThis.os = os;\n" + "globalThis.experiment = experiment;\n"; + JSValue throwaway = ejsEvalBuffer1(embeddedJS, str, strlen(str), "", JS_EVAL_TYPE_MODULE, &evalStatus); + } + + return embeddedJS; +} + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/fdpoll.c b/c/fdpoll.c index 2ea3da6e4..9c8064297 100755 --- a/c/fdpoll.c +++ b/c/fdpoll.c @@ -13,6 +13,10 @@ #endif +/* xlclang and clang demand a prototype */ + +int BPXPOL(); + int fdPoll(PollItem* fds, short nmqs, short nfds, int timeout, int *returnCode, int *reasonCode) { int returnValue; int *reasonCodePtr; @@ -45,4 +49,4 @@ int fdPoll(PollItem* fds, short nmqs, short nfds, int timeout, int *returnCode, #else #error No implemention for fdpoll.h -#endif \ No newline at end of file +#endif diff --git a/c/httpserver.c b/c/httpserver.c index 3b8643727..1553f3d41 100644 --- a/c/httpserver.c +++ b/c/httpserver.c @@ -2464,6 +2464,19 @@ static char *getCookieValue(HttpRequest *request, char *cookieName){ return NULL; } +#ifdef __ZOWE_OS_ZOS +int isLowerCasePasswordAllowed(){ + RCVT* rcvt = getCVT()->cvtrac; + return (RCVTFLG3_BIT_RCVTPLC & (rcvt->rcvtflg3)); + /* if lower-case pw allowed */ +} +#else +int isLowerCasePasswordAllowed(){ + return TRUE; +} +#endif + + #ifdef __ZOWE_OS_ZOS static int safAuthenticate(HttpService *service, HttpRequest *request, AuthResponse *authResponse){ int safStatus = 0, racfStatus = 0, racfReason = 0; diff --git a/c/json.c b/c/json.c index 02f6ec0a0..5a1b12b27 100644 --- a/c/json.c +++ b/c/json.c @@ -16,25 +16,26 @@ #include #include #include +#include #include #include #include +#include #include "metalio.h" #include "qsam.h" #else -#ifndef _XOPEN_SOURCE -#define _XOPEN_SOURCE 600 -#endif - #include #include #include #include #include +#include #include #include +#include +#include #include #endif @@ -48,10 +49,24 @@ #include "json.h" #include "charsets.h" +#ifdef __ZOWE_OS_WINDOWS +typedef int64_t ssize_t; +#endif + +/* Having lots of problems including unistd, so here's a hack */ +#ifdef __ZOWE_OS_ZOS +ssize_t write(int fd, const void *buf, size_t count); +#endif + + /* - * - c89 -DTEST_JSON_PARSER "-Wc,langlvl(extc99),gonum,goff,hgpr,roconst,ASM,asmlib('SYS1.MACLIB')" -o parser -I ../h json.c utils.c alloc.c bpxskt.c charsets.c - c89 -DTEST_JSON_PRINTER "-Wc,langlvl(extc99),gonum,goff,hgpr,roconst,ASM,asmlib('SYS1.MACLIB')" -o printer -I ../h json.c utils.c alloc.c bpxskt.c charsets.c + + Some unit tests for parsing and printing. Updated 2022 for current filenames and modularity, and 64-bitness. + + xlc -q64 -DTEST_JSON_PARSER -D_OPEN_SYS_FILE_EXT=1 -D_XOPEN_SOURCE=600 -D_OPEN_THREADS=1 -DSUBPOOL=132 "-Wc,float(ieee),longname,langlvl(extc99),gonum,goff,ASM,asmlib('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN')" -I ../h -o jsonparser json.c zosfile.c collections.c charsets.c logging.c zos.c recovery.c scheduling.c le.c timeutls.c utils.c alloc.c + + xlc -q64 -DTEST_JSON_PRINTER -D_OPEN_SYS_FILE_EXT=1 -D_XOPEN_SOURCE=600 -D_OPEN_THREADS=1 -DSUBPOOL=132 "-Wc,float(ieee),longname,langlvl(extc99),gonum,goff,ASM,asmlib('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN')" -I ../h -o jsonprinter json.c zosfile.c collections.c charsets.c logging.c zos.c recovery.c scheduling.c le.c timeutls.c utils.c alloc.c + */ #define CONVERSION_BUFFER_DEFAULT_SIZE 8192 @@ -65,13 +80,15 @@ #define JSON_DEBUG(...) /* fprintf(stderr, __VA_ARGS__) */ #define DUMPBUF($b, $l) /* dumpBufferToStream($b, $l, stderr) */ +/* JOE ERROR() is a defined macro in windows, so it needs a more specific name here */ + #ifdef METTLE -# define ERROR(...) /* TODO */ +# define JSONERROR(...) /* TODO */ #else /* TODO Implement good error handling: the caller must have a way to know if * an error happened. Most methods currently are void and there's no error * flag on the printer structure */ -# define ERROR(...) fprintf(stderr, __VA_ARGS__) +# define JSONERROR(...) fprintf(stderr, __VA_ARGS__) #endif static @@ -88,10 +105,7 @@ int jsonIsError(Json *json) { static void writeToBuffer(struct jsonPrinter_tag *p, char *text, int len); -jsonPrinter *makeJsonPrinter(int fd) { - jsonPrinter *p = (jsonPrinter*) safeMalloc(sizeof (jsonPrinter), "JSON Printer"); - - p->fd = fd; +void jsonPrinterReset(jsonPrinter *p){ p->depth = 0; p->indentString = " "; p->isStart = TRUE; @@ -99,6 +113,12 @@ jsonPrinter *makeJsonPrinter(int fd) { p->_conversionBufferSize = 0; p->_conversionBuffer = NULL; p->mode = JSON_MODE_NATIVE_CHARSET; +} + +jsonPrinter *makeJsonPrinter(int fd) { + jsonPrinter *p = (jsonPrinter*) safeMalloc(sizeof (jsonPrinter), "JSON Printer"); + p->fd = fd; + jsonPrinterReset(p); return p; } @@ -115,13 +135,7 @@ jsonPrinter *makeCustomJsonPrinter(void (*writeMethod)(jsonPrinter *, char *, in p->isCustom = TRUE; p->customWrite = writeMethod; p->customObject = object; - p->depth = 0; - p->indentString = " "; - p->isStart = TRUE; - p->isFirstLine = TRUE; - p->_conversionBufferSize = 0; - p->_conversionBuffer = NULL; - p->mode = JSON_MODE_NATIVE_CHARSET; + jsonPrinterReset(p); return p; } @@ -164,7 +178,17 @@ void jsonWriteBufferInternal(jsonPrinter *p, char *text, int len) { p->customWrite(p, text, len); } else { while (bytesWritten < len) { -#ifndef METTLE +#if defined(__ZOWE_OS_WINDOWS) + int newWriteReturn = -666; /* send(p->fd,text+bytesWritten,len-bytesWritten,0); */ + if (p->fd == _fileno(stdout)){ + newWriteReturn = fwrite(text, 1, len, stdout); + } else if (p->fd == _fileno(stderr)){ + newWriteReturn = fwrite(text, 1, len, stderr); + } else { + JSONERROR("JSON: can only write to stderr or stdout on Windows, not p->fd=%d\n",p->fd); + jsonSetIOErrorFlag(p); + } +#elif !defined(METTLE) int newWriteReturn = write(p->fd,text+bytesWritten,len-bytesWritten); #else int newWriteReturn = socketWrite((Socket *)p->fd,text+bytesWritten,len-bytesWritten, @@ -173,14 +197,14 @@ void jsonWriteBufferInternal(jsonPrinter *p, char *text, int len) { loopCount++; if (newWriteReturn < 0) { /* TODO: Replace by zowelog(...) */ - ERROR("JSON: write error, rc %d, return code %d, reason code %08X\n", + JSONERROR("JSON: write error, rc %d, return code %d, reason code %08X\n", newWriteReturn, returnCode, reasonCode); jsonSetIOErrorFlag(p); break; } if (loopCount > 10) { /* TODO: Replace by zowelog(...) */ - ERROR("JSON: write error, too many attempts\n"); + JSONERROR("JSON: write error, too many attempts\n"); jsonSetIOErrorFlag(p); break; } @@ -194,16 +218,10 @@ void jsonWriteBufferInternal(jsonPrinter *p, char *text, int len) { #define ESCAPE_LEN 6 /* \u0123 */ #define ESCAPE_NATIVE_BUF_SIZE 12 /* some bytes have been added for safety */ -#if defined(__ZOWE_OS_ZOS) -# pragma convert("UTF8") -#endif -static char UTF8_QUOTE = '"'; -static char UTF8_BACKSLASH = '\\'; -static char UTF8_ESCAPED_QUOTE[2] = "\\\""; -static char UTF8_ESCAPED_BACKSLASH[2]= "\\\\"; -#if defined(__ZOWE_OS_ZOS) -# pragma convert(pop) -#endif +static char UTF8_QUOTE = 0x22; +static char UTF8_BACKSLASH = 0x5C; +static char UTF8_ESCAPED_QUOTE[2] = { 0x5C, 0x22 }; +static char UTF8_ESCAPED_BACKSLASH[2]={ 0x5C, 0x5C}; #define UTF8_GET_NEXT_CHAR($size, $buf, $idx, $outChar, $err) do { \ if ((($buf)[*($idx)] & 0x80) == 0) { \ @@ -244,8 +262,8 @@ convertToUtf8(jsonPrinter *p, size_t len, char text[len], int inputCCSID) { newBuf = safeRealloc(p->_conversionBuffer, newSize, p->_conversionBufferSize, "JSON conversion buffer"); if (newBuf == NULL) { - //the old buffer will be free'd by freeJsonPrinter() if allocated - ERROR("JSON: error, not enough memory to convert\n"); + /* the old buffer will be free'd by freeJsonPrinter() if allocated */ + JSONERROR("JSON: error, not enough memory to convert\n"); return -1; } p->_conversionBufferSize = newSize; @@ -257,14 +275,14 @@ convertToUtf8(jsonPrinter *p, size_t len, char text[len], int inputCCSID) { if (convRc == CHARSET_CONVERSION_SUCCESS) { return convesionOutputLength; } else if (convRc != CHARSET_SHORT_BUFFER) { - ERROR("JSON: conversion error, rc %d, reason %d\n", convRc, + JSONERROR("JSON: conversion error, rc %d, reason %d\n", convRc, reasonCode); return -1; } } while (reallocLimit-- > 0); if (reallocLimit == 0) { //the old buffer will be free'd by freeJsonPrinter() if allocated - ERROR("JSON: error, not enough memory to convert\n"); + JSONERROR("JSON: error, not enough memory to convert\n"); return -1; } return len; @@ -291,13 +309,15 @@ writeBufferWithEscaping(jsonPrinter *p, size_t len, char text[len]) { UTF8_GET_NEXT_CHAR(len, text, &i, &utf8Char, &getCharErr); if (getCharErr != 0) { - ERROR("JSON: invalid UTF-8, rc %d\n", getCharErr); + JSONERROR("JSON: invalid UTF-8, rc %d\n", getCharErr); return -1; } - JSON_DEBUG("character at %p + %d, len %d, char %x\n", text, i, len, utf8Char); - if (((utf8Char >= 0) && (utf8Char <= effectiveControlCharBoundary)) - || (utf8Char == UTF8_BACKSLASH) || (utf8Char == UTF8_QUOTE)) { + JSON_DEBUG("character at %p + %d, len %d, char %x\n", text, i, len, utf8Char); + if ((utf8Char <= effectiveControlCharBoundary) || + (utf8Char == UTF8_BACKSLASH) || + (utf8Char == UTF8_QUOTE)) { + chunkLen = i - 1 - currentChunkOffset; if (chunkLen > 0) { jsonWriteBufferInternal(p, text + currentChunkOffset, chunkLen); @@ -326,7 +346,7 @@ writeBufferWithEscaping(jsonPrinter *p, size_t len, char text[len]) { CHARSET_OUTPUT_USE_BUFFER, &ptrToUtf8Buf, sizeof (escapeUtf), CCSID_UTF_8, NULL, &convesionOutputLength, &reasonCode); if (convRc != CHARSET_CONVERSION_SUCCESS) { - ERROR("JSON: conversion error, rc %d, reason %d\n", + JSONERROR("JSON: conversion error, rc %d, reason %d\n", convRc, reasonCode); return -1; } @@ -362,8 +382,8 @@ void jsonConvertAndWriteBuffer(jsonPrinter *p, char *text, size_t len, DUMPBUF(text, len); newLen = convertToUtf8(p, len, text, inputCCSID); if (newLen < 0) { - ERROR("jsonConvertAndWriteBuffer() error: newLen = %d\n", - newLen); + JSONERROR("jsonConvertAndWriteBuffer() error: newLen = %d\n", + (int)newLen); return; } JSON_DEBUG("utf8, len %d:\n", newLen); @@ -372,11 +392,9 @@ void jsonConvertAndWriteBuffer(jsonPrinter *p, char *text, size_t len, DUMPBUF(text, len); } if (escape) { - size_t bytesWritten; - - bytesWritten = writeBufferWithEscaping(p, len, text); - if (bytesWritten < 0) { - ERROR("jsonConvertAndWriteBuffer() error: bytesWritten = %d\n", + ssize_t bytesWritten = writeBufferWithEscaping(p, len, text); + if (bytesWritten < 0){ + JSONERROR("jsonConvertAndWriteBuffer() error: bytesWritten = %zd\n", bytesWritten); return; } @@ -399,6 +417,17 @@ void jsonWrite(jsonPrinter *p, char *text, bool escape, int inputCCSID) { jsonConvertAndWriteBuffer(p, text, strlen(text), escape, inputCCSID); } +void jsonWriteParseably(jsonPrinter *p, char *text, int len, bool quote, bool escape, int inputCCSID){ + if (quote){ + jsonWrite(p, "\"", false, inputCCSID); + } + jsonConvertAndWriteBuffer(p, text, len, escape, inputCCSID); + if (quote){ + jsonWrite(p, "\"", false, inputCCSID); + } +} + + static void jsonIndent(jsonPrinter *p) { int depth = p->depth; @@ -448,7 +477,14 @@ void jsonWriteUInt(jsonPrinter *p, unsigned int value) { static void jsonWriteInt64(jsonPrinter *p, int64 value) { char buffer[64]; - snprintf(buffer, sizeof (buffer), "%lld", value); + snprintf(buffer, sizeof (buffer), "%lld", INT64_LL(value)); + jsonWrite(p, buffer, false, SOURCE_CODE_CHARSET); +} + +static +void jsonWriteDouble(jsonPrinter *p, double value) { + char buffer[64]; + snprintf(buffer, sizeof (buffer), "%f", value); jsonWrite(p, buffer, false, SOURCE_CODE_CHARSET); } @@ -760,6 +796,20 @@ void jsonAddInt64(jsonPrinter *p, char *keyOrNull, int64 value) { jsonWriteInt64(p, value); } +void jsonAddDouble(jsonPrinter *p, char *keyOrNull, double value) { + if (jsonShouldStopWriting(p)) { + return; + } + if (p->isFirstLine) { + p->isFirstLine = FALSE; + } else { + jsonNewLine(p); + } + jsonWriteKeyAndSemicolon(p, keyOrNull); + jsonWriteDouble(p, value); +} + + void jsonSetIOErrorFlag(jsonPrinter *p) { p->ioErrorFlag = TRUE; } @@ -792,6 +842,12 @@ void freeJsonBuffer(JsonBuffer *buf) { safeFree((void *)buf, sizeof (*buf)); } +char *jsonBufferCopy(JsonBuffer *buf){ + char *data = safeMalloc(buf->len,"copyJsonBuffer"); + memcpy(data,buf->data,buf->len); + return data; +} + void jsonBufferRewind(JsonBuffer *buf) { buf->len = 0; memset(buf->data, '0', buf->size); @@ -829,17 +885,6 @@ writeToBuffer(struct jsonPrinter_tag *p, char *text, int len) { #define JSON_TOKEN_BUFFER_SIZE_LIMIT 104857600 /* 100 MB (for one token) */ #endif -typedef struct JsonParser_tag JsonParser; -typedef struct JsonTokenizer_tag JsonTokenizer; -typedef struct JsonToken_tag JsonToken; - -struct JsonParser_tag { - JsonTokenizer *tokenizer; - JsonToken *unreadToken; - ShortLivedHeap *slh; - CharStream *in; - Json *jsonError; -}; struct JsonTokenizer_tag { CharStream *in; @@ -875,6 +920,8 @@ struct JsonToken_tag { #define JSON_TOKEN_BAD_COMMENT 17 #define JSON_TOKEN_UNEXPECTED_CHAR 18 #define JSON_TOKEN_STRING_TOO_LONG 19 +#define JSON_TOKEN_INT64 20 +#define JSON_TOKEN_FLOAT 21 #define JSON_TOKEN_UNMATCHED 666 int type; char *text; @@ -895,7 +942,8 @@ static Json *jsonGetString(JsonParser* parser); static Json *jsonParse(JsonParser* parser); static JsonParser *makeJsonParser(ShortLivedHeap *slh, char *jsonString, int len); static JsonToken *getReservedWordToken(JsonTokenizer *tokenizer); -static JsonToken *getNumberToken(JsonTokenizer *tokenizer); +static JsonToken *getNumberTokenV1(JsonTokenizer *tokenizer); +static JsonToken *getNumberTokenV2(JsonTokenizer *tokenizer); static JsonToken *getStringToken(JsonTokenizer *tokenizer); static JsonToken *getLineCommentToken(JsonTokenizer *tokenizer); static JsonToken *getBlockCommentToken(JsonTokenizer *tokenizer); @@ -910,7 +958,7 @@ static void freeJsonParser(JsonParser *parser); static void freeJsonTokenizer(JsonTokenizer *tokenizer); static void jsonArrayAddElement(JsonParser *parser, JsonArray *arr, Json *element); static void jsonPrintInternal(jsonPrinter* printer, char* keyOrNull, Json *json); -static void jsonOjectAddProperty(JsonParser *parser, JsonObject *obj, char *key, Json *value); +static void jsonObjectAddProperty(JsonParser *parser, JsonObject *obj, char *key, Json *value); static void jsonParseFail(JsonParser *parser, char *formatString, ...); static void jsonTokenizerSkipWhitespace(JsonTokenizer *t); @@ -951,6 +999,8 @@ char *getTokenTypeString(int type) { return "unmatched token"; case JSON_TOKEN_STRING_TOO_LONG: return "string too long"; + case JSON_TOKEN_FLOAT: + return "float"; default: return "unknown token"; } @@ -1203,7 +1253,7 @@ JsonToken *getStringToken(JsonTokenizer *tokenizer) { } static -JsonToken *getNumberToken(JsonTokenizer *tokenizer) { +JsonToken *getNumberTokenV1(JsonTokenizer *tokenizer) { char *buffer = tokenizer->buffer; int pos = 0; int badNumber = FALSE; @@ -1231,6 +1281,59 @@ JsonToken *getNumberToken(JsonTokenizer *tokenizer) { return makeJsonToken(tokenizer, badNumber ? JSON_TOKEN_BAD_NUMBER : JSON_TOKEN_NUMBER, text); } +static int addTokenChar(JsonTokenizer *tokenizer, int pos, int c){ + if (pos < tokenizer->bufferSize){ + tokenizer->buffer[pos++] = c; + } + return pos; +} + +static +JsonToken *getNumberTokenV2(JsonTokenizer *tokenizer) { + char *buffer = tokenizer->buffer; + int pos = 0; + bool isInteger = true; + bool badNumber = false; + + if (jsonTokenizerLookahead(tokenizer) == '-') { + buffer[pos++] = jsonTokenizerRead(tokenizer); + } + if (isdigit(jsonTokenizerLookahead(tokenizer))) { + buffer[pos++] = jsonTokenizerRead(tokenizer); + while (isdigit(jsonTokenizerLookahead(tokenizer))) { + int c = jsonTokenizerRead(tokenizer); + pos = addTokenChar(tokenizer,pos,c); + } + if (jsonTokenizerLookahead(tokenizer) == '.'){ + isInteger = false; + pos = addTokenChar(tokenizer,pos,jsonTokenizerRead(tokenizer)); + if (isdigit(jsonTokenizerLookahead(tokenizer))){ + pos = addTokenChar(tokenizer,pos,jsonTokenizerRead(tokenizer)); + while (isdigit(jsonTokenizerLookahead(tokenizer))) { + int c = jsonTokenizerRead(tokenizer); + pos = addTokenChar(tokenizer,pos,c); + } + } else { + badNumber = true; + } + } + /* must have at least one number, followed by a lookahead to a non-number here */ + } else { + badNumber = true; + int c = jsonTokenizerRead(tokenizer); + pos = addTokenChar(tokenizer,pos,c); + } + char *text = jsonTokenizerAlloc(tokenizer, pos + 1); + memcpy(text, buffer, pos); + if (badNumber){ + return makeJsonToken(tokenizer,JSON_TOKEN_BAD_NUMBER,text); + } else if (isInteger){ + return makeJsonToken(tokenizer,JSON_TOKEN_INT64,text); + } else { + return makeJsonToken(tokenizer,JSON_TOKEN_FLOAT,text); + } +} + static JsonToken *getReservedWordToken(JsonTokenizer *tokenizer) { char *buffer = tokenizer->buffer; @@ -1368,10 +1471,15 @@ JsonToken *jsonNextToken(JsonParser* parser) { lookahead = jsonTokenizerLookahead(tokenizer); tokenizer->lastTokenLineNumber = tokenizer->lineNumber; tokenizer->lastTokenColumnNumber = tokenizer->columnNumber; + /* printf("JOE jsonNextToken lookahead=0x%x '%c'\n",(int)lookahead,(char)lookahead); + fflush(stdout); + */ if (lookahead == '\"') { token = getStringToken(tokenizer); } else if (isdigit(lookahead) || lookahead == '-') { - token = getNumberToken(tokenizer); + token = (parser->version >= 2 ? + getNumberTokenV2(tokenizer) : + getNumberTokenV1(tokenizer)); } else if (isalpha(lookahead)) { token = getReservedWordToken(tokenizer); } else if (lookahead == '{') { @@ -1401,12 +1509,13 @@ JsonToken *jsonNextToken(JsonParser* parser) { jsonParseFail(parser, "unexpected character %c(0x%x)\n", lookahead, lookahead); token = makeJsonToken(tokenizer, JSON_TOKEN_UNEXPECTED_CHAR, ""); } - + if (token->type == JSON_TOKEN_LINE_COMMENT || token->type == JSON_TOKEN_BLOCK_COMMENT) { token = jsonNextToken(parser); } jsonValidateToken(parser, token); - + + /* printf("JOE returning next token=0x%p\n",token);fflush(stdout); */ return token; } @@ -1475,6 +1584,35 @@ Json *jsonGetNumber(JsonParser* parser) { return json; } +static +Json *jsonGetInt64(JsonParser* parser) { + Json *json = NULL; + JsonToken *token = jsonMatchToken(parser, JSON_TOKEN_INT64); + if (!jsonIsTokenUnmatched(token)) { + json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_INT64; + json->data.integerValue = strtoll(token->text,NULL,10); + } else { + json = parser->jsonError; + } + return json; +} + +static +Json *jsonGetFloat(JsonParser* parser) { + Json *json = NULL; + JsonToken *token = jsonMatchToken(parser, JSON_TOKEN_FLOAT); + if (!jsonIsTokenUnmatched(token)) { + json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_DOUBLE; + json->data.floatValue = strtod(token->text,NULL); + } else { + json = parser->jsonError; + } + return json; +} + + static Json *jsonGetString(JsonParser* parser) { Json *json = NULL; @@ -1517,7 +1655,7 @@ Json *jsonGetNull(JsonParser* parser) { } static -void jsonOjectAddProperty(JsonParser *parser, JsonObject *obj, char *key, Json *value) { +void jsonObjectAddProperty(JsonParser *parser, JsonObject *obj, char *key, Json *value) { JsonProperty *property = (JsonProperty*) jsonParserAlloc(parser, sizeof (JsonProperty)); property->key = key; property->value = value; @@ -1600,10 +1738,13 @@ int jsonArrayContainsString(JsonArray *array, const char *s){ static Json *jsonGetObject(JsonParser* parser) { JsonToken *token = jsonMatchToken(parser, JSON_TOKEN_OBJECT_START); + /* printf("JGO ck.1\n");fflush(stdout); */ if (jsonIsTokenUnmatched(token)) { + /* printf("JGO ck.1.1\n");fflush(stdout); */ return parser->jsonError; } Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + /* printf("JGO ck.2\n");fflush(stdout); */ JsonObject *obj = (JsonObject*) jsonParserAlloc(parser, sizeof (JsonObject)); json->type = JSON_TYPE_OBJECT; json->data.object = obj; @@ -1619,7 +1760,7 @@ static Json *jsonGetObject(JsonParser* parser) { if (jsonIsError(valueJSON)) { return parser->jsonError; } - jsonOjectAddProperty(parser, obj, keyToken->text, valueJSON); + jsonObjectAddProperty(parser, obj, keyToken->text, valueJSON); lookahead = jsonLookaheadToken(parser); if (lookahead->type == JSON_TOKEN_COMMA) { jsonMatchToken(parser, JSON_TOKEN_COMMA); @@ -1678,10 +1819,17 @@ static Json *jsonParse(JsonParser* parser) { JsonToken *lookahead = jsonLookaheadToken(parser); Json *json = NULL; if (lookahead) { + /* printf("JOE: lookahead type %d\n",lookahead->type);fflush(stdout); */ switch (lookahead->type) { case JSON_TOKEN_NUMBER: json = jsonGetNumber(parser); break; + case JSON_TOKEN_INT64: + json = jsonGetInt64(parser); + break; + case JSON_TOKEN_FLOAT: + json = jsonGetFloat(parser); + break; case JSON_TOKEN_STRING: json = jsonGetString(parser); break; @@ -1705,6 +1853,237 @@ static Json *jsonParse(JsonParser* parser) { return json; } + +JsonBuilder *makeJsonBuilder(ShortLivedHeap *slh){ + JsonBuilder *builder = (JsonBuilder*) safeMalloc(sizeof (JsonBuilder), "JSON Builder"); + memset(builder,0,sizeof(JsonBuilder)); + JsonParser *parser = (JsonParser*)builder; + parser->slh = slh; + return builder; +} + +#define ADD_TO_ROOT -1 +#define ADD_TO_OBJECT -2 +#define ADD_TO_ARRAY -3 + +void freeJsonBuilder(JsonBuilder *builder, bool freeSLH){ + if (freeSLH){ + JsonParser *parser = (JsonParser*)builder; + SLHFree(parser->slh); + } + safeFree((char*)builder,sizeof(JsonBuilder)); +} + +static int checkParentLValue(Json *parent, + char *parentKey){ + if (parent){ + switch (parent->type){ + case JSON_TYPE_OBJECT: + return (parentKey ? ADD_TO_OBJECT : JSON_BUILD_FAIL_NO_KEY_ON_OBJECT); + case JSON_TYPE_ARRAY: + return (parentKey ? JSON_BUILD_FAIL_KEY_ON_ARRAY : ADD_TO_ARRAY); + default: + return JSON_BUILD_FAIL_PARENT_IS_SCALAR; + } + } else { + return (parentKey ? JSON_BUILD_FAIL_KEY_ON_ROOT : ADD_TO_ROOT); + } +} + +static void addToParent(JsonBuilder *b, + int lvalueStatus, + Json *parent, + char *parentKey, + Json *json){ + JsonParser *p = (JsonParser*)b; + switch (lvalueStatus){ + case ADD_TO_ROOT: + b->root = json; + break; + case ADD_TO_OBJECT: + jsonObjectAddProperty(p,parent->data.object,parentKey,json); + break; + case ADD_TO_ARRAY: + jsonArrayAddElement(p,parent->data.array,json); + break; + } +} + +Json *jsonBuildObject(JsonBuilder *b, + Json *parent, + char *parentKey, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + JsonObject *obj = (JsonObject*) jsonParserAlloc(parser, sizeof (JsonObject)); + json->type = JSON_TYPE_OBJECT; + json->data.object = obj; + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + } else { + *errorCode = lvalueStatus; + return NULL; + } +} + +Json *jsonBuildArray(JsonBuilder *b, + Json *parent, + char *parentKey, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + JsonArray *arr = (JsonArray*) jsonParserAlloc(parser, sizeof (JsonArray)); + arr->count = 0; + arr->capacity = 8; + arr->elements = (Json**) jsonParserAlloc(parser, sizeof (Json*) * arr->capacity); + json->type = JSON_TYPE_ARRAY; + json->data.array = arr; + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + + } else { + *errorCode = lvalueStatus; + return NULL; + } +} + +Json *jsonBuildString(JsonBuilder *b, + Json *parent, + char *parentKey, + char *s, + int sLen, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_STRING; + char *copy = jsonParserAlloc(parser,sLen+1); + memcpy(copy,s,sLen); + copy[sLen] = 0; + json->data.string = copy; + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + + } else { + *errorCode = lvalueStatus; + return NULL; + } +} + +Json *jsonBuildInt(JsonBuilder *b, + Json *parent, + char *parentKey, + int i, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_NUMBER; + json->data.number = i; + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + } else { + *errorCode = lvalueStatus; + return NULL; + } + +} + +Json *jsonBuildInt64(JsonBuilder *b, + Json *parent, + char *parentKey, + int64 i, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_INT64; + json->data.integerValue = i; + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + } else { + *errorCode = lvalueStatus; + return NULL; + } +} + +Json *jsonBuildDouble(JsonBuilder *b, + Json *parent, + char *parentKey, + double d, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_DOUBLE; + json->data.floatValue = d; + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + } else { + *errorCode = lvalueStatus; + return NULL; + } +} + +Json *jsonBuildBool(JsonBuilder *b, + Json *parent, + char *parentKey, + bool truthValue, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_BOOLEAN; + json->data.boolean = (truthValue ? 1 : 0); + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + } else { + *errorCode = lvalueStatus; + return NULL; + } +} + +Json *jsonBuildNull(JsonBuilder *b, + Json *parent, + char *parentKey, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_NULL; + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + } else { + *errorCode = lvalueStatus; + return NULL; + } +} + +char* jsonBuildKey(JsonBuilder *b, const char *key, int len) { + JsonParser *parser = (JsonParser*)b; + char *keyCopy = jsonParserAlloc(parser, len + 1); + snprintf(keyCopy, len+1, "%.*s", len, key); + return keyCopy; +} + int jsonIsObject(Json *json) { return json->type == JSON_TYPE_OBJECT; } @@ -1722,7 +2101,17 @@ int jsonIsBoolean(Json *json) { } int jsonIsNumber(Json *json) { - return json->type == JSON_TYPE_NUMBER; + return (json->type == JSON_TYPE_NUMBER || /* old-skool int */ + json->type == JSON_TYPE_DOUBLE || + json->type == JSON_TYPE_INT64); +} + +int jsonIsInt64(Json *json) { + return json->type == JSON_TYPE_INT64; +} + +int jsonIsDouble(Json *json) { + return json->type == JSON_TYPE_DOUBLE; } int jsonIsNull(Json *json) { @@ -1760,10 +2149,81 @@ int jsonAsBoolean(Json *json) { int jsonAsNumber(Json *json) { if (json->type == JSON_TYPE_NUMBER) { return json->data.number; + } else if (json->type == JSON_TYPE_INT64){ + return json->data.integerValue; } return 0; } +int64_t jsonAsInt64(Json *json){ + switch (json->type){ + case JSON_TYPE_NUMBER: + return (int64)json->data.number; + case JSON_TYPE_INT64: + return json->data.integerValue; + case JSON_TYPE_DOUBLE: + return (int64)json->data.floatValue; + default: + return 0; + } +} + +double jsonAsDouble(Json *json){ + switch (json->type){ + case JSON_TYPE_NUMBER: + return (double)json->data.number; + case JSON_TYPE_INT64: + return (double)json->data.integerValue; + case JSON_TYPE_DOUBLE: + return json->data.floatValue; + default: + return 0; + } +} + +static int64 hashString(char *s){ + int64 hash = 5381; + int i; + int len = strlen(s); + + for (i=0; itype) << 56; + int64_t mask = (1ll<<56)-1; + switch (json->type) { + case JSON_TYPE_NUMBER: + hash |= (mask & jsonAsNumber(json)); + break; + case JSON_TYPE_INT64: + hash |= (mask & jsonAsInt64(json)); + break; + case JSON_TYPE_DOUBLE: + hash |= (mask & (int64)jsonAsDouble(json)); + break; + case JSON_TYPE_STRING: + hash |= (mask & hashString(jsonAsString(json))); + break; + case JSON_TYPE_BOOLEAN: + hash |= (jsonAsBoolean(json) ? 1 : 0); + break; + case JSON_TYPE_NULL: + hash |= 0; + break; + case JSON_TYPE_OBJECT: + hash |= (int64)json; + break; + case JSON_TYPE_ARRAY: + hash |= (int64)json; + break; + } + return hash; +} + JsonArray *jsonArrayGetArray(JsonArray *array, int i) { Json *json = jsonArrayGetItem(array, i); if (json && jsonIsArray(json)) { @@ -1870,6 +2330,14 @@ void jsonPrintProperty(jsonPrinter* printer, JsonProperty *property) { jsonPrintInternal(printer, jsonPropertyGetKey(property), jsonPropertyGetValue(property)); } +static bool filterFromPrinting(jsonPrinter *printer, char *keyOrNull, Json *value){ + if (printer->filter){ + return printer->filter(printer->filterContext,keyOrNull,value); + } else { + return false; + } +} + void jsonPrintObject(jsonPrinter* printer, JsonObject *object) { JsonProperty *property; @@ -1891,6 +2359,12 @@ void jsonPrintInternal(jsonPrinter* printer, char* keyOrNull, Json *json) { case JSON_TYPE_NUMBER: jsonAddInt(printer, keyOrNull, jsonAsNumber(json)); break; + case JSON_TYPE_INT64: + jsonAddInt64(printer, keyOrNull, jsonAsInt64(json)); + break; + case JSON_TYPE_DOUBLE: + jsonAddDouble(printer, keyOrNull, jsonAsDouble(json)); + break; case JSON_TYPE_STRING: jsonAddString(printer, keyOrNull, jsonAsString(json)); break; @@ -1962,7 +2436,8 @@ Json *jsonParseUnterminatedUtf8String(ShortLivedHeap *slh, int outputCCSID, errorBufferOrNull, errorBufferSize); } -Json *jsonParseFile(ShortLivedHeap *slh, const char *filename, char* errorBufferOrNull, int errorBufferSize) { +static Json *jsonParseFileInternal(ShortLivedHeap *slh, const char *filename, char* errorBufferOrNull, int errorBufferSize, + int version){ Json *json = NULL; int returnCode = 0; int reasonCode = 0; @@ -1971,11 +2446,17 @@ Json *jsonParseFile(ShortLivedHeap *slh, const char *filename, char* errorBuffer UnixFile *file = NULL; int fileLen = 0; + /* + printf("JOE jsonParseFile\n"); + fflush(stdout); + */ + file = fileOpen(filename, FILE_OPTION_READ_ONLY, 0, 1024, &returnCode, &reasonCode); status = fileInfo(filename, &info, &returnCode, &reasonCode); if (file != NULL && status == 0) { fileLen = (int)fileInfoSize(&info); JsonParser *parser = makeJsonFileParser(slh, file, fileLen); + parser->version = version; json = jsonParse(parser); JsonToken *token = jsonMatchToken(parser, JSON_TOKEN_EOF); if (jsonIsError(json) || jsonIsTokenUnmatched(token)) { @@ -1990,9 +2471,18 @@ Json *jsonParseFile(ShortLivedHeap *slh, const char *filename, char* errorBuffer snprintf(errorBufferOrNull, errorBufferSize, "Couldn't open '%s'", filename); } } + /* printf("JOE return json\n");fflush(stdout); */ return json; } +Json *jsonParseFile(ShortLivedHeap *slh, const char *filename, char* errorBufferOrNull, int errorBufferSize){ + return jsonParseFileInternal(slh,filename,errorBufferOrNull,errorBufferSize,JSON_PARSE_VERSION_1); +} + +Json *jsonParseFile2(ShortLivedHeap *slh, const char *filename, char* errorBufferOrNull, int errorBufferSize){ + return jsonParseFileInternal(slh,filename,errorBufferOrNull,errorBufferSize,JSON_PARSE_VERSION_2); +} + Json *jsonParseString(ShortLivedHeap *slh, char *jsonString, char* errorBufferOrNull, int errorBufferSize) { return jsonParseUnterminatedString(slh,jsonString,strlen(jsonString),errorBufferOrNull,errorBufferSize); } @@ -2061,13 +2551,288 @@ JsonObject *jsonObjectProperty(JsonObject *object, char *propertyName, int *stat } } +/* JSON Merging */ + +typedef struct JsonMerger_tag { + JsonBuilder builder; /* must be first member to support casts */ + int flags; +} JsonMerger; + +static void copyJson(JsonBuilder *builder, Json *parent, char *parentKey, Json *json){ + int errorCode = 0; /* consumed and discarded because existing tree is valid */ + if (jsonIsObject(json)){ + JsonObject *jsonObject = jsonAsObject(json); + JsonProperty *prop = jsonObjectGetFirstProperty(jsonObject); + Json *copyObject = jsonBuildObject(builder,parent,parentKey,&errorCode); + while (prop){ + copyJson(builder,copyObject,prop->key,prop->value); + prop = jsonObjectGetNextProperty(prop); + } + } else if (jsonIsArray(json)){ + JsonArray *jsonArray = jsonAsArray(json); + Json *copyArray = jsonBuildArray(builder,parent,parentKey,&errorCode); + int size = jsonArrayGetCount(jsonArray); + for (int i=0; itype); + return; + } +} + +static int mergeJson1(JsonMerger *merger, Json *parent, char *parentKey, Json *overrides, Json *base){ + int errorCode = 0; + JsonBuilder *builder = &merger->builder; + /* + printf("jsonMerge1 typeO=%d typeB=%d builder=0x%p\n",overrides->type,base->type,builder); + fflush(stdout); + */ + if (jsonIsObject(base)){ + JsonObject *baseObject = jsonAsObject(base); + if (jsonIsObject(overrides)){ + JsonObject *overridesObject = jsonAsObject(overrides); + Json *merged = jsonBuildObject(builder,parent,parentKey,&errorCode); + JsonProperty *baseProp = jsonObjectGetFirstProperty(baseObject); + int status = JSON_MERGE_STATUS_SUCCESS; + while (baseProp && !status ){ + /* printf(" base loop %s\n",baseProp->key);fflush(stdout); */ + Json *baseValue = baseProp->value; + Json *overrideValue = jsonObjectGetPropertyValue(overridesObject,baseProp->key); + if (overrideValue == NULL){ /* NOT JSON NULL, really null */ + copyJson(builder,merged,baseProp->key,baseValue); + status = 0; + } else { + status = mergeJson1(merger,merged,baseProp->key,overrideValue,baseValue); + } + /* need to set result */ + baseProp = jsonObjectGetNextProperty(baseProp); + } + JsonProperty *overrideProp = jsonObjectGetFirstProperty(overridesObject); + while (overrideProp && !status){ + /* printf(" override loop %s\n",overrideProp->key);fflush(stdout); */ + if (!jsonObjectHasKey(baseObject,overrideProp->key)){ + copyJson(builder,merged,overrideProp->key,overrideProp->value); + } + overrideProp = jsonObjectGetNextProperty(overrideProp); + } + return status; + } else { + copyJson(builder,parent,parentKey,overrides); + return JSON_MERGE_STATUS_SUCCESS; + } + } else if (jsonIsArray(base)){ + JsonArray *baseArray = jsonAsArray(base); + if (jsonIsArray(overrides)){ + JsonArray *overridesArray = jsonAsArray(overrides); + int arrayPolicy = merger->flags & JSON_MERGE_FLAG_ARRAY_POLICY_MASK; + Json *merged = jsonBuildArray(builder,parent,parentKey,&errorCode); + int sizeB = jsonArrayGetCount(baseArray); + int sizeO = jsonArrayGetCount(overridesArray); + int i; + switch (arrayPolicy){ + case JSON_MERGE_FLAG_CONCATENATE_ARRAYS: /* len(merge) = len(a)+len(b) */ + for (i=0; i sizeO ? sizeB : sizeO); + for (i=0; i")); + fflush(stdout); + */ + copyJson(builder,parent,parentKey,overrides); + return JSON_MERGE_STATUS_SUCCESS; + } +} + +Json *jsonMerge(ShortLivedHeap *slh, Json *overrides, Json *base, int flags, int *statusPtr){ + JsonMerger merger; + memset(&merger,0,sizeof(JsonMerger)); + merger.flags = flags; + merger.builder.parser.slh = slh; + + int status = mergeJson1(&merger,NULL,NULL,overrides,base); + *statusPtr = status; + if (status){ + return NULL; + } else { + return merger.builder.root; + } +} + +Json *jsonCopy(ShortLivedHeap *slh, Json *value){ + JsonBuilder builder; + memset(&builder,0,sizeof(JsonBuilder)); + builder.parser.slh = slh; + copyJson(&builder,NULL,NULL,value); + return builder.root; +} + +/****** JSON Pointers ******************/ + +static JsonPointerElement *makePointerElement(char *token, int type){ + JsonPointerElement *element = (JsonPointerElement*)safeMalloc(sizeof(JsonPointerElement),"JsonPointerElement"); + element->string = token; + element->type = type; + return element; +} + +/* See RFC 6901 - absolute pointes only, so far */ +JsonPointer *parseJsonPointer(char *s){ + int len = strlen(s); + JsonPointer *jp = (JsonPointer*)safeMalloc(sizeof(JsonPointer),"JSONPointer"); + ArrayList *list = &(jp->elements); + initEmbeddedArrayList(list,NULL); + if (len == 0){ + return jp; + } else if (s[0] != '/'){ + safeFree((char*)jp,sizeof(JsonPointer)); + return NULL; + } + int pos = 1; + bool failed = false; + while (pos < len){ + int nextSlash = indexOf(s,len,'/',pos); + int end = (nextSlash == -1 ? len : nextSlash); + int tokenLen = end-pos+1; + char *token = safeMalloc(tokenLen,"JSON Ptr Token"); + memset(token,0,tokenLen); + /* Step 2: reduce the following digraphs + ~0 -> '~' + ~1 -> '/' + */ + bool pendingTilde = false; + int tPos = 0; + bool allDigits = true; + for (int i=pos; i '9'){ + allDigits = false; + } + if (pendingTilde){ + if (c == '0'){ + token[tPos++] = '~'; + } else if (c == '1'){ + token[tPos++] = '/'; + } else { + failed = true; + goto end; + } + pendingTilde = false; + } else { + if (c == '~'){ + pendingTilde = true; + } else { + token[tPos++] = c; + } + } + } + if (pendingTilde){ + failed = true; + goto end; + } + + arrayListAdd(list, + makePointerElement(token, + (allDigits ? JSON_POINTER_INTEGER : JSON_POINTER_NORMAL_KEY))); + + if (nextSlash == -1){ + break; + } else { + pos = nextSlash + 1; + } + } + end: + if (failed){ + safeFree((char*)jp,sizeof(JsonPointer)); + return NULL; + } else { + return jp; + } +} + +void freeJsonPointer(JsonPointer *jp){ + ArrayList *list = &jp->elements; + for (int i=0; isize; i++){ + JsonPointerElement *element = (JsonPointerElement*)arrayListElement(list,i); + safeFree((char*)element,sizeof(JsonPointerElement)); + } + safeFree((char*)jp,sizeof(JsonPointerElement)); +} + +void printJsonPointer(FILE *out, JsonPointer *jp){ + ArrayList *list = &jp->elements; + fprintf(out,"JSON Pointer:\n"); + for (int i=0; isize; i++){ + JsonPointerElement *element = (JsonPointerElement*)arrayListElement(list,i); + fprintf(out," 0x%04X: %s\n",element->type,element->string); + } +} + + + + + void reportJSONDataProblem(void *jsonObject, int status, char *propertyName){ switch (status){ case JSON_PROPERTY_NOT_FOUND: - printf("JSON property '%s' not found in object at 0x%x\n",propertyName,jsonObject); + printf("JSON property '%s' not found in object at 0x%p\n",propertyName,jsonObject); break; case JSON_PROPERTY_UNEXPECTED_TYPE: - printf("JSON property '%s' has wrong type in object at 0x%x\n",propertyName,jsonObject); + printf("JSON property '%s' has wrong type in object at 0x%p\n",propertyName,jsonObject); break; } } diff --git a/c/jsonschema.c b/c/jsonschema.c new file mode 100644 index 000000000..e65e27836 --- /dev/null +++ b/c/jsonschema.c @@ -0,0 +1,1655 @@ + + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#ifdef METTLE + +#include +#include +#include +#include +#include +#include +#include +#include +#include "metalio.h" +#include "qsam.h" + +#else + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 600 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#endif + + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "openprims.h" +#include "bpxnet.h" +#include "unixfile.h" +#include "json.h" +#include "charsets.h" +#include "collections.h" + +#ifdef __ZOWE_OS_WINDOWS +#include "winregex.h" +#else +#include "psxregex.h" /* works on ZOS and Linux */ +#endif + +#include "jsonschema.h" + +#ifdef __ZOWE_OS_WINDOWS +typedef int64_t ssize_t; +#endif + +#define JSTYPE_NULL 0 +#define JSTYPE_BOOLEAN 1 +#define JSTYPE_OBJECT 2 +#define JSTYPE_ARRAY 3 +#define JSTYPE_STRING 4 +#define JSTYPE_NUMBER 5 +#define JSTYPE_INTEGER 6 +#define JSTYPE_MAX 6 + +static char *getJSTypeName(int type){ + switch (type){ + case JSTYPE_NULL: return "null"; + case JSTYPE_BOOLEAN: return "boolean"; + case JSTYPE_OBJECT: return "objct"; + case JSTYPE_ARRAY: return "array"; + case JSTYPE_STRING: return "string"; + case JSTYPE_NUMBER: return "number"; + case JSTYPE_INTEGER: return "integer"; + default: return "unknown"; + } +} + +#define JS_VALIDATOR_MULTOF 0x00000001 +#define JS_VALIDATOR_MAX 0x00000002 +#define JS_VALIDATOR_MIN 0x00000004 +#define JS_VALIDATOR_MAX_CONTAINS 0x00000008 +#define JS_VALIDATOR_MIN_CONTAINS 0x00000010 +#define JS_VALIDATOR_MAX_ITEMS 0x00000020 +#define JS_VALIDATOR_MIN_ITEMS 0x00000040 +#define JS_VALIDATOR_MAX_PROPS 0x00000080 +#define JS_VALIDATOR_MIN_PROPS 0x00000100 +#define JS_VALIDATOR_MAX_LENGTH 0x00000200 +#define JS_VALIDATOR_MIN_LENGTH 0x00000400 + +typedef struct PatternProperty_tag { + char *pattern; + regex_t *compiledPattern; + int compilationError; + struct JSValueSpec_tag *valueSpec; + struct PatternProperty_tag *next; +} PatternProperty; + +typedef struct JSValueSpec_tag { + int typeMask; + int enumeratedValuesCount; + Json **enumeratedValues; + Json *constValue; + char *id; + char *hostName; /* from ID if set */ + char *fileName; /* from ID if set */ + char *description; + char *title; + char *anchor; + char *ref; + bool deprecated; + bool nullable; + /* shared sub-schema definitions go here */ + hashtable *definitions; + /* Quantifying/Compositing */ + int allOfCount; + struct JSValueSpec_tag **allOf; + int anyOfCount; + struct JSValueSpec_tag **anyOf; + int oneOfCount; + struct JSValueSpec_tag **oneOf; + struct JSValueSpec_tag *not; + + /* optional validators have flag for presence */ + uint64_t validatorFlags; + /* only filled for top level JSValueSpecs, ie no parent, or $id is present */ + hashtable *anchorTable; + /* everything with an ID under the top level */ + hashtable *idTable; + + double minimum; + double maximum; + int64_t minContains; + int64_t maxContains; + int64_t minItems; + int64_t maxItems; + + /* for Objects */ + int64_t minProperties; + int64_t maxProperties; + int requiredCount; + char **required; + hashtable *properties; + hashtable *dependentRequired; + bool additionalProperties; + PatternProperty *firstPatternProperty; + PatternProperty *lastPatternProperty; + + /* for numbers */ + bool exclusiveMaximum; + bool exclusiveMinimum; + double multipleOf; + /* bool isInteger; */ + + /* for arrays */ + struct JSValueSpec_tag *itemSpec; + bool uniqueItems; + + /* for strings */ + int64_t minLength; + int64_t maxLength; + char *pattern; + regex_t *compiledPattern; + int regexCompilationError; + + /* back/up pointer */ + struct JSValueSpec_tag *parent; +} JSValueSpec; + +typedef struct AccessPathUnion_tag { + int dummy; +} AccessPathUnion; + +static void printAccessPath(FILE *out, AccessPath *path){ + for (int i=0; icurrentSize; i++){ + if (path->elements[i].isName){ + fprintf(out,"/%s",path->elements[i].key.name); + } else { + fprintf(out,"/%d",path->elements[i].key.index); + } + } + fprintf(out,"\n"); +} + +static char *makeAccessPathString(char *buffer, int bufferLen, AccessPath *path){ + int pos = 0; + for (int i=0; icurrentSize; i++){ + if (pos + 100 >= bufferLen){ + sprintf(buffer+pos,"..."); + break; + } + if (path->elements[i].isName){ + pos += sprintf(buffer+pos,"/%s",path->elements[i].key.name); + } else { + pos += sprintf(buffer+pos,"/%d",path->elements[i].key.index); + } + } + return buffer; +} + + +static void accessPathPushIndex(AccessPath *accessPath, int index){ + if (accessPath->currentSize < MAX_SCHEMA_DEPTH){ + accessPath->elements[accessPath->currentSize].isName = false; + accessPath->elements[accessPath->currentSize++].key.index = index; + } +} + +static void accessPathPushName(AccessPath *accessPath, char *name){ + if (accessPath->currentSize < MAX_SCHEMA_DEPTH){ + accessPath->elements[accessPath->currentSize].isName = true; + accessPath->elements[accessPath->currentSize++].key.name = name; + } +} + +static void accessPathPop(AccessPath *accessPath){ + if (accessPath->currentSize > 0){ + accessPath->currentSize--; + } +} + + + +#define ERROR_MAX 1024 + +static void validationThrow(JsonValidator *validator, int errorCode, char *formatString, ...){ + va_list argPointer; + char *text = safeMalloc(ERROR_MAX,"ErrorBuffer"); + va_start(argPointer,formatString); + vsnprintf(text,ERROR_MAX,formatString,argPointer); + va_end(argPointer); + validator->errorCode = errorCode; + validator->errorMessage = text; + validator->errorMessageLength = ERROR_MAX; + longjmp(validator->recoveryData,1); +} + +static ValidityException *noteValidityException(JsonValidator *validator, + bool isSpeculating, + int invalidityCode, + char *formatString, ...){ + if (isSpeculating){ + return NULL; + } + va_list argPointer; + ValidityException *exception = (ValidityException*)safeMalloc(sizeof(ValidityException),"ValidityException"); + exception->code = invalidityCode; + exception->next = NULL; + va_start(argPointer,formatString); + vsnprintf(exception->message,MAX_VALIDITY_EXCEPTION_MSG,formatString,argPointer); + va_end(argPointer); + if (validator->firstValidityException == NULL){ + validator->firstValidityException = exception; + validator->lastValidityException = exception; + } else { + validator->lastValidityException->next = exception; + validator->lastValidityException = exception; + } + return exception; +} + +JsonValidator *makeJsonValidator(void){ + JsonValidator *validator = (JsonValidator*)safeMalloc(sizeof(JsonValidator),"JsonValidator"); + memset(validator,0,sizeof(JsonValidator)); + validator->accessPath = (AccessPath*)safeMalloc(sizeof(AccessPath),"JsonValidatorAccessPath"); + validator->accessPath->currentSize = 0; + return validator; +} + +void freeJsonValidator(JsonValidator *validator){ + safeFree((char*)validator->accessPath,sizeof(AccessPath)); + safeFree((char*)validator,sizeof(JsonValidator)); +} + +static char *validatorAccessPath(JsonValidator *validator){ + return makeAccessPathString(validator->accessPathBuffer,MAX_ACCESS_PATH,validator->accessPath); +} + +static JSValueSpec *getTopLevelAncestor(JSValueSpec *valueSpec){ + JSValueSpec *ancestor = valueSpec; + while (ancestor->parent){ + if (ancestor->id != NULL){ + return ancestor; + } + ancestor = ancestor->parent; + } + return ancestor; +} + +static JSValueSpec *getOutermostAncestor(JSValueSpec *valueSpec){ + JSValueSpec *ancestor = valueSpec; + while (ancestor->parent){ + ancestor = ancestor->parent; + } + return ancestor; +} + + +/* The return types of these validators is bool to allow validator to signal whether to continue + to gather more validation exceptions in this part of the JSON tree/graph. Returning false says + this part is invalid enough such that further evaluation would probably confuse the user with + contradictory information. */ + +typedef enum VResult_tag { + InvalidStop = 0, + ValidStop = 1, + InvalidContinue = 2, + ValidContinue = 3 +} VResult; + +static bool vResultValid(VResult r){ + return (r == ValidStop) || (r == ValidContinue); +} + +static VResult invertValidity(VResult r){ + return (VResult)(((int)r)^0x1); +} + +static bool validateType(JsonValidator *validator, + bool isSpeculating, + int typeCode, + JSValueSpec *valueSpec){ + if (validator->traceLevel >= 1){ + printf("typeCode=%d shifted=0x%x mask=0x%x\n",typeCode,(1 << typeCode),valueSpec->typeMask); + } + if (((1 << typeCode) & valueSpec->typeMask) == 0){ + noteValidityException(validator,isSpeculating,12,"type '%s' not permitted at %s", + getJSTypeName(typeCode),validatorAccessPath(validator)); + return false; + } else { + return true; + } +} + +static regex_t *compileRegex(char *pattern, int *regexStatus){ + regex_t *rx = regexAlloc(); + int compStatus = regexComp(rx,pattern,REG_EXTENDED); + if (compStatus){ + regexFree(rx); + *regexStatus = compStatus; + printf("*** WARNING *** pattern '%s' failed to compile with status %d\n", + pattern,compStatus); + return NULL; + } else { + *regexStatus = 0; + return rx; + } +} + +static JSValueSpec *getPropertyValueSpec(JSValueSpec *valueSpec, char *propertyName){ + JSValueSpec *propertyValueSpec = NULL; + if (valueSpec->properties){ + propertyValueSpec = (JSValueSpec*)htGet(valueSpec->properties,propertyName); + } + if (propertyValueSpec != NULL){ + return propertyValueSpec; + } + PatternProperty *patternProperty = valueSpec->firstPatternProperty; + while (patternProperty){ + if (patternProperty->compilationError == 0){ + if (patternProperty->compiledPattern == NULL){ + patternProperty->compiledPattern = compileRegex(patternProperty->pattern,&patternProperty->compilationError); + } + if (patternProperty->compiledPattern){ + int regexStatus = regexExec(patternProperty->compiledPattern,propertyName,0,NULL,0); + if (regexStatus == 0){ + return patternProperty->valueSpec; + } + } + } + patternProperty = patternProperty->next; + } + return NULL; +} + +/* for mutual recursion */ +static VResult validateJSON(JsonValidator *validator, bool isSpeculating, Json *value, JSValueSpec *valueSpec); + +static VResult validateJSONObject(JsonValidator *validator, + bool isSpeculating, + JsonObject *object, + JSValueSpec *valueSpec){ + int invalidCount = 0; + AccessPath *accessPath = validator->accessPath; + if (validator->traceLevel >= 1){ + printf("validateJSONObject required=0x%p\n",valueSpec->required); + printAccessPath(stdout,accessPath); + fflush(stdout); + } + if (valueSpec->required){ + for (int r=0; rrequiredCount; r++){ + char *requiredProperty = valueSpec->required[r]; + if (jsonObjectGetPropertyValue(object,requiredProperty) == NULL){ + noteValidityException(validator,isSpeculating,12,"missing required property '%s' at '%s'", + requiredProperty,validatorAccessPath(validator)); + } + } + } + // dependent required should go here + int propertyCount = 0; + JsonProperty *property = jsonObjectGetFirstProperty(object); + while (property){ + propertyCount++; + char *propertyName = jsonPropertyGetKey(property); + if (validator->traceLevel >= 1){ + printf("validate object pname=%s\n",propertyName); + fflush(stdout); + } + Json *propertyValue = jsonPropertyGetValue(property); + JSValueSpec *propertySpec = getPropertyValueSpec(valueSpec,propertyName); + accessPathPushName(accessPath,propertyName); + + if (propertySpec != NULL){ + VResult propResult = validateJSON(validator,isSpeculating,propertyValue,propertySpec); + if (!vResultValid(propResult)){ + invalidCount++; + } + } else { + if (valueSpec->additionalProperties == false){ + noteValidityException(validator,isSpeculating,12,"unspecified additional property not allowed: '%s' at '%s'", + propertyName,validatorAccessPath(validator)); + } else if (validator->flags && VALIDATOR_WARN_ON_UNDEFINED_PROPERTIES){ + printf("*WARNING* unspecified property seen, '%s', and checking code is not complete, vspec->props=0x%p\n", + propertyName,valueSpec->properties); + fflush(stdout); + } + if (valueSpec->properties){ + /* htDump(valueSpec->properties); */ + } + } + accessPathPop(accessPath); + property = jsonObjectGetNextProperty(property); + } + + // Interdependencies + if (valueSpec->dependentRequired != NULL){ + printf("*WARNING* depenentRequired not yet implemented\n"); + } + + + if (valueSpec->validatorFlags & JS_VALIDATOR_MAX_PROPS){ + int64_t lim = valueSpec->minProperties; + if (propertyCount > lim){ + invalidCount++; + noteValidityException(validator,isSpeculating,12,"too many properties, %d > MAX=%d at %s", + propertyCount,lim,validatorAccessPath(validator)); + } + } + if (valueSpec->validatorFlags & JS_VALIDATOR_MIN_PROPS){ + int64_t lim = valueSpec->minProperties; + if (propertyCount < lim){ + invalidCount++; + noteValidityException(validator,isSpeculating,12,"too few properties, %d < MIN=%d at %s", + propertyCount,lim,validatorAccessPath(validator)); + } + } + return (invalidCount > 0 ? InvalidContinue : ValidContinue); +} + +static VResult validateJSONArray(JsonValidator *validator, + bool isSpeculating, + JsonArray *array, + JSValueSpec *valueSpec){ + AccessPath *accessPath = validator->accessPath; + int invalidCount = 0; + //boolean uniqueItems; + // Long maxContains; + // Long minContains; + int elementCount = jsonArrayGetCount(array); + if (valueSpec->validatorFlags & JS_VALIDATOR_MAX_ITEMS){ + int64_t lim = valueSpec->maxItems; + if (elementCount > lim){ + invalidCount++; + noteValidityException(validator,isSpeculating,12, + "too many array items, %d > MAX=%d at %s", + elementCount,lim,validatorAccessPath(validator)); + } + } + if (valueSpec->validatorFlags & JS_VALIDATOR_MIN_ITEMS){ + int64_t lim = valueSpec->minItems; + if (elementCount < lim){ + invalidCount++; + noteValidityException(validator,isSpeculating,12, + "too few array ites, %d < MIN=%d at %s", + elementCount,lim,validatorAccessPath(validator)); + } + } + /* 'uniqueItems' is poorly specified regarding equality predicates. + for now it will use native equals() from JSValue. + It's kinda-crazy to do a true equal on printed representation on things that are + unlimited in size and complexity. + */ + LongHashtable *uniquenessSet = NULL; + if (valueSpec->uniqueItems){ + uniquenessSet = lhtCreate(257,NULL); + } + + for (int i=0; iitemSpec != NULL){ + VResult elementStatus = validateJSON(validator,isSpeculating,itemValue,valueSpec->itemSpec); + if (!vResultValid(elementStatus)){ + invalidCount++; + } + + } + if (valueSpec->uniqueItems){ + long longHash = jsonLongHash(itemValue); + if (lhtGet(uniquenessSet,longHash) != NULL){ + invalidCount++; + noteValidityException(validator,isSpeculating,12,"array uniqueItems violation %s is duplicate at %s", + itemValue,i,validatorAccessPath(validator)); + } + } + accessPathPop(accessPath); + } + return (invalidCount > 0 ? InvalidContinue : ValidContinue); +} + + +static VResult validateJSONString(JsonValidator *validator, + bool isSpeculating, + Json *string, + JSValueSpec *valueSpec){ + int invalidCount = 0; + char *s = jsonAsString(string); + int len = strlen(s); + if (valueSpec->validatorFlags & JS_VALIDATOR_MAX_LENGTH){ + long lim = valueSpec->maxLength; + if (len > lim){ + invalidCount++; + noteValidityException(validator,isSpeculating,12,"string too long, %d > MAX=%d at %s", + len,lim,validatorAccessPath(validator)); + } + } + if (valueSpec->validatorFlags & JS_VALIDATOR_MIN_LENGTH){ + long lim = valueSpec->minLength; + if (len < lim){ + invalidCount++; + noteValidityException(validator,isSpeculating,12,"string too short, %d < MAX=%d at %s", + len,lim,validatorAccessPath(validator)); + } + } + if (valueSpec->pattern && (valueSpec->regexCompilationError == 0)){ + if (valueSpec->compiledPattern == NULL){ + valueSpec->compiledPattern = compileRegex(valueSpec->pattern,&valueSpec->regexCompilationError); + } + if (valueSpec->compiledPattern){ + int regexStatus = regexExec(valueSpec->compiledPattern,s,0,NULL,0); + if (regexStatus != 0){ + invalidCount++; + noteValidityException(validator,isSpeculating,12,"string pattern match fail s='%s', pat='%s', at %s", + s,valueSpec->pattern,validatorAccessPath(validator)); + } + } + } + return (invalidCount > 0 ? InvalidContinue : ValidContinue); +} + +static VResult validateJSONNumber(JsonValidator *validator, + bool isSpeculating, + Json *doubleOrLegacyInt, + JSValueSpec *valueSpec){ + // Number multipleOf; // can be a filled, and must be > 0 + // Number exclusiveMaximum; + // Number exclusiveMinimum; + int invalidCount = 0; + double d = (doubleOrLegacyInt->type == JSON_TYPE_NUMBER ? + (double)jsonAsNumber(doubleOrLegacyInt) : + jsonAsDouble(doubleOrLegacyInt)); + if (valueSpec->validatorFlags & JS_VALIDATOR_MAX){ + double lim = valueSpec->maximum; + if (valueSpec->exclusiveMaximum ? (d >= lim) : (d > lim)){ + invalidCount++; + noteValidityException(validator,isSpeculating,12,"value too large, %f %s MAX=%f at %s", + d, + (valueSpec->exclusiveMaximum ? ">=" : ">"), + lim, + validatorAccessPath(validator)); + } + } + if (valueSpec->validatorFlags & JS_VALIDATOR_MIN){ + double lim = valueSpec->minimum; + if (valueSpec->exclusiveMinimum ? (d <= lim) : (d < lim)){ + invalidCount++; + noteValidityException(validator,isSpeculating,12,"value too small, %f %s MAX=%f at %s", + d, + (valueSpec->exclusiveMinimum ? "<=" : "<"), + lim, + validatorAccessPath(validator)); + } + } + return (invalidCount > 0 ? InvalidContinue : ValidContinue); +} + +static VResult validateJSONInteger(JsonValidator *validator, + bool isSpeculating, + Json *value64, + JSValueSpec *valueSpec){ + // Number multipleOf; // can be a filled, and must be > 0 + // Number exclusiveMaximum; + // Number exclusiveMinimum; + int invalidCount = 0; + int64_t i = jsonAsInt64(value64); + if (valueSpec->validatorFlags & JS_VALIDATOR_MAX){ + int64_t lim = (int64_t)valueSpec->maximum; + if (valueSpec->exclusiveMaximum ? (i >= lim) : (i > lim)){ + invalidCount++; + noteValidityException(validator,isSpeculating,12,"value too large, %lld %s MAX=%lld at %s", + i, + (valueSpec->exclusiveMaximum ? ">=" : ">"), + lim, + validatorAccessPath(validator)); + } + } + if (valueSpec->validatorFlags & JS_VALIDATOR_MIN){ + int64_t lim = (int64_t)valueSpec->minimum; + if (valueSpec->exclusiveMinimum ? (i <= lim) : (i < lim)){ + invalidCount++; + noteValidityException(validator,isSpeculating,12,"value too small, %lld %s MAX=%lld at %s", + i, + (valueSpec->exclusiveMinimum ? "<=" : "<"), + lim, + validatorAccessPath(validator)); + } + } + return (invalidCount > 0 ? InvalidContinue : ValidContinue); +} + +static char *httpPattern = "^(https?://[^/]+)/([^#]*)(#.*)?$"; +static char *filePattern = "^/([^#]*)(#.*)?$"; + +static int httpMatch(JsonValidator *v, char *s){ + if (v->httpRegexError){ + return v->httpRegexError; + } else if (v->httpRegex == NULL){ + v->httpRegex = compileRegex(httpPattern,&v->httpRegexError); + } + if (v->httpRegex){ + return regexExec(v->httpRegex,s,MAX_VALIDATOR_MATCHES,v->matches,0); + } else { + return v->httpRegexError; + } +} + +static int fileMatch(JsonValidator *v, char *s){ + if (v->fileRegexError){ + return v->fileRegexError; + } else if (v->fileRegex == NULL){ + v->fileRegex = compileRegex(filePattern,&v->fileRegexError); + } + if (v->fileRegex){ + return regexExec(v->fileRegex,s,MAX_VALIDATOR_MATCHES,v->matches,0); + } else { + return v->fileRegexError; + } +} + +static JSValueSpec *getTopSchemaByName(JsonValidator *validator, + char *hostName, int hostNameLength, + char *fileName, int fileNameLength){ + int len = hostNameLength+fileNameLength+8; + char *key = safeMalloc(len,"topSchemaKey"); + snprintf(key,len,"%*.*s/%*.*s", + hostNameLength,hostNameLength,hostName, + fileNameLength,fileNameLength,fileName); + hashtable *definitionTable = validator->topSchema->topValueSpec->definitions; + JSValueSpec *schema = (definitionTable ? htGet(definitionTable,key) : NULL); + if (validator->traceLevel >= 1){ + printf("getTopSchema for key='%s' yields 0x%p (from HT)\n",key,schema); + } + if (schema == NULL){ + for (int i=0; iotherSchemaCount; i++){ + JsonSchema *otherSchema = validator->otherSchemas[i]; + JSValueSpec *otherTopSpec = otherSchema->topValueSpec; + + if (otherTopSpec->id && !strcmp(otherTopSpec->id,key)){ + schema = otherTopSpec; + if (validator->traceLevel >= 1){ + printf("getTopSchema for key='%s' yields 0x%p (from Other Schemas)\n",key,schema); + } + break; + } + } + + } + safeFree(key,len); + return schema; +} + +static int matchLen(regmatch_t *match){ + return (int)(match->rm_eo - match->rm_so); +} + +static JSValueSpec *resolveCrossSchemaRef(JsonValidator *validator, bool isSpeculating, + JSValueSpec *referredTopSchema, char *ref, regmatch_t *anchorMatch){ + if (anchorMatch->rm_so == -1){ + return referredTopSchema; + } else if (referredTopSchema->anchorTable == NULL){ + noteValidityException(validator,isSpeculating,12,"schema '%s' does not define any anchors for ref '%s'", + (referredTopSchema->id ? referredTopSchema->id : ""), + ref); + return NULL; + } else { + char *anchor = ref+anchorMatch->rm_so+1; /* +1 to skip the '#' char */ + JSValueSpec *spec = (JSValueSpec*)htGet(referredTopSchema->anchorTable,anchor); + if (validator->traceLevel >= 1){ + printf("anchor '%s' resolves to 0x%p\n",anchor,spec); + } + if (spec == NULL){ + noteValidityException(validator,isSpeculating,12,"anchor schema ref '%s' does not resolve in referred schema",ref); + } + return spec; + } +} + + +static JSValueSpec *resolveRef(JsonValidator *validator, bool isSpeculating, JSValueSpec *valueSpec){ + char *ref = valueSpec->ref; + int refLen = strlen(ref); + int matchStatus = 0; + if (validator->traceLevel >= 1){ + printf("resolveRef ref=%s\n",valueSpec->ref); + } + JSValueSpec *containingSpec = getTopLevelAncestor(valueSpec); + if (containingSpec == NULL){ /* this code is wrong if topLevel (parent chain) is no good */ + printf("*** PANIC *** no top level\n"); + return NULL; + } + + /* Case 1 local reference w/o anchor */ + if (refLen >= 8 && !memcmp(ref,"#/$defs/",8)){ /* local resolution */ + int refLen = strlen(ref); + int pos = 2; + char *key = ref+8; + JSValueSpec *topSchema = validator->topSchema->topValueSpec; + if (topSchema->definitions == NULL){ + noteValidityException(validator,isSpeculating,12,"schema '%s' does not define shared '$defs'", + (topSchema->id ? topSchema->id : "")); + return NULL; /* for the compiler, bless its heart */ + } + JSValueSpec *resolvedSchema = htGet(topSchema->definitions,key); + if (resolvedSchema == NULL){ + noteValidityException(validator,isSpeculating,12,"path schema ref '%s' does not resolve against '$defs'",ref); + return NULL; + } else { + return resolvedSchema; + } + /* Case 2, global URL reference */ + } else if (refLen >= 10 && ((matchStatus = httpMatch(validator,ref)) == 0)){ + regmatch_t domainMatch = validator->matches[1]; + regmatch_t fileMatch = validator->matches[2]; + regmatch_t anchorMatch = validator->matches[3]; + JSValueSpec *referredTopSchema = getTopSchemaByName(validator, + ref+domainMatch.rm_so,matchLen(&domainMatch), + ref+fileMatch.rm_so,matchLen(&fileMatch)); + if (referredTopSchema == NULL){ + noteValidityException(validator,isSpeculating,12,"cross-domain schema not found for ref '%s'",ref); + return NULL; + } else { + return resolveCrossSchemaRef(validator,isSpeculating,referredTopSchema,ref,&anchorMatch); + } + /* Case 3, same-domain URL fragment (ie file) reference */ + } else if (refLen >= 1 && ((matchStatus = fileMatch(validator,ref)) == 0)){ + regmatch_t fileMatch = validator->matches[1]; + regmatch_t anchorMatch = validator->matches[2]; + JSValueSpec *referredTopSchema = getTopSchemaByName(validator, + containingSpec->hostName,strlen(containingSpec->hostName), + ref+fileMatch.rm_so,matchLen(&fileMatch)); + if (referredTopSchema == NULL){ + noteValidityException(validator,isSpeculating,12,"same-domain schema not found for ref '%s'",ref); + return NULL; + } else { + return resolveCrossSchemaRef(validator,isSpeculating,referredTopSchema,ref,&anchorMatch); + } + /* Case 4, just an anchor in same schema */ + } else if (refLen >= 1 && ref[0] == '#'){ + if (containingSpec->anchorTable == NULL){ + noteValidityException(validator,isSpeculating,12,"schema '%s' does not define any anchors for ref '%s'", + (containingSpec->id ? containingSpec->id : ""), + ref); + return NULL; + } + JSValueSpec *resolvedSchema = (JSValueSpec*)htGet(containingSpec->anchorTable,ref+1); + if (resolvedSchema == NULL){ + noteValidityException(validator,isSpeculating,12,"anchor schema ref '%s' does not resolve in containing schema",ref); + return NULL; + } else { + return resolvedSchema; + } + /* Other cases not known or supported */ + } else { + /* notes, + need to merge URL according W3 rules and pass to pluggable resolver that + we hope this validator has. + */ + noteValidityException(validator,isSpeculating,12,"bad or unimplemented ref at '%s' at %s", + valueSpec->ref,validatorAccessPath(validator)); + return NULL; + } +} + +static VResult validateJSONSimple(JsonValidator *validator, bool isSpeculating, Json *value, JSValueSpec *valueSpec){ + if (jsonIsNull(value)){ + return (validateType(validator,isSpeculating,JSTYPE_NULL,valueSpec) ? + ValidContinue : InvalidStop); + } else if (jsonIsBoolean(value)){ + return (validateType(validator,isSpeculating,JSTYPE_BOOLEAN,valueSpec) ? + ValidContinue : InvalidStop); + } else if (jsonIsObject(value)){ + return (validateType(validator,isSpeculating,JSTYPE_OBJECT,valueSpec) ? + validateJSONObject(validator,isSpeculating,jsonAsObject(value),valueSpec) : + InvalidStop); + } else if (jsonIsArray(value)){ + return (validateType(validator,isSpeculating,JSTYPE_ARRAY,valueSpec) ? + validateJSONArray(validator,isSpeculating,jsonAsArray(value),valueSpec) : + InvalidStop); + } else if (jsonIsString(value)){ + return (validateType(validator,isSpeculating,JSTYPE_STRING,valueSpec) ? + validateJSONString(validator,isSpeculating,value,valueSpec) : + InvalidStop); + } else if (jsonIsNumber(value)){ + if (!jsonIsInt64(value)){ + return (validateType(validator,isSpeculating,JSTYPE_NUMBER,valueSpec) ? + validateJSONNumber(validator,isSpeculating,value,valueSpec) : /* general, comparisons done as doubles */ + InvalidStop); + } else { + return (validateType(validator,isSpeculating,JSTYPE_INTEGER,valueSpec) ? + validateJSONInteger(validator,isSpeculating,value,valueSpec) : + InvalidStop); + } + + } else { + printf("*** PANIC *** unhandled JS Value with type = %d\n",value->type); + return InvalidStop; + } +} + +#define QUANT_ALL 1 +#define QUANT_ANY 2 +#define QUANT_ONE 3 + +static VResult validateQuantifiedSpecs(JsonValidator *validator, + bool isSpeculating, + Json *value, JSValueSpec **valueSpecs, + int specCount, int quantType){ + int validCount = 0; + + for (int i=0; itraceLevel >= 1){ + printf("vQS i=%d type=%d valid=%d vCount=%d\n",i,quantType,eltValid,validCount); + } + switch (quantType){ + case QUANT_ALL: + if (!eltValid){ + noteValidityException(validator,isSpeculating,12,"not allOf schemas at '%s' are valid", + validatorAccessPath(validator)); + return InvalidStop; + } + break; + case QUANT_ANY: + if (eltValid){ + return ValidStop; + } + case QUANT_ONE: + if (validCount > 1){ + noteValidityException(validator,isSpeculating,12,"more than oneOf schemas at '%s' are valid", + validatorAccessPath(validator)); + return InvalidStop; + } + break; + } + } + switch (quantType){ + case QUANT_ALL: + return ValidContinue; + case QUANT_ANY: + noteValidityException(validator,isSpeculating,12,"not anyOf schemas at '%s' are valid", + validatorAccessPath(validator)); + + return ValidContinue; + case QUANT_ONE: + /* printf("quantOne vCount=%d\n",validCount); */ + return (validCount == 1) ? ValidContinue : InvalidContinue; + default: + /* can't really get here */ + return InvalidStop; + } +} + +static bool jsonEquals(Json *a, Json *b){ + if (jsonIsArray(a) || jsonIsArray(b)){ + return false; + } else if (jsonIsObject(a) || jsonIsObject(b)){ + return false; + } else if (jsonIsString(a)){ + if (jsonIsString(b)){ + char *aStr = jsonAsString(a); + char *bStr = jsonAsString(b); + return !strcmp(aStr,bStr); + } else { + return false; + } + } else { + return !memcmp(a,b,sizeof(Json)); + } +} + +static VResult validateJSON(JsonValidator *validator, bool isSpeculating,Json *value, JSValueSpec *valueSpec){ + while (valueSpec->ref){ + valueSpec = resolveRef(validator,isSpeculating,valueSpec); + if (valueSpec == NULL){ + return InvalidStop; + } + } + if (validator->traceLevel >= 1){ + printf("validate JSON value->type=%d specTypeMask=0x%x\n",value->type,valueSpec->typeMask); + fflush(stdout); + } + if (valueSpec->constValue){ + bool eq = jsonEquals(value,valueSpec->constValue); + if (!eq){ + noteValidityException(validator,isSpeculating,12,"unequal constant value at %s", + validatorAccessPath(validator)); + return InvalidStop; + } else { + return ValidContinue; + } + } else if (valueSpec->enumeratedValues){ + bool matched = false; + for (int ee=0; eeenumeratedValuesCount; ee++){ + Json *enumValue = valueSpec->enumeratedValues[ee]; + if (jsonEquals(value,enumValue)){ + matched = true; + break; + } + } + if (!matched){ + noteValidityException(validator,isSpeculating,12,"no matching enum value at %s", + validatorAccessPath(validator)); + return InvalidStop; + } else { + return ValidContinue; + } + } else { + /* type is string or array, does this mean to collapse ValueSpecTypes + should we flag errors on out-of-bounds validation keywords for type keyword */ + VResult validity = validateJSONSimple(validator,isSpeculating,value,valueSpec); + /* here, mix in the quantified and compound */ + if (!vResultValid(validity)) return validity; + if (valueSpec->allOf){ + validity = validateQuantifiedSpecs(validator,isSpeculating,value,valueSpec->allOf,valueSpec->allOfCount,QUANT_ALL); + } + if (!vResultValid(validity)) return InvalidStop; + if (valueSpec->anyOf){ + validity = validateQuantifiedSpecs(validator,isSpeculating,value,valueSpec->anyOf,valueSpec->anyOfCount,QUANT_ANY); + } + if (!vResultValid(validity)) return InvalidStop; + if (valueSpec->oneOf){ + validity = validateQuantifiedSpecs(validator,isSpeculating,value,valueSpec->oneOf,valueSpec->oneOfCount,QUANT_ONE); + } + if (!vResultValid(validity)) return InvalidStop; + if (valueSpec->not){ + validity = validateJSON(validator,true,value,valueSpec->not); + if (vResultValid(validity)){ + noteValidityException(validator,isSpeculating,12,"negated schema at %s is valid", + validatorAccessPath(validator)); + } + validity = invertValidity(validity); + } + return validity; + } +} + +int jsonValidateSchema(JsonValidator *validator, Json *value, JsonSchema *topSchema, + JsonSchema **otherSchemas, int otherSchemaCount){ + if (setjmp(validator->recoveryData) == 0) { /* normal execution */ + validator->topSchema = topSchema; + validator->otherSchemas = otherSchemas; + validator->otherSchemaCount = otherSchemaCount; + VResult validity = validateJSON(validator,false,value,topSchema->topValueSpec); + printf("after validate without throw, should show validation exceptions\n"); + fflush(stdout); + if (validator->firstValidityException == NULL){ + return JSON_VALIDATOR_NO_EXCEPTIONS; + } else { + return JSON_VALIDATOR_HAS_EXCEPTIONS; + } + } else { + printf("validation failed '%s'\n",validator->errorMessage); + fflush(stdout); + return JSON_VALIDATOR_INTERNAL_FAILURE; + } +} + + +static void schemaThrow(JsonSchemaBuilder *builder, int errorCode, char *formatString, ...){ + va_list argPointer; + char *text = safeMalloc(ERROR_MAX,"ErrorBuffer"); + va_start(argPointer,formatString); + vsnprintf(text,ERROR_MAX,formatString,argPointer); + va_end(argPointer); + + builder->errorCode = errorCode; + builder->errorMessage = text; + builder->errorMessageLength = ERROR_MAX; + + longjmp(builder->recoveryData,1); +} + +static char *getStringOrFail(JsonSchemaBuilder *builder, JsonObject *object, char *propertyName) { + Json *propertyValue = jsonObjectGetPropertyValue(object,propertyName); + if (propertyValue == NULL){ + schemaThrow(builder,12,"object must have property: %s",propertyName); + return NULL; /* unreachable, but compiler doesn't know */ + } else if (jsonIsString(propertyValue)){ + return jsonAsString(propertyValue); + } else { + schemaThrow(builder,12,"property '%s' must contain a string, not %s",propertyName,propertyValue); + return NULL; /* unreachable, but compiler doesn't know */ + } +} + +static char *getString(JsonSchemaBuilder *builder, JsonObject *object, char *propertyName, char *defaultValue) { + Json *propertyValue = jsonObjectGetPropertyValue(object,propertyName); + if (propertyValue == NULL){ + return defaultValue; + } else if (jsonIsString(propertyValue)){ + return jsonAsString(propertyValue); + } else { + schemaThrow(builder,12,"property '%s' must contain a string, not %s",propertyName,propertyValue); + return NULL; /* unreachable, but compiler doesn't know */ + } +} + +static bool getBooleanValue(JsonSchemaBuilder *builder, JsonObject *object, char *propertyName, bool defaultValue){ + Json *propertyValue = jsonObjectGetPropertyValue(object,propertyName); + if (propertyValue == NULL){ + return defaultValue; + } else if (jsonIsBoolean(propertyValue)){ + return (bool)jsonAsBoolean(propertyValue); + } else { + schemaThrow(builder,12,"property '%s' must contain a number, not %s",propertyName,propertyValue); + return false; /* unreachable, but compiler doesn't know */ + } +} + +static double getNumber(JsonSchemaBuilder *builder, JsonObject *object, char *propertyName, double defaultValue){ + Json *propertyValue = jsonObjectGetPropertyValue(object,propertyName); + if (propertyValue == NULL){ + return defaultValue; + } else if (jsonIsNumber(propertyValue)){ + return jsonAsDouble(propertyValue); + } else { + schemaThrow(builder,12,"property '%s' must contain a number, not %s",propertyName,propertyValue); + return 0.0; /* unreachable, but compiler doesn't know */ + } +} + +static int64_t getIntegralValue(JsonSchemaBuilder *builder, JsonObject *object, char *propertyName, int64_t defaultValue){ + Json *propertyValue = jsonObjectGetPropertyValue(object,propertyName); + if (propertyValue == NULL){ + return defaultValue; + } else if (jsonIsNumber(propertyValue)){ + if (jsonIsInt64(propertyValue)){ + return jsonAsInt64(propertyValue); + } else { + schemaThrow(builder,12,"property '%s' must contain an integer, not %s",propertyName,propertyValue); + return 0; /* unreachable, but compiler doesn't know */ + } + } else { + schemaThrow(builder,12,"property '%s' must contain a number, not %s",propertyName,propertyValue); + return 0; /* unreachable, but compiler doesn't know */ + } +} + +static JsonArray *getArrayValue(JsonSchemaBuilder *builder, JsonObject *object, char *propertyName) { + Json *propertyValue = jsonObjectGetPropertyValue(object,propertyName); + if (propertyValue == NULL){ + return NULL; + } else if (jsonIsArray(propertyValue)){ + return jsonAsArray(propertyValue); + } else { + schemaThrow(builder,12,"property '%s' must contain an array, not %s",propertyName,propertyValue); + return NULL; /* unreachable, but compiler doesn't know */ + } +} + +static JsonObject *getObjectValue(JsonSchemaBuilder *builder, JsonObject *object, char *propertyName) { + Json *propertyValue = jsonObjectGetPropertyValue(object,propertyName); + if (propertyValue == NULL){ + return NULL; + } else if (jsonIsObject(propertyValue)){ + return jsonAsObject(propertyValue); + } else { + schemaThrow(builder,12,"property '%s' must contain an object, not %s",propertyName,propertyValue); + return NULL; /* unreachable, but compiler doesn't know */ + } +} + +static int getJSTypeForName(JsonSchemaBuilder *builder, char *name){ + if (!strcmp(name,"null")){ + return JSTYPE_NULL; + } else if (!strcmp(name,"boolean")){ + return JSTYPE_BOOLEAN; + } else if (!strcmp(name,"object")){ + return JSTYPE_OBJECT; + } else if (!strcmp(name,"array")){ + return JSTYPE_ARRAY; + } else if (!strcmp(name,"number")){ + return JSTYPE_NUMBER; + } else if (!strcmp(name,"string")){ + return JSTYPE_STRING; + } else if (!strcmp(name,"integer")){ + return JSTYPE_INTEGER; + } else { + schemaThrow(builder,12,"'%s' is not valid JSON Schema type",name); + return 0; + } +} + +static void checkNonNegative(JsonSchemaBuilder *builder, int64 value, char *name) { + if (value < 0){ + schemaThrow(builder,12,"%s must be non-negative",name); + } +} + +static void checkPositiveDouble(JsonSchemaBuilder *builder, double value, char *name) { + if (value <= 0){ + schemaThrow(builder,12,"%s must be positive",name); + } +} + +static char** builderStringArray(JsonSchemaBuilder *builder, int count){ + char **result = (char**)SLHAlloc(builder->slh,count*sizeof(char*)); + return result; +} + +static char **getStringArrayOrFail(JsonSchemaBuilder *builder, JsonArray *a, int *count){ + ShortLivedHeap *slh = builder->slh; + int aLen = jsonArrayGetCount(a); + char **result = builderStringArray(builder,aLen); + for (int i=0; itype); + return NULL; + } + property = jsonObjectGetNextProperty(property); + } + return map; +} + +static bool ensureUniqueKeys(JsonSchemaBuilder *builder, JsonObject *o){ + /* + The Java JSON Reader we use guarantees unique property keys, but + */ + return true; +} + +// Tolerate both "id" and "$id" +static char *getID(JsonSchemaBuilder *builder, JsonObject *o) { + char *id = getString(builder,o,"$id",NULL); + if (id == NULL){ + id = getString(builder,o,"id",NULL); + } + return id; +} + +/* forwarding for complex recursion */ +static JSValueSpec *build(JsonSchemaBuilder *builder, JSValueSpec *parent, Json *jsValue, bool isTopLevel); + +static void indexByAnchor(JSValueSpec *valueSpec){ + JSValueSpec *topLevelAncestor = getTopLevelAncestor(valueSpec); + if (topLevelAncestor->anchorTable == NULL){ + topLevelAncestor->anchorTable = htCreate(257,stringHash,stringCompare,NULL,NULL); + } + htPut(topLevelAncestor->anchorTable,valueSpec->anchor,valueSpec); +} + +static void indexByID(JSValueSpec *valueSpec){ + JSValueSpec *outermostAncestor = getOutermostAncestor(valueSpec); + if (outermostAncestor->idTable == NULL){ + outermostAncestor->idTable = htCreate(257,stringHash,stringCompare,NULL,NULL); + } + htPut(outermostAncestor->idTable,valueSpec->id,valueSpec); +} + +static char *extractString(JsonSchemaBuilder *b, char *s, regmatch_t *match){ + if (match->rm_so != -1){ + int len = (int)(match->rm_eo - match->rm_so); + char *copy = SLHAlloc(b->slh,len+1); + memcpy(copy,s+match->rm_so,len); + copy[len] = 0; + return copy; + } else { + return NULL; + } +} + + +static JSValueSpec *makeValueSpec(JsonSchemaBuilder *builder, + JSValueSpec *parent, + char *id, + char *description, + char *title, + char **typeNameArray, + int typeNameArrayLength){ + JSValueSpec *spec = (JSValueSpec*)SLHAlloc(builder->slh,sizeof(JSValueSpec)); + memset(spec,0,sizeof(JSValueSpec)); + spec->id = id; + spec->description = description; + spec->title = title; + spec->parent = parent; + /* parent must be set */ + if (id){ + indexByID(spec); + regmatch_t matches[10]; + + int regexStatus = 0; + regex_t *regex = compileRegex(httpPattern,®exStatus); + if (regex){ + int matchStatus = regexExec(regex,id,10,matches,0); + if (matchStatus){ + fprintf(stderr,"id does not look like a URL: '%s', regexMatchStatus=%d\n",id,matchStatus); + } else { + spec->hostName = extractString(builder,id,&matches[1]); + spec->fileName = extractString(builder,id,&matches[2]); + } + regexFree(regex); + } + + } + for (int i=0; itypeMask |= (1 << typeCode); + if (typeCode == JSTYPE_NUMBER){ + /* numbers allow integers, but not the converse */ + spec->typeMask |= (1 << JSTYPE_INTEGER); + } + } + return spec; +} + + +/* Map */ +static hashtable *getDefinitions(JsonSchemaBuilder *builder, JSValueSpec *parent, JsonObject *object){ + AccessPath *accessPath = builder->accessPath; + JsonObject *definitionsObject = getObjectValue(builder,object,"$defs"); + if (definitionsObject != NULL){ + hashtable *definitionMap = htCreate(101,stringHash,stringCompare,NULL,NULL); + accessPathPushName(accessPath,"definitions"); + JsonProperty *property = jsonObjectGetFirstProperty(definitionsObject); + while (property){ + char *propertyName = jsonPropertyGetKey(property); + Json *propertyValue = jsonPropertyGetValue(property); + if (propertyValue == NULL){ + schemaThrow(builder,12,"sub definition %s is NULL, which is not allowed",propertyName); + return NULL; + } + accessPathPushName(accessPath,propertyName); + htPut(definitionMap,propertyName,build(builder,parent,propertyValue,false)); + accessPathPop(accessPath); + property = jsonObjectGetNextProperty(property); + } + accessPathPop(accessPath); + return definitionMap; + } else { + return NULL; + } +} + +static void addPatternProperties(JsonSchemaBuilder *builder, JSValueSpec *valueSpec, JsonObject *object){ + JsonObject *patternObject = getObjectValue(builder,object,"patternProperties"); + if (patternObject != NULL){ + AccessPath *accessPath = builder->accessPath; + accessPathPushName(accessPath,"patternProperties"); + JsonProperty *property = jsonObjectGetFirstProperty(patternObject); + while (property){ + char *propertyName = jsonPropertyGetKey(property); + Json *propertyValue = jsonPropertyGetValue(property); + if (propertyValue == NULL){ + schemaThrow(builder,12,"patternProperty %s is NULL, which is not allowed",propertyName); + return; + } + accessPathPushName(accessPath,propertyName); + PatternProperty *patternProperty = (PatternProperty*)SLHAlloc(builder->slh,sizeof(PatternProperty)); + if (valueSpec->firstPatternProperty == NULL){ + valueSpec->firstPatternProperty = patternProperty; + valueSpec->lastPatternProperty = patternProperty; + } else { + valueSpec->lastPatternProperty->next = patternProperty; + valueSpec->lastPatternProperty = patternProperty; + } + memset(patternProperty,0,sizeof(PatternProperty)); + patternProperty->pattern = propertyName; + patternProperty->valueSpec = build(builder,valueSpec,propertyValue,false); + accessPathPop(accessPath); + property = jsonObjectGetNextProperty(property); + } + accessPathPop(accessPath); + } +} + +static JSValueSpec **getComposite(JsonSchemaBuilder *builder, JSValueSpec *parent, + JsonObject *object, char *key, int *countPtr){ + AccessPath *accessPath = builder->accessPath; + JsonArray *array = jsonObjectGetArray(object,key); + if (array != NULL){ + int len = jsonArrayGetCount(array); + *countPtr = len; + JSValueSpec **schemata = (JSValueSpec**)SLHAlloc(builder->slh,len*sizeof(JSValueSpec*)); + accessPathPushName(accessPath,key); + for (int i=0; iaccessPath; + if (builder->traceLevel >= 2){ + printf("JSONSchema build\n"); + printAccessPath(stdout,accessPath); + fflush(stdout); + } + if (jsonIsObject(jsValue)){ + JsonObject *object = jsonAsObject(jsValue); + ensureUniqueKeys(builder,object); + Json *typeValue = jsonObjectGetPropertyValue(object,"type"); + char **typeArray = NULL; + int typeArrayCount = 0; + if (typeValue == NULL){ + if (builder->traceLevel >= 1){ + printf("*** INFO *** untyped value at path:\n"); + printAccessPath(stdout,accessPath); + } + typeArray = allJSTypeNames; + typeArrayCount = ALL_TYPES_COUNT; + } else { + if (jsonIsString(typeValue)){ + typeArray = builderStringArray(builder,1); + typeArray[0] = getStringOrFail(builder,object,"type"); + typeArrayCount = 1; + } else if (jsonIsArray(typeValue)){ + typeArray = getStringArrayOrFail(builder,jsonAsArray(typeValue),&typeArrayCount); + } else { + schemaThrow(builder,12,"'type' must be either a single string or an array of string, not type=%d",typeValue->type); + } + } + + char *description = getString(builder,object,"description",NULL); + char *title = getString(builder,object,"title",NULL); + char *id = getID(builder,object); + JSValueSpec *valueSpec = makeValueSpec(builder,parent,id,description,title,typeArray,typeArrayCount); + valueSpec->ref = getString(builder,object,"$ref",NULL); + valueSpec->anchor = getString(builder,object,"$anchor",NULL); + if (valueSpec->anchor){ + indexByAnchor(valueSpec); + } + if (valueSpec->ref){ + /* THe JSON Schema spec says that if a $ref is defined it will be the only thing looked at, so + we short-cicuit here. + + "In Draft 4-7, $ref behaves a little differently. When an object + contains a $ref property, the object is considered a reference, + not a schema. Therefore, any other properties you put in that + object will not be treated as JSON Schema keywords and will be + ignored by the validator. $ref can only be used where a schema + is expected." + + */ + return valueSpec; + } + valueSpec->definitions = getDefinitions(builder,valueSpec,object); + valueSpec->additionalProperties = getBooleanValue(builder,object,"additionalProperties",true); + addPatternProperties(builder,valueSpec,object); + valueSpec->allOf = getComposite(builder,valueSpec,object,"allOf",&valueSpec->allOfCount); + valueSpec->anyOf = getComposite(builder,valueSpec,object,"anyOf",&valueSpec->anyOfCount); + valueSpec->oneOf = getComposite(builder,valueSpec,object,"oneOf",&valueSpec->oneOfCount); + Json *notSpec = jsonObjectGetPropertyValue(object,"not"); + if (notSpec != NULL){ + accessPathPushName(accessPath,"not"); + valueSpec->not = build(builder,valueSpec,notSpec,false); + accessPathPop(accessPath); + } + Json *constValue = jsonObjectGetPropertyValue(object,"const"); + JsonArray *enumValues = getArrayValue(builder,object,"enum"); + if (constValue && enumValues){ + schemaThrow(builder,12,"cannot specify both 'const' and 'enum' for the same schema"); + } + if (constValue){ + valueSpec->constValue = jsonCopy(builder->slh,constValue); + } + if (enumValues){ + valueSpec->enumeratedValuesCount = jsonArrayGetCount(enumValues); + valueSpec->enumeratedValues = (Json**)SLHAlloc(builder->slh,valueSpec->enumeratedValuesCount*sizeof(Json*)); + for (int ee=0; eeenumeratedValuesCount; ee++){ + valueSpec->enumeratedValues[ee] = jsonCopy(builder->slh,jsonArrayGetItem(enumValues,ee)); + } + } + /// valueSpec.not = getComposite(object,accessPath); + bool handledNumeric = false; + for (int typeCode=0; typeCode<=JSTYPE_MAX; typeCode++){ + if ((1 << typeCode) & valueSpec->typeMask){ + switch (typeCode){ + case JSTYPE_OBJECT: + { + JSValueSpec *objectSpec = valueSpec; + Json *properties = jsonObjectGetPropertyValue(object,"properties"); + objectSpec->maxProperties = getIntegralValue(builder,object,"maxProperties",MISSING_VALIDATOR); + objectSpec->minProperties = getIntegralValue(builder,object,"minProperties",MISSING_VALIDATOR); + if (objectSpec->maxProperties != MISSING_VALIDATOR){ + checkNonNegative(builder,objectSpec->maxProperties,"maxProperties"); + objectSpec->validatorFlags |= JS_VALIDATOR_MAX_PROPS; + } + if (objectSpec->maxProperties != MISSING_VALIDATOR){ + checkNonNegative(builder,objectSpec->minProperties,"minProperties"); + objectSpec->validatorFlags |= JS_VALIDATOR_MIN_PROPS; + } + JsonArray *required = getArrayValue(builder,object,"required"); + if (required != NULL){ + int count = 0; + objectSpec->required = getStringArrayOrFail(builder,required,&count); + objectSpec->requiredCount = count; + } + if (properties != NULL){ + if (jsonIsObject(properties)){ + JsonObject *propertiesObject = jsonAsObject(properties); + JsonProperty *property = jsonObjectGetFirstProperty(propertiesObject); + while (property){ + char *propertyName = jsonPropertyGetKey(property); + Json *propertyValue = jsonPropertyGetValue(property); + accessPathPushName(accessPath,propertyName); + if (valueSpec->properties == NULL){ + valueSpec->properties = htCreate(101,stringHash,stringCompare,NULL,NULL); + } + htPut(valueSpec->properties,propertyName,build(builder,valueSpec,propertyValue,false)); + accessPathPop(accessPath); + property = jsonObjectGetNextProperty(property); + } + } else { + schemaThrow(builder,12,"properties is filled by JSON value that is not object with type=%d\n",properties->type); + } + } else { + // properties is not required + } + JsonObject *dependentRequiredSpec = getObjectValue(builder,object,"dependentRequired"); + if (dependentRequiredSpec != NULL){ + objectSpec->dependentRequired = getStringToStringsMapOrFail(builder,dependentRequiredSpec); + } + } + break; + case JSTYPE_ARRAY: + { + JSValueSpec *arraySpec = valueSpec; + + arraySpec->maxItems = getIntegralValue(builder,object,"maxItems",MISSING_VALIDATOR); + arraySpec->minItems = getIntegralValue(builder,object,"minItems",MISSING_VALIDATOR); + arraySpec->uniqueItems = getBooleanValue(builder,object,"uniqueItems",false); + arraySpec->maxContains = getIntegralValue(builder,object,"maxContains",MISSING_VALIDATOR); + arraySpec->minContains = getIntegralValue(builder,object,"minContains",MISSING_VALIDATOR); + if (arraySpec->maxItems != MISSING_VALIDATOR){ + checkNonNegative(builder,arraySpec->maxItems,"maxItems"); + arraySpec->validatorFlags |= JS_VALIDATOR_MAX_ITEMS; + } + if (arraySpec->minItems != MISSING_VALIDATOR){ + checkNonNegative(builder,arraySpec->minItems,"minItems"); + arraySpec->validatorFlags |= JS_VALIDATOR_MIN_ITEMS; + } + if (arraySpec->maxContains != MISSING_VALIDATOR){ + checkNonNegative(builder,arraySpec->maxContains,"maxContains"); + arraySpec->validatorFlags |= JS_VALIDATOR_MAX_CONTAINS; + } + if (arraySpec->minContains != MISSING_VALIDATOR){ + checkNonNegative(builder,arraySpec->minContains,"minContains"); + arraySpec->validatorFlags |= JS_VALIDATOR_MIN_CONTAINS; + } + // recursion on items + Json *items = jsonObjectGetPropertyValue(object,"items"); + if (items != NULL){ + accessPathPushName(accessPath,"items"); + arraySpec->itemSpec = build(builder,valueSpec,items,false); + accessPathPop(accessPath); + } else { + // what does this default to or error + } + } + break; + case JSTYPE_STRING: + { + JSValueSpec *stringSpec = valueSpec; + stringSpec->maxLength = getIntegralValue(builder,object,"maxLength",MISSING_VALIDATOR); + stringSpec->minLength = getIntegralValue(builder,object,"minLength",MISSING_VALIDATOR); + if (stringSpec->maxLength != MISSING_VALIDATOR){ + checkNonNegative(builder,stringSpec->maxLength,"maxLength"); + stringSpec->validatorFlags |= JS_VALIDATOR_MAX_LENGTH; + } + if (stringSpec->minLength != MISSING_VALIDATOR){ + checkNonNegative(builder,stringSpec->minLength,"maxLength"); + stringSpec->validatorFlags |= JS_VALIDATOR_MIN_LENGTH; + } + stringSpec->pattern = getString(builder,object,"pattern",NULL); // hard to support on ANSI C + } + break; + case JSTYPE_NULL: + { + // no customizations ?! + } + break; + case JSTYPE_BOOLEAN: + { + // no customizations ?! + } + break; + case JSTYPE_NUMBER: + case JSTYPE_INTEGER: + { + if (!handledNumeric){ + handledNumeric = true; + JSValueSpec *numericSpec = valueSpec; + // new JSNumericSpec(description,title,(jsType == JSType.INTEGER)); + numericSpec->multipleOf = getNumber(builder,object,"multipleOf",MISSING_FLOAT_VALIDATOR); + numericSpec->maximum = getNumber(builder,object,"maximum",MISSING_FLOAT_VALIDATOR); + numericSpec->exclusiveMaximum = getBooleanValue(builder,object,"exclusiveMaximum",false); + numericSpec->minimum = getNumber(builder,object,"minimum",MISSING_FLOAT_VALIDATOR); + numericSpec->exclusiveMinimum = getBooleanValue(builder,object,"exclusiveMinimum",false); + if (numericSpec->maximum != MISSING_FLOAT_VALIDATOR){ + numericSpec->validatorFlags |= JS_VALIDATOR_MAX; + } + if (numericSpec->minimum != MISSING_FLOAT_VALIDATOR){ + numericSpec->validatorFlags |= JS_VALIDATOR_MIN; + } + if (numericSpec->multipleOf != MISSING_FLOAT_VALIDATOR){ + checkPositiveDouble(builder,numericSpec->multipleOf,"multipleOf"); + numericSpec->validatorFlags |= JS_VALIDATOR_MULTOF; + } + } + } + break; + default: + printf("*** PANIC *** unknown JSType code=%d\n",typeCode); + } + + } + }/* for (jsType... */ + return valueSpec; + } else { + schemaThrow(builder,12,"top level schema must be an object"); + return NULL; + } +} + + +JsonSchemaBuilder *makeJsonSchemaBuilder(int version){ + JsonSchemaBuilder *builder = (JsonSchemaBuilder*)safeMalloc(sizeof(JsonSchemaBuilder),"JsonBuilder"); + memset(builder,0,sizeof(JsonSchemaBuilder)); + builder->slh = makeShortLivedHeap(0x10000, 100); + builder->version = version; + builder->accessPath = (AccessPath*)SLHAlloc(builder->slh,sizeof(AccessPath)); + builder->accessPath->currentSize = 0; + return builder; +} + +void freeJsonSchemaBuilder(JsonSchemaBuilder *builder){ + if (builder->slh){ + SLHFree(builder->slh); + } + safeFree((char*)builder,sizeof(JsonSchemaBuilder)); +} + +JsonSchema *jsonBuildSchema(JsonSchemaBuilder *builder, + Json *schemaJson){ + if (setjmp(builder->recoveryData) == 0) { /* normal execution */ + if (builder->traceLevel >= 2){ + printf("after setjmp normal\n"); + fflush(stdout); + } + JSValueSpec *topValueSpec = build(builder,NULL,schemaJson,true); + JsonSchema *schema = (JsonSchema*)SLHAlloc(builder->slh,sizeof(JsonSchema)); + schema->topValueSpec = topValueSpec; + schema->version = builder->version; + /* Take ownership of the SLH if build is successful */ + schema->slh = builder->slh; + builder->slh = NULL; + return schema; + } else { /* throw handling */ + if (builder->traceLevel >= 2){ + printf("schema build fail %s\n",builder->errorMessage); + fflush(stdout); + } + return NULL; + } +} + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/le.c b/c/le.c index d4e243560..2da29fe01 100644 --- a/c/le.c +++ b/c/le.c @@ -86,7 +86,7 @@ LibraryFunction libraryFunctionTable[LIBRARY_FUNCTION_COUNT] #endif }; -char *getCAA(){ +char *getCAA(void){ char *realCAA = NULL; #if !defined(METTLE) && defined(_LP64) @@ -162,15 +162,15 @@ static LibraryFunction *findLibraryFunction(int rtlVectorOffset){ #define ESTIMATED_RTL_VECTOR_SIZE 0xB00 -void showRTL(){ +void showRTL(void){ CAA *caa = (CAA*)getCAA(); void **rtlVector = caa->runtimeLibraryVectorTable; - printf("RTL Vector at 0x%x\n",rtlVector); + printf("RTL Vector at 0x%p\n",rtlVector); dumpbuffer((char*)rtlVector,ESTIMATED_RTL_VECTOR_SIZE); int estimatedEntries = ESTIMATED_RTL_VECTOR_SIZE / 4; for (int i=2; i +#include +#include +#include +#include +#include +#include +#include +#include "metalio.h" +#include "qsam.h" + +#else + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 600 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#endif + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "openprims.h" +#include "bpxnet.h" +#include "unixfile.h" +#include "json.h" +#include "parsetools.h" +#include "microjq.h" + + +/* + Grammar + + Expr = Operation (PIPE* Operation)* + + Opertion = + Literal + CreateArray + CreateObject + Filter + + Literal + Number + String dquote and squote'd + + CreateArray + LBRACK Expr (COMMA Expr) RBRACK + + CreateObject + LPAREN KeyValue (COMMA KeyValue) RPAREN + + KeyValue + Key COLON Expression + + Key Identifier + + + Test Filters + '.results[] | {name, age}' + + Real JQ + + set JQ="c:\temp" + %JQ%\jq-win64 ... + + */ + + +#define TOP (FIRST_GRULE_ID+0) +#define EXPR (FIRST_GRULE_ID+1) +#define EXPR_TAIL (FIRST_GRULE_ID+2) +#define EXPR_PIPE_OP (FIRST_GRULE_ID+3) +#define OPERATION (FIRST_GRULE_ID+4) +#define CREATE_ARRAY (FIRST_GRULE_ID+5) +#define CREATE_ARRAY_TAIL (FIRST_GRULE_ID+6) +#define COMMA_EXPR (FIRST_GRULE_ID+7) +#define CREATE_OBJECT (FIRST_GRULE_ID+8) +#define CREATE_OBJECT_TAIL (FIRST_GRULE_ID+9) +#define COMMA_KEY_VALUE (FIRST_GRULE_ID+10) +#define KEY_VALUE (FIRST_GRULE_ID+11) +#define FILTER (FIRST_GRULE_ID+12) +#define TRAVERSAL (FIRST_GRULE_ID+13) +#define MORE_TRAVERSALS (FIRST_GRULE_ID+14) +#define INDEX (FIRST_GRULE_ID+15) +#define PICK (FIRST_GRULE_ID+16) +#define EXPLODE (FIRST_GRULE_ID+17) +#define CONTEXT (FIRST_GRULE_ID+18) + +static char *getJQRuleName(int id){ + switch (id){ + case TOP: return "TOP"; + case EXPR: return "EXPR"; + case EXPR_TAIL: return "EXPR_TAIL"; + case EXPR_PIPE_OP: return "EXPR_PIPE_OP"; + case OPERATION: return "OPERATION"; + case CREATE_ARRAY: return "CREATE_ARRAY"; + case CREATE_ARRAY_TAIL: return "CREATE_ARRAY_TAIL"; + case COMMA_EXPR: return "COMMA_EXPR"; + case CREATE_OBJECT: return "CREATE_OBJECT"; + case CREATE_OBJECT_TAIL: return "CREATE_OBJECT_TAIL"; + case COMMA_KEY_VALUE: return "COMMA_KEY_VALUE"; + case KEY_VALUE: return "KEY_VALUE"; + case FILTER: return "FILTER"; + case TRAVERSAL: return "TRAVERSAL"; + case MORE_TRAVERSALS: return "MORE_TRAVERSALS"; + case INDEX: return "INDEX"; + case PICK: return "PICK"; + case EXPLODE: return "EXPLODE"; + case CONTEXT: return "CONTEXT"; + default: + return "UNKNOWN_RULE"; + } +} + +static GRuleSpec jqGrammar[] = { + { G_SEQ, TOP, .sequence = { EXPR, EOF_TOKEN, G_END} }, + { G_SEQ, EXPR, .sequence = { OPERATION, EXPR_TAIL, G_END }}, + { G_STAR, EXPR_TAIL, .star = EXPR_PIPE_OP }, + { G_SEQ, EXPR_PIPE_OP, .sequence = { JTOKEN_VBAR, OPERATION, G_END }}, + { G_ALT, OPERATION, .alternates = { JTOKEN_INTEGER, + /* JTOKEN_DQUOTE_STRING, + JTOKEN_SQUOTE_STRING, + CREATE_ARRAY, + CREATE_OBJECT, --> how are these distinguishable from traversals?? */ + FILTER, + G_END }}, + { G_SEQ, CREATE_ARRAY, .sequence = { JTOKEN_LBRACK, EXPR, CREATE_ARRAY_TAIL, JTOKEN_RBRACK, G_END }}, + { G_STAR, CREATE_ARRAY_TAIL, .star = COMMA_EXPR }, + { G_SEQ, COMMA_EXPR, .sequence = { JTOKEN_COMMA, EXPR, G_END }}, + { G_SEQ, CREATE_OBJECT, .sequence = { JTOKEN_LBRACE, KEY_VALUE, CREATE_OBJECT_TAIL, JTOKEN_RBRACE, G_END }}, + { G_STAR, CREATE_OBJECT_TAIL, .star = COMMA_KEY_VALUE }, + { G_SEQ, COMMA_KEY_VALUE, .sequence = {JTOKEN_COMMA, KEY_VALUE, G_END }}, + { G_SEQ, KEY_VALUE, .sequence = { JTOKEN_IDENTIFIER, JTOKEN_COLON, EXPR, G_END }}, + { G_SEQ, FILTER, .sequence = { TRAVERSAL, MORE_TRAVERSALS, G_END }}, + { G_ALT, TRAVERSAL, .alternates = { INDEX, PICK, EXPLODE, CONTEXT, G_END }}, + { G_STAR, MORE_TRAVERSALS, .star = TRAVERSAL }, + { G_SEQ, INDEX, .sequence = { JTOKEN_LBRACK, JTOKEN_INTEGER, JTOKEN_RBRACK, G_END}}, + { G_SEQ, PICK, .sequence = { JTOKEN_DOT, JTOKEN_IDENTIFIER, G_END }}, + { G_SEQ, EXPLODE, .sequence = { JTOKEN_LBRACK, JTOKEN_RBRACK, G_END }}, + { G_SEQ, CONTEXT, .sequence = { JTOKEN_DOT, G_END}}, + { G_END } +}; + + +Json *parseJQ(JQTokenizer *jqt, ShortLivedHeap *slh, int traceLevel){ + AbstractTokenizer *tokenizer = (AbstractTokenizer*)jqt; + tokenizer->lowTokenID = FIRST_REAL_TOKEN_ID; + tokenizer->highTokenID = LAST_JTOKEN_ID; + tokenizer->nextToken = getNextJToken; + tokenizer->getTokenIDName = getJTokenName; + tokenizer->getTokenJsonType = getJTokenJsonType; + GParseContext *ctx = gParse(jqGrammar,TOP,tokenizer,jqt->data,jqt->length,getJQRuleName,traceLevel); + if (ctx->status > 0){ + return gBuildJSON(ctx,slh); + } else { + return NULL; + } +} + +#define JQ_ERROR_MAX 1024 + + + +typedef struct JQEvalContext_tag { + FILE *out; + jmp_buf recoveryData; + int flags; + int traceLevel; + int errorCode; + char *errorMessage; + int errorMessageLength; + jsonPrinter *printer; + FILE *traceOut; +} JQEvalContext; + +#define JQ_SUCCESS 0 +#define JQ_MISSING_PROPERTY 8 +#define JQ_BAD_PROPERTY_TYPE 12 +#define JQ_NOT_AN_OBJECT 16 +#define JQ_NOT_AN_ARRAY 20 +#define JQ_UNEXPECTED_OPERATION 24 +#define JQ_INTERNAL_NULL_POINTER 28 +#define JQ_NON_INTEGER_ARRAY_INDEX_NOT_YET_SUPPORTED 32 +#define JQ_ARRAY_INDEX_OUT_OF_BOUNDS 36 + +static void jqEvalThrow(JQEvalContext *ctx, int errorCode, char *formatString, ...){ + va_list argPointer; + char *text = safeMalloc(JQ_ERROR_MAX,"ErrorBuffer"); + va_start(argPointer,formatString); + vsnprintf(text,JQ_ERROR_MAX,formatString,argPointer); + va_end(argPointer); + ctx->errorCode = errorCode; + ctx->errorMessage = text; + ctx->errorMessageLength = JQ_ERROR_MAX; + longjmp(ctx->recoveryData,1); +} + +static Json *getJQProperty(JQEvalContext *ctx, JsonObject *value, char *key, bool isRequired){ + if (value == NULL){ + jqEvalThrow(ctx,JQ_INTERNAL_NULL_POINTER,"NULL pointer seen when looking for key = '%s'",key); + return NULL; + } + Json *propertyValue = jsonObjectGetPropertyValue(value,key); + if (isRequired && (propertyValue == NULL)){ + jqEvalThrow(ctx,JQ_MISSING_PROPERTY,"Missing required property %s",key); + } + return propertyValue; +} + +static int getJQInt(JQEvalContext *ctx, JsonObject *value, char *key, bool isRequired, int defaultValue){ + Json *intProperty = getJQProperty(ctx,value,key,isRequired); + if (jsonIsNumber(intProperty)){ + return jsonAsNumber(intProperty); + } else if (isRequired){ + jqEvalThrow(ctx,JQ_BAD_PROPERTY_TYPE,"Property '%s' is not int",key); + return 0; /* unreachable */ + } else { + return defaultValue; + } +} + +static JsonObject *getJQObject(JQEvalContext *ctx, JsonObject *value, char *key, bool isRequired){ + Json *p = getJQProperty(ctx,value,key,isRequired); + if (jsonIsObject(p)){ + return jsonAsObject(p); + } else if (isRequired){ + jqEvalThrow(ctx,JQ_BAD_PROPERTY_TYPE,"Property '%s' is not object",key); + return NULL; /* unreachable */ + } else { + return NULL; + } +} + +static char *getJQString(JQEvalContext *ctx, JsonObject *value, char *key, bool isRequired){ + Json *p = getJQProperty(ctx,value,key,isRequired); + if (jsonIsString(p)){ + return jsonAsString(p); + } else if (isRequired){ + jqEvalThrow(ctx,JQ_BAD_PROPERTY_TYPE,"Property '%s' is not string",key); + return NULL; /* unreachable */ + } else { + return NULL; + } +} + +static JsonArray *getJQArray(JQEvalContext *ctx, JsonObject *value, char *key, bool isRequired){ + Json *p = getJQProperty(ctx,value,key,isRequired); + if (jsonIsArray(p)){ + return jsonAsArray(p); + } else if (isRequired){ + jqEvalThrow(ctx,JQ_BAD_PROPERTY_TYPE,"Property '%s' is not array",key); + return NULL; /* unreachable */ + } else { + return NULL; + } +} + +static JsonObject *jqCastToObject(JQEvalContext *ctx, Json *json){ + if (json == NULL){ + jqEvalThrow(ctx,JQ_INTERNAL_NULL_POINTER,"NULL pointer seen when casting to JsonObject"); + return NULL; + } + if (jsonIsObject(json)){ + return jsonAsObject(json); + } else { + jqEvalThrow(ctx,JQ_NOT_AN_OBJECT,"Attempt to use value as object that is not an object"); + return NULL; /* unreachable */ + } +} + +static JsonArray *jqCastToArray(JQEvalContext *ctx, Json *json){ + if (jsonIsArray(json)){ + return jsonAsArray(json); + } else { + jqEvalThrow(ctx,JQ_NOT_AN_ARRAY,"Attempt to use value as array that is not an array"); + return NULL; /* unreachable */ + } +} + +static void jqPrint(JQEvalContext *ctx, Json *json){ + if (jsonIsString(json) && ctx->flags & JQ_FLAG_RAW_STRINGS){ + char *s = jsonAsString(json); + fprintf(ctx->out,"%s",s); + } else { + jsonPrint(ctx->printer,json); + } +} + +static void evalTraversal(JQEvalContext *ctx, Json *value, Json *filter, int index, int moreCount){ + JsonObject *filterObject = jqCastToObject(ctx,filter); + if (ctx->traceLevel >= 1){ + fprintf(ctx->traceOut,"evalTraversal, index=%d mcount=%d\n",index,moreCount); + } + Json *firstTraversal = getJQProperty(ctx,filterObject,"TRAVERSAL",true); + JsonArray *moreTraversals = getJQArray(ctx,filterObject,"MORE_TRAVERSALS",true); + Json *traversal = (index == 0 ) ? firstTraversal : jsonArrayGetItem(moreTraversals,index-1); + JsonObject *traversalObject = jqCastToObject(ctx,traversal); + int traversalType = getJQInt(ctx,traversalObject,"altID",true,0); + Json *traversalDetails = getJQProperty(ctx,traversalObject,"value",true); + switch (traversalType){ + case INDEX: + { + JsonObject *detailsObject = jqCastToObject(ctx,traversalDetails); + int arrayIndex = getJQInt(ctx,detailsObject,"INTEGER",false,-1); + if (arrayIndex == -1){ + jqEvalThrow(ctx,JQ_NON_INTEGER_ARRAY_INDEX_NOT_YET_SUPPORTED,"Index must be integer"); + } else { + JsonArray *valueArray = jqCastToArray(ctx,value); + if (arrayIndex >= 0 || arrayIndex < jsonArrayGetCount(valueArray)){ + Json *valueForIndex = jsonArrayGetItem(valueArray,arrayIndex); + if (index+1 <= moreCount){ + evalTraversal(ctx,valueForIndex,filter,index+1,moreCount); + } else { + jqPrint(ctx,valueForIndex); + } + } else { + jqEvalThrow(ctx,JQ_ARRAY_INDEX_OUT_OF_BOUNDS,"% is not in size of array",arrayIndex); + } + } + } + break; + case PICK: + { + JsonObject *detailsObject = jqCastToObject(ctx,traversalDetails); + char *identifier = getJQString(ctx,detailsObject,"ID",true); + JsonObject *valueObject = jqCastToObject(ctx,value); + Json *valueForID = getJQProperty(ctx,valueObject,identifier,true); + if (index+1 <= moreCount){ + evalTraversal(ctx,valueForID,filter,index+1,moreCount); + } else { + jqPrint(ctx,valueForID); + } + } + break; + case EXPLODE: + { + JsonArray *valueArray = jqCastToArray(ctx,value); + int valueCount = jsonArrayGetCount(valueArray); + for (int v=0; vprinter); + Json *element = jsonArrayGetItem(valueArray,v); + if (index+1 <= moreCount){ + evalTraversal(ctx,element,filter,index+1,moreCount); + } else { + jqPrint(ctx,element); + } + fprintf(ctx->out,"\n"); + } + } + break; + case CONTEXT: + if (index+1 <= moreCount){ + evalTraversal(ctx,value,filter,index+1,moreCount); + } else { + jqPrint(ctx,value); + } + break; + default: + jqEvalThrow(ctx,JQ_UNEXPECTED_OPERATION,"unexpeted operation %d",traversalType); + } +} + +static void evalFilter(JQEvalContext *ctx, Json *value, Json *filter){ + JsonObject *filterObject = jqCastToObject(ctx,filter); + JsonArray *moreTraversals = getJQArray(ctx,filterObject,"MORE_TRAVERSALS",true); + int moreCount = jsonArrayGetCount(moreTraversals); + evalTraversal(ctx,value,filter,0,moreCount); +} + +static void evalOperation(JQEvalContext *ctx, Json *value, Json *operation){ + JsonObject *operationObject = jqCastToObject(ctx,operation); + int altID = getJQInt(ctx,operationObject,"altID",true,-1); + switch (altID){ + case FILTER: + evalFilter(ctx,value,getJQProperty(ctx,operationObject,"value",true)); + break; + default: + jqEvalThrow(ctx,JQ_UNEXPECTED_OPERATION,"unexpected operation %d",altID); + } +} + +static void evalJQExpr(JQEvalContext *ctx, Json *value, Json *expr){ + JsonObject *exprObject = jqCastToObject(ctx,expr); + Json *operation = getJQProperty(ctx,exprObject,"OPERATION",true); + Json *exprTail = getJQProperty(ctx,exprObject,"EXPR_TAIL",true); + evalOperation(ctx,value,operation); + /* ignoring tail for now */ +} + +int evalJQ(Json *value, Json *jqTree, FILE *out, int flags, int traceLevel){ + JQEvalContext ctx; + memset(&ctx,0,sizeof(JQEvalContext)); + ctx.out = out; +#ifdef __ZOWE_OS_WINDOWS + int fd = _fileno(out); +#else + int fd = fileno(out); +#endif + ctx.printer = makeJsonPrinter(fd); + if (flags & JQ_FLAG_PRINT_PRETTY){ + jsonEnablePrettyPrint(ctx.printer); + } + ctx.traceLevel = traceLevel; + ctx.traceOut = stderr; + ctx.flags = flags; + if (setjmp(ctx.recoveryData) == 0) { /* normal execution */ + evalJQExpr(&ctx,value,getJQProperty(&ctx,jqCastToObject(&ctx,jqTree),"EXPR",true)); + return JQ_SUCCESS; + } else { + fprintf(stderr,"uJQ error message: %s\n",ctx.errorMessage); + return ctx.errorCode; + } +} + +/* + Known examples from zowe-install-plugins + +arguments: "-r .apimlServices.static.file" +arguments: "-r .apimlServices.static[].file" +arguments: "-r .appfwPlugins[0].path" +arguments: "-r .commands.configure" +arguments: "-r .commands.preConfigure" +arguments: "-r .commands.start" +arguments: "-r .commands.validate" +arguments: "-r .components.caching-service.storage.mode" +arguments: "-r .gatewaySharedLibs[0]" +arguments: "-r .zOSMF.host" +arguments: "-r .zOSMF.port" +arguments: "-r .zowe.launchScript.logLevel" +arguments: "-r .zowe.runtimeDirectory" +arguments: "-r .zowe.setup.certificate.dname.caCommonName" +arguments: "-r .zowe.setup.certificate.dname.commonName" +arguments: "-r .zowe.setup.certificate.dname.country" +arguments: "-r .zowe.setup.certificate.dname.locality" +arguments: "-r .zowe.setup.certificate.dname.org" +arguments: "-r .zowe.setup.certificate.dname.orgUnit" +arguments: "-r .zowe.setup.certificate.dname.state" +arguments: "-r .zowe.setup.certificate.importCertificateAuthorities" +arguments: "-r .zowe.setup.certificate.pkcs12.caAlias" +arguments: "-r .zowe.setup.certificate.pkcs12.caPassword" +arguments: "-r .zowe.setup.certificate.pkcs12.directory" +arguments: "-r .zowe.setup.certificate.pkcs12.import.alias" +arguments: "-r .zowe.setup.certificate.pkcs12.import.keystore" +arguments: "-r .zowe.setup.certificate.pkcs12.import.password" +arguments: "-r .zowe.setup.certificate.pkcs12.name" +arguments: "-r .zowe.setup.certificate.pkcs12.password" +arguments: "-r .zowe.setup.certificate.san" +arguments: "-r .zowe.setup.certificate.type" +arguments: "-r .zowe.setup.certificate.validity" +arguments: "-r .zowe.setup.certificate.zOSMF.ca" +arguments: "-r .zowe.setup.certificate.zOSMF.user" +arguments: "-r .zowe.setup.mvs.authLoadlib" +arguments: "-r .zowe.setup.mvs.authPluginLib" +arguments: "-r .zowe.setup.mvs.hlq" +arguments: "-r .zowe.setup.mvs.jcllib" +arguments: "-r .zowe.setup.mvs.parmlib" +arguments: "-r .zowe.setup.mvs.proclib" +arguments: "-r .zowe.setup.security.groups.admin" +arguments: "-r .zowe.setup.security.groups.stc" +arguments: "-r .zowe.setup.security.groups.sysProg" +arguments: "-r .zowe.setup.security.product" +arguments: "-r .zowe.setup.security.stcs.aux" +arguments: "-r .zowe.setup.security.stcs.xmem" +arguments: "-r .zowe.setup.security.stcs.zowe" +arguments: "-r .zowe.setup.security.users.aux" +arguments: "-r .zowe.setup.security.users.xmem" +arguments: "-r .zowe.setup.security.users.zowe" +arguments: "-r .zowe.verifyCertificates" +arguments: "-r .zowe.workspaceDirectory" + */ + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/parsetools.c b/c/parsetools.c new file mode 100644 index 000000000..8568b35e0 --- /dev/null +++ b/c/parsetools.c @@ -0,0 +1,889 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#include +#include +#include +#include +#include +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "parsetools.h" +#include "json.h" + +/* + + clang -I../h -I../platform/windows -D_CRT_SECURE_NO_WARNINGS -Dstrdup=_strdup -DYAML_DECLARE_STATIC=1 -Wdeprecated-declarations --rtlib=compiler-rt -o parsetools.exe parsetools.c timeutls.c utils.c alloc.c + + Grammar + (PHRASEREF n) + (TOKENREF n) + (ALT n) + (SEQ n) + (STAR ) + (PLUS n) + + */ + +static int cp1047to1208[256] = { + 0x00, 0x01, 0x02, 0x03, 0x9c, 0x09, 0x86, 0x7f, 0x97, 0x8d, 0x8e, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x9d, 0x0a, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8f, 0x1c, 0x1d, 0x1e, 0x1f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1b, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x05, 0x06, 0x07, + 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9a, 0x9b, 0x14, 0x15, 0x9e, 0x1a, + 0x20, 0xa0, 0xe2, 0xe4, 0xe0, 0xe1, 0xe3, 0xe5, 0xe7, 0xf1, 0xa2, 0x2e, 0x3c, 0x28, 0x2b, 0x7c, + 0x26, 0xe9, 0xea, 0xeb, 0xe8, 0xed, 0xee, 0xef, 0xec, 0xdf, 0x21, 0x24, 0x2a, 0x29, 0x3b, 0x5e, + 0x2d, 0x2f, 0xc2, 0xc4, 0xc0, 0xc1, 0xc3, 0xc5, 0xc7, 0xd1, 0xa6, 0x2c, 0x25, 0x5f, 0x3e, 0x3f, + 0xf8, 0xc9, 0xca, 0xcb, 0xc8, 0xcd, 0xce, 0xcf, 0xcc, 0x60, 0x3a, 0x23, 0x40, 0x27, 0x3d, 0x22, + 0xd8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xab, 0xbb, 0xf0, 0xfd, 0xfe, 0xb1, + 0xb0, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0xaa, 0xba, 0xe6, 0xb8, 0xc6, 0xa4, + 0xb5, 0x7e, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0xa1, 0xbf, 0xd0, 0x5b, 0xde, 0xae, + 0xac, 0xa3, 0xa5, 0xb7, 0xa9, 0xa7, 0xb6, 0xbc, 0xbd, 0xbe, 0xdd, 0xa8, 0xaf, 0x5d, 0xb4, 0xd7, + 0x7b, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xad, 0xf4, 0xf6, 0xf2, 0xf3, 0xf5, + 0x7d, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0xb9, 0xfb, 0xfc, 0xf9, 0xfa, 0xff, + 0x5c, 0xf7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0xb2, 0xd4, 0xd6, 0xd2, 0xd3, 0xd5, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xb3, 0xdb, 0xdc, 0xd9, 0xda, 0x9f +}; + +bool testCharProp(int ccsid, int c, int prop){ + switch(ccsid){ + case IBM1047: + return (charProps[cp1047to1208[c]] & prop) != 0; + case UTF8: + return (charProps[c] & prop) != 0; + default: + printf("*** PANIC *** Unsupported CCSID = %d\n",ccsid); + return false; + } +} + +#define MAX_DEPTH 200 + + +typedef struct SavedState_tag { + GRule *rule; + int index; + int pos; + GRule *parent; + int indexInParent; + /* AST building */ + SyntaxNode *node; + int nodeChildCount; + +} SavedState; + +static void setParseState(SavedState *state, + GRule *rule, + int index, + int pos, + GRule *parent, + int indexInParent){ + state->rule = rule; + state->index = index; + state->pos = pos; + state->parent = parent; + state->indexInParent = indexInParent; +} + +static int countRuleSet(GRule *rules){ + int count = 0; + while (rules[count].type != G_END){ + count++; + } + return count; +} + +static bool matchToken(AbstractTokenizer *tokenizer, + char *s, int len, + int pos, int *nextPos, + int desiredTokenID, + int *tokenIDPtr){ + int tokenID = tokenizer->nextToken(tokenizer,s,len,pos,nextPos); + *tokenIDPtr = tokenID; + return (tokenID == desiredTokenID); +} + +static bool isTokenReference(AbstractTokenizer *tokenizer, int id){ + return ((id == EOF_TOKEN) || + (id >= tokenizer->lowTokenID && id <= tokenizer->highTokenID)); +} + +static GRule *getRule(GRule *ruleSet, int count, int id){ + /* printf("get rule %d\n",id); */ + fflush(stdout); + if (id < FIRST_GRULE_ID){ + printf("rule id %d lower than min %d\n",id,FIRST_GRULE_ID); + return NULL; + } + for (int i=0; irefs = (GRuleRef*)SLHAlloc(slh,rule->subCount*sizeof(GRuleRef)); + for (int i=0; isubCount; i++){ + int index = indices[i]; + /* printf("setRRA i=%d\n",i); */ + GRuleRef *ref = &rule->refs[i]; + ref->isTokenRef = isTokenReference(tokenizer,index); + if (ref->isTokenRef){ + ref->tokenID = index; + } else { + GRule *referent = getRule(ruleSet,ruleCount,index); + if (referent == NULL){ + return false; + } + ref->parent = rule; + ref->rule = referent; + ref->indexInParent = index; + } + } + return true; +} + +static GRule *makeGRules(GRuleSpec *ruleSpecs, AbstractTokenizer *tokenizer, ShortLivedHeap *slh, int *ruleCountPtr){ + int ruleCount = 0; + while (ruleSpecs[ruleCount].type != G_END){ + ruleCount++; + } + *ruleCountPtr = ruleCount; + GRule *rules = (GRule*)SLHAlloc(slh,ruleCount*sizeof(GRule)); + int r; + for (r=0; rtype = spec->type; + rule->id = spec->id; + /* printf("rule at 0x%p with id=%d\n",rule,rule->id); */ + } + for (r=0; rtype){ + case G_SEQ: + { + int subCount = 0; + while (spec->sequence.subs[subCount] != G_END) subCount++; + rule->subCount = subCount; + if (!setRuleRefArray(slh,tokenizer,rule,rules,ruleCount,&spec->sequence.subs[0])){ + return NULL; + } + } + break; + case G_ALT: + { + int subCount = 0; + while (spec->alternates.subs[subCount] != G_END) subCount++; + rule->subCount = subCount; + if (!setRuleRefArray(slh,tokenizer,rule,rules,ruleCount,&spec->alternates.subs[0])){ + return NULL; + } + } + break; + case G_STAR: + { + int subCount = 1; + rule->subCount = subCount; + if (!setRuleRefArray(slh,tokenizer,rule,rules,ruleCount,&spec->star.sub)){ + return NULL; + } + } + break; + default: + printf("*** PANIC unhandled rule type in compilation %d\n",rule->type); + } + } + return rules; +} + + +typedef struct GContinuation_tag { + struct GContinuation_tag *previous; +#ifdef METTLE +#error "need setjmp" +#else + jmp_buf resumeData; +#endif + int indexInRule; + int position; + int depth; + int buildStepHWM; +} GContinuation; + +static char *getBuildStepTypeName(int type){ + switch (type){ + case BUILD_OBJECT: return "Object"; + case BUILD_ARRAY: return "Array"; + case BUILD_STRING: return "String"; + case BUILD_INT: return "int"; + case BUILD_INT64: return "int64"; + case BUILD_DOUBLE: return "double"; + case BUILD_BOOL: return "boolean"; + case BUILD_NULL: return "null"; + case BUILD_KEY: return "key"; + case BUILD_POP: return "pop()"; + case BUILD_ALT: return "altWrap()"; + default: return "UnknownBuildStep"; + } + +} + +#define GPARSE_SUCCESS(x) ((x)>=0) +#define GPARSE_FAIL (-1) +#define setResumePoint(cPtr) setjmp((cPtr)->resumeData) + +static void setStepRef(GBuildStep *step, GRuleRef *ref){ + step->tokenID = (ref->isTokenRef ? ref->tokenID : NO_VALID_TOKEN); + step->rule = ref->rule; +} + +static void setAltChoice(GBuildStep *step, GRuleRef *ref){ + /* printf("setAltChoice isTokenRef=%d, tokenID=%d\n",ref->isTokenRef,ref->tokenID); */ + step->altChoice = ref; +} + +static GBuildStep *makeBuildStep(GParseContext *ctx, int type, GRuleRef *ref, GRule *rule){ + if (ctx->buildStepCount >= ctx->buildStepsSize){ + printf("**** PANIC **** out of build steps\n"); + printf("**** PANIC **** out of build steps\n"); + printf("**** PANIC **** out of build steps\n"); + printf("**** PANIC **** out of build steps\n"); + return NULL; + } + GBuildStep *step = &ctx->buildSteps[ctx->buildStepCount++]; + memset(step,0,sizeof(GBuildStep)); + step->type = type; + if (ref){ + setStepRef(step,ref); + } else if (rule){ + step->tokenID = NO_VALID_TOKEN; + step->rule = rule; + } else { + step->tokenID = NO_VALID_TOKEN; + step->rule = NULL; + } + step->valueStart = STEP_UNDEFINED_POSITION; + step->valueEnd = STEP_UNDEFINED_POSITION; + return step; +} + +static void buildTokenValue(GParseContext *ctx, GRuleRef *ref, int lastTokenStart, int lastTokenEnd){ + int jsonType = JSON_TYPE_STRING; + if (ctx->tokenizer->getTokenJsonType){ + jsonType = ctx->tokenizer->getTokenJsonType(ref->tokenID); + } + int buildType = BUILD_STRING; + switch (jsonType){ + case JSON_TYPE_NUMBER: + buildType = BUILD_NUMBER; + break; + } + GBuildStep *step = makeBuildStep(ctx,buildType,ref,NULL); + step->valueStart = lastTokenStart; + step->valueEnd = lastTokenEnd; +} + +static void buildSeqElement(GParseContext *ctx, GRuleRef *ref){ + makeBuildStep(ctx,BUILD_KEY,ref,NULL); +} + +static GBuildStep *buildAltValue(GParseContext *ctx, GRule *rule){ + return makeBuildStep(ctx,BUILD_ALT,NULL,rule); +} + +static void buildObject(GParseContext *ctx, GRule *rule){ + makeBuildStep(ctx,BUILD_OBJECT,NULL,rule); +} + +static void buildArray(GParseContext *ctx, GRule *rule){ + makeBuildStep(ctx,BUILD_ARRAY,NULL,rule); +} + +static void buildPop(GParseContext *ctx){ + makeBuildStep(ctx,BUILD_POP,NULL,NULL); +} + +static void indent(int depth){ + for (int i=0; itraceLevel >= 1){ + printf("-------> Backtracking... build step back from %d to %d\n", + ctx->buildStepCount, + continuation->buildStepHWM); + fflush(stdout); + } + longjmp(continuation->resumeData,1); +} + +static int runTokenMatch(GParseContext *ctx, GRuleRef *ref, int pos, int depth){ + AbstractTokenizer *tokenizer = ctx->tokenizer; + int nextPos = 0; + int tokenID = 0; + if (ctx->traceLevel >= 1){ + indent(depth); + printf("runTokenMatch trying id=0x%x, (%s) pos=%d\n",ref->tokenID,tokenizer->getTokenIDName(ref->tokenID),pos); + fflush(stdout); + } + bool matched = matchToken(tokenizer,ctx->s,ctx->len,pos,&nextPos,ref->tokenID,&tokenID); + if (matched){ + buildTokenValue(ctx,ref,tokenizer->lastTokenStart,tokenizer->lastTokenEnd); + return nextPos; + } else if (tokenID == NO_VALID_TOKEN){ + printf("*** Tokenizer failed, parse must stop\n"); + printf("*** Tokenizer failed, parse must stop\n"); + printf("*** Tokenizer failed, parse must stop\n"); + printf("*** Tokenizer failed, parse must stop\n"); + /* should throw to outer error handler */ + return 0x7FFFFFFF; + } else { + return GPARSE_FAIL; + } +} + +static char *ruleName(GParseContext *ctx, GRule *rule){ + return ctx->ruleNamer(rule->id); +} + +static int runSeq(GParseContext *ctx, GRule *rule, int startPos, int depth, GContinuation *resumePoint){ + if (ctx->traceLevel >= 1){ + indent(depth); + printf("runSeq (%s) at pos=%d, resume=0x%p\n",ruleName(ctx,rule),startPos,resumePoint); + fflush(stdout); + } + int pos = startPos; + for (int index=0; indexsubCount; index++){ + GRuleRef *ref = &rule->refs[index]; + buildSeqElement(ctx,ref); + int matchResult = + (ref->isTokenRef ? + runTokenMatch(ctx,ref,pos,depth) : + parseDispatch(ctx,ref->rule,pos,depth,resumePoint)); + if (ctx->traceLevel >= 1){ + indent(depth+1); + printf("seq submatch index=%d result=%d\n",index,matchResult); + } + if (GPARSE_SUCCESS(matchResult)){ + pos = matchResult; + } else if (resumePoint){ + backtrack(ctx,resumePoint); + } else { + return GPARSE_FAIL; + } + } + return pos; +} + +static int runAlt(GParseContext *ctx, GRule *rule, int startPos, int depth, GContinuation *resumePoint){ + if (ctx->traceLevel >= 1){ + indent(depth); + printf("runAlt (%s) at pos=%d, resume=0x%p\n",ruleName(ctx,rule),startPos,resumePoint); + fflush(stdout); + } + GBuildStep *altStep = buildAltValue(ctx,rule); + GContinuation continuation; + continuation.previous = resumePoint; + continuation.indexInRule = 0; + continuation.position = startPos; + continuation.depth = depth; + continuation.buildStepHWM = ctx->buildStepCount; + int isResumption = setResumePoint(&continuation); + int index = (isResumption ? continuation.indexInRule : 0); + int pos = (isResumption ? continuation.position : startPos); + ctx->buildStepCount = continuation.buildStepHWM; + for (; indexsubCount; index++){ + GRuleRef *ref = &rule->refs[index]; + setAltChoice(altStep,ref); + continuation.indexInRule = index+1; + if (ref->isTokenRef){ + int matchResult = runTokenMatch(ctx,ref,pos,depth); + if (GPARSE_SUCCESS(matchResult)){ + if (ctx->traceLevel >= 1){ + indent(depth+1); + printf("alt token success matchResult=%d\n",matchResult); + } + return matchResult; + } else if (index+1 < rule->subCount){ + // just roll forward + if (ctx->traceLevel >= 1){ + indent(depth+1); + printf("alt token roll forward\n"); + } + } else if (resumePoint){ + if (ctx->traceLevel >= 1){ + indent(depth+1); + printf("alt token backtrack\n"); + } + backtrack(ctx,resumePoint); + } else { + if (ctx->traceLevel >= 1){ + indent(depth+1); + printf("alt token fail\n"); + } + return GPARSE_FAIL; + } + } else { + GContinuation *failContinuation = ((index + 1 < rule->subCount) ? + &continuation : + resumePoint); + int parseResult = parseDispatch(ctx,ref->rule,pos,depth,failContinuation); + if (ctx->traceLevel >= 1){ + indent(depth+1); + printf("alt subrule res = %d\n",parseResult); + } + return parseResult; // we never loop except when hitting a token + } + } + if (ctx->traceLevel >= 1){ + indent(depth+1); + printf("out of ALT's\n"); + } + return GPARSE_FAIL; +} + +// can easily by plus or repeat, or opt +static int runStar(GParseContext *ctx, GRule *rule, int startPos, int depth, GContinuation *resumePoint){ + if (ctx->traceLevel >= 1){ + indent(depth); + printf("runStar (%s) at pos=%d, resume=0x%p\n",ruleName(ctx,rule),startPos,resumePoint); + fflush(stdout); + } + GContinuation continuation; + continuation.previous = resumePoint; + continuation.indexInRule = 0; + continuation.position = startPos; + continuation.depth = depth; + continuation.buildStepHWM = ctx->buildStepCount; + GRuleRef *ref = &rule->refs[0]; + int isResumption = setResumePoint(&continuation); + if (isResumption){ + ctx->buildStepCount = continuation.buildStepHWM; + return continuation.position; // still always a success + } else { + /* this is a "greedy" star */ + while (true){ + int matchResult = + (ref->isTokenRef ? + runTokenMatch(ctx,ref,continuation.position,depth) : + parseDispatch(ctx,ref->rule,continuation.position,depth,&continuation)); + if (GPARSE_SUCCESS(matchResult)){ + continuation.position = matchResult; + continuation.buildStepHWM = ctx->buildStepCount; + continuation.indexInRule++; + // keep looping + } else { + return continuation.position; // it's always a success, even if zero matched + } + } + } +} + +static int parseDispatch(GParseContext *ctx, GRule *rule, int startPos, int depth, GContinuation *resumePoint){ + switch (rule->type){ + case G_SEQ: + { + buildObject(ctx,rule); + int res = runSeq(ctx,rule,startPos,depth+1,resumePoint); + buildPop(ctx); + return res; + } + case G_ALT: + { + int res = runAlt(ctx,rule,startPos,depth+1,resumePoint); + buildPop(ctx); + return res; + } + case G_STAR: + { + buildArray(ctx,rule); + int res = runStar(ctx,rule,startPos,depth+1,resumePoint); + buildPop(ctx); + return res; + } + default: + printf("*** PANIC *** Unhandled ruleType = %d\n",rule->type); + return GPARSE_FAIL; /* should bounce to "total failure setjmp" */ + } +} + +void showBuildSteps(GParseContext *ctx){ + for (int i=0; ibuildStepCount; i++){ + GBuildStep *step = &ctx->buildSteps[i]; + if (step->type == BUILD_ALT){ + bool isToken = (step->tokenID != NO_VALID_TOKEN); + int markingID = (step->tokenID != NO_VALID_TOKEN) ? step->tokenID : step->rule->id; + printf("Step(%d): ALT choice byToken=%d id=%d\n",i,isToken,markingID); + } else if (step->tokenID != NO_VALID_TOKEN){ + printf("Step(%d) %s: token %s text from %d to %d\n", + i, + getBuildStepTypeName(step->type), + ctx->tokenizer->getTokenIDName(step->tokenID), + step->valueStart,step->valueEnd); + } else if (step->rule){ + printf("Step(%d) %s: %s\n",i,getBuildStepTypeName(step->type),ctx->ruleNamer(step->rule->id)); + } else { + printf("Step(%d) %s\n",i,getBuildStepTypeName(step->type)); + } + } +} + +Json *gBuildJSON(GParseContext *ctx, ShortLivedHeap *slh){ + if (slh == NULL){ + slh = ctx->slh; + } + JsonBuilder *b = makeJsonBuilder(slh); + int sp = 0; + Json **parentStack = (Json**)SLHAlloc(ctx->slh,MAX_DEPTH*sizeof(Json*)); + Json *currentParent = NULL; + char *currentKey = NULL; + for (int i=0; ibuildStepCount; i++){ + if (ctx->traceLevel >= 1){ + printf("build step %d, currKey=%s\n",i,currentKey); + } + GBuildStep *step = &ctx->buildSteps[i]; + int errorCode = 0; + switch (step->type){ + case BUILD_OBJECT: + { + Json *o = jsonBuildObject(b,currentParent,currentKey,&errorCode); + parentStack[sp++] = currentParent; + currentParent = o; + currentKey = NULL; + } + break; + case BUILD_ARRAY: + { + Json *a = jsonBuildArray(b,currentParent,currentKey,&errorCode); + parentStack[sp++] = currentParent; + currentParent = a; + currentKey = NULL; + } + break; + case BUILD_POP: + { + currentParent = parentStack[--sp]; + currentKey = NULL; + } + break; + case BUILD_KEY: + { + /* unless EOF */ + switch (step->tokenID){ + case NO_VALID_TOKEN: + currentKey = ctx->ruleNamer(step->rule->id); + break; + case EOF_TOKEN: + currentKey = ""; + break; + default: + currentKey = ctx->tokenizer->getTokenIDName(step->tokenID); + break; + } + } + break; + case BUILD_ALT: + { + Json *alt = jsonBuildObject(b,currentParent,currentKey,&errorCode); + parentStack[sp++] = currentParent; + currentParent = alt; + GRuleRef *choiceRef = step->altChoice; + int choiceID = (choiceRef->isTokenRef ? choiceRef->tokenID : choiceRef->rule->id); + jsonBuildInt(b,alt,"altID",choiceID,&errorCode); + currentKey = "value"; + } + break; + case BUILD_STRING: + if (currentKey == NULL){ + printf("*** WARNING *** no parent for string!!!\n"); + } else { + int len = step->valueEnd - step->valueStart; + jsonBuildString(b,currentParent,currentKey,ctx->s+step->valueStart,len,&errorCode); + currentKey = NULL; /* because was consumed */ + } + break; + case BUILD_NUMBER: + { + int len = step->valueEnd - step->valueStart; + char *numberText = SLHAlloc(ctx->slh,len+1); + memcpy(numberText,ctx->s+step->valueStart,len); + numberText[len] = 0; + int64_t val = strtoll(numberText,NULL,10); + if (val < 0x7FFFFFFF){ + jsonBuildInt(b,currentParent,currentKey,(int)val,&errorCode); + } else { + jsonBuildInt64(b,currentParent,currentKey,val,&errorCode); + } + } + break; + case BUILD_INT64: + case BUILD_INT: + case BUILD_DOUBLE: + case BUILD_BOOL: + case BUILD_NULL: + printf("unhandled JSON value type for build type %d\n",step->type); + break; + default: + printf("*** PANIC *** unhandled JSON builder step type %d\n",step->type); + break; + } + /* + } else if (step->type + } else if (step->tokenID != NO_VALID_TOKEN){ + } else if (step->rule){ + printf("Step %s: %s\n",getBuildStepTypeName(step->type),ctx->ruleNamer(step->rule->id)); + } else { + printf("Step %s\n",getBuildStepTypeName(step->type)); + } + */ + if (errorCode){ + printf("last build step failed with code = %d\n",errorCode); + } + } + Json *result = b->root; + freeJsonBuilder(b,false); + return result; +} + +GParseContext *gParse(GRuleSpec *ruleSet, int topRuleID, AbstractTokenizer *tokenizer, char *s, int len, + char *(*ruleNamer)(int id), + int traceLevel){ + ShortLivedHeap *slh = makeShortLivedHeap(0x10000, 100); + GParseContext *ctx = (GParseContext *)SLHAlloc(slh,sizeof(GParseContext)); + memset(ctx,0,sizeof(GParseContext)); + ctx->tokenizer = tokenizer; + ctx->s = s; + ctx->len = len; + ctx->ruleNamer = ruleNamer; + ctx->slh = slh; + ctx->buildStepsSize = MAX_BUILD_STEPS; + ctx->buildSteps = (GBuildStep*)SLHAlloc(slh,ctx->buildStepsSize*sizeof(GBuildStep)); + ctx->traceLevel = traceLevel; + int ruleSetSize = 0; + GRule *rules = makeGRules(ruleSet,tokenizer,slh,&ruleSetSize); + ctx->rules = rules; + GRule *topRule = getRule(rules,ruleSetSize,topRuleID); + if (topRule->type == G_SEQ){ + int parseResult = parseDispatch(ctx,topRule,0,0,NULL); + ctx->status = parseResult; + if (ctx->traceLevel >= 1){ + printf("parseResult=%d\n",parseResult); + showBuildSteps(ctx); + } + return ctx; + } else { + printf("top rule must be G_SEQ\n"); + ctx->status = -1; + return ctx; + } +} + +/* JLexer (Flexible Tokenizer for java-ish languages) */ + +#define STATE_INITIAL 0 +#define STATE_INTEGER 1 +#define STATE_IDENTIFIER 2 +#define STATE_SQUOTE 3 +#define STATE_DQUOTE 4 + +char *getJTokenName(int id){ + switch (id){ + case EOF_TOKEN: return "EOF"; + case JTOKEN_IDENTIFIER: return "ID"; + case JTOKEN_INTEGER: return "INTEGER"; + case JTOKEN_DQUOTE_STRING: return "DQUOTE_STRING"; + case JTOKEN_SQUOTE_STRING: return "SQUOTE_STRING"; + case JTOKEN_LPAREN: return "LPAREN"; + case JTOKEN_RPAREN: return "RPAREN"; + case JTOKEN_LBRACK: return "LBRACK"; + case JTOKEN_RBRACK: return "RBRACK"; + case JTOKEN_LBRACE: return "LBRACE"; + case JTOKEN_RBRACE: return "RBRACE"; + case JTOKEN_DOT: return "DOT"; + case JTOKEN_COMMA: return "COMMA"; + case JTOKEN_VBAR: return "VBAR"; + case JTOKEN_COLON: return "COLON"; + case JTOKEN_QMARK: return "QMARK"; + case JTOKEN_PLUS: return "PLUS"; + case JTOKEN_DASH: return "DASH"; + case JTOKEN_STAR: return "STAR"; + case JTOKEN_SLASH: return "SLASH"; + case JTOKEN_PERCENT: return "PERCENT"; + default:return "BAD_TOKEN"; + } +} + +static bool isLetter(JQTokenizer *jqt, int c){ + return testCharProp(jqt->ccsid,c,CPROP_LETTER); +} + +static bool isDigit(JQTokenizer *jqt, int c){ + return testCharProp(jqt->ccsid,c,CPROP_DIGIT); +} + +static bool isWhite(JQTokenizer *jqt, int c){ + return testCharProp(jqt->ccsid,c,CPROP_WHITE); +} + +int jqtToken(JQTokenizer *jqt){ + /* belt *AND* suspenders */ + AbstractTokenizer *tokenizer = (AbstractTokenizer*)jqt; + if (jqt->pos >= jqt->length){ + tokenizer->lastTokenStart = jqt->pos; /* 0-length token */ + tokenizer->lastTokenEnd = jqt->pos; + return EOF_TOKEN; + } + int tokenID = NO_VALID_TOKEN; + while(tokenID == NO_VALID_TOKEN){ + int c = -1; // EOF by defaultx + int charPos = jqt->pos; + if (jqt->pos < jqt->length){ + c = (int)jqt->data[jqt->pos++]; + } + /* printf("at pos=%d, c=0x%x state=%d\n",jqt->pos,c,jqt->tokenState); */ + switch (jqt->tokenState){ + case STATE_INITIAL: + if (c == -1){ + tokenID = EOF_TOKEN; + tokenizer->lastTokenStart = jqt->length; + } else if (isLetter(jqt,c) || + (c == UNI_DOLLAR) || + (c == UNI_UNDER)){ + jqt->tokenState = STATE_IDENTIFIER; + tokenizer->lastTokenStart = charPos; + } else if (isDigit(jqt,c)){ + jqt->tokenState = STATE_INTEGER; + tokenizer->lastTokenStart = charPos; + } else if (isWhite(jqt,c)){ + // just roll forward + } else if (c == UNI_DQUOTE){ + jqt->tokenState = STATE_DQUOTE; + tokenizer->lastTokenStart = charPos; + } else if (c == UNI_SQUOTE){ + jqt->tokenState = STATE_SQUOTE; + tokenizer->lastTokenStart = charPos; + } else { + /* Single-char-punc tokens */ + switch (c){ + case UNI_LPAREN: tokenID = JTOKEN_LPAREN; break; + case UNI_RPAREN: tokenID = JTOKEN_RPAREN; break; + case UNI_LBRACK: tokenID = JTOKEN_LBRACK; break; + case UNI_RBRACK: tokenID = JTOKEN_RBRACK; break; + case UNI_LBRACE: tokenID = JTOKEN_LBRACE; break; + case UNI_RBRACE: tokenID = JTOKEN_RBRACE; break; + case UNI_DOT: tokenID = JTOKEN_DOT; break; + case UNI_COMMA: tokenID = JTOKEN_COMMA; break; + case UNI_COLON: tokenID = JTOKEN_COLON; break; + case UNI_VBAR: tokenID = JTOKEN_VBAR; break; + case UNI_QMARK: tokenID = JTOKEN_QMARK; break; + case UNI_PLUS: tokenID = JTOKEN_PLUS; break; + case UNI_DASH: tokenID = JTOKEN_DASH; break; + case UNI_STAR: tokenID = JTOKEN_STAR; break; + case UNI_SLASH: tokenID = JTOKEN_SLASH; break; + case UNI_PERCENT: tokenID = JTOKEN_PERCENT; break; + default: + /* unhandled char, kill the parse */ + return NO_VALID_TOKEN; + } + tokenizer->lastTokenStart = charPos; + } + break; + case STATE_INTEGER: + if (!isDigit(jqt,c)){ + if (c != -1) jqt->pos--; /* nothing to push back */ + tokenID = JTOKEN_INTEGER; + } + break; + case STATE_IDENTIFIER: + if (isLetter(jqt,c) || + (c == UNI_DOLLAR) || + (c == UNI_UNDER)){ + // accumulate + } else { + if (c != -1) jqt->pos--; /* nothing to push back */ + tokenID = JTOKEN_IDENTIFIER; + } + break; + case STATE_DQUOTE: + if (c == -1){ /* EOF in string */ + return NO_VALID_TOKEN; + } else if (c == UNI_DQUOTE){ + tokenID = JTOKEN_DQUOTE_STRING; + } + break; + case STATE_SQUOTE: + if (c == -1){ /* EOF in string */ + return NO_VALID_TOKEN; + } else if (c == UNI_SQUOTE){ + tokenID = JTOKEN_SQUOTE_STRING; + } + default: + printf("*** PANIC *** internal error unknown tokenizer state = %d\n",jqt->tokenState); + return NO_VALID_TOKEN; + // unknow state, kill the parse + } + if (tokenID != NO_VALID_TOKEN){ + tokenizer->lastTokenEnd = jqt->pos; + jqt->tokenState = STATE_INITIAL; + return tokenID; + } + } + return 0; +} + +int getNextJToken(AbstractTokenizer *tokenizer, char *s, int len, int pos, int *nextPos){ + JQTokenizer *jqt = (JQTokenizer*)tokenizer; + jqt->pos = pos; + int id = jqtToken(jqt); + *nextPos = jqt->pos; + return id; +} + +int getJTokenJsonType(int tokenID){ + switch (tokenID){ + case JTOKEN_INTEGER: + return JSON_TYPE_NUMBER; /* see json.h */ + default: + return JSON_TYPE_STRING; + } +} + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/psxskt.c b/c/psxskt.c new file mode 100644 index 000000000..f3f299650 --- /dev/null +++ b/c/psxskt.c @@ -0,0 +1,1492 @@ + + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#include +#include +#include +#include +#include +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "bpxnet.h" +#include "unixfile.h" + +#include +#include + +#include +#include +#include + +#ifndef __ZOWE_OS_AIX +#include +#endif + +#include +#include + +#include +#include /* gethostbyname */ +#include +#include +#include + +#ifdef USE_RS_SSL +#include "rs_ssl.h" +#endif + +int socketInit(char *uniqueName){ + /* don't do anything in POSIX */ + return 0; +} + +static int socketTrace = 0; +int setSocketTrace(int toWhat) { + int was = socketTrace; + socketTrace = toWhat; + return was; +} + +typedef union SockAddrAll_tag { + struct sockaddr_in v4; + struct sockaddr_in6 v6; +} SockAddrAll; + +#define IPPROTO_EVENT 254 /* semi-official "experimental" protocol number */ + +static +SocketAddress * setSocketAddr(SocketAddress *address, + const InetAddr *addr, + unsigned short port) +{ + memset(address,0,sizeof(SocketAddress)); + + address->family = (uint8_t) (addr ? addr->type : AF_INET); + if (addr && (port == 0)) { + address->port = (uint16_t) addr->port; + } else { + address->port = (uint16_t) htons(port); + } + + if (addr) { + switch (address->family) { + case AF_INET: + address->internalAddress.v4Address.sin_family = AF_INET; + address->internalAddress.v4Address.sin_port = (in_port_t) (port == 0 ? address->port : htons(port)); + memcpy(&(address->internalAddress.v4Address.sin_addr), &(addr->data.data4), sizeof(struct in_addr)); + break; + + case AF_INET6: + address->internalAddress.v6Address.sin6_family = AF_INET6; + address->internalAddress.v6Address.sin6_port = (in_port_t) (port == 0 ? address->port : htons(port)); + memcpy(&(address->internalAddress.v6Address.sin6_addr), &(addr->data.data6), sizeof(struct in6_addr)); + break; + + default: + if (socketTrace){ + printf("setSocketAddr: Unknown address family %d\n", (int) address->family); + } + return NULL; + } + } + +if (socketTrace) { + dumpbuffer(address, sizeof(SocketAddress)); +} + + return address; +} + +SocketAddress *makeSocketAddr(InetAddr *addr, + unsigned short port){ + SocketAddress *address = (SocketAddress*)safeMalloc(sizeof(SocketAddress),"SocketAddress"); + if (0 == setSocketAddr(address, addr, port)) { + freeSocketAddr(address); + address = (SocketAddress *) NULL; + } + return address; +} + +void freeSocketAddr(SocketAddress *address){ + safeFree((char*)address,sizeof(SocketAddress)); +} + +static +int getEndPointName(int sd, InetAddr *name, int which /* 0 = this end, other = peer */) +{ + SockAddrAll socketAddr; + memset(&socketAddr, 0, sizeof(SockAddrAll)); + int status = 0; + socklen_t sockLen = sizeof(SockAddrAll); + if (which == 0) { + status = getsockname(sd, (struct sockaddr *)&socketAddr, &sockLen); + } else { + status = getpeername(sd, (struct sockaddr *)&socketAddr, &sockLen); + } + + if (0 == status) { + /* fill in the address information */ + memset(name, 0, sizeof(InetAddr)); + sa_family_t domain = socketAddr.v4.sin_family; /* same place for all address types */ + name->type = (uint16_t) domain; + switch(domain) { + case AF_INET: + name->port = (uint16_t) socketAddr.v4.sin_port; + memcpy(&(name->data), &socketAddr.v4.sin_addr, sizeof(struct in_addr)); + break; + + case AF_INET6: + name->port = (uint16_t) socketAddr.v6.sin6_port; + memcpy(&(name->data), &socketAddr.v6.sin6_addr, sizeof(struct in6_addr)); + break; + + default: + status = EAFNOSUPPORT; + break; + } + } else { + status = errno; + if (socketTrace){ + printf("get%sname failed, errno=%d (%s)\n", + (which == 0 ? "sock" : "peer"), status, strerror(status)); + } + } + return status; +} + +/*--5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---*/ + +/* + TBC-JLC: This code currently supports only IPv4. + */ +Socket *tcpClient2(SocketAddress *socketAddress, + int timeoutInMillis, + int *returnCode, /* errnum */ + int *reasonCode){ /* errnum - JR's */ + + sa_family_t domain = 0; + + int sd = socket(AF_INET,SOCK_STREAM,IPPROTO_TCP); + if (sd < 0) { + *reasonCode = *returnCode = errno; + if (socketTrace){ + printf("Failed to create client socket, errno=%d (%s)\n", + *returnCode, strerror(*returnCode)); + } + return NULL; + } else { + if (socketTrace){ + printf("Created client socket, sd=%d\n", sd); + } + } + + int socketVector[2] = {0}; + int status; + int madeConnection = FALSE; + struct sockaddr *sockAddr = (struct sockaddr*)&(socketAddress->internalAddress.v4Address); + + { + int socketAddrSize = sizeof(SocketAddress); + if (timeoutInMillis >= 0){ + Socket tempSocket; + tempSocket.sd = sd; + + /* setSocketBlockingMode(&tempSocket, TRUE, returnCode, reasonCode); */ + status = connect(sd,sockAddr,sizeof(struct sockaddr_in)); /* v4 only */ + *returnCode = *reasonCode = errno; + if (socketTrace || (status < 0)) { + printf("connect: status=%d errno=%d (%s)\n",status,*returnCode, strerror(*returnCode)); + fflush(stdout); + } + /* EINPROGRESS is the expected return code here but we are just being careful by checking + for EWOULDBLOCK as well + */ + if (status == 0){ + *returnCode = 0; + *reasonCode = 0; + status = tcpStatus(&tempSocket, timeoutInMillis, 1, returnCode, reasonCode); +#ifdef DEBUG + printf("after tcpStatus() = %d\n",status); +#endif + if (status == SD_STATUS_TIMEOUT) { + int sd = socketVector[0]; + if (socketTrace) { + printf("Failed to connect socket, will clean up sd=%d\n", sd); + } + *returnCode = 0; + *reasonCode = 0; + status = close(sd); + if (socketTrace) { + printf("closeSocket() for time out connect status %d\n",status); + } + return NULL; + } +#ifdef DEBUG + printf("tcpStatus return and did not timeout status=%d\n",status); + fflush(stdout); +#endif + *returnCode = 0; + *reasonCode = 0; + /* setSocketBlockingMode(&tempSocket, FALSE, returnCode, reasonCode); */ + int optionLength = sizeof(int); + int optionData = 0; + *returnCode = 0; + *reasonCode = 0; + status = getSocketOption(&tempSocket, SO_ERROR, &optionLength, (char *)&optionData, returnCode, reasonCode); + if( optionData > 0 ){ + if (socketTrace){ + printf("Failed to connect socket, returnCode %d\n", optionData); + } + *returnCode = optionData; + madeConnection = FALSE; + } else { + madeConnection = TRUE; + } + } + if (socketTrace){ + printf("did we make connection (timeout case) = %d\n",madeConnection); + } + } else{ /* if timeouts are not specified */ + status = connect(sd,sockAddr,sizeof(struct sockaddr_in)); /* v4 only */ + *returnCode = *reasonCode = errno; + if (socketTrace || (status < 0)) { + printf("connect: status=%d errno=%d (%s)\n",status,*returnCode, strerror(*returnCode)); + fflush(stdout); + } + madeConnection = (status == 0 ? TRUE : FALSE); + } + if (!madeConnection){ + if (socketTrace){ + printf("Failed to connect socket, will clean up, sd was %d\n",sd); + } + status = close(sd); + if (socketTrace || (status < 0)) { + int error = (status < 0 ? errno : 0); + printf("close(%d) for failed connect: status=%d, errno=%d (%s)\n", + sd, status, error, strerror(error)); + } + return NULL; + } else{ + Socket *socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + socket->sd = sd; + sprintf(socket->debugName,"SD=%u",(unsigned) socket->sd); + socket->isServer = FALSE; + /* manual says returnCode and value only meaningful if return value = -1 */ + *returnCode = 0; + *reasonCode = 0; + socket->protocol = IPPROTO_TCP; + return socket; + } + } +} + +/* pointer to length is stupid */ +int getSocketOption(Socket *socket, int optionName, int *optionDataLength, char *optionData, + int *returnCode, int *reasonCode){ + int sd = socket->sd; + int getOption = 1; + int setOption = 2; + int setIBMOption = 3; + int level = 0x0000FFFF; + + if (socketTrace){ + printf("before get socket option optionName=0x%x dataBufferLen=%d\n",optionName,*optionDataLength); + } + int status = setsockopt(sd, + SOL_SOCKET, /* is this always right */ + optionName, + optionData, + *optionDataLength); + if (socketTrace){ + printf("after getsockopt, status=%d ret code %d reason 0x%x\n",status,*returnCode,*reasonCode); + } + if (status < 0){ + *returnCode = errno; + *reasonCode = *returnCode; + if (socketTrace) { + printf("get sockopt failed, ret code %d reason 0x%x\n", *returnCode, *reasonCode); + } + return -1; + } else { + *returnCode = 0; + *reasonCode = 0; + return status; + } +} + + +Socket *tcpClient(SocketAddress *socketAddress, + int *returnCode, /* errnum */ + int *reasonCode){ /* errnum - JR's */ + return tcpClient2(socketAddress,-1,returnCode,reasonCode); +} + +void socketFree(Socket *socket){ + if ((IPPROTO_EVENT != socket->protocol) && + (0 != socket->socketAddr)) { + safeFree((char*)socket->socketAddr,sizeof(InetAddr)); + } + safeFree((char*)socket,sizeof(Socket)); +} + +int udpReceiveFrom(Socket *socket, + SocketAddress *sourceAddress, /* will be overwritten */ + char *buffer, int bufferLength, + int *returnCode, int *reasonCode){ + int status = 0; + int sd = socket->sd; + int flags = 0; + struct sockaddr *sockAddr = (struct sockaddr*)&(sourceAddress->internalAddress.v4Address); + int socketAddressSize = sizeof(struct sockaddr); + + *returnCode = *reasonCode = 0; + + if (socketTrace > 2){ + printf("receiveFrom into buffer=0x%x bufLen=%d\n", buffer,bufferLength); + } + + int bytesReadOrError = recvfrom(sd, buffer, bufferLength, flags, sockAddr, &socketAddressSize); + + if (bytesReadOrError < 0){ + *returnCode = errno; + *reasonCode = *returnCode; + return -1; + } else { + if (socketTrace > 1){ + printf("read %d bytes\n",bytesReadOrError); + dumpbuffer(buffer, bytesReadOrError); + } + *returnCode = 0; + *reasonCode = 0; + + if (!(socket->lastDestination.port == sourceAddress->port) && + ((socket->lastDestination.family == AF_INET && + (0 == memcmp(&(socket->lastDestination.internalAddress.v4Address), + &(sourceAddress->internalAddress.v4Address), + sizeof(sourceAddress->internalAddress.v4Address)))) + || + (socket->lastDestination.family == AF_INET6 && + (0 == memcmp(&(socket->lastDestination.internalAddress.v6Address), + &(sourceAddress->internalAddress.v6Address), + sizeof(sourceAddress->internalAddress.v6Address)))))) + { + if (socket->lastDestination.family == 0) { + *reasonCode = 1; // Indicate no last destination was set for the socket + } else { + *reasonCode = 2; // Indicate source address/port did not match the last destination + } + } else { + memset(&socket->lastDestination,0,sizeof(SocketAddress)); + } + return bytesReadOrError; + } + +} + +void setSocketLastDestination(Socket *socket, SocketAddress *sourceAddress) { + socket->lastDestination = *sourceAddress; +} + +void clearSocketLastDestination(Socket *socket) { + memset(&socket->lastDestination,0,sizeof(SocketAddress)); +} + +int udpSendTo(Socket *socket, + SocketAddress *destinationAddress, + char *buffer, int desiredBytes, + int *returnCode, int *reasonCode){ + int status = 0; + int sd = socket->sd; + int flags = 0; + struct sockaddr *sockAddr = (struct sockaddr*)&(destinationAddress->internalAddress.v4Address); + int socketAddressSize = sizeof(struct sockaddr); + + *returnCode = *reasonCode = 0; + + if (socketTrace > 2){ + printf("sendTo desired=%d\n", desiredBytes); + dumpbuffer(buffer, desiredBytes); + dumpbuffer(destinationAddress, sizeof(SocketAddress)); + } + + int bytesSent = sendto(sd, buffer, desiredBytes, flags, sockAddr, socketAddressSize); + + if (bytesSent < 0){ + memset(&socket->lastDestination,0,sizeof(SocketAddress)); + if (socketTrace > 1){ + printf("error: %d\n", errno); + } + *returnCode = errno; + *reasonCode = *returnCode; + return -1; + } else { + socket->lastDestination = *destinationAddress; + if (socketTrace > 1){ + printf("sent %d bytes\n",bytesSent); + } + *returnCode = 0; + *reasonCode = 0; + return bytesSent; + } +} + +int getSocketKeepAliveMode(Socket *socket, int *returnCode, int *reasonCode){ + + int optval = 0; + int optlen = sizeof(optval); + int returnValue = getSocketOption(socket,SO_KEEPALIVE,&optlen,&optval, + returnCode,reasonCode); + if (returnValue < 0) { // If an error occurred + printf("Unable to get socket option SO_KEEPALIVE.\n"); + return returnValue; + } + returnValue = optval; + + *returnCode = 0; + *reasonCode = 0; + return returnValue; +} + +// - An enableKeepAlive value of 0 will disable keepalive processing +// - An enableKeepAlive value of >0 will enable keepalive processing with the default +// time values. +int setSocketKeepAliveMode(Socket *socket, int enableKeepAlive, + int *returnCode, int *reasonCode){ + + // Enable/disable SO_KEEPALIVE, as requested + int optval = (enableKeepAlive ? 1 : 0); + int returnValue = setSocketOption(socket,SOL_SOCKET,SO_KEEPALIVE,sizeof(optval),&optval, + returnCode,reasonCode); + if (returnValue < 0) { // If an error occurred + printf("Unable to set socket option SO_KEEPALIVE to %s.\n",(optval? "on" : "off")); + return returnValue; + } + + *returnCode = 0; + *reasonCode = 0; + return returnValue; +} + +int socketRead(Socket *socket, char *buffer, int desiredBytes, + int *returnCode, int *reasonCode){ + int sd = socket->sd; + int flags = 0; /* no special behaviors */ + +#ifdef USE_RS_SSL + if (NULL != socket->sslHandle) { /* sslHandle is RS_SSL_CONNECTION in this case */ + int bytesRead = 0; + int status = rs_ssl_read(socket->sslHandle, buffer, desiredBytes, &bytesRead); + if (0 != status) { + if (socketTrace){ + printf("socketRead: rs_ssl_read failed with status: %d\n", status); + } + *returnCode = status; + *reasonCode = *returnCode; + return -1; + } else { + *returnCode = 0; + *reasonCode = 0; + return bytesRead; + } + } + else +#endif + { + int bytesReadOrError = (int) read(sd,buffer,(size_t) desiredBytes); + if (bytesReadOrError < 0){ + *returnCode = errno; + *reasonCode = *returnCode; + return -1; + } else { + if (socketTrace > 1){ + printf("read %d bytes\n",bytesReadOrError); + } + *returnCode = 0; + *reasonCode = 0; + return bytesReadOrError; + } + } +} + +int socketWrite(Socket *socket, const char *buffer, int desiredBytes, + int *returnCode, int *reasonCode){ + int sd = socket->sd; + int flags = 0; + + if (socketTrace > 1){ + printf("socketWrite(%s, %d)\n", + socket->debugName, desiredBytes); + if (socketTrace > 2){ + dumpbuffer(buffer, desiredBytes); + } + } + +#ifdef USE_RS_SSL + if (NULL != socket->sslHandle) { /* sslHandle is RS_SSL_CONNECTION in this case */ + int bytesWritten = 0; + int status = rs_ssl_write(socket->sslHandle, buffer, desiredBytes, &bytesWritten); + if (0 != status) { + if (socketTrace) { + printf("socketWrite: rs_ssl_write failed with status: %d\n", status); + } + *returnCode = status; + *reasonCode = *returnCode; + return -1; + } else { + *returnCode = 0; + *reasonCode = 0; + return bytesWritten; + } + } + else +#endif + { + int bytesWrittenOrError = (int) write(sd,buffer,(size_t) desiredBytes); + if (bytesWrittenOrError < 0){ + *returnCode = errno; + *reasonCode = *returnCode; + return -1; + } else { + if (socketTrace > 2){ + printf("socketWrite(%s, %d)wrote %d bytes\n", + socket->debugName, desiredBytes, bytesWrittenOrError); + fflush(stdout); + } + *returnCode = 0; + *reasonCode = 0; + return bytesWrittenOrError; + } + } +} + + +static int tcpStatusInternal(Socket *socket, int timeout, int checkWrite, + int checkRead, int *returnCode, int *reasonCode){ + *reasonCode = 0; // unnecessary argument for POSIX implementation + *returnCode = 0; + + int socketDescriptor = socket->sd; + int maxSoc = socketDescriptor + 1; + + fd_set setRead; + fd_set setWrite; + fd_set setError; + + FD_ZERO(&setRead); + FD_ZERO(&setWrite); + FD_ZERO(&setError); + + struct timeval timeOut; + timeOut.tv_sec = timeout / 1000; + timeOut.tv_usec = (timeout % 1000) * 1000; + struct timeval *timeoutPtr = NULL; + + if (timeout >= 0){ + timeoutPtr = &timeOut; + } + + if (checkRead){ + FD_SET(socketDescriptor, &setRead); + } + + FD_SET(socketDescriptor, &setError); + + if (checkWrite){ + FD_SET(socketDescriptor, &setWrite); + } + + int retValue = select(maxSoc, (checkRead ? &setRead : NULL), (checkWrite ? &setWrite : NULL), &setError, timeoutPtr); + + if (retValue < 0) { + *reasonCode = *returnCode = errno; + if (socketTrace) { + printf("Select failed for sd %d, socket %s: errno=%d (%s)\n", + socketDescriptor, socket->debugName, *returnCode, strerror(*returnCode)); + } + return SD_STATUS_FAILED; + } + + if (retValue > 0){ + int sdStatus = 0; + + if (FD_ISSET(socketDescriptor, &setRead)){ + sdStatus |= SD_STATUS_RD_RDY; + } + if (FD_ISSET(socketDescriptor, &setWrite)){ + sdStatus |= SD_STATUS_WR_RDY; + } + if (FD_ISSET(socketDescriptor, &setError)){ + return SD_STATUS_ERROR; + } + + return sdStatus; + } + + return SD_STATUS_TIMEOUT; + } + +int tcpStatus(Socket *socket, + int timeout, /* in milliseconds */ + int checkWrite, /* otherwise check for read and error */ + int *returnCode, int *reasonCode){ + return tcpStatusInternal(socket,timeout,checkWrite,!checkWrite,returnCode,reasonCode); +} + +#define F_GETFL 3 +#define F_SETFL 4 + +int setSocketBlockingMode(Socket *socket, int isNonBlocking, + int *returnCode, int *reasonCode){ + int sd = socket->sd; + int status = 0; + long command = FIONBIO; + unsigned long argument = (isNonBlocking ? 1 : 0); + + status = ioctl(sd,command,&argument); + + if (status < 0){ + *reasonCode = *returnCode = errno; + if (socketTrace) { + printf("setSocketBlockingMode(%s, %s) failed: errno = %d (%s)\n", + socket->debugName, (isNonBlocking ? "NONBLOCKING (true)" : "BLOCKING (false)"), + *reasonCode, strerror(*reasonCode)); + } + return -1; + } else { + if (socketTrace){ + printf("setSocketBlockingMode(%s, %s) succeeded\n", + socket->debugName, (isNonBlocking ? "NONBLOCKING (true)" : "BLOCKING (false)")); + } + *returnCode = *reasonCode = 0; + return 0; + } +} + +int setSocketReuseAddr(Socket *socket, int *returnCode, int *reasonCode){ + int on = 1; + return setSocketOption(socket,SOL_SOCKET,SO_REUSEADDR,sizeof(int),(char*)&on,returnCode,reasonCode); +} + +Socket *udpPeer(SocketAddress *socketAddress, int *returnCode, /* errnum */ + int *reasonCode) { /* errnum - JR's */ + int status; + struct sockaddr *sockAddr = (struct sockaddr*) &(socketAddress->internalAddress.v4Address); + + int sd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + + if (sd == -1) { + *returnCode = errno; + *reasonCode = *returnCode; + if (socketTrace) { + printf("Failed to create socket, error=%d\n", *returnCode); + } + return NULL; + } + else { + int socketAddressSize = sizeof(SocketAddress); + + status = bind(sd, sockAddr, sizeof(struct sockaddr_in)); + if (status != 0) { + *returnCode = errno; + *reasonCode = *returnCode; + } + if (socketTrace) { + printf("bind status %d returnCode %x reasonCode %x\n", status, *returnCode, *reasonCode); + } + if (status != 0) { + /* polaris noted that we are leaking the socket in this failure to bind case*/ + close(sd); + return NULL; + } + else { + Socket *socket = (Socket*) safeMalloc(sizeof(Socket), "Socket"); + socket->sd = sd; + sprintf(socket->debugName, "SD=%u", (unsigned) socket->sd); + socket->isServer = FALSE; + /* manual says returnCode and value only meaningful if return value = -1 */ + *returnCode = 0; + *reasonCode = 0; + socket->protocol = IPPROTO_UDP; + + return socket; + } + } +} + +Socket *tcpServer(InetAddr *addr, /* usually NULL/0 */ + int port, + int *returnCode, + int *reasonCode){ + int status; + socklen_t sockLen = 0; + SockAddrAll socketAddr; + memset(&socketAddr, 0, sizeof(SockAddrAll)); + + sa_family_t domain = (sa_family_t) (addr ? addr->type : AF_INET); + switch(domain) { + case AF_INET: + socketAddr.v4.sin_family = domain; + socketAddr.v4.sin_port = htons((in_port_t)port); + if (addr) { + memcpy(&(socketAddr.v4.sin_addr.s_addr), &(addr->data.data4), sizeof(struct in_addr)); + } + sockLen = sizeof(struct sockaddr_in); + break; + + case AF_INET6: + socketAddr.v6.sin6_family = domain; + socketAddr.v6.sin6_port = (in_port_t)port; + if (addr) { + memcpy(&(socketAddr.v6.sin6_addr.s6_addr), &(addr->data.data6), sizeof(struct in6_addr)); + } + sockLen = sizeof(struct sockaddr_in6); + break; + + default: + if (socketTrace){ + printf("tcpserver: Unknown address family %d\n", domain); + } + *reasonCode = *returnCode = EAFNOSUPPORT; + return NULL; + } + + int sd = socket(domain,SOCK_STREAM,IPPROTO_TCP); + if (sd == -1){ + *reasonCode = *returnCode = errno; + if (socketTrace){ + printf("socket(%d,SOCK_STREAM,IPPROTO_TCP) failed, errno=%d (%s)\n", + domain, *returnCode, strerror(*returnCode)); + } + return NULL; + } + + status = setSocketReuseAddr(&sd, + returnCode, + reasonCode); + if (status != 0){ + if (socketTrace){ + printf("Failed to set SO_REUSEADDR errno=%d reason=0x%x\n", + *returnCode,*reasonCode); + } + /* treat as not fatal?*/ + } + + if (port != 0) { + /* bind to the port provided */ + status = bind(sd,(struct sockaddr*)&socketAddr,sockLen); + if (status != 0){ + *reasonCode = *returnCode = errno; + if (socketTrace){ + printf("Failed to bind server socket errno=%d (%s)\n", + *returnCode,strerror(*returnCode)); + } + close(sd); + return NULL; + } + } + + int backlogQueueLength = 100; + status = listen(sd,backlogQueueLength); + if (status != 0){ + *returnCode = errno; + *reasonCode = *returnCode; + if (socketTrace){ + printf("Failed to start server socket listen, errno=%d (%s)\n", + *returnCode,strerror(*returnCode)); + } + close(sd); + return NULL; + } + + /* Yea, team - we now have a listening socket. */ + + *reasonCode = *returnCode = 0; + Socket *socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + memset(socket, 0, sizeof(Socket)); + socket->sd = sd; + socket->isServer = TRUE; + socket->protocol = IPPROTO_TCP; + + InetAddr name; + if (0 == getEndPointName(sd, &name, 0)) { + /* fill in the address information */ + socket->socketAddr = (InetAddr *)safeMalloc(sizeof(InetAddr),"InetAddr"); + memcpy(socket->socketAddr, &name, sizeof(InetAddr)); + port = (int) name.port; + } + sprintf(socket->debugName,"SD=%d,port=%u",socket->sd, (unsigned) port); + + if (socketTrace) { + printf("Immediately before return from tcpServer, socket contains:\n"); + dumpbuffer((char*)socket, sizeof(Socket)); + } + + return socket; +} + +Socket *tcpServer2(InetAddr *addr, + int port, + int tlsFlags, + int *returnCode, + int *reasonCode) +{ + Socket* socket = NULL; + socket = tcpServer(addr, port, returnCode, reasonCode); + if (socket != NULL) { + socket->tlsFlags = tlsFlags; + } + + return socket; +} +/*--5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---*/ + + +Socket *socketAccept(Socket *serverSocket, int *returnCode, int *reasonCode){ + + SockAddrAll newSocketAddr; + socklen_t newSocketAddrSize = sizeof(SockAddrAll); + int newSD = accept(serverSocket->sd,(struct sockaddr *)&newSocketAddr,&newSocketAddrSize); + if (newSD < 0){ + *reasonCode = *returnCode = errno; + if (socketTrace){ + printf("Failed to accept new socket errno=%d (%s)\n", + *returnCode,strerror(*returnCode)); + } + return NULL; + } + + Socket *socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + memset(socket,0,sizeof(Socket)); + socket->sd = newSD; + socket->isServer = 0; + socket->protocol = IPPROTO_TCP; + socket->tlsFlags = serverSocket->tlsFlags; + sprintf(socket->debugName,"SD=%d",socket->sd); + return socket; +} + + +#define DOT 1 +#define FIRST 2 +#define SECOND 3 +#define THIRD 4 + +int isV4Numeric(char *chars, int *addr){ + int i,len = strlen(chars); + int state = DOT; + int dotCount = 0; + int value; + int addressValue = 0; + for (i=0; i= '0' && ch <= '9'){ + if (dotCount == 0 && ch == '0') + return 0; + else + state = FIRST; + } else { + return 0; + } + break; + case FIRST: + if (ch >= '0' && ch <= '9'){ + state = SECOND; + } else if (ch == '.'){ + state = DOT; + value = (chars[i-1] - '0'); + addressValue = (addressValue << 8) | value; + dotCount++; + } else{ + return 0; + } + break; + case SECOND: + if (ch >= '0' && ch <= '9'){ + state = THIRD; + } else if (ch == '.'){ + state = DOT; + value = ((10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + addressValue = (addressValue << 8) | value; + dotCount++; + } else{ + return 0; + } + break; + case THIRD: + if (ch >= '0' && ch <= '9'){ + return 0; + } else if (ch == '.'){ + state = DOT; + value = ((100 * (chars[i-3] - '0')) + + (10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + addressValue = (addressValue << 8) | value; + if (value < 256) + dotCount++; + else + return 0; + } else{ + return 0; + } + break; + } + } + if (dotCount == 3){ + switch (state){ + case FIRST: + value = (chars[i-1] - '0'); + break; + case SECOND: + value = ((10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + break; + case THIRD: + value = ((100 * (chars[i-3] - '0')) + + (10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + break; + } + addressValue = (addressValue << 8) | value; + *addr = htonl(addressValue); + return (value < 256); + } else{ + return 0; + } +} + +/* the return value is in network byte order */ + +int getV4HostByName(char *hostName){ + int status = 0; + int returnCode = 0; + int reasonCode = 0; + int len = strlen(hostName); + struct hostent *hostEntPtr; + + hostEntPtr = gethostbyname(hostName); + if (socketTrace){ + printf("hostent addr = %x\n",*((int*)hostEntPtr)); + } + if (hostEntPtr){ + int i = 0; + int numericAddress = 0; + if (socketTrace){ + dumpbuffer((char*)hostEntPtr,20); + } + if (socketTrace) { + printf("hostent->length=%d name=%s\n", hostEntPtr->h_length, hostEntPtr->h_name); + fflush(stdout); + } + do { + char *hostAddr = hostEntPtr->h_addr_list[i]; + if (hostAddr == NULL){ + break; + } + numericAddress = *((int*)hostAddr); /*very IP-v4 here */ + if (socketTrace) { + printf("hostAddr is at 0x%x\n", hostAddr); + printf("numeric=0x%x == %d.%d.%d.%d\n", numericAddress, (int) hostAddr[0], (int) hostAddr[1], (int) hostAddr[2], + (int) hostAddr[3]); + } + i++; + } while (TRUE); + return numericAddress; + } else{ + returnCode = errno; + reasonCode = returnCode; + if (socketTrace) { + printf("getHostName V4 failure, returnCode %d reason code %d\n", returnCode, reasonCode); + } + return 0; + } +} + + +InetAddr *getAddressByName(char *addressString){ + int numericAddress = 0x7F123456; + + if (!isV4Numeric(addressString,&numericAddress)) { /* updates numericAddress if addressString was a V4 IP address */ + numericAddress = getV4HostByName(addressString); + } + + if ((numericAddress != 0x7F123456) && + (0 != numericAddress)) + { + InetAddr *addr = (InetAddr*)safeMalloc(sizeof(InetAddr),"Inet Address"); + memset(addr,0,sizeof(InetAddr)); + addr->type = AF_INET; + addr->port = 0; + memcpy(&(addr->data.data4),&numericAddress,4); +#ifdef DEBUG + printf("InetAddr structure: AF_INET=0x%x\n",AF_INET); + dumpbuffer((char*)addr,sizeof(InetAddr)); + fflush(stdout); +#endif + return addr; + } else{ + return NULL; + } +} + +int getLocalHostName(char* inout_hostname, + unsigned int* inout_hostname_len, + int *returnCode, int *reasonCode) +{ + int result = 0; + /* Parameter check code copied from z/OS implementation. + Seems odd that this one routine would bother... */ + if ((NULL == inout_hostname) || + (NULL == inout_hostname_len) || + (NULL == returnCode) || (NULL == reasonCode)) + { + result = -1; + } else { + memset(inout_hostname, 0, (size_t)(*inout_hostname_len)); + result = gethostname(inout_hostname, *inout_hostname_len); + if (result < 0) { + *reasonCode = *returnCode = errno; + if (socketTrace) { + printf("gethostname(*,%d) failed, errno=%d (%s)\n", + *returnCode, strerror(*returnCode)); + } + } else { + /* gethostname doesn't provide a length, it just null-terminates + its output, if there is space. But the caller wants the length, + so figure it out. Note that, idiotically, the length includes + the trailing null (that's how z/OS does it). */ + if (inout_hostname[(*inout_hostname_len)-1] == 0) { + /* it fit, including the terminator */ + *inout_hostname_len = 1+(int)strlen(inout_hostname); + } /* else - it filled the available space. */ + } + } + return result; +} + +int socketClose(Socket *socket, int *returnCode, int *reasonCode){ + if (socketTrace) { + printf("socketClose(%s): isServer=%s,proto=%d: ", + socket->debugName, (socket->isServer ? "true" : "false"), (int) socket->protocol); + } + + int status = 0; + *reasonCode = *returnCode = 0; + +#ifdef USE_RS_SSL + if ((0 == socket->isServer) && (NULL != socket->sslHandle)) { + /* sslHandle is RS_SSL_CONNECTION in this case */ + /* this call does a C stdlib shutdown/close internally */ + status = rs_ssl_releaseConnection(socket->sslHandle); + return status; + } +#endif + + status = close(socket->sd); + + if (0 == status) { + if (socketTrace) { + printf(" OK\n"); + } + } else { + *reasonCode = *returnCode = errno; + if (socketTrace) { + printf("failed: %d (%s)\n", errno, strerror(errno)); + } + } + return status; +} + +int getSocketName(Socket *socket, SocketAddress *socketAddress) +{ + InetAddr name; + int status = getEndPointName(socket->sd, &name, 0); + if (0 == status) { + setSocketAddr(socketAddress, &name, 0); + } + return status; +} + +int getSocketName2(Socket *socket, SocketAddress *socketAddress) +{ + InetAddr name; + int status = getEndPointName(socket->sd, &name, 1); + if (0 == status) { + setSocketAddr(socketAddress, &name, 0); + } + return status; +} + +/***************************************************************/ + +#ifdef __ZOWE_OS_AIX +Socket** makeEventSockets() +{ + int eventFD[2]; + int sts = 0; + Socket** rtnSockets = NULL; + + do { + sts = pipe(eventFD); + if (sts < 0) { + if (socketTrace) { + printf("makeEventSockets failed in pipe command. errno = %d (%s)\n", errno, strerror(errno)); + break; + } + } + + rtnSockets = (Socket**)safeMalloc(2*sizeof(Socket*), "evensockets array"); + rtnSockets[0] = (Socket*)safeMalloc(sizeof(Socket),"Socket1"); + memset(rtnSockets[0], 0, sizeof(Socket)); + rtnSockets[0]->sd = eventFD[0]; + rtnSockets[0]->protocol = IPPROTO_EVENT; + sprintf(rtnSockets[0]->debugName,"EVENT0=%d",eventFD[0]); + if (socketTrace) { + printf("makeEventSocket() returned %s\n", rtnSockets[0]->debugName); + } + rtnSockets[1] = (Socket*)safeMalloc(sizeof(Socket),"Socket2"); + memset(rtnSockets[1], 0, sizeof(Socket)); + rtnSockets[1]->sd = eventFD[1]; + rtnSockets[1]->protocol = IPPROTO_EVENT; + sprintf(rtnSockets[1]->debugName,"EVENT1=%d",eventFD[1]); + if (socketTrace) { + printf("makeEventSocket() returned %s\n", rtnSockets[1]->debugName); + } + } while(0); + + return rtnSockets; +} +#else +Socket* makeEventSocket() +{ + int eventFD; + if (0 > (eventFD = eventfd(0, 0))) { + if (socketTrace) { + int error = errno; + printf("makeEventSocket() failed creating eventfd. errno=%d (%s)\n", + error, strerror(error)); + } + return NULL; + } + if (eventFD >= FD_SETSIZE) { + if (socketTrace) { + printf("makeEventSocket() failed; read side FD (%d) exceeds FD_SETSIZE (%d)\n", + eventFD, (int) FD_SETSIZE); + } + return NULL; + } + Socket* socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + memset(socket, 0, sizeof(Socket)); + socket->sd = eventFD; + socket->protocol = IPPROTO_EVENT; + sprintf(socket->debugName,"EVENT=%d",eventFD); + if (socketTrace) { + printf("makeEventSocket() returned %s\n", socket->debugName); + } + return socket; +} +#endif + +int postEventSocket(Socket* socket) +{ + if (socket->protocol != IPPROTO_EVENT) { + if (socketTrace) { + printf("postEventSocket called for non-event socket %s\n", + socket->debugName); + return EINVAL; + } + } + + uint64_t bump = 1; + if (socketTrace > 1) { + printf("postEventSocket called for %s\n", socket->debugName); + } + ssize_t written = write(socket->sd, &bump, sizeof(uint64_t)); + if (written != sizeof(uint64_t)) { + if (socketTrace) { + int error = errno; + printf("postEventSocket write for %s failed. errno=%d (%s)\n", + socket->debugName, error, strerror(error)); + return error; + } + } + return 0; +} + +uint64_t waitForEventSocket(Socket* socket) +{ + if (socket->protocol != IPPROTO_EVENT) { + if (socketTrace) { + printf("waitForEventSocket called for non-event socket %s\n", + socket->debugName); + return 0; + } + } + + uint64_t count = 0; + ssize_t bytesRead = read(socket->sd, &count, sizeof(uint64_t)); + if (bytesRead != sizeof(uint64_t)) { + if (socketTrace) { + int error = errno; + printf("waitForEventSocket read for %s failed. errno=%d (%s)\n", + socket->debugName, error, strerror(error)); + } + count = 0; + } else { + if (socketTrace) { + printf("waitForEventSocket called for %s; returned %" PRIu64 "\n", + socket->debugName, count); + } + } + return count; +} + +void clearEventSocket(Socket* socket) +{ + /* Cheesy */ + if (0 == postEventSocket(socket)) { + uint64_t count = waitForEventSocket(socket); + if (socketTrace) { + if (count > 0) { + /* success */ + printf("clearEventSocket called for %s; there were %" PRIu64 " pending notifications\n", + socket->debugName, count-1); + } else { + printf("clearEventSocket called for %s; wait failed.\n", socket->debugName); + } + } + } else { + if (socketTrace) { + printf("clearEventSocket called for %s; post failed.\n", socket->debugName); + } + } +} + +SocketSet *makeSocketSet(int highestAllowedSD){ + if ((highestAllowedSD < 1) || (highestAllowedSD >= FD_SETSIZE)) { + if (socketTrace){ + printf("makeSocketSet(%d) invalid\n", highestAllowedSD); + } + return NULL; + } + SocketSet *set = (SocketSet*)safeMalloc(sizeof(SocketSet),"SocketSet"); + memset(set,0,sizeof(SocketSet)); + size_t socketArraySize = (size_t) ((highestAllowedSD+1)*sizeof(Socket*)); + set->sockets = (Socket**)safeMalloc(socketArraySize, "SocketArray"); + memset(set->sockets, 0, socketArraySize); + set->highestAllowedSD = highestAllowedSD; + FD_ZERO(&(set->allSDs)); + return set; +} + +void freeSocketSet(SocketSet *set){ + safeFree((char*) set, sizeof(SocketSet)); +} + +void setSocketSetIdleTimeLimit(SocketSet *set, int idleSeconds) { + set->idleTimeLimit = (uint64_t)idleSeconds; +} + +int socketSetAdd(SocketSet *set, Socket *socket){ + int sd = socket->sd; + + if ((sd > set->highestAllowedSD) || (sd < 0)){ + if (socketTrace){ + printf("socketSetAdd for SD=%d (%s) out of range (> %d)\n",sd, socket->debugName, set->highestAllowedSD); + } + return 12; + } + + if (FD_ISSET(sd, &(set->allSDs)) || (set->sockets[sd] != NULL)) { + if (socketTrace){ + printf("socketSetAdd for SD=%d (%s) failed - socket already in set\n",sd, socket->debugName); + } + return 12; + } + + if (NULL == set->sockets[sd]) + { + if (socketTrace > 1){ + printf("socketSetAdd for SD=%d (%s) succeeded\n",sd, socket->debugName); + } + FD_SET(sd, &(set->allSDs)); + set->sockets[sd] = socket; + ++(set->socketCount); + } + + return 0; +} + +int socketSetRemove(SocketSet *set, Socket *socket){ + int sd = socket->sd; + + if ((sd > set->highestAllowedSD) || (sd < 0)){ + if (socketTrace){ + printf("socketSetRemove for SD=%d (%s) out of range (> %d)\n",sd, socket->debugName, set->highestAllowedSD); + } + return 12; + } + if (!FD_ISSET(sd, &(set->allSDs)) || (set->sockets[sd] == NULL)) { + if (socketTrace){ + printf("socketSetRemove for SD=%d (%s) failed - socket not in set\n",sd, socket->debugName); + } + return 12; + } + + if (NULL != set->sockets[sd]) + { + if (socketTrace > 1){ + printf("socketSetRemove for SD=%d (%s) succeeded\n",sd, socket->debugName); + } + FD_CLR(sd, &(set->allSDs)); + set->sockets[sd] = NULL; + --(set->socketCount); + } + + return 0; +} + +static +void printSocketSetStatus(SocketSet *set, fd_set* fdset, const char* label) +{ + printf("%s: maxSD=%d\n", label, set->highestAllowedSD); + for (int i = 0; i < set->highestAllowedSD+1; ++i) { + if (FD_ISSET(i, fdset)) { + Socket* s = set->sockets[i]; + printf(" [%02d]: %s\n", i, (s ? s->debugName : "INVALID")); + } + } +} + +int extendedSelect(SocketSet *set, + int timeout, /* in milliseconds */ + int checkWrite, int checkRead, + int *returnCode, int *reasonCode){ + *reasonCode = *returnCode = 0; + struct timeval theTimeout = {0,0}; + struct timeval* theTimeoutPtr; + + memcpy(&(set->scratchReadMask), &(set->allSDs), sizeof(fd_set)); + memcpy(&(set->scratchWriteMask), &(set->allSDs), sizeof(fd_set)); + memcpy(&(set->scratchErrorMask), &(set->allSDs), sizeof(fd_set)); + + if (timeout >= 0) { + theTimeout.tv_sec = timeout/1000; + theTimeout.tv_usec = 1000*(timeout%1000); + theTimeoutPtr = &theTimeout; + } else { + theTimeoutPtr = (struct timeval*) NULL; + } + + int maxSoc = set->highestAllowedSD+1; + + int status = select(maxSoc, + (checkRead ? &(set->scratchReadMask) : (fd_set*) 0), + (checkWrite ? &(set->scratchWriteMask) : (fd_set*) 0), + &(set->scratchErrorMask), + theTimeoutPtr); + *reasonCode = *returnCode = (status < 0) ? errno : 0; + + if (socketTrace){ + printf("extendedSelect(maxSoc=%d, timeout={%d,%d}, write=%s, read=%s) " + "returns %d with errno=%d (%s)\n", + maxSoc, timeout/1000, 1000*(timeout%1000), + (checkWrite ? "true" : "false"), (checkRead ? "true" : "false"), + status, *returnCode, strerror(*returnCode)); + if (socketTrace > 1) { + if (socketTrace > 1) { + printSocketSetStatus(set, &(set->allSDs), "allSDs"); + } + if (checkRead) { + printSocketSetStatus(set, &(set->scratchReadMask), "read"); + } + if (checkWrite) { + printSocketSetStatus(set, &(set->scratchWriteMask), "write"); + } + if (socketTrace > 2) { + printSocketSetStatus(set, &(set->scratchErrorMask), "error"); + } + } + } + return status; +} + +static +int setSocketOptionEx(Socket *socket, + int level, const char* levelDesc, + int optionName, const char* optionDesc, + int optionDataLength, char *optionData, + int *returnCode, int *reasonCode) +{ + int status = setsockopt(socket->sd, level, optionName, optionData, (socklen_t) optionDataLength); + if (status < 0){ + *reasonCode = *returnCode = errno; + if (socketTrace){ + printf("setsockopt(%s, level=%d%s, option=%d%s, len=%d) failed, errno=%d (%s)\n", + socket->debugName, level, levelDesc, optionName, optionDesc, optionDataLength, *returnCode, strerror(*returnCode)); + } + } else { + *reasonCode = *returnCode = 0; + if (socketTrace){ + printf("setsockopt(%s, level=%d%s, option=%d%s, len=%d) OK\n", + socket->debugName, level, levelDesc, optionName, optionDesc, optionDataLength); + } + } + return status; +} + +int setSocketOption(Socket *socket, int level, int optionName, int optionDataLength, char *optionData, + int *returnCode, int *reasonCode) +{ + return setSocketOptionEx(socket, level, "", optionName, "", optionDataLength, optionData, returnCode, reasonCode); +} + +void setSocketIdleTimeoutMode(Socket *socket, int enableIdleTimeout) { + // Socket idle timeout support is currently only supported for TCPIP + if (socket->protocol == IPPROTO_TCP) { + if (enableIdleTimeout == 0) { + socket->idleTimeoutFlags = IDLE_TIMEOUT_DISABLED; + } else { + socket->idleTimeoutFlags = IDLE_TIMEOUT_ENABLED; + if (sxSocketIsReady(socket)) { // TLS is not requested or handshake is complete + socket->readyTime = socket->createTime; // Use create for ready time + } + } + } +} + +int setSocketNoDelay(Socket *socket, int noDelay, int *returnCode, int *reasonCode){ + return setSocketOptionEx(socket,IPPROTO_TCP, " (IPPROTO_TCP)",TCP_NODELAY, " (TCP_NODELAY)", sizeof(int),(char*)&noDelay,returnCode,reasonCode); +} + +int setSocketWriteBufferSize(Socket *socket, int bufferSize, int *returnCode, int *reasonCode){ + return setSocketOptionEx(socket,SOL_SOCKET, " (SOL_SOCKET)",SO_SNDBUF, " (SO_SNDBUF)", sizeof(int),(char*)&bufferSize,returnCode,reasonCode); +} + +int setSocketReadBufferSize(Socket *socket, int bufferSize, int *returnCode, int *reasonCode){ + return setSocketOptionEx(socket,SOL_SOCKET, " (SOL_SOCKET)",SO_RCVBUF, " (SO_RCVBUF)", sizeof(int),(char*)&bufferSize,returnCode,reasonCode); +} + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/recovery.c b/c/recovery.c index 06710d299..18d5b2b46 100644 --- a/c/recovery.c +++ b/c/recovery.c @@ -2004,9 +2004,9 @@ void recoveryGetABENDCode(SDWA *sdwa, int *completionCode, int *reasonCode) { if (flag & 0x04) { char *sdwadata = (char*)sdwa; - SDWAPTRS *sdwaptrs = (SDWAPTRS *)(sdwa->sdwaxpad); + SDWAPTRS *sdwaptrs = (SDWAPTRS *)INT2PTR(sdwa->sdwaxpad); if (sdwaptrs != NULL) { - char *sdwarc1 = (char *)sdwaptrs->sdwasrvp; + char *sdwarc1 = (char *)INT2PTR(sdwaptrs->sdwasrvp); if (sdwarc1 != NULL) { rsn = *(int * __ptr32)(sdwarc1 + 44); } diff --git a/c/scheduling.c b/c/scheduling.c index e27bc275f..7e8ea9c8e 100644 --- a/c/scheduling.c +++ b/c/scheduling.c @@ -484,6 +484,7 @@ int zosWait(void *ecb, int clearFirst){ : : "r"(ecbAsInteger) : "r15"); + return 0; } int zosWaitList(void *ecbList, int numberToWait){ @@ -499,6 +500,7 @@ int zosWaitList(void *ecbList, int numberToWait){ : : "r"(ecbListAsInteger), "r"(numberToWait) : "r15"); + return 0; } int zosPost(void *ecb, int completionCode){ @@ -507,6 +509,7 @@ int zosPost(void *ecb, int completionCode){ : : "r"(ecbAsInteger), "r"(completionCode) : "r15"); + return 0; } #define schedulingDSECTs SKDDSECT diff --git a/c/timeutls.c b/c/timeutls.c index ef26a79c5..a0f22a700 100644 --- a/c/timeutls.c +++ b/c/timeutls.c @@ -26,7 +26,13 @@ #include #include #include + +#ifndef _MSC_VER /* WINDOWS */ #include +#else +#include +#endif /* WINDOWS */ + #endif #include "zowetypes.h" @@ -231,10 +237,10 @@ int stckToTimestamp(int64 stck, char *output) STCKCONVPlist * pPlist = (STCKCONVPlist *) safeMalloc31(sizeof(STCKCONVPlist), "STCKCONVPlist"); int * value = (int *)(&stck); - int * __ptr32 CVTPTR = (int* __ptr32) ((int* __ptr32) 16); - int * __ptr32 CVT = (int * __ptr32) CVTPTR[0]; - int * __ptr32 SFT = (int * __ptr32) (CVT[772/4]); - int serviceRoutine = SFT[304/4]; + int *mem = (int*)0; + void *cvt = INT2PTR(mem[0x10/4]); + int *sft = (int*)INT2PTR(((int*)cvt)[772/4]); + int serviceRoutine = sft[304/4]; pPlist->dateFlags = 0x03; @@ -260,10 +266,10 @@ int stckToTimestamp(int64 stck, char *output) int timestampToSTCK(char *todText, int64 *stck, int64 offset){ int *value = (int*)stck; - int *CVTPTR = (int*)((int*)16); - int *CVT = (int*)CVTPTR[0]; - int *SFT = (int*)(CVT[772/4]); - int serviceRoutine = SFT[352/4]; + int *mem = (int*)0; + void *cvt = INT2PTR(mem[0x10/4]); + int *sft = (int*)INT2PTR(((int*)cvt)[772/4]); + int serviceRoutine = sft[352/4]; int res = 0; CONVTODPlist plist; @@ -292,7 +298,7 @@ int timeZoneDifferenceFor(int64 theTime) return -1; } -#elif defined(__ZOWE_OS_LINUX) || defined(__ZOWE_OS_AIX) +#elif defined(__ZOWE_OS_LINUX) || defined(__ZOWE_OS_AIX) || defined(__ZOWE_OS_WINDOWS) #include @@ -353,13 +359,78 @@ uint64 sleazyToSeconds(const struct tm* time) return result; } +/* a platform sensitive wrapper */ +static void platformGMTime(time_t *time, + struct tm *tm){ +#ifdef __ZOWE_OS_WINDOWS + memcpy(tm,gmtime(time),sizeof(struct tm)); +#else + gmtime_r(time,tm); +#endif +} + +/* A missing function in Windows - + * This implementation came from A MS VisStudio Support pag + */ + +#ifdef __ZOWE_OS_WINDOWS + +#define DELTA_EPOCH_IN_MICROSECS 11644473600000000Ui64 + +struct timezone +{ + int tz_minuteswest; /* minutes W of Greenwich */ + int tz_dsttime; /* type of dst correction */ +}; + +static int gettimeofday(struct timeval *tv, struct timezone *tz) +{ + FILETIME ft; + unsigned __int64 tmpres = 0; + static int tzflag; + + if (NULL != tv) + { + GetSystemTimeAsFileTime(&ft); + + tmpres |= ft.dwHighDateTime; + tmpres <<= 32; + tmpres |= ft.dwLowDateTime; + + /*converting file time to unix epoch*/ + tmpres -= DELTA_EPOCH_IN_MICROSECS; + tmpres /= 10; /*convert into microseconds*/ + tv->tv_sec = (long)(tmpres / 1000000UL); + tv->tv_usec = (long)(tmpres % 1000000UL); + } + + if (NULL != tz) + { + if (!tzflag) + { + _tzset(); + tzflag++; + } + tz->tz_minuteswest = _timezone / 60; + tz->tz_dsttime = _daylight; + } + + return 0; +} + +#endif /* WINDOWS gettimeofday implementation */ + static int tzsetHasBeenCalled = 0; static inline void tzsetWrapper() { if (!tzsetHasBeenCalled) { +#ifdef __ZOWE_OS_WINDOWS + _tzset(); +#else tzset(); +#endif tzsetHasBeenCalled = 1; } } @@ -371,7 +442,7 @@ int timeZoneDifferenceInternal(time_t base, struct tm* local) int result = 0; tzsetWrapper(); struct tm utc; - gmtime_r(&base, &utc); + platformGMTime(&base, &utc); uint64 utcBase = sleazyToSeconds(&utc); uint64 localBase = sleazyToSeconds(local); if (utcBase > localBase) { @@ -384,6 +455,7 @@ int timeZoneDifferenceInternal(time_t base, struct tm* local) return result; } + /* Compute the difference between UTC and local time for the specified Unix time (seconds since the Unix Epoch). @@ -392,8 +464,19 @@ int timeZoneDifferenceFor(int64 theTime) { /* There has to be a better way... */ time_t base = (time_t) theTime; - struct tm local; + struct tm local = { 0 }; +#ifdef __ZOWE_OS_WINDOWS + struct tm *tempTM = localtime(&base); + if (tempTM){ + memcpy(&local,tempTM,sizeof(struct tm)); + } else { + /* Nothing good to do here, and return value will be v.bad. + Garbage In, Garbage Out! + */ + } +#else localtime_r(&base, &local); +#endif return timeZoneDifferenceInternal(base, &local); } @@ -463,7 +546,7 @@ int stckToTimestamp(int64 stck, char *output) stckToTimeval(&timeval, stck); time_t time = (time_t)timeval.tv_sec; struct tm split; - gmtime_r(&time, &split); + platformGMTime(&time, &split); memset(output, 0, TIMESTAMP_LENGTH); writePackedDecimal(output,0,10,split.tm_hour); @@ -473,6 +556,7 @@ int stckToTimestamp(int64 stck, char *output) writePackedDecimal(output,8,1000,split.tm_year+1900); writePackedDecimal(output,10,10,split.tm_mon+1); writePackedDecimal(output,11,10,split.tm_mday); + return 0; } int timestampToSTCK(char *todText, int64 *stck, int64 offset) @@ -516,7 +600,6 @@ int timestampToSTCK(char *todText, int64 *stck, int64 offset) *stck = (int64)rawStck; return 0; } - #else #error OS unknown #endif @@ -764,9 +847,9 @@ void unixToTimestamp(uint64 unixTime, char *output) { convertIntToString(output + 10, 2, minutes); convertIntToString(output + 12, 2, seconds); -#elif defined(__ZOWE_OS_LINUX) || defined(__ZOWE_OS_AIX) +#elif defined(__ZOWE_OS_LINUX) || defined(__ZOWE_OS_AIX) || defined(__ZOWE_OS_WINDOWS) struct tm split; - gmtime_r((time_t*)&unixTime, &split); + platformGMTime((time_t*)&unixTime, &split); memset(output, 0x0, 16); // Copy date @@ -801,18 +884,24 @@ int snprintLocalTime(char *buffer, int length, int tzSource) { gmtime_r(&tt, result); #elif defined(__ZOWE_OS_LINUX) || defined(__ZOWE_OS_AIX) localtime_r(&tt, result); +#elif defined(__ZOWE_OS_WINDOWS) + memcpy(result,localtime(&tt),sizeof(struct tm)); #else #error Unknown OS #endif } else if (tzSource == TZ_FROM_TZ) { +#ifndef __ZOWE_OS_WINDOWS localtime_r(&tt, result); +#else + memcpy(result,localtime(&tt),sizeof(struct tm)); +#endif } else { return 0; } return snprintf(buffer, length, "%04d-%02d-%02d-%02d-%02d-%02d.%06d", 1900 + result->tm_year, result->tm_mon + 1, result->tm_mday, result->tm_hour, result->tm_min, result->tm_sec, - tv->tv_usec); + (int)tv->tv_usec); #endif } diff --git a/c/utils.c b/c/utils.c index 72d53c665..f727af5a1 100644 --- a/c/utils.c +++ b/c/utils.c @@ -23,7 +23,10 @@ #else #include #include -#include +#include +#ifndef _MSC_VER /* Windows always has to be the oddball */ +#include +#endif #include #endif @@ -708,7 +711,7 @@ void dumpbufferA(const char *buffer, int length){ #ifdef METTLE printf("the buffer is empty at %x\n",buffer); #else - fprintf((FILE*)traceOut,"the buffer is empty at %x\n",buffer); + fprintf((FILE*)traceOut,"the buffer is empty at %p\n",buffer); #endif while (index <= last_index){ @@ -1051,8 +1054,8 @@ static char binToEB64[] ={0xc1,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xd1,0xd2 char *encodeBase64(ShortLivedHeap *slh, const char buf[], int size, int *resultSize, int useEbcdic){ int allocSize = BASE64_ENCODE_SIZE(size)+1; /* +1 for null term */ - char *result = NULL; - if (result = (slh ? SLHAlloc(slh,allocSize) : safeMalloc31(allocSize,"BASE64"))) { + char *result = (slh ? SLHAlloc(slh,allocSize) : safeMalloc31(allocSize,"BASE64")); + if (result){ encodeBase64NoAlloc(buf, size, result, resultSize, useEbcdic); return result; } else{ @@ -1078,8 +1081,8 @@ void encodeBase64NoAlloc(const char buf[], int size, char result[], int *resultS int byte1 = data[inCursor++] & 0xff; int byte2 = data[inCursor++] & 0xff; *resPtr++ = (char)(translation[byte0 >> 2]); - *resPtr++ = (char)(translation[(byte0 << 4) & 0x3f | (byte1 >> 4)]); - *resPtr++ = (char)(translation[(byte1 << 2) & 0x3f | (byte2 >> 6)]); + *resPtr++ = (char)(translation[((byte0 << 4) & 0x3f) | (byte1 >> 4)]); + *resPtr++ = (char)(translation[((byte1 << 2) & 0x3f) | (byte2 >> 6)]); *resPtr++ = (char)(translation[byte2 & 0x3f]); } @@ -1093,7 +1096,7 @@ void encodeBase64NoAlloc(const char buf[], int size, char result[], int *resultS } else{ int byte1 = data[inCursor++] & 0xff; - *resPtr++ = (char)(translation[(byte0 << 4) & 0x3f | (byte1 >> 4)]); + *resPtr++ = (char)(translation[((byte0 << 4) & 0x3f) | (byte1 >> 4)]); *resPtr++ = (char)(translation[(byte1 << 2) & 0x3f]); *resPtr++ = equalsChar; } @@ -1297,19 +1300,19 @@ int base32Decode (int alphabet, if (ch2 == -2) { break; } - output [outputIndex++] = (char) ((ch1 << 6) | (ch2 << 1) | (ch3 >> 4) & 0xff); + output [outputIndex++] = (char) ((ch1 << 6) | (ch2 << 1) | ((ch3 >> 4) & 0xff)); if (ch4 == -2) { break; } - output [outputIndex++] = (char) ((ch3 << 4) | (ch4 >> 1) & 0xff); + output [outputIndex++] = (char) ((ch3 << 4) | ((ch4 >> 1) & 0xff)); if (ch5 == -2) { break; } - output [outputIndex++] = (char) ((ch4 << 7) | (ch5 << 2) | (ch6 >> 3) & 0xff); + output [outputIndex++] = (char) ((ch4 << 7) | (ch5 << 2) | ((ch6 >> 3) & 0xff)); if (ch7 == -2) { break; } - output [outputIndex++] = (char) ((ch6 << 5) | (ch7) & 0xff); + output [outputIndex++] = (char) ((ch6 << 5) | ((ch7) & 0xff)); } *outputLength = outputIndex; return 0; @@ -1356,12 +1359,12 @@ int base32Encode (int alphabet, byte3 = *input++ & 0xff; byte4 = *input++ & 0xff; output [outputIndex++] = (char)(encodeTable [(byte0 >> 3) & 0x1f]); - output [outputIndex++] = (char)(encodeTable [(byte0 << 2) & 0x1f | (byte1 >> 6)]); + output [outputIndex++] = (char)(encodeTable [((byte0 << 2) & 0x1f) | (byte1 >> 6)]); output [outputIndex++] = (char)(encodeTable [(byte1 >> 1) & 0x1f]); - output [outputIndex++] = (char)(encodeTable [(byte1 << 4) & 0x1f | (byte2 >> 4)]); - output [outputIndex++] = (char)(encodeTable [(byte2 << 1) & 0x1f | (byte3 >> 7)]); + output [outputIndex++] = (char)(encodeTable [((byte1 << 4) & 0x1f) | (byte2 >> 4)]); + output [outputIndex++] = (char)(encodeTable [((byte2 << 1) & 0x1f) | (byte3 >> 7)]); output [outputIndex++] = (char)(encodeTable [(byte3 >> 2) & 0x1f]); - output [outputIndex++] = (char)(encodeTable [(byte3 << 3) & 0x1f | (byte4 >> 5)]); + output [outputIndex++] = (char)(encodeTable [((byte3 << 3) & 0x1f) | (byte4 >> 5)]); output [outputIndex++] = (char)(encodeTable [(byte4 & 0x1f)]); } if (numCharsLeft) { @@ -1379,7 +1382,7 @@ int base32Encode (int alphabet, } else if (numCharsLeft == 2) { int byte1 = *input++; - output [outputIndex++] = (char)(encodeTable [(byte0 << 2) & 0x1f | (byte1 >> 6)]); + output [outputIndex++] = (char)(encodeTable [((byte0 << 2) & 0x1f) | (byte1 >> 6)]); output [outputIndex++] = (char)(encodeTable [(byte1 >> 1) & 0x1f]); output [outputIndex++] = (char)(encodeTable [(byte1 << 4) & 0x1f]); output [outputIndex++] = equals; @@ -1390,9 +1393,9 @@ int base32Encode (int alphabet, else if (numCharsLeft == 3) { int byte1 = *input++; int byte2 = *input++; - output [outputIndex++] = (char)(encodeTable [(byte0 << 2) & 0x1f | (byte1 >> 6)]); + output [outputIndex++] = (char)(encodeTable [((byte0 << 2) & 0x1f) | (byte1 >> 6)]); output [outputIndex++] = (char)(encodeTable [(byte1 >> 1) & 0x1f]); - output [outputIndex++] = (char)(encodeTable [(byte1 << 4) & 0x1f | (byte2 >> 4)]); + output [outputIndex++] = (char)(encodeTable [((byte1 << 4) & 0x1f) | (byte2 >> 4)]); output [outputIndex++] = (char)(encodeTable [(byte2 << 1) & 0x1f]); output [outputIndex++] = equals; output [outputIndex++] = equals; @@ -1402,10 +1405,10 @@ int base32Encode (int alphabet, int byte1 = *input++; int byte2 = *input++; int byte3 = *input++; - output [outputIndex++] = (char)(encodeTable [(byte0 << 2) & 0x1f | (byte1 >> 6)]); + output [outputIndex++] = (char)(encodeTable [((byte0 << 2) & 0x1f) | (byte1 >> 6)]); output [outputIndex++] = (char)(encodeTable [(byte1 >> 1) & 0x1f]); - output [outputIndex++] = (char)(encodeTable [(byte1 << 4) & 0x1f | (byte2 >> 4)]); - output [outputIndex++] = (char)(encodeTable [(byte2 << 1) & 0x1f | (byte3 >> 7)]); + output [outputIndex++] = (char)(encodeTable [((byte1 << 4) & 0x1f) | (byte2 >> 4)]); + output [outputIndex++] = (char)(encodeTable [((byte2 << 1) & 0x1f) | (byte3 >> 7)]); output [outputIndex++] = (char)(encodeTable [(byte3 >> 2) & 0x1f]); output [outputIndex++] = (char)(encodeTable [(byte3 << 3) & 0x1f]); output [outputIndex++] = equals; @@ -1452,7 +1455,7 @@ static void reportSLHFailure(ShortLivedHeap *slh, int size){ while (chain){ char *data = chain->data-4; int size = *((int*)data); - printf(" segment %d at 0x%x, size = 0x%x\n",i,data,size+4); + printf(" segment %d at 0x%p, size = 0x%x\n",i,data,size+4); chain = chain->next; i++; } @@ -1477,13 +1480,15 @@ char *SLHAlloc(ShortLivedHeap *slh, int size){ size += (8-rem); } char *data; - /* printf("slh=%d\n",slh);fflush(stdout); - printf("SLHAlloc me=0x%x size=%d bc=%d\n",slh,size,slh->blockCount);fflush(stdout); */ + /* + printf("slh=0x%p\n",slh);fflush(stdout); + printf("SLHAlloc me=0x%p size=%d bc=%d\n",slh,size,slh->blockCount);fflush(stdout); + */ int remainingHeapBytes = (slh->blockSize * (slh->maxBlocks - slh->blockCount)); if (size > remainingHeapBytes){ printf("SLH at 0x%p cannot allocate above block size %d > %d mxbl %d bkct %d bksz %d\n", slh,size,remainingHeapBytes,slh->maxBlocks,slh->blockCount,slh->blockSize); - + fflush(stdout); char *mem = (char*)0; mem[0] = 13; return NULL; @@ -1736,7 +1741,7 @@ CharStream *makeBufferCharStream(char *buffer, int len, int trace){ s->positionMethod = getBufferPosition; s->closeMethod = NULL; if (trace){ - printf("mbcs out 0x%x, '%.*s'\n",s,len,s); + printf("mbcs out 0x%p, '%.*s'\n",s,len,buffer); } return s; } @@ -1980,6 +1985,10 @@ int decimalToOctal(int decimal) { #define debugPrintf(formatString, ...) #endif +static int incrementPlaceValues(int *placeValues, + int lim, int digits); + + int matchWithWildcards(char *pattern, int patternLen, char *s, int len, int flags){ @@ -2211,17 +2220,6 @@ void trimRight(char *str, int length) { } } -#ifdef __ZOWE_OS_ZOS -int isLowerCasePasswordAllowed(){ - RCVT* rcvt = getCVT()->cvtrac; - return (RCVTFLG3_BIT_RCVTPLC & (rcvt->rcvtflg3)); /* if lower-case pw allowed */ -} -#else -int isLowerCasePasswordAllowed(){ - return TRUE; -} -#endif - bool isPassPhrase(const char *password) { return strlen(password) > 8; } diff --git a/c/winskt.c b/c/winskt.c new file mode 100644 index 000000000..d7894627d --- /dev/null +++ b/c/winskt.c @@ -0,0 +1,1136 @@ + + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#include +#include +#include +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "openprims.h" +#include "collections.h" +#include "bpxnet.h" +#include "unixfile.h" + +/* + this pragma tells the compiler/linker to link in the right, modern WinSock stuff + + */ + +#pragma comment(lib, "Ws2_32.lib") + +/* + https://msdn.microsoft.com/en-us/library/windows/desktop/ms741394(v=vs.85).aspx + https://tangentsoft.net/wskfaq/newbie.html + + fcntl = ioctlsocket + + IOCTL / FCNTL + https://msdn.microsoft.com/en-us/library/windows/desktop/ms740126(v=vs.85).aspx + + + Various C language run-time systems use the IOCTLs for purposes + unrelated to Windows Sockets. As a consequence, the ioctlsocket + function and the WSAIoctl function were defined to handle socket + functions that were performed by IOCTL and fcntl in the Berkeley + Software Distribution. + */ + +int tcpInit(char *uniqueName){ + /* do nothing for now */ + return 0; +} + +static int socketTrace = TRUE; + +SocketAddress *makeSocketAddr(InetAddr *addr, + unsigned short port){ + SocketAddress *address = (SocketAddress*)safeMalloc(sizeof(SocketAddress),"SocketAddress"); + memset(address,0,sizeof(SocketAddress)); + if (socketTrace){ + printf("socket address at 0x%p\n",address); + } + address->family = AF_INET; + address->port = ((port & 0xff)<<8)|((port&0xff00)>>8); + if (addr){ + address->v4Address = addr->data.data4; + } else{ + memset(&(address->v4Address),0,sizeof(struct in_addr)); + } + if (socketTrace){ + printf("about to return socket address at 0x%p\n",address); + } + return address; +} + +void freeSocketAddr(SocketAddress *address){ + safeFree((char*)address,sizeof(SocketAddress)); +} + +/*--5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---*/ + +#define SO_ERROR 0x1007 + +Socket *tcpClient2(SocketAddress *socketAddress, + int timeoutInMillis, + int *returnCode, /* errnum */ + int *reasonCode){ /* errnum - JR's */ + int socketVector[2]; + int *reasonCodePtr; + int status; + int madeConnection = FALSE; + SOCKET windowsSocket = socket(AF_INET,SOCK_STREAM,IPPROTO_TCP); + struct sockaddr *windowsSocketAddress = (struct sockaddr *)socketAddress; + + printf("socket = 0x%llx\n",windowsSocket); + if (windowsSocket == INVALID_SOCKET){ + if (socketTrace){ + printf("Failed to create socket\n"); + } + return NULL; + } else{ + int socketAddrSize = sizeof(SocketAddress); + if (timeoutInMillis >= 0){ + Socket tempSocket; + tempSocket.windowsSocket = windowsSocket; + + /* setSocketBlockingMode(&tempSocket, TRUE, returnCode, reasonCode); */ + printf("windowsSocket:\n"); + dumpbuffer((char*)windowsSocketAddress,20); + status = connect(windowsSocket,windowsSocketAddress,sizeof(struct sockaddr_in)); + *returnCode = WSAGetLastError(); + printf("socket connect status=%d errno=%d\n",status,*returnCode); + fflush(stdout); + if (socketTrace) { + printf("connect() status %d\n",status); + } + /* EINPROGRESS is the expected return code here but we are just being careful by checking + for EWOULDBLOCK as well + + WINDOWS NOTE: status needs to 0, opposite of z/OS. But maybe the zOS code is getting + lucky. The windows return value for connect() seems to have a clearer + interpretation here. + */ + if (status == 0){ + *returnCode = 0; + *reasonCode = 0; + status = tcpStatus(&tempSocket, timeoutInMillis, 1, returnCode, reasonCode); + printf("after tcpStatus() = %d\n",status); + if (status == SD_STATUS_TIMEOUT) { + int sd = socketVector[0]; + if (socketTrace) { + printf("Failed to connect socket, will clean up sd=%d\n", sd); + } + *returnCode = 0; + *reasonCode = 0; + status = closesocket(windowsSocket); + if (socketTrace) { + printf("closeSocket() for time out connect status %d\n",status); + } + return NULL; + } + printf("tcpStatus return and did not timeout status=%d\n",status); + fflush(stdout); + *returnCode = 0; + *reasonCode = 0; + /* setSocketBlockingMode(&tempSocket, FALSE, returnCode, reasonCode); */ + int optionLength = sizeof(int); + int optionData = 0; + *returnCode = 0; + *reasonCode = 0; + status = getSocketOption(&tempSocket, SO_ERROR, &optionLength, (char *)&optionData, returnCode, reasonCode); + if( optionData > 0 ){ + if (socketTrace){ + printf("Failed to connect socket, returnCode %d\n", optionData); + } + *returnCode = optionData; + madeConnection = FALSE; + } else { + madeConnection = TRUE; + } + } + if (socketTrace){ + printf("did we make connection (timeout case) = %d\n",madeConnection); + } + } else{ /* if timeouts are not specified */ + status = connect(windowsSocket,windowsSocketAddress,14); + if (socketTrace){ + printf("BPXCON status %d returnCode %x reasonCode %x\n", + status,*returnCode,*reasonCode); + } + madeConnection = (status == 0 ? TRUE : FALSE); + } + if (!madeConnection){ + if (socketTrace){ + printf("Failed to connect socket, will clean up, SOCKET at 0x%llx\n",windowsSocket); + } + *returnCode = 0; + *reasonCode = 0; + status = closesocket(windowsSocket); + if (socketTrace){ + printf("closesocket() for failed connect status %d\n",status); + } + return NULL; + } else{ + Socket *socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + socket->windowsSocket = windowsSocket; + sprintf(socket->debugName,"HNDL=0X%llx",socket->windowsSocket); + socket->isServer = FALSE; + /* manual says returnCode and value only meaningful if return value = -1 */ + *returnCode = 0; + *reasonCode = 0; + socket->protocol = IPPROTO_TCP; + return socket; + } + } +} + +/* pointer to length is stupid */ +int getSocketOption(Socket *socket, int optionName, int *optionDataLength, char *optionData, + int *returnCode, int *reasonCode){ + SOCKET windowsSocket = socket->windowsSocket; + int getOption = 1; + int setOption = 2; + int setIBMOption = 3; + int level = 0x0000FFFF; + + if (socketTrace){ + printf("before get socket option optionName=0x%x dataBufferLen=%d\n",optionName,*optionDataLength); + } + int status = setsockopt(windowsSocket, + SOL_SOCKET, /* is this always right */ + optionName, + optionData, + *optionDataLength); + if (socketTrace){ + printf("after getsockopt, status=%d ret code %d reason 0x%x\n",status,*returnCode,*reasonCode); + } + if (status < 0){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + printf("get sockopt failed, ret code %d reason 0x%x\n",*returnCode,*reasonCode); + return -1; + } else { + *returnCode = 0; + *reasonCode = 0; + return status; + } +} + + +Socket *tcpClient(SocketAddress *socketAddress, + int *returnCode, /* errnum */ + int *reasonCode){ /* errnum - JR's */ + return tcpClient2(socketAddress,-1,returnCode,reasonCode); +} + +void socketFree(Socket *socket){ + safeFree((char*)socket,sizeof(Socket)); +} + +static +int setSocketOptionEx(Socket *socket, + int level, const char* levelDesc, + int optionName, const char* optionDesc, + int optionDataLength, char *optionData, + int *returnCode, int *reasonCode) +{ + SOCKET windowsSocket = socket->windowsSocket; + int status = setsockopt(windowsSocket, level, optionName, optionData, (socklen_t) optionDataLength); + if (status < 0){ + *reasonCode = *returnCode = errno; + if (socketTrace){ + printf("setsockopt(%s, level=%d%s, option=%d%s, len=%d) failed, errno=%d (%s)\n", + socket->debugName, level, levelDesc, optionName, optionDesc, optionDataLength, *returnCode, strerror(*returnCode)); + } + } else { + *reasonCode = *returnCode = 0; + if (socketTrace){ + printf("setsockopt(%s, level=%d%s, option=%d%s, len=%d) OK\n", + socket->debugName, level, levelDesc, optionName, optionDesc, optionDataLength); + } + } + return status; +} + +int setSocketOption(Socket *socket, int level, int optionName, int optionDataLength, char *optionData, + int *returnCode, int *reasonCode) +{ + return setSocketOptionEx(socket, level, "", optionName, "", optionDataLength, optionData, returnCode, reasonCode); +} + + +int getSocketKeepAliveMode(Socket *socket, int *returnCode, int *reasonCode){ + + int optval = 0; + int optlen = sizeof(optval); + int returnValue = getSocketOption(socket,SO_KEEPALIVE,&optlen,(char*)&optval, + returnCode,reasonCode); + if (returnValue < 0) { // If an error occurred + printf("Unable to get socket option SO_KEEPALIVE.\n"); + return returnValue; + } + returnValue = optval; + + *returnCode = 0; + *reasonCode = 0; + return returnValue; +} + +// - An enableKeepAlive value of 0 will disable keepalive processing +// - An enableKeepAlive value of >0 will enable keepalive processing with the default +// time values. +int setSocketKeepAliveMode(Socket *socket, int enableKeepAlive, + int *returnCode, int *reasonCode){ + + // Enable/disable SO_KEEPALIVE, as requested + int optval = (enableKeepAlive ? 1 : 0); + int returnValue = setSocketOption(socket,SOL_SOCKET,SO_KEEPALIVE,sizeof(optval),(char*)&optval, + returnCode,reasonCode); + if (returnValue < 0) { // If an error occurred + printf("Unable to set socket option SO_KEEPALIVE to %s.\n",(optval? "on" : "off")); + return returnValue; + } + + *returnCode = 0; + *reasonCode = 0; + return returnValue; +} + +int socketRead(Socket *socket, char *buffer, int desiredBytes, + int *returnCode, int *reasonCode){ + SOCKET windowsSocket = socket->windowsSocket; + int flags = 0; /* no special behaviors */ + + int bytesReadOrError = recv(windowsSocket,buffer,desiredBytes,flags); + if (bytesReadOrError < 0){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + return -1; + } else { + if (socketTrace){ + printf("read %d bytes\n",bytesReadOrError); + } + *returnCode = 0; + *reasonCode = 0; + return bytesReadOrError; + } +} + +int socketWrite(Socket *socket, const char *buffer, int desiredBytes, + int *returnCode, int *reasonCode){ + SOCKET windowsSocket = socket->windowsSocket; + int flags = 0; + + if (socketTrace){ + printf("writing desired=%d retCode=%d reasonCode=%d\n", + desiredBytes,*returnCode,*reasonCode); + dumpbuffer(buffer, desiredBytes); + } + + int bytesWrittenOrError = send(windowsSocket,buffer,desiredBytes,0); + if (bytesWrittenOrError < 0){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + printf("write failed, socketHandle=0x%llx,desired write len %d buffer at 0x%p, ret code %d reason 0x%x\n", + windowsSocket,desiredBytes,buffer,*returnCode,*reasonCode); + return -1; + } else { + if (socketTrace){ + printf("wrote %d bytes\n",bytesWrittenOrError);fflush(stdout); + } + *returnCode = 0; + *reasonCode = 0; + return bytesWrittenOrError; + } +} + +static void setupFDSets(fd_set *readFDs, + fd_set *writeFDs, + fd_set *exceptionFDs, + SOCKET windowsSocket, + int checkWrite, + int checkRead){ + FD_ZERO(readFDs); + FD_ZERO(writeFDs); + FD_ZERO(exceptionFDs); + + if (checkWrite){ + FD_SET(windowsSocket,writeFDs); + } + if (checkRead){ + FD_SET(windowsSocket,readFDs); + FD_SET(windowsSocket,exceptionFDs); + } + +} + +static int tcpStatusInternal(Socket *socket, + int timeout, /* in milliseconds */ + int checkWrite, int checkRead, + int *returnCode, int *reasonCode){ + /* what does it MEAN?? */ + int status = 0; + SOCKET windowsSocket = socket->windowsSocket; + fd_set readFDs, writeFDs, exceptionFDs; + struct timeval timeValue; + const struct timeval *timeValuePtr; + + if (timeout >= 0) + { + timeValue.tv_sec = timeout/1000; /* millis to seconds */ + timeValue.tv_usec = (timeout-(timeout/1000)*1000)*1000; /* micros */ + timeValuePtr = &timeValue; + } else { + timeValuePtr = NULL; + } + + setupFDSets(&readFDs,&writeFDs,&exceptionFDs,windowsSocket,checkWrite,checkRead); + status = select(0,&readFDs,&writeFDs,&exceptionFDs,timeValuePtr); + + if (socketTrace){ + printf("BPXSEL status %d returnCode %x reasonCode %x\n", + status,*returnCode,*reasonCode); + } + if (status > 0){ + int sdStatus = 0; + + if (FD_ISSET(windowsSocket,&readFDs)){ + sdStatus |= SD_STATUS_RD_RDY; + } + if (FD_ISSET(windowsSocket,&writeFDs)){ + sdStatus |= SD_STATUS_WR_RDY; + } + return sdStatus; + } else if (status == 0){ /* timeout */ + return SD_STATUS_TIMEOUT; + } else if (FD_ISSET(windowsSocket,&exceptionFDs)){ + return SD_STATUS_ERROR; + } else{ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + return SD_STATUS_FAILED; /* look in errno */ + } +} + +int tcpStatus(Socket *socket, + int timeout, /* in milliseconds */ + int checkWrite, /* otherwise check for read and error */ + int *returnCode, int *reasonCode){ + return tcpStatusInternal(socket,timeout,checkWrite,!checkWrite,returnCode,reasonCode); +} + +#define O_NONBLOCK 0x04 +#define F_GETFL 3 +#define F_SETFL 4 + +/* Note: Windows "ioctl" known as ioctlsocket is not as complete + as it is in posix/unix. See WSAIoctl for richer functionality */ + +int setSocketBlockingMode(Socket *socket, int isNonBlocking, + int *returnCode, int *reasonCode){ + SOCKET windowsSocket = socket->windowsSocket; + int status = 0; + long command = FIONBIO; + unsigned long argument = (isNonBlocking ? 1 : 0); + + status = ioctlsocket(windowsSocket,command,&argument); + + if (status < 0){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + + printf("ioctlsocket(1) failed, ret code %d reason 0x%x\n",*returnCode,*reasonCode); + return -1; + } else { + if (socketTrace){ + printf("ioctlsocket succeeded, nonblocking=%d \n",isNonBlocking); + } + *returnCode = 0; + *reasonCode = 0; + return 0; + } +} + +Socket *udpPeer(SocketAddress *socketAddress, + int *returnCode, /* errnum */ + int *reasonCode){ /* errnum - JR's */ + int status; + struct sockaddr *windowsSocketAddress = (struct sockaddr *)socketAddress; + + SOCKET windowsSocket = socket(AF_INET,SOCK_DGRAM,IPPROTO_UDP); + + if (windowsSocket == INVALID_SOCKET){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + if (socketTrace){ + printf("Failed to create socket, error=%d\n",*returnCode); + } + return NULL; + } else{ + int socketAddressSize = sizeof(SocketAddress); + status = bind(windowsSocket,windowsSocketAddress,sizeof(struct sockaddr_in)); + if (status != 0){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + } + if (socketTrace) { + printf("BPXBND status %d returnCode %x reasonCode %x\n", + status, *returnCode, *reasonCode); + } + if (status != 0){ + return NULL; + } else{ + Socket *socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + socket->windowsSocket = windowsSocket; + sprintf(socket->debugName,"HNDL=0X%llx",socket->windowsSocket); + socket->isServer = FALSE; + /* manual says returnCode and value only meaningful if return value = -1 */ + *returnCode = 0; + *reasonCode = 0; + socket->protocol = IPPROTO_UDP; + return socket; + } + } +} + +Socket *tcpServer(InetAddr *addr, /* usually NULL/0 */ + int port, + int *returnCode, + int *reasonCode){ + int socketVector[2]; + int status; + + SOCKET windowsSocket = socket(AF_INET,SOCK_STREAM,IPPROTO_TCP); + if (windowsSocket == INVALID_SOCKET){ + if (socketTrace){ + printf("Failed to create socket\n"); + } + return NULL; + } else{ + int sd = socketVector[0]; + int socketAddressSize = sizeof(SocketAddress); + int inetAddressSize = sizeof(InetAddr); + SocketAddress *socketAddress = makeSocketAddr(addr,(unsigned short)port); + struct sockaddr *windowsSocketAddress = (struct sockaddr *)socketAddress; + InetAddr inetAddr; + memset(&inetAddr,0,sizeof(InetAddr)); + inetAddr.type = AF_INET; + inetAddr.port = (unsigned short)port; + memset(&inetAddr.data.data4,0,4); + if (socketTrace){ + printf("server SocketAddress = %d\n",sd); + dumpbuffer((char*)socketAddress,sizeof(SocketAddress)); + dumpbuffer((char*)&inetAddr,sizeof(InetAddr)); + } + status = bind(windowsSocket,windowsSocketAddress,sizeof(struct sockaddr_in)); + if (status != 0){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + } + if (status != 0){ + if (socketTrace){ + printf("Failed to bind server socket errno=%d reason=0x%x\n", + *returnCode,*reasonCode); + } + safeFree((char*)socketAddress,sizeof(SocketAddress)); + return NULL; + } else{ + int backlogQueueLength = 100; + status = listen(windowsSocket,backlogQueueLength); + + if (status != 0){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + if (socketTrace){ + printf("Failed to start server socket listen errno=%d reason=0x%x\n", + *returnCode,*reasonCode); + } + return NULL; + } else { + Socket *socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + /* manual says returnCode and value only meaningful if return value = -1 */ + *returnCode = 0; + *reasonCode = 0; + + socket->windowsSocket = windowsSocket; + sprintf(socket->debugName,"HNDL=0X%llx",socket->windowsSocket); + socket->isServer = TRUE; + socket->protocol = IPPROTO_TCP; + if (socketTrace) { + printf("Immediately before return from tcpServer, socket contains:\n"); + dumpbuffer((char*)socket, sizeof(Socket)); + } + return socket; + } + } + } + +} + +/*--5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---*/ + + +Socket *socketAccept(Socket *serverSocket, int *returnCode, int *reasonCode){ + SocketAddress *newSocketAddress + = (SocketAddress*)safeMalloc(sizeof(SocketAddress),"Socket Address"); + struct sockaddr *windowsSocketAddress = (struct sockaddr *)newSocketAddress; + SOCKET windowsSocket = serverSocket->windowsSocket; + int status; + int newSocketAddressSize = sizeof(struct sockaddr_in); + + printf("newSocketAddressSize=%d sizeof(SocketAddress)=%lld\n",newSocketAddressSize,sizeof(SocketAddress)); + fflush(stdout); + + SOCKET newWindowsSocket = accept(windowsSocket,windowsSocketAddress,&newSocketAddressSize); + if (newWindowsSocket == INVALID_SOCKET){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + if (socketTrace){ + printf("Failed to accept new socket errno=%d reasonCode=0x%x\n", + *returnCode,*reasonCode); + } + return NULL; + } else{ + Socket *socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + memset(socket,0,sizeof(Socket)); + socket->windowsSocket = newWindowsSocket; + socket->isServer = 0; + socket->protocol = IPPROTO_TCP; + return socket; + } +} + + +#define DOT 1 +#define FIRST 2 +#define SECOND 3 +#define THIRD 4 + +int isV4Numeric(char *chars, int *addr){ + int i,len = strlen(chars); + int state = DOT; + int dotCount = 0; + int value; + int addressValue = 0; + for (i=0; i= '0' && ch <= '9'){ + if (dotCount == 0 && ch == '0') + return 0; + else + state = FIRST; + } else { + return 0; + } + break; + case FIRST: + if (ch >= '0' && ch <= '9'){ + state = SECOND; + } else if (ch == '.'){ + state = DOT; + value = (chars[i-1] - '0'); + addressValue = (addressValue << 8) | value; + dotCount++; + } else{ + return 0; + } + break; + case SECOND: + if (ch >= '0' && ch <= '9'){ + state = THIRD; + } else if (ch == '.'){ + state = DOT; + value = ((10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + addressValue = (addressValue << 8) | value; + dotCount++; + } else{ + return 0; + } + break; + case THIRD: + if (ch >= '0' && ch <= '9'){ + return 0; + } else if (ch == '.'){ + state = DOT; + value = ((100 * (chars[i-3] - '0')) + + (10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + addressValue = (addressValue << 8) | value; + if (value < 256) + dotCount++; + else + return 0; + } else{ + return 0; + } + break; + } + } + if (dotCount == 3){ + switch (state){ + case FIRST: + value = (chars[i-1] - '0'); + break; + case SECOND: + value = ((10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + break; + case THIRD: + value = ((100 * (chars[i-3] - '0')) + + (10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + break; + } + addressValue = (addressValue << 8) | value; + *addr = addressValue; + return (value < 256); + } else{ + return 0; + } +} + +/* the return value is in network byte order */ + +int getV4HostByName(char *hostName){ + int status = 0; + int returnCode = 0; + int reasonCode = 0; + int len = strlen(hostName); + struct hostent *hostEntPtr; + + hostEntPtr = gethostbyname(hostName); + if (socketTrace){ + printf("hostent addr = %x\n",*((int*)hostEntPtr)); + } + if (hostEntPtr){ + int i = 0; + int numericAddress = 0; + if (socketTrace){ + dumpbuffer((char*)hostEntPtr,20); + } + printf("hostent->length=%d name=%s\n",hostEntPtr->h_length,hostEntPtr->h_name); + fflush(stdout); + do { + char *hostAddr = hostEntPtr->h_addr_list[i]; + if (hostAddr == NULL){ + break; + } + printf("hostAddr = 0x%p %s\n",hostAddr,hostAddr); + numericAddress = *((int*)hostAddr); /*very IP-v4 here */ + printf("numeric=0x%x\n",numericAddress); + i++; + } while (TRUE); + return numericAddress; + } else{ + returnCode = WSAGetLastError(); + reasonCode = returnCode; + printf("getHostName V4 failure, returnCode %d reason code %d\n",returnCode,reasonCode); + return 0; + } +} + + +InetAddr *getAddressByName(char *addressString){ + int numericAddress = 0x7F123456; + + if (!isV4Numeric(addressString,&numericAddress)){ + numericAddress = getV4HostByName(addressString); + /* printf("Host names is DNS NAME (or crap), %x\n",numericAddress); */ + } else{ + /* printf("Host name is numeric %x\n",numericAddress); */ + } + + if (numericAddress != 0x7F123456){ + InetAddr *addr = (InetAddr*)safeMalloc(sizeof(InetAddr),"Inet Address"); + memset(addr,0,sizeof(InetAddr)); + addr->type = AF_INET; + addr->port = 0; + memcpy(&(addr->data.data4),&numericAddress,4); + printf("InetAddr structure: AF_INET=0x%x\n",AF_INET); + dumpbuffer((char*)addr,sizeof(InetAddr)); + fflush(stdout); + return addr; + } else{ + return NULL; + } +} + +int socketInit(char *uniqueName){ + WORD wVersionRequested; + WSADATA wsaData; + int err; + + /* Use the MAKEWORD(lowbyte, highbyte) macro declared in Windef.h */ + wVersionRequested = MAKEWORD(2, 0); + + err = WSAStartup(wVersionRequested, &wsaData); + if (err != 0) { + /* Tell the user that we could not find a usable */ + /* Winsock DLL. */ + printf("WSAStartup failed with error: %d\n", err); + return 1; + } + return 0; +} + +int socketClose(Socket *socket, int *returnCode, int *reasonCode){ + SOCKET windowsSocket = socket->windowsSocket; + int status; + + status = closesocket(windowsSocket); + if (status == 0){ + return 0; + } else { + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + return -1; + } +} + +/*** File/Directory stuff ***/ + +static int fileTrace = FALSE; + +int setFileTrace(int toWhat) { + int was = fileTrace; + fileTrace = toWhat; + return was; +} + +UnixFile *fileOpen(const char *filename, int options, int mode, int bufferSize, int *returnCode, int *reasonCode){ + int returnValue = 0; + int *reasonCodePtr; + int status; + int len = strlen(filename); + char *unixMode = "r"; + + /* r Open text file for reading. The stream is positioned at the + beginning of the file. + + r+ Open for reading and writing. The stream is positioned at the + beginning of the file. + + w Truncate file to zero length or create text file for writing. + The stream is positioned at the beginning of the file. + + w+ Open for reading and writing. The file is created if it does + not exist, otherwise it is truncated. The stream is + positioned at the beginning of the file. + + a Open for appending (writing at end of file). The file is + created if it does not exist. The stream is positioned at the + end of the file. + + a+ Open for reading and appending (writing at end of file). The + file is created if it does not exist. The initial file + position for reading is at the beginning of the file, but + output is always appended to the end of the file. + */ + if (options & FILE_OPTION_APPEND){ + unixMode = (options & FILE_OPTION_READ_ONLY) ? "a+" : "a"; + } else if (options & FILE_OPTION_WRITE_ONLY){ + if (options & FILE_OPTION_READ_ONLY){ + unixMode = (options & FILE_OPTION_TRUNCATE) ? "w+" : "r+"; + } else { + if (options & FILE_OPTION_TRUNCATE){ + unixMode = "w"; + } else { + *returnCode = -1; + return NULL; /* parameter error */ + } + } + } else if (options & FILE_OPTION_READ_ONLY){ + unixMode = "r"; + } else { + *returnCode = -1; + return NULL; /* parameter error */ + } + + FILE *internalFile = fopen(filename,unixMode); + if (internalFile == NULL){ + *returnCode = -1; + return NULL; /* open failed */ + } + UnixFile *file = (UnixFile*)safeMalloc(sizeof(UnixFile),"OMVS File"); + memset(file,0,sizeof(UnixFile)); + file->internalFile = internalFile; + file->fd = _fileno(internalFile); + file->pathname = safeMalloc(strlen(filename)+1,"Unix File Name"); + strcpy(file->pathname, filename); + file->isDirectory = FALSE; + if (bufferSize > 0){ + file->buffer = safeMalloc(bufferSize,"OMVS File Buffer"); + file->bufferSize = bufferSize; + file->bufferPos = bufferSize; + file->bufferFill = bufferSize; + } + return file; +} + +int fileRead(UnixFile *file, char *buffer, int desiredBytes, + int *returnCode, int *reasonCode){ + FILE *internalFile = file->internalFile; + + size_t bytesRead = fread(buffer,1,desiredBytes,internalFile); + *returnCode = 0; + if (bytesRead < desiredBytes){ + file->eofKnown = TRUE; + if (ferror(internalFile)){ + *returnCode = errno; + *reasonCode = errno; + } + } + return (int)bytesRead; +} + +int fileWrite(UnixFile *file, const char *buffer, int desiredBytes, + int *returnCode, int *reasonCode) { + FILE *internalFile = file->internalFile; + + size_t bytesWritten = fwrite(buffer,1,desiredBytes,internalFile); + *returnCode = 0; + if (bytesWritten < desiredBytes){ + file->eofKnown = TRUE; + if (ferror(internalFile)){ + *returnCode = errno; + *reasonCode = errno; + } + } + return (int)bytesWritten; +} + +int fileGetChar(UnixFile *file, int *returnCode, int *reasonCode){ + if (file->bufferSize == 0){ + *returnCode = 8; + *reasonCode = 0xBFF; + return -1; + } else if (file->bufferPos < file->bufferFill){ + return (int)(file->buffer[file->bufferPos++])&0xFF; + } else if (file->eofKnown){ + return -1; + } else{ + /* go after next buffer and pray */ + int bytesRead = fileRead(file,file->buffer,file->bufferSize,returnCode,reasonCode); + if (bytesRead >= 0) { + if (bytesRead < file->bufferSize) { /* out of data after this read */ + file->eofKnown = TRUE; + } + file->bufferFill = bytesRead; + file->bufferPos = 1; + return file->buffer[0]&0xFF; + } else{ + return -1; + } + } +} + +int fileInfo(const char *filename, FileInfo *info, int *returnCode, int *reasonCode){ + int nameLength = strlen(filename); + + int status = _stat(filename,info); + if (status != 0){ + *returnCode = errno; + *reasonCode = errno; + } + return status; +} + +int fileEOF(const UnixFile *file){ + return ((file->bufferPos >= file->bufferFill) && file->eofKnown); +} + +int fileClose(UnixFile *file, int *returnCode, int *reasonCode){ + FILE *internalFile = file->internalFile; + + int status = fclose(internalFile); + if (status != 0){ + *returnCode = errno; + *reasonCode = errno; + } + + if (file->pathname != NULL) { + safeFree(file->pathname, strlen(file->pathname) + 1); + file->pathname = NULL; + } + if (file->buffer != NULL) { + safeFree(file->buffer, file->bufferSize); + file->buffer = NULL; + } + safeFree((char *)file, sizeof(UnixFile)); + file = NULL; + + return status; +} + +int fileInfoIsDirectory(const FileInfo *info){ + return info->st_mode & _S_IFDIR; +} + +int64 fileInfoSize(const FileInfo *info){ + return info->st_size; +} + +int fileInfoCCSID(const FileInfo *info){ + printf("***WARNING*** guessing about File's code page in Windows\n"); + return CP_UTF8; +} + +int fileInfoUnixCreationTime(const FileInfo *info){ + return info->st_ctime; +} + +UnixFile *directoryOpen(const char *directoryName, int *returnCode, int *reasonCode){ + printf("d0.0\n"); + fflush(stdout); + + int nameLength = strlen(directoryName); + WIN32_FIND_DATA findData; + int directoryNameBufferSize = nameLength + 10; + char *directoryNameBuffer = safeMalloc(directoryNameBufferSize,"Windows Directory Name"); + HANDLE hFind = INVALID_HANDLE_VALUE; + + int len = sprintf(directoryNameBuffer,"%s\\*",directoryName); + hFind = FindFirstFile(directoryNameBuffer, &findData); + + safeFree(directoryNameBuffer, directoryNameBufferSize); + directoryNameBuffer = NULL; + + printf("dO.1 hFind=0x%p\n",hFind); + fflush(stdout); + if (hFind == INVALID_HANDLE_VALUE){ + *returnCode = GetLastError(); + *reasonCode = *returnCode; + return NULL; + } else { + UnixFile *directory = (UnixFile*)safeMalloc(sizeof(UnixFile),"Windows Directory"); + memset(directory,0,sizeof(UnixFile)); + directory->internalFile = NULL; + directory->fd = 0; + directory->hFind = hFind; + memcpy(&(directory->findData),&findData,sizeof(WIN32_FIND_DATA)); + directory->pathname = safeMalloc(strlen(directoryName)+1,"Unix File Name"); + strcpy(directory->pathname, directoryName); + directory->isDirectory = TRUE; + directory->hasMoreEntries = TRUE; + + *returnCode = 0; + *reasonCode = 0; + return directory; + } +} + +int directoryRead(UnixFile *directory, char *entryBuffer, int entryBufferLength, int *returnCode, int *reasonCode){ + HANDLE hFind = directory->hFind; + WIN32_FIND_DATA *findData = &(directory->findData); + + if (directory->hasMoreEntries){ + int filenameLength = strlen(findData->cFileName); + DirectoryEntry *entry = (DirectoryEntry*)entryBuffer; + entry->entryLength = filenameLength+2; + entry->nameLength = filenameLength; + memcpy(entry->name,findData->cFileName,filenameLength); + if (FindNextFile(hFind, findData)){ + directory->hasMoreEntries = TRUE; + } else { + /* find loop is done */ + directory->hasMoreEntries = FALSE; + } + return 1; + } else { + return 0; + } +} + +int directoryClose(UnixFile *directory, int *returnCode, int *reasonCode){ + HANDLE hFind = directory->hFind; + + FindClose(hFind); + if (directory->pathname != NULL) { + safeFree(directory->pathname, strlen(directory->pathname)+1); + directory->pathname = NULL; + } + safeFree((char*)directory,sizeof(UnixFile)); + *returnCode = 0; + *reasonCode = 0; + return 0; +} + +SocketSet *makeSocketSet(int highestAllowedSD){ + SocketSet *set = (SocketSet*)safeMalloc(sizeof(SocketSet),"SocketSet"); + memset(set,0,sizeof(SocketSet)); + + set->highestAllowedSD = highestAllowedSD; + + int socketArrayLength = highestAllowedSD; + set->sockets = (Socket**)safeMalloc(socketArrayLength*sizeof(Socket*),"SocketSetSocketArray"); + memset(set->sockets,0,socketArrayLength*sizeof(Socket*)); + + set->revisionNumber = 0; + return set; +} + +void freeSocketSet(SocketSet *set) { + int socketArrayLength = set->highestAllowedSD; + safeFree((char*)(void*)set->sockets, socketArrayLength * sizeof(Socket*)); + safeFree((char*)set, sizeof(SocketSet)); + return; +} + +int socketSetAdd(SocketSet *set, Socket *socket){ + printf("winskt.socketSetAdd set=0x%p socket=0x%p\n",set,socket); + fflush(stdout); + SOCKET windowsSocket = socket->windowsSocket; + if (socketTrace) + { + printf ("Adding socket, SOCKET HANDLE=0x%llx\n", windowsSocket); + } + + if (set->socketCount >= set->highestAllowedSD){ + printf("Socket set is full at %d sockets\n",set->highestAllowedSD); + return 12; + } + + set->sockets[set->socketCount] = socket; + set->socketCount++; + set->revisionNumber++; + + return 0; +} + +int socketSetRemove(SocketSet *set, Socket *socket){ + SOCKET windowsSocket = socket->windowsSocket; + if (socketTrace) + { + printf ("Removing socket, SOCKET HANDLE=0x%llx\n", windowsSocket); + } + + set->socketCount--; + set->sockets[set->socketCount] = NULL; + set->revisionNumber++; + return 0; +} + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/xlate.c b/c/xlate.c index 91d4b59ef..068c6a267 100644 --- a/c/xlate.c +++ b/c/xlate.c @@ -17,8 +17,11 @@ #else #include #include +#include #endif +#include "zowetypes.h" +#include "alloc.h" #include "xlate.h" typedef struct { @@ -76,6 +79,52 @@ char* a2e(char *buffer, int len) return buffer; } +/** A Charset oracle is a string digester that can guess charsets + based upon what it is fed. It's heuristic, so YMMV. +*/ + + +CharsetOracle *makeCharsetOracle(void){ + CharsetOracle *oracle = (CharsetOracle*)safeMalloc(sizeof(CharsetOracle),"CharsetOracle"); + memset(oracle,0,sizeof(CharsetOracle)); + return oracle; +} + +void freeCharsetOracle(CharsetOracle *oracle){ + safeFree((char*)oracle,sizeof(CharsetOracle)); +} + +void charsetOracleDigest(CharsetOracle *oracle, char *s, int len){ + for (int i=0; irangeBuckets[(c&0xF0)>>4]++; + oracle->charBuckets[c&0xff]++; + } + oracle->total += len; +} + +#define min(a,b) (((a) < (b)) ? (a) : (b)) +#define max(a,b) (((a) > (b)) ? (a) : (b)) + +int guessCharset(CharsetOracle *oracle, double *confidence){ + /* should also take variety of character */ + int *buckets = oracle->rangeBuckets; + int sub20 = buckets[0] + buckets[1]; + int twenties = buckets[2]; + int thirties = buckets[3]; + int range40To7F = buckets[4] + buckets[5] + buckets[6] + buckets[7]; + int low = sub20 + twenties + thirties + range40To7F; + int total = oracle->total; + int range80ToFF = total-low; + + double lengthConfidence = min(1.0,((double)(oracle->total-50))/50); + *confidence = lengthConfidence; + if (range40To7F > range80ToFF){ + return CHARSET_ORACLE_UTF_FAMILY; + } else { + return CHARSET_ORACLE_EBCDIC_FAMILY; + } +} /* This program and the accompanying materials are diff --git a/c/yaml2json.c b/c/yaml2json.c new file mode 100644 index 000000000..c62189b53 --- /dev/null +++ b/c/yaml2json.c @@ -0,0 +1,894 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + + +#include + /* #include */ + +#include +#include +#include +#include +#include + +#ifdef NDEBUG +#undef NDEBUG +#endif +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "charsets.h" +#include "json.h" +#include "jsonschema.h" + +#ifdef __ZOWE_OS_ZOS + +int __atoe_l(char *bufferptr, int leng); +int __etoa_l(char *bufferptr, int leng); + +#endif + + +#if defined(__ZOWE_OS_ZOS) +# define SOURCE_CODE_CHARSET CCSID_IBM1047 +#else +# define SOURCE_CODE_CHARSET CCSID_UTF_8 +#endif + + +static int convertToNative(char *buf, size_t size) { +#ifdef __ZOWE_OS_ZOS + return __atoe_l(buf, size); +#endif + return 0; +} + +static int convertFromNative(char *buf, size_t size) { +#ifdef __ZOWE_OS_ZOS + return __etoa_l(buf, size); +#endif + return 0; +} + +static char *tokenTypeName(yaml_token_type_t type){ + switch (type){ + case YAML_NO_TOKEN: return "NO_TOKEN"; + case YAML_STREAM_START_TOKEN: return "STREAM_START"; + case YAML_STREAM_END_TOKEN: return "STREAM_END"; + case YAML_VERSION_DIRECTIVE_TOKEN: return "VERSION_DIRECTIVE"; + case YAML_TAG_DIRECTIVE_TOKEN: return "TAG_DIRECTIVE"; + case YAML_DOCUMENT_START_TOKEN: return "DOCUMENT_START"; + case YAML_DOCUMENT_END_TOKEN: return "DOCUMENT_END"; + case YAML_BLOCK_SEQUENCE_START_TOKEN: return "BLOCK_SEQUENCE_START"; + case YAML_BLOCK_MAPPING_START_TOKEN: return "BLOCK_MAPPING_START"; + case YAML_BLOCK_END_TOKEN: return "BLOCK_END"; + case YAML_FLOW_SEQUENCE_START_TOKEN: return "FLOW_SEQUENCE_START"; + case YAML_FLOW_SEQUENCE_END_TOKEN: return "FLOW_SEQUENCE_END"; + case YAML_FLOW_MAPPING_START_TOKEN: return "FLOW_MAPPING_START"; + case YAML_FLOW_MAPPING_END_TOKEN: return "FLOW_MAPPING_END"; + case YAML_BLOCK_ENTRY_TOKEN: return "BLOCK_ENTRY"; + case YAML_FLOW_ENTRY_TOKEN: return "FLOW_ENTRY"; + case YAML_KEY_TOKEN: return "KEY"; + case YAML_VALUE_TOKEN: return "VALUE"; + case YAML_ALIAS_TOKEN: return "ALIAS"; + case YAML_ANCHOR_TOKEN: return "ANCHOR"; + case YAML_TAG_TOKEN: return "TAG"; + case YAML_SCALAR_TOKEN: return "SCALAR"; + default: + { + printf("unknown token seen with type = %d\n",(int)type); + return "UNKNOWN_TOKEN"; + } + } +} + +static char *getScalarStyleName(yaml_scalar_style_t style){ + switch (style){ + case YAML_ANY_SCALAR_STYLE: return "any"; + case YAML_PLAIN_SCALAR_STYLE: return "plain"; + case YAML_SINGLE_QUOTED_SCALAR_STYLE: return "single"; + case YAML_DOUBLE_QUOTED_SCALAR_STYLE: return "double"; + case YAML_LITERAL_SCALAR_STYLE: return "literal"; + case YAML_FOLDED_SCALAR_STYLE: return "folded"; + default: return "UNKNOWN_STYLE"; + } +} + +static int yamlReadHandler(void *data, unsigned char *buffer, size_t size, size_t *sizeRead) { + FILE *fp = data; + int rc = 1; + size_t bytesRead = fread(buffer, 1, size, fp); + if (bytesRead > 0) { + if (convertFromNative((char *)buffer, bytesRead) == -1) { + fprintf (stderr, "failed to convert yaml input - %s\n", strerror(errno)); + rc = 0; + } + } + if (ferror(fp)) { + fprintf (stderr, "failed to read yaml input - %s\n", strerror(errno)); + rc = 0; + } + *sizeRead = bytesRead; + return rc; +} + +static void decodeParserError(yaml_parser_t *parser, char *errorBuf, size_t errorBufSize) { + switch (parser->error) { + case YAML_MEMORY_ERROR: + snprintf(errorBuf, errorBufSize, "YAML memory error: not enough memory for parsing"); + break; + case YAML_READER_ERROR: { + size_t problemLen = strlen(parser->problem); + char problemNative[problemLen + 1]; + snprintf (problemNative, problemLen + 1, "%s", parser->problem); + convertToNative(problemNative, problemLen); + if (parser->problem_value != -1) { + snprintf(errorBuf, errorBufSize, "YAML reader error: %s: #%X at %ld", problemNative, parser->problem_value, (long)parser->problem_offset); + } else { + snprintf(errorBuf, errorBufSize, "YAML reader error: %s at %ld", problemNative, (long)parser->problem_offset); + } + break; + } + case YAML_SCANNER_ERROR: + if (parser->context) { + snprintf(errorBuf, errorBufSize, "YAML scanner error: %s at line %d, column %d" + "%s at line %d, column %d\n", parser->context, + (int)parser->context_mark.line+1, (int)parser->context_mark.column+1, + parser->problem, (int)parser->problem_mark.line+1, + (int)parser->problem_mark.column+1); + } else { + snprintf(errorBuf, errorBufSize, "YAML scanner error: %s at line %d, column %d", + parser->problem, (int)parser->problem_mark.line+1, + (int)parser->problem_mark.column+1); + } + break; + case YAML_PARSER_ERROR: + if (parser->context) { + snprintf(errorBuf, errorBufSize, "YAML parser error: %s at line %d, column %d\n" + "%s at line %d, column %d", parser->context, + (int)parser->context_mark.line+1, (int)parser->context_mark.column+1, + parser->problem, (int)parser->problem_mark.line+1, + (int)parser->problem_mark.column+1); + } else { + snprintf(errorBuf, errorBufSize, "YAML parser error: %s at line %d, column %d", + parser->problem, (int)parser->problem_mark.line+1, + (int)parser->problem_mark.column+1); + } + break; + default: + snprintf(errorBuf, errorBufSize, "YAML parser: unknown error"); + } +} + +yaml_document_t *readYAML(const char *filename, char *errorBuf, size_t errorBufSize) { + FILE *file = NULL; + yaml_document_t *document = NULL; + yaml_parser_t parser = {0}; + bool done = false; + + do { + if (!(document = (yaml_document_t*)safeMalloc(sizeof(yaml_document_t), "YAML Doc"))) { + snprintf(errorBuf, errorBufSize, "failed to alloc memory for YAML doc"); + break; + } + memset(document, 0, sizeof(yaml_document_t)); + if (!(file = fopen(filename, "rb"))) { + snprintf(errorBuf, errorBufSize, "failed to read '%s' - %s", filename, strerror(errno)); + break; + } + if (!yaml_parser_initialize(&parser)) { + snprintf(errorBuf, errorBufSize, "failed to initialize YAML parser"); + break; + } + yaml_parser_set_input(&parser, yamlReadHandler, file); + if (!yaml_parser_load(&parser, document)) { + decodeParserError(&parser, errorBuf, errorBufSize); + break; + } + if (!yaml_document_get_root_node(document)){ + snprintf(errorBuf, errorBufSize, "failed to get root node in YAML '%s'", filename); + break; + } + done = true; + } while (0); + + if (!done && document) { + yaml_document_delete(document); + safeFree((char*)document, sizeof(yaml_document_t)); + document = NULL; + } + if (file) { + fclose(file); + } + yaml_parser_delete(&parser); + return document; +} + +static void indent(int x){ + for (int i=0; idata.scalar.length; + int printLength = dataLen > SCALAR_SIZE_LIMIT ? 40 : (int)dataLen; + char val[printLength + 1]; + snprintf(val, printLength + 1, "%.*s", printLength, node->data.scalar.value); + convertToNative(val, printLength); + printf("%s%s", val, eol ? "\n" : ""); +} + + +static void pprintYAML1(yaml_document_t *doc, yaml_node_t *node, int depth){ + switch (node->type){ + case YAML_NO_NODE: + { + indent(depth); + printf("NoNode\n"); + } + break; + case YAML_SCALAR_NODE: + { + indent(depth); + size_t dataLen = node->data.scalar.length; + printf("Scalar: (len=%d)", (int)node->data.scalar.length); + printYamlScalar(node, true); + } + break; + case YAML_SEQUENCE_NODE: + { + yaml_node_item_t *item; + indent(depth);printf("SequenceStart (%s)\n",getSequenceStyleName(node->data.sequence.style)); + for (item = node->data.sequence.items.start; item < node->data.sequence.items.top; item++) { + yaml_node_t *nextNode = yaml_document_get_node(doc, *item); + if (nextNode){ + pprintYAML1(doc,nextNode,depth+2); + } else { + indent(depth+4); + printf("dead end item\n"); + } + } + indent(depth);printf("SequenceEnd\n"); + } + break; + case YAML_MAPPING_NODE: + { + yaml_node_pair_t *pair; + indent(depth);printf("MapStart\n"); + for (pair = node->data.mapping.pairs.start; pair < node->data.mapping.pairs.top; pair++) { + yaml_node_t *keyNode = yaml_document_get_node(doc, pair->key); + /* printf("keyNode 0x%p\n",keyNode); */ + if (keyNode) { + if (keyNode->type != YAML_SCALAR_NODE){ + printf("*** UNEXPECTED NON SCALAR KEY ***\n"); + } else { + indent(depth+2); + size_t dataLen = keyNode->data.scalar.length; + int printLength = dataLen > SCALAR_SIZE_LIMIT ? 40 : (int)dataLen; + /* printf("printLength=%d dataLen=%lld\n",printLength,dataLen); */ + printYamlScalar(keyNode, false); + printf(":\n"); + } + } else { + printf("dead end key\n"); + } + yaml_node_t *valueNode = yaml_document_get_node(doc, pair->value); + if (valueNode){ + pprintYAML1(doc,valueNode,depth+4); + } else{ + printf("dead end value\n"); + } + } + indent(depth);printf("MapEnd\n"); + } + break; + default: + printf("unexpected yaml node type %d\n",node->type); + break; + } +} + +void pprintYAML(yaml_document_t *document){ + pprintYAML1(document,yaml_document_get_root_node(document),0); +} + +#define MAX_JSON_KEY 256 +#define MAX_JSON_STRING 65536 + +/* this needs to be smarter some day + Refer to Yaml spec http://yaml.org/spec/1.2-old/spec.html#id2805071 + */ + +static bool isSyntacticallyInteger(yaml_char_t *data, int length){ + if (length == 0){ + /* printf("empty string not integer\n"); */ + return false; + } + for (int i=0; i= '0' && c <= '9'){ + + } else { + /* printf("%s is NOT an integer\n",data); */ + return false; + } + } + /* printf("%s is an integer\n",data); */ + return true; +} + +static bool isSyntacticallyBool(yaml_char_t *data, int length){ + if ((length == 4) && + (!memcmp(data,"true",4) || + !memcmp(data,"True",4) || + !memcmp(data,"TRUE",4))){ + return true; + } else if ((length == 5) && + (!memcmp(data,"false",5) || + !memcmp(data,"False",5) || + !memcmp(data,"FALSE",5))){ + return true; + } else { + return false; + } +} + +static bool isSyntacticallyNull(yaml_char_t *data, int length){ + if ((length == 1) && + !memcmp(data,"~",1)){ + return true; + } else if ((length == 4) && + (!memcmp(data,"null",4) || + !memcmp(data,"Null",4) || + !memcmp(data,"NULL",4))){ + return true; + } else { + return false; + } +} + +static bool isSyntacticallyTemplate(yaml_char_t *data, int length){ + if (strstr((char*)data,"${{")){ + return true; + } else { + return false; + } +} + +static int64_t readInt(yaml_char_t *data, int length, bool *valid){ + int64_t val64 = 0; + bool allDecimal = true; + + for (int i=0; i= '0' && c <= '9'){ + val64 = (10 * val64) + (c-'0'); + } else { + allDecimal = false; + } + } + + *valid = allDecimal; + return val64; +} + +static bool readBool(yaml_char_t *data, int length, bool *valid){ + if ((length == 4) && + (!memcmp(data,"true",4) || + !memcmp(data,"True",4) || + !memcmp(data,"TRUE",4))){ + *valid = true; + return true; + } else if ((length == 5) && + (!memcmp(data,"false",5) || + !memcmp(data,"False",5) || + !memcmp(data,"FALSE",5))){ + *valid = true; + return false; + } else { + *valid = false; + return false; + } +} + + +#define JSON_FAIL_NOT_HANDLED 100 +#define JSON_FAIL_BAD_INTEGER 104 +#define JSON_FAIL_BAD_BOOL 108 +#define JSON_FAIL_BAD_NULL 112 +#define JSON_FAIL_BAD_TEMPLATE 116 + +static char *extractString(JsonBuilder *b, char *s, char *e){ + int len = e-s; + char *copy = SLHAlloc(b->parser.slh,len+1); + memcpy(copy,s,len); + copy[len] = 0; + return copy; +} + +static void addPlusIfNecessary(jsonPrinter *p, int sourceCCSID, bool *firstPtr){ + bool first = *firstPtr; + if (!first){ + jsonWriteParseably(p," + ",3,false,false,sourceCCSID); + } + *firstPtr = false; +} + +static int buildTemplateJSON(JsonBuilder *b, Json *parent, char *parentKey, + char *nativeValue, int valueLength){ + char *tail = nativeValue; + JsonBuffer *buffer = makeJsonBuffer(); + int sourceCCSID = CCSID_UTF_8; + /* printf ("buildTemplateJSON nativeValue '%.*s' sourceCodeCharset %d\n", valueLength, nativeValue, sourceCCSID);*/ + jsonPrinter *p = makeBufferJsonPrinter(sourceCCSID,buffer); + int status = 0; + bool first = true; + while (true){ + char *nextExpr = strstr(tail,"${{"); + + if (nextExpr){ + if (nextExpr > tail){ + char *frag = extractString(b,tail,nextExpr); + /* printf("frag = '%s'\n",frag); */ + addPlusIfNecessary(p,sourceCCSID,&first); + jsonWriteParseably(p,frag,strlen(frag),true,false,sourceCCSID); + } + char *end = strstr(nextExpr,"}}"); + if (end){ + char *exprText = extractString(b,nextExpr+3,end); + tail = end+2; + addPlusIfNecessary(p,sourceCCSID,&first); + jsonWriteParseably(p,exprText,strlen(exprText),false,false,sourceCCSID); + } else { + status = JSON_FAIL_BAD_TEMPLATE; + break; + } + } else { + char *lastFrag = extractString(b,tail,nativeValue+valueLength); + /* printf("lastFrag = '%s'\n",lastFrag); */ + if (strlen(lastFrag) > 0){ + addPlusIfNecessary(p,sourceCCSID,&first); + jsonWriteParseably(p,lastFrag,strlen(lastFrag),true,false,sourceCCSID); + } + break; + } + } + if (status == 0){ + Json *object = jsonBuildObject(b,parent,parentKey,&status); + if (!status){ + jsonBuildString(b,object,ZOWE_INTERNAL_TYPE,ZOWE_UNEVALUATED,strlen(ZOWE_UNEVALUATED),&status); + if (!status){ + jsonBufferTerminateString(buffer); + char *sourceCode = jsonBufferCopy(buffer); + if (b->traceLevel >= 1){ + printf("embedded source code is (tl=%d): %s\n",b->traceLevel,sourceCode); + dumpbuffer(sourceCode,strlen(sourceCode)); + } + jsonBuildString(b,object,"source",sourceCode,strlen(sourceCode),&status); + } + } + } + freeJsonPrinter(p); + freeJsonBuffer(buffer); + return status; +} + + +static Json *yaml2JSON1(JsonBuilder *b, Json *parent, char *parentKey, + yaml_document_t *doc, yaml_node_t *node, int depth){ + int buildStatus = 0; + switch (node->type){ + case YAML_NO_NODE: + { + printf("*** WARNING *** NoNode\n"); + return NULL; + } + case YAML_SCALAR_NODE: + { + size_t dataLen = node->data.scalar.length; + int valueLength = (int)dataLen; + if (dataLen > MAX_JSON_STRING){ + printf("*** WARNING *** oversize JSON string!\n"); + valueLength = MAX_JSON_STRING; + } + // Negative numbers, hexadecimal + yaml_scalar_style_t style = node->data.scalar.style; + switch (style){ + case YAML_ANY_SCALAR_STYLE: + case YAML_PLAIN_SCALAR_STYLE: + case YAML_SINGLE_QUOTED_SCALAR_STYLE: + case YAML_DOUBLE_QUOTED_SCALAR_STYLE: + case YAML_LITERAL_SCALAR_STYLE: + case YAML_FOLDED_SCALAR_STYLE: + { + char nativeValue[valueLength+1]; + char *tag = (char*)node->tag; + int tagLen = strlen(tag); + char nativeTag[tagLen + 1]; + snprintf(nativeValue, valueLength+1, "%.*s", valueLength, (const char *)node->data.scalar.value); + snprintf(nativeTag, tagLen + 1, "%.*s", tagLen, tag); + convertToNative(nativeValue, valueLength); + convertToNative(nativeTag, tagLen); + if (b->traceLevel >= 2){ + printf("tag = %s scalarStyle=%s\n",nativeTag,getScalarStyleName(node->data.scalar.style)); + } + Json *scalar = NULL; + // HERE, make test with float, int, bool, null, ddate + if (!strcmp(nativeTag,YAML_NULL_TAG) || + (!strcmp(nativeTag,YAML_STR_TAG) && + (style == YAML_PLAIN_SCALAR_STYLE) && + isSyntacticallyNull((yaml_char_t*)nativeValue,valueLength))){ + scalar = jsonBuildNull(b,parent,parentKey,&buildStatus); + } else if (!strcmp(nativeTag,YAML_INT_TAG) || + (!strcmp(nativeTag,YAML_STR_TAG) && + (style == YAML_PLAIN_SCALAR_STYLE) && + isSyntacticallyInteger((yaml_char_t*)nativeValue,valueLength))){ + bool valid; + int64_t x = readInt((yaml_char_t*)nativeValue,valueLength,&valid); + if (valid){ + scalar = jsonBuildInt64(b,parent,parentKey,x,&buildStatus); + } else { + buildStatus = JSON_FAIL_BAD_INTEGER; + } + } else if (!strcmp(nativeTag,YAML_BOOL_TAG) || + (!strcmp(nativeTag,YAML_STR_TAG) && + (style == YAML_PLAIN_SCALAR_STYLE) && + isSyntacticallyBool((yaml_char_t*)nativeValue,valueLength))){ + bool valid; + bool x = readBool((yaml_char_t*)nativeValue,valueLength,&valid); + if (valid){ + scalar = jsonBuildBool(b,parent,parentKey,x,&buildStatus); + } else { + buildStatus = JSON_FAIL_BAD_BOOL; + } + } else if (!strcmp(nativeTag,YAML_STR_TAG)){ + /* + Must be plain + ${{ data.scalar.style); + return NULL; + } + } + case YAML_SEQUENCE_NODE: + { + yaml_node_item_t *item; + Json *jsonArray = jsonBuildArray(b,parent,parentKey,&buildStatus); + if (jsonArray){ + for (item = node->data.sequence.items.start; item < node->data.sequence.items.top; item++) { + yaml_node_t *nextNode = yaml_document_get_node(doc, *item); + if (nextNode){ + yaml2JSON1(b,jsonArray,NULL,doc,nextNode,depth+2); + } else { + printf("*** WARNING *** dead end item\n"); + } + } + return jsonArray; + } else { + printf("*** WARNING *** Failed to add json array err=%d\n",buildStatus); + return NULL; + } + } + case YAML_MAPPING_NODE: + { + yaml_node_pair_t *pair; + Json *jsonObject = jsonBuildObject(b,parent,parentKey,&buildStatus); + if (jsonObject){ + for (pair = node->data.mapping.pairs.start; pair < node->data.mapping.pairs.top; pair++) { + yaml_node_t *keyNode = yaml_document_get_node(doc, pair->key); + /* printf("keyNode 0x%p\n",keyNode); */ + char *key = NULL; + int keyLength; + if (keyNode) { + if (keyNode->type != YAML_SCALAR_NODE){ + printf("*** WARNING *** Non Scalar key\n"); + } else { + size_t dataLen = keyNode->data.scalar.length; + keyLength = (int)dataLen; + if (keyLength > MAX_JSON_KEY){ + printf("*** WARNING *** key too long '%*.*s...'\n", + MAX_JSON_KEY,MAX_JSON_KEY,keyNode->data.scalar.value); + } else { + key = (char*)keyNode->data.scalar.value; + } + } + } else { + printf("*** WARNING *** dead end key\n"); + } + if (key){ + char *keyNative = jsonBuildKey(b, key, keyLength); + convertToNative(keyNative, keyLength); + yaml_node_t *valueNode = yaml_document_get_node(doc, pair->value); + if (valueNode){ + yaml2JSON1(b,jsonObject,keyNative,doc,valueNode,depth+2); + } else{ + printf("*** WARNING *** dead end value\n"); + } + } + } + return jsonObject; + } else { + printf("*** WARNING *** Failed to add json object err=%d\n",buildStatus); + return NULL; + } + } + break; + default: + printf("*** WARNING *** unexpected yaml node type %d\n",node->type); + return NULL; + } +} + +typedef struct ByteOutputStream_tag{ + int size; + int capacity; + int chunkSize; + char *data; +} ByteOutputStream; + +ByteOutputStream *makeByteOutputStream(int chunkSize){ + ByteOutputStream *bos = (ByteOutputStream*)safeMalloc(sizeof(ByteOutputStream),"ByteOutputStream"); + bos->size = 0; + bos->capacity = chunkSize; + bos->chunkSize = chunkSize; + bos->data = safeMalloc(chunkSize,"ByteOutputStream Chunk"); + return bos; +} + +int bosWrite(ByteOutputStream *bos, char *data, int dataSize){ + if (bos->size+dataSize > bos->capacity){ + int extendSize = (bos->chunkSize > dataSize) ? bos->chunkSize : dataSize; + printf("bos extend currSize=0x%x dataSize=0x%x chunk=0x%x extend=0x%x\n", + bos->size,dataSize,bos->chunkSize,extendSize); + int newCapacity = bos->capacity + extendSize; + char *newData = safeMalloc(newCapacity,"BOS extend"); + memcpy(newData,bos->data,bos->size); + safeFree(bos->data,bos->capacity); + bos->data = newData; + bos->capacity = newCapacity; + } + memcpy(bos->data+bos->size,data,dataSize); + bos->size += dataSize; + return bos->size; +} + +int bosAppendString(ByteOutputStream *bos, char *s){ + return bosWrite(bos,s,strlen(s)); +} + +int bosAppendChar(ByteOutputStream *bos, char c){ + return bosWrite(bos,&c,1); +} + +char *bosNullTerminateAndUse(ByteOutputStream *bos){ + char c = 0; + bosWrite(bos,&c,1); + return bos->data; +} + +char *bosUse(ByteOutputStream *bos){ + return bos->data; +} + +void bosReset(ByteOutputStream *bos){ + bos->size = 0; +} + +void bosFree(ByteOutputStream *bos, bool freeBuffer){ + if (freeBuffer){ + safeFree(bos->data,bos->capacity); + } + safeFree((char*)bos,sizeof(ByteOutputStream)); +} + +/* Friday, expand it */ + +Json *yaml2JSON(yaml_document_t *document, ShortLivedHeap *slh){ + JsonBuilder *builder = makeJsonBuilder(slh); + Json *json = yaml2JSON1(builder,NULL,NULL,document,yaml_document_get_root_node(document),0); + freeJsonBuilder(builder,false); + return json; +} + + +static void yaml2JS1(ByteOutputStream *bos, + yaml_document_t *doc, yaml_node_t *node, int depth, + int traceLevel){ + int buildStatus = 0; + switch (node->type){ + case YAML_NO_NODE: + { + printf("*** WARNING *** NoNode\n"); + return; + } + case YAML_SCALAR_NODE: + { + size_t dataLen = node->data.scalar.length; + int valueLength = (int)dataLen; + if (dataLen > MAX_JSON_STRING){ + printf("*** WARNING *** oversize JSON string!\n"); + valueLength = MAX_JSON_STRING; + } + // Negative numbers, hexadecimal + yaml_scalar_style_t style = node->data.scalar.style; + switch (style){ + case YAML_ANY_SCALAR_STYLE: + case YAML_PLAIN_SCALAR_STYLE: + case YAML_SINGLE_QUOTED_SCALAR_STYLE: + case YAML_DOUBLE_QUOTED_SCALAR_STYLE: + case YAML_LITERAL_SCALAR_STYLE: + case YAML_FOLDED_SCALAR_STYLE: + { + char nativeValue[valueLength+1]; + char *tag = (char*)node->tag; + int tagLen = strlen(tag); + char nativeTag[tagLen + 1]; + snprintf(nativeValue, valueLength+1, "%.*s", valueLength, (const char *)node->data.scalar.value); + snprintf(nativeTag, tagLen + 1, "%.*s", tagLen, tag); + convertToNative(nativeValue, valueLength); + convertToNative(nativeTag, tagLen); + if (traceLevel >= 2){ + printf("tag = %s scalarStyle=%s\n",nativeTag,getScalarStyleName(node->data.scalar.style)); + } + Json *scalar = NULL; + /* HERE, make test with float, int, bool, null, date */ + if (!strcmp(nativeTag,YAML_NULL_TAG)){ + } else if (!strcmp(nativeTag,YAML_NULL_TAG)){ + /* Json *scalar = jsonBuildNull(b,parent,parentKey,&buildStatus); */ + } else if (!strcmp(nativeTag,YAML_BOOL_TAG)){ + /* Json *scalar = jsonBuildBool(b,parent,parentKey,"FOO",3,&buildStatus); */ + } else if (!strcmp(nativeTag,YAML_INT_TAG) || + (!strcmp(nativeTag,YAML_STR_TAG) && + (style == YAML_PLAIN_SCALAR_STYLE) && + isSyntacticallyInteger((yaml_char_t*)nativeValue,valueLength))){ + bool valid; + int64_t x = readInt((yaml_char_t*)nativeValue,valueLength,&valid); + if (valid){ + /* this is not yet done */ + } else { + buildStatus = JSON_FAIL_BAD_INTEGER; + } + /* Json *scalar = jsonBuildInt(b,parent,parentKey,"FOO",3,&buildStatus); */ + } else if (!strcmp(nativeTag,YAML_STR_TAG)){ + printf("%*.*s\n",valueLength,valueLength,nativeValue); + } else if (!strcmp(nativeTag,YAML_FLOAT_TAG)){ + printf("*** Warning don't know how to handle float yet\n"); + buildStatus = JSON_FAIL_NOT_HANDLED; + } else if (!strcmp(nativeTag,YAML_TIMESTAMP_TAG)){ + printf("*** Warning don't know how to handle timestamp yet\n"); + buildStatus = JSON_FAIL_NOT_HANDLED; + } + if (buildStatus){ + printf("*** WARNING *** Failed to add property/scalar err=%d\n",buildStatus); + } + return; + } + default: + printf("*** WARNING *** - unknown scalar style %d",node->data.scalar.style); + return; + } + } + case YAML_SEQUENCE_NODE: + { + yaml_node_item_t *item; + printf("[\n"); + for (item = node->data.sequence.items.start; item < node->data.sequence.items.top; item++) { + yaml_node_t *nextNode = yaml_document_get_node(doc, *item); + if (nextNode){ + yaml2JS1(bos,doc,nextNode,depth+2,traceLevel); + } else { + printf("*** WARNING *** dead end item\n"); + } + } + printf("]\n"); + } + case YAML_MAPPING_NODE: + { + yaml_node_pair_t *pair; + /* Json *jsonObject = jsonBuildObject(b,parent,parentKey,&buildStatus); */ + for (pair = node->data.mapping.pairs.start; pair < node->data.mapping.pairs.top; pair++) { + yaml_node_t *keyNode = yaml_document_get_node(doc, pair->key); + /* printf("keyNode 0x%p\n",keyNode); */ + char *key = NULL; + int keyLength; + if (keyNode) { + if (keyNode->type != YAML_SCALAR_NODE){ + printf("*** WARNING *** Non Scalar key\n"); + } else { + size_t dataLen = keyNode->data.scalar.length; + keyLength = (int)dataLen; + if (keyLength > MAX_JSON_KEY){ + printf("*** WARNING *** key too long '%*.*s...'\n", + MAX_JSON_KEY,MAX_JSON_KEY,keyNode->data.scalar.value); + } else { + key = (char*)keyNode->data.scalar.value; + } + } + } else { + printf("*** WARNING *** dead end key\n"); + } + if (key){ + char keyNative[keyLength+1]; + memcpy(keyNative,key,keyLength); + keyNative[keyLength] = 0; + convertToNative(keyNative, keyLength); + yaml_node_t *valueNode = yaml_document_get_node(doc, pair->value); + if (valueNode){ + printf("%s\n",keyNative); + yaml2JS1(bos,doc,valueNode,depth+2,traceLevel); + } else{ + printf("*** WARNING *** dead end value\n"); + } + } + } + return; + } + break; + default: + printf("*** WARNING *** unexpected yaml node type %d\n",node->type); + return; + } +} + +/* this is not fully done, and might not be needed */ +static void yaml2JS(yaml_document_t *document, ShortLivedHeap *slh){ + ByteOutputStream *bos = makeByteOutputStream(0x1000); + yaml2JS1(bos,document,yaml_document_get_root_node(document),0,1); +} + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/zos.c b/c/zos.c index 08e0c4d2d..0b54395aa 100644 --- a/c/zos.c +++ b/c/zos.c @@ -108,14 +108,14 @@ int supervisorMode(int enable){ } int setKey(int key){ + int shiftedKey = key << 4; int oldKey; __asm(" XR 2,2 \n" - " SLL %1,4 \n" " MODESET KEYREG=(%1),SAVEKEY=(2) \n" " SRL 2,4 \n" " ST 2,%0 \n" :"=m"(oldKey) - :"&r"(key) + :"r"(shiftedKey) :"r2"); return oldKey; } @@ -222,15 +222,14 @@ ExternalSecurityManager getExternalSecurityManager(void) { } CVT *getCVT(void) { - int * __ptr32 mem = (int * __ptr32) 0; - int * __ptr32 theCVT = (int * __ptr32)(*(mem+(0x10/4))); - return (CVT*)theCVT; + int *mem = (int*)0; + return (CVT*)INT2PTR(mem[0x10/4]); } Addr31 getATCVT(void) { - int * __ptr32 mem = (int * __ptr32) 0; - int * __ptr32 theATCVT = (int * __ptr32)(*(mem+(ATCVT_ADDRESS/4))); - return theATCVT; + int *mem = (int*)0; + /* int * __ptr32 theATCVT = (int * __ptr32)(*(mem+(ATCVT_ADDRESS/4))); */ + return (Addr31)INT2PTR(mem[ATCVT_ADDRESS/4]); } void *getIEACSTBL(void) { @@ -260,7 +259,7 @@ char *getSystemName (void) { TCB *getTCB(void) { int *mem = (int*)0; - return (TCB*)mem[CURRENT_TCB/sizeof(int)]; + return (TCB*)INT2PTR(mem[CURRENT_TCB/4]); } STCB *getSTCB(void) { @@ -275,7 +274,7 @@ OTCB *getOTCB(void) { ASCB *getASCB(void) { int *mem = (int*)0; - return (ASCB*)(mem[CURRENT_ASCB/sizeof(int)]&0x7FFFFFFF); + return (ASCB*)INT2PTR((mem[CURRENT_ASCB/4])&0x7FFFFFFF); } ASXB *getASXB(void) { @@ -458,6 +457,8 @@ typedef struct safp_tag{ #define SAF_VERIFY_NESTED_COPY 0x20 #define SAF_VERIFY_NO_MFA 0x10 +#pragma pack(packed) + typedef struct safVerifyRequest_tag{ char initlen; char initsubp; @@ -504,6 +505,13 @@ typedef struct safVerifyRequest_tag{ void * __ptr32 idta; } safVerifyRequest; +/* Helper function to workaround xlclang FE and backend not agreeing on packed ptr32 size */ + +static int sizeofSafVerifyRequest(void){ + return sizeof(safVerifyRequest); +} + + /* first flag set */ #define SAF_AUTH_RACFIND_SPECIFIED 0x80 #define SAF_AUTH_RACFIND_YES 0x40 @@ -530,15 +538,59 @@ typedef struct safVerifyRequest_tag{ #define SAF_AUTH_STATUS_WRITEONLY 0x20 #define SAF_AUTH_STATUS_ACCESS 0x10 +typedef struct safAuthRequest_tag{ + char achkleng; + char old_installationDataAddress[3]; + char achkflg1; + char old_entityNameAddress[3]; + char achkflg2; + char old_classNameAddress[3]; + char achkflg3; + char old_volser3ByteAddress[3]; + void * __ptr32 old_volserAddress; + void * __ptr32 applName; + void * __ptr32 acee; + void * __ptr32 owner; /* ? */ + void * __ptr32 installationData; + void * __ptr32 entityName; + void * __ptr32 className; + void * __ptr32 volser; + void * __ptr32 acclvl1; + void * __ptr32 acclvl2; + short fileSequenceNumber; + char tapeFlags; + char achkflg4; + void * __ptr32 userid; + void * __ptr32 groupName; + void * __ptr32 ddname; + void * __ptr32 reserved1; + void * __ptr32 utoken; + void * __ptr32 rtoken; + void * __ptr32 logstr; + void * __ptr32 recvr; +} safAuthRequest; + +typedef struct safStatRequest_tag{ + void * __ptr32 className; + void * __ptr32 CDTentry; + short statLength; + char reserved[2]; + void * __ptr32 classCopy; + int classCopyLength; + void * __ptr32 statNext; +} safStatRequest; + +#pragma pack(reset) + static int SAF(safp * __ptr32 safwrapper, int useSupervisorMode) { int returnCode = 0; int supervisorState = 0; - char * __ptr32 cvt = * (char * __ptr32 * __ptr32 ) 16; - char * __ptr32 safVectorTable = (char * __ptr32) *(int *)(cvt + 248); - char * __ptr32 safRouter = (char * __ptr32) *(int *)(safVectorTable + 12); + CVT *cvt = getCVT(); + int *safVectorTable = (int*)INT2PTR(cvt->cvtsaf); + int safRouter = safVectorTable[3]; /* not using ptr type becuz embedded ASM does not care */ ALLOC_STRUCT31( STRUCT31_NAME(below2G), @@ -592,47 +644,6 @@ void setSafTrace(int traceLevel, void *traceFile){ safTraceFile = traceFile; } -typedef struct safAuthRequest_tag{ - char achkleng; - char old_installationDataAddress[3]; - char achkflg1; - char old_entityNameAddress[3]; - char achkflg2; - char old_classNameAddress[3]; - char achkflg3; - char old_volser3ByteAddress[3]; - void * __ptr32 old_volserAddress; - void * __ptr32 applName; - void * __ptr32 acee; - void * __ptr32 owner; /* ? */ - void * __ptr32 installationData; - void * __ptr32 entityName; - void * __ptr32 className; - void * __ptr32 volser; - void * __ptr32 acclvl1; - void * __ptr32 acclvl2; - short fileSequenceNumber; - char tapeFlags; - char achkflg4; - void * __ptr32 userid; - void * __ptr32 groupName; - void * __ptr32 ddname; - void * __ptr32 reserved1; - void * __ptr32 utoken; - void * __ptr32 rtoken; - void * __ptr32 logstr; - void * __ptr32 recvr; -} safAuthRequest; - -typedef struct safStatRequest_tag{ - void * __ptr32 className; - void * __ptr32 CDTentry; - short statLength; - char reserved[2]; - void * __ptr32 classCopy; - int classCopyLength; - void * __ptr32 statNext; -} safStatRequest; static safp *makeSAFCallData(int requestNumber, int useSupervisorMode, @@ -649,7 +660,7 @@ static safp *makeSAFCallData(int requestNumber, version = SAFPR192; switch (requestNumber){ case SAFPVER: - specificDataSize = sizeof(safVerifyRequest); + specificDataSize = sizeofSafVerifyRequest(); flags = SAFP_FLAG_R18; if (useSupervisorMode){ flags |= SAFP_FLAG_SYST; @@ -780,6 +791,20 @@ static char *makeCountedString(char *name, return result; } +/* forward decl */ +static int safVerifyInternal(int options, + char *userid, + char *password, + char *newPassword, + ACEE **aceeHandle, + void **messageAreaPtr, + int subpool, + char *applicationName, + int sessionType, + int *racfStatus, + int *racfReason, + IDTA *idta); + int safVerify(int options, char *userid, char *password, ACEE **aceeHandle, int *racfStatus, int *racfReason){ @@ -978,7 +1003,7 @@ static int safVerifyInternal(int options, } } - verifyRequest->initlen = sizeof(safVerifyRequest); + verifyRequest->initlen = sizeofSafVerifyRequest(); if (options & VERIFY_CREATE){ verifyFlags1 |= SAF_VERIFY_CREATE; } else if (options & VERIFY_DELETE){ @@ -1019,8 +1044,8 @@ static int safVerifyInternal(int options, flags 1 is zero */ *ACEEPtr = *aceeHandle; if (safTrace){ - fprintf(safTraceFile,"setting acee anchor to %0.8X', %0.8X'\n",aceeHandle, *aceeHandle); - fprintf(safTraceFile,"setting real acee anchor to %0.8X', %0.8X'\n",ACEEPtr, *ACEEPtr); + fprintf(safTraceFile,"setting acee anchor to 0x%p', 0x%p'\n",aceeHandle, *aceeHandle); + fprintf(safTraceFile,"setting real acee anchor to 0x%p', 0x%p'\n",ACEEPtr, *ACEEPtr); } verifyRequest->aceeAnchor = ACEEPtr; /* placeholder for system-created ACEE */ } @@ -1029,12 +1054,12 @@ static int safVerifyInternal(int options, verifyRequest->initflg3 = verifyFlags3; if (safTrace){ - fprintf(safTraceFile,"countedUserid %0.8X' password %0.8X' passphrase %0.8X'\n", + fprintf(safTraceFile,"countedUserid 0x%p' password 0x%p' passphrase 0x%p'\n", countedUserid, countedPassword, countedPassphrase); - fprintf(safTraceFile,"about to go call saf wrapper at %0.8X' verify block at %0.8X' acee %0.8X'\n",safWrapper,verifyRequest, + fprintf(safTraceFile,"about to go call saf wrapper at 0x%p' verify block at 0x%p' acee 0x%p'\n",safWrapper,verifyRequest, (aceeHandle != NULL ? *aceeHandle : NULL)); dumpbuffer((char*)safWrapper,sizeof(safp)); - fprintf(safTraceFile,"verify:\n",0); + fprintf(safTraceFile,"verify: request at 0x%p\n",verifyRequest); dumpbuffer((char*)verifyRequest,sizeof(safVerifyRequest)); } safStatus = SAF(safWrapper,useSupervisorMode); @@ -1138,21 +1163,21 @@ static int safAuth_internal(int options, char *safClass, char *entity, int acces } if (safTrace){ - fprintf(safTraceFile,"before calling SAF, wrapper=%0.8X':\n",safWrapper); + fprintf(safTraceFile,"before calling SAF, wrapper=0x%p':\n",safWrapper); dumpbuffer((char*)safWrapper,sizeof(safp)); - fprintf(safTraceFile,"before calling SAF, auth=%0.8X':\n", authRequest); + fprintf(safTraceFile,"before calling SAF, auth=0x%p':\n", authRequest); dumpbuffer((char*)authRequest,sizeof(safAuthRequest)); - fprintf(safTraceFile,"before calling SAF, class=%0.8X':\n", countedClass); + fprintf(safTraceFile,"before calling SAF, class=0x%p':\n", countedClass); dumpbuffer((char*)countedClass,countedClass[0]+1); - fprintf(safTraceFile,"before calling SAF, entity=%0.8X':\n", countedEntity); + fprintf(safTraceFile,"before calling SAF, entity=0x%p':\n", countedEntity); dumpbuffer((char*)countedEntity,countedEntity[3]+4); } safStatus = SAF(safWrapper, useSupervisorMode); if (safTrace){ fprintf(safTraceFile,"after calling SAF, safStatus=%d\n",safStatus); - fprintf(safTraceFile,"after calling SAF, wrapper=%0.8X':\n",safWrapper); + fprintf(safTraceFile,"after calling SAF, wrapper=0x%p':\n",safWrapper); dumpbuffer((char*)safWrapper,sizeof(safp)); - fprintf(safTraceFile,"after calling SAF, auth=%0.8X':\n", authRequest); + fprintf(safTraceFile,"after calling SAF, auth=0x%p':\n", authRequest); dumpbuffer((char*)authRequest,sizeof(safAuthRequest)); } *racfStatus = safWrapper->safprret; @@ -1210,15 +1235,15 @@ int safStat(int options, char *safClass, char *copy, int copyLength, int *racfSt statRequest->statLength = sizeof(safStatRequest); if (safTrace){ - fprintf(safTraceFile,"about to call saf, wrapper %0.8X:\n",safWrapper); + fprintf(safTraceFile,"about to call saf, wrapper 0x%p:\n",safWrapper); dumpbuffer((char*)safWrapper,sizeof(safp)); - fprintf(safTraceFile,"stat %0.8X:\n",statRequest); + fprintf(safTraceFile,"stat 0x%p:\n",statRequest); dumpbuffer((char*)statRequest,sizeof(safStatRequest)); - fprintf(safTraceFile,"className %0.8X:\n",classBuffer); + fprintf(safTraceFile,"className 0x%p:\n",classBuffer); dumpbuffer((char*)classBuffer,12); - fprintf(safTraceFile,"classCopy %0.8X:\n",copy); + fprintf(safTraceFile,"classCopy 0x%p:\n",copy); dumpbuffer((char*)copy,copyLength); - fprintf(safTraceFile,"workArea %0.8X\n",workArea); + fprintf(safTraceFile,"workArea 0x%p\n",workArea); fflush(safTraceFile); } safStatus = SAF(safWrapper, useSupervisorMode); @@ -1291,7 +1316,7 @@ int locate(char *dsn, int *volserCount, char *firstVolser){ memcpy(below2G->dsn44,dsn,dsnLength); /* - printf("dsn44 %x len %d\n",dsn44,dsnLength); + printf("dsn44 0x%p len %d\n",dsn44,dsnLength); dumpbuffer(dsn44,44); printf("arg %x:\n",svc26Arg); dumpbuffer(svc26Arg,16); @@ -1394,7 +1419,7 @@ static void *loadByNameInternal(char *moduleName, int *statusPtr, if (status){ return 0; /* entry point not found */ } else{ - return (void*)entryPoint; + return INT2PTR(entryPoint); } } diff --git a/c/zosfile.c b/c/zosfile.c index 4dfc9400d..f72b6fc5f 100644 --- a/c/zosfile.c +++ b/c/zosfile.c @@ -120,6 +120,28 @@ #define BPXGPN BPX1GPN #endif +/* Better compilers need these symbols to be declared as functions */ +int BPXRED(); +int BPXOPN(); +int BPXWRT(); +int BPXREN(); +int BPXCHR(); +int BPXCHM(); +int BPXCLO(); +int BPXLCO(); +int BPXSTA(); +int BPXUNL(); +int BPXOPD(); +int BPXMKD(); +int BPXRDD(); +int BPXRMD(); +int BPXCLD(); +int BPXUMK(); +int BPXFCT(); +int BPXLST(); +int BPXGGN(); +int BPXGPN(); + #define MAX_ENTRY_BUFFER_SIZE 2550 #define MAX_NUM_ENTRIES 1000 @@ -454,8 +476,8 @@ int fileChangeTagPure(const char *fileName, int *returnCode, int *reasonCode, int fileChangeTag(const char *fileName, int *returnCode, int *reasonCode, int ccsid) { bool pure = true; - fileChangeTagPure(fileName, returnCode, reasonCode, ccsid, pure); - } + return fileChangeTagPure(fileName, returnCode, reasonCode, ccsid, pure); +} int fileChangeTagPure(const char *fileName, int *returnCode, int *reasonCode, int ccsid, bool pure) { diff --git a/h/collections.h b/h/collections.h index 406b3be0b..83be11c77 100644 --- a/h/collections.h +++ b/h/collections.h @@ -20,6 +20,7 @@ */ #include "zowetypes.h" +#include "utils.h" #ifndef __ZOWE_OS_ZOS #include "openprims.h" @@ -50,6 +51,13 @@ #define makeQueue MAKELCFQ #define destroyQueue DSTRLCFQ +#define makeArrayList ALSTMAKE +#define arrayListAdd ALSTADD +#define arrayListElement ALSTELMT +#define arrayListSort ALSTSORT +#define arrayListShallowCopy ALSHLCPY +#define initEmbbededArrayList ALINEMAR + #endif typedef struct fixedBlockMgr_tag{ @@ -307,8 +315,23 @@ void qInsert(Queue *q, void *newData) QueueAmode64; */ void *qRemove(Queue *q) QueueAmode64; -#endif +typedef struct ArrayList_tag{ + int capacity; + int size; + void **array; + ShortLivedHeap *slh; +} ArrayList; + +ArrayList *makeArrayList(); +void arrayListFree(ArrayList *list); +void arrayListAdd(ArrayList *list, void *thing); +void *arrayListElement(ArrayList *list, int i); +void initEmbeddedArrayList(ArrayList *list, + ShortLivedHeap *slh); +void *arrayListShallowCopy(ArrayList *source, ArrayList *target); +void arrayListSort(ArrayList *list, int (*comparator)(const void *a, const void *b)); +#endif /* This program and the accompanying materials are diff --git a/h/crossmemory.h b/h/crossmemory.h index e8fcc6978..f80360813 100644 --- a/h/crossmemory.h +++ b/h/crossmemory.h @@ -174,8 +174,8 @@ typedef struct ELXLIST_tag { #define CROSS_MEMORY_SERVER_MAX_CMD_TASK_NUM 30 -typedef struct CrossMemoryServerGlobalArea_tag; -typedef struct CrossMemoryService_tag; +struct CrossMemoryServerGlobalArea_tag; +struct CrossMemoryService_tag; typedef int (CrossMemoryServiceFunction)(struct CrossMemoryServerGlobalArea_tag *globalArea, struct CrossMemoryService_tag *service, void *parm); @@ -231,7 +231,7 @@ typedef struct CrossMemoryServerGlobalArea_tag { int pcLogLevel; - PAD_LONG(0, RecoveryStatePool *pcssRecoveryPool); + PAD_LONG(0, struct RecoveryStatePool_tag *pcssRecoveryPool); CPID pcssStackPool; char reserved3[492]; diff --git a/h/discovery.h b/h/discovery.h index 22d7f8326..50672888c 100644 --- a/h/discovery.h +++ b/h/discovery.h @@ -164,6 +164,7 @@ typedef struct TSBInfo_tag{ IKTTSBX *tsbx; } TSBInfo; +struct DiscoveryContext_tag; typedef void SubsystemVisitor(struct DiscoveryContext_tag *context, SSCT *ssctChain, diff --git a/h/dynalloc.h b/h/dynalloc.h index b75987233..7cebd8e4a 100644 --- a/h/dynalloc.h +++ b/h/dynalloc.h @@ -25,8 +25,8 @@ #error z/OS targets are supported only #endif -#define turn_on_HOB(x) x = (TextUnit* __ptr32) ((int) x | 0x80000000) -#define turn_off_HOB(x) x = (TextUnit* __ptr32) ((int) x & 0x7FFFFFFF) +#define turn_on_HOB(x) x = (TextUnit* __ptr32) INT2PTR((int) x | 0x80000000) +#define turn_off_HOB(x) x = (TextUnit* __ptr32) INT2PTR((int) x & 0x7FFFFFFF) #define S99VRBAL 0x01 /* Allocation */ #define S99VRBUN 0x02 /* Unallocation */ diff --git a/h/embeddedjs.h b/h/embeddedjs.h new file mode 100644 index 000000000..1d6bb613c --- /dev/null +++ b/h/embeddedjs.h @@ -0,0 +1,54 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#ifndef __ZOWE_EMBEDDEDJS__ +#define __ZOWE_EMBEDDEDJS__ 1 + +#include "zowetypes.h" + +struct EmbeddedJS_tag; + +typedef struct EmbeddedJS_tag EmbeddedJS; + +struct JSValueBox_tag; + +typedef struct JSValueBox_tag JSValueBox; + +JSValueBox ejsEvalBuffer(EmbeddedJS *ejs, + const void *buffer, int bufferLength, + const char *filename, int eval_flags, + int *statusPtr); + +int ejsEvalFile(EmbeddedJS *ejs, const char *filename, int loadMode); + +void ejsFreeJSValue(EmbeddedJS *ejs, JSValueBox valueBox); + + +/* usually pass NULL as arg unless trying to build complex embedded JS + application that can run multiple threads or other multitasking. +*/ +EmbeddedJS *makeEmbeddedJS(EmbeddedJS *sharedEJS); + +Json *ejsJSToJson(EmbeddedJS *ejs, JSValueBox value, ShortLivedHeap *slh); +JSValueBox ejsJsonToJS(EmbeddedJS *ejs, Json *json); +int ejsSetGlobalProperty(EmbeddedJS *ejs, const char *propetyName, JSValueBox valueBox); +Json *evaluateJsonTemplates(EmbeddedJS *ejs, ShortLivedHeap *slh, Json *json); + +#endif + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/h/httpserver.h b/h/httpserver.h index 116710686..2cddc8631 100644 --- a/h/httpserver.h +++ b/h/httpserver.h @@ -590,6 +590,8 @@ int setHttpCloseConversationTrace(int toWhat); int setHttpAuthTrace(int toWhat); #endif +int isLowerCasePasswordAllowed(); + int httpServerInitJwtContext(HttpServer *self, bool legacyFallback, const char *pkcs11TokenName, diff --git a/h/json.h b/h/json.h index e990501b6..1e413f251 100644 --- a/h/json.h +++ b/h/json.h @@ -16,11 +16,14 @@ #ifdef METTLE # include # include +# include #else # include +# include #endif /* METTLE */ #include "zowetypes.h" #include "utils.h" +#include "collections.h" /** \file * \brief json.h is an implementation of an efficient low-level JSON writer and parser. @@ -29,6 +32,8 @@ * and chunked input and output. */ +typedef struct Json_tag Json; + /** * \brief jsonPrinter represents a high-level stream to write JSON. * @@ -57,6 +62,10 @@ typedef struct jsonPrinter_tag { char *_conversionBuffer; int ioErrorFlag; int isInMultipartString; + bool (*filter)(void *filterContext, + char *keyOrNull, + Json *value); + void *filterContext; } jsonPrinter; typedef struct jsonBuffer_tag { @@ -81,6 +90,11 @@ jsonPrinter *makeCustomUtf8JsonPrinter( void (*writeMethod)(jsonPrinter *, char *, int), void *object, int inputCCSID); +/** + * \brief Reset a printer to its starting state. + */ +void jsonPrinterReset(jsonPrinter *printer); + jsonPrinter *makeBufferJsonPrinter(int inputCCSID, JsonBuffer *buf); /** @@ -142,6 +156,13 @@ void jsonEndArray(jsonPrinter *p); void jsonAddString(jsonPrinter *p, char *keyOrNull, char *value); +/** + * + * A special method for writing javascript data that goes beyond standard JSON writing. + * + */ +void jsonWriteParseably(jsonPrinter *p, char *text, int len, bool quote, bool escape, int inputCCSID); + /** * \brief Allows the caller to place a string representing well-formed JSON * content in the output stream into an enclosing object or array @@ -332,8 +353,8 @@ JsonBuffer *makeJsonBuffer(void); void jsonBufferTerminateString(JsonBuffer *buffer); void jsonBufferRewind(JsonBuffer *buffer); void freeJsonBuffer(JsonBuffer *buffer); +char *jsonBufferCopy(JsonBuffer *buffer); -typedef struct Json_tag Json; typedef struct JsonObject_tag JsonObject; typedef struct JsonArray_tag JsonArray; typedef struct JsonProperty_tag JsonProperty; @@ -346,10 +367,18 @@ struct Json_tag { #define JSON_TYPE_NULL 3 #define JSON_TYPE_OBJECT 4 #define JSON_TYPE_ARRAY 5 + /* V2 parsing can yield the following additional types */ +#define JSON_TYPE_INT64 6 +#define JSON_TYPE_DOUBLE 7 #define JSON_TYPE_ERROR 666 int type; union { int number; + /* There two fields are being introduced to support 'proper' parsing of + numbers. Up until 2022, this json library was not very friendly to floating + point and integers with a >32-bit mantissa. */ + int64_t integerValue; + double floatValue; char *string; int boolean; JsonObject *object; @@ -382,6 +411,8 @@ struct JsonError_tag { Json *jsonParseString(ShortLivedHeap *slh, char *jsonString, char* errorBufferOrNull, int errorBufferSize); Json *jsonParseUnterminatedString(ShortLivedHeap *slh, char *jsonString, int len, char* errorBufferOrNull, int errorBufferSize); Json *jsonParseFile(ShortLivedHeap *slh, const char *filename , char* errorBufferOrNull, int errorBufferSize); +/* parseFile2 supports full-width integers and floating-point numbers */ +Json *jsonParseFile2(ShortLivedHeap *slh, const char *filename, char* errorBufferOrNull, int errorBufferSize); Json *jsonParseUnterminatedUtf8String(ShortLivedHeap *slh, int outputCCSID, char *jsonUtf8String, int len, char *errorBufferOrNull, int errorBufferSize); @@ -395,6 +426,8 @@ void jsonPrintArray(jsonPrinter* printer, JsonArray *array); int jsonAsBoolean(Json *json); int jsonAsNumber(Json *json); +int64_t jsonAsInt64(Json *json); +double jsonAsDouble(Json *json); char *jsonAsString(Json *json); JsonArray *jsonAsArray(Json *json); JsonObject *jsonAsObject(Json *json); @@ -405,9 +438,16 @@ int jsonIsArray(Json *json); int jsonIsBoolean(Json *json); int jsonIsNull(Json *json); int jsonIsNumber(Json *json); +/* V2 more specific number type */ +int jsonIsDouble(Json *json); +int jsonIsInt64(Json *json); int jsonIsObject(Json *json); int jsonIsString(Json *json); +/************** High Uniqueness Hash ******************/ + +int64_t jsonLongHash(Json *json); + /************** Array Accessors ***********************/ int jsonArrayGetCount(JsonArray *array); @@ -517,6 +557,135 @@ char *jsonStringProperty(JsonObject *object, char *propertyName, int *stat JsonObject *jsonObjectProperty(JsonObject *object, char *propertyName, int *status); void reportJSONDataProblem(void *jsonObject, int status, char *propertyName); +/* JSON Builder interface. This Interface allows the translation to JSON + form other formats, without going through json syntax parsing. +*/ + +typedef struct JsonParser_tag JsonParser; +typedef struct JsonTokenizer_tag JsonTokenizer; +typedef struct JsonToken_tag JsonToken; + +#define JSON_PARSE_VERSION_1 1 /* incorrect numbers (only 32-bit integers) */ +#define JSON_PARSE_VERSION_2 2 /* numbers parse to int64 or a double */ + +struct JsonParser_tag { + JsonTokenizer *tokenizer; + JsonToken *unreadToken; + ShortLivedHeap *slh; + CharStream *in; + Json *jsonError; + int version; +}; + +typedef struct JsonBuilder_tag { + JsonParser parser; /* must be first member to support casts */ + Json *root; + int traceLevel; +} JsonBuilder; + +#define ZOWE_INTERNAL_TYPE "__zowe_internal_type__" +#define ZOWE_UNEVALUATED "unevaluated" + +JsonBuilder *makeJsonBuilder(ShortLivedHeap *slh); + +#define JSON_BUILD_FAIL_PARENT_IS_SCALAR 12 +#define JSON_BUILD_FAIL_KEY_ON_ARRAY 16 +#define JSON_BUILD_FAIL_KEY_ON_ROOT 20 +#define JSON_BUILD_FAIL_NO_KEY_ON_OBJECT 24 + +void freeJsonBuilder(JsonBuilder *builder, bool freeSLH); + +Json *jsonBuildObject(JsonBuilder *b, + Json *parent, + char *parentKey, + int *errorCode); + +Json *jsonBuildArray(JsonBuilder *b, + Json *parent, + char *parentKey, + int *errorCode); + +/** + This copies the string into memory owned by the builder/parser. That is + the input string arg "s" is not incorporated into the result. +*/ +Json *jsonBuildString(JsonBuilder *b, + Json *parent, + char *parentKey, + char *s, + int sLen, + int *errorCode); + +Json *jsonBuildInt(JsonBuilder *b, + Json *parent, + char *parentKey, + int i, + int *errorCode); + +/* This builds a V2 compatible integer, jsonBuiltInt does the legacy 32 bit integer */ +Json *jsonBuildInt64(JsonBuilder *b, + Json *parent, + char *parentKey, + int64 i, + int *errorCode); + +Json *jsonBuildDouble(JsonBuilder *b, + Json *parent, + char *parentKey, + double d, + int *errorCode); + + +Json *jsonBuildBool(JsonBuilder *b, + Json *parent, + char *parentKey, + bool truthValue, + int *errorCode); + +Json *jsonBuildNull(JsonBuilder *b, + Json *parent, + char *parentKey, + int *errorCode); + + +char* jsonBuildKey(JsonBuilder *b, const char *key, int len); + +#define JSON_MERGE_STATUS_SUCCESS 0 +#define JSON_MERGE_STATUS_UNMERGEABLE_TYPES 1 +#define JSON_MERGE_STATUS_UNIMPLEMENTED 2 + +/* Merging occurs when arrays match up to arays, or objects to objects */ + +#define JSON_MERGE_FLAG_ARRAY_POLICY_MASK 0x000F + +#define JSON_MERGE_FLAG_CONCATENATE_ARRAYS 0x0001 /* len(merge) = len(a)+len(b) */ +#define JSON_MERGE_FLAG_MERGE_IN_PLACE 0x0002 /* len(merge) = max(len(a),len(b)), a overrides corresponding elements of b */ +#define JSON_MERGE_FLAG_TAKE_BASE 0x0003 /* len(merge) = len(b) */ +#define JSON_MERGE_FLAG_TAKE_OVERRIDES 0x0004 /* len(merge) = len(a) */ + +Json *jsonMerge(ShortLivedHeap *slh, Json *overrides, Json *base, int flags, int *statusPtr); +Json *jsonCopy(ShortLivedHeap *slh, Json *value); + +/* JSON Pointers */ + +#define JSON_POINTER_NORMAL_KEY 1 +#define JSON_POINTER_INTEGER 2 + +typedef struct JsonPointerElement_tag { + char *string; + int type; +} JsonPointerElement; + +/* These JSON pointers are *ABSOLUTE*, supporting relative is more work */ + +typedef struct JsonPointer_tag { + ArrayList elements; +} JsonPointer; + +JsonPointer *parseJsonPointer(char *s); +void freeJsonPointer(JsonPointer *jp); +void printJsonPointer(FILE *out, JsonPointer *jp); + #endif /* __JSON__ */ diff --git a/h/jsonschema.h b/h/jsonschema.h new file mode 100644 index 000000000..ce5ff0176 --- /dev/null +++ b/h/jsonschema.h @@ -0,0 +1,129 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#ifndef __ZOWE_JSONSCHEMA__ +#define __ZOWE_JSONSCHEMA__ 1 + +#ifdef METTLE + +#else +#include +#endif + +#ifdef __ZOWE_OS_WINDOWS +#include "winregex.h" +#else +#include "psxregex.h" +#endif + + +typedef struct JsonSchema_tag { + ShortLivedHeap *slh; + int version; + struct JSValueSpec_tag *topValueSpec; +} JsonSchema; + +#define MAX_SCHEMA_DEPTH 100 + +typedef struct AccessPath_tag { + struct { + union { + char *name; + int index; + } key; + bool isName; + } elements[MAX_SCHEMA_DEPTH]; + int currentSize; +} AccessPath; + +typedef struct JsonSchemaBuilder_tag { + JsonSchema *schema; + ShortLivedHeap *slh; + int version; + int traceLevel; + AccessPath *accessPath; + int errorCode; + char *errorMessage; + int errorMessageLength; +#ifdef METTLE +#error "need setjmp" +#else + jmp_buf recoveryData; +#endif +} JsonSchemaBuilder; + +#define MAX_ACCESS_PATH 1024 + +#define MAX_VALIDITY_EXCEPTION_MSG 1024 + +typedef struct ValidityException_tag { + int code; + struct ValidityException_tag *next; + char message[MAX_VALIDITY_EXCEPTION_MSG]; +} ValidityException; + +#define VALIDATOR_WARN_ON_UNDEFINED_PROPERTIES 0x0001 + +#define MAX_VALIDATOR_MATCHES 8 + +typedef struct JsonValidator_tag { + JsonSchema *topSchema; + /* Other schemas that are referred to by the top and each other can be loaded into + one validator */ + JsonSchema **otherSchemas; + int otherSchemaCount; + int errorCode; + char *errorMessage; + int errorMessageLength; + char accessPathBuffer[MAX_ACCESS_PATH]; + ValidityException *firstValidityException; + ValidityException *lastValidityException; + int flags; + int traceLevel; + AccessPath *accessPath; + /* we use these pattern many times and manage the resources from here */ + regmatch_t matches[MAX_VALIDATOR_MATCHES]; + regex_t *httpRegex; + int httpRegexError; + regex_t *fileRegex; + int fileRegexError; + jmp_buf recoveryData; +} JsonValidator; + +#define JSON_SCHEMA_DRAFT_4 400 +#define DEFAULT_JSON_SCHEMA_VERSION JSON_SCHEMA_DRAFT_4 + + +JsonSchemaBuilder *makeJsonSchemaBuilder(int version); +void freeJsonSchemaBuilder(JsonSchemaBuilder *builder); +JsonSchema *jsonBuildSchema(JsonSchemaBuilder *builder, Json *schemaJson); + +JsonValidator *makeJsonValidator(); +void freeJsonValidator(JsonValidator *validator); + +#define JSON_VALIDATOR_NO_EXCEPTIONS 0 +#define JSON_VALIDATOR_HAS_EXCEPTIONS 4 +#define JSON_VALIDATOR_INTERNAL_FAILURE 8 + +int jsonValidateSchema(JsonValidator *validator, Json *value, JsonSchema *topSchema, + JsonSchema **otherSchemas, int otherSchemaCount); + + +#endif + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/h/le.h b/h/le.h index 964fed720..224f6c779 100644 --- a/h/le.h +++ b/h/le.h @@ -245,8 +245,8 @@ ZOWE_PRAGMA_PACK_RESET #endif -char *getCAA(); -void showRTL(); +char *getCAA(void); +void showRTL(void); #else /* not __ZOWE_OS_ZOS - */ diff --git a/h/microjq.h b/h/microjq.h new file mode 100644 index 000000000..a99934576 --- /dev/null +++ b/h/microjq.h @@ -0,0 +1,13 @@ +#ifndef __ZOWE_MICROJQ__ +#define __ZOWE_MICROJQ__ 1 + +#define JQ_FLAG_RAW_STRINGS 0x0001 +#define JQ_FLAG_PRINT_PRETTY 0x0002 + +#include "json.h" +#include "parsetools.h" + +Json *parseJQ(JQTokenizer *jqt, ShortLivedHeap *slh, int traceLevel); +int evalJQ(Json *value, Json *jqTree, FILE *out, int flags, int traceLevel); + +#endif diff --git a/h/parsetools.h b/h/parsetools.h new file mode 100644 index 000000000..129f0202e --- /dev/null +++ b/h/parsetools.h @@ -0,0 +1,503 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#ifndef __ZOWE_PARSETOOLS__ +#define __ZOWE_PARSETOOLS__ 1 + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "json.h" + +/* Common Tokenization/Lexing Infrastucture */ + +#define EOF_TOKEN -1 +#define NO_VALID_TOKEN 0 +#define FIRST_REAL_TOKEN_ID 1 + + +#define CPROP_ALPHA 0x0001 +#define CPROP_DIGIT 0x0002 +#define CPROP_HEX 0x0004 +#define CPROP_CONTROL 0x0008 +#define CPROP_LETTER 0x0010 +#define CPROP_LOWER 0x0020 +#define CPROP_WHITE 0x0040 + +static int charProps[256] = { + CPROP_CONTROL, /* 0x0 */ + CPROP_CONTROL, /* 0x1 */ + CPROP_CONTROL, /* 0x2 */ + CPROP_CONTROL, /* 0x3 */ + CPROP_CONTROL, /* 0x4 */ + CPROP_CONTROL, /* 0x5 */ + CPROP_CONTROL, /* 0x6 */ + CPROP_CONTROL, /* 0x7 */ + CPROP_CONTROL, /* 0x8 */ + CPROP_CONTROL |CPROP_WHITE, /* 0x9 */ + CPROP_CONTROL |CPROP_WHITE, /* 0xa */ + CPROP_CONTROL |CPROP_WHITE, /* 0xb */ + CPROP_CONTROL |CPROP_WHITE, /* 0xc */ + CPROP_CONTROL |CPROP_WHITE, /* 0xd */ + CPROP_CONTROL, /* 0xe */ + CPROP_CONTROL, /* 0xf */ + CPROP_CONTROL, /* 0x10 */ + CPROP_CONTROL, /* 0x11 */ + CPROP_CONTROL, /* 0x12 */ + CPROP_CONTROL, /* 0x13 */ + CPROP_CONTROL, /* 0x14 */ + CPROP_CONTROL, /* 0x15 */ + CPROP_CONTROL, /* 0x16 */ + CPROP_CONTROL, /* 0x17 */ + CPROP_CONTROL, /* 0x18 */ + CPROP_CONTROL, /* 0x19 */ + CPROP_CONTROL, /* 0x1a */ + CPROP_CONTROL, /* 0x1b */ + CPROP_CONTROL |CPROP_WHITE, /* 0x1c */ + CPROP_CONTROL |CPROP_WHITE, /* 0x1d */ + CPROP_CONTROL |CPROP_WHITE, /* 0x1e */ + CPROP_CONTROL |CPROP_WHITE, /* 0x1f */ + CPROP_WHITE, /* 0x20 */ + 0, /* 0x21 */ + 0, /* 0x22 */ + 0, /* 0x23 */ + 0, /* 0x24 */ + 0, /* 0x25 */ + 0, /* 0x26 */ + 0, /* 0x27 */ + 0, /* 0x28 */ + 0, /* 0x29 */ + 0, /* 0x2a */ + 0, /* 0x2b */ + 0, /* 0x2c */ + 0, /* 0x2d */ + 0, /* 0x2e */ + 0, /* 0x2f */ + CPROP_DIGIT |CPROP_HEX, /* 0x30 */ + CPROP_DIGIT |CPROP_HEX, /* 0x31 */ + CPROP_DIGIT |CPROP_HEX, /* 0x32 */ + CPROP_DIGIT |CPROP_HEX, /* 0x33 */ + CPROP_DIGIT |CPROP_HEX, /* 0x34 */ + CPROP_DIGIT |CPROP_HEX, /* 0x35 */ + CPROP_DIGIT |CPROP_HEX, /* 0x36 */ + CPROP_DIGIT |CPROP_HEX, /* 0x37 */ + CPROP_DIGIT |CPROP_HEX, /* 0x38 */ + CPROP_DIGIT |CPROP_HEX, /* 0x39 */ + 0, /* 0x3a */ + 0, /* 0x3b */ + 0, /* 0x3c */ + 0, /* 0x3d */ + 0, /* 0x3e */ + 0, /* 0x3f */ + 0, /* 0x40 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER, /* 0x41 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER, /* 0x42 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER, /* 0x43 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER, /* 0x44 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER, /* 0x45 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER, /* 0x46 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x47 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x48 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x49 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x4a */ + CPROP_ALPHA |CPROP_LETTER, /* 0x4b */ + CPROP_ALPHA |CPROP_LETTER, /* 0x4c */ + CPROP_ALPHA |CPROP_LETTER, /* 0x4d */ + CPROP_ALPHA |CPROP_LETTER, /* 0x4e */ + CPROP_ALPHA |CPROP_LETTER, /* 0x4f */ + CPROP_ALPHA |CPROP_LETTER, /* 0x50 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x51 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x52 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x53 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x54 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x55 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x56 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x57 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x58 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x59 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x5a */ + 0, /* 0x5b */ + 0, /* 0x5c */ + 0, /* 0x5d */ + 0, /* 0x5e */ + 0, /* 0x5f */ + 0, /* 0x60 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER |CPROP_LOWER, /* 0x61 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER |CPROP_LOWER, /* 0x62 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER |CPROP_LOWER, /* 0x63 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER |CPROP_LOWER, /* 0x64 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER |CPROP_LOWER, /* 0x65 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER |CPROP_LOWER, /* 0x66 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x67 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x68 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x69 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x6a */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x6b */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x6c */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x6d */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x6e */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x6f */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x70 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x71 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x72 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x73 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x74 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x75 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x76 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x77 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x78 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x79 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x7a */ + 0, /* 0x7b */ + 0, /* 0x7c */ + 0, /* 0x7d */ + 0, /* 0x7e */ + CPROP_CONTROL, /* 0x7f */ + CPROP_CONTROL, /* 0x80 */ + CPROP_CONTROL, /* 0x81 */ + CPROP_CONTROL, /* 0x82 */ + CPROP_CONTROL, /* 0x83 */ + CPROP_CONTROL, /* 0x84 */ + CPROP_CONTROL, /* 0x85 */ + CPROP_CONTROL, /* 0x86 */ + CPROP_CONTROL, /* 0x87 */ + CPROP_CONTROL, /* 0x88 */ + CPROP_CONTROL, /* 0x89 */ + CPROP_CONTROL, /* 0x8a */ + CPROP_CONTROL, /* 0x8b */ + CPROP_CONTROL, /* 0x8c */ + CPROP_CONTROL, /* 0x8d */ + CPROP_CONTROL, /* 0x8e */ + CPROP_CONTROL, /* 0x8f */ + CPROP_CONTROL, /* 0x90 */ + CPROP_CONTROL, /* 0x91 */ + CPROP_CONTROL, /* 0x92 */ + CPROP_CONTROL, /* 0x93 */ + CPROP_CONTROL, /* 0x94 */ + CPROP_CONTROL, /* 0x95 */ + CPROP_CONTROL, /* 0x96 */ + CPROP_CONTROL, /* 0x97 */ + CPROP_CONTROL, /* 0x98 */ + CPROP_CONTROL, /* 0x99 */ + CPROP_CONTROL, /* 0x9a */ + CPROP_CONTROL, /* 0x9b */ + CPROP_CONTROL, /* 0x9c */ + CPROP_CONTROL, /* 0x9d */ + CPROP_CONTROL, /* 0x9e */ + CPROP_CONTROL, /* 0x9f */ + 0, /* 0xa0 */ + 0, /* 0xa1 */ + 0, /* 0xa2 */ + 0, /* 0xa3 */ + 0, /* 0xa4 */ + 0, /* 0xa5 */ + 0, /* 0xa6 */ + 0, /* 0xa7 */ + 0, /* 0xa8 */ + 0, /* 0xa9 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xaa */ + 0, /* 0xab */ + 0, /* 0xac */ + 0, /* 0xad */ + 0, /* 0xae */ + 0, /* 0xaf */ + 0, /* 0xb0 */ + 0, /* 0xb1 */ + 0, /* 0xb2 */ + 0, /* 0xb3 */ + 0, /* 0xb4 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xb5 */ + 0, /* 0xb6 */ + 0, /* 0xb7 */ + 0, /* 0xb8 */ + 0, /* 0xb9 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xba */ + 0, /* 0xbb */ + 0, /* 0xbc */ + 0, /* 0xbd */ + 0, /* 0xbe */ + 0, /* 0xbf */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc0 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc1 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc2 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc3 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc4 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc5 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc6 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc7 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc8 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc9 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xca */ + CPROP_ALPHA |CPROP_LETTER, /* 0xcb */ + CPROP_ALPHA |CPROP_LETTER, /* 0xcc */ + CPROP_ALPHA |CPROP_LETTER, /* 0xcd */ + CPROP_ALPHA |CPROP_LETTER, /* 0xce */ + CPROP_ALPHA |CPROP_LETTER, /* 0xcf */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd0 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd1 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd2 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd3 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd4 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd5 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd6 */ + 0, /* 0xd7 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd8 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd9 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xda */ + CPROP_ALPHA |CPROP_LETTER, /* 0xdb */ + CPROP_ALPHA |CPROP_LETTER, /* 0xdc */ + CPROP_ALPHA |CPROP_LETTER, /* 0xdd */ + CPROP_ALPHA |CPROP_LETTER, /* 0xde */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xdf */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe0 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe1 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe2 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe3 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe4 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe5 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe6 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe7 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe8 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe9 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xea */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xeb */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xec */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xed */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xee */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xef */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf0 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf1 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf2 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf3 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf4 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf5 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf6 */ + 0, /* 0xf7 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf8 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf9 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xfa */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xfb */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xfc */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xfd */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xfe */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER /* 0xff */ +}; + +#define IBM1047 1047 +#define UTF8 1208 + +#define UNI_TAB 0x09 +#define UNI_LF 0x0A +#define UNI_CR 0X0D +#define UNI_SPACE 0x20 +#define UNI_BANG 0x21 +#define UNI_DQUOTE 0x22 +#define UNI_HASH 0x23 +#define UNI_DOLLAR 0x24 +#define UNI_PERCENT 0x25 +#define UNI_AMP 0x26 +#define UNI_SQUOTE 0x27 +#define UNI_LPAREN 0x28 +#define UNI_RPAREN 0x29 +#define UNI_STAR 0x2A +#define UNI_PLUS 0x2B +#define UNI_COMMA 0x2C +#define UNI_DASH 0x2D +#define UNI_DOT 0x2E +#define UNI_SLASH 0x2F +#define UNI_COLON 0x3A +#define UNI_SEMI 0x3B +#define UNI_LESS 0x3C +#define UNI_EQUAL 0x3D +#define UNI_GREATER 0x3E +#define UNI_QMARK 0x3F +#define UNI_AT 0x40 +#define UNI_LBRACK 0x5B +#define UNI_BSLASH 0x5C +#define UNI_RBRACK 0x5D +#define UNI_CARET 0x5E +#define UNI_UNDER 0x5F +#define UNI_BQUOTE 0x60 +#define UNI_LBRACE 0x7B +#define UNI_VBAR 0x7C +#define UNI_RBRACE 0x7D +#define UNI_TILDE 0x7E + +typedef struct AbstractTokenizer_tag { + int lowTokenID; + int highTokenID; + int lastTokenStart; + int lastTokenEnd; + int (*nextToken)(struct AbstractTokenizer_tag *tokenizer, char *s, int len, int pos, int *nextPos); + char *(*getTokenIDName)(int tokenID); + int (*getTokenJsonType)(int tokenID); /* if not specified, you get a string! */ +} AbstractTokenizer; + +bool testCharProp(int ccsid, int c, int prop); + +/* Common Parsing infrastructure */ + +#define FIRST_GRULE_ID 10000 + +#define G_END 0 +#define G_SEQ 1 +#define G_ALT 2 +#define G_STAR 3 +#define G_PLUS 4 + +#define G_MAX_SUBS 12 + +typedef struct GRuleSpec_tag { + int type; + int id; + union { + struct { int subs[G_MAX_SUBS]; } sequence; + struct { int subs[G_MAX_SUBS]; } alternates; + struct { int sub; } star; + struct { int sub; } plus; + }; +} GRuleSpec; + +typedef struct GRuleRef_tag { + struct GRule_tag *parent; + struct GRule_tag *rule; + int indexInParent; + int tokenID; + bool isTokenRef; +} GRuleRef; + +typedef struct GRule_tag { + int type; + int id; + int subCount; + GRuleRef *refs; +} GRule; + + + +#define BUILD_OBJECT 1 +#define BUILD_ARRAY 2 +#define BUILD_STRING 3 +#define BUILD_NUMBER 4 +#define BUILD_INT 5 +#define BUILD_INT64 6 +#define BUILD_DOUBLE 7 +#define BUILD_BOOL 8 +#define BUILD_NULL 9 +#define BUILD_KEY 10 +#define BUILD_POP 11 +#define BUILD_ALT 12 + + +#define MAX_BUILD_STEPS 10000 /* who in hell knows? JK, actually should be a parameter to GParse() */ +#define STEP_UNDEFINED_POSITION (-1) + +typedef struct GBuildStep_tag { + int type; /* BUILD_xxx above */ + GRule *rule; + int tokenID; + char *keyInParent; + int valueStart; /* index into parse source */ + int valueEnd; /* index into parse source */ + GRuleRef *altChoice; +} GBuildStep; + +typedef struct GParseContext_tag { + AbstractTokenizer *tokenizer; + GRule *rules; + char *s; + int len; + ShortLivedHeap *slh; + char *(*ruleNamer)(int id); + int buildStepsSize; + int buildStepCount; + GBuildStep *buildSteps; + int status; + int traceLevel; + int errorPosition; + char *errorMessage; +} GParseContext; + +typedef struct SyntaxNode_tag { + int type; /* owned by the user of this facility */ + int childCapacity; + int childCount; + struct SyntaxNode_tag **children; +} SyntaxNode; + +typedef struct SyntaxTree_tag { + ShortLivedHeap slh; + SyntaxNode *topNode; +} SyntaxTree; + +typedef struct GParseResult_tag { + SyntaxTree *tree; +} GParseResult; + +GParseContext *gParse(GRuleSpec *ruleSet, int topRuleID, AbstractTokenizer *tokenizer, char *s, int len, + char *(*ruleName)(int id), + int traceLevel); + +Json *gBuildJSON(GParseContext *ctx, ShortLivedHeap *slh); + + +/* These definitions are for JLexer, a flexible tokenizer for java/javascript-ish syntaxes */ + +typedef struct JQTokenizer_tag { + AbstractTokenizer tokenizer; /* cheesy "subclassing" idiom */ + int ccsid; + int tokenState; /* = STATE_INITIAL; */ + char *data; + int length; + int pos; +} JQTokenizer; + +#define JTOKEN_IDENTIFIER FIRST_REAL_TOKEN_ID +#define JTOKEN_INTEGER (FIRST_REAL_TOKEN_ID+1) +#define JTOKEN_DQUOTE_STRING FIRST_REAL_TOKEN_ID+2 +#define JTOKEN_SQUOTE_STRING FIRST_REAL_TOKEN_ID+3 +#define JTOKEN_LPAREN FIRST_REAL_TOKEN_ID+4 +#define JTOKEN_RPAREN FIRST_REAL_TOKEN_ID+5 +#define JTOKEN_LBRACK FIRST_REAL_TOKEN_ID+6 +#define JTOKEN_RBRACK FIRST_REAL_TOKEN_ID+7 +#define JTOKEN_LBRACE FIRST_REAL_TOKEN_ID+8 +#define JTOKEN_RBRACE FIRST_REAL_TOKEN_ID+9 +#define JTOKEN_DOT FIRST_REAL_TOKEN_ID+10 +#define JTOKEN_COMMA FIRST_REAL_TOKEN_ID+11 +#define JTOKEN_VBAR FIRST_REAL_TOKEN_ID+12 +#define JTOKEN_COLON FIRST_REAL_TOKEN_ID+13 +#define JTOKEN_QMARK FIRST_REAL_TOKEN_ID+14 +#define JTOKEN_PLUS FIRST_REAL_TOKEN_ID+15 +#define JTOKEN_DASH FIRST_REAL_TOKEN_ID+16 +#define JTOKEN_STAR FIRST_REAL_TOKEN_ID+17 +#define JTOKEN_SLASH FIRST_REAL_TOKEN_ID+18 +#define JTOKEN_PERCENT FIRST_REAL_TOKEN_ID+19 + +#define LAST_JTOKEN_ID JTOKEN_PERCENT /* KEEP ME UPDATED!!! - whenever a new token ID is added*/ + +char *getJTokenName(int id); +/* testing interface */ +int jqtToken(JQTokenizer *jqt); +/* proper interface for use in GParser */ +int getNextJToken(AbstractTokenizer *tokenizer, char *s, int len, int pos, int *nextPos); +int getJTokenJsonType(int tokenID); + + +#endif + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/h/qsam.h b/h/qsam.h index 221c7370e..19c9634d2 100644 --- a/h/qsam.h +++ b/h/qsam.h @@ -117,15 +117,15 @@ #define DCB_RECFM_FBS 0x98 #define DCB_RECFM_VBS 0x58 -typedef __packed struct dcbExtension{ /* Aka, the DCBE */ +typedef _Packed struct dcbExtension{ /* Aka, the DCBE */ char id[4]; short int len; char rsvd006[2]; void * __ptr32 dcb; unsigned int rela; - __packed union { + _Packed union { char flg1; /* Set by OPEN processing */ - __packed struct { + _Packed struct { int open : 1; int md31 : 1; int slbi : 1; @@ -133,9 +133,9 @@ typedef __packed struct dcbExtension{ /* Aka, the DCBE */ int rsvd010_4 : 4; }; }; - __packed union { + _Packed union { char flg2; /* Set by requestor prior to OPEN */ - __packed struct { + _Packed struct { int rmode31 : 1; int pasteod : 1; int rsvd011_2 : 1; @@ -147,9 +147,9 @@ typedef __packed struct dcbExtension{ /* Aka, the DCBE */ }; }; short int nstr; - __packed union { + _Packed union { char flg3; /* Set by requestor prior to OPEN */ - __packed struct { + _Packed struct { int large : 1; int fixedbuf: 1; int eadscb : 1; @@ -170,8 +170,8 @@ typedef __packed struct dcbExtension{ /* Aka, the DCBE */ } DCBExtension; /* map dcbDevice at 0 size is 20 */ -typedef __packed struct dcbDevice{ - __packed struct dcbExtension * __ptr32 dcbe; +typedef _Packed struct dcbDevice{ + _Packed struct dcbExtension * __ptr32 dcbe; char unmapped[16]; /* A complex, device type sensitive, portion of the DCB that we do not need */ } DCBDevice; @@ -179,7 +179,7 @@ typedef __packed struct dcbDevice{ #define DCB_COMMON_OFFSET 0x14 #define VB_RECORD_OFFSET 4 -typedef __packed struct dcbCommon{ +typedef _Packed struct dcbCommon{ /* OFFSET 0x14 */ char bufno; /* 0 if no pool */ int bufcb:24; /* buffer pool control block = 000001 if no pool */ @@ -221,7 +221,7 @@ typedef __packed struct dcbCommon{ /* map dcbBBQCommon at 52 (0x34) BSAM/BPAM/QSAM common - size 20 */ #define DCB_BBQ_COMMON_OFFSET 0x34 -typedef __packed struct dcbBBQCommon { +typedef _Packed struct dcbBBQCommon { char optionCodes; /* OPTCD */ int check:24; /* or internal QSAM ERROR RTN */ int synad; /* synchns err rtn addr, =1 if not set */ @@ -237,7 +237,7 @@ typedef __packed struct dcbBBQCommon { /* map dcbBsamBpamInterface at 72 (0x48) */ #define DCB_BSAM_BPAM_OFFSET 0x48 -typedef __packed struct dcbBsamBpamInterface{ +typedef _Packed struct dcbBsamBpamInterface{ char ncp; /* MAX NUM OF OUTSTANDING READ/WRITES */ int eobr:24; /* internal AM use */ int eobw; /* internal AM use */ @@ -246,7 +246,7 @@ typedef __packed struct dcbBsamBpamInterface{ int cnp; /* control, note, point */ } DCBBBQInterface; -typedef __packed struct { +typedef _Packed struct { DCBDevice Device; DCBCommon Common; DCBBBQCommon BBQCommon; @@ -256,7 +256,7 @@ typedef __packed struct { /* There is lots of code that adds 4 bytes to skip past the DCB open list that preceeds the DCB. Do not declare additional variables prior to the DCB. */ -typedef __packed struct { +typedef _Packed struct { union { DCBSAM * __ptr32 OpenCloseList; unsigned int OpenCloseListOptions; @@ -279,7 +279,7 @@ typedef __packed struct { #define OPEN_CLOSE_EXTEND 0x0E000000 #define OPEN_CLOSE_OUTINX 0x06000000 -typedef __packed struct { +typedef _Packed struct { char flag1; char flag2; char sens0; @@ -295,7 +295,7 @@ typedef __packed struct { char seek[8]; } IOB; -typedef __packed struct DECB_tag{ +typedef _Packed struct DECB_tag{ int ecb; char type1; /* 0x80 - "S" coded for length */ char type2; /* 0x80 is a suggestion */ diff --git a/h/recovery.h b/h/recovery.h index ec7d91850..fc131e241 100644 --- a/h/recovery.h +++ b/h/recovery.h @@ -295,7 +295,7 @@ typedef struct RecoveryContext_tag { char sdumpxSaveArea[72]; } RecoveryContext; -typedef struct RecoveryStatePool_tag RecoveryStatePool; +struct RecoveryStatePool_tag; ZOWE_PRAGMA_PACK_RESET @@ -401,7 +401,7 @@ int recoveryEstablishRouter(int flags); * of the RC_RCV_xxxx error codes. *****************************************************************************/ int recoveryEstablishRouter2(RecoveryContext *userContext, - RecoveryStatePool *userStatePool, + struct RecoveryStatePool_tag *userStatePool, int flags); /***************************************************************************** @@ -415,7 +415,7 @@ int recoveryEstablishRouter2(RecoveryContext *userContext, * Return value: * State pool structure on success or NULL on failure. *****************************************************************************/ -RecoveryStatePool *recoveryMakeStatePool(unsigned int stateCount); +struct RecoveryStatePool_tag *recoveryMakeStatePool(unsigned int stateCount); /***************************************************************************** * Remove a user state pool. @@ -428,7 +428,7 @@ RecoveryStatePool *recoveryMakeStatePool(unsigned int stateCount); * Return value: * N/A *****************************************************************************/ -void recoveryRemoveStatePool(RecoveryStatePool *statePool); +void recoveryRemoveStatePool(struct RecoveryStatePool_tag *statePool); #endif /* RCVR_CPOOL_STATES */ diff --git a/h/unixfile.h b/h/unixfile.h index 58d8560ae..0cef65145 100644 --- a/h/unixfile.h +++ b/h/unixfile.h @@ -269,8 +269,8 @@ typedef struct BPXYATT_tag { /* End of Version 1 */ int fileFormat:8; int reserved2:24; - int fileTagCCSID:16; - int fileTagFlags:16; + unsigned int fileTagCCSID:16; + unsigned int fileTagFlags:16; char reserved3[8]; /* End of Version 2 */ int64 accessTime2; diff --git a/h/xlate.h b/h/xlate.h index 291d28b1d..0adab9ce7 100644 --- a/h/xlate.h +++ b/h/xlate.h @@ -70,6 +70,25 @@ extern "C" char* e2a(char *buffer, int len); char* a2e(char *buffer, int len); + +typedef struct CharsetOracle_tag { + int total; + int charBuckets[0x100]; + int rangeBuckets[0x10]; +} CharsetOracle; + +#define CHARSET_ORACLE_EBCDIC_FAMILY 0x10000 +#define CHARSET_ORACLE_UTF_FAMILY 0x20000 + +/** A Charset oracle is a string digester that can guess charsets + based upon what it is fed. It's heuristic, so YMMV. +*/ + +CharsetOracle *makeCharsetOracle(void); + +void freeCharsetOracle(CharsetOracle *oracle); +void charsetOracleDigest(CharsetOracle *oracle, char *s, int len); +int guessCharset(CharsetOracle *oracle, double *confidence); #ifdef __cplusplus } diff --git a/h/yaml2json.h b/h/yaml2json.h new file mode 100644 index 000000000..7af751201 --- /dev/null +++ b/h/yaml2json.h @@ -0,0 +1,10 @@ +#ifndef __ZOWE_YAML2JSON__ +#define __ZOWE_YAML2JSON__ 1 + +#include "yaml.h" + +yaml_document_t *readYAML(const char *filename, char *errorBuf, size_t errorBufSize); +void pprintYAML(yaml_document_t *document); +Json *yaml2JSON(yaml_document_t *document, ShortLivedHeap *slh); + +#endif diff --git a/h/zowetypes.h b/h/zowetypes.h index f4532bc82..52df8f73c 100644 --- a/h/zowetypes.h +++ b/h/zowetypes.h @@ -84,6 +84,7 @@ __ZOWE_COMP_GCC __ZOWE_COMP_VCPP __ZOWE_COMP_XLC __ZOWE_COMP_IXLC +__ZOWE_COMP_CLANG - new in 2022! __ZOWE_64BIT @@ -123,6 +124,10 @@ no ifdef means XLC LE on ZOS, and everything else. This is effectively our "def /* Base macros for Windows */ #ifdef _MSC_VER +#define ZOWE_PRAGMA_PACK +#define ZOWE_PRAGMA_PACK_RESET + + #define __ZOWE_OS_WINDOWS 1 #ifdef _WIN64 @@ -133,11 +138,17 @@ no ifdef means XLC LE on ZOS, and everything else. This is effectively our "def #endif /* _MSC_VER */ + + /* Base Macros for zOS */ #if defined(__MVS__) && (defined (__IBMC__) || defined (__IBMCPP__)) && (defined (_LP64) || defined (_ILP32)) #define __ZOWE_OS_ZOS #define __ZOWE_ARCH_Z +#define __ZOWE_COMP_XLC +#ifdef __clang__ +#define __ZOWE_COMP_XLCLANG /* new in 2022, xlclang defines *BOTH* __IBMC__ *AND* __clang__ */ +#endif #ifdef _LP64 #define __ZOWE_64 @@ -149,6 +160,26 @@ no ifdef means XLC LE on ZOS, and everything else. This is effectively our "def #endif /* __MVS__ __IBMC__ */ + +/* new clang on ZOS cases */ +#if defined(__MVS__) && defined(__clang__) && (defined (_LP64) || defined (_ILP32)) + +#define __ZOWE_OS_ZOS +#define __ZOWE_ARCH_Z +#define __ZOWE_COMP_CLANG + +#ifdef _LP64 +#define __ZOWE_64 +#else +#define __ZOWE_32 +#endif + +/* Structure packing */ +#define ZOWE_PRAGMA_PACK _Pragma ( "pack(packed)" ) +#define ZOWE_PRAGMA_PACK_RESET _Pragma ( "pack(reset)" ) + +#endif /* __MVS__ __clang__ */ + #if defined(_AIX) && (defined (__IBMC__) || defined (__IBMCPP__)) #define __ZOWE_OS_AIX 1 #define __ZOWE_COMP_XLC 1 @@ -165,7 +196,7 @@ no ifdef means XLC LE on ZOS, and everything else. This is effectively our "def #endif /* AIX */ /* Base Macros for GCC, which is usually Linux or Unix, but Z,P and X architectures can use GCC so analysis is more complex */ -#ifdef __GNUC__ +#if defined(__GNUC__) && !defined(__clang__) #ifdef _LP64 #define __ZOWE_64 @@ -249,6 +280,9 @@ typedef uint64_t uint64; #define INT64_INT(x) ((int)x) #define INT_INT64(x) ((int64)x) #define CHARPTR_INT64(x) ((int64)x) +#define INT2PTR(x) ((void*)((int64_t)(x))) +#define INT64_LL(x) ((long long)(x)) +#define UINT64_ULL(x) ((unsigned long long)(x)) #else @@ -258,6 +292,9 @@ typedef unsigned long long uint64; #define INT64_INT(x) ((int)x) #define INT_INT64(x) ((int64)x) #define CHARPTR_INT64(x) ((int64)((int)x)) +#define INT2PTR(x) ((void*)(x)) +#define INT64_LL(x) ((long long)(x)) +#define UINT64_ULL(x) ((unsigned long long)(x)) #endif diff --git a/platform/README.md b/platform/README.md new file mode 100644 index 000000000..847fe030c --- /dev/null +++ b/platform/README.md @@ -0,0 +1,15 @@ +# Platform Notes + +The zowe-common-c library is primarily driven by the need to have C programming interfaces for +making ZOS programming on C less painful regarding OS aware features. However, the many parts +of this library can run on other platforms, which makes development and testing easier. Non-ZOS +platform support files go in sub-directories here. Since xlc and it's runtime libraries are +highly POSIX compliant, there is not too much needed for Linux and other Unix's. Windows needs +more support for quirky non-compatible IO, systems calls, time functions, regular-expressions, etc. + +## CPP or not to be + +C++ is not used on ZOS, because ZOWE-common-c targets both LE (Posix) runtime environents and Metal +(traditional MVS) environments. IBM's xlc C++ does not support Metal and probably never will. So C++ +is out in general. However, to provide platform support on Windows, C++ is needed because API's are +often *only* published in C++. diff --git a/platform/posix/psxregex.c b/platform/posix/psxregex.c new file mode 100644 index 000000000..a9a8a4578 --- /dev/null +++ b/platform/posix/psxregex.c @@ -0,0 +1,32 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#include "psxregex.h" +#include "zowetypes.h" +#include "alloc.h" + +regex_t *regexAlloc(){ + return (regex_t*)safeMalloc(sizeof(regex_t),"regex_t"); +} + +void regexFree(regex_t *r){ + regfree(r); + safeFree((char*)r,sizeof(regex_t)); +} + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/platform/posix/psxregex.h b/platform/posix/psxregex.h new file mode 100644 index 000000000..bc0cbdeaf --- /dev/null +++ b/platform/posix/psxregex.h @@ -0,0 +1,32 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#ifndef __ZOWE_PSXREGEX__ +#define __ZOWE_PSXREGEX__ + +#include + +regex_t *regexAlloc(); +void regexFree(regex_t *m); + +#define regexComp regcomp +#define regexExec regexec + +#endif + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/platform/windows/cppregex.cpp b/platform/windows/cppregex.cpp new file mode 100644 index 000000000..fb1efaa33 --- /dev/null +++ b/platform/windows/cppregex.cpp @@ -0,0 +1,54 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#include "cppregex.h" +#include + +CPPRegex::CPPRegex() +{ + /* value = start; */ +} + +int CPPRegex::compile(const char *pattern, int cflags) +{ + try { + rx.assign(pattern,std::regex::ECMAScript); + } catch (const std::regex_error& e){ + return e.code(); + } + return 0; +} + +enum reg_errcode_t { + REG_NOERROR = 0, REG_NOMATCH, REG_BADPAT, REG_ECOLLATE, + REG_ECTYPE, REG_EESCAPE, REG_ESUBREG, REG_EBRACK, + REG_EPAREN, REG_EBRACE, REG_BADBR, REG_ERANGE, + REG_ESPACE, REG_BADRPT, REG_EEND, REG_ESIZE, + REG_ERPAREN +}; + +bool CPPRegex::exec(const char *first, const char *last, std::cmatch& matchResults){ + try { + bool status = regex_search(first, last, matchResults, rx); + return status; + } catch (const std::regex_error& e){ + return e.code(); + } +} + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/platform/windows/cppregex.h b/platform/windows/cppregex.h new file mode 100644 index 000000000..7106a903a --- /dev/null +++ b/platform/windows/cppregex.h @@ -0,0 +1,37 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#ifndef __CPPREGEX_H__ +#define __CPPREGEX_H__ + +#include + +class CPPRegex +{ + public: + CPPRegex(); + int compile(const char *pat, int cflags); + bool exec(const char *first, const char *last, std::cmatch& results); + + private: + std::regex rx; +}; + +#endif // __CPPEGEX_H__ + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/platform/windows/winregex.cpp b/platform/windows/winregex.cpp new file mode 100644 index 000000000..7eadf5e69 --- /dev/null +++ b/platform/windows/winregex.cpp @@ -0,0 +1,82 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#include +#include "winregex.h" +#include "cppregex.h" + + +regex_t *regexAlloc(){ + regex_t *m; + CPPRegex *obj; + + m = (decltype(m))malloc(sizeof(*m)); + obj = new CPPRegex(); + m->obj = obj; + + return m; +} + +void regexFree(regex_t *m){ + if (m == NULL) + return; + delete static_cast(m->obj); + free(m); +} + +int regexComp(regex_t *r, const char *pat, int cflags){ + CPPRegex *obj; + + if (r == NULL){ + return REG_E_UNKNOWN; + } + + obj = static_cast(r->obj); + return obj->compile(pat,cflags); +} + +int regexExec(regex_t *r, const char *str, size_t nmatch, + regmatch_t *pmatch, int eflags){ + CPPRegex *obj; + std::cmatch matchResults; + + if (r == NULL){ + return REG_E_UNKNOWN; + } + + obj = static_cast(r->obj); + // here + // 1 worry about additional arguments vs. linue DIE + // 2 get pmatch data structure to fill in. + bool matched = obj->exec(str,str+strlen(str),matchResults); + if (matched){ + for (int i=0; i + +/* regcomp() cflags */ + +#define REG_EXTENDED 0x001 /* Use Extended RE syntax rules */ +#define REG_ICASE 0x002 /* Ignore case in match */ +#define REG_NEWLINE 0x004 /* Convert to + */ +#define REG_NOSUB 0x008 /* regexec() not report + subexpressions */ + + +/* regexec() eflags */ + +#define REG_NOTBOL 0x100 /* First character not start of line */ +#define REG_NOTEOL 0x200 /* Last character not end of line */ + + +/* Regular Expression error codes */ + +#define REG_NOMATCH 1 /* RE pattern not found */ +#define REG_BADPAT 2 /* Invalid Regular Expression */ +#define REG_ECOLLATE 3 /* Invalid collating element */ +#define REG_ECTYPE 4 /* Invalid character class */ +#define REG_EESCAPE 5 /* Last character is \ */ +#define REG_ESUBREG 6 /* Invalid number in \digit */ +#define REG_EBRACK 7 /* imbalance */ +#define REG_EPAREN 8 /* \( \) or () imbalance */ +#define REG_EBRACE 9 /* \{ \} or { } imbalance */ +#define REG_BADBR 10 /* Invalid \{ \} range exp */ +#define REG_ERANGE 11 /* Invalid range exp endpoint */ +#define REG_ESPACE 12 /* Out of memory */ +#define REG_BADRPT 13 /* ?*+ not preceded by valid RE */ +#define REG_ECHAR 14 /* invalid multibyte character */ +#define REG_EBOL 15 /* ^ anchor and not BOL */ +#define REG_EEOL 16 /* $ anchor and not EOL */ +#define REG_EMPTY 17 +#define REG_E_MEMORY 15 +#define REG_E_UNKNOWN 21 + + /* + static const reg_error_t REG_BADPAT = 2; // Invalid pattern. + static const reg_error_t REG_BADPAT = 2; // Invalid pattern. + static const reg_error_t REG_ECOLLATE = 3; // Undefined collating element. + static const reg_error_t REG_ECTYPE = 4; // Invalid character class name. + static const reg_error_t REG_EESCAPE = 5; // Trailing backslash. + static const reg_error_t REG_ESUBREG = 6; // Invalid back reference. + static const reg_error_t REG_EBRACK = 7; // Unmatched left bracket. + static const reg_error_t REG_EPAREN = 8; // Parenthesis imbalance. + static const reg_error_t REG_EBRACE = 9; // Unmatched \{. + static const reg_error_t REG_BADBR = 10; // Invalid contents of \{\}. + static const reg_error_t REG_ERANGE = 11; // Invalid range end. + static const reg_error_t REG_ESPACE = 12; // Ran out of memory. + static const reg_error_t REG_BADRPT = 13; // No preceding re for repetition op. + static const reg_error_t REG_EEND = 14; // unexpected end of expression + static const reg_error_t REG_ESIZE = 15; // expression too big + static const reg_error_t REG_ERPAREN = 8; // = REG_EPAREN : unmatched right parenthesis + static const reg_error_t REG_EMPTY = 17; // empty expression + static const reg_error_t REG_E_MEMORY = 15; // = REG_ESIZE : out of memory + static const reg_error_t REG_ECOMPLEXITY = 18; // complexity too high + static const reg_error_t REG_ESTACK = 19; // out of stack space + static const reg_error_t REG_E_PERL = 20; // Perl (?...) error + static const reg_error_t REG_E_UNKNOWN = 21; // unknown error + static const reg_error_t REG_ENOSYS = 21; // = REG_E_UNKNOWN : Reserved. + */ + + +typedef struct winregex { + void *obj; +} regex_t; + +typedef int64_t regoff_t; + +typedef struct regmatch{ + regoff_t rm_so; /* should be regoff_t */ + regoff_t rm_eo; +} regmatch_t; + +regex_t *regexAlloc(); +void regexFree(regex_t *m); + +int regexComp(regex_t *r, const char *pattern, int cflags); +int regexExec(regex_t *r, const char *str, size_t nmatch, + regmatch_t *pmatch, int eflags); + +#ifdef __cplusplus +} +#endif + +#endif /* __WINREGEX_H__ */ + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 000000000..26fd55f6a --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,90 @@ +################################################################################ +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. +################################################################################ + +# Makefile for z/OS +# make prepare && make +CC:=xlclang +BITS:= +#BITS:=lp64 + +ifeq ($(BITS),lp64) + ENHANCED_ASCII:=-qascii -D_ENHANCED_ASCII_EXT=0xFFFFFFFF +else + ENHANCED_ASCII:=-qascii -D_ENHANCED_ASCII_EXT=0xFFFFFFFF -Wc,NOXPLINK +endif + +MAJOR:=0 +MINOR:=2 +PATCH:=5 +VERSION:=\"$(MAJOR).$(MINOR).$(PATCH)\" +DEFINESYAML:=-DYAML_VERSION_MAJOR=$(MAJOR) -DYAML_VERSION_MINOR=$(MINOR) -DYAML_VERSION_PATCH=$(PATCH) -DYAML_VERSION_STRING=$(VERSION) -DYAML_DECLARE_STATIC=1 +DEFINES:=-D_XOPEN_SOURCE=600 -D_OPEN_THREADS=1 -D_OPEN_SYS_FILE_EXT=1 -DCONFIG_VERSION=\"2021-03-27\" + +CC_FLAGS:=-Iquickjs-portable -Ilibyaml/include -I../h -I../platform/posix $(DEFINES) -Wc,dll,expo,langlvl\(extc99\),gonum,goff,hgpr,roconst,ASM,asmlib\('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN'\) -Wc,agg,list\(\),so\(\),off,xref,$(BITS) +LD_FLAGS:=-Wl,$(BITS) + +LIBYAMLOBJS:=api.o reader.o scanner.o parser.o loader.o writer.o emitter.o dumper.o +SCHEMATESTOBJS:=schematest.o yaml2json.o jsonschema.o json.o xlate.o charsets.o bpxskt.o logging.o collections.o timeutls.o timeutls.o utils.o alloc.o embeddedjs.o zosfile.o zos.o le.o scheduling.o recovery.o psxregex.o +QUICKJSOBS:=cutils.o quickjs.o quickjs-libc.o libunicode.o libregexp.o polyfill.o +CONFIGMGROBJ:=configmgr.o yaml2json.o jsonschema.o json.o xlate.o charsets.o bpxskt.o logging.o collections.o timeutls.o timeutls.o utils.o alloc.o embeddedjs.o zosfile.o zos.o le.o scheduling.o recovery.o psxregex.o + +.PHONY: clean all prepare test_configmgr + +all: configmgr schematest + +configmgr: $(CONFIGMGROBJ) $(LIBYAMLOBJS) $(QUICKJSOBS) + $(CC) $(LD_FLAGS) -o $@ $^ + +schematest: $(SCHEMATESTOBJS) $(LIBYAMLOBJS) $(QUICKJSOBS) + $(CC) $(LD_FLAGS) -o $@ $^ + +schematest.o: schematest.c + $(CC) $(CC_FLAGS) -c schematest.c + +%.o: ../c/%.c + $(CC) $(CC_FLAGS) -c $< + +%.o: ../platform/posix/%.c + $(CC) $(CC_FLAGS) -c $< + +%.o: libyaml/src/%.c + $(CC) $(CC_FLAGS) $(DEFINESYAML) $(ENHANCED_ASCII) -c $< + +%.o: quickjs-portable/%.c + $(CC) $(CC_FLAGS) $(DEFINESYAML) $(ENHANCED_ASCII) -c $< + +%.o: quickjs-portable/porting/%.c + $(CC) $(CC_FLAGS) $(DEFINESYAML) $(ENHANCED_ASCII) -c $< + +libyaml: + git clone git@github.com:yaml/libyaml.git + +quickjs-portable: + git clone git@github.com:JoeNemo/quickjs-portable.git + +prepare: libyaml quickjs-portable + +clean: + rm -f configmgr schematest *.o + +test_configmgr: configmgr + ./configmgr -s ./schemadata -p 'FILE(./schemadata/zoweoverrides.yaml):FILE(./schemadata/zowebase.yaml)' validate + ./configmgr -s ./schemadata -p 'FILE(./schemadata/zoweoverrides.yaml):FILE(./schemadata/zowebase.yaml)' env out.env && cat out.env + + +################################################################################ +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. +################################################################################ \ No newline at end of file diff --git a/tests/charsetguesser.c b/tests/charsetguesser.c new file mode 100644 index 000000000..e1d2b0bff --- /dev/null +++ b/tests/charsetguesser.c @@ -0,0 +1,78 @@ +#include +/* #include */ + +#include +#include +#include +#include + +#ifdef NDEBUG +#undef NDEBUG +#endif +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "unixfile.h" +#include "json.h" +#include "xlate.h" + + +/* + Notes: + + Windows Build ______________________________ + + Assuming clang is installed + + clang -I%YAML%/include -I./src -I../h -I ../platform/windows -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -o charsetguesser.exe charsetguesser.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c + + ZOS Build _________________________________ + + + + Running the Test ________________________________ + + charsetguesser + + */ + +int main(int argc, char *argv[]) +{ + char *filename = argv[1]; + UnixFile *in = NULL; + int maxChars = 1000; + int returnCode = 0; + int reasonCode = 0; + + + in = fileOpen(filename, FILE_OPTION_READ_ONLY, 0, 1024, &returnCode, &reasonCode); + /* int status = (in ? fileInfo(filename, &info, &returnCode, &reasonCode) : 12); */ + if (in != NULL){ /* && status == 0) { */ + CharsetOracle *oracle = makeCharsetOracle(); + + for (int i=0; i maxChars){ + break; + } + char c = (char)(b&0xff); + charsetOracleDigest(oracle,&c,1); + } + fileClose(in,&returnCode,&reasonCode); + double confidence; + int charset = guessCharset(oracle,&confidence); + printf("charset guess=0x%08x, confidence=%f\n",charset,confidence); + for (int i=0; i<16; i++){ + printf(" 0x%02X: %d\n",i<<4,oracle->rangeBuckets[i]); + } + freeCharsetOracle(oracle); + } else { + printf("could not open %s ret=%d reason=0x%x\n",filename,returnCode,reasonCode); + } + + return 0; +} + diff --git a/tests/jqtest.c b/tests/jqtest.c new file mode 100644 index 000000000..d311c0ca9 --- /dev/null +++ b/tests/jqtest.c @@ -0,0 +1,130 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#ifdef METTLE + +#include +#include +#include +#include +#include +#include +#include +#include +#include "metalio.h" +#include "qsam.h" + +#else + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 600 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#endif + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "openprims.h" +#include "bpxnet.h" +#include "unixfile.h" +#include "json.h" +#include "parsetools.h" +#include "microjq.h" + +/* + _Compiling on Windows_ + + clang -I../platform/windows -I ..\h -Wdeprecated-declarations -D_CRT_SECURE_NO_WARNINGS -o jqtest.exe jqtest.c ../c/microjq.c ../c/parsetools.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c + + */ + +static void pprintJSON(Json *tree){ +#ifdef __ZOWE_OS_WINDOWS + int stdoutFD = _fileno(stdout); +#else + int stdoutFD = STDOUT_FILENO; +#endif + if (tree){ + jsonPrinter *p = makeJsonPrinter(stdoutFD); + jsonEnablePrettyPrint(p); + printf("parse result as json\n"); + jsonPrint(p,tree); + } +} + +int main(int argc, char **argv){ + char *command = argv[1]; + char *filter = argv[2]; + char *filename = argv[3]; + JQTokenizer jqt; + memset(&jqt,0,sizeof(JQTokenizer)); + jqt.data = argv[2]; + jqt.length = strlen(jqt.data); + jqt.ccsid = 1208; + ShortLivedHeap *slh = makeShortLivedHeap(0x10000, 100); + if (!strcmp(command,"parse")){ + printf("parse...\n"); + Json *tree = parseJQ(&jqt,slh,1); + if (tree){ + pprintJSON(tree); + } else { + printf("Parse failed\n"); + } + } else if (!strcmp(command,"eval") || + !strcmp(command,"evalTrace") || + !strcmp(command,"evalCompact")){ + Json *jqTree = parseJQ(&jqt,slh,0); + if (jqTree){ + int errorBufferSize = 1024; + char *errorBuffer = safeMalloc(errorBufferSize,"ErrorBuffer"); + memset(errorBuffer,0,errorBufferSize); + Json *json = jsonParseFile2(slh,filename,errorBuffer,errorBufferSize); + if (json == NULL){ + printf("json parsed json=0x%p, with error '%s'\n",json,errorBuffer); + return 8; + } + int traceLevel = 0; + int flags = JQ_FLAG_PRINT_PRETTY; + if (!strcmp(command,"evalTrace")){ + traceLevel = 1; + } + if (!strcmp(command,"evalRaw")){ + flags |= JQ_FLAG_RAW_STRINGS; + } + if (!strcmp(command,"evalCompact")){ + flags ^= JQ_FLAG_PRINT_PRETTY; + } + int evalStatus = evalJQ(json,jqTree,stdout,flags,traceLevel); + if (evalStatus != 0){ + fprintf(stderr,"micro jq eval problem %d\n",evalStatus); + } + } else { + printf("Parse failed\n"); + } + } else { + printf("bad command: %s\n",command); + } + return 0; +} diff --git a/tests/jsonmergetest.c b/tests/jsonmergetest.c new file mode 100644 index 000000000..dc26dfd57 --- /dev/null +++ b/tests/jsonmergetest.c @@ -0,0 +1,101 @@ +#include +#include +#include +#include +#include + +#ifdef NDEBUG +#undef NDEBUG +#endif +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "json.h" + +/* + Notes: + + (all work assumed to be done from shell in this directory) + + Windows Build ______________________________ + + + clang -I./src -I../h -I ../platform/windows -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -o jsonmergetest.exe jsonmergetest.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc + + + ZOS Build _________________________________ + + + + + Running the Test ________________________________ + + jsonschematest + + */ + +int main(int argc, char *argv[]) +{ +#ifdef __ZOWE_OS_WINDOWS + int stdoutFD = _fileno(stdout); +#else + int stdoutFD = STDOUT_FILENO; +#endif + + if (argc < 4){ + printf("Usage jsonmergetest \n"); + printf(" ArrayMergePolicy: \n"); + printf(" concat: take all base elements followed by all overrides - len(merge) = len(o)+len(b)\n"); + printf(" merge: take from both arrays favoring overrides - len(merge) = max(len(o),len(b))\n"); + printf(" base: take base array only - len(merge) = len(b)\n"); + printf(" overrides: take overrides array only - len(merge) = len(a)\n"); + } + char *arrayPolicy = argv[1]; + char *overridesFile = argv[2]; + char *baseFile = argv[3]; + int flags = 0; + if (!strcmp(arrayPolicy,"concat")){ + flags = JSON_MERGE_FLAG_CONCATENATE_ARRAYS; + } else if (!strcmp(arrayPolicy,"merge")){ + flags = JSON_MERGE_FLAG_MERGE_IN_PLACE; + } else if (!strcmp(arrayPolicy,"base")){ + flags = JSON_MERGE_FLAG_TAKE_BASE; + } else if (!strcmp(arrayPolicy,"overrides")){ + flags = JSON_MERGE_FLAG_TAKE_OVERRIDES; + } else { + printf("unknown array merge policy, '%s'\n",arrayPolicy); + } + int errorBufferSize = 1024; + char *errorBuffer = safeMalloc(errorBufferSize,"ErrorBuffer"); + memset(errorBuffer,0,errorBufferSize); + + ShortLivedHeap *slh = makeShortLivedHeap(0x10000, 100); + + Json *base = jsonParseFile2(slh,baseFile,errorBuffer,errorBufferSize); + if (base == NULL){ + printf("Json parse fail in %s, with error '%s'\n",baseFile,errorBuffer); + return 12; + } + printf("read base\n"); fflush(stdout); + Json *overrides = jsonParseFile2(slh,overridesFile,errorBuffer,errorBufferSize); + if (overrides == NULL){ + printf("Json parse fail in %s, with error '%s'\n",overridesFile,errorBuffer); + return 12; + } + printf("read overrides\n"); fflush(stdout); + jsonPrinter *p = makeJsonPrinter(stdoutFD); + jsonEnablePrettyPrint(p); + int status = 0; + Json *merged = jsonMerge(slh,overrides,base,flags,&status); + printf("merged\n"); fflush(stdout); + if (merged){ + jsonPrint(p,merged); + fflush(stdout); + } else { + printf("failed to merge, status = %d\n",status); + } + return 0; +} + diff --git a/tests/jstest.c b/tests/jstest.c new file mode 100644 index 000000000..f42fef944 --- /dev/null +++ b/tests/jstest.c @@ -0,0 +1,153 @@ +#include +#include +#include +#include +#include + +#include "cutils.h" +#include "quickjs-libc.h" + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "json.h" +#include "yaml2json.h" +#include "embeddedjs.h" + +/* + + set QJS=c:\repos\quickjs + + set YAML=c:\repos\libyaml ## Wherever you git clone'd libyaml + + clang++ -c ../platform/windows/cppregex.cpp ../platform/windows/winregex.cpp + + clang -I %QJS%/porting -I%YAML%/include -I../platform/windows -I %QJS% -I ..\h -DCONFIG_VERSION=\"2021-03-27\" -D_CRT_SECURE_NO_WARNINGS -Dstrdup=_strdup -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 -Wdeprecated-declarations --rtlib=compiler-rt -o jstest.exe jstest.c %QJS%\quickjs.c %QJS%\cutils.c %QJS%\quickjs-libc.c %QJS%\libbf.c %QJS%\libregexp.c %QJS%\libunicode.c %QJS%\porting\winpthread.c %QJS%\porting\wintime.c %QJS%\porting\windirent.c %QJS%\porting\winunistd.c %QJS%\repl.c %YAML%/src/api.c %YAML%/src/reader.c %YAML%/src/scanner.c %YAML%/src/parser.c %YAML%/src/loader.c %YAML%/src/writer.c %YAML%/src/emitter.c %YAML%/src/dumper.c ../c/yaml2json.c ../c/embeddedjs.c ../c/jsonschema.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c cppregex.o winregex.o + + + A software configuration is a virtual hierarchical document + - representable as JSON in memory (Javascript data structures, more precisely) + - not necessarily in one source document + - can be cloned with modifications for HA or Provisioning Needs + - can be semantically validated as a pre-flight mechanism to launch components, or from config tools + - not all things can be validate, but most can + + The ConfigManager is the single interface to the software configuation + - it is callable/embeddabl from all points + - Windows, Linux, ZOS at minimum + - JavaScript,C Java Bindings + - output to STDOUT for scripting + - it is embodied in a staticly linked executable that is part of the base software install + - like JDBC client. You get the data and query semantics w/o exactly knowing where every + - it is bootstrapped from a minimal set of schemas and file system paths + - it self-augments through property values that bring in more schemas and path elements + - it provides output of its state as environment variables for components that are + not highly configuration + - is THE ONLY means of ZOWE component learning its parameters, outside of legacy components that read envvars + + A configuration source is a user (or UI) created document + - is owned by the user + - is text + - "No Dark Matter" - all source/paths must be known + - Zowe-wide Changes to sources/path must be reviewed + - can be place under Version Control, eg Git + - is not modified by installers, launchers, etc. + - can be syntactically validated and prompted for syntax in editors + - configuration sources have template capabilities, ie are not static data structures, + - can always co-reference other parts of same source + - can reference after merge/overlay any part of schema + - can be evaluated with additional args/globals defined at different times in the InstallationTarget's lifecycle + - SSL may be an exception to these rules - + + An RunTarget is the unit software instance request + - RunTargets can require other IT's by name and version. + - RunTargets have sections of the SoftwareConfiguration that probably come from separate ConfigSource + - RunTargets form a directed acyclic graph (like Make targets, OSGI Modules, NPM Packages) to be instantiale + - RunTargets have a lifecycle - + initialInstall + upgrade + clone() + start + kill (maybe, for optional cleanup) + uninstall + */ + + +int main(int argc, char **argv){ + int i; + int include_count = 0; + char *command = argv[1]; + int loadMode = FILE_LOAD_AUTODETECT; + int errorBufferSize = 1024; +#ifdef __ZOWE_OS_WINDOWS + int stdoutFD = _fileno(stdout); +#else + int stdoutFD = STDOUT_FILENO; +#endif + char *errorBuffer = safeMalloc(errorBufferSize,"ErrorBuffer"); + + printf("JSTest Start \n"); + fflush(stdout); + + EmbeddedJS *ejs = makeEmbeddedJS(NULL); + + + /* no includes yet + for(i = 0; i < include_count; i++) { + if (eval_file(ctx, include_list[i], module)) + goto fail; + } + */ + int evalStatus = 0; + + /* JOE, here, eval buffer, or file, or go interactive */ + if (!strcmp(command,"expr")){ + char *sourceCode = argv[2]; + JSValue output = ejsEvalBuffer(ejs, sourceCode, strlen(sourceCode), "", 0, &evalStatus); + if (evalStatus){ + printf("evaluation failed, status=%d\n",evalStatus); + } else { + ejsFreeJSValue(ejs,output); + } + } else if (!strcmp(command,"file")){ + char *filename = argv[2]; + if (ejsEvalFile(ejs, filename, loadMode)){ + printf("file eval failed\n"); + } + } else if (!strcmp(command,"yamlcopy") || + !strcmp(command,"yamleval")){ + char *filename = argv[2]; + ShortLivedHeap *slh = makeShortLivedHeap(0x10000, 100); + yaml_document_t *doc = readYAML(filename, errorBuffer, errorBufferSize); + if (doc == NULL){ + printf("yaml file is bad: %s\n",errorBuffer); + return 0; + } + Json *json = yaml2JSON(doc,slh); + jsonPrinter *p = makeJsonPrinter(stdoutFD); + jsonEnablePrettyPrint(p); + printf("Yaml file as json\n"); + jsonPrint(p,json); + + if (!strcmp(command,"yamlcopy")){ + JSValue jsValue = ejsJsonToJS(ejs,json); + printf("made jsValue\n"); + fflush(stdout); + ejsSetGlobalProperty(ejs,"magic1",jsValue); + char *s1 = "console.log(\"theThing=\"+JSON.stringify(magic1))"; + JSValue throwAway = ejsEvalBuffer(ejs, s1, strlen(s1), "", 0, &evalStatus); + printf("evalStatus=%d\n",evalStatus); + Json *json2 = ejsJSToJson(ejs,jsValue,slh); + printf("JSON translated back to zowe-common-c.json\n"); + jsonPrint(p,json2); + } else { + Json *resultantJSON = evaluateJsonTemplates(ejs,slh,json); + printf("JSON templates evaluated\n"); + jsonPrint(p,resultantJSON); + } + } + + printf("JSTest done\n"); + fflush(stdout); + return 0; +} diff --git a/tests/parsetest.c b/tests/parsetest.c new file mode 100644 index 000000000..517d9ad21 --- /dev/null +++ b/tests/parsetest.c @@ -0,0 +1,215 @@ +#include +#include +#include +#include +#include +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "json.h" +#include "parsetools.h" + +/* + + clang -I../h -I../platform/windows -D_CRT_SECURE_NO_WARNINGS -Dstrdup=_strdup -DYAML_DECLARE_STATIC=1 -Wdeprecated-declarations --rtlib=compiler-rt -o parsetest.exe parsetest.c ../c/parsetools.c ../c/json.c ../c/winskt.c ../c/xlate.c ../c/charsets.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c + + Grammar + (PHRASEREF n) + (TOKENREF n) + (ALT n) + (SEQ n) + (STAR ) + (PLUS n) + + */ + +static int myNextToken(AbstractTokenizer *tokenizer, char *s, int len, int pos, int *nextPos){ + if (pos < len){ + char c = s[pos]; + *nextPos = (pos+1); + return c&0xFF; + } else { + *nextPos = pos; + return EOF_TOKEN; + } +} + +static char *myTokenIDName(int id){ + // leaky + switch(id){ + case EOF_TOKEN: return ""; + case NO_VALID_TOKEN: return ""; + default: + { + char *buffer = safeMalloc(8,"Token ID"); + snprintf(buffer,8,"<%c>",id); + return buffer; + } + } +} + +#define TOP (FIRST_GRULE_ID+0) +#define R1 (FIRST_GRULE_ID+1) +#define R2 (FIRST_GRULE_ID+2) +#define EE (FIRST_GRULE_ID+3) +#define EEE (FIRST_GRULE_ID+4) +#define BBB (FIRST_GRULE_ID+5) + +static GRuleSpec testSet1[] = { + { G_SEQ, TOP, .sequence = { (int)'A', R1, (int)'D', (int)EOF_TOKEN, G_END} }, + { G_ALT, R1, .alternates = { (int)'B', (int)'C', G_END}}, + { G_END } +}; + + +static GRuleSpec testSet2[] = { + { G_SEQ, TOP, .sequence = { (int)'A', R1, (int)'D', (int)EOF_TOKEN, G_END} }, + { G_ALT, R1, .alternates = { EEE, EE, G_END}}, + { G_SEQ, EEE, .sequence = { (int)'E', (int)'E', (int)'E', G_END}}, + { G_SEQ, EE, .sequence = { (int)'E', (int)'E', G_END}}, + { G_END } +}; + +static GRuleSpec testSet3[] = { + { G_SEQ, TOP, .sequence = { (int)'A', R1, (int)'D', (int)EOF_TOKEN, G_END} }, + { G_STAR, R1, .star = R2 }, + { G_ALT, R2, .alternates = { BBB, EE, G_END}}, + { G_SEQ, BBB, .sequence = { (int)'B', (int)'B', (int)'B', G_END}}, + { G_SEQ, EE, .sequence = { (int)'E', (int)'E', G_END}}, + { G_END } +}; + +static char *getTestSetRuleName(int id){ + switch (id){ + case TOP: return "TOP"; + case R1: return "R1"; + case R2: return "R2"; + case EE: return "EE"; + case EEE: return "EEE"; + case BBB: return "BBB"; + default: + return "UNKNOWN_RULE"; + } +} + + + +static int old_main(int argc, char **argv){ + AbstractTokenizer tokenizer; + tokenizer.lowTokenID = 1; + tokenizer.highTokenID = 127; + tokenizer.nextToken = myNextToken; + tokenizer.getTokenIDName = myTokenIDName; + GParseContext *ctx = gParse(testSet3,TOP,&tokenizer,argv[1],strlen(argv[1]),getTestSetRuleName); + return 0; +} + +static void tokenTest(JQTokenizer *jqt){ + AbstractTokenizer *tokenizer = (AbstractTokenizer*)jqt; + printf("length = %d\n",jqt->length); + while (true){ + int tokenID = jqtToken(jqt); + printf("token id=%d, %s pos is now=%d\n",tokenID,getJTokenName(tokenID),jqt->pos); + printf(" from %d to %d\n",tokenizer->lastTokenStart,tokenizer->lastTokenEnd); + if (tokenID == NO_VALID_TOKEN){ + printf("bad token seen near %d\n",jqt->pos); + break; + } else if (tokenID == EOF_TOKEN){ + printf("EOF Seen\n"); + break; + } + } +} + +#define TOP (FIRST_GRULE_ID+0) +#define EXPR (FIRST_GRULE_ID+1) +#define EXPR_TAIL (FIRST_GRULE_ID+2) +#define EXPR_ADD_SUB (FIRST_GRULE_ID+3) +#define PLUS_MINUS (FIRST_GRULE_ID+4) +#define TERM (FIRST_GRULE_ID+5) +#define TERM_TAIL (FIRST_GRULE_ID+6) +#define TERM_MULT_DIV (FIRST_GRULE_ID+7) +#define STAR_SLASH (FIRST_GRULE_ID+8) +#define FACTOR (FIRST_GRULE_ID+9) + +static char *getExprRuleName(int id){ + switch (id){ + case TOP: return "TOP"; + case EXPR: return "EXPR"; + case EXPR_TAIL: return "EXPR_TAIL"; + case EXPR_ADD_SUB: return "EXPR_ADD_SUB"; + case PLUS_MINUS: return "PLUS_MINUS"; + case TERM: return "TERM"; + case TERM_TAIL: return "TERM_TAIL"; + case TERM_MULT_DIV: return "TERM_MULT_DIV"; + case STAR_SLASH: return "STAR_SLASH"; + case FACTOR: return "FACTOR"; + default: + return "UNKNOWN_RULE"; + } +} + +static GRuleSpec expressionGrammar[] = { + { G_SEQ, TOP, .sequence = { EXPR, EOF_TOKEN, G_END} }, + { G_SEQ, EXPR, .sequence = { TERM, EXPR_TAIL, G_END }}, + { G_STAR, EXPR_TAIL, .star = EXPR_ADD_SUB }, + { G_SEQ, EXPR_ADD_SUB, .sequence = { PLUS_MINUS, TERM, G_END }}, + { G_ALT, PLUS_MINUS, .alternates = { JTOKEN_PLUS, JTOKEN_DASH, G_END }}, + { G_SEQ, TERM, .sequence = { FACTOR, TERM_TAIL, G_END }}, + { G_STAR, TERM_TAIL, .star = TERM_MULT_DIV }, + { G_SEQ, TERM_MULT_DIV, .sequence = { STAR_SLASH, FACTOR, G_END}}, + { G_ALT, STAR_SLASH, .alternates = { JTOKEN_STAR, JTOKEN_SLASH, G_END }}, + { G_ALT, FACTOR, .alternates = { JTOKEN_IDENTIFIER, JTOKEN_INTEGER, G_END }}, /* add expression recursion later */ + { G_END } +}; + + +static void parseTest1(JQTokenizer *jqt){ + AbstractTokenizer *tokenizer = (AbstractTokenizer*)jqt; + tokenizer->lowTokenID = FIRST_REAL_TOKEN_ID; + tokenizer->highTokenID = LAST_JTOKEN_ID; + tokenizer->nextToken = getNextJToken; + tokenizer->getTokenIDName = getJTokenName; + GParseContext *ctx = gParse(expressionGrammar,TOP,tokenizer,jqt->data,jqt->length,getExprRuleName); + printf("parse ctx = 0x%p\n",ctx); + if (ctx->status > 0){ + #ifdef __ZOWE_OS_WINDOWS + int stdoutFD = _fileno(stdout); +#else + int stdoutFD = STDOUT_FILENO; +#endif + ShortLivedHeap *slh = makeShortLivedHeap(0x10000, 100); + Json *tree = gBuildJSON(ctx,slh); + if (tree){ + jsonPrinter *p = makeJsonPrinter(stdoutFD); + jsonEnablePrettyPrint(p); + printf("parse result as json\n"); + jsonPrint(p,tree); + } + } +} + +int main(int argc, char **argv){ + char *command = argv[1]; + char *filter = argv[2]; + char *filename = argv[3]; + JQTokenizer jqt; + memset(&jqt,0,sizeof(JQTokenizer)); + jqt.data = argv[2]; + jqt.length = strlen(jqt.data); + jqt.ccsid = 1208; + if (!strcmp(command,"tokenize")){ + printf("tokenize...\n"); + tokenTest(&jqt); + } else if (!strcmp(command,"parse")){ + printf("parse...\n"); + parseTest1(&jqt); + } else { + printf("bad command: %s\n",command); + } + return 0; +} + + diff --git a/tests/schemadata/bundle1.json b/tests/schemadata/bundle1.json new file mode 100644 index 000000000..81247f3ad --- /dev/null +++ b/tests/schemadata/bundle1.json @@ -0,0 +1,6 @@ +{ + "A": "apple", + "listenerPort": 344444, + "BB": "FOO", + "Z": "606094" +} diff --git a/tests/schemadata/example-zowe.yaml b/tests/schemadata/example-zowe.yaml new file mode 100644 index 000000000..ee6f585ac --- /dev/null +++ b/tests/schemadata/example-zowe.yaml @@ -0,0 +1,613 @@ +################################################################################ +# This program and the accompanying materials are made available under the terms of the +# Eclipse Public License v2.0 which accompanies this distribution, and is available at +# https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. +################################################################################ + +#=============================================================================== +# This is a YAML configuration file for Zowe instance. +# +# YAML is a human-friendly data serialization language for all programming languages. +# To learn more about YAML specifications, please check https://yaml.org/. +# +# To learn more details about the entries, please check https://docs.zowe.org/. +# +# For first time users, or for the most common use cases, please pay more +# attention on the entries marked with "**COMMONLY_CUSTOMIZED**". +# +# If you modify any settings listed in "zwe init --help" command, you may need to +# re-run "zwe init" command to make them take effect. +#=============================================================================== + +#------------------------------------------------------------------------------- +# Zowe global configurations +# +# This section includes Zowe setup information used by `zwe install` and +# `zwe init` command, as well as default configurations for Zowe runtime. +#------------------------------------------------------------------------------- +zowe: + + #------------------------------------------------------------------------------- + # These configurations are used by "zwe install" or "zwe init" commands. + #------------------------------------------------------------------------------- + setup: + # MVS data set related configurations + mvs: + # **COMMONLY_CUSTOMIZED** + # where Zowe MVS data sets will be installed + # hlq: IBMUSER.ZWEV2 + hlq: IBMUSER.F$$ + # **COMMONLY_CUSTOMIZED** + # PROCLIB where Zowe STCs will be copied over + proclib: USER.PROCLIB + # **COMMONLY_CUSTOMIZED** + # Zowe PARMLIB + parmlib: IBMUSER.ZWEV2.CUST.PARMLIB + # **COMMONLY_CUSTOMIZED** + # JCL library where Zowe will store temporary JCLs during initialization + jcllib: IBMUSER.ZWEV2.CUST.JCLLIB + # APF authorized LOADLIB for Zowe + # Optional. If it's empty, .SZWEAUTH will be APF authorized. + authLoadlib: + # **COMMONLY_CUSTOMIZED** + # APF authorized LOADLIB for Zowe ZIS Plugins + authPluginLib: IBMUSER.ZWEV2.CUST.ZWESAPL + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # # Security related configurations. This setup is optional. + # security: + # # security product name. Can be RACF, ACF2 or TSS + # product: RACF + # # security group name + # groups: + # # Zowe admin user group + # admin: ZWEADMIN + # # Zowe STC group + # stc: ZWEADMIN + # # Zowe SysProg group + # sysProg: ZWEADMIN + # # security user name + # users: + # # Zowe runtime user name of main service + # zowe: ZWESVUSR + # # Zowe runtime user name of Cross Memory Server + # xmem: ZWESIUSR + # # Zowe runtime user name of Auxilary Service + # aux: ZWESIUSR + # # STC names + # stcs: + # # STC name of Zowe main service + # zowe: ZWESLSTC + # # STC name of Zowe Cross Memory Server + # xmem: ZWESISTC + # # STC name of Zowe Auxilary Service + # aux: ZWESASTC + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # Certificate related configurations + # + # There are 5 configurations cases. Please choose one from below. + + # >>>> Certificate setup scenario 1 + # PKCS12 (keystore) with Zowe generate certificates. + certificate: + # Type of certificate storage. Valid values are: PKCS12 or JCERACFKS + type: PKCS12 + pkcs12: + # **COMMONLY_CUSTOMIZED** + # Keystore directory + directory: /var/zowe/keystore + # **COMMONLY_CUSTOMIZED** + # # Certificate alias name. Optional, default value is localhost. + # # Note: please use all lower cases as alias. + # name: localhost + # **COMMONLY_CUSTOMIZED** + # # Keystore password. Optional, default value is password. + # password: password + # **COMMONLY_CUSTOMIZED** + # # Alias name of self-signed certificate authority. Optional, default value is local_ca. + # # Note: please use all lower cases as alias. + # caAlias: local_ca + # **COMMONLY_CUSTOMIZED** + # # Password of keystore stored self-signed certificate authority. Optional, default value is local_ca_password. + # caPassword: local_ca_password + # # Distinguished name for Zowe generated certificates. All optional. + # dname: + # caCommonName: "" + # commonName: "" + # orgUnit: "" + # org: "" + # locality: "" + # state: "" + # country: "" + # # Validity days for Zowe generated certificates + # validity: 3650 + # # Domain names and IPs should be added into certificate SAN + # # If this field is not defined, `zwe init` command will use + # # `zowe.externalDomains`. + # san: + # - dvipa.my-company.com + # # sample IP address + # - 12.34.56.78 + + # # >>>> Certificate setup scenario 2 + # # PKCS12 (keystore) with importing certificate generated by other CA. + # certificate: + # # Type of certificate storage. Valid values are: PKCS12 or JCERACFKS + # type: PKCS12 + # pkcs12: + # # **COMMONLY_CUSTOMIZED** + # # Keystore directory + # directory: /var/zowe/keystore + # # # Certificate alias name. Optional, default value is localhost. + # # # Note: please use all lower cases as alias. + # # name: localhost + # # # Keystore password. Optional, default value is password. + # # password: password + # import: + # # **COMMONLY_CUSTOMIZED** + # # Existing PKCS12 keystore which holds the certificate issued by external CA. + # keystore: /var/zowe/extcerts/dummy_certs.keystore.p12 + # # **COMMONLY_CUSTOMIZED** + # # Password of the above keystore + # password: dummycert + # # **COMMONLY_CUSTOMIZED** + # # Certificate alias will be imported + # # Note: please use all lower cases as alias. + # alias: dummy_certs + # # **COMMONLY_CUSTOMIZED** + # # PEM format certificate authorities will also be imported and trusted. + # importCertificateAuthorities: + # # This should be the certificate authority signed the certificate will be imported. + # - /var/zowe/extcerts/dummy_ca.cer + + # # >>>> Certificate setup scenario 3 + # # JCERACFKS (z/OS Keyring) with Zowe generated certificates. + # certificate: + # # Type of certificate storage. Valid values are: PKCS12 or JCERACFKS + # type: JCERACFKS + # keyring: + # # **COMMONLY_CUSTOMIZED** + # # keyring name + # name: Zowe2Keyring + # # **COMMONLY_CUSTOMIZED** + # # # Label of Zowe certificate. Optional, default value is localhost. + # # label: Primary Zowe v2 Cert + # # **COMMONLY_CUSTOMIZED** + # # # label of Zowe CA certificate. Optional, default value is localca. + # # caLabel: Zowe v2 CA + # # # Distinguished name for Zowe generated certificates. All optional. + # # dname: + # # caCommonName: "" + # # commonName: "" + # # orgUnit: "" + # # org: "" + # # locality: "" + # # state: "" + # # country: "" + # # # Validity days for Zowe generated certificates + # # validity: 3650 + # # # Domain names and IPs should be added into certificate SAN + # # # If this field is not defined, `zwe init` command will use + # # # `zowe.externalDomains`. + # # # **NOTE**: due to the limitation of RACDCERT command, this field should + # # # contain exactly 2 entries with the domain name and IP address. + # # san: + # # - dvipa.my-company.com + # # - 12.34.56.78 + + # # >>>> Certificate setup scenario 4 + # # JCERACFKS (z/OS Keyring) and connect to existing certificate + # certificate: + # # Type of certificate storage. Valid values are: PKCS12 or JCERACFKS + # type: JCERACFKS + # keyring: + # # **COMMONLY_CUSTOMIZED** + # # keyring name + # name: Zowe2Keyring + # connect: + # # **COMMONLY_CUSTOMIZED** + # # Current owner of the existing certificate, can be SITE or an user ID. + # user: IBMUSER + # # **COMMONLY_CUSTOMIZED** + # # Label of the existing certificate will be connected to Zowe keyring. + # label: Dummy Cert + # # **COMMONLY_CUSTOMIZED** + # # If you have other certificate authorities want to be trusted in Zowe keyring, + # # list the certificate labels here. + # # **NOTE**, due to the limitation of RACDCERT command, this field should + # # contain maximum 2 entries. + # importCertificateAuthorities: + # - Dummy CA + + # # >>>> Certificate setup scenario 5 + # # JCERACFKS (z/OS Keyring) with importing certificate stored in data set + # certificate: + # # Type of certificate storage. Valid values are: PKCS12 or JCERACFKS + # type: JCERACFKS + # keyring: + # # **COMMONLY_CUSTOMIZED** + # # keyring name + # name: Zowe2Keyring + # # **COMMONLY_CUSTOMIZED** + # # # Label of Zowe certificate. Optional, default value is localhost. + # # label: Primary Zowe v2 Cert + # import: + # # **COMMONLY_CUSTOMIZED** + # # Name of the data set holds the certificate issued by other CA. + # # This data set should be in PKCS12 format and contain private key. + # dsName: IBMUSER.DUMMY.CERTP12 + # # **COMMONLY_CUSTOMIZED** + # # Password for the PKCS12 data set. + # password: dummycert + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # VSAM configurations if you are using VSAM as Caching Service storage + vsam: + # VSAM data set with Record-Level-Sharing enabled or not + # Valid values could be: NONRLS or RLS. + mode: NONRLS + # Volume name if you are using VSAM in NONRLS mode + volume: VOL123 + # Storage class name if you are using VSAM in RLS mode + storageClass: + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # **COMMONLY_CUSTOMIZED** + # Zowe runtime (root) directory + # + # **NOTE**: if it's not specified and you passed "--update-config" argument + # when you run "zwe init" command, this value will be updated with the Zowe + # runtime where the "zwe" command is located. + # + # This value is required by ZWESLSTC to know where is Zowe runtime. + runtimeDirectory: "" + + # **COMMONLY_CUSTOMIZED** + # Where to store runtime logs + logDirectory: /global/zowe/instance/logs + + # **COMMONLY_CUSTOMIZED** + # Zowe runtime workspace directory + workspaceDirectory: /global/zowe/instance/workspace + + # **COMMONLY_CUSTOMIZED** + # Where extensions are installed + extensionDirectory: /global/zowe/extensions + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # runtime z/OS job name + job: + # Zowe JES job name + name: ZWE1SV$ + # Prefix of component address space + prefix: ZWE1 + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # **COMMONLY_CUSTOMIZED** + # You can list your external domains how you want to access Zowe. + # This should be the domain list you would like to put into your web browser + # address bar. + externalDomains: + # this should be the domain name of you Dynamic VIP Address (DVIPA) + - dvipa.my-company.com + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # This is the port you use to access Zowe Gateway from your web browser. + # + # In many use cases, this should be same as `components.gateway.port`. But in + # some use cases, like containerization, this port could be different. + externalPort: 65236 + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # You can define any extra environment variables as key/value pairs here + environments: + # Example of a global environment variable for all components + # MY_ENV_VAR: my_env_val + + # Another example to customize SSH port for VT Terminal Desktop app + # ZWED_SSH_PORT: 22 + # ZWED_TN3270_PORT: 23 + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # Enable debug mode for zowe launch scripts + launchScript: + # set to "debug" or "trace" to display extra debug information + logLevel: "" + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # Default Zowe certificate + # + # **NOTE**: these fields can be updated automatically if you pass + # `--update-config` to `zwe init` command. The generated value will base on + # your setup in `zowe.setup.certificate` section. + certificate: + keystore: + type: PKCS12 + file: /global/zowe/keystore/localhost/localhost.keystore.p12 + password: password + alias: localhost + truststore: + type: PKCS12 + file: /global/zowe/keystore/localhost/localhost.truststore.p12 + password: password + pem: + key: /global/zowe/keystore/localhost/localhost.key + certificate: /global/zowe/keystore/localhost/localhost.cer + certificateAuthorities: /global/zowe/keystore/local_ca/local_ca.cer + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # How we want to verify SSL certificates of services. Valid values are: + # - STRICT: will validate if the certificate is trusted in our trust store and + # if the certificate Command Name and Subject Alternative Name (SAN) + # is validate. This is recommendate for the best security. + # - NONSTRICT: will validate if the certificate is trusted in our trust store. + # This mode does not validate certificate Common Name and Subject + # Alternative Name (SAN). + # - DISABLED: disable certificate validation. This is NOT recommendated for + # security + verifyCertificates: STRICT + + +#------------------------------------------------------------------------------- +# Java configuraiton +# +# Some Zowe components requires Java. Define the path where you have your Java +# is installed. +# +# **NOTE**: this field can be updated automatically if you pass `--update-config` +# to `zwe init` command. +#------------------------------------------------------------------------------- +java: + # **COMMONLY_CUSTOMIZED** + # Path to your Java home directory + home: "" + + +#------------------------------------------------------------------------------- +# node.js configuraiton +# +# Some Zowe components requires node.js. Define the path where you have your +# node.js is installed. +# +# **NOTE**: this field can be updated automatically if you pass `--update-config` +# to `zwe init` command. +#------------------------------------------------------------------------------- +node: + # **COMMONLY_CUSTOMIZED** + # Path to your node.js home directory + home: "" + + +#------------------------------------------------------------------------------- +# z/OSMF configuraiton +# +# If your Zowe instance is configured to use z/OSMF for authentication or other +# features. You need to define how to access your z/OSMF instance. +#------------------------------------------------------------------------------- +zOSMF: + # **COMMONLY_CUSTOMIZED** + # host name of your z/OSMF instance + host: dvipa.my-company.com + # **COMMONLY_CUSTOMIZED** + port: "443" + applId: IZUDFLT + + +#------------------------------------------------------------------------------- +# Zowe components default configurations +# +# This section includes default configurations for all Zowe components installed +# on the Zowe instance. +# +# Every component should define their own section under `components` with their +# component ID. +# +# For each component, they can always have "enabled" property and "certificate" +# property. More configurations for each component can be found in component +# manifest file. +#------------------------------------------------------------------------------- +components: + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + gateway: + enabled: true + port: 7554 + debug: false + + apiml: + security: + auth: + provider: zosmf + zosmf: + jwtAutoconfiguration: auto + serviceId: zosmf + authorization: + endpoint: + enabled: false + provider: + jwtInitializerTimeout: 5 + x509: + enabled: false + zosmf: + applid: IZUDFLT + service: + allowEncodedSlashes: true + corsEnabled: false + server: + internal: + # gateway supports internal connector + enabled: false + port: 7556 + ssl: + enabled: false + # internal connector can use different certificate + # certificates: + # alias: + + # If we customize this to use different external certificate, than should also + # define "server.internal.ssl.certificate" and enable "server.internal.ssl.enabled". + # certificates: + # alias: + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + metrics-service: + enabled: false + port: 7551 + debug: false + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + api-catalog: + enabled: true + port: 7552 + debug: false + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + discovery: + enabled: true + port: 7553 + debug: false + # Define this value to match your number of Discovery StatefulSet if you are running containerized Zowe + # replicas: 1 + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + caching-service: + enabled: true + port: 7555 + debug: false + + storage: + evictionStrategy: reject + # can be inMemory, VSAM + mode: VSAM + size: 10000 + vsam: + # your VSAM data set created by ZWECSVSM job + # this is required if storage mode is VSAM + name: + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + app-server: + enabled: true + port: 7556 + # we can customize any component with custom certificate + # the missing definitions will be picked from "zowe.certificate" + # certificate: + # keystore: + # alias: app-server + # pem: + # key: /global/zowe/keystore/localhost/localhost.keystore.app-server.key + # certificate: /global/zowe/keystore/localhost/localhost.keystore.app-server.cer-ebcdic + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + zss: + enabled: true + port: 7557 + crossMemoryServerName: ZWESIS_STD + tls: true + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + jobs-api: + enabled: false + port: 7558 + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + files-api: + enabled: false + port: 7559 + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + explorer-jes: + enabled: true + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + explorer-mvs: + enabled: true + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + explorer-uss: + enabled: true + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # Each extension can have dedicated definition similar to core components. + # my-extension: + # enabled: true + + +#------------------------------------------------------------------------------- +# Zowe high availability instances customizations +# +# This section includes customizations for each Zowe high availability instance. +# +# You can start each HA instance with `zwe start --ha-instance `. +#------------------------------------------------------------------------------- +# haInstances: +# +# # HA instance ID +# lpar1: +# # hostname where this instance will be started +# hostname: lpar1.my-company.com +# # Your &SYSNAME for this LPAR +# # This sysname will be used to route your JES command to target system. +# sysname: LPR1 +# # for this HA instance, we didn't customize "components", so it will use default value. + +# # HA instance ID, we will start 2 instances on LPAR2 +# # **NOTE**, we can only start one gateway in same LPAR. +# lpar2a: +# # hostname where this instance will be started +# hostname: lpar2.my-company.com +# # Your &SYSNAME for this LPAR +# # This sysname will be used to route your JES command to target system. +# sysname: LPR2 + +# # These configurations will overwrite highest level default "components" configuration +# components: +# discovery: +# # use customized port on this instance +# port: 17553 +# api-catalog: +# port: 17552 +# app-server: +# # no app-server in this instance +# enabled: false +# zss: +# # no app-server in this instance +# enabled: false +# jobs-api: +# port: 18545 +# enabled: true +# files-api: +# port: 18547 +# enabled: true +# caching-service: +# port: 17555 +# +# lpar2b: +# hostname: lpar2.my-company.com +# # your &SYSNAME for this LPAR +# sysname: LPR2 + +# # These configurations will overwrite highest level default "components" configuration +# components: +# gateway: +# enabled: false +# discovery: +# enabled: false +# api-catalog: +# enabled: false +# app-server: +# enabled: false +# port: 28544 +# zss: +# port: 28542 +# jobs-api: +# enabled: true +# files-api: +# enabled: false +# caching-service: +# enabled: false diff --git a/tests/schemadata/food.schema b/tests/schemadata/food.schema new file mode 100644 index 000000000..1fb95f443 --- /dev/null +++ b/tests/schemadata/food.schema @@ -0,0 +1,42 @@ +{ + "name": "zowe_yaml_schema", + "$id": "http//zowe.org/schemas/yaml-2.0.json", + "type": "object", + "required": [ "fruit", "beverage" ], + "additionalProperties": false, + "patternProperties": { + "^course.$": { "type": "number"}}, + "properties": { + "fruit": { + "allOf": [ + { "type": "string" }, + { "maxLength": 5 } + ] + }, + "vegetable": { + "anyOf": [ + { "type": "number", + "minimum": 13 }, + { "type": "string", + "pattern": "^c.*$"} + ] + }, + "entree": { + "const": "beef", + }, + "carbs": { + "enum": [ "potato", "rice" ], + }, + "beverage": { + "oneOf": [ + { "type": "string", + "pattern": "^.*k$"}, + { "type": "string", + "pattern": "^m.*$"} + ] + }, + "dessert": { + "not": { "type": "boolean" } + } + } +} diff --git a/tests/schemadata/food1.json b/tests/schemadata/food1.json new file mode 100644 index 000000000..a11ce5d57 --- /dev/null +++ b/tests/schemadata/food1.json @@ -0,0 +1,10 @@ +{ + "fruit": "grape", + "vegetable": 33, + "beverage": "millstone", + "carbs": "rice", + "dessert": 3, + "entree": "beef", + "course1": 333, + +} diff --git a/tests/schemadata/yamltypes.yaml b/tests/schemadata/yamltypes.yaml new file mode 100644 index 000000000..85c6c6cad --- /dev/null +++ b/tests/schemadata/yamltypes.yaml @@ -0,0 +1,22 @@ +--- +zowe: + default: + port: 3000 + hlq: "ZWE.PROD" + setup: + certificate: + pkcs12: ~ + type: PKCS12 + mvs: + authLoadlib: ~ + authPluginLib: IBMUSER.ZWEV2.CUST.ZWESAPL + foo: 3 + goforit: true + hay: "Hay \" " + hlq: "${{ zowe.default.hlq+\".BAZ\" }}" + hlq2: ${{ zowe.default.hlq+".BAZ" }} + port: ${{ zowe.default.port+1 }} + jcllib: IBMUSER.ZWEV2.CUST.JCLLIB + parmlib: IBMUSER.ZWEV2.CUST.PARMLIB + proclib: VENDOR.PROCLIB + \ No newline at end of file diff --git a/tests/schemadata/zoweappserver.json b/tests/schemadata/zoweappserver.json new file mode 100644 index 000000000..d22f5aeaa --- /dev/null +++ b/tests/schemadata/zoweappserver.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09", + "$id": "https://zarf.org/schema/appserver", + "allOf": [ + { + "$ref": "https://zowe.org/schema/base" + }, + { + "$ref": "https://zowe.org/schema/common#zoweComponent" + }, + { + "type": "object", + "required": [ + "BB" + ], + "properties": { + "BB": { + "const": "FOO" + } + } + } + ], +} diff --git a/tests/schemadata/zowebase.json b/tests/schemadata/zowebase.json new file mode 100644 index 000000000..b4c2277f8 --- /dev/null +++ b/tests/schemadata/zowebase.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09", + "$id": "https://zowe.org/schema/base", + "type": "object", + "properties": { + "A": { + "type": "string" + }, + "Z": { + "$ref": "/schema/common#zoweZip" + } + } +} + diff --git a/tests/schemadata/zowebundle.json b/tests/schemadata/zowebundle.json new file mode 100644 index 000000000..a84d3e320 --- /dev/null +++ b/tests/schemadata/zowebundle.json @@ -0,0 +1,66 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09", + "$id": "https://zarf.org/schema/appserver", + "allOf": [ + { + "$ref": "https://zowe.org/schema/base" + }, + { + "$ref": "https://zowe.org/schema/common#zoweComponent" + }, + { + "type": "object", + "required": [ + "BB" + ], + "properties": { + "BB": { + "const": "FOO" + } + } + } + ], + "$defs": { + "https://zowe.org/schema/base": { + "$schema": "https://json-schema.org/draft/2019-09", + "$id": "https://zowe.org/schema/base", + "type": "object", + "properties": { + "A": { + "type": "string" + }, + "Z": { + "$ref": "/schema/common#zoweZip" + } + } + }, + "https://zowe.org/schema/common": { + "$schema": "https://json-schema.org/draft/2019-09", + "$id": "https://zowe.org/schema/common", + "$defs": { + "component": { + "type": "object", + "$anchor": "zoweComponent", + "required": [ + "listenerPort" + ], + "properties": { + "listenerPort": { + "$ref": "#zowePort" + } + } + }, + "port": { + "$anchor": "zowePort", + "type": "integer", + "maximum": 65535 + }, + "zipCode": { + "$anchor": "zoweZip", + "type": "string", + "pattern": "^[0-9]{5}$" + } + } + } + } +} diff --git a/tests/schemadata/zowecommon.json b/tests/schemadata/zowecommon.json new file mode 100644 index 000000000..f8f51ff7a --- /dev/null +++ b/tests/schemadata/zowecommon.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09", + "$id": "https://zowe.org/schema/common", + "$defs": { + "component": { + "type": "object", + "$anchor": "zoweComponent", + "required": [ + "listenerPort" + ], + "properties": { + "listenerPort": { + "$ref": "#zowePort" + } + } + }, + "port": { + "$anchor": "zowePort", + "type": "integer", + "maximum": 65535 + }, + "zipCode": { + "$anchor": "zoweZip", + "type": "string", + "pattern": "^[0-9]{5}$" + } + } +} diff --git a/tests/schemadata/zowedefault.yaml b/tests/schemadata/zowedefault.yaml new file mode 100644 index 000000000..8ac2c0ea1 --- /dev/null +++ b/tests/schemadata/zowedefault.yaml @@ -0,0 +1,27 @@ +zowe: + + #------------------------------------------------------------------------------- + # These configurations are used by "zwe install" or "zwe init" commands. + #------------------------------------------------------------------------------- + setup: + # MVS data set related configurations + mvs: + xxx: "FOOBAR" + hlq: ${{ zowe.setup.mvs.xxx+".ZWEV2" }} + proclib: ${{ zowe.setup.mvs.hlq+".PROCLIB" }} + parmlib: IBMUSER.ZWEV2.CUST.PARMLIB + jcllib: IBMUSER.ZWEV2.CUST.JCLLIB + authLoadlib: + authPluginLib: IBMUSER.ZWEV2.CUST.ZWESAPL + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # Certificate related configurations + # + # There are 5 configurations cases. Please choose one from below. + + # >>>> Certificate setup scenario 1 + # PKCS12 (keystore) with Zowe generate certificates. + certificate: + # Type of certificate storage. Valid values are: PKCS12 or JCERACFKS + type: PKCS12 + pkcs12: diff --git a/tests/schemadata/zoweoverrides.yaml b/tests/schemadata/zoweoverrides.yaml new file mode 100644 index 000000000..51b1f1a88 --- /dev/null +++ b/tests/schemadata/zoweoverrides.yaml @@ -0,0 +1,4 @@ +zowe: + setup: + mvs: + jcllib: ${{ zowe.setup.mvs.hlq + ".FRED" }} diff --git a/tests/schemadata/zoweyaml.schema b/tests/schemadata/zoweyaml.schema new file mode 100644 index 000000000..2dd3fc3e4 --- /dev/null +++ b/tests/schemadata/zoweyaml.schema @@ -0,0 +1,72 @@ +{ + "name": "zowe_yaml_schema", + "$id": "http//zowe.org/schemas/yaml-2.0.json", + "type": "object", + "required": [ "zowe" ], + "properties": { + "zowe": { + "type": "object", + + "description": "This is the top-level Zowe Server Configuration Object", + "properties": { + "setup": { + "type": "object", + "properties": { + "mvs": { + "type": "object", + "description": "MVS (ZOS) Dataset names and prefixes", + "properties": { + "proclib": { + "title": "PROCLIB", + "description": "JCL Procedure Library", + "examples": [ "FOO.BAR.BLETCH" ], + "$ref": "#/$defs/dataset" }, + "parmlib": { + "description": "Parameter Library (44-char ZOS dotted name)", + "$ref": "#/$defs/dataset" }, + "hlq": { "$ref": "#/$defs/dataset" } + } + } + } + }, + "job": { + "type": "object", + "required": [ "name", "prefix" ], + "properties": { + "name": { "$ref": "#/$defs/jobname" }, + "prefix": { + "type": "string" + } + } + }, + "externalPort": { + "description": "This must be a valid TCP Port or offset reference", + "$ref": "#/$defs/tcpPort" }, + "logDirectory": { + "description": "Must be a valid USS (Unix-style) path", + "type": "string" + } + } + } + }, + "$defs": { + "dataset": { + "type": "string", + "description": "A 44-char all caps dotted ZOS name", + "pattern": "^([A-Z0-9\\$\\#\\@]){1,8}(\\.([A-Z0-9\\$\\#\\@]){1,8}){0,11}$", + "minLength": 3, + "maxLength": 44 + }, + "jobname": { + "type": "string", + "pattern": "^([A-Z0-9\\$\\#\\@]){1,8}$", + "minLength": 3, + "maxLength": 8 + }, + "tcpPort": { + "type": "integer", + "minimum": 1024, + "maximum": 65535 + } + } +} diff --git a/tests/schematest.c b/tests/schematest.c new file mode 100644 index 000000000..70e046652 --- /dev/null +++ b/tests/schematest.c @@ -0,0 +1,166 @@ +#include +/* #include */ + +#include +#include +#include +#include + +#ifdef NDEBUG +#undef NDEBUG +#endif +#include + +#include "zowetypes.h" + +#ifndef __ZOWE_OS_WINDOWS +#include +#endif + +#include "alloc.h" +#include "utils.h" +#include "json.h" +#include "jsonschema.h" +#include "yaml2json.h" + +/* + Notes: + + (all work assumed to be done from shell in this directory) + + Windows Build ______________________________ + + Assuming clang is installed + + set YAML=c:\repos\libyaml ## Wherever you git clone'd libyaml + + clang++ -c ../platform/windows/cppregex.cpp ../platform/windows/winregex.cpp + + clang -I%YAML%/include -I./src -I../h -I ../platform/windows -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 -o schematest.exe schematest.c %YAML%/src/api.c %YAML%/src/reader.c %YAML%/src/scanner.c %YAML%/src/parser.c %YAML%/src/loader.c %YAML%/src/writer.c %YAML%/src/emitter.c %YAML%/src/dumper.c ../c/yaml2json.c ../c/jsonschema.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c cppregex.o winregex.o + + ZOS Build _________________________________ + + + + + Running the Test ________________________________ + + schematest ? + + schematest yaml schemadata/example-zowe.yaml utf8 schemadata/zoweyaml.schema + */ + +int main(int argc, char *argv[]){ + int number; + char *syntax = argv[1]; + char *filename = argv[2]; + char *charset = argv[3]; + char *schemaFilename = (argc >= 5 ? argv[4] : NULL); + int errorBufferSize = 1024; +#ifdef __ZOWE_OS_WINDOWS + int stdoutFD = _fileno(stdout); +#else + int stdoutFD = STDOUT_FILENO; +#endif + char *errorBuffer = safeMalloc(errorBufferSize,"ErrorBuffer"); + + /* + Arg for charset + functionalize the char tests + materialize the char constants + see if we can parse on other alphabets + "soft" fork to JoeNemo + */ + + if (argc < 4) { + printf(" ? \n"); + return 0; + } + + Json *json = NULL; + ShortLivedHeap *slh = makeShortLivedHeap(0x10000, 100); + + if (!strcmp(syntax,"json")){ + memset(errorBuffer,0,errorBufferSize); + json = jsonParseFile2(slh,filename,errorBuffer,errorBufferSize); + if (json == NULL){ + printf("json parsed json=0x%p, with error '%s'\n",json,errorBuffer); + } + + } else if (!strcmp(syntax,"yaml")){ + printf("yaml syntax case\n"); + fflush(stdout); + yaml_document_t *doc = readYAML(filename, errorBuffer, errorBufferSize); + printf("yaml doc at 0x%p\n",doc); + if (doc){ + pprintYAML(doc); + json = yaml2JSON(doc,slh); + } else { + printf("%s\n", errorBuffer); + } + } else { + printf("unhandled syntax\n"); + } + + if (json){ + printf("Showing internal JSON whether from Yaml, JSON or PARMLIB\n"); + jsonPrinter *p = makeJsonPrinter(stdoutFD); + jsonEnablePrettyPrint(p); + jsonPrint(p,json); + + if (schemaFilename){ + memset(errorBuffer,0,errorBufferSize); + Json *schemaJSON = jsonParseFile2(slh,schemaFilename,errorBuffer,errorBufferSize); + if (schemaJSON == NULL){ + printf("json parsed schema json=0x%p, with error '%s'\n",json,errorBuffer); + fflush(stdout); + } + if (schemaJSON){ + printf("Regurgitating JSON Schema before digesting it for real\n"); + jsonPrint(p,schemaJSON); + printf("\n"); + fflush(stdout); + JsonSchemaBuilder *builder = makeJsonSchemaBuilder(DEFAULT_JSON_SCHEMA_VERSION); + JsonSchema *schema = jsonBuildSchema(builder,schemaJSON); + if (schema){ + JsonValidator *validator = makeJsonValidator(); + printf("Before Validate\n");fflush(stdout); + int validateStatus = jsonValidateSchema(validator,json,schema,NULL,0); + switch (validateStatus){ + case JSON_VALIDATOR_NO_EXCEPTIONS: + printf("No validity Exceptions\n"); + break; + case JSON_VALIDATOR_HAS_EXCEPTIONS: + { + printf("Validity Exceptions:\n"); + ValidityException *e = validator->firstValidityException; + while (e){ + printf(" %s\n",e->message); + e = e->next; + } + } + break; + case JSON_VALIDATOR_INTERNAL_FAILURE: + printf("validation internal failure"); + break; + } + freeJsonValidator(validator); + printf("Done with Validation Test\n"); + fflush(stdout); + } else { + printf("Schema Build Failed: %s\n",builder->errorMessage); + } + freeJsonSchemaBuilder(builder); + } else { + printf("Failed to read schema JSON\n"); + } + } + } else{ + printf("was not able to make JSON from source\n"); + + } + printf("JSON Test end\n"); + fflush(stdout); + return 0; +} + diff --git a/tests/setkeytest.c b/tests/setkeytest.c new file mode 100644 index 000000000..6b293192b --- /dev/null +++ b/tests/setkeytest.c @@ -0,0 +1,42 @@ +#include +#include +#include +#include +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "zos.h" + + +/* + xlclang -q64 -D_OPEN_SYS_FILE_EXT=1 -D_XOPEN_SOURCE=600 -D_OPEN_THREADS=1 -DSUBPOOL=132 "-Wc,float(ieee),longname,langlvl(extc99),gonum,goff,ASM,asmlib('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN')" "-Wl,ac=1" -I ../h -I ../platform/posix -Wbitwise-op-parentheses -o setkeytest setkeytest.c ../c/zos.c ../c/timeutls.c ../c/utils.c ../c/alloc.c + + */ + +static int getPrefix(){ + int32_t prefix = 0; + __asm(" STPX %0 " + : + :"m"(prefix) + :"r15"); + return prefix; +} + + +int main(int argc, char **argv){ + if (argc >= 2 && !strcmp(argv[1],"true")){ + supervisorMode(true); + int oldKey = setKey(0); + int prefix = getPrefix(); + printf("WITH PRIV: this ZOS has prefix 0x%x\n",prefix); + setKey(oldKey); + supervisorMode(false); + printf("oldKey was = %d\n",oldKey); + } else{ + int prefix = getPrefix(); + printf("NO PRIV: this ZOS has prefix 0x%x\n",prefix); + } + return 0; +}