From 6e4dc886a2cca68498bff5c492eccec6fcf38af6 Mon Sep 17 00:00:00 2001 From: joepun76 Date: Thu, 27 Jan 2022 17:53:56 -0500 Subject: [PATCH 01/42] remove datasetjson.c pls refer to zss (PR #400) Signed-off-by: joepun76 --- c/datasetjson.c | 2533 ----------------------------------------------- h/datasetjson.h | 83 -- 2 files changed, 2616 deletions(-) delete mode 100644 c/datasetjson.c delete mode 100644 h/datasetjson.h diff --git a/c/datasetjson.c b/c/datasetjson.c deleted file mode 100644 index 07483263d..000000000 --- a/c/datasetjson.c +++ /dev/null @@ -1,2533 +0,0 @@ - - -/* - This program and the accompanying materials are - made available under the terms of the Eclipse Public License v2.0 which accompanies - this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html - - SPDX-License-Identifier: EPL-2.0 - - Copyright Contributors to the Zowe Project. -*/ - -#ifdef METTLE -/* HAS NOT BEEN COMPILED WITH METTLE BEFORE */ -#else -#include -#include -#include -#include -#include -#include -#include -#include "zowetypes.h" -#include "alloc.h" -#include "utils.h" -#include "json.h" -#include "bpxnet.h" -#include "logging.h" -#include "unixfile.h" -#ifdef __ZOWE_OS_ZOS -#include "zos.h" -#endif - -#include "charsets.h" - -#include "socketmgmt.h" -#include "httpserver.h" -#include "datasetjson.h" -#include "pdsutil.h" -#include "jcsi.h" -#include "impersonation.h" -#include "dynalloc.h" -#include "utils.h" -#include "vsam.h" -#include "qsam.h" -#include "icsf.h" - -#define INDEXED_DSCB 96 - -static char defaultDatasetTypesAllowed[3] = {'A','D','X'}; -static char clusterTypesAllowed[3] = {'C','D','I'}; /* TODO: support 'I' type DSNs */ -static int clusterTypesCount = 3; -static char *datasetStart = "//'"; -static char *defaultCSIFields[] ={ "NAME ", "TYPE ", "VOLSER "}; -static int defaultCSIFieldCount = 3; -static char *defaultVSAMCSIFields[] ={"AMDCIREC", "AMDKEY ", "ASSOC ", "VSAMTYPE"}; -static int defaultVSAMCSIFieldCount = 4; -static char vsamCSITypes[5] = {'R', 'D', 'G', 'I', 'C'}; - -static char getRecordLengthType(char *dscb); -static int getMaxRecordLength(char *dscb); - -const static int DSCB_TRACE = FALSE; - -typedef struct DatasetName_tag { - char value[44]; /* space-padded */ -} DatasetName; - -typedef struct DatasetMemberName_tag { - char value[8]; /* space-padded */ -} DatasetMemberName; - -typedef struct DDName_tag { - char value[8]; /* space-padded */ -} DDName; - -typedef struct Volser_tag { - char value[6]; /* space-padded */ -} Volser; - -static int getVolserForDataset(const DatasetName *dataset, Volser *volser); -static bool memberExists(char* dsName, DynallocMemberName daMemberName); - - -static int getLreclOrRespondError(HttpResponse *response, const DatasetName *dsn, const char *ddPath) { - int lrecl = 0; - - FileInfo info; - int returnCode; - int reasonCode; - FILE *in = fopen(ddPath, "r"); - if (in == NULL) { - respondWithError(response,HTTP_STATUS_NOT_FOUND,"File could not be opened or does not exist"); - return 0; - } - - Volser volser; - memset(&volser.value, ' ', sizeof(volser.value)); - - int volserSuccess = getVolserForDataset(dsn, &volser); - int handledThroughDSCB = FALSE; - - if (!volserSuccess){ - - char dscb[INDEXED_DSCB] = {0}; - int rc = obtainDSCB1(dsn->value, sizeof(dsn->value), - volser.value, sizeof(volser.value), - dscb); - if (rc == 0){ - if (DSCB_TRACE){ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "DSCB for %.*s found\n", sizeof(dsn->value), dsn->value); - dumpbuffer(dscb,INDEXED_DSCB); - } - - lrecl = getMaxRecordLength(dscb); - char recordType = getRecordLengthType(dscb); - if (recordType == 'U'){ - fclose(in); - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"Undefined-length dataset"); - return 0; - } - handledThroughDSCB = TRUE; - } - } - if (!handledThroughDSCB){ - FileInfo info; - fldata_t fileinfo = {0}; - char filenameOutput[100]; - int returnCode = fldata(in,filenameOutput,&fileinfo); - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "FLData request rc=0x%x\n",returnCode); - if (!returnCode) { - if (fileinfo.__recfmU) { - fclose(in); - respondWithError(response, HTTP_STATUS_BAD_REQUEST, - "Undefined-length dataset"); - return 0; - } - lrecl = fileinfo.__maxreclen; - } else { - fclose(in); - respondWithError(response, HTTP_STATUS_INTERNAL_SERVER_ERROR, - "Could not read dataset information"); - return 0; - } - } - fclose(in); - - return lrecl; -} - -/* - TODO this duplicates a lot of stremDataset. Thinking of putting conditionals on streamDataset writing to json stream, but then function becomes misleading. - */ -static char *getDatasetETag(char *filename, int recordLength, int *rc, int *eTagReturnLength) { - -#ifdef __ZOWE_OS_ZOS - int rcEtag = 0; - int eTagLength = 0; - - // Note: to allow processing of zero-length records set _EDC_ZERO_RECLEN=Y - int defaultSize = DATA_STREAM_BUFFER_SIZE; - FILE *in; - if (recordLength < 1){ - recordLength = defaultSize; - in = fopen(filename,"rb"); - } - else { - in = fopen(filename,"rb, type=record"); - } - - ICSFDigest digest; - char hash[32]; - - int bufferSize = recordLength+1; - char buffer[bufferSize]; - int contentLength = 0; - int bytesRead = 0; - if (in) { - rcEtag = icsfDigestInit(&digest, ICSF_DIGEST_SHA1); - if (rcEtag) { //if etag generation has an error, just don't send it. - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_WARNING, "ICSF error for SHA etag init, %d\n",rcEtag); - } else { - while (!feof(in)){ - bytesRead = fread(buffer,1,recordLength,in); - if (bytesRead > 0 && !ferror(in)) { - rcEtag = icsfDigestUpdate(&digest, buffer, bytesRead); - contentLength = contentLength + bytesRead; - } else if (bytesRead == 0 && !feof(in) && !ferror(in)) { - // empty record - } else if (ferror(in)) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Error reading DSN=%s, rc=%d\n", filename, bytesRead); - break; - } - } - } - fclose(in); - if (!rcEtag) { rcEtag = icsfDigestFinish(&digest, hash); } - if (rcEtag) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_WARNING, "ICSF error for SHA etag, %d\n",rcEtag); - } - } - else { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "FAILED TO OPEN FILE\n"); - } - - *rc = rcEtag; - - if (!rcEtag) { - // Convert hash text to hex. - eTagLength = digest.hashLength*2; - *eTagReturnLength = eTagLength; - char *eTag = safeMalloc(eTagLength+1, "datasetetag"); - memset(eTag, '\0', eTagLength+1); - simpleHexPrint(eTag, hash, digest.hashLength); - return eTag; - } - -#else /* not __ZOWE_OS_ZOS */ - - /* Currently nothing else has "datasets" */ - /* TBD: Is it really necessary to provide this empty array? - It seems like the safest approach, not knowing anyting about the client.. - */ - -#endif /* not __ZOWE_OS_ZOS */ - return NULL; -} - -int streamDataset(Socket *socket, char *filename, int recordLength, jsonPrinter *jPrinter){ -#ifdef __ZOWE_OS_ZOS - // Note: to allow processing of zero-length records set _EDC_ZERO_RECLEN=Y - int defaultSize = DATA_STREAM_BUFFER_SIZE; - FILE *in; - if (recordLength < 1){ - recordLength = defaultSize; - in = fopen(filename,"rb"); - } - else { - in = fopen(filename,"rb, type=record"); - } - - int rcEtag; - ICSFDigest digest; - char hash[32]; - - int bufferSize = recordLength+1; - char buffer[bufferSize]; - jsonStartArray(jPrinter,"records"); - int contentLength = 0; - int bytesRead = 0; - if (in) { - rcEtag = icsfDigestInit(&digest, ICSF_DIGEST_SHA1); - if (rcEtag) { //if etag generation has an error, just don't send it. - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_WARNING, "ICSF error for SHA etag init, %d\n",rcEtag); - } - while (!feof(in)){ - bytesRead = fread(buffer,1,recordLength,in); - if (bytesRead > 0 && !ferror(in)) { - if (!rcEtag) { rcEtag = icsfDigestUpdate(&digest, buffer, bytesRead); } - jsonAddUnterminatedString(jPrinter, NULL, buffer, bytesRead); - contentLength = contentLength + bytesRead; - } else if (bytesRead == 0 && !feof(in) && !ferror(in)) { - // empty record - jsonAddString(jPrinter, NULL, ""); - } else if (ferror(in)) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Error reading DSN=%s, rc=%d\n", filename, bytesRead); - break; - } - } - fclose(in); - if (!rcEtag) { rcEtag = icsfDigestFinish(&digest, hash); } - } - else { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "FAILED TO OPEN FILE\n"); - } - - jsonEndArray(jPrinter); - - if (!rcEtag) { - // Convert hash text to hex. - int eTagLength = digest.hashLength*2; - char eTag[eTagLength+1]; - memset(eTag, '\0', eTagLength+1); - simpleHexPrint(eTag, hash, digest.hashLength); - jsonAddString(jPrinter, "etag", eTag); - } - -#else /* not __ZOWE_OS_ZOS */ - - /* Currently nothing else has "datasets" */ - /* TBD: Is it really necessary to provide this empty array? - It seems like the safest approach, not knowing anyting about the client.. - */ - jsonStartArray(jPrinter,"records"); - jsonEndArray(jPrinter); - int contentLength = 0; -#endif /* not __ZOWE_OS_ZOS */ - return contentLength; -} - -int streamVSAMDataset(HttpResponse* response, char *acb, int maxRecordLength, int maxRecords, int maxBytes, - int keyLoc, int keyLen, jsonPrinter *jPrinter) { -#ifdef __ZOWE_OS_ZOS - int defaultSize = DATA_STREAM_BUFFER_SIZE; - - int returnCode = 0; - int reasonCode = 0; - RPLCommon *rpl = NULL; - - if (acb) { - rpl = (RPLCommon *)(acb+RPL_COMMON_OFFSET+8); - } else { - /* TODO: should never happen, but regardless, close out the JSON before this error so it doesn't look weird? */ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "We were not passed an ACB.\n"); - /* respondWithError(response, HTTP_STATUS_INTERNAL_SERVER_ERROR,"Missing ACB"); should really return this */ - return 8; - } - int bufferSize = rpl->bufLen + 1; - int keyBufferSize = rpl->keyLen ? rpl->keyLen + 1 : 4; - char *buffer = safeMalloc(bufferSize, "VSAM buffer"); - char *keyBuffer = safeMalloc(keyBufferSize, "VSAM key buffer"); - memset(buffer,0,bufferSize); - memset(keyBuffer,0,keyBufferSize); - jsonStartArray(jPrinter,"records"); - - int contentLength = 0; - int encodedLength = 0; - int encodedKeyLength = 0; - char *encodedRecord = NULL; - char *encodedKey = NULL; - int bytesRead = 0; - int rbaFound = 0; - int status = 0; - - /* TODO: limit the bytes or records returned if those values are set */ - while (!(rpl->status)) { - status = getRecord(acb, buffer, &bytesRead); - /* TODO: if the user enters a parm to skip the first record, we can call continue in this loop after getRecord, but before JSON gets sent */ - if (bytesRead > bufferSize) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "CRITICAL ERROR: Catalog was wrong about maximum record size.\n"); - jsonAddString(jPrinter, NULL, "_ERROR: Catalog Error. Record found was too large."); - jsonEndArray(jPrinter); - } - memset(buffer+bytesRead,0,bufferSize-bytesRead); - if (rpl->keyLen) { - memcpy(keyBuffer, buffer+keyLoc, keyLen); - keyBuffer[keyLen] = 0; - } else { - rbaFound = *(int *)((rpl->rba)+4); - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "rbaFound = %d\n", rbaFound); - memcpy(keyBuffer, &rbaFound, 4); - } - if (!bytesRead && rpl->status) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Read 0 bytes with error: ACB=%08x, rc=%d, rplRC = %0x%06x\n", acb, status, rpl->status, rpl->feedback); - break; - } else if (rpl->status && (rpl->feedback & 0xFF) == 0x04) { /* TODO: extra eof info to pass into json? */ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "eof found after RBA %d\n", rbaFound); - /* TODO: I suggest we remove our hashtable entry, close the ACB, and free their relative memories here - but confirm this with team first. */ - break; - } else { - contentLength = contentLength + bytesRead; - if (rpl->keyLen) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "found (\"%s\",\"%s\") Total bytes = %d.\n", keyBuffer, buffer, contentLength); - } else{ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "found (0x%08x,\"%s\") Total bytes = %d.\n", *(int *)keyBuffer, buffer, contentLength); - } - encodedRecord = encodeBase64(NULL, buffer, bytesRead, &encodedLength, 1); - encodedKey = encodeBase64(NULL, keyBuffer, keyLen ? keyLen : 4, &encodedKeyLength, 1); - jsonStartObject(jPrinter, NULL); - jsonAddString(jPrinter, "key", encodedKey); - jsonAddString(jPrinter, "record", encodedRecord); - jsonEndObject(jPrinter); - safeFree(encodedRecord, encodedLength); - safeFree(encodedKey, encodedKeyLength); - } - if (rpl->status) zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "New Error: ACB=%08x, rc=%d, rplRC = %02x%06x\n", acb, status, rpl->status, rpl->feedback); - } - - jsonEndArray(jPrinter); - safeFree(buffer, bufferSize); - safeFree(keyBuffer, keyBufferSize); - -#else /* not __ZOWE_OS_ZOS */ - - /* Currently nothing else has VSAM */ - /* TBD: Is it really necessary to provide this empty array? - It seems like the safest approach, not knowing anything about the client.. - */ - jsonStartArray(jPrinter,"records"); - jsonEndArray(jPrinter); - int contentLength = 0; -#endif /* not __ZOWE_OS_ZOS */ - return contentLength; -} - - -static void addDetailsFromDSCB(char *dscb, jsonPrinter *jPrinter, int *isPDS) { -#ifdef __ZOWE_OS_ZOS - int posOffset = 44; - - int recfm = dscb[84-posOffset]; - - jsonStartObject(jPrinter,"recfm"); - { - char recLen; - if ((recfm & 0xC0) == 0xC0){/*undefined*/ - jsonAddString(jPrinter,"recordLength","U"); - recLen = 'U'; - } - else if (recfm & 0x80){/*fixed*/ - jsonAddString(jPrinter,"recordLength","F"); - recLen = 'F'; - } - else if (recfm & 0x40){/*variable*/ - jsonAddString(jPrinter,"recordLength","V"); - recLen = 'V'; - } - - if (recfm & 0x10){/*blocked*/ - jsonAddBoolean(jPrinter,"isBlocked",TRUE); - } - if (recfm & 0x0f){/*standard (or spanned?!)*/ - if (recLen == 'V'){ - jsonAddBoolean(jPrinter,"isSpanned",TRUE); - } - else if (recLen == 'F'){ - jsonAddBoolean(jPrinter,"isStandard",TRUE); - } - } - - if (recfm & 0x04){ - jsonAddString(jPrinter,"carriageControl","ansi"); - } - else if (recfm & 0x02){ - jsonAddString(jPrinter,"carriageControl","machine"); - } - else{ - jsonAddString(jPrinter,"carriageControl","unknown"); - } - } - jsonEndObject(jPrinter); - - - int dsorgHigh = dscb[82-posOffset]; - int isUnmovable = FALSE; - - jsonStartObject(jPrinter,"dsorg"); - { - if (dsorgHigh & 0x40){/*Physical Sequential / PS*/ - jsonAddString(jPrinter,"organization","sequential"); - } - else if (dsorgHigh & 0x20){/*Direct Organization / DA*/ - - } - else if (dsorgHigh & 0x10){/*BTAM or QTAM / CX*/ - - } - else if (dsorgHigh & 0x02){ /*Partitioned / PO*/ - int pdsCheck = dscb[78-posOffset] & 0x0a; - if (pdsCheck == 0x0a){/*pdse & pdsex = hfs*/ - jsonAddString(jPrinter,"organization","hfs"); - } - else{ - *isPDS = TRUE; - jsonAddString(jPrinter,"organization","partitioned"); - jsonAddBoolean(jPrinter,"isPDSDir",TRUE); - if (pdsCheck == 0x08){ /*only pdse = pdse*/ - jsonAddBoolean(jPrinter,"isPDSE",TRUE); - } - else{ - /*is pds*/ - } - } - } - if (dsorgHigh & 0x01){ - isUnmovable = TRUE; - } - - int dsorgLow = dscb[83-posOffset]; - - if (dsorgLow & 0x80){/*Graphics / GS*/ - - } - else if (dsorgLow & 0x08){/*VSAM*/ - jsonAddString(jPrinter,"organization","vsam"); - int keylen = dscb[90-posOffset] << 12; - int keyoffset = dscb[91-posOffset] << 8; - jsonAddInt(jPrinter,"keylen",keylen); - jsonAddInt(jPrinter,"keyoffset",keyoffset); - } - - int lrecl = (dscb[88-posOffset] << 8) | dscb[89-posOffset]; - jsonAddInt(jPrinter,"maxRecordLen",lrecl); - - int blockSize = (dscb[86-posOffset] << 8 | dscb[87-posOffset]); - jsonAddInt(jPrinter,"totalBlockSize",blockSize); - } - jsonEndObject(jPrinter); -#endif /* __ZOWE_OS_ZOS */ -} - -#ifdef __ZOWE_OS_ZOS -/* https://www.ibm.com/support/knowledgecenter/en/SSLTBW_2.1.0/com.ibm.zos.v2r1.idas300/s3013.htm */ -static int isPartionedDataset(char *dscb) { - int posOffset = 44; - int dsorgHigh = dscb[82-posOffset]; - - if (dsorgHigh & 0x02) { - return TRUE; - } - - return FALSE; -} -#endif - -static bool isSupportedWriteDsorg(char *dscb, bool *isPds) { - int posOffset = 44; - int dsorgHigh = dscb[82-posOffset]; - - if (dsorgHigh & 0x40){/*Physical Sequential / PS*/ - return true; - } - else if (dsorgHigh & 0x20){/*Direct Organization / DA*/ - return false; - } - else if (dsorgHigh & 0x10){/*BTAM or QTAM / CX*/ - return false; - } - else if (dsorgHigh & 0x02){ /*Partitioned / PO*/ - int pdsCheck = dscb[78-posOffset] & 0x0a; - if (pdsCheck == 0x0a){/*pdse & pdsex = hfs*/ - return false; - } - else{ - *isPds = true; - return true; - } - } - if (dsorgHigh & 0x01){//"unmovable" - return false; - } - - int dsorgLow = dscb[83-posOffset]; - - if (dsorgLow & 0x80){/*Graphics / GS*/ - return false; - } - else if (dsorgLow & 0x08){/*VSAM*/ - return false; - } -} - -static int obtainDSCB1(const char *dsname, unsigned int dsnameLength, - const char *volser, unsigned int volserLength, - char *dscb1) { - -#define DSCB1_SIZE 96 - -#define SVC27_WORKAREA_SIZE 140 -#define SVC27_OPCODE_SEARCH 0xC100 -#define SVC27_OPCODE_SEEK 0xC080 - -#define SVC27_OPTION_NO_TIOT_ENQ 0x80 -#define SVC27_OPTION_NO_DUMMY_DSCB1 0x40 -#define SVC27_OPTION_NO_CAT_ALLOC 0x20 -#define SVC27_OPTION_HIDE_NAME 0x10 -#define SVC27_OPTION_EADSCB_OK 0x08 - - ALLOC_STRUCT31( - STRUCT31_NAME(mem31), - STRUCT31_FIELDS( - char dsnameSpacePadded[44]; - char volserSpacePadded[6]; - char workArea[SVC27_WORKAREA_SIZE]; - __packed struct { - int operationCode : 16; - int option : 8; - int dscbNumber : 8; - char * __ptr32 dsname; - char * __ptr32 volser; - char * __ptr32 workArea; - } parmList; - ) - ); - - memset(mem31->dsnameSpacePadded, ' ', sizeof(mem31->dsnameSpacePadded)); - memcpy(mem31->dsnameSpacePadded, dsname, dsnameLength); - memset(mem31->volserSpacePadded, ' ', sizeof(mem31->volserSpacePadded)); - memcpy(mem31->volserSpacePadded, volser, volserLength); - - memset(mem31->workArea, 0, sizeof(mem31->workArea)); - - mem31->parmList.operationCode = SVC27_OPCODE_SEARCH; - mem31->parmList.option = SVC27_OPTION_EADSCB_OK; - mem31->parmList.dscbNumber = 0; - mem31->parmList.dsname = mem31->dsnameSpacePadded; - mem31->parmList.volser = mem31->volserSpacePadded; - mem31->parmList.workArea = mem31->workArea; - - int rc = 0; - - __asm( - " LA 1,0(%1) \n" - " SVC 27 \n" - " ST 15,%0 \n" - : "=m"(rc) - : "r"(&mem31->parmList) - : "r0", "r15" - ); - - memcpy(dscb1, mem31->workArea, DSCB1_SIZE); - - FREE_STRUCT31( - STRUCT31_NAME(mem31) - ); - - return rc; -} - -#ifdef __ZOWE_OS_ZOS -void addDetailedDatasetMetadata(char *datasetName, int nameLength, - char *volser, int volserLength, - jsonPrinter *jPrinter) { - - int isPDS = FALSE; - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Going to check dataset %s attributes\n",datasetName); - char dscb[INDEXED_DSCB] = {0}; - int rc = obtainDSCB1(datasetName, nameLength, - volser, volserLength, - dscb); - if (rc == 0){ - if (DSCB_TRACE){ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "DSCB for %s found\n",datasetName); - dumpbuffer(dscb,INDEXED_DSCB); - } - addDetailsFromDSCB(dscb,jPrinter,&isPDS); - } - else{ - char buffer[100]; - sprintf(buffer,"Type 1 or 8 DSCB for dataset %s not found",datasetName); - jsonAddString(jPrinter,"error",buffer); - } -} -#endif /* __ZOWE_OS_ZOS */ - -#ifdef __ZOWE_OS_ZOS -void addMemberedDatasetMetadata(char *datasetName, int nameLength, - char *volser, int volserLength, - char *memberQuery, int memberLength, - jsonPrinter *jPrinter, - int includeUnprintable) { - - int isPDS = FALSE; - char dscb[INDEXED_DSCB] = {0}; - int rc = obtainDSCB1(datasetName, nameLength, - volser, volserLength, - dscb); - - if (rc != 0) { - char buffer[100]; - sprintf(buffer, "Type 1 or 8 DSCB for dataset %s not found", datasetName); - jsonAddString(jPrinter, "error", buffer); - return; - } - - /* Use the DSCB to determine if it's - * a PDS or not, then free the - * memory. - */ - isPDS = isPartionedDataset(dscb); - - /* If it's not a PDS, we exit. */ - if (!isPDS) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Cannot print members. Selected dataset is not a PDS."); - return; - } - - char *theQuery = (memberLength < 1) ? "*":memberQuery; - if (memberLength < 1){ memberLength = 1;} - StringList *memberList = getPDSMembers(datasetName); - int memberCount = stringListLength(memberList); - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "--PDS %s has %d members\n",datasetName,memberCount); - if (memberCount > 0){ - jsonStartArray(jPrinter,"members"); - StringListElt *stringElement = firstStringListElt(memberList); - for (int i = 0; i < memberCount; i++){ - char *memberName = stringElement->string; - if (matchWithWildcards(theQuery,memberLength,memberName,8,0)) { - char percentName[24]; - int namePos = 0; - int containsUnprintable = FALSE; - for (int j = 0; j < 8; j++) { - if (memberName[j] < 0x40){ - containsUnprintable = TRUE; - if (includeUnprintable == FALSE){ - break; - } - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Member pos=%d has hex of 0x%x, will change.\n",j,memberName[j]); - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Member name: %s, percent name:%s\n",memberName,percentName); - sprintf(&percentName[namePos],"%%%2.2x",memberName[j]); - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "After: percentName: %s\n",percentName); - namePos = namePos+3; - } - else { - percentName[namePos] = memberName[j]; - namePos++; - } - } - - if (containsUnprintable == FALSE || includeUnprintable == TRUE){ - percentName[namePos] = '\0'; - - jsonStartObject(jPrinter,NULL); - { - jsonAddString(jPrinter,"name",percentName); - } - jsonEndObject(jPrinter); - - } - } - stringElement = stringElement->next; - } - jsonEndArray(jPrinter); - } - SLHFree(memberList->slh); -} -#endif /* __ZOWE_OS_ZOS */ - -#ifdef __ZOWE_OS_ZOS - -#define DSPATH_PREFIX "//\'" -#define DSPATH_SUFFIX "\'" - -static bool isDatasetPathValid(const char *path) { - - /* Basic check. The fopen() dataset path format is //'dsn(member)' */ - - /* Min dataset file name: - * //' - 3 - * min ds name - 1 - * ' - 1 - */ -#define PATH_MIN_LENGTH 5 - - /* Max dataset file name: - * //' - 3 - * man ds name - 44 - * ( - 1 - * max member name - 8 - * ) - 1 - * ' - 1 - */ -#define PATH_MAX_LENGTH 58 - -#define DSN_MIN_LENGTH 1 -#define DSN_MAX_LENGTH 44 -#define MEMBER_MIN_LENGTH 1 -#define MEMBER_MAX_LENGTH 8 - - size_t pathLength = strlen(path); - - if (pathLength < PATH_MIN_LENGTH || PATH_MAX_LENGTH < pathLength) { - return false; - } - - if (memcmp(path, DSPATH_PREFIX, strlen(DSPATH_PREFIX))) { - return false; - } - if (memcmp(&path[pathLength - 1], DSPATH_SUFFIX, strlen(DSPATH_SUFFIX))) { - return false; - } - - const char *dsnStart = path + strlen(DSPATH_PREFIX); - const char *dsnEnd = path + pathLength - 1 - strlen(DSPATH_SUFFIX); - - const char *leftParen = strchr(dsnStart, '('); - const char *rightParen = strchr(dsnStart, ')'); - - if (!leftParen ^ !rightParen) { - return false; - } - - if (leftParen) { - - ptrdiff_t dsnLength = leftParen - dsnStart; - if (dsnLength < DSN_MIN_LENGTH || DSN_MAX_LENGTH < dsnLength) { - return false; - } - - if (rightParen != dsnEnd) { - return false; - } - - ptrdiff_t memberNameLength = rightParen - leftParen - 1; - if (memberNameLength < MEMBER_MIN_LENGTH || MEMBER_MAX_LENGTH < memberNameLength) { - return false; - } - - } else { - - ptrdiff_t dsnLength = dsnEnd - dsnStart + 1; - if (dsnLength < DSN_MIN_LENGTH || DSN_MAX_LENGTH < dsnLength) { - return false; - } - - } - -#undef PATH_MIN_LENGTH -#undef PATH_MAX_LENGTH - -#undef DSN_MIN_LENGTH -#undef DSN_MAX_LENGTH -#undef MEMBER_MIN_LENGTH -#undef MEMBER_MAX_LENGTH - - return true; - -} - -static void extractDatasetAndMemberName(const char *datasetPath, - DatasetName *dsn, - DatasetMemberName *memberName) { - - memset(&dsn->value, ' ', sizeof(dsn->value)); - memset(&memberName->value, ' ', sizeof(memberName->value)); - - size_t pathLength = strlen(datasetPath); - - const char *dsnStart = datasetPath + strlen(DSPATH_PREFIX); - const char *leftParen = strchr(datasetPath, '('); - - if (leftParen) { - memcpy(dsn->value, dsnStart, leftParen - dsnStart); - const char *rightParen = strchr(datasetPath, ')'); - memcpy(memberName->value, leftParen + 1, rightParen - leftParen - 1); - } else { - memcpy(dsn->value, dsnStart, - pathLength - strlen(DSPATH_PREFIX""DSPATH_SUFFIX)); - } - - for (int i = 0; i < sizeof(dsn->value); i++) { - dsn->value[i] = toupper(dsn->value[i]); - } - - for (int i = 0; i < sizeof(memberName->value); i++) { - memberName->value[i] = toupper(memberName->value[i]); - } - -} - -#undef DSPATH_PREFIX -#undef DSPATH_SUFFIX - -static void respondWithDYNALLOCError(HttpResponse *response, - int rc, int sysRC, int sysRSN, - const DynallocDatasetName *dsn, - const DynallocMemberName *member, - const char *site) { - - if (rc == RC_DYNALLOC_SVC99_FAILED && sysRC == 4) { - - if (sysRSN == 0x020C0000 || sysRSN == 0x02100000) { - respondWithMessage(response, HTTP_STATUS_FORBIDDEN, - "Dataset \'%44.44s(%8.8s)\' busy (%s)", - dsn->name, member->name, site); - return; - } - - if (sysRSN == 0x02180000) { - respondWithMessage(response, HTTP_STATUS_NOT_FOUND, - "Device not available for dataset \'%44.44s(%8.8s)\' " - "(%s)", dsn->name, member->name, site); - return; - } - - if (sysRSN == 0x023C0000) { - respondWithMessage(response, HTTP_STATUS_NOT_FOUND, - "Catalog not available for dataset \'%44.44s(%8.8s)\' " - "(%s)", dsn->name, member->name, site); - return; - } - - } - - respondWithMessage(response, HTTP_STATUS_INTERNAL_SERVER_ERROR, - "DYNALLOC failed with RC = %d, DYN RC = %d, RSN = 0x%08X, " - "dsn=\'%44.44s(%8.8s)\', (%s)", rc, sysRC, sysRSN, - dsn->name, member->name, site); - -} - -#define IS_DAMEMBER_EMPTY($member) \ - (!memcmp(&($member), &(DynallocMemberName){" "}, sizeof($member))) - -static void updateDatasetWithJSONInternal(HttpResponse* response, - const char *datasetPath, /* backward compatibility */ - const DatasetName *dsn, - const DDName *ddName, - JsonObject *json) { - - char ddPath[16]; - snprintf(ddPath, sizeof(ddPath), "DD:%8.8s", ddName->value); - - JsonArray *recordArray = jsonObjectGetArray(json,"records"); - int recordCount = jsonArrayGetCount(recordArray); - int maxRecordLength = 80; - int isFixed = FALSE; - - /*Check if valid type of dataset to be written to*/ - Volser volser; - memset(&volser.value, ' ', sizeof(volser.value)); - - int volserSuccess = getVolserForDataset(dsn, &volser); - if (!volserSuccess){ - - char dscb[INDEXED_DSCB] = {0}; - int rc = obtainDSCB1(dsn->value, sizeof(dsn->value), - volser.value, sizeof(volser.value), - dscb); - if (rc == 0){ - if (DSCB_TRACE){ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "DSCB for %.*s found\n", sizeof(dsn->value), dsn->value); - dumpbuffer(dscb,INDEXED_DSCB); - } - bool isPds = false; - if (!isSupportedWriteDsorg(dscb, &isPds)) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"Unsupported dataset type"); - return; - } else if (isPds) { - bool isMember = false; - int memberStart=44; - int memberEnd=memberStart+8; - for (int i = memberStart; i < memberEnd; i++){ - if (*(dsn->value+i) != 0x40){ - isMember = true; - break; - } - } - if (!isMember){ - respondWithError(response, HTTP_STATUS_BAD_REQUEST, "Overwrite of PDS not supported"); - return; - } - } - - maxRecordLength = getMaxRecordLength(dscb); - char recordType = getRecordLengthType(dscb); - if (recordType == 'F'){ - isFixed = TRUE; - } else if (recordType == 'U') { - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"Undefined-length dataset"); - return; - } - } - } - else{ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "fallback for record length discovery\n"); - fldata_t fileinfo = {0}; - char filenameOutput[100]; - FILE *datasetRead = fopen(datasetPath, "rb, recfm=*, type=record"); - if (datasetRead == NULL) { - respondWithError(response,HTTP_STATUS_NOT_FOUND,"File could not be opened or does not exist"); - return; - } - - int returnCode = fldata(datasetRead,filenameOutput,&fileinfo); - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "FLData request rc=0x%x\n",returnCode); - fflush(stdout); - if (!returnCode) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_WARNING, - "fldata concat=%d, mem=%d, hiper=%d, temp=%d, vsam=%d, hfs=%d, device=%s\n", - fileinfo.__dsorgConcat, fileinfo.__dsorgMem, fileinfo.__dsorgHiper, - fileinfo.__dsorgTemp, fileinfo.__dsorgVSAM, fileinfo.__dsorgHFS, fileinfo.__device); - - if (fileinfo.__dsorgVSAM || fileinfo.__dsorgHFS || fileinfo.__dsorgHiper) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST, "Dataset type not supported"); - fclose(datasetRead); - return; - } - if (fileinfo.__maxreclen){ - maxRecordLength = fileinfo.__maxreclen; - } - else { - respondWithError(response,HTTP_STATUS_INTERNAL_SERVER_ERROR,"Could not discover record length"); - fclose(datasetRead); - return; - } - if (fileinfo.__recfmF) { - isFixed = TRUE; - } else if (fileinfo.__recfmU) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"Undefined-length dataset"); - fclose(datasetRead); - return; - } - } - else { - respondWithError(response,HTTP_STATUS_INTERNAL_SERVER_ERROR,"Could not read dataset information"); - fclose(datasetRead); - return; - } - fclose(datasetRead); - } - /*end dataset type check*/ - - - /*record length validation*/ - for ( int i = 0; i < recordCount; i++) { - Json *item = jsonArrayGetItem(recordArray,i); - if (jsonIsString(item) == TRUE) { - char *jsonString = jsonAsString(item); - int recordLength = strlen(jsonString); - if (recordLength > maxRecordLength) { - for (int j = recordLength; j > maxRecordLength-1; j--){ - if (jsonString[j] > 0x40){ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Invalid record for dataset, recordLength=%d but max for dataset is %d\n", recordLength, maxRecordLength); - char errorMessage[1024]; - int errorLength = sprintf(errorMessage,"Record #%d with contents \"%s\" is longer than the max record length of %d",i+1,jsonString,maxRecordLength); - errorMessage[errorLength] = '\0'; - respondWithError(response, HTTP_STATUS_BAD_REQUEST,errorMessage); - return; - } - } - recordLength = maxRecordLength; - } - if (isFixed && recordLength < maxRecordLength) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "UPDATE DATASET: record given for fixed datset less than maxLength=%d, len=%d, data:%s\n",maxRecordLength,recordLength,jsonString); - } - } - else { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Incorrectly formatted array!\n"); - char errorMessage[1024]; - int errorLength = sprintf(errorMessage,"Array position %d is not a string, but must be for record updating",i); - errorMessage[errorLength] = '\0'; - respondWithError(response, HTTP_STATUS_BAD_REQUEST,errorMessage); - return; - } - } - /*passed record length check and type check*/ - - FILE *outDataset = fopen(datasetPath, "wb, recfm=*, type=record"); - if (outDataset == NULL) { - respondWithError(response,HTTP_STATUS_NOT_FOUND,"File could not be opened or does not exist"); - return; - } - - int bytesWritten = 0; - int recordsWritten = 0; - char recordBuffer[maxRecordLength+1]; - - ICSFDigest digest; - char hash[32]; - - int rcEtag = icsfDigestInit(&digest, ICSF_DIGEST_SHA1); - if (rcEtag) { //if etag generation has an error, just don't send it. - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_WARNING, "ICSF error for SHA etag init for write, %d\n",rcEtag); - } - - for (int i = 0; i < recordCount; i++) { - char *record = jsonArrayGetString(recordArray,i); - int recordLength = strlen(record); - if (recordLength == 0) { //this is a hack, which will be removed as we move away from fwrite - record = " "; - recordLength = 1; - } - int len; - if (isFixed) { - // pad with spaces/or trim if needed - len = snprintf (recordBuffer, sizeof(recordBuffer), "%-*s", maxRecordLength, record); - } else { - // trim if needed - len = snprintf (recordBuffer, sizeof(recordBuffer), "%s", record); - } - bytesWritten = fwrite(recordBuffer,1,len,outDataset); - recordsWritten++; - if (bytesWritten < 0 && ferror(outDataset)){ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Error writing to dataset, rc=%d\n", bytesWritten); - respondWithError(response,HTTP_STATUS_INTERNAL_SERVER_ERROR,"Error writing to dataset"); - fclose(outDataset); - break; - } else if (!rcEtag) { - rcEtag = icsfDigestUpdate(&digest, recordBuffer, bytesWritten); - } - } - - if (!rcEtag) { rcEtag = icsfDigestFinish(&digest, hash); } - if (rcEtag) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_WARNING, "ICSF error for SHA etag, %d\n",rcEtag); - } - - /*success!*/ - - jsonPrinter *p = respondWithJsonPrinter(response); - setResponseStatus(response, 201, "Created"); - setDefaultJSONRESTHeaders(response); - writeHeader(response); - jsonStart(p); - - char msgBuffer[128]; - snprintf(msgBuffer, sizeof(msgBuffer), "Updated dataset %s with %d records", datasetPath, recordsWritten); - jsonAddString(p, "msg", msgBuffer); - - if (!rcEtag) { - // Convert hash text to hex. - int eTagLength = digest.hashLength*2; - char eTag[eTagLength+1]; - memset(eTag, '\0', eTagLength); - int len = digest.hashLength; - simpleHexPrint(eTag, hash, digest.hashLength); - jsonAddString(p, "etag", eTag); - } - jsonEnd(p); - - finishResponse(response); - - fclose(outDataset); -} - -static void updateDatasetWithJSON(HttpResponse *response, JsonObject *json, char *datasetPath, - const char *lastEtag, bool force) { - - HttpRequest *request = response->request; - - if (!isDatasetPathValid(datasetPath)) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST, "Invalid dataset name"); - return; - } - - if (!lastEtag && !force) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST, "No etag given"); - return; - } - - DatasetName dsn; - DatasetMemberName memberName; - extractDatasetAndMemberName(datasetPath, &dsn, &memberName); - - DynallocDatasetName daDsn; - DynallocMemberName daMember; - memcpy(daDsn.name, dsn.value, sizeof(daDsn.name)); - memcpy(daMember.name, memberName.value, sizeof(daMember.name)); - DynallocDDName daDDname = {.name = "????????"}; - - int daRC = RC_DYNALLOC_OK, daSysRC = 0, daSysRSN = 0; - daRC = dynallocAllocDataset( - &daDsn, - IS_DAMEMBER_EMPTY(daMember) ? NULL : &daMember, - &daDDname, - DYNALLOC_DISP_OLD, - DYNALLOC_ALLOC_FLAG_NO_CONVERSION | DYNALLOC_ALLOC_FLAG_NO_MOUNT, - &daSysRC, &daSysRSN - ); - - if (daRC != RC_DYNALLOC_OK) { - zowelog(NULL, LOG_COMP_DATASERVICE, ZOWE_LOG_DEBUG, - "error: ds alloc dsn=\'%44.44s\', member=\'%8.8s\', dd=\'%8.8s\'," - " rc=%d sysRC=%d, sysRSN=0x%08X (update)\n", - daDsn.name, daMember.name, daDDname.name, daRC, daSysRC, daSysRSN, "update"); - respondWithDYNALLOCError(response, daRC, daSysRC, daSysRSN, - &daDsn, &daMember, "w"); - return; - } - - zowelog(NULL, LOG_COMP_DATASERVICE, ZOWE_LOG_DEBUG, - "debug: updating dsn=\'%44.44s\', member=\'%8.8s\', dd=\'%8.8s\'\n", - daDsn.name, daMember.name, daDDname.name); - - DDName ddName; - memcpy(&ddName.value, &daDDname.name, sizeof(ddName.value)); - - char ddPath[16]; - snprintf(ddPath, sizeof(ddPath), "DD:%8.8s", ddName.value); - - int eTagRC = 0; - if (!force) { //do not write dataset if current contents do not match contents client expected, unless forced - int eTagReturnLength = 0; - int lrecl = getLreclOrRespondError(response, &dsn, ddPath); - if (lrecl) { - char *eTag = getDatasetETag(ddPath, lrecl, &eTagRC, &eTagReturnLength); - zowelog(NULL, LOG_COMP_DATASERVICE, ZOWE_LOG_INFO, "Given etag=%s, current etag=%s\n",lastEtag, eTag); - if (!eTag) { - respondWithError(response, HTTP_STATUS_INTERNAL_SERVER_ERROR, "Could not generate etag"); - } else if (strcmp(eTag, lastEtag)) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST, "Provided etag did not match system etag. To write, read the dataset again and resolve the difference, then retry."); - safeFree(eTag,eTagReturnLength+1); - } else { - safeFree(eTag,eTagReturnLength+1); - updateDatasetWithJSONInternal(response, datasetPath, &dsn, &ddName, json); - } - } - } else { - updateDatasetWithJSONInternal(response, datasetPath, &dsn, &ddName, json); - } - - daRC = dynallocUnallocDatasetByDDName(&daDDname, DYNALLOC_UNALLOC_FLAG_NONE, - &daSysRC, &daSysRSN); - if (daRC != RC_DYNALLOC_OK) { - zowelog(NULL, LOG_COMP_DATASERVICE, ZOWE_LOG_DEBUG, - "error: ds unalloc dsn=\'%44.44s\', member=\'%8.8s\', dd=\'%8.8s\'," - " rc=%d sysRC=%d, sysRSN=0x%08X (update)\n", - daDsn.name, daMember.name, daDDname.name, daRC, daSysRC, daSysRSN, "update"); - } -} - -#endif /* __ZOWE_OS_ZOS */ - -#ifdef __ZOWE_OS_ZOS -/* TODO: this has not yet been tested with real, live JSON */ -static void updateVSAMDatasetWithJSON(HttpResponse *response, JsonObject *json, char *dsn, hashtable *acbTable) { - JsonArray *recordArray = jsonObjectGetArray(json,"records"); - int recordCount = jsonArrayGetCount(recordArray); - char *username = response->request->username; - int maxRecordLength = 80; - /*ACEE *newACEE; - int impersonationBegan = startZOSImpersonation(username,&newACEE); - if (!impersonationBegan) { - respondWithError(response,HTTP_STATUS_INTERNAL_SERVER_ERROR,"Failed privilege check"); - endZOSImpersonation(&newACEE); - return; - }*/ - char dsnUidPair[44+8+1]; - memset(dsnUidPair, ' ', 44+8); - memcpy(dsnUidPair, dsn, 44); - memcpy(dsnUidPair+44, username, 8); - StatefulACB *state = (StatefulACB *)htGet(acbTable, dsnUidPair); - char *outACB = state ? state->acb : NULL; - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "htGet on \"%s\" returned 0x%0x\n", dsnUidPair, outACB); - - if (outACB == NULL) { /* TODO: openACB for output */ - /* TODO: open CSI to find relevant fields if ACB needs to be opened. */ - } - RPLCommon *rpl = (RPLCommon *)(outACB+RPL_COMMON_OFFSET+8); - maxRecordLength = rpl->bufLen; - - /* Time to check the JSON for errors while parsing it into a readable format */ - /* TODO: However you wish to format the JSON to easily be read later is your choice. Below is an unfinished, theoretical approach. */ - for ( int i = 0; i < recordCount; i++) { - Json *item = jsonArrayGetItem(recordArray,i); - if (jsonIsString(item) == TRUE) { - /* TODO: find and B64 decrypt the (key,record) pairs and write them as a block of chars */ - char *jsonString = jsonAsString(item); /* TODO: should be looking at a "str:str" or "str: str" value here? */ - int recordLength = strlen(jsonString); /* TODO: more correctly calculate the recordLength AND keyLength */ - if (recordLength > maxRecordLength) { /* TODO: check recordLength AND keyLength (rpl->keyLen for keyLength) */ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Invalid record for dataset, recordLength=%d but max for dataset is %d\n",recordLength,maxRecordLength); - char errorMessage[1024]; - int errorLength = sprintf(errorMessage,"Record #%d with contents \"%s\" is longer than the max record length of %d",i+1,jsonString,maxRecordLength); - errorMessage[errorLength] = '\0'; - respondWithError(response, HTTP_STATUS_BAD_REQUEST,errorMessage); - return; - } - } else { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Incorrectly formatted array!\n"); - char errorMessage[1024]; - int errorLength = sprintf(errorMessage,"Array position %d is not a string, but must be for record updating",i); - errorMessage[errorLength] = '\0'; - respondWithError(response, HTTP_STATUS_BAD_REQUEST,errorMessage); - return; - } - } - /* Passed JSON check */ - - /* TODO: parse all of the user request parms here. Currently, the only planned one is "replace". */ - int update = 0; - /* TODO: make sure RPL is correctly formatted. If it is not, add a modRPL call similar to the one in respondWithVSAMDataset() */ - /* TODO: the only user parm we need to add to the RPL's list is RPL_OPTCD_UPD (replace) here */ - - int recordsWritten = 0; - int recordLength = 0; - char *lastKey; - char *tempRecord = safeMalloc(maxRecordLength, "Temporary Record Bufer"); /* TODO: does it make sense to SLHAlloc this instead? */ - for (int i = 0; i < recordCount; i++) { - /* TODO: note that non-KSDS uses a pointer to the ? */ - /* TODO: get Key before storing */ - char *key = jsonArrayGetString(recordArray,i); /* TODO: get key AND record here */ - char *record; /* TODO: correctly parse the record, minding your decided format from the above loop. */ - /* TODO: point to the key, based on which type of dataset it is. See pointByXXX functions in respondWithVSAMDataset() for example syntax */ - if (update) { - getRecord(outACB, tempRecord, &recordLength); /* in VSAM, we need to GET for update before we can write over a record with a later PUT for update */ - } - putRecord(outACB, record, recordLength); - if (rpl->status) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Error writing to dataset, rc=%02x%06x\n",rpl->status,rpl->feedback); - respondWithError(response,HTTP_STATUS_INTERNAL_SERVER_ERROR,"Error writing to dataset"); - break; - } - recordsWritten++; - lastKey = key; /* TODO: we are keeping this so the caller knows where we left off */ - } - /*success!*/ - safeFree(tempRecord, maxRecordLength); - - char successMessage[1024]; - int successMessageLength = sprintf(successMessage,"Updated dataset %s with %d records",dsn,recordsWritten); /* TODO: Note last key */ - successMessage[successMessageLength] = '\0'; - respondWithError(response,HTTP_STATUS_OK,successMessage); /*why do we call it respondWithError if we can use successful messages too*/ - /*endZOSImpersonation(&newACEE);*/ -} -#endif /* __ZOWE_OS_ZOS */ - -#ifndef NATIVE_CODEPAGE -#define NATIVE_CODEPAGE CCSID_EBCDIC_1047 -#endif - -/* - Dataset only, no wildcards or pds members accepted - beware that this should be wrapped with authentication - */ -static int getVolserForDataset(const DatasetName *dataset, Volser *volser) { - if (dataset == NULL){ - return -1; - } - int length = sizeof(dataset->value); - - char dsnNullTerm[45] = {0}; - memcpy(dsnNullTerm, dataset->value, sizeof(dataset->value)); - - int lParenIndex = indexOf(dsnNullTerm, length, '(', 0); - int asterixIndex = indexOf(dsnNullTerm, length, '*', 0); - int dollarIndex = indexOf(dsnNullTerm, length, '$', 0); - if (lParenIndex > -1 || asterixIndex > -1 || dollarIndex > -1){ - return -1; - } - csi_parmblock * __ptr32 returnParms = (csi_parmblock* __ptr32)safeMalloc31(sizeof(csi_parmblock),"CSI ParmBlock"); - EntryDataSet *entrySet = returnEntries(dsnNullTerm, defaultDatasetTypesAllowed,3, 0, defaultCSIFields, defaultCSIFieldCount, NULL, NULL, returnParms); - - EntryData *entry = entrySet->entries ? entrySet->entries[0] : NULL; - int rc = -1; - if (entry){ - char *fieldData = (char*)entry+sizeof(EntryData); - unsigned short *fieldLengthArray = ((unsigned short *)((char*)entry+sizeof(EntryData))); - char *fieldValueStart = (char*)entry+sizeof(EntryData)+defaultCSIFieldCount*sizeof(short); - for (int j=0; jvalue,fieldValueStart,6); - rc = 0; - break; - } - fieldValueStart += fieldLengthArray[j]; - } - } - - for (int i = 0; i < entrySet->length; i++){ - EntryData *currentEntry = entrySet->entries[i]; - int fieldDataLength = currentEntry->data.fieldInfoHeader.totalLength; - int entrySize = sizeof(EntryData)+fieldDataLength-4; - safeFree((char*)(currentEntry),entrySize); - } - if (entrySet->entries != NULL) { - safeFree((char*)(entrySet->entries),sizeof(EntryData*)*entrySet->size); - entrySet->entries = NULL; - } - safeFree((char*)entrySet,sizeof(EntryDataSet)); - safeFree31((char*)returnParms,sizeof(csi_parmblock)); - return rc; -} - -static char getRecordLengthType(char *dscb){ - int posOffset = 44; - int recfm = dscb[84 - posOffset]; - if ((recfm & 0xc0) == 0xc0){ - return 'U'; - } - else if (recfm & 0x80){ - return 'F'; - } - else if (recfm & 0x40){ - return 'V'; - } -} - -static int getMaxRecordLength(char *dscb){ - int posOffset = 44; - int lrecl = (dscb[88-posOffset] << 8) | dscb[89-posOffset]; - return lrecl; -} - -void updateDataset(HttpResponse* response, char* absolutePath, int jsonMode) { -#ifdef __ZOWE_OS_ZOS - if (jsonMode != TRUE) { /*TODO add support for updating files with raw bytes instead of JSON*/ - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"Cannot update file without JSON formatted record request"); - return; - } - - HttpRequest *request = response->request; - HttpRequestParam *forceParam = getCheckedParam(request,"force"); - char *forceArg = (forceParam ? forceParam->stringValue : NULL); - bool force = (forceArg != NULL && !strcmp(forceArg,"true")); - - FileInfo info; - int returnCode; - int reasonCode; - - char *contentBody = response->request->contentBody; - int bodyLength = strlen(contentBody); - - char *convertedBody = safeMalloc(bodyLength*4,"writeDatasetConvert"); - int conversionBufferLength = bodyLength*4; - int translationLength; - int outCCSID = NATIVE_CODEPAGE; - - returnCode = convertCharset(contentBody, - bodyLength, - CCSID_UTF_8, - CHARSET_OUTPUT_USE_BUFFER, - &convertedBody, - conversionBufferLength, - outCCSID, - NULL, - &translationLength, - &reasonCode); - - if(returnCode == 0) { - int blockSize = 0x10000; - int maxBlockCount = (translationLength*2)/blockSize; - if (!maxBlockCount){ - maxBlockCount = 0x10; - } - ShortLivedHeap *slh = makeShortLivedHeap(blockSize,maxBlockCount); - char errorBuffer[2048]; - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "UPDATE DATASET: before JSON parse dataLength=0x%x\n",bodyLength); - Json *json = jsonParseUnterminatedString(slh, - convertedBody, translationLength, - errorBuffer, sizeof(errorBuffer)); - if (json) { - if (jsonIsObject(json)) { - JsonObject *jsonObject = jsonAsObject(json); - char *etag = jsonObjectGetString(jsonObject,"etag"); - if (!etag) { - HttpHeader *etagHeader = getHeader(request, "etag"); - if (etagHeader) { - etag = etagHeader->nativeValue; - } - } - updateDatasetWithJSON(response, jsonObject, absolutePath, etag, force); - } else{ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "*** INTERNAL ERROR *** message is JSON, but not an object\n"); - } - } - else { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "UPDATE DATASET: body was not JSON!\n"); - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"POST body could not be parsed as JSON format"); - } - SLHFree(slh); - } - else { - respondWithError(response,HTTP_STATUS_INTERNAL_SERVER_ERROR,"Could not translate character set to EBCDIC"); - } - safeFree(convertedBody,conversionBufferLength); -#endif /* __ZOWE_OS_ZOS */ -} - -void deleteDatasetOrMember(HttpResponse* response, char* absolutePath) { -#ifdef __ZOWE_OS_ZOS - HttpRequest *request = response->request; - if (!isDatasetPathValid(absolutePath)) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST, "Invalid dataset name"); - return; - } - - DatasetName datasetName; - DatasetMemberName memberName; - extractDatasetAndMemberName(absolutePath, &datasetName, &memberName); - DynallocDatasetName daDatasetName; - DynallocMemberName daMemberName; - memcpy(daDatasetName.name, datasetName.value, sizeof(daDatasetName.name)); - memcpy(daMemberName.name, memberName.value, sizeof(daMemberName.name)); - DynallocDDName daDDName = {.name = "????????"}; - - char CSIType = getCSIType(absolutePath); - if (CSIType == '') { - respondWithMessage(response, HTTP_STATUS_NOT_FOUND, - "Dataset or member does not exist \'%44.44s(%8.8s)\' " - "(%s)", daDatasetName.name, daMemberName.name, "r"); - return; - } - if (isVsam(CSIType)) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST, - "VSAM dataset detected. Please use regular dataset route"); - return; - } - - int daReturnCode = RC_DYNALLOC_OK, daSysReturnCode = 0, daSysReasonCode = 0; - daReturnCode = dynallocAllocDataset( - &daDatasetName, - NULL, - &daDDName, - DYNALLOC_DISP_OLD, - DYNALLOC_ALLOC_FLAG_NO_CONVERSION | DYNALLOC_ALLOC_FLAG_NO_MOUNT, - &daSysReturnCode, &daSysReasonCode - ); - - if (daReturnCode != RC_DYNALLOC_OK) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, - "error: ds alloc dsn=\'%44.44s\', member=\'%8.8s\', dd=\'%8.8s\'," - " rc=%d sysRC=%d, sysRSN=0x%08X (read)\n", - daDatasetName.name, daMemberName.name, daDDName.name, - daReturnCode, daSysReturnCode, daSysReasonCode); - respondWithDYNALLOCError(response, daReturnCode, daSysReturnCode, - daSysReasonCode, &daDatasetName, &daMemberName, - "r"); - return; - } - - bool isMemberEmpty = IS_DAMEMBER_EMPTY(daMemberName); - - if (isMemberEmpty) { - daReturnCode = dynallocUnallocDatasetByDDName2(&daDDName, DYNALLOC_UNALLOC_FLAG_NONE, - &daSysReturnCode, &daSysReasonCode, - TRUE /* Delete data set on deallocation */ - ); - if (daReturnCode != RC_DYNALLOC_OK) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, - "error: ds alloc dsn=\'%44.44s\', member=\'%8.8s\', dd=\'%8.8s\'," - " rc=%d sysRC=%d, sysRSN=0x%08X (read)\n", - daDatasetName.name, daMemberName.name, daDDName.name, - daReturnCode, daSysReturnCode, daSysReasonCode); - respondWithDYNALLOCError(response, daReturnCode, daSysReturnCode, - daSysReasonCode, &daDatasetName, &daMemberName, - "r"); - return; - } - } - else { - char dsNameNullTerm[DATASET_NAME_LEN + 1] = {0}; - memcpy(dsNameNullTerm, datasetName.value, sizeof(datasetName.value)); - - char *dcb = openSAM(daDDName.name, /* The data set must be opened by supplying a dd name */ - OPEN_CLOSE_OUTPUT, /* To delete a pds data set member, this option must be set */ - FALSE, /* Indicates that this data set is not QSAM */ - 0, /* Record format (zero if unknown) */ - 0, /* Record length (zero if unknown) */ - 0); /* Block size (zero if unknown) */ - - if (dcb == NULL) { - respondWithError(response, HTTP_STATUS_INTERNAL_SERVER_ERROR, "Data set could not be opened"); - return; - } - - if (!memberExists(dsNameNullTerm, daMemberName)) { - respondWithError(response, HTTP_STATUS_NOT_FOUND, "Data set member does not exist"); - closeSAM(dcb, 0); - daReturnCode = dynallocUnallocDatasetByDDName(&daDDName, DYNALLOC_UNALLOC_FLAG_NONE, - &daSysReturnCode, &daSysReasonCode); - return; - } - - char *belowMemberName = NULL; - belowMemberName = malloc24(DATASET_MEMBER_NAME_LEN); /* This must be allocated below the line */ - - if (belowMemberName == NULL) { - respondWithError(response, HTTP_STATUS_INTERNAL_SERVER_ERROR, "Could not allocate member name"); - closeSAM(dcb, 0); - return; - } - - memset(belowMemberName, ' ', DATASET_MEMBER_NAME_LEN); - memcpy(belowMemberName, memberName.value, DATASET_MEMBER_NAME_LEN); - - int stowReturnCode = 0, stowReasonCode = 0; - stowReturnCode = bpamDeleteMember(dcb, belowMemberName, &stowReasonCode); - - /* Free member name and dcb as they are no longer needed */ - free24(belowMemberName, DATASET_MEMBER_NAME_LEN); - closeSAM(dcb, 0); - - daReturnCode = dynallocUnallocDatasetByDDName(&daDDName, DYNALLOC_UNALLOC_FLAG_NONE, - &daSysReturnCode, &daSysReasonCode); - - if (stowReturnCode != 0) { - char responseMessage[128]; - snprintf(responseMessage, sizeof(responseMessage), "Member %8.8s could not be deleted\n", daMemberName.name); - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, - "error: stowReturnCode=%d, stowReasonCode=%d\n", - stowReturnCode, stowReasonCode); - respondWithError(response, HTTP_STATUS_INTERNAL_SERVER_ERROR, responseMessage); - return; - } - - if (daReturnCode != RC_DYNALLOC_OK) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, - "error: ds alloc dsn=\'%44.44s\', member=\'%8.8s\', dd=\'%8.8s\'," - " rc=%d sysRC=%d, sysRSN=0x%08X (read)\n", - daDatasetName.name, daMemberName.name, daDDName.name, - daReturnCode, daSysReturnCode, daSysReasonCode); - respondWithDYNALLOCError(response, daReturnCode, daSysReturnCode, - daSysReasonCode, &daDatasetName, &daMemberName, - "r"); - return; - } - } - - jsonPrinter *p = respondWithJsonPrinter(response); - setResponseStatus(response, 200, "OK"); - setDefaultJSONRESTHeaders(response); - - writeHeader(response); - - jsonStart(p); - char responseMessage[128]; - if (isMemberEmpty) { - char* dsName; - dsName = absolutePath+3; - dsName[strlen(dsName) - 1] = '\0'; - snprintf(responseMessage, sizeof(responseMessage), "Data set %s was deleted successfully", dsName); - jsonAddString(p, "msg", responseMessage); - } - else { - snprintf(responseMessage, sizeof(responseMessage), "Data set member %8.8s was deleted successfully", daMemberName.name); - jsonAddString(p, "msg", responseMessage); - } - jsonEnd(p); - - finishResponse(response); - -#endif /* __ZOWE_OS_ZOS */ -} - -bool memberExists(char* dsName, DynallocMemberName daMemberName) { - bool found = false; - StringList *memberList = getPDSMembers(dsName); - int memberCount = stringListLength(memberList); - if (memberCount > 0){ - StringListElt *stringElement = firstStringListElt(memberList); - for (int i = 0; i < memberCount; i++){ - char *memName = stringElement->string; - char dest[9]; - strncpy(dest, daMemberName.name, 8); - dest[8] = '\0'; - if (strcmp(memName, dest) == 0) { - found = true; - } - stringElement = stringElement->next; - } - } - SLHFree(memberList->slh); - return found; -} - -bool isVsam(char CSIType) { - int index = indexOf(vsamCSITypes, strlen(vsamCSITypes), CSIType, 0); - if (index == -1) { - return false; - } else { - return true; - } -} - -char getCSIType(char* absolutePath) { - char *typesArg = defaultDatasetTypesAllowed; - int datasetTypeCount = (typesArg == NULL) ? 3 : strlen(typesArg); - int workAreaSizeArg = 0; - int fieldCount = defaultCSIFieldCount; - char **csiFields = defaultCSIFields; - - csi_parmblock * __ptr32 returnParms = (csi_parmblock* __ptr32)safeMalloc31(sizeof(csi_parmblock),"CSI ParmBlock"); - - DatasetName datasetName; - DatasetMemberName memberName; - extractDatasetAndMemberName(absolutePath, &datasetName, &memberName); - - char dsNameNullTerm[DATASET_NAME_LEN + 1] = {0}; - memcpy(dsNameNullTerm, datasetName.value, sizeof(datasetName.value)); - - EntryDataSet *entrySet = returnEntries(dsNameNullTerm, typesArg, datasetTypeCount, - workAreaSizeArg, csiFields, fieldCount, - NULL, NULL, returnParms); - - EntryData *entry = entrySet->entries[0]; - - if (entrySet->length == 1) { - if (entry) { - return entry->type; - } - } else if (entrySet->length == 0) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "No entries for the dataset name found"); - } else { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "More than one entry found for dataset name"); - } - - return ''; -} - -void deleteVSAMDataset(HttpResponse* response, char* absolutePath) { -#ifdef __ZOWE_OS_ZOS - HttpRequest *request = response->request; - - if (!isDatasetPathValid(absolutePath)) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST, "Invalid dataset name"); - return; - } - char CSIType = getCSIType(absolutePath); - - if (CSIType == '') { - respondWithError(response, HTTP_STATUS_NOT_FOUND, "Dataset not found"); - return; - } - - if (!isVsam(CSIType)) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST, - "Non VSAM dataset detected. Please use regular dataset route"); - return; - } - - char* dsName; - dsName = absolutePath+3; - dsName[strlen(dsName) - 1] = '\0'; - for (int i = 0; i < strlen(dsName); i++) { - if (isalpha(dsName[i])) { - dsName[i] = toupper(dsName[i]); - } - } - - int rc = deleteCluster(dsName); - char responseMessage[128]; - - if (rc == 0) { - snprintf(responseMessage, sizeof(responseMessage), "VSAM dataset %s was successfully deleted", dsName); - jsonPrinter *p = respondWithJsonPrinter(response); - setResponseStatus(response, 200, "OK"); - setDefaultJSONRESTHeaders(response); - writeHeader(response); - jsonStart(p); - jsonAddString(p, "msg", responseMessage); - jsonEnd(p); - - finishResponse(response); - } else { - snprintf(responseMessage, sizeof(responseMessage), "Invalid VSAM delete with IDCAMS with return code: %d", rc); - jsonPrinter *p = respondWithJsonPrinter(response); - setResponseStatus(response, 403, "Forbidden"); - setDefaultJSONRESTHeaders(response); - writeHeader(response); - jsonStart(p); - jsonAddString(p, "msg", responseMessage); - jsonEnd(p); - - finishResponse(response); - - } - - -#endif /* __ZOWE_OS_ZOS */ -} - - -void updateVSAMDataset(HttpResponse* response, char* absolutePath, hashtable *acbTable, int jsonMode) { -#ifdef __ZOWE_OS_ZOS - if (jsonMode != TRUE) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"Cannot update file without JSON formatted record request"); - return; - } - - HttpRequest *request = response->request; - - int returnCode; - int reasonCode; - - char *contentBody = response->request->contentBody; - int bodyLength = strlen(contentBody); - - char *convertedBody = safeMalloc(bodyLength*4,"writeDatasetConvert"); - int conversionBufferLength = bodyLength*4; - int translationLength; - int outCCSID = NATIVE_CODEPAGE; - - returnCode = convertCharset(contentBody, - bodyLength, - CCSID_UTF_8, - CHARSET_OUTPUT_USE_BUFFER, - &convertedBody, - conversionBufferLength, - outCCSID, - NULL, - &translationLength, - &reasonCode); - - if(returnCode == 0) { - - ShortLivedHeap *slh = makeShortLivedHeap(0x10000,0x10); - char errorBuffer[2048]; - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "UPDATE DATASET: before JSON parse dataLength=0x%x\n",bodyLength); - Json *json = jsonParseUnterminatedString(slh, - convertedBody, translationLength, - errorBuffer, sizeof(errorBuffer)); - if (json) { - if (jsonIsObject(json)){ - updateVSAMDatasetWithJSON(response, jsonAsObject(json), absolutePath, acbTable); - } else{ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "*** INTERNAL ERROR *** message is JSON, but not an object\n"); - } - } - else { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "UPDATE DATASET: body was not JSON!\n"); - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"POST body could not be parsed as JSON format"); - } - SLHFree(slh); - } else { - respondWithError(response,HTTP_STATUS_INTERNAL_SERVER_ERROR,"Could not translate character set to EBCDIC"); - } - safeFree(convertedBody,conversionBufferLength); -#endif /* __ZOWE_OS_ZOS */ -} - -/* - write = openSAM(name,OPEN_CLOSE_OUTPUT,TRUE,recfm,lrecl,blksize); - read = openSAM(name,OPEN_CLOSE_INPUT,TRUE,recfm,lrecl,blksize); - - */ - -static void respondWithDatasetInternal(HttpResponse* response, - const char *datasetPath, - const DatasetName *dsn, - const DDName *ddName, - int jsonMode) { -#ifdef __ZOWE_OS_ZOS - HttpRequest *request = response->request; - - char ddPath[16]; - snprintf(ddPath, sizeof(ddPath), "DD:%8.8s", ddName->value); - - int lrecl = getLreclOrRespondError(response, dsn, ddPath); - if (!lrecl) { - return; - } - - jsonPrinter *jPrinter = respondWithJsonPrinter(response); - setResponseStatus(response, 200, "OK"); - setDefaultJSONRESTHeaders(response); - - writeHeader(response); - - if (lrecl){ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Streaming data for %s\n", datasetPath); - - jsonStart(jPrinter); - int status = streamDataset(response->socket, ddPath, lrecl, jPrinter); - jsonEnd(jPrinter); - } - finishResponse(response); -#endif /* __ZOWE_OS_ZOS */ -} - -void respondWithDataset(HttpResponse* response, char* absolutePath, int jsonMode) { - - HttpRequest *request = response->request; - - if (!isDatasetPathValid(absolutePath)) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST, "Invalid dataset name"); - return; - } - - DatasetName dsn; - DatasetMemberName memberName; - extractDatasetAndMemberName(absolutePath, &dsn, &memberName); - - DynallocDatasetName daDsn; - DynallocMemberName daMember; - memcpy(daDsn.name, dsn.value, sizeof(daDsn.name)); - memcpy(daMember.name, memberName.value, sizeof(daMember.name)); - DynallocDDName daDDname = {.name = "????????"}; - - int daRC = RC_DYNALLOC_OK, daSysRC = 0, daSysRSN = 0; - daRC = dynallocAllocDataset( - &daDsn, - IS_DAMEMBER_EMPTY(daMember) ? NULL : &daMember, - &daDDname, - DYNALLOC_DISP_SHR, - DYNALLOC_ALLOC_FLAG_NO_CONVERSION | DYNALLOC_ALLOC_FLAG_NO_MOUNT, - &daSysRC, &daSysRSN - ); - - if (daRC != RC_DYNALLOC_OK) { - zowelog(NULL, LOG_COMP_DATASERVICE, ZOWE_LOG_DEBUG, - "error: ds alloc dsn=\'%44.44s\', member=\'%8.8s\', dd=\'%8.8s\'," - " rc=%d sysRC=%d, sysRSN=0x%08X (read)\n", - daDsn.name, daMember.name, daDDname.name, daRC, daSysRC, daSysRSN); - respondWithDYNALLOCError(response, daRC, daSysRC, daSysRSN, - &daDsn, &daMember, "r"); - return; - } - - zowelog(NULL, LOG_COMP_DATASERVICE, ZOWE_LOG_DEBUG, - "debug: reading dsn=\'%44.44s\', member=\'%8.8s\', dd=\'%8.8s\'\n", - daDsn.name, daMember.name, daDDname.name); - - DDName ddName; - memcpy(&ddName.value, &daDDname.name, sizeof(ddName.value)); - respondWithDatasetInternal(response, absolutePath, &dsn, &ddName, jsonMode); - - daRC = dynallocUnallocDatasetByDDName(&daDDname, DYNALLOC_UNALLOC_FLAG_NONE, - &daSysRC, &daSysRSN); - if (daRC != RC_DYNALLOC_OK) { - zowelog(NULL, LOG_COMP_DATASERVICE, ZOWE_LOG_DEBUG, - "error: ds unalloc dsn=\'%44.44s\', member=\'%8.8s\', dd=\'%8.8s\'," - " rc=%d sysRC=%d, sysRSN=0x%08X (read)\n", - daDsn.name, daMember.name, daDDname.name, daRC, daSysRC, daSysRSN, "read"); - } - -} - -#define CSI_VSAMTYPE_KSDS 0x8000 -#define CSI_VSAMTYPE_RRDS 0x0200 -#define CSI_VSAMTYPE_LDS 0x0004 -#define CSI_VSAMTYPE_VRRDS 0x0001 -#define CSI_VSAMTYPE_CLOER 0x0020 /* Error on last close - stats may be inaccurate */ - -/* In development - this is hardcoded for now */ -void respondWithVSAMDataset(HttpResponse* response, char* absolutePath, hashtable *acbTable, int jsonMode) { -#ifdef __ZOWE_OS_ZOS - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "begin %s\n", __FUNCTION__); - HttpRequest *request = response->request; - - HttpRequestParam *closeParam = getCheckedParam(request,"closeAfter"); - char *closeArg = (closeParam ? closeParam->stringValue : NULL); - int closeAfter = (closeArg != NULL && !strcmp(closeArg,"true")); - - int returnCode; - int reasonCode; - char dsn[45] = ""; - int rplParms = 0; - memcpy(dsn, absolutePath, 44); - if (strlen(absolutePath) <= 44) { - memset(dsn + strlen(absolutePath), ' ', 44 - strlen(absolutePath)); - dsn[44] = 0; - } else { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "DSN of size %d is too large.\n", strlen(absolutePath)); - respondWithError(response, HTTP_STATUS_BAD_REQUEST, "Dataset Name must be 44 bytes of less"); - return; - } - char *username = response->request->username; - - int maxRecords = 0; - int maxBytes = 0; - /* TODO: parse out the parms from the HttpRequest* request */ - - unsigned int vsamType = 0; - unsigned int ciSize = 0; - unsigned int maxlrecl = 0; - unsigned int maxbuffer = 0; - unsigned int keyLoc = 0; - unsigned int keyLen = 0; - char dsnCluster[45] = ""; - char dsnData[45] = ""; - - /* TODO: We should not need to do this on every call - only if the ACB needs to be opened */ - /* TODO: How to access the CSI in cases where the entry is archived? Is this possible? */ - csi_parmblock * __ptr32 returnParms = (csi_parmblock* __ptr32)safeMalloc31(sizeof(csi_parmblock),"CSI ParmBlock"); - EntryDataSet *entrySet = returnEntries(dsn, clusterTypesAllowed, clusterTypesCount, 0, defaultVSAMCSIFields, defaultVSAMCSIFieldCount, NULL, NULL, returnParms); - EntryData *entry = entrySet->entries[0]; - if (entry){ - if (entry->type == 'I') { /* TODO: how do we want to handle INDEX datasets? Some editors use */ - /* TODO: free the entire entrySet, then call CSI again */ - } - if (entry->type == 'C') { - unsigned short *fieldLengthArray = ((unsigned short *)((char*)entry+sizeof(EntryData))); - char *fieldValueStart = (char*)entry+sizeof(EntryData)+defaultVSAMCSIFieldCount*sizeof(short); - for (int j=0; jlength; i++){ - EntryData *currentEntry = entrySet->entries[i]; - int fieldDataLength = currentEntry->data.fieldInfoHeader.totalLength; - int entrySize = sizeof(EntryData)+fieldDataLength-4; - memset((char*)(currentEntry),0,entrySize); - safeFree((char*)(currentEntry),entrySize); - } - memset((char*)(entrySet->entries),0,sizeof(EntryData*)*entrySet->length); - safeFree((char*)(entrySet->entries),sizeof(EntryData*)*entrySet->size); - memset((char*)entrySet,0,sizeof(EntryDataSet)); - safeFree((char*)entrySet,sizeof(EntryDataSet)); - - EntryDataSet *entrySet = returnEntries(dsnData, clusterTypesAllowed, clusterTypesCount, 0, defaultVSAMCSIFields, defaultVSAMCSIFieldCount, NULL, NULL, returnParms); - entry = entrySet->entries[0]; - } else if (entry->type != 'D') { - safeFree31((char*)returnParms,sizeof(csi_parmblock)); - for (int i = 0; i < entrySet->length; i++){ - EntryData *currentEntry = entrySet->entries[i]; - if (!(entrySet->entries)) break; - int fieldDataLength = currentEntry->data.fieldInfoHeader.totalLength; - int entrySize = sizeof(EntryData)+fieldDataLength-4; - safeFree((char*)(currentEntry),entrySize); - } - safeFree((char*)(entrySet->entries),sizeof(EntryData*)*entrySet->length); - safeFree((char*)entrySet,sizeof(EntryDataSet)); - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"Not Found in Catalog"); - return; - } - - if (entry->type == 'D') { - memcpy(dsnData, entry->name, 44); - unsigned short *fieldLengthArray = ((unsigned short *)((char*)entry+sizeof(EntryData))); - char *fieldValueStart = (char*)entry+sizeof(EntryData)+defaultVSAMCSIFieldCount*sizeof(short); - for (int j=0; j>= (sizeof(int) * 8 - 16); /* in case of weird int sizes */ - } - if (!strcmp(defaultVSAMCSIFields[j],"ASSOC ") && fieldLengthArray[j]){ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "CLUSTER = '%44.44s' [%c]\n", fieldValueStart+1, *fieldValueStart); - memcpy(dsnCluster,fieldValueStart+1,44); - } - if (!strcmp(defaultVSAMCSIFields[j],"AMDCIREC") && fieldLengthArray[j]){ - memcpy(&ciSize,fieldValueStart,4); - ciSize >>= (sizeof(int) * 8 - 32); /* in case of weird int sizes */ - memcpy(&maxlrecl,fieldValueStart+4,4); - maxlrecl >>= (sizeof(int) * 8 - 32); /* in case of weird int sizes */ - } - if (!strcmp(defaultVSAMCSIFields[j],"AMDKEY ") && fieldLengthArray[j]){ - memcpy(&keyLoc,fieldValueStart,2); - keyLoc >>= (sizeof(int) * 8 - 16); /* in case of weird int sizes */ - memcpy(&keyLen,fieldValueStart+2,2); - keyLen >>= (sizeof(int) * 8 - 16); /* in case of weird int sizes */ - } - fieldValueStart += fieldLengthArray[j]; - } - } - } else { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Catalog Entry not found for \"%s\"\n", dsn); - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"Not Found in Catalog"); - return; - } /* end Catalog Search */ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "vsamType = 0x%0x, ciSize = %d, maxlrecl = %d, keyLoc = %d, keyLen = %d\n", vsamType, ciSize, maxlrecl, keyLoc, keyLen); - - safeFree31((char*)returnParms,sizeof(csi_parmblock)); - for (int i = 0; i < entrySet->length; i++){ - EntryData *currentEntry = entrySet->entries[i]; - if (!(entrySet->entries)) break; - if (currentEntry == (EntryData *)0x000a0000) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "... how did this happen:\n"); - dumpbuffer((char *)entrySet, 1000); - } - int fieldDataLength = currentEntry->data.fieldInfoHeader.totalLength; - int entrySize = sizeof(EntryData)+fieldDataLength-4; - safeFree((char*)(currentEntry),entrySize); - } - safeFree((char*)(entrySet->entries),sizeof(EntryData*)*entrySet->size); - safeFree((char*)entrySet,sizeof(EntryDataSet)); - - char *dsnUidPair = safeMalloc(44+8+1, "DSN,UID Pair Entry"); /* TODO: plug this leak for each time it is htPut below. */ - memset(dsnUidPair, ' ', 44+8); /* TODO: we will want to free it when the ACB closes. */ - memcpy(dsnUidPair, dsnData, 44); - memcpy(dsnUidPair+44, username, 8); - StatefulACB *state = (StatefulACB *)htGet(acbTable, dsnUidPair); - char *inACB = state ? state->acb : NULL; - int searchArg = 0; - void *argPtr = NULL; - - if (inACB) { /* an ACB exists */ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "ACB found at 0x%0x\n", inACB); - /* TODO: if opened for output, close and reopen for input by preserving existing macrf, rpl, maxrecl parms. Reset arg. */ - /* TODO: if the user submits a start key parm, this is where it should be entered */ - switch (state->type) { - case VSAM_TYPE_KSDS: - argPtr = state->argPtr.key; - break; - case VSAM_TYPE_ESDS: - argPtr = &(state->argPtr.rba); - break; - case VSAM_TYPE_LDS: - argPtr = &(state->argPtr.ci); - break; - case VSAM_TYPE_RRDS: - argPtr = &(state->argPtr.record); - break; - } - safeFree(dsnUidPair, 44+8+1); - } else { /* need to open an ACB */ - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "about to dynalloc a DD to %s\n", dsn); - DynallocInputParms inputParms; - memcpy(inputParms.dsName, dsn, DATASET_NAME_LEN); - memcpy(inputParms.ddName, "MVD00000", DD_NAME_LEN); - inputParms.disposition = DISP_SHARE; - char ddname[9] = "MVD00000"; - returnCode = dynallocDataset(&inputParms, &reasonCode); - - int ddNumber = 1; - while (reasonCode==0x4100000 && ddNumber < 100000) { - sprintf(inputParms.ddName, "MVD%05d", ddNumber); - sprintf(ddname, "MVD%05d", ddNumber); - returnCode = dynallocDataset(&inputParms, &reasonCode); - ddNumber++; - } - if (returnCode) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Dynalloc RC = %d, reasonCode = %x\n", returnCode, reasonCode); - respondWithError(response, HTTP_STATUS_INTERNAL_SERVER_ERROR, "Unable to allocate a DD for ACB"); - return; - } - - int macrfParms = 0; - maxbuffer = maxlrecl; - state = (StatefulACB *)safeMalloc(sizeof(StatefulACB), "Stateful ACB Entry"); - if (vsamType & CSI_VSAMTYPE_KSDS) { - macrfParms = ACB_MACRF_KEY | ACB_MACRF_SEQ | ACB_MACRF_IN; - rplParms = RPL_OPTCD_KEY | RPL_OPTCD_SEQ; - state->type = VSAM_TYPE_KSDS; - } else if (vsamType & (CSI_VSAMTYPE_RRDS | CSI_VSAMTYPE_VRRDS)) { - macrfParms = ACB_MACRF_SEQ | ACB_MACRF_IN; - rplParms = RPL_OPTCD_KEY | RPL_OPTCD_SEQ; - state->type = VSAM_TYPE_RRDS; - } else if (vsamType & CSI_VSAMTYPE_LDS) { - macrfParms = ACB_MACRF_CNV | ACB_MACRF_SEQ | ACB_MACRF_IN; - rplParms = RPL_OPTCD_CNV | RPL_OPTCD_SEQ; - state->type = VSAM_TYPE_LDS; - maxbuffer = ciSize; - } else { /* assume ESDS otherwise */ - macrfParms = ACB_MACRF_ADR | ACB_MACRF_SEQ | ACB_MACRF_IN; - rplParms = RPL_OPTCD_ADR | RPL_OPTCD_SEQ; - state->type = VSAM_TYPE_ESDS; - } - inACB = openACB(ddname, ACB_MODE_INPUT, macrfParms, 0, rplParms, maxlrecl, maxbuffer); - if (!inACB) { - /* Failed to open ACB! */ - /* TODO: free the few things that we created above that will leak */ - respondWithError(response, HTTP_STATUS_INTERNAL_SERVER_ERROR,"ACB Not Opened"); - return; - } - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "ACB for %s opened at %08x\n", ddname, inACB); - /* TODO: if the user submits a start key parm, enter it in the point function calls below */ - state->acb = inACB; - switch (state->type) { - case VSAM_TYPE_KSDS: - state->argPtr.key = (char *)safeMalloc(keyLen+1, "Stateful ACB Key Buffer"); - memset(state->argPtr.key, 0, keyLen+1); - argPtr = state->argPtr.key; - /* returnCode = pointByKey(inACB, (char *)argPtr, keyLen); TODO: add only when the user provides a known key */ - break; - case VSAM_TYPE_ESDS: - state->argPtr.rba = 0; - argPtr = &(state->argPtr.rba); - returnCode = pointByRBA(inACB, argPtr); - break; - case VSAM_TYPE_LDS: - state->argPtr.ci = 0; - argPtr = &(state->argPtr.ci); - returnCode = pointByCI(inACB, argPtr); - break; - case VSAM_TYPE_RRDS: - state->argPtr.record = 1; - argPtr = &(state->argPtr.record); - returnCode = pointByRecord(inACB, argPtr); - break; - } - - if (returnCode) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "point failed with RC = %08x\n", returnCode); - /* TODO: free the few things that we created above that will otherwise leak */ - respondWithError(response, HTTP_STATUS_INTERNAL_SERVER_ERROR, "Could not POINT to the designated search argument."); - return; - } - htPut(acbTable, dsnUidPair, state); - } /* end Open */ - - RPLCommon *inRPL = (RPLCommon *) (inACB+RPL_COMMON_OFFSET+8); - /* TODO: if the user submits a backwards parm, this is where it should be added to rplParms RPL_OPTCD_BWD here */ - modRPL(inACB, inRPL->rplType, inRPL->keyLen, inRPL->workArea, inRPL->arg, - rplParms, inRPL->optcd2, inRPL->nextRPL, inRPL->recLen, inRPL->bufLen); - - jsonPrinter *jPrinter = respondWithJsonPrinter(response); - setResponseStatus(response, 200, "OK"); - setDefaultJSONRESTHeaders(response); - - writeHeader(response); - - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Streaming data for %s\n", absolutePath); - jsonStart(jPrinter); - /* TODO: if the user submits parms that limit the length of the returned data, modifying the below maxRecords or maxBytes will be needed */ - returnCode = streamVSAMDataset(response, inACB, maxlrecl, maxRecords, maxBytes, keyLoc, keyLen, jPrinter); - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Dataset bytesRead = %d\n", returnCode); - jsonEnd(jPrinter); - dumpbuffer((char *)state, sizeof(StatefulACB)); - - finishResponse(response); - if (closeAfter == TRUE){ - closeACB(inACB,ACB_MODE_INPUT); - htRemove(acbTable,dsnUidPair); - safeFree(dsnUidPair, 44+8+1); - safeFree((char*)state,sizeof(StatefulACB)); - } - -#endif /* __ZOWE_OS_ZOS */ -} - -int decodePercentByte(char *inString, int inLength, char *outString, int *outStringLength) { - int outPos = 0; - for (int i = 0; i < inLength; i++) { - if (inString[i] == '%') { - if (i >= inLength-2) { - zowelog(NULL, LOG_COMP_RESTDATASET, ZOWE_LOG_DEBUG, "Error: Percent seen without following 2 hex characters for '%s'\n",inString); - return 2; - } - char hex[3]; - hex[2] = '\0'; - strncpy(hex,&inString[i+1],2); - char hexChar = (char)(0xff & strtol(hex,NULL,16)); - memcpy(&outString[outPos],&hexChar,1); - //outString[outPos] = hexChar; - outPos++; - i = i+2; - } - else { - outString[outPos] = inString[i]; - outPos++; - } - } - outString[outPos] = '\0'; - *outStringLength = outPos; - return 0; -} - -void respondWithDatasetMetadata(HttpResponse *response) { -#ifdef __ZOWE_OS_ZOS - HttpRequest *request = response->request; - char *datasetOrMember = stringListPrint(request->parsedFile, 2, 2, "?", 0); /*get search term*/ - - if (datasetOrMember == NULL || strlen(datasetOrMember) < 1){ - respondWithError(response,HTTP_STATUS_BAD_REQUEST,"No dataset name given"); - return; - } - char *username = response->request->username; - int dsnLen = strlen(datasetOrMember); - char *percentDecoded = cleanURLParamValue(response->slh, datasetOrMember); - char *absDsPathTemp = stringConcatenate(response->slh, "//'", percentDecoded); - char *absDsPath = stringConcatenate(response->slh, absDsPathTemp, "'"); - - if(!isDatasetPathValid(absDsPath)){ - respondWithError(response,HTTP_STATUS_BAD_REQUEST,"Invalid dataset path"); - return; - } - - /* From here on, we know we have a valid data path */ - int lParenIndex = indexOf(datasetOrMember, dsnLen, '(', 0); - int rParenIndex = indexOf(datasetOrMember, dsnLen, ')', 0); - DatasetName dsnName; - DatasetMemberName memName; - int memberNameLength = 0; - - extractDatasetAndMemberName(absDsPath, &dsnName, &memName); - memberNameLength = (unsigned int)rParenIndex - (unsigned int)lParenIndex -1; - - HttpRequestParam *addQualifiersParam = getCheckedParam(request,"addQualifiers"); - char *addQualifiersArg = (addQualifiersParam ? addQualifiersParam->stringValue : NULL); - - HttpRequestParam *detailParam = getCheckedParam(request,"detail"); - char *detailArg = (detailParam ? detailParam->stringValue : NULL); - - HttpRequestParam *typesParam = getCheckedParam(request,"types"); - char *typesArg = (typesParam ? typesParam->stringValue : defaultDatasetTypesAllowed); - - HttpRequestParam *listMembersParam = getCheckedParam(request,"listMembers"); - char *listMembersArg = (listMembersParam ? listMembersParam->stringValue : NULL); - - int datasetTypeCount = (typesArg == NULL) ? 3 : strlen(typesArg); - - HttpRequestParam *workAreaSizeParam = getCheckedParam(request,"workAreaSize"); - int workAreaSizeArg = (workAreaSizeParam ? workAreaSizeParam->intValue : 0); - - HttpRequestParam *migratedParam = getCheckedParam(request,"includeMigrated"); - char *migratedArg = (migratedParam ? migratedParam->stringValue : NULL); - - HttpRequestParam *unprintableParam = getCheckedParam(request,"includeUnprintable"); - char *unprintableArg = (unprintableParam ? unprintableParam->stringValue : ""); - int includeUnprintable = !strcmp(unprintableArg, "true") ? TRUE : FALSE; - - HttpRequestParam *resumeNameParam = getCheckedParam(request,"resumeName"); - char *resumeNameArg = (resumeNameParam ? resumeNameParam->stringValue : NULL); - - HttpRequestParam *resumeCatalogNameParam = getCheckedParam(request,"resumeCatalogName"); - char *resumeCatalogNameArg = (resumeCatalogNameParam ? resumeCatalogNameParam->stringValue : NULL); - - if (resumeNameArg != NULL) { - if (strlen(resumeNameArg) > 44) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"Malformed resume dataset name"); - } - if (resumeCatalogNameArg == NULL) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"Missing resume catalog name"); - } - else if (strlen(resumeCatalogNameArg) > 44) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"Malformed resume catalog name"); - } - } - else if (resumeCatalogNameArg != NULL) { - if (strlen(resumeCatalogNameArg) > 44) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"Malformed resume catalog name"); - } - if (resumeNameArg == NULL) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"Missing resume dataset name"); - } - else if (strlen(resumeNameArg) > 44) { - respondWithError(response, HTTP_STATUS_BAD_REQUEST,"Malformed resume dataset name"); - } - } - - if(addQualifiersArg != NULL) { - int addQualifiers = !strcmp(addQualifiersArg, "true"); -#define DSN_MAX_LEN 44 - char dsnNameNullTerm[DSN_MAX_LEN + 1] = {0}; //+1 for null term - memcpy(dsnNameNullTerm, dsnName.value, sizeof(dsnName.value)); - nullTerminate(dsnNameNullTerm, sizeof(dsnNameNullTerm) - 1); - if (addQualifiers && dsnLen <= DSN_MAX_LEN) { - int dblAsteriskPos = indexOfString(dsnNameNullTerm, dsnLen, "**", 0); - int periodPos = lastIndexOf(dsnNameNullTerm, dsnLen, '.'); - if (!(dblAsteriskPos == dsnLen - 2 && periodPos == dblAsteriskPos - 1)) { - if (dsnLen <= DSN_MAX_LEN - 3) { - snprintf(dsnNameNullTerm, DSN_MAX_LEN + 1, "%s.**", dsnNameNullTerm); - } - } - } - memcpy(dsnName.value, dsnNameNullTerm, strlen(dsnNameNullTerm)); -#undef DSN_MAX_LEN - } - - int fieldCount = defaultCSIFieldCount; - char **csiFields = defaultCSIFields; - char dsnNameNullTerm[45] = {0}; - memcpy(dsnNameNullTerm, dsnName.value, sizeof(dsnName.value)); - nullTerminate(dsnNameNullTerm, sizeof(dsnNameNullTerm) - 1); - csi_parmblock * __ptr32 returnParms = (csi_parmblock* __ptr32)safeMalloc31(sizeof(csi_parmblock),"CSI ParmBlock"); - EntryDataSet *entrySet = returnEntries(dsnNameNullTerm, typesArg,datasetTypeCount, workAreaSizeArg, csiFields, fieldCount, resumeNameArg, resumeCatalogNameArg, returnParms); - char *resumeName = returnParms->resume_name; - char *catalogName = returnParms->catalog_name; - int isResume = (returnParms->is_resume == 'Y'); - - jsonPrinter *jPrinter = respondWithJsonPrinter(response); - setResponseStatus(response, 200, "OK"); - setDefaultJSONRESTHeaders(response); - writeHeader(response); - char volser[7]; - memset(volser,0,7); - jsonStart(jPrinter); - jsonAddString(jPrinter,"_objectType","com.rs.mvd.base.dataset.metadata"); - jsonAddString(jPrinter,"_metadataVersion","1.1"); - { - if (lParenIndex > 0){ - /*requested a pds member.*/ - /*jsonAddBoolean(jPrinter,"dataFromPDSDirectory",TRUE);*/ - } - jsonAddInt(jPrinter,"hasMore",isResume); - if (isResume) { - jsonAddUnterminatedString(jPrinter,"resumeName",resumeName,44); - jsonAddUnterminatedString(jPrinter,"resumeCatalogName",catalogName,44); - } - jsonStartArray(jPrinter,"datasets"); - for (int i = 0; i < entrySet->length; i++){ - EntryData *entry = entrySet->entries[i]; - - if (entry) { - int fieldDataLength = entry->data.fieldInfoHeader.totalLength; - int entrySize = sizeof(EntryData)+fieldDataLength-4; /* -4 for the fact that the length is 4 from end of EntryData */ - int isMigrated = FALSE; - jsonStartObject(jPrinter, NULL); - int datasetNameLength = sizeof(entry->name); - char *datasetName = entry->name; - jsonAddUnterminatedString(jPrinter, "name", datasetName, datasetNameLength); - jsonAddUnterminatedString(jPrinter, "csiEntryType", &entry->type, 1); - int volserLength = 0; - memset(volser, 0, sizeof(volser)); - char type = entry->type; - if (type == 'A' || type == 'B' || type == 'D' || type == 'H'){ - char *fieldData = (char*)entry+sizeof(EntryData); - unsigned short *fieldLengthArray = ((unsigned short *)((char*)entry+sizeof(EntryData))); - char *fieldValueStart = (char*)entry+sizeof(EntryData)+fieldCount*sizeof(short); - for (int j=0; j 0); - int detail = !strcmp(detailArg, "true"); - - if (detail){ - if (!isMigrated || !strcmp(migratedArg, "true")){ - addDetailedDatasetMetadata(datasetName, datasetNameLength, - volser, volserLength, - jPrinter); - } - } - if (shouldListMembers) { - if (!isMigrated || !strcmp(migratedArg, "true")){ - addMemberedDatasetMetadata(datasetName, datasetNameLength, - volser, volserLength, - memName.value, memberNameLength, - jPrinter, includeUnprintable); - } - } - jsonEndObject(jPrinter); - safeFree((char*)(entry),entrySize); - } - } - jsonEndArray(jPrinter); - } - jsonEnd(jPrinter); - finishResponse(response); - safeFree31((char*)returnParms,sizeof(csi_parmblock)); - safeFree((char*)(entrySet->entries),sizeof(EntryData*)*entrySet->size); - safeFree((char*)entrySet,sizeof(EntryDataSet)); -#endif /* __ZOWE_OS_ZOS */ -} - - -static const char hlqFirstChar[29] = {'A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z','$','@','#'}; - -void respondWithHLQNames(HttpResponse *response, MetadataQueryCache *metadataQueryCache) { -#ifdef __ZOWE_OS_ZOS - HttpRequest *request = response->request; - setResponseStatus(response, 200, "OK"); - setDefaultJSONRESTHeaders(response); - jsonPrinter *jPrinter = respondWithJsonPrinter(response); - writeHeader(response); - EntryDataSet *hlqSet; - csi_parmblock * __ptr32 * __ptr32 returnParmsArray; - char **csiFields = defaultCSIFields; - int fieldCount = defaultCSIFieldCount; - - HttpRequestParam *cacheParam = getCheckedParam(request,"updateCache"); - char *cacheArg = (cacheParam ? cacheParam->stringValue : NULL); - - int updateCache = (cacheArg != NULL && !strcmp(cacheArg,"true")); - - if (metadataQueryCache->cachedHLQSet && updateCache==FALSE){ - hlqSet = metadataQueryCache->cachedHLQSet; - returnParmsArray = metadataQueryCache->cachedCSIParmblocks; - } - else{ - if (metadataQueryCache->cachedHLQSet != NULL){ - EntryData **entries = metadataQueryCache->cachedHLQSet->entries; - for (int i = 0; i < metadataQueryCache->cachedHLQSet->length; i++){ - EntryData *entry = entries[i]; - int fieldDataLength = entry->data.fieldInfoHeader.totalLength; - int entrySize = sizeof(EntryData)+fieldDataLength-4; /* -4 for the fact that the length is 4 from end of EntryData */ - safeFree((char*)entries[i],entrySize); - safeFree((char*)metadataQueryCache->cachedCSIParmblocks[i],sizeof(csi_parmblock)); - } - safeFree((char*)metadataQueryCache->cachedHLQSet->entries,sizeof(EntryData*)*(metadataQueryCache->cachedHLQSet->length)); - safeFree((char*)metadataQueryCache->cachedHLQSet,sizeof(EntryDataSet)); - } - HttpRequestParam *typesParam = getCheckedParam(request,"types"); - char *typesArg = (typesParam ? typesParam->stringValue : defaultDatasetTypesAllowed); - int datasetTypeCount = (typesArg == NULL) ? 3 : strlen(typesArg); - - HttpRequestParam *workAreaSizeParam = getCheckedParam(request,"workAreaSize"); - int workAreaSizeArg = (workAreaSizeParam ? workAreaSizeParam->intValue : 0); - - returnParmsArray = (csi_parmblock* __ptr32 * __ptr32)safeMalloc31(29*sizeof(csi_parmblock* __ptr32),"CSI Parm Results"); - hlqSet = getHLQs(typesArg, datasetTypeCount, workAreaSizeArg, csiFields, fieldCount, returnParmsArray); - } - jsonStart(jPrinter); - { - char letterOrSymbol[2]; - letterOrSymbol[1] = '\0'; - jsonStartArray(jPrinter,"csiResults"); - for (int i = 0; i < 29; i++){ - strncpy(letterOrSymbol,hlqFirstChar+i,1); - jsonStartObject(jPrinter,NULL); - jsonAddString(jPrinter, "name", letterOrSymbol); - csi_parmblock *returnParms = returnParmsArray[i]; - int isResume = (returnParms->is_resume == 'Y'); - jsonAddInt(jPrinter,"hasMore",isResume); - if (isResume){ - jsonAddUnterminatedString(jPrinter,"resumeName",returnParms->resume_name,sizeof(returnParms->resume_name)); - jsonAddUnterminatedString(jPrinter,"resumeCatalogName",returnParms->catalog_name,sizeof(returnParms->catalog_name)); - } - jsonEndObject(jPrinter); - } - jsonEndArray(jPrinter); - - jsonStartArray(jPrinter,"datasets"); - fflush(stdout); - for (int i = 0; i < hlqSet->length;i++){ - EntryData *entry = hlqSet->entries[i]; - if (entry) { - int fieldDataLength = entry->data.fieldInfoHeader.totalLength; - int entrySize = sizeof(EntryData)+fieldDataLength-4; /* -4 for the fact that the length is 4 from end of EntryData */ - if (isBlanks(entry->name, 0, sizeof(entry->name))){ - continue; - } - jsonStartObject(jPrinter, NULL); - jsonAddUnterminatedString(jPrinter, "name", entry->name, sizeof(entry->name)); - jsonAddUnterminatedString(jPrinter, "type", &entry->type, 1); - char type = entry->type; - if (type == 'A' || type == 'B' || type == 'D' || type == 'H'){ - int fieldDataLength = entry->data.fieldInfoHeader.totalLength; - char *fieldData = (char*)entry+sizeof(EntryData); - unsigned short *fieldLengthArray = ((unsigned short *)((char*)entry+sizeof(EntryData))); - char *fieldValueStart = (char*)entry+sizeof(EntryData)+fieldCount*sizeof(short); - for (int j=0; jcachedHLQSet = hlqSet; - metadataQueryCache->cachedCSIParmblocks = returnParmsArray; - finishResponse(response); -#endif /* __ZOWE_OS_ZOS */ -} - - -#endif /* not METTLE - the whole module */ - - -/* - This program and the accompanying materials are - made available under the terms of the Eclipse Public License v2.0 which accompanies - this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html - - SPDX-License-Identifier: EPL-2.0 - - Copyright Contributors to the Zowe Project. -*/ - diff --git a/h/datasetjson.h b/h/datasetjson.h deleted file mode 100644 index 8993b57fd..000000000 --- a/h/datasetjson.h +++ /dev/null @@ -1,83 +0,0 @@ - - -/* - This program and the accompanying materials are - made available under the terms of the Eclipse Public License v2.0 which accompanies - this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html - - SPDX-License-Identifier: EPL-2.0 - - Copyright Contributors to the Zowe Project. -*/ - -#ifndef __DATASETJSON__ -#define __DATASETJSON__ 1 - - -#include "stcbase.h" -#include "json.h" -#include "xml.h" -#include "jcsi.h" - -#define DATA_STREAM_BUFFER_SIZE 4096 - -#define SAF_AUTHORIZATION_READ 0x04 -#define SAF_AUTHORIZATION_UPDATE 0x08 - -typedef struct MetadataQueryCache_tag{ - EntryDataSet *cachedHLQSet; - csi_parmblock * __ptr32 * __ptr32 cachedCSIParmblocks; -} MetadataQueryCache; - -typedef struct serveVSAMCache_tag{ - hashtable *acbTable; -} serveVSAMCache; - -typedef struct StatefulACB_tag { - char *acb; /* contains 8-byte plist */ -#define VSAM_TYPE_KSDS 1 -#define VSAM_TYPE_ESDS 2 -#define VSAM_TYPE_LDS 3 -#define VSAM_TYPE_RRDS 4 - int type; - union { - char *key; - int rba; - int ci; - int record; - } argPtr; -} StatefulACB; - -int streamDataset(Socket *socket, char *filename, int recordLength, jsonPrinter *jPrinter); -int streamVSAMDataset(HttpResponse* response, char *acb, int maxRecordLength, int maxRecords, int maxBytes, int keyLoc, int keyLen, jsonPrinter *jPrinter); -void addDetailedDatasetMetadata(char *datasetName, int nameLength, - char *volser, int volserLength, - jsonPrinter *jPrinter); -void addMemberedDatasetMetadata(char *datasetName, int nameLength, - char *volser, int volserLength, - char *memberQuery, int memberLength, - jsonPrinter *jPrinter, - int includeUnprintable); -void respondWithDataset(HttpResponse* response, char* absolutePath, int jsonMode); -void respondWithVSAMDataset(HttpResponse* response, char* absolutePath, hashtable *acbTable, int jsonMode); -void respondWithDatasetMetadata(HttpResponse *response); -void respondWithHLQNames(HttpResponse *response, MetadataQueryCache *metadataQueryCache); -void updateDataset(HttpResponse* response, char* absolutePath, int jsonMode); -void updateVSAMDataset(HttpResponse* response, char* absolutePath, hashtable *acbTable, int jsonMode); -void deleteVSAMDataset(HttpResponse* response, char* absolutePath); -void deleteDatasetOrMember(HttpResponse* response, char* absolutePath); -char getCSIType(char* absolutePath); -bool isVsam(char CSIType); -#endif - - -/* - This program and the accompanying materials are - made available under the terms of the Eclipse Public License v2.0 which accompanies - this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html - - SPDX-License-Identifier: EPL-2.0 - - Copyright Contributors to the Zowe Project. -*/ - From 3e0d21f61ae021827d634a3b7880fbd7c94a1a31 Mon Sep 17 00:00:00 2001 From: Joe Date: Sat, 29 Jan 2022 17:49:26 -0500 Subject: [PATCH 02/42] support for json schema, including restoration of ability to compile and test on Windows and Linux Signed-off-by: Joe --- c/alloc.c | 4 +- c/charsets.c | 4 + c/collections.c | 30 +- c/json.c | 499 ++++++++++++++-- c/jsonschema.c | 1012 ++++++++++++++++++++++++++++++++ c/psxskt.c | 1482 +++++++++++++++++++++++++++++++++++++++++++++++ c/timeutls.c | 97 +++- c/utils.c | 22 +- c/winskt.c | 1123 +++++++++++++++++++++++++++++++++++ c/xlate.c | 1 + h/json.h | 93 +++ h/jsonschema.h | 69 +++ h/zowetypes.h | 6 + 13 files changed, 4374 insertions(+), 68 deletions(-) create mode 100644 c/jsonschema.c create mode 100644 c/psxskt.c create mode 100644 c/winskt.c create mode 100644 h/jsonschema.h diff --git a/c/alloc.c b/c/alloc.c index f6f1f567c..33742901a 100644 --- a/c/alloc.c +++ b/c/alloc.c @@ -518,8 +518,10 @@ static char *safeMalloc64Internal(int size, char *site, long long token){ } #if defined(METTLE) && defined(_LP64) res = getmain64((long long)sizeInMegabytes,NULL,NULL); -#elif defined(_LP64) /* LE case */ +#elif defined(_LP64) /* LE case */ res = malloc(size); /* According to Redpaper http://www.redbooks.ibm.com/redpapers/pdfs/redp9110.pdf allocated above bar */ +#elif defined(_MSC_VER) && defined(_M_X64) /* Windows 64 case */ + res = malloc(size); #else res = NULL; /* Invalid if not compiled _LP64 */ #endif diff --git a/c/charsets.c b/c/charsets.c index a9d1947fc..1a6e6ddcc 100644 --- a/c/charsets.c +++ b/c/charsets.c @@ -35,6 +35,10 @@ #ifdef __ZOWE_OS_WINDOWS +/* JOE 1/20/22 */ +#include + + /* Windows doc diff --git a/c/collections.c b/c/collections.c index 34d4e155b..608d70219 100644 --- a/c/collections.c +++ b/c/collections.c @@ -516,13 +516,13 @@ void htDump(hashtable *ht){ for (i=0; ibackboneSize; i++){ hashentry *entry = ht->backbone[i]; if (entry != NULL){ - printf("in slot %d: entry=%x\n",i,entry); + printf("in slot %d: entry=0x%p\n",i,entry); fflush(stdout); while (entry != NULL){ if (isString){ - printf(" key=%x '%s' value: %x\n",entry->key,entry->key,entry->value); + printf(" key=0x%p '%s' value: 0x%p\n",entry->key,(char*)entry->key,entry->value); } else{ - printf(" key=%x value: %x\n",entry->key,entry->value); + printf(" key=0x%p value: 0x%p\n",entry->key,entry->value); } fflush(stdout); entry = entry->next; @@ -793,7 +793,7 @@ void lhtMap(LongHashtable *ht, void (*visitor)(void *, int64, void *), void *use static void lruHashVisitor(void *key, void *value){ LRUElement *element = (LRUElement*)value; - printf(" %16.16s: 0x%x containing 0x%x\n",key,value,element->data); + printf(" %16.16s: 0x%p containing 0x%p\n",(char*)key,value,element->data); } LRUCache *makeLRUCache(int size){ @@ -822,13 +822,13 @@ void lruDump(LRUCache *cache){ LRUElement *element = cache->newest; printf(" Newest To Oldest:\n"); while (element){ - printf(" Elt: %16.16s -> 0x%x\n",element->digest,element->data); + printf(" Elt: %16.16s -> 0x%p\n",(char*)element->digest,element->data); element = element->older; } element = cache->oldest; printf(" Oldest To Newest:\n"); while (element){ - printf(" Elt: %16.16s -> 0x%x\n",element->digest,element->data); + printf(" Elt: %16.16s -> 0x%p\n",(char*)element->digest,element->data); element = element->newer; } printf(" In Hash Order:\n"); @@ -838,7 +838,7 @@ void lruDump(LRUCache *cache){ /* can change recency */ void *lruGet(LRUCache *cache, char *digest){ if (cache->trace){ - printf("LRU Get digest: 0x%x\n",digest); + printf("LRU Get digest: 0x%p\n",digest); dumpbuffer(digest,16); } LRUElement *element = (LRUElement*)htGet(cache->ht,digest); @@ -868,14 +868,14 @@ static LRUElement *allocLRUElement(LRUCache *cache){ */ void *lruStore(LRUCache *cache, char *digest, void *thing){ if (cache->trace){ - printf("LRU Store digest: 0x%x\n",digest); + printf("LRU Store digest: 0x%p\n",digest); dumpbuffer(digest,16); } hashtable *ht = cache->ht; LRUElement *existingElement = htGet(ht,digest); if (cache->trace){ - printf("lruStore existing = 0x%x\n",existingElement); + printf("lruStore existing = 0x%p\n",existingElement); } if (existingElement){ if (existingElement == cache->newest){ @@ -886,7 +886,7 @@ void *lruStore(LRUCache *cache, char *digest, void *thing){ } else if (existingElement == cache->oldest){ LRUElement *newOldest = cache->oldest->newer; if (cache->trace){ - printf("recaching oldest: rotate to front, newOldest = 0x%x\n",newOldest); + printf("recaching oldest: rotate to front, newOldest = 0x%p\n",newOldest); } existingElement->newer = NULL; @@ -903,7 +903,7 @@ void *lruStore(LRUCache *cache, char *digest, void *thing){ LRUElement *prev = existingElement->older; LRUElement *next = existingElement->newer; if (cache->trace){ - printf("recaching middle, prun between older=0x%x and newer 0x%x\n", + printf("recaching middle, prun between older=0x%p and newer 0x%p\n", prev->data,next->data); } @@ -924,14 +924,14 @@ void *lruStore(LRUCache *cache, char *digest, void *thing){ if (cache->oldest && cache->newest){ LRUElement *oldNewest = cache->newest; if (cache->trace){ - printf(">1 room case newest->data=0x%x\n",oldNewest->data); + printf(">1 room case newest->data=0x%p\n",oldNewest->data); } newElement->older = cache->newest; cache->newest = newElement; oldNewest->newer = newElement; } else{ if (cache->trace){ - printf("== 0 case, go for it, newElement=0x%x\n",newElement); + printf("== 0 case, go for it, newElement=0x%p\n",newElement); } newElement->older = NULL; cache->newest = newElement; @@ -946,7 +946,7 @@ void *lruStore(LRUCache *cache, char *digest, void *thing){ LRUElement *secondOldest = oldOldest->newer; LRUElement *recycledElement = (LRUElement*)htGet(ht,oldOldest->digest); if (cache->trace){ - printf("recycle case oldOldest=0x%x recycled = 0x%x\n",oldOldest,recycledElement); + printf("recycle case oldOldest=0x%p recycled = 0x%p\n",oldOldest,recycledElement); } void *decached = recycledElement->data; htRemove(ht,oldOldest->digest); @@ -965,7 +965,7 @@ void *lruStore(LRUCache *cache, char *digest, void *thing){ LRUElement *recycledElement = cache->oldest; if (cache->trace){ - printf("recycle case (size=1) recycled = 0x%x\n",recycledElement); + printf("recycle case (size=1) recycled = 0x%p\n",recycledElement); } void *decached = recycledElement->data; htRemove(ht,recycledElement->digest); diff --git a/c/json.c b/c/json.c index 02679ce12..86422f5cd 100644 --- a/c/json.c +++ b/c/json.c @@ -19,6 +19,7 @@ #include #include #include +#include #include "metalio.h" #include "qsam.h" @@ -35,6 +36,9 @@ #include #include #include +#include /* JOE 1/20/22 */ +#include /* JOE 1/20/22 */ +#include /* JOE 1/20/22 */ #include #endif @@ -48,6 +52,11 @@ #include "json.h" #include "charsets.h" +#ifdef __ZOWE_OS_WINDOWS +typedef int64_t ssize_t; +#endif + + /* * c89 -DTEST_JSON_PARSER "-Wc,langlvl(extc99),gonum,goff,hgpr,roconst,ASM,asmlib('SYS1.MACLIB')" -o parser -I ../h json.c utils.c alloc.c bpxskt.c charsets.c @@ -65,13 +74,15 @@ #define DEBUG(...) /* fprintf(stderr, __VA_ARGS__) */ #define DUMPBUF($b, $l) /* dumpBufferToStream($b, $l, stderr) */ +/* JOE ERROR() is a defined macro in windows, so it needs a more specific name here */ + #ifdef METTLE -# define ERROR(...) /* TODO */ +# define JSONERROR(...) /* TODO */ #else /* TODO Implement good error handling: the caller must have a way to know if * an error happened. Most methods currently are void and there's no error * flag on the printer structure */ -# define ERROR(...) fprintf(stderr, __VA_ARGS__) +# define JSONERROR(...) fprintf(stderr, __VA_ARGS__) #endif static @@ -164,7 +175,16 @@ void jsonWriteBufferInternal(jsonPrinter *p, char *text, int len) { p->customWrite(p, text, len); } else { while (bytesWritten < len) { -#ifndef METTLE +#if defined(__ZOWE_OS_WINDOWS) + int newWriteReturn = -666; /* send(p->fd,text+bytesWritten,len-bytesWritten,0); */ + if (p->fd == _fileno(stdout)){ + newWriteReturn = fwrite(text, 1, len, stdout); + } else if (p->fd == _fileno(stderr)){ + newWriteReturn = fwrite(text, 1, len, stderr); + } else { + printf("JOE WINDOWS FAKE SOCKET WRITE bad case\n"); + } +#elif !defined(METTLE) int newWriteReturn = write(p->fd,text+bytesWritten,len-bytesWritten); #else int newWriteReturn = socketWrite((Socket *)p->fd,text+bytesWritten,len-bytesWritten, @@ -173,14 +193,14 @@ void jsonWriteBufferInternal(jsonPrinter *p, char *text, int len) { loopCount++; if (newWriteReturn < 0) { /* TODO: Replace by zowelog(...) */ - ERROR("JSON: write error, rc %d, return code %d, reason code %08X\n", + JSONERROR("JSON: write error, rc %d, return code %d, reason code %08X\n", newWriteReturn, returnCode, reasonCode); jsonSetIOErrorFlag(p); break; } if (loopCount > 10) { /* TODO: Replace by zowelog(...) */ - ERROR("JSON: write error, too many attempts\n"); + JSONERROR("JSON: write error, too many attempts\n"); jsonSetIOErrorFlag(p); break; } @@ -245,7 +265,7 @@ convertToUtf8(jsonPrinter *p, size_t len, char text[len], int inputCCSID) { p->_conversionBufferSize, "JSON conversion buffer"); if (newBuf == NULL) { //the old buffer will be free'd by freeJsonPrinter() if allocated - ERROR("JSON: error, not enough memory to convert\n"); + JSONERROR("JSON: error, not enough memory to convert\n"); return -1; } p->_conversionBufferSize = newSize; @@ -257,14 +277,14 @@ convertToUtf8(jsonPrinter *p, size_t len, char text[len], int inputCCSID) { if (convRc == CHARSET_CONVERSION_SUCCESS) { return convesionOutputLength; } else if (convRc != CHARSET_SHORT_BUFFER) { - ERROR("JSON: conversion error, rc %d, reason %d\n", convRc, + JSONERROR("JSON: conversion error, rc %d, reason %d\n", convRc, reasonCode); return -1; } } while (reallocLimit-- > 0); if (reallocLimit == 0) { //the old buffer will be free'd by freeJsonPrinter() if allocated - ERROR("JSON: error, not enough memory to convert\n"); + JSONERROR("JSON: error, not enough memory to convert\n"); return -1; } return len; @@ -291,7 +311,7 @@ writeBufferWithEscaping(jsonPrinter *p, size_t len, char text[len]) { UTF8_GET_NEXT_CHAR(len, text, &i, &utf8Char, &getCharErr); if (getCharErr != 0) { - ERROR("JSON: invalid UTF-8, rc %d\n", getCharErr); + JSONERROR("JSON: invalid UTF-8, rc %d\n", getCharErr); return -1; } DEBUG("character at %p + %d, len %d, char %x\n", text, i, len, utf8Char); @@ -326,7 +346,7 @@ writeBufferWithEscaping(jsonPrinter *p, size_t len, char text[len]) { CHARSET_OUTPUT_USE_BUFFER, &ptrToUtf8Buf, sizeof (escapeUtf), CCSID_UTF_8, NULL, &convesionOutputLength, &reasonCode); if (convRc != CHARSET_CONVERSION_SUCCESS) { - ERROR("JSON: conversion error, rc %d, reason %d\n", + JSONERROR("JSON: conversion error, rc %d, reason %d\n", convRc, reasonCode); return -1; } @@ -362,8 +382,8 @@ void jsonConvertAndWriteBuffer(jsonPrinter *p, char *text, size_t len, DUMPBUF(text, len); newLen = convertToUtf8(p, len, text, inputCCSID); if (newLen < 0) { - ERROR("jsonConvertAndWriteBuffer() error: newLen = %d\n", - newLen); + JSONERROR("jsonConvertAndWriteBuffer() error: newLen = %d\n", + (int)newLen); return; } DEBUG("utf8, len %d:\n", newLen); @@ -376,8 +396,8 @@ void jsonConvertAndWriteBuffer(jsonPrinter *p, char *text, size_t len, bytesWritten = writeBufferWithEscaping(p, len, text); if (bytesWritten < 0) { - ERROR("jsonConvertAndWriteBuffer() error: bytesWritten = %d\n", - bytesWritten); + JSONERROR("jsonConvertAndWriteBuffer() error: bytesWritten = %d\n", + (int)bytesWritten); return; } } else { @@ -452,6 +472,13 @@ void jsonWriteInt64(jsonPrinter *p, int64 value) { jsonWrite(p, buffer, false, SOURCE_CODE_CHARSET); } +static +void jsonWriteDouble(jsonPrinter *p, double value) { + char buffer[64]; + snprintf(buffer, sizeof (buffer), "%f", value); + jsonWrite(p, buffer, false, SOURCE_CODE_CHARSET); +} + static void jsonWriteBoolean(jsonPrinter *p, int value) { jsonWrite(p, value ? "true" : "false", false, SOURCE_CODE_CHARSET); @@ -760,6 +787,20 @@ void jsonAddInt64(jsonPrinter *p, char *keyOrNull, int64 value) { jsonWriteInt64(p, value); } +void jsonAddDouble(jsonPrinter *p, char *keyOrNull, double value) { + if (jsonShouldStopWriting(p)) { + return; + } + if (p->isFirstLine) { + p->isFirstLine = FALSE; + } else { + jsonNewLine(p); + } + jsonWriteKeyAndSemicolon(p, keyOrNull); + jsonWriteDouble(p, value); +} + + void jsonSetIOErrorFlag(jsonPrinter *p) { p->ioErrorFlag = TRUE; } @@ -829,17 +870,6 @@ writeToBuffer(struct jsonPrinter_tag *p, char *text, int len) { #define JSON_TOKEN_BUFFER_SIZE_LIMIT 104857600 /* 100 MB (for one token) */ #endif -typedef struct JsonParser_tag JsonParser; -typedef struct JsonTokenizer_tag JsonTokenizer; -typedef struct JsonToken_tag JsonToken; - -struct JsonParser_tag { - JsonTokenizer *tokenizer; - JsonToken *unreadToken; - ShortLivedHeap *slh; - CharStream *in; - Json *jsonError; -}; struct JsonTokenizer_tag { CharStream *in; @@ -875,6 +905,8 @@ struct JsonToken_tag { #define JSON_TOKEN_BAD_COMMENT 17 #define JSON_TOKEN_UNEXPECTED_CHAR 18 #define JSON_TOKEN_STRING_TOO_LONG 19 +#define JSON_TOKEN_INT64 20 +#define JSON_TOKEN_FLOAT 21 #define JSON_TOKEN_UNMATCHED 666 int type; char *text; @@ -895,7 +927,8 @@ static Json *jsonGetString(JsonParser* parser); static Json *jsonParse(JsonParser* parser); static JsonParser *makeJsonParser(ShortLivedHeap *slh, char *jsonString, int len); static JsonToken *getReservedWordToken(JsonTokenizer *tokenizer); -static JsonToken *getNumberToken(JsonTokenizer *tokenizer); +static JsonToken *getNumberTokenV1(JsonTokenizer *tokenizer); +static JsonToken *getNumberTokenV2(JsonTokenizer *tokenizer); static JsonToken *getStringToken(JsonTokenizer *tokenizer); static JsonToken *getLineCommentToken(JsonTokenizer *tokenizer); static JsonToken *getBlockCommentToken(JsonTokenizer *tokenizer); @@ -910,7 +943,7 @@ static void freeJsonParser(JsonParser *parser); static void freeJsonTokenizer(JsonTokenizer *tokenizer); static void jsonArrayAddElement(JsonParser *parser, JsonArray *arr, Json *element); static void jsonPrintInternal(jsonPrinter* printer, char* keyOrNull, Json *json); -static void jsonOjectAddProperty(JsonParser *parser, JsonObject *obj, char *key, Json *value); +static void jsonObjectAddProperty(JsonParser *parser, JsonObject *obj, char *key, Json *value); static void jsonParseFail(JsonParser *parser, char *formatString, ...); static void jsonTokenizerSkipWhitespace(JsonTokenizer *t); @@ -951,6 +984,8 @@ char *getTokenTypeString(int type) { return "unmatched token"; case JSON_TOKEN_STRING_TOO_LONG: return "string too long"; + case JSON_TOKEN_FLOAT: + return "float"; default: return "unknown token"; } @@ -1047,6 +1082,8 @@ void freeJsonParser(JsonParser *parser) { static void jsonParseFail(JsonParser *parser, char *formatString, ...) { + printf("JSON PARSE FAIL!!!!!\n"); + fflush(stdout); if (parser->jsonError == NULL) { int size = 1024; char *buffer = jsonParserAlloc(parser, size); @@ -1203,7 +1240,7 @@ JsonToken *getStringToken(JsonTokenizer *tokenizer) { } static -JsonToken *getNumberToken(JsonTokenizer *tokenizer) { +JsonToken *getNumberTokenV1(JsonTokenizer *tokenizer) { char *buffer = tokenizer->buffer; int pos = 0; int badNumber = FALSE; @@ -1231,6 +1268,59 @@ JsonToken *getNumberToken(JsonTokenizer *tokenizer) { return makeJsonToken(tokenizer, badNumber ? JSON_TOKEN_BAD_NUMBER : JSON_TOKEN_NUMBER, text); } +static int addTokenChar(JsonTokenizer *tokenizer, int pos, int c){ + if (pos < tokenizer->bufferSize){ + tokenizer->buffer[pos++] = c; + } + return pos; +} + +static +JsonToken *getNumberTokenV2(JsonTokenizer *tokenizer) { + char *buffer = tokenizer->buffer; + int pos = 0; + bool isInteger = true; + bool badNumber = false; + + if (jsonTokenizerLookahead(tokenizer) == '-') { + buffer[pos++] = jsonTokenizerRead(tokenizer); + } + if (isdigit(jsonTokenizerLookahead(tokenizer))) { + buffer[pos++] = jsonTokenizerRead(tokenizer); + while (isdigit(jsonTokenizerLookahead(tokenizer))) { + int c = jsonTokenizerRead(tokenizer); + pos = addTokenChar(tokenizer,pos,c); + } + if (jsonTokenizerLookahead(tokenizer) == '.'){ + isInteger = false; + pos = addTokenChar(tokenizer,pos,jsonTokenizerRead(tokenizer)); + if (isdigit(jsonTokenizerLookahead(tokenizer))){ + pos = addTokenChar(tokenizer,pos,jsonTokenizerRead(tokenizer)); + while (isdigit(jsonTokenizerLookahead(tokenizer))) { + int c = jsonTokenizerRead(tokenizer); + pos = addTokenChar(tokenizer,pos,c); + } + } else { + badNumber = true; + } + } + /* must have at least one number, followed by a lookahead to a non-number here */ + } else { + badNumber = true; + int c = jsonTokenizerRead(tokenizer); + pos = addTokenChar(tokenizer,pos,c); + } + char *text = jsonTokenizerAlloc(tokenizer, pos + 1); + memcpy(text, buffer, pos); + if (badNumber){ + return makeJsonToken(tokenizer,JSON_TOKEN_BAD_NUMBER,text); + } else if (isInteger){ + return makeJsonToken(tokenizer,JSON_TOKEN_INT64,text); + } else { + return makeJsonToken(tokenizer,JSON_TOKEN_FLOAT,text); + } +} + static JsonToken *getReservedWordToken(JsonTokenizer *tokenizer) { char *buffer = tokenizer->buffer; @@ -1368,10 +1458,15 @@ JsonToken *jsonNextToken(JsonParser* parser) { lookahead = jsonTokenizerLookahead(tokenizer); tokenizer->lastTokenLineNumber = tokenizer->lineNumber; tokenizer->lastTokenColumnNumber = tokenizer->columnNumber; + /* printf("JOE jsonNextToken lookahead=0x%x '%c'\n",(int)lookahead,(char)lookahead); + fflush(stdout); + */ if (lookahead == '\"') { token = getStringToken(tokenizer); } else if (isdigit(lookahead) || lookahead == '-') { - token = getNumberToken(tokenizer); + token = (parser->version >= 2 ? + getNumberTokenV2(tokenizer) : + getNumberTokenV1(tokenizer)); } else if (isalpha(lookahead)) { token = getReservedWordToken(tokenizer); } else if (lookahead == '{') { @@ -1401,12 +1496,13 @@ JsonToken *jsonNextToken(JsonParser* parser) { jsonParseFail(parser, "unexpected character %c(0x%x)\n", lookahead, lookahead); token = makeJsonToken(tokenizer, JSON_TOKEN_UNEXPECTED_CHAR, ""); } - + if (token->type == JSON_TOKEN_LINE_COMMENT || token->type == JSON_TOKEN_BLOCK_COMMENT) { token = jsonNextToken(parser); } jsonValidateToken(parser, token); - + + /* printf("JOE returning next token=0x%p\n",token);fflush(stdout); */ return token; } @@ -1475,6 +1571,35 @@ Json *jsonGetNumber(JsonParser* parser) { return json; } +static +Json *jsonGetInt64(JsonParser* parser) { + Json *json = NULL; + JsonToken *token = jsonMatchToken(parser, JSON_TOKEN_INT64); + if (!jsonIsTokenUnmatched(token)) { + json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_INT64; + json->data.integerValue = strtoll(token->text,NULL,10); + } else { + json = parser->jsonError; + } + return json; +} + +static +Json *jsonGetFloat(JsonParser* parser) { + Json *json = NULL; + JsonToken *token = jsonMatchToken(parser, JSON_TOKEN_FLOAT); + if (!jsonIsTokenUnmatched(token)) { + json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_DOUBLE; + json->data.floatValue = strtod(token->text,NULL); + } else { + json = parser->jsonError; + } + return json; +} + + static Json *jsonGetString(JsonParser* parser) { Json *json = NULL; @@ -1517,7 +1642,7 @@ Json *jsonGetNull(JsonParser* parser) { } static -void jsonOjectAddProperty(JsonParser *parser, JsonObject *obj, char *key, Json *value) { +void jsonObjectAddProperty(JsonParser *parser, JsonObject *obj, char *key, Json *value) { JsonProperty *property = (JsonProperty*) jsonParserAlloc(parser, sizeof (JsonProperty)); property->key = key; property->value = value; @@ -1600,10 +1725,13 @@ int jsonArrayContainsString(JsonArray *array, const char *s){ static Json *jsonGetObject(JsonParser* parser) { JsonToken *token = jsonMatchToken(parser, JSON_TOKEN_OBJECT_START); + /* printf("JGO ck.1\n");fflush(stdout); */ if (jsonIsTokenUnmatched(token)) { + /* printf("JGO ck.1.1\n");fflush(stdout); */ return parser->jsonError; } Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + /* printf("JGO ck.2\n");fflush(stdout); */ JsonObject *obj = (JsonObject*) jsonParserAlloc(parser, sizeof (JsonObject)); json->type = JSON_TYPE_OBJECT; json->data.object = obj; @@ -1619,7 +1747,7 @@ static Json *jsonGetObject(JsonParser* parser) { if (jsonIsError(valueJSON)) { return parser->jsonError; } - jsonOjectAddProperty(parser, obj, keyToken->text, valueJSON); + jsonObjectAddProperty(parser, obj, keyToken->text, valueJSON); lookahead = jsonLookaheadToken(parser); if (lookahead->type == JSON_TOKEN_COMMA) { jsonMatchToken(parser, JSON_TOKEN_COMMA); @@ -1678,10 +1806,17 @@ static Json *jsonParse(JsonParser* parser) { JsonToken *lookahead = jsonLookaheadToken(parser); Json *json = NULL; if (lookahead) { + /* printf("JOE: lookahead type %d\n",lookahead->type);fflush(stdout); */ switch (lookahead->type) { case JSON_TOKEN_NUMBER: json = jsonGetNumber(parser); break; + case JSON_TOKEN_INT64: + json = jsonGetInt64(parser); + break; + case JSON_TOKEN_FLOAT: + json = jsonGetFloat(parser); + break; case JSON_TOKEN_STRING: json = jsonGetString(parser); break; @@ -1705,6 +1840,193 @@ static Json *jsonParse(JsonParser* parser) { return json; } + +JsonBuilder *makeJsonBuilder(ShortLivedHeap *slh){ + JsonBuilder *builder = (JsonBuilder*) safeMalloc(sizeof (JsonBuilder), "JSON Builder"); + memset(builder,0,sizeof(JsonBuilder)); + JsonParser *parser = (JsonParser*)builder; + parser->slh = slh; + return builder; +} + +#define ADD_TO_ROOT -1 +#define ADD_TO_OBJECT -2 +#define ADD_TO_ARRAY -3 + +void freeJsonBuilder(JsonBuilder *builder, bool freeSLH){ + if (freeSLH){ + JsonParser *parser = (JsonParser*)builder; + SLHFree(parser->slh); + } + safeFree((char*)builder,sizeof(JsonBuilder)); +} + +static int checkParentLValue(Json *parent, + char *parentKey){ + if (parent){ + switch (parent->type){ + case JSON_TYPE_OBJECT: + return (parentKey ? ADD_TO_OBJECT : JSON_BUILD_FAIL_NO_KEY_ON_OBJECT); + case JSON_TYPE_ARRAY: + return (parentKey ? JSON_BUILD_FAIL_KEY_ON_ARRAY : ADD_TO_ARRAY); + default: + return JSON_BUILD_FAIL_PARENT_IS_SCALAR; + } + } else { + return (parentKey ? JSON_BUILD_FAIL_KEY_ON_ROOT : ADD_TO_ROOT); + } +} + +static void addToParent(JsonBuilder *b, + int lvalueStatus, + Json *parent, + char *parentKey, + Json *json){ + JsonParser *p = (JsonParser*)b; + switch (lvalueStatus){ + case ADD_TO_ROOT: + b->root = json; + break; + case ADD_TO_OBJECT: + jsonObjectAddProperty(p,parent->data.object,parentKey,json); + break; + case ADD_TO_ARRAY: + jsonArrayAddElement(p,parent->data.array,json); + break; + } +} + +Json *jsonBuildObject(JsonBuilder *b, + Json *parent, + char *parentKey, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + JsonObject *obj = (JsonObject*) jsonParserAlloc(parser, sizeof (JsonObject)); + json->type = JSON_TYPE_OBJECT; + json->data.object = obj; + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + } else { + *errorCode = lvalueStatus; + return NULL; + } +} + +Json *jsonBuildArray(JsonBuilder *b, + Json *parent, + char *parentKey, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + JsonArray *arr = (JsonArray*) jsonParserAlloc(parser, sizeof (JsonArray)); + arr->count = 0; + arr->capacity = 8; + arr->elements = (Json**) jsonParserAlloc(parser, sizeof (Json*) * arr->capacity); + json->type = JSON_TYPE_ARRAY; + json->data.array = arr; + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + + } else { + *errorCode = lvalueStatus; + return NULL; + } +} + +Json *jsonBuildString(JsonBuilder *b, + Json *parent, + char *parentKey, + char *s, + int sLen, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_STRING; + char *copy = jsonParserAlloc(parser,sLen+1); + memcpy(copy,s,sLen); + copy[sLen] = 0; + json->data.string = copy; + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + + } else { + *errorCode = lvalueStatus; + return NULL; + } + +} + +Json *jsonBuildInt(JsonBuilder *b, + Json *parent, + char *parentKey, + int i, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_NUMBER; + json->data.number = i; + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + } else { + *errorCode = lvalueStatus; + return NULL; + } + +} + +Json *jsonBuildBool(JsonBuilder *b, + Json *parent, + char *parentKey, + bool *truthValue, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_BOOLEAN; + json->data.boolean = (truthValue ? 1 : 0); + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + } else { + *errorCode = lvalueStatus; + return NULL; + } +} + +Json *jsonBuildNull(JsonBuilder *b, + Json *parent, + char *parentKey, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_NULL; + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + } else { + *errorCode = lvalueStatus; + return NULL; + } +} + + + int jsonIsObject(Json *json) { return json->type == JSON_TYPE_OBJECT; } @@ -1722,7 +2044,17 @@ int jsonIsBoolean(Json *json) { } int jsonIsNumber(Json *json) { - return json->type == JSON_TYPE_NUMBER; + return (json->type == JSON_TYPE_NUMBER || /* old-skool int */ + json->type == JSON_TYPE_DOUBLE || + json->type == JSON_TYPE_INT64); +} + +int jsonIsInt64(Json *json) { + return json->type == JSON_TYPE_INT64; +} + +int jsonIsDouble(Json *json) { + return json->type == JSON_TYPE_DOUBLE; } int jsonIsNull(Json *json) { @@ -1760,10 +2092,81 @@ int jsonAsBoolean(Json *json) { int jsonAsNumber(Json *json) { if (json->type == JSON_TYPE_NUMBER) { return json->data.number; + } else if (json->type == JSON_TYPE_INT64){ + return json->data.integerValue; } return 0; } +int64_t jsonAsInt64(Json *json){ + switch (json->type){ + case JSON_TYPE_NUMBER: + return (int64)json->data.number; + case JSON_TYPE_INT64: + return json->data.integerValue; + case JSON_TYPE_DOUBLE: + return (int64)json->data.floatValue; + default: + return 0; + } +} + +double jsonAsDouble(Json *json){ + switch (json->type){ + case JSON_TYPE_NUMBER: + return (double)json->data.number; + case JSON_TYPE_INT64: + return (double)json->data.integerValue; + case JSON_TYPE_DOUBLE: + return json->data.floatValue; + default: + return 0; + } +} + +static int64 hashString(char *s){ + int64 hash = 5381; + int i; + int len = strlen(s); + + for (i=0; itype) << 56; + int64_t mask = (1ll<<56)-1; + switch (json->type) { + case JSON_TYPE_NUMBER: + hash |= (mask & jsonAsNumber(json)); + break; + case JSON_TYPE_INT64: + hash |= (mask & jsonAsInt64(json)); + break; + case JSON_TYPE_DOUBLE: + hash |= (mask & (int64)jsonAsDouble(json)); + break; + case JSON_TYPE_STRING: + hash |= (mask & hashString(jsonAsString(json))); + break; + case JSON_TYPE_BOOLEAN: + hash |= (jsonAsBoolean(json) ? 1 : 0); + break; + case JSON_TYPE_NULL: + hash |= 0; + break; + case JSON_TYPE_OBJECT: + hash |= (int64)json; + break; + case JSON_TYPE_ARRAY: + hash |= (int64)json; + break; + } + return hash; +} + JsonArray *jsonArrayGetArray(JsonArray *array, int i) { Json *json = jsonArrayGetItem(array, i); if (json && jsonIsArray(json)) { @@ -1891,6 +2294,12 @@ void jsonPrintInternal(jsonPrinter* printer, char* keyOrNull, Json *json) { case JSON_TYPE_NUMBER: jsonAddInt(printer, keyOrNull, jsonAsNumber(json)); break; + case JSON_TYPE_INT64: + jsonAddInt64(printer, keyOrNull, jsonAsInt64(json)); + break; + case JSON_TYPE_DOUBLE: + jsonAddDouble(printer, keyOrNull, jsonAsDouble(json)); + break; case JSON_TYPE_STRING: jsonAddString(printer, keyOrNull, jsonAsString(json)); break; @@ -1962,7 +2371,8 @@ Json *jsonParseUnterminatedUtf8String(ShortLivedHeap *slh, int outputCCSID, errorBufferOrNull, errorBufferSize); } -Json *jsonParseFile(ShortLivedHeap *slh, const char *filename, char* errorBufferOrNull, int errorBufferSize) { +static Json *jsonParseFileInternal(ShortLivedHeap *slh, const char *filename, char* errorBufferOrNull, int errorBufferSize, + int version){ Json *json = NULL; int returnCode = 0; int reasonCode = 0; @@ -1971,11 +2381,17 @@ Json *jsonParseFile(ShortLivedHeap *slh, const char *filename, char* errorBuffer UnixFile *file = NULL; int fileLen = 0; + /* + printf("JOE jsonParseFile\n"); + fflush(stdout); + */ + file = fileOpen(filename, FILE_OPTION_READ_ONLY, 0, 1024, &returnCode, &reasonCode); status = fileInfo(filename, &info, &returnCode, &reasonCode); if (file != NULL && status == 0) { fileLen = (int)fileInfoSize(&info); JsonParser *parser = makeJsonFileParser(slh, file, fileLen); + parser->version = version; json = jsonParse(parser); JsonToken *token = jsonMatchToken(parser, JSON_TOKEN_EOF); if (jsonIsError(json) || jsonIsTokenUnmatched(token)) { @@ -1990,9 +2406,18 @@ Json *jsonParseFile(ShortLivedHeap *slh, const char *filename, char* errorBuffer snprintf(errorBufferOrNull, errorBufferSize, "Couldn't open '%s'", filename); } } + /* printf("JOE return json\n");fflush(stdout); */ return json; } +Json *jsonParseFile(ShortLivedHeap *slh, const char *filename, char* errorBufferOrNull, int errorBufferSize){ + return jsonParseFileInternal(slh,filename,errorBufferOrNull,errorBufferSize,JSON_PARSE_VERSION_1); +} + +Json *jsonParseFile2(ShortLivedHeap *slh, const char *filename, char* errorBufferOrNull, int errorBufferSize){ + return jsonParseFileInternal(slh,filename,errorBufferOrNull,errorBufferSize,JSON_PARSE_VERSION_2); +} + Json *jsonParseString(ShortLivedHeap *slh, char *jsonString, char* errorBufferOrNull, int errorBufferSize) { return jsonParseUnterminatedString(slh,jsonString,strlen(jsonString),errorBufferOrNull,errorBufferSize); } @@ -2064,10 +2489,10 @@ JsonObject *jsonObjectProperty(JsonObject *object, char *propertyName, int *stat void reportJSONDataProblem(void *jsonObject, int status, char *propertyName){ switch (status){ case JSON_PROPERTY_NOT_FOUND: - printf("JSON property '%s' not found in object at 0x%x\n",propertyName,jsonObject); + printf("JSON property '%s' not found in object at 0x%p\n",propertyName,jsonObject); break; case JSON_PROPERTY_UNEXPECTED_TYPE: - printf("JSON property '%s' has wrong type in object at 0x%x\n",propertyName,jsonObject); + printf("JSON property '%s' has wrong type in object at 0x%p\n",propertyName,jsonObject); break; } } diff --git a/c/jsonschema.c b/c/jsonschema.c new file mode 100644 index 000000000..93e845307 --- /dev/null +++ b/c/jsonschema.c @@ -0,0 +1,1012 @@ + + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#ifdef METTLE + +#include +#include +#include +#include +#include +#include +#include +#include +#include "metalio.h" +#include "qsam.h" + +#else + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 600 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#endif + +/* +#ifdef __ZOWE_OS_WINDOWS +#include "winregex.h" +#else +#include "regex.h" +#endif +*/ + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "openprims.h" +#include "bpxnet.h" +#include "unixfile.h" +#include "json.h" +#include "charsets.h" +#include "collections.h" +#include "jsonschema.h" + +#ifdef __ZOWE_OS_WINDOWS +typedef int64_t ssize_t; +#endif + +#define JSTYPE_NULL 0 +#define JSTYPE_BOOLEAN 1 +#define JSTYPE_OBJECT 2 +#define JSTYPE_ARRAY 3 +#define JSTYPE_STRING 4 +#define JSTYPE_NUMBER 5 +#define JSTYPE_INTEGER 6 +#define JSTYPE_MAX 6 + +#define JS_VALIDATOR_MULTOF 0x00000001 +#define JS_VALIDATOR_MAX 0x00000002 +#define JS_VALIDATOR_MIN 0x00000004 +#define JS_VALIDATOR_MAX_CONTAINS 0x00000008 +#define JS_VALIDATOR_MIN_CONTAINS 0x00000010 +#define JS_VALIDATOR_MAX_ITEMS 0x00000020 +#define JS_VALIDATOR_MIN_ITEMS 0x00000040 +#define JS_VALIDATOR_MAX_PROPS 0x00000080 +#define JS_VALIDATOR_MIN_PROPS 0x00000100 +#define JS_VALIDATOR_MAX_LENGTH 0x00000200 +#define JS_VALIDATOR_MIN_LENGTH 0x00000400 + +typedef struct JSValueSpec_tag { + int typeMask; + Json **enumeratedValues; + Json *constValue; + char *id; + char *description; + char *title; + char *ref; + bool deprecated; + bool nullable; + /* shared sub-schema definitions go here */ + hashtable *definitions; + /* Compositing */ + struct JSValueSpec_tag **allOf; + struct JSValueSpec_tag **anyOf; + struct JSValueSpec_tag **oneOf; + struct JSValueSpec_tag *not; + + /* optional validators have flag for presence */ + uint64_t validatorFlags; + + double minimum; + double maximum; + int64_t minContains; + int64_t maxContains; + int64_t minItems; + int64_t maxItems; + int64_t minProperties; + int64_t maxProperties; + int requiredCount; + char **required; + hashtable *properties; + hashtable *dependentRequired; + + /* for numbers */ + bool exclusiveMaximum; + bool exclusiveMinimum; + double multipleOf; + bool isInteger; + + /* for arrays */ + struct JSValueSpec_tag *itemSpec; + bool uniqueItems; + + /* for strings */ + int64_t minLength; + int64_t maxLength; + char *pattern; + /* regex_t *compiledPattern; */ +} JSValueSpec; + +typedef struct AccessPathUnion_tag { + +} AccessPathUnion; + + + +static void printAccessPath(FILE *out, AccessPath *path){ + for (int i=0; icurrentSize; i++){ + if (path->elements[i].isName){ + fprintf(out,"/%s",path->elements[i].key.name); + } else { + fprintf(out,"/%d",path->elements[i].key.index); + } + } + fprintf(out,"\n"); +} + +static char *makeAccessPathString(char *buffer, int bufferLen, AccessPath *path){ + int pos = 0; + for (int i=0; icurrentSize; i++){ + if (pos + 100 >= bufferLen){ + sprintf(buffer+pos,"..."); + break; + } + if (path->elements[i].isName){ + pos += sprintf(buffer+pos,"/%s",path->elements[i].key.name); + } else { + pos += sprintf(buffer+pos,"/%d",path->elements[i].key.index); + } + } + return buffer; +} + + +static void accessPathPushIndex(AccessPath *accessPath, int index){ + if (accessPath->currentSize < MAX_SCHEMA_DEPTH){ + accessPath->elements[accessPath->currentSize].isName = false; + accessPath->elements[accessPath->currentSize++].key.index = index; + } +} + +static void accessPathPushName(AccessPath *accessPath, char *name){ + if (accessPath->currentSize < MAX_SCHEMA_DEPTH){ + accessPath->elements[accessPath->currentSize].isName = true; + accessPath->elements[accessPath->currentSize++].key.name = name; + } +} + +static void accessPathPop(AccessPath *accessPath){ + if (accessPath->currentSize > 0){ + accessPath->currentSize--; + } +} + + +#define ERROR_MAX 1024 + +static void validationThrow(JsonValidator *validator, int errorCode, char *formatString, ...){ + va_list argPointer; + char *text = safeMalloc(ERROR_MAX,"ErrorBuffer"); + va_start(argPointer,formatString); + vsnprintf(text,ERROR_MAX,formatString,argPointer); + va_end(argPointer); + validator->errorCode = errorCode; + validator->errorMessage = text; + validator->errorMessageLength = ERROR_MAX; + longjmp(validator->recoveryData,1); +} + +JsonValidator *makeJsonValidator(){ + JsonValidator *validator = (JsonValidator*)safeMalloc(sizeof(JsonValidator),"JsonValidator"); + memset(validator,0,sizeof(JsonValidator)); + validator->accessPath = (AccessPath*)safeMalloc(sizeof(AccessPath),"JsonValidatorAccessPath"); + validator->accessPath->currentSize = 0; + return validator; +} + +void freeJsonValidator(JsonValidator *validator){ + safeFree((char*)validator->accessPath,sizeof(AccessPath)); + safeFree((char*)validator,sizeof(JsonValidator)); +} + +static char *validatorAccessPath(JsonValidator *validator){ + return makeAccessPathString(validator->accessPathBuffer,MAX_ACCESS_PATH,validator->accessPath); +} + +static void validateType(JsonValidator *validator, + int typeCode, + JSValueSpec *valueSpec){ + printf("validateType.1 typeCode=%d vspecMask=0x%x\n",typeCode,valueSpec->typeMask);fflush(stdout); + if (((1 << typeCode) & valueSpec->typeMask) == 0){ + validationThrow(validator,12,"type %d not permitted at %s",typeCode,validatorAccessPath(validator)); + } + printf("validateType.2\n");fflush(stdout); +} + +/* for mutual recursion */ +static void validateJSON(JsonValidator *validator, Json *value, JSValueSpec *valueSpec); + +static void validateJSONObject(JsonValidator *validator, + JsonObject *object, + JSValueSpec *valueSpec){ + AccessPath *accessPath = validator->accessPath; + printf("validateJSONObject required=0x%p\n",valueSpec->required); + printAccessPath(stdout,accessPath); + fflush(stdout); + if (valueSpec->required){ + for (int r=0; rrequiredCount; r++){ + char *requiredProperty = valueSpec->required[r]; + printf("VJO requiredProp=%s\n",requiredProperty);fflush(stdout); + if (jsonObjectGetPropertyValue(object,requiredProperty) == NULL){ + validationThrow(validator,12,"missing required property %s at %s", + requiredProperty,validatorAccessPath(validator)); + } + } + } + // dependent required should go here + int propertyCount = 0; + JsonProperty *property = jsonObjectGetFirstProperty(object); + while (property){ + propertyCount++; + char *propertyName = jsonPropertyGetKey(property); + printf("validate object pname=%s\n",propertyName); + fflush(stdout); + Json *propertyValue = jsonPropertyGetValue(property); + JSValueSpec *propertySpec = (valueSpec->properties ? + (JSValueSpec*)htGet(valueSpec->properties,propertyName) : + NULL); + accessPathPushName(accessPath,propertyName); + + if (propertySpec != NULL){ + validateJSON(validator,propertyValue,propertySpec); + } else { + printf("*WARNING* unspecified property seen, '%s', and checking code is not complete, vspec->props=0x%p\n", + propertyName,valueSpec->properties); + fflush(stdout); + if (valueSpec->properties){ + /* htDump(valueSpec->properties); */ + } + } + accessPathPop(accessPath); + property = jsonObjectGetNextProperty(property); + } + + // Interdependencies + if (valueSpec->dependentRequired != NULL){ + printf("*WARNING* depenentRequired not yet implemented\n"); + } + + + if (valueSpec->validatorFlags & JS_VALIDATOR_MAX_PROPS){ + int64_t lim = valueSpec->minProperties; + if (propertyCount > lim){ + validationThrow(validator,12,"too many properties, %d > MAX=%d at %s", + propertyCount,lim,validatorAccessPath(validator)); + } + } + if (valueSpec->validatorFlags & JS_VALIDATOR_MIN_PROPS){ + int64_t lim = valueSpec->minProperties; + if (propertyCount < lim){ + validationThrow(validator,12,"too few properties, %d < MIN=%d at %s", + propertyCount,lim,validatorAccessPath(validator)); + } + } + +} + +void validateJSONArray(JsonValidator *validator, + JsonArray *array, + JSValueSpec *valueSpec){ + AccessPath *accessPath = validator->accessPath; + //boolean uniqueItems; + // Long maxContains; + // Long minContains; + int elementCount = jsonArrayGetCount(array); + if (valueSpec->validatorFlags & JS_VALIDATOR_MAX_ITEMS){ + int64_t lim = valueSpec->maxItems; + if (elementCount > lim){ + validationThrow(validator,12, + "too many array items, %d > MAX=%d at %s", + elementCount,lim,validatorAccessPath(validator)); + } + } + if (valueSpec->validatorFlags & JS_VALIDATOR_MIN_ITEMS){ + int64_t lim = valueSpec->minItems; + if (elementCount < lim){ + validationThrow(validator,12, + "too few array ites, %d < MIN=%d at %s", + elementCount,lim,validatorAccessPath(validator)); + } + } + /* 'uniqueItems' is poorly specified regarding equality predicates. + for now it will use native equals() from JSValue. + It's kinda-crazy to do a true equal on printed representation on things that are + unlimited in size and complexity. + */ + LongHashtable *uniquenessSet = NULL; + if (valueSpec->uniqueItems){ + uniquenessSet = lhtCreate(257,NULL); + } + + for (int i=0; iitemSpec != NULL){ + validateJSON(validator,itemValue,valueSpec->itemSpec); + } + if (valueSpec->uniqueItems){ + long longHash = jsonLongHash(itemValue); + if (lhtGet(uniquenessSet,longHash) != NULL){ + validationThrow(validator,12,"array uniqueItems violation %s is duplicate at %s", + itemValue,i,validatorAccessPath(validator)); + } + } + accessPathPop(accessPath); + } +} + +static void validateJSONString(JsonValidator *validator, + Json *string, + JSValueSpec *valueSpec){ + char *s = jsonAsString(string); + int len = strlen(s); + if (valueSpec->validatorFlags & JS_VALIDATOR_MAX_LENGTH){ + long lim = valueSpec->maxLength; + if (len > lim){ + validationThrow(validator,12,"string too long, %d > MAX=%d at %s", + len,lim,validatorAccessPath(validator)); + } + } + if (valueSpec->validatorFlags & JS_VALIDATOR_MIN_LENGTH){ + long lim = valueSpec->minLength; + if (len < lim){ + validationThrow(validator,12,"string too short, %d < MAX=%d at %s", + len,lim,validatorAccessPath(validator)); + } + } + // String pattern; // ECMA-262 regex + // prefix items + // additionalProperties +} + +static void validateJSONNumber(JsonValidator *validator, + Json *doubleOrLegacyInt, + JSValueSpec *valueSpec){ + // Number multipleOf; // can be a filled, and must be > 0 + // Number exclusiveMaximum; + // Number exclusiveMinimum; + double d = (doubleOrLegacyInt->type == JSON_TYPE_NUMBER ? + (double)jsonAsNumber(doubleOrLegacyInt) : + jsonAsDouble(doubleOrLegacyInt)); + if (valueSpec->validatorFlags & JS_VALIDATOR_MAX){ + double lim = valueSpec->maximum; + if (valueSpec->exclusiveMaximum ? (d >= lim) : (d > lim)){ + validationThrow(validator,12,"value too large, %f %s MAX=%f at %s", + d, + (valueSpec->exclusiveMaximum ? ">=" : ">"), + lim, + validatorAccessPath(validator)); + } + } + if (valueSpec->validatorFlags & JS_VALIDATOR_MIN){ + double lim = valueSpec->minimum; + if (valueSpec->exclusiveMinimum ? (d <= lim) : (d < lim)){ + validationThrow(validator,12,"value too small, %f %s MAX=%f at %s", + d, + (valueSpec->exclusiveMinimum ? "<=" : "<"), + lim, + validatorAccessPath(validator)); + } + } +} + +static void validateJSONInteger(JsonValidator *validator, + Json *value64, + JSValueSpec *valueSpec){ + // Number multipleOf; // can be a filled, and must be > 0 + // Number exclusiveMaximum; + // Number exclusiveMinimum; + int64_t i = jsonAsInt64(value64); + if (valueSpec->validatorFlags & JS_VALIDATOR_MAX){ + int64_t lim = (int64_t)valueSpec->maximum; + if (valueSpec->exclusiveMaximum ? (i >= lim) : (i > lim)){ + validationThrow(validator,12,"value too large, %lld %s MAX=%lld at %s", + i, + (valueSpec->exclusiveMaximum ? ">=" : ">"), + lim, + validatorAccessPath(validator)); + } + } + if (valueSpec->validatorFlags & JS_VALIDATOR_MIN){ + int64_t lim = (int64_t)valueSpec->minimum; + if (valueSpec->exclusiveMinimum ? (i <= lim) : (i < lim)){ + validationThrow(validator,12,"value too small, %lld %s MAX=%lld at %s", + i, + (valueSpec->exclusiveMinimum ? "<=" : "<"), + lim, + validatorAccessPath(validator)); + } + } +} + +static JSValueSpec *resolveRef(JsonValidator *validator, JSValueSpec *valueSpec){ + char *ref = valueSpec->ref; + if (!memcmp(ref,"#/$defs/",8)){ /* local resolution */ + int refLen = strlen(ref); + int pos = 2; + char *key = ref+8; + JSValueSpec *topSchema = validator->schema->topValueSpec; + if (topSchema->definitions == NULL){ + validationThrow(validator,12,"schema '%s' does not define shared '$defs'", + (topSchema->id ? topSchema->id : "")); + return NULL; /* for the compiler, bless its heart */ + } + JSValueSpec *resolvedSchema = htGet(topSchema->definitions,key); + if (resolvedSchema == NULL){ + validationThrow(validator,12,"schema ref '%s' does not resolve against '$defs'",ref); + return NULL; + } + return resolvedSchema; + } else { + /* notes, + need to merge URL according W3 rules and pass to pluggable resolver that + we hope this validator has. + */ + validationThrow(validator,12,"external refs not yet implemented",ref); + return NULL; + } +} + +static void validateJSON(JsonValidator *validator, Json *value, JSValueSpec *valueSpec){ + while (valueSpec->ref){ + valueSpec = resolveRef(validator,valueSpec); + } + printf("validate JSON value->type=%d\n",value->type); + fflush(stdout); + // type is string or array, does this mean to collapse ValueSpecTypes + // should we flag errors on out-of-bounds validation keywords for type keyword + if (jsonIsNull(value)){ + validateType(validator,JSTYPE_NULL,valueSpec); + } else if (jsonIsBoolean(value)){ + validateType(validator,JSTYPE_BOOLEAN,valueSpec); + } else if (jsonIsObject(value)){ + validateType(validator,JSTYPE_OBJECT,valueSpec); + validateJSONObject(validator,jsonAsObject(value),valueSpec); + } else if (jsonIsArray(value)){ + validateType(validator,JSTYPE_ARRAY,valueSpec); + validateJSONArray(validator,jsonAsArray(value),valueSpec); + } else if (jsonIsString(value)){ + validateType(validator,JSTYPE_STRING,valueSpec); + validateJSONString(validator,value,valueSpec); + } else if (jsonIsNumber(value)){ + if (!jsonIsInt64(value)){ + validateType(validator,JSTYPE_NUMBER,valueSpec); + validateJSONInteger(validator,value,valueSpec); /* general, comparisons done as doubles */ + } else { + validateType(validator,JSTYPE_INTEGER,valueSpec); + validateJSONNumber(validator,value,valueSpec); + } + + } else { + printf("*** PANIC *** unhandled JS Value with type = %d\n",value->type); + } +} + +int jsonValidateSchema(JsonValidator *validator, Json *value, JsonSchema *schema){ + if (setjmp(validator->recoveryData) == 0) { /* normal execution */ + validator->schema = schema; + validateJSON(validator,value,schema->topValueSpec); + printf("after validate without throw\n"); + fflush(stdout); + return 0; + } else { + printf("validation failed '%s'\n",validator->errorMessage); + fflush(stdout); + return validator->errorCode; + } +} + + +static void schemaThrow(JsonSchemaBuilder *builder, int errorCode, char *formatString, ...){ + va_list argPointer; + char *text = safeMalloc(ERROR_MAX,"ErrorBuffer"); + va_start(argPointer,formatString); + vsnprintf(text,ERROR_MAX,formatString,argPointer); + va_end(argPointer); + + builder->errorCode = errorCode; + builder->errorMessage = text; + builder->errorMessageLength = ERROR_MAX; + + longjmp(builder->recoveryData,1); +} + +static char *getStringOrFail(JsonSchemaBuilder *builder, JsonObject *object, char *propertyName) { + Json *propertyValue = jsonObjectGetPropertyValue(object,propertyName); + if (propertyValue == NULL){ + schemaThrow(builder,12,"object must have property: %s",propertyName); + return NULL; /* unreachable, but compiler doesn't know */ + } else if (jsonIsString(propertyValue)){ + return jsonAsString(propertyValue); + } else { + schemaThrow(builder,12,"property '%s' must contain a string, not %s",propertyName,propertyValue); + return NULL; /* unreachable, but compiler doesn't know */ + } +} + +static char *getString(JsonSchemaBuilder *builder, JsonObject *object, char *propertyName, char *defaultValue) { + Json *propertyValue = jsonObjectGetPropertyValue(object,propertyName); + if (propertyValue == NULL){ + return defaultValue; + } else if (jsonIsString(propertyValue)){ + return jsonAsString(propertyValue); + } else { + schemaThrow(builder,12,"property '%s' must contain a string, not %s",propertyName,propertyValue); + return NULL; /* unreachable, but compiler doesn't know */ + } +} + +static bool getBooleanValue(JsonSchemaBuilder *builder, JsonObject *object, char *propertyName, bool defaultValue){ + Json *propertyValue = jsonObjectGetPropertyValue(object,propertyName); + if (propertyValue == NULL){ + return defaultValue; + } else if (jsonIsBoolean(propertyValue)){ + return (bool)jsonAsBoolean(propertyValue); + } else { + schemaThrow(builder,12,"property '%s' must contain a number, not %s",propertyName,propertyValue); + return false; /* unreachable, but compiler doesn't know */ + } +} + +static double getNumber(JsonSchemaBuilder *builder, JsonObject *object, char *propertyName, double defaultValue){ + Json *propertyValue = jsonObjectGetPropertyValue(object,propertyName); + if (propertyValue == NULL){ + return defaultValue; + } else if (jsonIsNumber(propertyValue)){ + return jsonAsDouble(propertyValue); + } else { + schemaThrow(builder,12,"property '%s' must contain a number, not %s",propertyName,propertyValue); + return 0.0; /* unreachable, but compiler doesn't know */ + } +} + +static int64_t getIntegralValue(JsonSchemaBuilder *builder, JsonObject *object, char *propertyName, int64_t defaultValue){ + Json *propertyValue = jsonObjectGetPropertyValue(object,propertyName); + if (propertyValue == NULL){ + return defaultValue; + } else if (jsonIsNumber(propertyValue)){ + if (jsonIsInt64(propertyValue)){ + return jsonAsInt64(propertyValue); + } else { + schemaThrow(builder,12,"property '%s' must contain an integer, not %s",propertyName,propertyValue); + return 0; /* unreachable, but compiler doesn't know */ + } + } else { + schemaThrow(builder,12,"property '%s' must contain a number, not %s",propertyName,propertyValue); + return 0; /* unreachable, but compiler doesn't know */ + } +} + +static JsonArray *getArrayValue(JsonSchemaBuilder *builder, JsonObject *object, char *propertyName) { + Json *propertyValue = jsonObjectGetPropertyValue(object,propertyName); + if (propertyValue == NULL){ + return NULL; + } else if (jsonIsArray(propertyValue)){ + return jsonAsArray(propertyValue); + } else { + schemaThrow(builder,12,"property '%s' must contain an array, not %s",propertyName,propertyValue); + return NULL; /* unreachable, but compiler doesn't know */ + } +} + +static JsonObject *getObjectValue(JsonSchemaBuilder *builder, JsonObject *object, char *propertyName) { + Json *propertyValue = jsonObjectGetPropertyValue(object,propertyName); + if (propertyValue == NULL){ + return NULL; + } else if (jsonIsObject(propertyValue)){ + return jsonAsObject(propertyValue); + } else { + schemaThrow(builder,12,"property '%s' must contain an object, not %s",propertyName,propertyValue); + return NULL; /* unreachable, but compiler doesn't know */ + } +} + +static int getJSTypeForName(JsonSchemaBuilder *builder, char *name){ + if (!strcmp(name,"null")){ + return JSTYPE_NULL; + } else if (!strcmp(name,"boolean")){ + return JSTYPE_BOOLEAN; + } else if (!strcmp(name,"object")){ + return JSTYPE_OBJECT; + } else if (!strcmp(name,"array")){ + return JSTYPE_ARRAY; + } else if (!strcmp(name,"number")){ + return JSTYPE_NUMBER; + } else if (!strcmp(name,"string")){ + return JSTYPE_STRING; + } else if (!strcmp(name,"integer")){ + return JSTYPE_INTEGER; + } else { + schemaThrow(builder,12,"'%s' is not valid JSON Schema type",name); + return 0; + } +} + +static void checkNonNegative(JsonSchemaBuilder *builder, int64 value, char *name) { + if (value < 0){ + schemaThrow(builder,12,"%s must be non-negative",name); + } +} + +static void checkPositiveDouble(JsonSchemaBuilder *builder, double value, char *name) { + if (value <= 0){ + schemaThrow(builder,12,"%s must be positive",name); + } +} + +static char** builderStringArray(JsonSchemaBuilder *builder, int count){ + char **result = (char**)SLHAlloc(builder->slh,count*sizeof(char*)); + return result; +} + +static char **getStringArrayOrFail(JsonSchemaBuilder *builder, JsonArray *a, int *count){ + ShortLivedHeap *slh = builder->slh; + int aLen = jsonArrayGetCount(a); + char **result = builderStringArray(builder,aLen); + for (int i=0; itype); + return NULL; + } + property = jsonObjectGetNextProperty(property); + } + return map; +} + +static bool ensureUniqueKeys(JsonSchemaBuilder *builder, JsonObject *o){ + /* + The Java JSON Reader we use guarantees unique property keys, but + */ + return true; +} + +// Tolerate both "id" and "$id" +static char *getID(JsonSchemaBuilder *builder, JsonObject *o) { + char *id = getString(builder,o,"$id",NULL); + if (id == NULL){ + id = getString(builder,o,"id",NULL); + } + return id; +} + +/* forwarding for complex recursion */ +static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopLevel); + +static JSValueSpec *makeValueSpec(JsonSchemaBuilder *builder, + char *id, + char *description, + char *title, + char **typeNameArray, + int typeNameArrayLength){ + JSValueSpec *spec = (JSValueSpec*)SLHAlloc(builder->slh,sizeof(JSValueSpec)); + memset(spec,0,sizeof(JSValueSpec)); + spec->id = id; + spec->description = description; + spec->title = title; + for (int i=0; itypeMask |= (1 << typeCode); + } + return spec; +} + + +/* Map */ +static hashtable *getDefinitions(JsonSchemaBuilder *builder, JsonObject *object){ + AccessPath *accessPath = builder->accessPath; + JsonObject *definitionsObject = getObjectValue(builder,object,"$defs"); + if (definitionsObject != NULL){ + hashtable *definitionMap = htCreate(101,stringHash,stringCompare,NULL,NULL); + accessPathPushName(accessPath,"definitions"); + JsonProperty *property = jsonObjectGetFirstProperty(definitionsObject); + while (property){ + char *propertyName = jsonPropertyGetKey(property); + Json *propertyValue = jsonPropertyGetValue(property); + if (propertyValue == NULL){ + schemaThrow(builder,12,"sub definition %s is NULL, which is not allowed",propertyName); + return NULL; + } + accessPathPushName(accessPath,propertyName); + htPut(definitionMap,propertyName,build(builder,propertyValue,false)); + accessPathPop(accessPath); + property = jsonObjectGetNextProperty(property); + } + accessPathPop(accessPath); + return definitionMap; + } else { + return NULL; + } +} + +static JSValueSpec **getComposite(JsonSchemaBuilder *builder, JsonObject *object, char *key){ + AccessPath *accessPath = builder->accessPath; + JsonArray *array = jsonObjectGetArray(object,key); + if (array != NULL){ + int len = jsonArrayGetCount(array); + JSValueSpec **schemata = (JSValueSpec**)SLHAlloc(builder->slh,len*sizeof(JSValueSpec*)); + accessPathPushName(accessPath,key); + for (int i=0; iaccessPath; + printf("JSONSchema build\n"); + printAccessPath(stdout,accessPath); + fflush(stdout); + if (jsonIsObject(jsValue)){ + JsonObject *object = jsonAsObject(jsValue); + ensureUniqueKeys(builder,object); + Json *typeValue = jsonObjectGetPropertyValue(object,"type"); + char **typeArray = NULL; + int typeArrayCount = 0; + if (typeValue == NULL){ + printf("*** INFO *** untyped value at path:\n"); + printAccessPath(stdout,accessPath); + typeArray = allJSTypeNames; + typeArrayCount = ALL_TYPES_COUNT; + } else { + if (jsonIsString(typeValue)){ + typeArray = builderStringArray(builder,1); + typeArray[0] = getStringOrFail(builder,object,"type"); + typeArrayCount = 1; + } else if (jsonIsArray(typeValue)){ + typeArray = getStringArrayOrFail(builder,jsonAsArray(typeValue),&typeArrayCount); + } else { + schemaThrow(builder,12,"'type' must be either a single string or an array of string, not type=%d",typeValue->type); + } + } + + char *description = getString(builder,object,"description",NULL); + char *title = getString(builder,object,"title",NULL); + + JSValueSpec *valueSpec = makeValueSpec(builder,getID(builder,object),description,title,typeArray,typeArrayCount); + valueSpec->ref = getString(builder,object,"$ref",NULL); + valueSpec->definitions = getDefinitions(builder,object); + valueSpec->allOf = getComposite(builder,object,"allOf"); + valueSpec->anyOf = getComposite(builder,object,"anyOf"); + valueSpec->oneOf = getComposite(builder,object,"oneOf"); + Json *notSpec = jsonObjectGetPropertyValue(object,"not"); + if (notSpec != NULL){ + accessPathPushName(accessPath,"not"); + valueSpec->not = build(builder,notSpec,false); + accessPathPop(accessPath); + } + /// valueSpec.not = getComposite(object,accessPath); + for (int typeCode=0; typeCode<=JSTYPE_MAX; typeCode++){ + if ((1 << typeCode) & valueSpec->typeMask){ + switch (typeCode){ + case JSTYPE_OBJECT: + { + JSValueSpec *objectSpec = valueSpec; + Json *properties = jsonObjectGetPropertyValue(object,"properties"); + objectSpec->maxProperties = getIntegralValue(builder,object,"maxProperties",MISSING_VALIDATOR); + objectSpec->minProperties = getIntegralValue(builder,object,"minProperties",MISSING_VALIDATOR); + if (objectSpec->maxProperties != MISSING_VALIDATOR){ + checkNonNegative(builder,objectSpec->maxProperties,"maxProperties"); + objectSpec->validatorFlags |= JS_VALIDATOR_MAX_PROPS; + } + if (objectSpec->maxProperties != MISSING_VALIDATOR){ + checkNonNegative(builder,objectSpec->minProperties,"minProperties"); + objectSpec->validatorFlags |= JS_VALIDATOR_MIN_PROPS; + } + JsonArray *required = getArrayValue(builder,object,"required"); + printf("builder required array = 0x%p\n",required); + if (required != NULL){ + int count = 0; + objectSpec->required = getStringArrayOrFail(builder,required,&count); + printf("during build, required is 0x%p\n",objectSpec->required); + objectSpec->requiredCount = count; + } + if (properties != NULL){ + if (jsonIsObject(properties)){ + JsonObject *propertiesObject = jsonAsObject(properties); + JsonProperty *property = jsonObjectGetFirstProperty(propertiesObject); + while (property){ + char *propertyName = jsonPropertyGetKey(property); + Json *propertyValue = jsonPropertyGetValue(property); + accessPathPushName(accessPath,propertyName); + if (valueSpec->properties == NULL){ + valueSpec->properties = htCreate(101,stringHash,stringCompare,NULL,NULL); + } + htPut(valueSpec->properties,propertyName,build(builder,propertyValue,false)); + accessPathPop(accessPath); + property = jsonObjectGetNextProperty(property); + } + } else { + schemaThrow(builder,12,"properties is filled by JSON value that is not object with type=%d\n",properties->type); + } + } else { + // properties is not required + } + JsonObject *dependentRequiredSpec = getObjectValue(builder,object,"dependentRequired"); + if (dependentRequiredSpec != NULL){ + objectSpec->dependentRequired = getStringToStringsMapOrFail(builder,dependentRequiredSpec); + } + } + break; + case JSTYPE_ARRAY: + { + JSValueSpec *arraySpec = valueSpec; + + arraySpec->maxItems = getIntegralValue(builder,object,"maxItems",MISSING_VALIDATOR); + arraySpec->minItems = getIntegralValue(builder,object,"minItems",MISSING_VALIDATOR); + arraySpec->uniqueItems = getBooleanValue(builder,object,"uniqueItems",false); + arraySpec->maxContains = getIntegralValue(builder,object,"maxContains",MISSING_VALIDATOR); + arraySpec->minContains = getIntegralValue(builder,object,"minContains",MISSING_VALIDATOR); + if (arraySpec->maxItems != MISSING_VALIDATOR){ + checkNonNegative(builder,arraySpec->maxItems,"maxItems"); + arraySpec->validatorFlags |= JS_VALIDATOR_MAX_ITEMS; + } + if (arraySpec->minItems != MISSING_VALIDATOR){ + checkNonNegative(builder,arraySpec->minItems,"minItems"); + arraySpec->validatorFlags |= JS_VALIDATOR_MIN_ITEMS; + } + if (arraySpec->maxContains != MISSING_VALIDATOR){ + checkNonNegative(builder,arraySpec->maxContains,"maxContains"); + arraySpec->validatorFlags |= JS_VALIDATOR_MAX_CONTAINS; + } + if (arraySpec->minContains != MISSING_VALIDATOR){ + checkNonNegative(builder,arraySpec->minContains,"minContains"); + arraySpec->validatorFlags |= JS_VALIDATOR_MIN_CONTAINS; + } + // recursion on items + Json *items = jsonObjectGetPropertyValue(object,"items"); + if (items != NULL){ + accessPathPushName(accessPath,"items"); + arraySpec->itemSpec = build(builder,items,false); + accessPathPop(accessPath); + } else { + // what does this default to or error + } + } + break; + case JSTYPE_STRING: + { + JSValueSpec *stringSpec = valueSpec; + stringSpec->maxLength = getIntegralValue(builder,object,"maxLength",MISSING_VALIDATOR); + stringSpec->minLength = getIntegralValue(builder,object,"minLength",MISSING_VALIDATOR); + if (stringSpec->maxLength != MISSING_VALIDATOR){ + checkNonNegative(builder,stringSpec->maxLength,"maxLength"); + stringSpec->validatorFlags |= JS_VALIDATOR_MAX_LENGTH; + } + if (stringSpec->minLength != MISSING_VALIDATOR){ + checkNonNegative(builder,stringSpec->minLength,"maxLength"); + stringSpec->validatorFlags |= JS_VALIDATOR_MIN_LENGTH; + } + stringSpec->pattern = getString(builder,object,"pattern",NULL); // hard to support on ANSI C + } + break; + case JSTYPE_NULL: + { + // no customizations ?! + } + break; + case JSTYPE_BOOLEAN: + { + // no customizations ?! + } + break; + case JSTYPE_NUMBER: + case JSTYPE_INTEGER: + { + JSValueSpec *numericSpec = valueSpec; + // new JSNumericSpec(description,title,(jsType == JSType.INTEGER)); + numericSpec->isInteger = (typeCode == JSTYPE_INTEGER); + numericSpec->multipleOf = getNumber(builder,object,"multipleOf",MISSING_FLOAT_VALIDATOR); + numericSpec->maximum = getNumber(builder,object,"maximum",MISSING_FLOAT_VALIDATOR); + numericSpec->exclusiveMaximum = getBooleanValue(builder,object,"exclusiveMaximum",false); + numericSpec->minimum = getNumber(builder,object,"minimum",MISSING_FLOAT_VALIDATOR); + numericSpec->exclusiveMinimum = getBooleanValue(builder,object,"exclusiveMinimum",false); + if (numericSpec->multipleOf != MISSING_FLOAT_VALIDATOR){ + checkPositiveDouble(builder,numericSpec->multipleOf,"multipleOf"); + numericSpec->validatorFlags |= JS_VALIDATOR_MULTOF; + } + } + break; + default: + printf("*** PANIC *** unknown JSType code=%d\n",typeCode); + } + + } + }// for (jsType... + return valueSpec; + } else { + schemaThrow(builder,12,"top level schema must be an object"); + return NULL; + } +} + + +JsonSchemaBuilder *makeJsonSchemaBuilder(int version){ + JsonSchemaBuilder *builder = (JsonSchemaBuilder*)safeMalloc(sizeof(JsonSchemaBuilder),"JsonBuilder"); + memset(builder,0,sizeof(JsonSchemaBuilder)); + builder->slh = makeShortLivedHeap(0x10000, 100); + builder->version = version; + builder->accessPath = (AccessPath*)SLHAlloc(builder->slh,sizeof(AccessPath)); + builder->accessPath->currentSize = 0; + return builder; +} + +void freeJsonSchemaBuilder(JsonSchemaBuilder *builder){ + if (builder->slh){ + SLHFree(builder->slh); + } + safeFree((char*)builder,sizeof(JsonSchemaBuilder)); +} + +JsonSchema *jsonBuildSchema(JsonSchemaBuilder *builder, Json *jsValue){ + if (setjmp(builder->recoveryData) == 0) { /* normal execution */ + printf("after setjmp normal\n"); + fflush(stdout); + JSValueSpec *topValueSpec = build(builder,jsValue,true); + JsonSchema *schema = (JsonSchema*)SLHAlloc(builder->slh,sizeof(JsonSchema)); + schema->topValueSpec = topValueSpec; + schema->version = builder->version; + /* Take ownership of the SLH if build is successful */ + schema->slh = builder->slh; + builder->slh = NULL; + return schema; + } else { /* throw handling */ + printf("schema build fail %s\n",builder->errorMessage); + fflush(stdout); + return NULL; + } +} diff --git a/c/psxskt.c b/c/psxskt.c new file mode 100644 index 000000000..2e50f879f --- /dev/null +++ b/c/psxskt.c @@ -0,0 +1,1482 @@ + + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#include +#include +#include +#include +#include +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "bpxnet.h" +#include "unixfile.h" + +#include +#include + +#include +#include +#include + +#ifndef __ZOWE_OS_AIX +#include +#endif + +#include +#include + +#include +#include /* gethostbyname */ +#include +#include +#include + +#ifdef USE_RS_SSL +#include "rs_ssl.h" +#endif + +int socketInit(char *uniqueName){ + /* don't do anything in POSIX */ + return 0; +} + +static int socketTrace = 0; +int setSocketTrace(int toWhat) { + int was = socketTrace; + socketTrace = toWhat; + return was; +} + +typedef union SockAddrAll_tag { + struct sockaddr_in v4; + struct sockaddr_in6 v6; +} SockAddrAll; + +#define IPPROTO_EVENT 254 /* semi-official "experimental" protocol number */ + +static +SocketAddress * setSocketAddr(SocketAddress *address, + const InetAddr *addr, + unsigned short port) +{ + memset(address,0,sizeof(SocketAddress)); + + address->family = (uint8_t) (addr ? addr->type : AF_INET); + if (addr && (port == 0)) { + address->port = (uint16_t) addr->port; + } else { + address->port = (uint16_t) htons(port); + } + + if (addr) { + switch (address->family) { + case AF_INET: + address->internalAddress.v4Address.sin_family = AF_INET; + address->internalAddress.v4Address.sin_port = (in_port_t) (port == 0 ? address->port : htons(port)); + memcpy(&(address->internalAddress.v4Address.sin_addr), &(addr->data.data4), sizeof(struct in_addr)); + break; + + case AF_INET6: + address->internalAddress.v6Address.sin6_family = AF_INET6; + address->internalAddress.v6Address.sin6_port = (in_port_t) (port == 0 ? address->port : htons(port)); + memcpy(&(address->internalAddress.v6Address.sin6_addr), &(addr->data.data6), sizeof(struct in6_addr)); + break; + + default: + if (socketTrace){ + printf("setSocketAddr: Unknown address family %d\n", (int) address->family); + } + return NULL; + } + } + +if (socketTrace) { + dumpbuffer(address, sizeof(SocketAddress)); +} + + return address; +} + +SocketAddress *makeSocketAddr(InetAddr *addr, + unsigned short port){ + SocketAddress *address = (SocketAddress*)safeMalloc(sizeof(SocketAddress),"SocketAddress"); + if (0 == setSocketAddr(address, addr, port)) { + freeSocketAddr(address); + address = (SocketAddress *) NULL; + } + return address; +} + +void freeSocketAddr(SocketAddress *address){ + safeFree((char*)address,sizeof(SocketAddress)); +} + +static +int getEndPointName(int sd, InetAddr *name, int which /* 0 = this end, other = peer */) +{ + SockAddrAll socketAddr; + memset(&socketAddr, 0, sizeof(SockAddrAll)); + int status = 0; + socklen_t sockLen = sizeof(SockAddrAll); + if (which == 0) { + status = getsockname(sd, (struct sockaddr *)&socketAddr, &sockLen); + } else { + status = getpeername(sd, (struct sockaddr *)&socketAddr, &sockLen); + } + + if (0 == status) { + /* fill in the address information */ + memset(name, 0, sizeof(InetAddr)); + sa_family_t domain = socketAddr.v4.sin_family; /* same place for all address types */ + name->type = (uint16_t) domain; + switch(domain) { + case AF_INET: + name->port = (uint16_t) socketAddr.v4.sin_port; + memcpy(&(name->data), &socketAddr.v4.sin_addr, sizeof(struct in_addr)); + break; + + case AF_INET6: + name->port = (uint16_t) socketAddr.v6.sin6_port; + memcpy(&(name->data), &socketAddr.v6.sin6_addr, sizeof(struct in6_addr)); + break; + + default: + status = EAFNOSUPPORT; + break; + } + } else { + status = errno; + if (socketTrace){ + printf("get%sname failed, errno=%d (%s)\n", + (which == 0 ? "sock" : "peer"), status, strerror(status)); + } + } + return status; +} + +/*--5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---*/ + +/* + TBC-JLC: This code currently supports only IPv4. + */ +Socket *tcpClient2(SocketAddress *socketAddress, + int timeoutInMillis, + int *returnCode, /* errnum */ + int *reasonCode){ /* errnum - JR's */ + + sa_family_t domain = 0; + + int sd = socket(AF_INET,SOCK_STREAM,IPPROTO_TCP); + if (sd < 0) { + *reasonCode = *returnCode = errno; + if (socketTrace){ + printf("Failed to create client socket, errno=%d (%s)\n", + *returnCode, strerror(*returnCode)); + } + return NULL; + } else { + if (socketTrace){ + printf("Created client socket, sd=%d\n", sd); + } + } + + int socketVector[2] = {0}; + int status; + int madeConnection = FALSE; + struct sockaddr *sockAddr = (struct sockaddr*)&(socketAddress->internalAddress.v4Address); + + { + int socketAddrSize = sizeof(SocketAddress); + if (timeoutInMillis >= 0){ + Socket tempSocket; + tempSocket.sd = sd; + + /* setSocketBlockingMode(&tempSocket, TRUE, returnCode, reasonCode); */ + status = connect(sd,sockAddr,sizeof(struct sockaddr_in)); /* v4 only */ + *returnCode = *reasonCode = errno; + if (socketTrace || (status < 0)) { + printf("connect: status=%d errno=%d (%s)\n",status,*returnCode, strerror(*returnCode)); + fflush(stdout); + } + /* EINPROGRESS is the expected return code here but we are just being careful by checking + for EWOULDBLOCK as well + */ + if (status == 0){ + *returnCode = 0; + *reasonCode = 0; + status = tcpStatus(&tempSocket, timeoutInMillis, 1, returnCode, reasonCode); +#ifdef DEBUG + printf("after tcpStatus() = %d\n",status); +#endif + if (status == SD_STATUS_TIMEOUT) { + int sd = socketVector[0]; + if (socketTrace) { + printf("Failed to connect socket, will clean up sd=%d\n", sd); + } + *returnCode = 0; + *reasonCode = 0; + status = close(sd); + if (socketTrace) { + printf("closeSocket() for time out connect status %d\n",status); + } + return NULL; + } +#ifdef DEBUG + printf("tcpStatus return and did not timeout status=%d\n",status); + fflush(stdout); +#endif + *returnCode = 0; + *reasonCode = 0; + /* setSocketBlockingMode(&tempSocket, FALSE, returnCode, reasonCode); */ + int optionLength = sizeof(int); + int optionData = 0; + *returnCode = 0; + *reasonCode = 0; + status = getSocketOption(&tempSocket, SO_ERROR, &optionLength, (char *)&optionData, returnCode, reasonCode); + if( optionData > 0 ){ + if (socketTrace){ + printf("Failed to connect socket, returnCode %d\n", optionData); + } + *returnCode = optionData; + madeConnection = FALSE; + } else { + madeConnection = TRUE; + } + } + if (socketTrace){ + printf("did we make connection (timeout case) = %d\n",madeConnection); + } + } else{ /* if timeouts are not specified */ + status = connect(sd,sockAddr,sizeof(struct sockaddr_in)); /* v4 only */ + *returnCode = *reasonCode = errno; + if (socketTrace || (status < 0)) { + printf("connect: status=%d errno=%d (%s)\n",status,*returnCode, strerror(*returnCode)); + fflush(stdout); + } + madeConnection = (status == 0 ? TRUE : FALSE); + } + if (!madeConnection){ + if (socketTrace){ + printf("Failed to connect socket, will clean up, sd was %d\n",sd); + } + status = close(sd); + if (socketTrace || (status < 0)) { + int error = (status < 0 ? errno : 0); + printf("close(%d) for failed connect: status=%d, errno=%d (%s)\n", + sd, status, error, strerror(error)); + } + return NULL; + } else{ + Socket *socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + socket->sd = sd; + sprintf(socket->debugName,"SD=%u",(unsigned) socket->sd); + socket->isServer = FALSE; + /* manual says returnCode and value only meaningful if return value = -1 */ + *returnCode = 0; + *reasonCode = 0; + socket->protocol = IPPROTO_TCP; + return socket; + } + } +} + +/* pointer to length is stupid */ +int getSocketOption(Socket *socket, int optionName, int *optionDataLength, char *optionData, + int *returnCode, int *reasonCode){ + int sd = socket->sd; + int getOption = 1; + int setOption = 2; + int setIBMOption = 3; + int level = 0x0000FFFF; + + if (socketTrace){ + printf("before get socket option optionName=0x%x dataBufferLen=%d\n",optionName,*optionDataLength); + } + int status = setsockopt(sd, + SOL_SOCKET, /* is this always right */ + optionName, + optionData, + *optionDataLength); + if (socketTrace){ + printf("after getsockopt, status=%d ret code %d reason 0x%x\n",status,*returnCode,*reasonCode); + } + if (status < 0){ + *returnCode = errno; + *reasonCode = *returnCode; + if (socketTrace) { + printf("get sockopt failed, ret code %d reason 0x%x\n", *returnCode, *reasonCode); + } + return -1; + } else { + *returnCode = 0; + *reasonCode = 0; + return status; + } +} + + +Socket *tcpClient(SocketAddress *socketAddress, + int *returnCode, /* errnum */ + int *reasonCode){ /* errnum - JR's */ + return tcpClient2(socketAddress,-1,returnCode,reasonCode); +} + +void socketFree(Socket *socket){ + if ((IPPROTO_EVENT != socket->protocol) && + (0 != socket->socketAddr)) { + safeFree((char*)socket->socketAddr,sizeof(InetAddr)); + } + safeFree((char*)socket,sizeof(Socket)); +} + +int udpReceiveFrom(Socket *socket, + SocketAddress *sourceAddress, /* will be overwritten */ + char *buffer, int bufferLength, + int *returnCode, int *reasonCode){ + int status = 0; + int sd = socket->sd; + int flags = 0; + struct sockaddr *sockAddr = (struct sockaddr*)&(sourceAddress->internalAddress.v4Address); + int socketAddressSize = sizeof(struct sockaddr); + + *returnCode = *reasonCode = 0; + + if (socketTrace > 2){ + printf("receiveFrom into buffer=0x%x bufLen=%d\n", buffer,bufferLength); + } + + int bytesReadOrError = recvfrom(sd, buffer, bufferLength, flags, sockAddr, &socketAddressSize); + + if (bytesReadOrError < 0){ + *returnCode = errno; + *reasonCode = *returnCode; + return -1; + } else { + if (socketTrace > 1){ + printf("read %d bytes\n",bytesReadOrError); + dumpbuffer(buffer, bytesReadOrError); + } + *returnCode = 0; + *reasonCode = 0; + + if (!(socket->lastDestination.port == sourceAddress->port) && + ((socket->lastDestination.family == AF_INET && + (0 == memcmp(&(socket->lastDestination.internalAddress.v4Address), + &(sourceAddress->internalAddress.v4Address), + sizeof(sourceAddress->internalAddress.v4Address)))) + || + (socket->lastDestination.family == AF_INET6 && + (0 == memcmp(&(socket->lastDestination.internalAddress.v6Address), + &(sourceAddress->internalAddress.v6Address), + sizeof(sourceAddress->internalAddress.v6Address)))))) + { + if (socket->lastDestination.family == 0) { + *reasonCode = 1; // Indicate no last destination was set for the socket + } else { + *reasonCode = 2; // Indicate source address/port did not match the last destination + } + } else { + memset(&socket->lastDestination,0,sizeof(SocketAddress)); + } + return bytesReadOrError; + } + +} + +void setSocketLastDestination(Socket *socket, SocketAddress *sourceAddress) { + socket->lastDestination = *sourceAddress; +} + +void clearSocketLastDestination(Socket *socket) { + memset(&socket->lastDestination,0,sizeof(SocketAddress)); +} + +int udpSendTo(Socket *socket, + SocketAddress *destinationAddress, + char *buffer, int desiredBytes, + int *returnCode, int *reasonCode){ + int status = 0; + int sd = socket->sd; + int flags = 0; + struct sockaddr *sockAddr = (struct sockaddr*)&(destinationAddress->internalAddress.v4Address); + int socketAddressSize = sizeof(struct sockaddr); + + *returnCode = *reasonCode = 0; + + if (socketTrace > 2){ + printf("sendTo desired=%d\n", desiredBytes); + dumpbuffer(buffer, desiredBytes); + dumpbuffer(destinationAddress, sizeof(SocketAddress)); + } + + int bytesSent = sendto(sd, buffer, desiredBytes, flags, sockAddr, socketAddressSize); + + if (bytesSent < 0){ + memset(&socket->lastDestination,0,sizeof(SocketAddress)); + if (socketTrace > 1){ + printf("error: %d\n", errno); + } + *returnCode = errno; + *reasonCode = *returnCode; + return -1; + } else { + socket->lastDestination = *destinationAddress; + if (socketTrace > 1){ + printf("sent %d bytes\n",bytesSent); + } + *returnCode = 0; + *reasonCode = 0; + return bytesSent; + } +} + +int getSocketKeepAliveMode(Socket *socket, int *returnCode, int *reasonCode){ + + int optval = 0; + int optlen = sizeof(optval); + int returnValue = getSocketOption(socket,SO_KEEPALIVE,&optlen,&optval, + returnCode,reasonCode); + if (returnValue < 0) { // If an error occurred + printf("Unable to get socket option SO_KEEPALIVE.\n"); + return returnValue; + } + returnValue = optval; + + *returnCode = 0; + *reasonCode = 0; + return returnValue; +} + +// - An enableKeepAlive value of 0 will disable keepalive processing +// - An enableKeepAlive value of >0 will enable keepalive processing with the default +// time values. +int setSocketKeepAliveMode(Socket *socket, int enableKeepAlive, + int *returnCode, int *reasonCode){ + + // Enable/disable SO_KEEPALIVE, as requested + int optval = (enableKeepAlive ? 1 : 0); + int returnValue = setSocketOption(socket,SOL_SOCKET,SO_KEEPALIVE,sizeof(optval),&optval, + returnCode,reasonCode); + if (returnValue < 0) { // If an error occurred + printf("Unable to set socket option SO_KEEPALIVE to %s.\n",(optval? "on" : "off")); + return returnValue; + } + + *returnCode = 0; + *reasonCode = 0; + return returnValue; +} + +int socketRead(Socket *socket, char *buffer, int desiredBytes, + int *returnCode, int *reasonCode){ + int sd = socket->sd; + int flags = 0; /* no special behaviors */ + +#ifdef USE_RS_SSL + if (NULL != socket->sslHandle) { /* sslHandle is RS_SSL_CONNECTION in this case */ + int bytesRead = 0; + int status = rs_ssl_read(socket->sslHandle, buffer, desiredBytes, &bytesRead); + if (0 != status) { + if (socketTrace){ + printf("socketRead: rs_ssl_read failed with status: %d\n", status); + } + *returnCode = status; + *reasonCode = *returnCode; + return -1; + } else { + *returnCode = 0; + *reasonCode = 0; + return bytesRead; + } + } + else +#endif + { + int bytesReadOrError = (int) read(sd,buffer,(size_t) desiredBytes); + if (bytesReadOrError < 0){ + *returnCode = errno; + *reasonCode = *returnCode; + return -1; + } else { + if (socketTrace > 1){ + printf("read %d bytes\n",bytesReadOrError); + } + *returnCode = 0; + *reasonCode = 0; + return bytesReadOrError; + } + } +} + +int socketWrite(Socket *socket, const char *buffer, int desiredBytes, + int *returnCode, int *reasonCode){ + int sd = socket->sd; + int flags = 0; + + if (socketTrace > 1){ + printf("socketWrite(%s, %d)\n", + socket->debugName, desiredBytes); + if (socketTrace > 2){ + dumpbuffer(buffer, desiredBytes); + } + } + +#ifdef USE_RS_SSL + if (NULL != socket->sslHandle) { /* sslHandle is RS_SSL_CONNECTION in this case */ + int bytesWritten = 0; + int status = rs_ssl_write(socket->sslHandle, buffer, desiredBytes, &bytesWritten); + if (0 != status) { + if (socketTrace) { + printf("socketWrite: rs_ssl_write failed with status: %d\n", status); + } + *returnCode = status; + *reasonCode = *returnCode; + return -1; + } else { + *returnCode = 0; + *reasonCode = 0; + return bytesWritten; + } + } + else +#endif + { + int bytesWrittenOrError = (int) write(sd,buffer,(size_t) desiredBytes); + if (bytesWrittenOrError < 0){ + *returnCode = errno; + *reasonCode = *returnCode; + return -1; + } else { + if (socketTrace > 2){ + printf("socketWrite(%s, %d)wrote %d bytes\n", + socket->debugName, desiredBytes, bytesWrittenOrError); + fflush(stdout); + } + *returnCode = 0; + *reasonCode = 0; + return bytesWrittenOrError; + } + } +} + + +static int tcpStatusInternal(Socket *socket, int timeout, int checkWrite, + int checkRead, int *returnCode, int *reasonCode){ + *reasonCode = 0; // unnecessary argument for POSIX implementation + *returnCode = 0; + + int socketDescriptor = socket->sd; + int maxSoc = socketDescriptor + 1; + + fd_set setRead; + fd_set setWrite; + fd_set setError; + + FD_ZERO(&setRead); + FD_ZERO(&setWrite); + FD_ZERO(&setError); + + struct timeval timeOut; + timeOut.tv_sec = timeout / 1000; + timeOut.tv_usec = (timeout % 1000) * 1000; + struct timeval *timeoutPtr = NULL; + + if (timeout >= 0){ + timeoutPtr = &timeOut; + } + + if (checkRead){ + FD_SET(socketDescriptor, &setRead); + } + + FD_SET(socketDescriptor, &setError); + + if (checkWrite){ + FD_SET(socketDescriptor, &setWrite); + } + + int retValue = select(maxSoc, (checkRead ? &setRead : NULL), (checkWrite ? &setWrite : NULL), &setError, timeoutPtr); + + if (retValue < 0) { + *reasonCode = *returnCode = errno; + if (socketTrace) { + printf("Select failed for sd %d, socket %s: errno=%d (%s)\n", + socketDescriptor, socket->debugName, *returnCode, strerror(*returnCode)); + } + return SD_STATUS_FAILED; + } + + if (retValue > 0){ + int sdStatus = 0; + + if (FD_ISSET(socketDescriptor, &setRead)){ + sdStatus |= SD_STATUS_RD_RDY; + } + if (FD_ISSET(socketDescriptor, &setWrite)){ + sdStatus |= SD_STATUS_WR_RDY; + } + if (FD_ISSET(socketDescriptor, &setError)){ + return SD_STATUS_ERROR; + } + + return sdStatus; + } + + return SD_STATUS_TIMEOUT; + } + +int tcpStatus(Socket *socket, + int timeout, /* in milliseconds */ + int checkWrite, /* otherwise check for read and error */ + int *returnCode, int *reasonCode){ + return tcpStatusInternal(socket,timeout,checkWrite,!checkWrite,returnCode,reasonCode); +} + +#define F_GETFL 3 +#define F_SETFL 4 + +int setSocketBlockingMode(Socket *socket, int isNonBlocking, + int *returnCode, int *reasonCode){ + int sd = socket->sd; + int status = 0; + long command = FIONBIO; + unsigned long argument = (isNonBlocking ? 1 : 0); + + status = ioctl(sd,command,&argument); + + if (status < 0){ + *reasonCode = *returnCode = errno; + if (socketTrace) { + printf("setSocketBlockingMode(%s, %s) failed: errno = %d (%s)\n", + socket->debugName, (isNonBlocking ? "NONBLOCKING (true)" : "BLOCKING (false)"), + *reasonCode, strerror(*reasonCode)); + } + return -1; + } else { + if (socketTrace){ + printf("setSocketBlockingMode(%s, %s) succeeded\n", + socket->debugName, (isNonBlocking ? "NONBLOCKING (true)" : "BLOCKING (false)")); + } + *returnCode = *reasonCode = 0; + return 0; + } +} + +int setSocketReuseAddr(Socket *socket, int *returnCode, int *reasonCode){ + int on = 1; + return setSocketOption(socket,SOL_SOCKET,SO_REUSEADDR,sizeof(int),(char*)&on,returnCode,reasonCode); +} + +Socket *udpPeer(SocketAddress *socketAddress, int *returnCode, /* errnum */ + int *reasonCode) { /* errnum - JR's */ + int status; + struct sockaddr *sockAddr = (struct sockaddr*) &(socketAddress->internalAddress.v4Address); + + int sd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); + + if (sd == -1) { + *returnCode = errno; + *reasonCode = *returnCode; + if (socketTrace) { + printf("Failed to create socket, error=%d\n", *returnCode); + } + return NULL; + } + else { + int socketAddressSize = sizeof(SocketAddress); + + status = bind(sd, sockAddr, sizeof(struct sockaddr_in)); + if (status != 0) { + *returnCode = errno; + *reasonCode = *returnCode; + } + if (socketTrace) { + printf("bind status %d returnCode %x reasonCode %x\n", status, *returnCode, *reasonCode); + } + if (status != 0) { + /* polaris noted that we are leaking the socket in this failure to bind case*/ + close(sd); + return NULL; + } + else { + Socket *socket = (Socket*) safeMalloc(sizeof(Socket), "Socket"); + socket->sd = sd; + sprintf(socket->debugName, "SD=%u", (unsigned) socket->sd); + socket->isServer = FALSE; + /* manual says returnCode and value only meaningful if return value = -1 */ + *returnCode = 0; + *reasonCode = 0; + socket->protocol = IPPROTO_UDP; + + return socket; + } + } +} + +Socket *tcpServer(InetAddr *addr, /* usually NULL/0 */ + int port, + int *returnCode, + int *reasonCode){ + int status; + socklen_t sockLen = 0; + SockAddrAll socketAddr; + memset(&socketAddr, 0, sizeof(SockAddrAll)); + + sa_family_t domain = (sa_family_t) (addr ? addr->type : AF_INET); + switch(domain) { + case AF_INET: + socketAddr.v4.sin_family = domain; + socketAddr.v4.sin_port = htons((in_port_t)port); + if (addr) { + memcpy(&(socketAddr.v4.sin_addr.s_addr), &(addr->data.data4), sizeof(struct in_addr)); + } + sockLen = sizeof(struct sockaddr_in); + break; + + case AF_INET6: + socketAddr.v6.sin6_family = domain; + socketAddr.v6.sin6_port = (in_port_t)port; + if (addr) { + memcpy(&(socketAddr.v6.sin6_addr.s6_addr), &(addr->data.data6), sizeof(struct in6_addr)); + } + sockLen = sizeof(struct sockaddr_in6); + break; + + default: + if (socketTrace){ + printf("tcpserver: Unknown address family %d\n", domain); + } + *reasonCode = *returnCode = EAFNOSUPPORT; + return NULL; + } + + int sd = socket(domain,SOCK_STREAM,IPPROTO_TCP); + if (sd == -1){ + *reasonCode = *returnCode = errno; + if (socketTrace){ + printf("socket(%d,SOCK_STREAM,IPPROTO_TCP) failed, errno=%d (%s)\n", + domain, *returnCode, strerror(*returnCode)); + } + return NULL; + } + + status = setSocketReuseAddr(&sd, + returnCode, + reasonCode); + if (status != 0){ + if (socketTrace){ + printf("Failed to set SO_REUSEADDR errno=%d reason=0x%x\n", + *returnCode,*reasonCode); + } + /* treat as not fatal?*/ + } + + if (port != 0) { + /* bind to the port provided */ + status = bind(sd,(struct sockaddr*)&socketAddr,sockLen); + if (status != 0){ + *reasonCode = *returnCode = errno; + if (socketTrace){ + printf("Failed to bind server socket errno=%d (%s)\n", + *returnCode,strerror(*returnCode)); + } + close(sd); + return NULL; + } + } + + int backlogQueueLength = 100; + status = listen(sd,backlogQueueLength); + if (status != 0){ + *returnCode = errno; + *reasonCode = *returnCode; + if (socketTrace){ + printf("Failed to start server socket listen, errno=%d (%s)\n", + *returnCode,strerror(*returnCode)); + } + close(sd); + return NULL; + } + + /* Yea, team - we now have a listening socket. */ + + *reasonCode = *returnCode = 0; + Socket *socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + memset(socket, 0, sizeof(Socket)); + socket->sd = sd; + socket->isServer = TRUE; + socket->protocol = IPPROTO_TCP; + + InetAddr name; + if (0 == getEndPointName(sd, &name, 0)) { + /* fill in the address information */ + socket->socketAddr = (InetAddr *)safeMalloc(sizeof(InetAddr),"InetAddr"); + memcpy(socket->socketAddr, &name, sizeof(InetAddr)); + port = (int) name.port; + } + sprintf(socket->debugName,"SD=%d,port=%u",socket->sd, (unsigned) port); + + if (socketTrace) { + printf("Immediately before return from tcpServer, socket contains:\n"); + dumpbuffer((char*)socket, sizeof(Socket)); + } + + return socket; +} + +Socket *tcpServer2(InetAddr *addr, + int port, + int tlsFlags, + int *returnCode, + int *reasonCode) +{ + Socket* socket = NULL; + socket = tcpServer(addr, port, returnCode, reasonCode); + if (socket != NULL) { + socket->tlsFlags = tlsFlags; + } + + return socket; +} +/*--5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---*/ + + +Socket *socketAccept(Socket *serverSocket, int *returnCode, int *reasonCode){ + + SockAddrAll newSocketAddr; + socklen_t newSocketAddrSize = sizeof(SockAddrAll); + int newSD = accept(serverSocket->sd,(struct sockaddr *)&newSocketAddr,&newSocketAddrSize); + if (newSD < 0){ + *reasonCode = *returnCode = errno; + if (socketTrace){ + printf("Failed to accept new socket errno=%d (%s)\n", + *returnCode,strerror(*returnCode)); + } + return NULL; + } + + Socket *socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + memset(socket,0,sizeof(Socket)); + socket->sd = newSD; + socket->isServer = 0; + socket->protocol = IPPROTO_TCP; + socket->tlsFlags = serverSocket->tlsFlags; + sprintf(socket->debugName,"SD=%d",socket->sd); + return socket; +} + + +#define DOT 1 +#define FIRST 2 +#define SECOND 3 +#define THIRD 4 + +int isV4Numeric(char *chars, int *addr){ + int i,len = strlen(chars); + int state = DOT; + int dotCount = 0; + int value; + int addressValue = 0; + for (i=0; i= '0' && ch <= '9'){ + if (dotCount == 0 && ch == '0') + return 0; + else + state = FIRST; + } else { + return 0; + } + break; + case FIRST: + if (ch >= '0' && ch <= '9'){ + state = SECOND; + } else if (ch == '.'){ + state = DOT; + value = (chars[i-1] - '0'); + addressValue = (addressValue << 8) | value; + dotCount++; + } else{ + return 0; + } + break; + case SECOND: + if (ch >= '0' && ch <= '9'){ + state = THIRD; + } else if (ch == '.'){ + state = DOT; + value = ((10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + addressValue = (addressValue << 8) | value; + dotCount++; + } else{ + return 0; + } + break; + case THIRD: + if (ch >= '0' && ch <= '9'){ + return 0; + } else if (ch == '.'){ + state = DOT; + value = ((100 * (chars[i-3] - '0')) + + (10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + addressValue = (addressValue << 8) | value; + if (value < 256) + dotCount++; + else + return 0; + } else{ + return 0; + } + break; + } + } + if (dotCount == 3){ + switch (state){ + case FIRST: + value = (chars[i-1] - '0'); + break; + case SECOND: + value = ((10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + break; + case THIRD: + value = ((100 * (chars[i-3] - '0')) + + (10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + break; + } + addressValue = (addressValue << 8) | value; + *addr = htonl(addressValue); + return (value < 256); + } else{ + return 0; + } +} + +/* the return value is in network byte order */ + +int getV4HostByName(char *hostName){ + int status = 0; + int returnCode = 0; + int reasonCode = 0; + int len = strlen(hostName); + struct hostent *hostEntPtr; + + hostEntPtr = gethostbyname(hostName); + if (socketTrace){ + printf("hostent addr = %x\n",*((int*)hostEntPtr)); + } + if (hostEntPtr){ + int i = 0; + int numericAddress = 0; + if (socketTrace){ + dumpbuffer((char*)hostEntPtr,20); + } + if (socketTrace) { + printf("hostent->length=%d name=%s\n", hostEntPtr->h_length, hostEntPtr->h_name); + fflush(stdout); + } + do { + char *hostAddr = hostEntPtr->h_addr_list[i]; + if (hostAddr == NULL){ + break; + } + numericAddress = *((int*)hostAddr); /*very IP-v4 here */ + if (socketTrace) { + printf("hostAddr is at 0x%x\n", hostAddr); + printf("numeric=0x%x == %d.%d.%d.%d\n", numericAddress, (int) hostAddr[0], (int) hostAddr[1], (int) hostAddr[2], + (int) hostAddr[3]); + } + i++; + } while (TRUE); + return numericAddress; + } else{ + returnCode = errno; + reasonCode = returnCode; + if (socketTrace) { + printf("getHostName V4 failure, returnCode %d reason code %d\n", returnCode, reasonCode); + } + return 0; + } +} + + +InetAddr *getAddressByName(char *addressString){ + int numericAddress = 0x7F123456; + + if (!isV4Numeric(addressString,&numericAddress)) { /* updates numericAddress if addressString was a V4 IP address */ + numericAddress = getV4HostByName(addressString); + } + + if ((numericAddress != 0x7F123456) && + (0 != numericAddress)) + { + InetAddr *addr = (InetAddr*)safeMalloc(sizeof(InetAddr),"Inet Address"); + memset(addr,0,sizeof(InetAddr)); + addr->type = AF_INET; + addr->port = 0; + memcpy(&(addr->data.data4),&numericAddress,4); +#ifdef DEBUG + printf("InetAddr structure: AF_INET=0x%x\n",AF_INET); + dumpbuffer((char*)addr,sizeof(InetAddr)); + fflush(stdout); +#endif + return addr; + } else{ + return NULL; + } +} + +int getLocalHostName(char* inout_hostname, + unsigned int* inout_hostname_len, + int *returnCode, int *reasonCode) +{ + int result = 0; + /* Parameter check code copied from z/OS implementation. + Seems odd that this one routine would bother... */ + if ((NULL == inout_hostname) || + (NULL == inout_hostname_len) || + (NULL == returnCode) || (NULL == reasonCode)) + { + result = -1; + } else { + memset(inout_hostname, 0, (size_t)(*inout_hostname_len)); + result = gethostname(inout_hostname, *inout_hostname_len); + if (result < 0) { + *reasonCode = *returnCode = errno; + if (socketTrace) { + printf("gethostname(*,%d) failed, errno=%d (%s)\n", + *returnCode, strerror(*returnCode)); + } + } else { + /* gethostname doesn't provide a length, it just null-terminates + its output, if there is space. But the caller wants the length, + so figure it out. Note that, idiotically, the length includes + the trailing null (that's how z/OS does it). */ + if (inout_hostname[(*inout_hostname_len)-1] == 0) { + /* it fit, including the terminator */ + *inout_hostname_len = 1+(int)strlen(inout_hostname); + } /* else - it filled the available space. */ + } + } + return result; +} + +int socketClose(Socket *socket, int *returnCode, int *reasonCode){ + if (socketTrace) { + printf("socketClose(%s): isServer=%s,proto=%d: ", + socket->debugName, (socket->isServer ? "true" : "false"), (int) socket->protocol); + } + + int status = 0; + *reasonCode = *returnCode = 0; + +#ifdef USE_RS_SSL + if ((0 == socket->isServer) && (NULL != socket->sslHandle)) { + /* sslHandle is RS_SSL_CONNECTION in this case */ + /* this call does a C stdlib shutdown/close internally */ + status = rs_ssl_releaseConnection(socket->sslHandle); + return status; + } +#endif + + status = close(socket->sd); + + if (0 == status) { + if (socketTrace) { + printf(" OK\n"); + } + } else { + *reasonCode = *returnCode = errno; + if (socketTrace) { + printf("failed: %d (%s)\n", errno, strerror(errno)); + } + } + return status; +} + +int getSocketName(Socket *socket, SocketAddress *socketAddress) +{ + InetAddr name; + int status = getEndPointName(socket->sd, &name, 0); + if (0 == status) { + setSocketAddr(socketAddress, &name, 0); + } + return status; +} + +int getSocketName2(Socket *socket, SocketAddress *socketAddress) +{ + InetAddr name; + int status = getEndPointName(socket->sd, &name, 1); + if (0 == status) { + setSocketAddr(socketAddress, &name, 0); + } + return status; +} + +/***************************************************************/ + +#ifdef __ZOWE_OS_AIX +Socket** makeEventSockets() +{ + int eventFD[2]; + int sts = 0; + Socket** rtnSockets = NULL; + + do { + sts = pipe(eventFD); + if (sts < 0) { + if (socketTrace) { + printf("makeEventSockets failed in pipe command. errno = %d (%s)\n", errno, strerror(errno)); + break; + } + } + + rtnSockets = (Socket**)safeMalloc(2*sizeof(Socket*), "evensockets array"); + rtnSockets[0] = (Socket*)safeMalloc(sizeof(Socket),"Socket1"); + memset(rtnSockets[0], 0, sizeof(Socket)); + rtnSockets[0]->sd = eventFD[0]; + rtnSockets[0]->protocol = IPPROTO_EVENT; + sprintf(rtnSockets[0]->debugName,"EVENT0=%d",eventFD[0]); + if (socketTrace) { + printf("makeEventSocket() returned %s\n", rtnSockets[0]->debugName); + } + rtnSockets[1] = (Socket*)safeMalloc(sizeof(Socket),"Socket2"); + memset(rtnSockets[1], 0, sizeof(Socket)); + rtnSockets[1]->sd = eventFD[1]; + rtnSockets[1]->protocol = IPPROTO_EVENT; + sprintf(rtnSockets[1]->debugName,"EVENT1=%d",eventFD[1]); + if (socketTrace) { + printf("makeEventSocket() returned %s\n", rtnSockets[1]->debugName); + } + } while(0); + + return rtnSockets; +} +#else +Socket* makeEventSocket() +{ + int eventFD; + if (0 > (eventFD = eventfd(0, 0))) { + if (socketTrace) { + int error = errno; + printf("makeEventSocket() failed creating eventfd. errno=%d (%s)\n", + error, strerror(error)); + } + return NULL; + } + if (eventFD >= FD_SETSIZE) { + if (socketTrace) { + printf("makeEventSocket() failed; read side FD (%d) exceeds FD_SETSIZE (%d)\n", + eventFD, (int) FD_SETSIZE); + } + return NULL; + } + Socket* socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + memset(socket, 0, sizeof(Socket)); + socket->sd = eventFD; + socket->protocol = IPPROTO_EVENT; + sprintf(socket->debugName,"EVENT=%d",eventFD); + if (socketTrace) { + printf("makeEventSocket() returned %s\n", socket->debugName); + } + return socket; +} +#endif + +int postEventSocket(Socket* socket) +{ + if (socket->protocol != IPPROTO_EVENT) { + if (socketTrace) { + printf("postEventSocket called for non-event socket %s\n", + socket->debugName); + return EINVAL; + } + } + + uint64_t bump = 1; + if (socketTrace > 1) { + printf("postEventSocket called for %s\n", socket->debugName); + } + ssize_t written = write(socket->sd, &bump, sizeof(uint64_t)); + if (written != sizeof(uint64_t)) { + if (socketTrace) { + int error = errno; + printf("postEventSocket write for %s failed. errno=%d (%s)\n", + socket->debugName, error, strerror(error)); + return error; + } + } + return 0; +} + +uint64_t waitForEventSocket(Socket* socket) +{ + if (socket->protocol != IPPROTO_EVENT) { + if (socketTrace) { + printf("waitForEventSocket called for non-event socket %s\n", + socket->debugName); + return 0; + } + } + + uint64_t count = 0; + ssize_t bytesRead = read(socket->sd, &count, sizeof(uint64_t)); + if (bytesRead != sizeof(uint64_t)) { + if (socketTrace) { + int error = errno; + printf("waitForEventSocket read for %s failed. errno=%d (%s)\n", + socket->debugName, error, strerror(error)); + } + count = 0; + } else { + if (socketTrace) { + printf("waitForEventSocket called for %s; returned %" PRIu64 "\n", + socket->debugName, count); + } + } + return count; +} + +void clearEventSocket(Socket* socket) +{ + /* Cheesy */ + if (0 == postEventSocket(socket)) { + uint64_t count = waitForEventSocket(socket); + if (socketTrace) { + if (count > 0) { + /* success */ + printf("clearEventSocket called for %s; there were %" PRIu64 " pending notifications\n", + socket->debugName, count-1); + } else { + printf("clearEventSocket called for %s; wait failed.\n", socket->debugName); + } + } + } else { + if (socketTrace) { + printf("clearEventSocket called for %s; post failed.\n", socket->debugName); + } + } +} + +SocketSet *makeSocketSet(int highestAllowedSD){ + if ((highestAllowedSD < 1) || (highestAllowedSD >= FD_SETSIZE)) { + if (socketTrace){ + printf("makeSocketSet(%d) invalid\n", highestAllowedSD); + } + return NULL; + } + SocketSet *set = (SocketSet*)safeMalloc(sizeof(SocketSet),"SocketSet"); + memset(set,0,sizeof(SocketSet)); + size_t socketArraySize = (size_t) ((highestAllowedSD+1)*sizeof(Socket*)); + set->sockets = (Socket**)safeMalloc(socketArraySize, "SocketArray"); + memset(set->sockets, 0, socketArraySize); + set->highestAllowedSD = highestAllowedSD; + FD_ZERO(&(set->allSDs)); + return set; +} + +void freeSocketSet(SocketSet *set){ + safeFree((char*) set, sizeof(SocketSet)); +} + +void setSocketSetIdleTimeLimit(SocketSet *set, int idleSeconds) { + set->idleTimeLimit = (uint64_t)idleSeconds; +} + +int socketSetAdd(SocketSet *set, Socket *socket){ + int sd = socket->sd; + + if ((sd > set->highestAllowedSD) || (sd < 0)){ + if (socketTrace){ + printf("socketSetAdd for SD=%d (%s) out of range (> %d)\n",sd, socket->debugName, set->highestAllowedSD); + } + return 12; + } + + if (FD_ISSET(sd, &(set->allSDs)) || (set->sockets[sd] != NULL)) { + if (socketTrace){ + printf("socketSetAdd for SD=%d (%s) failed - socket already in set\n",sd, socket->debugName); + } + return 12; + } + + if (NULL == set->sockets[sd]) + { + if (socketTrace > 1){ + printf("socketSetAdd for SD=%d (%s) succeeded\n",sd, socket->debugName); + } + FD_SET(sd, &(set->allSDs)); + set->sockets[sd] = socket; + ++(set->socketCount); + } + + return 0; +} + +int socketSetRemove(SocketSet *set, Socket *socket){ + int sd = socket->sd; + + if ((sd > set->highestAllowedSD) || (sd < 0)){ + if (socketTrace){ + printf("socketSetRemove for SD=%d (%s) out of range (> %d)\n",sd, socket->debugName, set->highestAllowedSD); + } + return 12; + } + if (!FD_ISSET(sd, &(set->allSDs)) || (set->sockets[sd] == NULL)) { + if (socketTrace){ + printf("socketSetRemove for SD=%d (%s) failed - socket not in set\n",sd, socket->debugName); + } + return 12; + } + + if (NULL != set->sockets[sd]) + { + if (socketTrace > 1){ + printf("socketSetRemove for SD=%d (%s) succeeded\n",sd, socket->debugName); + } + FD_CLR(sd, &(set->allSDs)); + set->sockets[sd] = NULL; + --(set->socketCount); + } + + return 0; +} + +static +void printSocketSetStatus(SocketSet *set, fd_set* fdset, const char* label) +{ + printf("%s: maxSD=%d\n", label, set->highestAllowedSD); + for (int i = 0; i < set->highestAllowedSD+1; ++i) { + if (FD_ISSET(i, fdset)) { + Socket* s = set->sockets[i]; + printf(" [%02d]: %s\n", i, (s ? s->debugName : "INVALID")); + } + } +} + +int extendedSelect(SocketSet *set, + int timeout, /* in milliseconds */ + int checkWrite, int checkRead, + int *returnCode, int *reasonCode){ + *reasonCode = *returnCode = 0; + struct timeval theTimeout = {0,0}; + struct timeval* theTimeoutPtr; + + memcpy(&(set->scratchReadMask), &(set->allSDs), sizeof(fd_set)); + memcpy(&(set->scratchWriteMask), &(set->allSDs), sizeof(fd_set)); + memcpy(&(set->scratchErrorMask), &(set->allSDs), sizeof(fd_set)); + + if (timeout >= 0) { + theTimeout.tv_sec = timeout/1000; + theTimeout.tv_usec = 1000*(timeout%1000); + theTimeoutPtr = &theTimeout; + } else { + theTimeoutPtr = (struct timeval*) NULL; + } + + int maxSoc = set->highestAllowedSD+1; + + int status = select(maxSoc, + (checkRead ? &(set->scratchReadMask) : (fd_set*) 0), + (checkWrite ? &(set->scratchWriteMask) : (fd_set*) 0), + &(set->scratchErrorMask), + theTimeoutPtr); + *reasonCode = *returnCode = (status < 0) ? errno : 0; + + if (socketTrace){ + printf("extendedSelect(maxSoc=%d, timeout={%d,%d}, write=%s, read=%s) " + "returns %d with errno=%d (%s)\n", + maxSoc, timeout/1000, 1000*(timeout%1000), + (checkWrite ? "true" : "false"), (checkRead ? "true" : "false"), + status, *returnCode, strerror(*returnCode)); + if (socketTrace > 1) { + if (socketTrace > 1) { + printSocketSetStatus(set, &(set->allSDs), "allSDs"); + } + if (checkRead) { + printSocketSetStatus(set, &(set->scratchReadMask), "read"); + } + if (checkWrite) { + printSocketSetStatus(set, &(set->scratchWriteMask), "write"); + } + if (socketTrace > 2) { + printSocketSetStatus(set, &(set->scratchErrorMask), "error"); + } + } + } + return status; +} + +static +int setSocketOptionEx(Socket *socket, + int level, const char* levelDesc, + int optionName, const char* optionDesc, + int optionDataLength, char *optionData, + int *returnCode, int *reasonCode) +{ + int status = setsockopt(socket->sd, level, optionName, optionData, (socklen_t) optionDataLength); + if (status < 0){ + *reasonCode = *returnCode = errno; + if (socketTrace){ + printf("setsockopt(%s, level=%d%s, option=%d%s, len=%d) failed, errno=%d (%s)\n", + socket->debugName, level, levelDesc, optionName, optionDesc, optionDataLength, *returnCode, strerror(*returnCode)); + } + } else { + *reasonCode = *returnCode = 0; + if (socketTrace){ + printf("setsockopt(%s, level=%d%s, option=%d%s, len=%d) OK\n", + socket->debugName, level, levelDesc, optionName, optionDesc, optionDataLength); + } + } + return status; +} + +int setSocketOption(Socket *socket, int level, int optionName, int optionDataLength, char *optionData, + int *returnCode, int *reasonCode) +{ + return setSocketOptionEx(socket, level, "", optionName, "", optionDataLength, optionData, returnCode, reasonCode); +} + +void setSocketIdleTimeoutMode(Socket *socket, int enableIdleTimeout) { + // Socket idle timeout support is currently only supported for TCPIP + if (socket->protocol == IPPROTO_TCP) { + if (enableIdleTimeout == 0) { + socket->idleTimeoutFlags = IDLE_TIMEOUT_DISABLED; + } else { + socket->idleTimeoutFlags = IDLE_TIMEOUT_ENABLED; + if (sxSocketIsReady(socket)) { // TLS is not requested or handshake is complete + socket->readyTime = socket->createTime; // Use create for ready time + } + } + } +} + +int setSocketNoDelay(Socket *socket, int noDelay, int *returnCode, int *reasonCode){ + return setSocketOptionEx(socket,IPPROTO_TCP, " (IPPROTO_TCP)",TCP_NODELAY, " (TCP_NODELAY)", sizeof(int),(char*)&noDelay,returnCode,reasonCode); +} + +int setSocketWriteBufferSize(Socket *socket, int bufferSize, int *returnCode, int *reasonCode){ + return setSocketOptionEx(socket,SOL_SOCKET, " (SOL_SOCKET)",SO_SNDBUF, " (SO_SNDBUF)", sizeof(int),(char*)&bufferSize,returnCode,reasonCode); +} + +int setSocketReadBufferSize(Socket *socket, int bufferSize, int *returnCode, int *reasonCode){ + return setSocketOptionEx(socket,SOL_SOCKET, " (SOL_SOCKET)",SO_RCVBUF, " (SO_RCVBUF)", sizeof(int),(char*)&bufferSize,returnCode,reasonCode); +} diff --git a/c/timeutls.c b/c/timeutls.c index ef26a79c5..a2b5dd3db 100644 --- a/c/timeutls.c +++ b/c/timeutls.c @@ -26,7 +26,13 @@ #include #include #include + +#ifndef _MSC_VER /* WINDOWS */ #include +#else +#include +#endif /* WINDOWS */ + #endif #include "zowetypes.h" @@ -292,7 +298,7 @@ int timeZoneDifferenceFor(int64 theTime) return -1; } -#elif defined(__ZOWE_OS_LINUX) || defined(__ZOWE_OS_AIX) +#elif defined(__ZOWE_OS_LINUX) || defined(__ZOWE_OS_AIX) || defined(__ZOWE_OS_WINDOWS) #include @@ -353,13 +359,78 @@ uint64 sleazyToSeconds(const struct tm* time) return result; } +/* a platform sensitive wrapper */ +static void platformGMTime(time_t *time, + struct tm *tm){ +#ifdef __ZOWE_OS_WINDOWS + memcpy(tm,gmtime(time),sizeof(struct tm)); +#else + gmtime_r(time,tm); +#endif +} + +/* A missing function in Windows - + * This implementation came from A MS VisStudio Support pag + */ + +#ifdef __ZOWE_OS_WINDOWS + +#define DELTA_EPOCH_IN_MICROSECS 11644473600000000Ui64 + +struct timezone +{ + int tz_minuteswest; /* minutes W of Greenwich */ + int tz_dsttime; /* type of dst correction */ +}; + +static int gettimeofday(struct timeval *tv, struct timezone *tz) +{ + FILETIME ft; + unsigned __int64 tmpres = 0; + static int tzflag; + + if (NULL != tv) + { + GetSystemTimeAsFileTime(&ft); + + tmpres |= ft.dwHighDateTime; + tmpres <<= 32; + tmpres |= ft.dwLowDateTime; + + /*converting file time to unix epoch*/ + tmpres -= DELTA_EPOCH_IN_MICROSECS; + tmpres /= 10; /*convert into microseconds*/ + tv->tv_sec = (long)(tmpres / 1000000UL); + tv->tv_usec = (long)(tmpres % 1000000UL); + } + + if (NULL != tz) + { + if (!tzflag) + { + _tzset(); + tzflag++; + } + tz->tz_minuteswest = _timezone / 60; + tz->tz_dsttime = _daylight; + } + + return 0; +} + +#endif /* WINDOWS gettimeofday implementation */ + static int tzsetHasBeenCalled = 0; static inline void tzsetWrapper() { if (!tzsetHasBeenCalled) { +#ifdef __ZOWE_OS_WINDOWS + _tzset(); +#else tzset(); +#endif tzsetHasBeenCalled = 1; } } @@ -371,7 +442,7 @@ int timeZoneDifferenceInternal(time_t base, struct tm* local) int result = 0; tzsetWrapper(); struct tm utc; - gmtime_r(&base, &utc); + platformGMTime(&base, &utc); uint64 utcBase = sleazyToSeconds(&utc); uint64 localBase = sleazyToSeconds(local); if (utcBase > localBase) { @@ -384,6 +455,7 @@ int timeZoneDifferenceInternal(time_t base, struct tm* local) return result; } + /* Compute the difference between UTC and local time for the specified Unix time (seconds since the Unix Epoch). @@ -393,7 +465,12 @@ int timeZoneDifferenceFor(int64 theTime) /* There has to be a better way... */ time_t base = (time_t) theTime; struct tm local; +#ifdef __ZOWE_OS_WINDOWS + struct tm *tempTM = localtime(&base); + memcpy(&local,tempTM,sizeof(struct tm)); +#else localtime_r(&base, &local); +#endif return timeZoneDifferenceInternal(base, &local); } @@ -463,7 +540,7 @@ int stckToTimestamp(int64 stck, char *output) stckToTimeval(&timeval, stck); time_t time = (time_t)timeval.tv_sec; struct tm split; - gmtime_r(&time, &split); + platformGMTime(&time, &split); memset(output, 0, TIMESTAMP_LENGTH); writePackedDecimal(output,0,10,split.tm_hour); @@ -473,6 +550,7 @@ int stckToTimestamp(int64 stck, char *output) writePackedDecimal(output,8,1000,split.tm_year+1900); writePackedDecimal(output,10,10,split.tm_mon+1); writePackedDecimal(output,11,10,split.tm_mday); + return 0; } int timestampToSTCK(char *todText, int64 *stck, int64 offset) @@ -516,7 +594,6 @@ int timestampToSTCK(char *todText, int64 *stck, int64 offset) *stck = (int64)rawStck; return 0; } - #else #error OS unknown #endif @@ -764,9 +841,9 @@ void unixToTimestamp(uint64 unixTime, char *output) { convertIntToString(output + 10, 2, minutes); convertIntToString(output + 12, 2, seconds); -#elif defined(__ZOWE_OS_LINUX) || defined(__ZOWE_OS_AIX) +#elif defined(__ZOWE_OS_LINUX) || defined(__ZOWE_OS_AIX) || defined(__ZOWE_OS_WINDOWS) struct tm split; - gmtime_r((time_t*)&unixTime, &split); + platformGMTime((time_t*)&unixTime, &split); memset(output, 0x0, 16); // Copy date @@ -801,18 +878,24 @@ int snprintLocalTime(char *buffer, int length, int tzSource) { gmtime_r(&tt, result); #elif defined(__ZOWE_OS_LINUX) || defined(__ZOWE_OS_AIX) localtime_r(&tt, result); +#elif defined(__ZOWE_OS_WINDOWS) + memcpy(result,localtime(&tt),sizeof(struct tm)); #else #error Unknown OS #endif } else if (tzSource == TZ_FROM_TZ) { +#ifndef __ZOWE_OS_WINDOWS localtime_r(&tt, result); +#else + memcpy(result,localtime(&tt),sizeof(struct tm)); +#endif } else { return 0; } return snprintf(buffer, length, "%04d-%02d-%02d-%02d-%02d-%02d.%06d", 1900 + result->tm_year, result->tm_mon + 1, result->tm_mday, result->tm_hour, result->tm_min, result->tm_sec, - tv->tv_usec); + (int)tv->tv_usec); #endif } diff --git a/c/utils.c b/c/utils.c index 72d53c665..60c8c931f 100644 --- a/c/utils.c +++ b/c/utils.c @@ -708,7 +708,7 @@ void dumpbufferA(const char *buffer, int length){ #ifdef METTLE printf("the buffer is empty at %x\n",buffer); #else - fprintf((FILE*)traceOut,"the buffer is empty at %x\n",buffer); + fprintf((FILE*)traceOut,"the buffer is empty at %p\n",buffer); #endif while (index <= last_index){ @@ -1051,8 +1051,8 @@ static char binToEB64[] ={0xc1,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xd1,0xd2 char *encodeBase64(ShortLivedHeap *slh, const char buf[], int size, int *resultSize, int useEbcdic){ int allocSize = BASE64_ENCODE_SIZE(size)+1; /* +1 for null term */ - char *result = NULL; - if (result = (slh ? SLHAlloc(slh,allocSize) : safeMalloc31(allocSize,"BASE64"))) { + char *result = (slh ? SLHAlloc(slh,allocSize) : safeMalloc31(allocSize,"BASE64")); + if (result){ encodeBase64NoAlloc(buf, size, result, resultSize, useEbcdic); return result; } else{ @@ -1452,7 +1452,7 @@ static void reportSLHFailure(ShortLivedHeap *slh, int size){ while (chain){ char *data = chain->data-4; int size = *((int*)data); - printf(" segment %d at 0x%x, size = 0x%x\n",i,data,size+4); + printf(" segment %d at 0x%p, size = 0x%x\n",i,data,size+4); chain = chain->next; i++; } @@ -1477,13 +1477,15 @@ char *SLHAlloc(ShortLivedHeap *slh, int size){ size += (8-rem); } char *data; - /* printf("slh=%d\n",slh);fflush(stdout); - printf("SLHAlloc me=0x%x size=%d bc=%d\n",slh,size,slh->blockCount);fflush(stdout); */ + /* + printf("slh=0x%p\n",slh);fflush(stdout); + printf("SLHAlloc me=0x%p size=%d bc=%d\n",slh,size,slh->blockCount);fflush(stdout); + */ int remainingHeapBytes = (slh->blockSize * (slh->maxBlocks - slh->blockCount)); if (size > remainingHeapBytes){ printf("SLH at 0x%p cannot allocate above block size %d > %d mxbl %d bkct %d bksz %d\n", slh,size,remainingHeapBytes,slh->maxBlocks,slh->blockCount,slh->blockSize); - + fflush(stdout); char *mem = (char*)0; mem[0] = 13; return NULL; @@ -1736,7 +1738,7 @@ CharStream *makeBufferCharStream(char *buffer, int len, int trace){ s->positionMethod = getBufferPosition; s->closeMethod = NULL; if (trace){ - printf("mbcs out 0x%x, '%.*s'\n",s,len,s); + printf("mbcs out 0x%p, '%.*s'\n",s,len,buffer); } return s; } @@ -1980,6 +1982,10 @@ int decimalToOctal(int decimal) { #define debugPrintf(formatString, ...) #endif +static int incrementPlaceValues(int *placeValues, + int lim, int digits); + + int matchWithWildcards(char *pattern, int patternLen, char *s, int len, int flags){ diff --git a/c/winskt.c b/c/winskt.c new file mode 100644 index 000000000..095cc1e8f --- /dev/null +++ b/c/winskt.c @@ -0,0 +1,1123 @@ + + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#include +#include +#include +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "openprims.h" +#include "collections.h" +#include "bpxnet.h" +#include "unixfile.h" + +/* + this pragma tells the compiler/linker to link in the right, modern WinSock stuff + + */ + +#pragma comment(lib, "Ws2_32.lib") + +/* + https://msdn.microsoft.com/en-us/library/windows/desktop/ms741394(v=vs.85).aspx + https://tangentsoft.net/wskfaq/newbie.html + + fcntl = ioctlsocket + + IOCTL / FCNTL + https://msdn.microsoft.com/en-us/library/windows/desktop/ms740126(v=vs.85).aspx + + + Various C language run-time systems use the IOCTLs for purposes + unrelated to Windows Sockets. As a consequence, the ioctlsocket + function and the WSAIoctl function were defined to handle socket + functions that were performed by IOCTL and fcntl in the Berkeley + Software Distribution. + */ + +int tcpInit(char *uniqueName){ + /* do nothing for now */ + return 0; +} + +static int socketTrace = TRUE; + +SocketAddress *makeSocketAddr(InetAddr *addr, + unsigned short port){ + SocketAddress *address = (SocketAddress*)safeMalloc(sizeof(SocketAddress),"SocketAddress"); + memset(address,0,sizeof(SocketAddress)); + if (socketTrace){ + printf("socket address at 0x%p\n",address); + } + address->family = AF_INET; + address->port = ((port & 0xff)<<8)|((port&0xff00)>>8); + if (addr){ + address->v4Address = addr->data.data4; + } else{ + memset(&(address->v4Address),0,sizeof(struct in_addr)); + } + if (socketTrace){ + printf("about to return socket address at 0x%p\n",address); + } + return address; +} + +void freeSocketAddr(SocketAddress *address){ + safeFree((char*)address,sizeof(SocketAddress)); +} + +/*--5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---*/ + +#define SO_ERROR 0x1007 + +Socket *tcpClient2(SocketAddress *socketAddress, + int timeoutInMillis, + int *returnCode, /* errnum */ + int *reasonCode){ /* errnum - JR's */ + int socketVector[2]; + int *reasonCodePtr; + int status; + int madeConnection = FALSE; + SOCKET windowsSocket = socket(AF_INET,SOCK_STREAM,IPPROTO_TCP); + struct sockaddr *windowsSocketAddress = (struct sockaddr *)socketAddress; + + printf("socket = 0x%llx\n",windowsSocket); + if (windowsSocket == INVALID_SOCKET){ + if (socketTrace){ + printf("Failed to create socket\n"); + } + return NULL; + } else{ + int socketAddrSize = sizeof(SocketAddress); + if (timeoutInMillis >= 0){ + Socket tempSocket; + tempSocket.windowsSocket = windowsSocket; + + /* setSocketBlockingMode(&tempSocket, TRUE, returnCode, reasonCode); */ + printf("windowsSocket:\n"); + dumpbuffer((char*)windowsSocketAddress,20); + status = connect(windowsSocket,windowsSocketAddress,sizeof(struct sockaddr_in)); + *returnCode = WSAGetLastError(); + printf("socket connect status=%d errno=%d\n",status,*returnCode); + fflush(stdout); + if (socketTrace) { + printf("connect() status %d\n",status); + } + /* EINPROGRESS is the expected return code here but we are just being careful by checking + for EWOULDBLOCK as well + + WINDOWS NOTE: status needs to 0, opposite of z/OS. But maybe the zOS code is getting + lucky. The windows return value for connect() seems to have a clearer + interpretation here. + */ + if (status == 0){ + *returnCode = 0; + *reasonCode = 0; + status = tcpStatus(&tempSocket, timeoutInMillis, 1, returnCode, reasonCode); + printf("after tcpStatus() = %d\n",status); + if (status == SD_STATUS_TIMEOUT) { + int sd = socketVector[0]; + if (socketTrace) { + printf("Failed to connect socket, will clean up sd=%d\n", sd); + } + *returnCode = 0; + *reasonCode = 0; + status = closesocket(windowsSocket); + if (socketTrace) { + printf("closeSocket() for time out connect status %d\n",status); + } + return NULL; + } + printf("tcpStatus return and did not timeout status=%d\n",status); + fflush(stdout); + *returnCode = 0; + *reasonCode = 0; + /* setSocketBlockingMode(&tempSocket, FALSE, returnCode, reasonCode); */ + int optionLength = sizeof(int); + int optionData = 0; + *returnCode = 0; + *reasonCode = 0; + status = getSocketOption(&tempSocket, SO_ERROR, &optionLength, (char *)&optionData, returnCode, reasonCode); + if( optionData > 0 ){ + if (socketTrace){ + printf("Failed to connect socket, returnCode %d\n", optionData); + } + *returnCode = optionData; + madeConnection = FALSE; + } else { + madeConnection = TRUE; + } + } + if (socketTrace){ + printf("did we make connection (timeout case) = %d\n",madeConnection); + } + } else{ /* if timeouts are not specified */ + status = connect(windowsSocket,windowsSocketAddress,14); + if (socketTrace){ + printf("BPXCON status %d returnCode %x reasonCode %x\n", + status,*returnCode,*reasonCode); + } + madeConnection = (status == 0 ? TRUE : FALSE); + } + if (!madeConnection){ + if (socketTrace){ + printf("Failed to connect socket, will clean up, SOCKET at 0x%llx\n",windowsSocket); + } + *returnCode = 0; + *reasonCode = 0; + status = closesocket(windowsSocket); + if (socketTrace){ + printf("closesocket() for failed connect status %d\n",status); + } + return NULL; + } else{ + Socket *socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + socket->windowsSocket = windowsSocket; + sprintf(socket->debugName,"HNDL=0X%llx",socket->windowsSocket); + socket->isServer = FALSE; + /* manual says returnCode and value only meaningful if return value = -1 */ + *returnCode = 0; + *reasonCode = 0; + socket->protocol = IPPROTO_TCP; + return socket; + } + } +} + +/* pointer to length is stupid */ +int getSocketOption(Socket *socket, int optionName, int *optionDataLength, char *optionData, + int *returnCode, int *reasonCode){ + SOCKET windowsSocket = socket->windowsSocket; + int getOption = 1; + int setOption = 2; + int setIBMOption = 3; + int level = 0x0000FFFF; + + if (socketTrace){ + printf("before get socket option optionName=0x%x dataBufferLen=%d\n",optionName,*optionDataLength); + } + int status = setsockopt(windowsSocket, + SOL_SOCKET, /* is this always right */ + optionName, + optionData, + *optionDataLength); + if (socketTrace){ + printf("after getsockopt, status=%d ret code %d reason 0x%x\n",status,*returnCode,*reasonCode); + } + if (status < 0){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + printf("get sockopt failed, ret code %d reason 0x%x\n",*returnCode,*reasonCode); + return -1; + } else { + *returnCode = 0; + *reasonCode = 0; + return status; + } +} + + +Socket *tcpClient(SocketAddress *socketAddress, + int *returnCode, /* errnum */ + int *reasonCode){ /* errnum - JR's */ + return tcpClient2(socketAddress,-1,returnCode,reasonCode); +} + +void socketFree(Socket *socket){ + safeFree((char*)socket,sizeof(Socket)); +} + +static +int setSocketOptionEx(Socket *socket, + int level, const char* levelDesc, + int optionName, const char* optionDesc, + int optionDataLength, char *optionData, + int *returnCode, int *reasonCode) +{ + SOCKET windowsSocket = socket->windowsSocket; + int status = setsockopt(windowsSocket, level, optionName, optionData, (socklen_t) optionDataLength); + if (status < 0){ + *reasonCode = *returnCode = errno; + if (socketTrace){ + printf("setsockopt(%s, level=%d%s, option=%d%s, len=%d) failed, errno=%d (%s)\n", + socket->debugName, level, levelDesc, optionName, optionDesc, optionDataLength, *returnCode, strerror(*returnCode)); + } + } else { + *reasonCode = *returnCode = 0; + if (socketTrace){ + printf("setsockopt(%s, level=%d%s, option=%d%s, len=%d) OK\n", + socket->debugName, level, levelDesc, optionName, optionDesc, optionDataLength); + } + } + return status; +} + +int setSocketOption(Socket *socket, int level, int optionName, int optionDataLength, char *optionData, + int *returnCode, int *reasonCode) +{ + return setSocketOptionEx(socket, level, "", optionName, "", optionDataLength, optionData, returnCode, reasonCode); +} + + +int getSocketKeepAliveMode(Socket *socket, int *returnCode, int *reasonCode){ + + int optval = 0; + int optlen = sizeof(optval); + int returnValue = getSocketOption(socket,SO_KEEPALIVE,&optlen,(char*)&optval, + returnCode,reasonCode); + if (returnValue < 0) { // If an error occurred + printf("Unable to get socket option SO_KEEPALIVE.\n"); + return returnValue; + } + returnValue = optval; + + *returnCode = 0; + *reasonCode = 0; + return returnValue; +} + +// - An enableKeepAlive value of 0 will disable keepalive processing +// - An enableKeepAlive value of >0 will enable keepalive processing with the default +// time values. +int setSocketKeepAliveMode(Socket *socket, int enableKeepAlive, + int *returnCode, int *reasonCode){ + + // Enable/disable SO_KEEPALIVE, as requested + int optval = (enableKeepAlive ? 1 : 0); + int returnValue = setSocketOption(socket,SOL_SOCKET,SO_KEEPALIVE,sizeof(optval),(char*)&optval, + returnCode,reasonCode); + if (returnValue < 0) { // If an error occurred + printf("Unable to set socket option SO_KEEPALIVE to %s.\n",(optval? "on" : "off")); + return returnValue; + } + + *returnCode = 0; + *reasonCode = 0; + return returnValue; +} + +int socketRead(Socket *socket, char *buffer, int desiredBytes, + int *returnCode, int *reasonCode){ + SOCKET windowsSocket = socket->windowsSocket; + int flags = 0; /* no special behaviors */ + + int bytesReadOrError = recv(windowsSocket,buffer,desiredBytes,flags); + if (bytesReadOrError < 0){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + return -1; + } else { + if (socketTrace){ + printf("read %d bytes\n",bytesReadOrError); + } + *returnCode = 0; + *reasonCode = 0; + return bytesReadOrError; + } +} + +int socketWrite(Socket *socket, const char *buffer, int desiredBytes, + int *returnCode, int *reasonCode){ + SOCKET windowsSocket = socket->windowsSocket; + int flags = 0; + + if (socketTrace){ + printf("writing desired=%d retCode=%d reasonCode=%d\n", + desiredBytes,*returnCode,*reasonCode); + dumpbuffer(buffer, desiredBytes); + } + + int bytesWrittenOrError = send(windowsSocket,buffer,desiredBytes,0); + if (bytesWrittenOrError < 0){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + printf("write failed, socketHandle=0x%llx,desired write len %d buffer at 0x%p, ret code %d reason 0x%x\n", + windowsSocket,desiredBytes,buffer,*returnCode,*reasonCode); + return -1; + } else { + if (socketTrace){ + printf("wrote %d bytes\n",bytesWrittenOrError);fflush(stdout); + } + *returnCode = 0; + *reasonCode = 0; + return bytesWrittenOrError; + } +} + +static void setupFDSets(fd_set *readFDs, + fd_set *writeFDs, + fd_set *exceptionFDs, + SOCKET windowsSocket, + int checkWrite, + int checkRead){ + FD_ZERO(readFDs); + FD_ZERO(writeFDs); + FD_ZERO(exceptionFDs); + + if (checkWrite){ + FD_SET(windowsSocket,writeFDs); + } + if (checkRead){ + FD_SET(windowsSocket,readFDs); + FD_SET(windowsSocket,exceptionFDs); + } + +} + +static int tcpStatusInternal(Socket *socket, + int timeout, /* in milliseconds */ + int checkWrite, int checkRead, + int *returnCode, int *reasonCode){ + /* what does it MEAN?? */ + int status = 0; + SOCKET windowsSocket = socket->windowsSocket; + fd_set readFDs, writeFDs, exceptionFDs; + struct timeval timeValue; + const struct timeval *timeValuePtr; + + if (timeout >= 0) + { + timeValue.tv_sec = timeout/1000; /* millis to seconds */ + timeValue.tv_usec = (timeout-(timeout/1000)*1000)*1000; /* micros */ + timeValuePtr = &timeValue; + } else { + timeValuePtr = NULL; + } + + setupFDSets(&readFDs,&writeFDs,&exceptionFDs,windowsSocket,checkWrite,checkRead); + status = select(0,&readFDs,&writeFDs,&exceptionFDs,timeValuePtr); + + if (socketTrace){ + printf("BPXSEL status %d returnCode %x reasonCode %x\n", + status,*returnCode,*reasonCode); + } + if (status > 0){ + int sdStatus = 0; + + if (FD_ISSET(windowsSocket,&readFDs)){ + sdStatus |= SD_STATUS_RD_RDY; + } + if (FD_ISSET(windowsSocket,&writeFDs)){ + sdStatus |= SD_STATUS_WR_RDY; + } + return sdStatus; + } else if (status == 0){ /* timeout */ + return SD_STATUS_TIMEOUT; + } else if (FD_ISSET(windowsSocket,&exceptionFDs)){ + return SD_STATUS_ERROR; + } else{ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + return SD_STATUS_FAILED; /* look in errno */ + } +} + +int tcpStatus(Socket *socket, + int timeout, /* in milliseconds */ + int checkWrite, /* otherwise check for read and error */ + int *returnCode, int *reasonCode){ + return tcpStatusInternal(socket,timeout,checkWrite,!checkWrite,returnCode,reasonCode); +} + +#define O_NONBLOCK 0x04 +#define F_GETFL 3 +#define F_SETFL 4 + +/* Note: Windows "ioctl" known as ioctlsocket is not as complete + as it is in posix/unix. See WSAIoctl for richer functionality */ + +int setSocketBlockingMode(Socket *socket, int isNonBlocking, + int *returnCode, int *reasonCode){ + SOCKET windowsSocket = socket->windowsSocket; + int status = 0; + long command = FIONBIO; + unsigned long argument = (isNonBlocking ? 1 : 0); + + status = ioctlsocket(windowsSocket,command,&argument); + + if (status < 0){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + + printf("ioctlsocket(1) failed, ret code %d reason 0x%x\n",*returnCode,*reasonCode); + return -1; + } else { + if (socketTrace){ + printf("ioctlsocket succeeded, nonblocking=%d \n",isNonBlocking); + } + *returnCode = 0; + *reasonCode = 0; + return 0; + } +} + +Socket *udpPeer(SocketAddress *socketAddress, + int *returnCode, /* errnum */ + int *reasonCode){ /* errnum - JR's */ + int status; + struct sockaddr *windowsSocketAddress = (struct sockaddr *)socketAddress; + + SOCKET windowsSocket = socket(AF_INET,SOCK_DGRAM,IPPROTO_UDP); + + if (windowsSocket == INVALID_SOCKET){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + if (socketTrace){ + printf("Failed to create socket, error=%d\n",*returnCode); + } + return NULL; + } else{ + int socketAddressSize = sizeof(SocketAddress); + status = bind(windowsSocket,windowsSocketAddress,sizeof(struct sockaddr_in)); + if (status != 0){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + } + if (socketTrace) { + printf("BPXBND status %d returnCode %x reasonCode %x\n", + status, *returnCode, *reasonCode); + } + if (status != 0){ + return NULL; + } else{ + Socket *socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + socket->windowsSocket = windowsSocket; + sprintf(socket->debugName,"HNDL=0X%llx",socket->windowsSocket); + socket->isServer = FALSE; + /* manual says returnCode and value only meaningful if return value = -1 */ + *returnCode = 0; + *reasonCode = 0; + socket->protocol = IPPROTO_UDP; + return socket; + } + } +} + +Socket *tcpServer(InetAddr *addr, /* usually NULL/0 */ + int port, + int *returnCode, + int *reasonCode){ + int socketVector[2]; + int status; + + SOCKET windowsSocket = socket(AF_INET,SOCK_STREAM,IPPROTO_TCP); + if (windowsSocket == INVALID_SOCKET){ + if (socketTrace){ + printf("Failed to create socket\n"); + } + return NULL; + } else{ + int sd = socketVector[0]; + int socketAddressSize = sizeof(SocketAddress); + int inetAddressSize = sizeof(InetAddr); + SocketAddress *socketAddress = makeSocketAddr(addr,(unsigned short)port); + struct sockaddr *windowsSocketAddress = (struct sockaddr *)socketAddress; + InetAddr inetAddr; + memset(&inetAddr,0,sizeof(InetAddr)); + inetAddr.type = AF_INET; + inetAddr.port = (unsigned short)port; + memset(&inetAddr.data.data4,0,4); + if (socketTrace){ + printf("server SocketAddress = %d\n",sd); + dumpbuffer((char*)socketAddress,sizeof(SocketAddress)); + dumpbuffer((char*)&inetAddr,sizeof(InetAddr)); + } + status = bind(windowsSocket,windowsSocketAddress,sizeof(struct sockaddr_in)); + if (status != 0){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + } + if (status != 0){ + if (socketTrace){ + printf("Failed to bind server socket errno=%d reason=0x%x\n", + *returnCode,*reasonCode); + } + safeFree((char*)socketAddress,sizeof(SocketAddress)); + return NULL; + } else{ + int backlogQueueLength = 100; + status = listen(windowsSocket,backlogQueueLength); + + if (status != 0){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + if (socketTrace){ + printf("Failed to start server socket listen errno=%d reason=0x%x\n", + *returnCode,*reasonCode); + } + return NULL; + } else { + Socket *socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + /* manual says returnCode and value only meaningful if return value = -1 */ + *returnCode = 0; + *reasonCode = 0; + + socket->windowsSocket = windowsSocket; + sprintf(socket->debugName,"HNDL=0X%llx",socket->windowsSocket); + socket->isServer = TRUE; + socket->protocol = IPPROTO_TCP; + if (socketTrace) { + printf("Immediately before return from tcpServer, socket contains:\n"); + dumpbuffer((char*)socket, sizeof(Socket)); + } + return socket; + } + } + } + +} + +/*--5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---*/ + + +Socket *socketAccept(Socket *serverSocket, int *returnCode, int *reasonCode){ + SocketAddress *newSocketAddress + = (SocketAddress*)safeMalloc(sizeof(SocketAddress),"Socket Address"); + struct sockaddr *windowsSocketAddress = (struct sockaddr *)newSocketAddress; + SOCKET windowsSocket = serverSocket->windowsSocket; + int status; + int newSocketAddressSize = sizeof(struct sockaddr_in); + + printf("newSocketAddressSize=%d sizeof(SocketAddress)=%lld\n",newSocketAddressSize,sizeof(SocketAddress)); + fflush(stdout); + + SOCKET newWindowsSocket = accept(windowsSocket,windowsSocketAddress,&newSocketAddressSize); + if (newWindowsSocket == INVALID_SOCKET){ + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + if (socketTrace){ + printf("Failed to accept new socket errno=%d reasonCode=0x%x\n", + *returnCode,*reasonCode); + } + return NULL; + } else{ + Socket *socket = (Socket*)safeMalloc(sizeof(Socket),"Socket"); + memset(socket,0,sizeof(Socket)); + socket->windowsSocket = newWindowsSocket; + socket->isServer = 0; + socket->protocol = IPPROTO_TCP; + return socket; + } +} + + +#define DOT 1 +#define FIRST 2 +#define SECOND 3 +#define THIRD 4 + +int isV4Numeric(char *chars, int *addr){ + int i,len = strlen(chars); + int state = DOT; + int dotCount = 0; + int value; + int addressValue = 0; + for (i=0; i= '0' && ch <= '9'){ + if (dotCount == 0 && ch == '0') + return 0; + else + state = FIRST; + } else { + return 0; + } + break; + case FIRST: + if (ch >= '0' && ch <= '9'){ + state = SECOND; + } else if (ch == '.'){ + state = DOT; + value = (chars[i-1] - '0'); + addressValue = (addressValue << 8) | value; + dotCount++; + } else{ + return 0; + } + break; + case SECOND: + if (ch >= '0' && ch <= '9'){ + state = THIRD; + } else if (ch == '.'){ + state = DOT; + value = ((10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + addressValue = (addressValue << 8) | value; + dotCount++; + } else{ + return 0; + } + break; + case THIRD: + if (ch >= '0' && ch <= '9'){ + return 0; + } else if (ch == '.'){ + state = DOT; + value = ((100 * (chars[i-3] - '0')) + + (10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + addressValue = (addressValue << 8) | value; + if (value < 256) + dotCount++; + else + return 0; + } else{ + return 0; + } + break; + } + } + if (dotCount == 3){ + switch (state){ + case FIRST: + value = (chars[i-1] - '0'); + break; + case SECOND: + value = ((10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + break; + case THIRD: + value = ((100 * (chars[i-3] - '0')) + + (10 * (chars[i-2] - '0')) + + (chars[i-1] - '0')); + break; + } + addressValue = (addressValue << 8) | value; + *addr = addressValue; + return (value < 256); + } else{ + return 0; + } +} + +/* the return value is in network byte order */ + +int getV4HostByName(char *hostName){ + int status = 0; + int returnCode = 0; + int reasonCode = 0; + int len = strlen(hostName); + struct hostent *hostEntPtr; + + hostEntPtr = gethostbyname(hostName); + if (socketTrace){ + printf("hostent addr = %x\n",*((int*)hostEntPtr)); + } + if (hostEntPtr){ + int i = 0; + int numericAddress = 0; + if (socketTrace){ + dumpbuffer((char*)hostEntPtr,20); + } + printf("hostent->length=%d name=%s\n",hostEntPtr->h_length,hostEntPtr->h_name); + fflush(stdout); + do { + char *hostAddr = hostEntPtr->h_addr_list[i]; + if (hostAddr == NULL){ + break; + } + printf("hostAddr = 0x%p %s\n",hostAddr,hostAddr); + numericAddress = *((int*)hostAddr); /*very IP-v4 here */ + printf("numeric=0x%x\n",numericAddress); + i++; + } while (TRUE); + return numericAddress; + } else{ + returnCode = WSAGetLastError(); + reasonCode = returnCode; + printf("getHostName V4 failure, returnCode %d reason code %d\n",returnCode,reasonCode); + return 0; + } +} + + +InetAddr *getAddressByName(char *addressString){ + int numericAddress = 0x7F123456; + + if (!isV4Numeric(addressString,&numericAddress)){ + numericAddress = getV4HostByName(addressString); + /* printf("Host names is DNS NAME (or crap), %x\n",numericAddress); */ + } else{ + /* printf("Host name is numeric %x\n",numericAddress); */ + } + + if (numericAddress != 0x7F123456){ + InetAddr *addr = (InetAddr*)safeMalloc(sizeof(InetAddr),"Inet Address"); + memset(addr,0,sizeof(InetAddr)); + addr->type = AF_INET; + addr->port = 0; + memcpy(&(addr->data.data4),&numericAddress,4); + printf("InetAddr structure: AF_INET=0x%x\n",AF_INET); + dumpbuffer((char*)addr,sizeof(InetAddr)); + fflush(stdout); + return addr; + } else{ + return NULL; + } +} + +int socketInit(char *uniqueName){ + WORD wVersionRequested; + WSADATA wsaData; + int err; + + /* Use the MAKEWORD(lowbyte, highbyte) macro declared in Windef.h */ + wVersionRequested = MAKEWORD(2, 0); + + err = WSAStartup(wVersionRequested, &wsaData); + if (err != 0) { + /* Tell the user that we could not find a usable */ + /* Winsock DLL. */ + printf("WSAStartup failed with error: %d\n", err); + return 1; + } + return 0; +} + +int socketClose(Socket *socket, int *returnCode, int *reasonCode){ + SOCKET windowsSocket = socket->windowsSocket; + int status; + + status = closesocket(windowsSocket); + if (status == 0){ + return 0; + } else { + *returnCode = WSAGetLastError(); + *reasonCode = *returnCode; + return -1; + } +} + +/*** File/Directory stuff ***/ + +static int fileTrace = FALSE; + +int setFileTrace(int toWhat) { + int was = fileTrace; + fileTrace = toWhat; + return was; +} + +UnixFile *fileOpen(const char *filename, int options, int mode, int bufferSize, int *returnCode, int *reasonCode){ + int returnValue = 0; + int *reasonCodePtr; + int status; + int len = strlen(filename); + char *unixMode = "r"; + + /* r Open text file for reading. The stream is positioned at the + beginning of the file. + + r+ Open for reading and writing. The stream is positioned at the + beginning of the file. + + w Truncate file to zero length or create text file for writing. + The stream is positioned at the beginning of the file. + + w+ Open for reading and writing. The file is created if it does + not exist, otherwise it is truncated. The stream is + positioned at the beginning of the file. + + a Open for appending (writing at end of file). The file is + created if it does not exist. The stream is positioned at the + end of the file. + + a+ Open for reading and appending (writing at end of file). The + file is created if it does not exist. The initial file + position for reading is at the beginning of the file, but + output is always appended to the end of the file. + */ + if (options & FILE_OPTION_APPEND){ + unixMode = (options & FILE_OPTION_READ_ONLY) ? "a+" : "a"; + } else if (options & FILE_OPTION_WRITE_ONLY){ + if (options & FILE_OPTION_READ_ONLY){ + unixMode = (options & FILE_OPTION_TRUNCATE) ? "w+" : "r+"; + } else { + if (options & FILE_OPTION_TRUNCATE){ + unixMode = "w"; + } else { + *returnCode = -1; + return NULL; /* parameter error */ + } + } + } else if (options & FILE_OPTION_READ_ONLY){ + unixMode = "r"; + } else { + *returnCode = -1; + return NULL; /* parameter error */ + } + + FILE *internalFile = fopen(filename,unixMode); + if (internalFile == NULL){ + *returnCode = -1; + return NULL; /* open failed */ + } + UnixFile *file = (UnixFile*)safeMalloc(sizeof(UnixFile),"OMVS File"); + memset(file,0,sizeof(UnixFile)); + file->internalFile = internalFile; + file->fd = _fileno(internalFile); + file->pathname = safeMalloc(strlen(filename)+1,"Unix File Name"); + strcpy(file->pathname, filename); + file->isDirectory = FALSE; + if (bufferSize > 0){ + file->buffer = safeMalloc(bufferSize,"OMVS File Buffer"); + file->bufferSize = bufferSize; + file->bufferPos = bufferSize; + file->bufferFill = bufferSize; + } + return file; +} + +int fileRead(UnixFile *file, char *buffer, int desiredBytes, + int *returnCode, int *reasonCode){ + FILE *internalFile = file->internalFile; + + size_t bytesRead = fread(buffer,1,desiredBytes,internalFile); + *returnCode = 0; + if (bytesRead < desiredBytes){ + file->eofKnown = TRUE; + if (ferror(internalFile)){ + *returnCode = errno; + *reasonCode = errno; + } + } + return (int)bytesRead; +} + +int fileWrite(UnixFile *file, const char *buffer, int desiredBytes, + int *returnCode, int *reasonCode) { + FILE *internalFile = file->internalFile; + + size_t bytesWritten = fwrite(buffer,1,desiredBytes,internalFile); + *returnCode = 0; + if (bytesWritten < desiredBytes){ + file->eofKnown = TRUE; + if (ferror(internalFile)){ + *returnCode = errno; + *reasonCode = errno; + } + } + return (int)bytesWritten; +} + +int fileGetChar(UnixFile *file, int *returnCode, int *reasonCode){ + if (file->bufferSize == 0){ + *returnCode = 8; + *reasonCode = 0xBFF; + return -1; + } else if (file->bufferPos < file->bufferFill){ + return (int)(file->buffer[file->bufferPos++]); + } else if (file->eofKnown){ + return -1; + } else{ + /* go after next buffer and pray */ + int bytesRead = fileRead(file,file->buffer,file->bufferSize,returnCode,reasonCode); + if (bytesRead >= 0) { + if (bytesRead < file->bufferSize) { /* out of data after this read */ + file->eofKnown = TRUE; + } + file->bufferFill = bytesRead; + file->bufferPos = 1; + return file->buffer[0]; + } else{ + return -1; + } + } +} + +int fileInfo(const char *filename, FileInfo *info, int *returnCode, int *reasonCode){ + int nameLength = strlen(filename); + + int status = _stat(filename,info); + if (status != 0){ + *returnCode = errno; + *reasonCode = errno; + } + return status; +} + +int fileEOF(const UnixFile *file){ + return ((file->bufferPos >= file->bufferFill) && file->eofKnown); +} + +int fileClose(UnixFile *file, int *returnCode, int *reasonCode){ + FILE *internalFile = file->internalFile; + + int status = fclose(internalFile); + if (status != 0){ + *returnCode = errno; + *reasonCode = errno; + } + + if (file->pathname != NULL) { + safeFree(file->pathname, strlen(file->pathname) + 1); + file->pathname = NULL; + } + if (file->buffer != NULL) { + safeFree(file->buffer, file->bufferSize); + file->buffer = NULL; + } + safeFree((char *)file, sizeof(UnixFile)); + file = NULL; + + return status; +} + +int fileInfoIsDirectory(const FileInfo *info){ + return info->st_mode & _S_IFDIR; +} + +int64 fileInfoSize(const FileInfo *info){ + return info->st_size; +} + +int fileInfoCCSID(const FileInfo *info){ + printf("***WARNING*** guessing about File's code page in Windows\n"); + return CP_UTF8; +} + +int fileInfoUnixCreationTime(const FileInfo *info){ + return info->st_ctime; +} + +UnixFile *directoryOpen(const char *directoryName, int *returnCode, int *reasonCode){ + printf("d0.0\n"); + fflush(stdout); + + int nameLength = strlen(directoryName); + WIN32_FIND_DATA findData; + int directoryNameBufferSize = nameLength + 10; + char *directoryNameBuffer = safeMalloc(directoryNameBufferSize,"Windows Directory Name"); + HANDLE hFind = INVALID_HANDLE_VALUE; + + int len = sprintf(directoryNameBuffer,"%s\\*",directoryName); + hFind = FindFirstFile(directoryNameBuffer, &findData); + + safeFree(directoryNameBuffer, directoryNameBufferSize); + directoryNameBuffer = NULL; + + printf("dO.1 hFind=0x%p\n",hFind); + fflush(stdout); + if (hFind == INVALID_HANDLE_VALUE){ + *returnCode = GetLastError(); + *reasonCode = *returnCode; + return NULL; + } else { + UnixFile *directory = (UnixFile*)safeMalloc(sizeof(UnixFile),"Windows Directory"); + memset(directory,0,sizeof(UnixFile)); + directory->internalFile = NULL; + directory->fd = 0; + directory->hFind = hFind; + memcpy(&(directory->findData),&findData,sizeof(WIN32_FIND_DATA)); + directory->pathname = safeMalloc(strlen(directoryName)+1,"Unix File Name"); + strcpy(directory->pathname, directoryName); + directory->isDirectory = TRUE; + directory->hasMoreEntries = TRUE; + + *returnCode = 0; + *reasonCode = 0; + return directory; + } +} + +int directoryRead(UnixFile *directory, char *entryBuffer, int entryBufferLength, int *returnCode, int *reasonCode){ + HANDLE hFind = directory->hFind; + WIN32_FIND_DATA *findData = &(directory->findData); + + if (directory->hasMoreEntries){ + int filenameLength = strlen(findData->cFileName); + printf("found %s\n",findData->cFileName); + if (FindNextFile(hFind, findData)){ + directory->hasMoreEntries = TRUE; + } else { + /* find loop is done */ + directory->hasMoreEntries = FALSE; + } + return 1; + } else { + return 0; + } +} + +int directoryClose(UnixFile *directory, int *returnCode, int *reasonCode){ + HANDLE hFind = directory->hFind; + + FindClose(hFind); + if (directory->pathname != NULL) { + safeFree(directory->pathname, strlen(directory->pathname)+1); + directory->pathname = NULL; + } + safeFree((char*)directory,sizeof(UnixFile)); + *returnCode = 0; + *reasonCode = 0; + return 0; +} + +SocketSet *makeSocketSet(int highestAllowedSD){ + SocketSet *set = (SocketSet*)safeMalloc(sizeof(SocketSet),"SocketSet"); + memset(set,0,sizeof(SocketSet)); + + set->highestAllowedSD = highestAllowedSD; + + int socketArrayLength = highestAllowedSD; + set->sockets = (Socket**)safeMalloc(socketArrayLength*sizeof(Socket*),"SocketSetSocketArray"); + memset(set->sockets,0,socketArrayLength*sizeof(Socket*)); + + set->revisionNumber = 0; + return set; +} + +void freeSocketSet(SocketSet *set) { + int socketArrayLength = set->highestAllowedSD; + safeFree((char*)(void*)set->sockets, socketArrayLength * sizeof(Socket*)); + safeFree((char*)set, sizeof(SocketSet)); + return; +} + +int socketSetAdd(SocketSet *set, Socket *socket){ + printf("winskt.socketSetAdd set=0x%p socket=0x%p\n",set,socket); + fflush(stdout); + SOCKET windowsSocket = socket->windowsSocket; + if (socketTrace) + { + printf ("Adding socket, SOCKET HANDLE=0x%llx\n", windowsSocket); + } + + if (set->socketCount >= set->highestAllowedSD){ + printf("Socket set is full at %d sockets\n",set->highestAllowedSD); + return 12; + } + + set->sockets[set->socketCount] = socket; + set->socketCount++; + set->revisionNumber++; + + return 0; +} + +int socketSetRemove(SocketSet *set, Socket *socket){ + SOCKET windowsSocket = socket->windowsSocket; + if (socketTrace) + { + printf ("Removing socket, SOCKET HANDLE=0x%llx\n", windowsSocket); + } + + set->socketCount--; + set->sockets[set->socketCount] = NULL; + set->revisionNumber++; + return 0; +} diff --git a/c/xlate.c b/c/xlate.c index 91d4b59ef..6c4b2d8f7 100644 --- a/c/xlate.c +++ b/c/xlate.c @@ -17,6 +17,7 @@ #else #include #include +#include #endif #include "xlate.h" diff --git a/h/json.h b/h/json.h index e990501b6..c7f5e4a45 100644 --- a/h/json.h +++ b/h/json.h @@ -16,8 +16,10 @@ #ifdef METTLE # include # include +# include #else # include +# include #endif /* METTLE */ #include "zowetypes.h" #include "utils.h" @@ -346,10 +348,18 @@ struct Json_tag { #define JSON_TYPE_NULL 3 #define JSON_TYPE_OBJECT 4 #define JSON_TYPE_ARRAY 5 + /* V2 parsing can yield the following additional types */ +#define JSON_TYPE_INT64 6 +#define JSON_TYPE_DOUBLE 7 #define JSON_TYPE_ERROR 666 int type; union { int number; + /* There two fields are being introduced to support 'proper' parsing of + numbers. Up until 2022, this json library was not very friendly to floating + point and integers with a >32-bit mantissa. */ + int64_t integerValue; + double floatValue; char *string; int boolean; JsonObject *object; @@ -382,6 +392,8 @@ struct JsonError_tag { Json *jsonParseString(ShortLivedHeap *slh, char *jsonString, char* errorBufferOrNull, int errorBufferSize); Json *jsonParseUnterminatedString(ShortLivedHeap *slh, char *jsonString, int len, char* errorBufferOrNull, int errorBufferSize); Json *jsonParseFile(ShortLivedHeap *slh, const char *filename , char* errorBufferOrNull, int errorBufferSize); +/* parseFile2 supports full-width integers and floating-point numbers */ +Json *jsonParseFile2(ShortLivedHeap *slh, const char *filename, char* errorBufferOrNull, int errorBufferSize); Json *jsonParseUnterminatedUtf8String(ShortLivedHeap *slh, int outputCCSID, char *jsonUtf8String, int len, char *errorBufferOrNull, int errorBufferSize); @@ -395,6 +407,8 @@ void jsonPrintArray(jsonPrinter* printer, JsonArray *array); int jsonAsBoolean(Json *json); int jsonAsNumber(Json *json); +int64_t jsonAsInt64(Json *json); +double jsonAsDouble(Json *json); char *jsonAsString(Json *json); JsonArray *jsonAsArray(Json *json); JsonObject *jsonAsObject(Json *json); @@ -405,9 +419,16 @@ int jsonIsArray(Json *json); int jsonIsBoolean(Json *json); int jsonIsNull(Json *json); int jsonIsNumber(Json *json); +/* V2 more specific number type */ +int jsonIsDouble(Json *json); +int jsonIsInt64(Json *json); int jsonIsObject(Json *json); int jsonIsString(Json *json); +/************** High Uniqueness Hash ******************/ + +int64_t jsonLongHash(Json *json); + /************** Array Accessors ***********************/ int jsonArrayGetCount(JsonArray *array); @@ -517,6 +538,78 @@ char *jsonStringProperty(JsonObject *object, char *propertyName, int *stat JsonObject *jsonObjectProperty(JsonObject *object, char *propertyName, int *status); void reportJSONDataProblem(void *jsonObject, int status, char *propertyName); +/* JSON Builder interface. This Interface allows the translation to JSON + form other formats, without going through json syntax parsing. +*/ + +typedef struct JsonParser_tag JsonParser; +typedef struct JsonTokenizer_tag JsonTokenizer; +typedef struct JsonToken_tag JsonToken; + +#define JSON_PARSE_VERSION_1 1 /* incorrect numbers (only 32-bit integers) */ +#define JSON_PARSE_VERSION_2 2 /* numbers parse to int64 or a double */ + +struct JsonParser_tag { + JsonTokenizer *tokenizer; + JsonToken *unreadToken; + ShortLivedHeap *slh; + CharStream *in; + Json *jsonError; + int version; +}; + +typedef struct JsonBuilder_tag { + JsonParser parser; + Json *root; +} JsonBuilder; + +JsonBuilder *makeJsonBuilder(ShortLivedHeap *slh); + +#define JSON_BUILD_FAIL_PARENT_IS_SCALAR 12 +#define JSON_BUILD_FAIL_KEY_ON_ARRAY 16 +#define JSON_BUILD_FAIL_KEY_ON_ROOT 20 +#define JSON_BUILD_FAIL_NO_KEY_ON_OBJECT 24 + +void freeJsonBuilder(JsonBuilder *builder, bool freeSLH); + +Json *jsonBuildObject(JsonBuilder *b, + Json *parent, + char *parentKey, + int *errorCode); + +Json *jsonBuildArray(JsonBuilder *b, + Json *parent, + char *parentKey, + int *errorCode); + +Json *jsonBuildString(JsonBuilder *b, + Json *parent, + char *parentKey, + char *s, + int sLen, + int *errorCode); + +Json *jsonBuildInt(JsonBuilder *b, + Json *parent, + char *parentKey, + int i, + int *errorCode); + +Json *jsonBuildBool(JsonBuilder *b, + Json *parent, + char *parentKey, + bool *truthValue, + int *errorCode); + +Json *jsonBuildNull(JsonBuilder *b, + Json *parent, + char *parentKey, + int *errorCode); + + + + + #endif /* __JSON__ */ diff --git a/h/jsonschema.h b/h/jsonschema.h new file mode 100644 index 000000000..3d3a5223f --- /dev/null +++ b/h/jsonschema.h @@ -0,0 +1,69 @@ +#ifndef __ZOWE_JSONSCHEMA__ +#define __ZOWE_JSONSCHEMA__ 1 + +#ifdef METTLE + +#else +#include +#endif + +typedef struct JsonSchema_tag { + ShortLivedHeap *slh; + int version; + struct JSValueSpec_tag *topValueSpec; +} JsonSchema; + +#define MAX_SCHEMA_DEPTH 100 + +typedef struct AccessPath_tag { + struct { + union { + char *name; + int index; + } key; + bool isName; + } elements[MAX_SCHEMA_DEPTH]; + int currentSize; +} AccessPath; + +typedef struct JsonSchemaBuilder_tag { + JsonSchema *schema; + ShortLivedHeap *slh; + int version; + AccessPath *accessPath; + int errorCode; + char *errorMessage; + int errorMessageLength; +#ifdef METTLE +#error "need setjmp" +#else + jmp_buf recoveryData; +#endif +} JsonSchemaBuilder; + +#define MAX_ACCESS_PATH 1024 + +typedef struct JsonValidator_tag { + JsonSchema *schema; + int errorCode; + char *errorMessage; + int errorMessageLength; + char accessPathBuffer[MAX_ACCESS_PATH]; + AccessPath *accessPath; + jmp_buf recoveryData; +} JsonValidator; + +#define JSON_SCHEMA_DRAFT_4 400 +#define DEFAULT_JSON_SCHEMA_VERSION JSON_SCHEMA_DRAFT_4 + + +JsonSchemaBuilder *makeJsonSchemaBuilder(int version); +void freeJsonSchemaBuilder(JsonSchemaBuilder *builder); +JsonSchema *jsonBuildSchema(JsonSchemaBuilder *builder, Json *jsValue); + +JsonValidator *makeJsonValidator(); +void freeJsonValidator(JsonValidator *validator); +int jsonValidateSchema(JsonValidator *validator, Json *value, JsonSchema *schema); + + +#endif diff --git a/h/zowetypes.h b/h/zowetypes.h index f4532bc82..d98bdac4a 100644 --- a/h/zowetypes.h +++ b/h/zowetypes.h @@ -123,6 +123,10 @@ no ifdef means XLC LE on ZOS, and everything else. This is effectively our "def /* Base macros for Windows */ #ifdef _MSC_VER +#define ZOWE_PRAGMA_PACK +#define ZOWE_PRAGMA_PACK_RESET + + #define __ZOWE_OS_WINDOWS 1 #ifdef _WIN64 @@ -133,6 +137,8 @@ no ifdef means XLC LE on ZOS, and everything else. This is effectively our "def #endif /* _MSC_VER */ + + /* Base Macros for zOS */ #if defined(__MVS__) && (defined (__IBMC__) || defined (__IBMCPP__)) && (defined (_LP64) || defined (_ILP32)) From 4014385dc02edc2c3edf3f77463d4895900d3ff9 Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 1 Feb 2022 10:58:30 -0500 Subject: [PATCH 03/42] More verifiers for JSON Schema including regex Signed-off-by: Joe --- c/configmgr.c | 133 ++++++++++++++++++ c/json.c | 21 ++- c/jsonschema.c | 251 +++++++++++++++++++++++----------- h/json.h | 8 ++ h/jsonschema.h | 18 +++ platform/README.md | 15 ++ platform/windows/cppregex.cpp | 36 +++++ platform/windows/cppregex.h | 17 +++ platform/windows/winregex.cpp | 65 +++++++++ platform/windows/winregex.h | 97 +++++++++++++ 10 files changed, 579 insertions(+), 82 deletions(-) create mode 100644 c/configmgr.c create mode 100644 platform/README.md create mode 100644 platform/windows/cppregex.cpp create mode 100644 platform/windows/cppregex.h create mode 100644 platform/windows/winregex.cpp create mode 100644 platform/windows/winregex.h diff --git a/c/configmgr.c b/c/configmgr.c new file mode 100644 index 000000000..d38fa8049 --- /dev/null +++ b/c/configmgr.c @@ -0,0 +1,133 @@ + + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#ifdef METTLE + +#include +#include +#include +#include +#include +#include +#include +#include +#include "metalio.h" +#include "qsam.h" + +#else + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 600 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include /* JOE 1/20/22 */ +#include /* JOE 1/20/22 */ +#include /* JOE 1/20/22 */ +#include + +#endif + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "openprims.h" +#include "bpxnet.h" +#include "unixfile.h" +#include "json.h" +#include "charsets.h" + +#ifdef __ZOWE_OS_WINDOWS +typedef int64_t ssize_t; +#endif + +/* A configuration manager is a facility providing access to a set of configuration source + (most likely textual) in many formats in one or more locations whose data definition is + provided by a set of JSON Schema Definions. + + Formats + YAML + JSON + PARMLIB + Environment Variables + + Paths + Configs and Schemas Live on paths of higher-to-lower precedence directories and libraries + (PDS/PDSE's). + + + References + + http://json-schema.org + + https://datatracker.ietf.org/doc/html/draft-bhutton-json-schema-validation) + +*/ + +#define CONFIG_PATH_OMVS_DIR 0x0001 +#define CONFIG_PATH_MVS_LIBRAY 0x0002 + +typedef struct ConfigPathElement_tag { + int flags; + char *name; + struct ConfigPathElement_tag *next; +} ConfigPathElement; + +typedef struct ConfigManager_tag { + ShortLivedHeap *slh; + ConfigPathElement *schemaPath; + configPathElement *configPath; + hashtable *schemaCache; + hashtable *configCache; +} ConfigManager; + +#define ZCFG_SUCCESS 0 + +/* Merging notes + defaulting and merging requires the same data shape at all levels and sources +*/ + +/* all calls return status */ + +/* zowe.setup.mvs.proclib */ + +int cfgGetInt(ConfigManager *mgr, int *value, ...){ /* path elements which are strings and ints for array indexing */ + +} + +int cfgGetInt64(ConfigManager *mgr, int64 *value, ...){ + +} + +#define ZCFG_ALLOC_HEAP 1 +#define ZCFG_ALLOC_SLH 2 +#define ZCFG_ALLOC_MEMCPY 3 + +int cfgGetString(ConfigManager *mgr, char *value, int allocOptions, void *mem, ...){ + return ZCFG_SUCCESS; +} + +int cfgGetInstanceType(ConfigManager *mgr, int *type, int *subType, ...){ + +} + + + + + + diff --git a/c/json.c b/c/json.c index 86422f5cd..eba3c1901 100644 --- a/c/json.c +++ b/c/json.c @@ -1963,7 +1963,6 @@ Json *jsonBuildString(JsonBuilder *b, *errorCode = lvalueStatus; return NULL; } - } Json *jsonBuildInt(JsonBuilder *b, @@ -1987,6 +1986,26 @@ Json *jsonBuildInt(JsonBuilder *b, } +Json *jsonBuildInt64(JsonBuilder *b, + Json *parent, + char *parentKey, + int64 i, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_INT64; + json->data.integerValue = i; + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + } else { + *errorCode = lvalueStatus; + return NULL; + } +} + Json *jsonBuildBool(JsonBuilder *b, Json *parent, char *parentKey, diff --git a/c/jsonschema.c b/c/jsonschema.c index 93e845307..7998a12c6 100644 --- a/c/jsonschema.c +++ b/c/jsonschema.c @@ -44,13 +44,6 @@ #endif -/* -#ifdef __ZOWE_OS_WINDOWS -#include "winregex.h" -#else -#include "regex.h" -#endif -*/ #include "zowetypes.h" #include "alloc.h" @@ -61,6 +54,13 @@ #include "json.h" #include "charsets.h" #include "collections.h" + +#ifdef __ZOWE_OS_WINDOWS +#include "winregex.h" +#else +#include "psxregex.h" /* works on ZOS and Linux */ +#endif + #include "jsonschema.h" #ifdef __ZOWE_OS_WINDOWS @@ -136,7 +136,8 @@ typedef struct JSValueSpec_tag { int64_t minLength; int64_t maxLength; char *pattern; - /* regex_t *compiledPattern; */ + regex_t *compiledPattern; + int regexCompilationError; } JSValueSpec; typedef struct AccessPathUnion_tag { @@ -194,6 +195,7 @@ static void accessPathPop(AccessPath *accessPath){ } + #define ERROR_MAX 1024 static void validationThrow(JsonValidator *validator, int errorCode, char *formatString, ...){ @@ -208,6 +210,24 @@ static void validationThrow(JsonValidator *validator, int errorCode, char *forma longjmp(validator->recoveryData,1); } +static ValidityException *noteValidityException(JsonValidator *validator, int invalidityCode, char *formatString, ...){ + va_list argPointer; + ValidityException *exception = (ValidityException*)safeMalloc(sizeof(ValidityException),"ValidityException"); + exception->code = invalidityCode; + exception->next = NULL; + va_start(argPointer,formatString); + vsnprintf(exception->message,MAX_VALIDITY_EXCEPTION_MSG,formatString,argPointer); + va_end(argPointer); + if (validator->firstValidityException == NULL){ + validator->firstValidityException = exception; + validator->lastValidityException = exception; + } else { + validator->lastValidityException->next = exception; + validator->lastValidityException = exception; + } + return exception; +} + JsonValidator *makeJsonValidator(){ JsonValidator *validator = (JsonValidator*)safeMalloc(sizeof(JsonValidator),"JsonValidator"); memset(validator,0,sizeof(JsonValidator)); @@ -225,20 +245,26 @@ static char *validatorAccessPath(JsonValidator *validator){ return makeAccessPathString(validator->accessPathBuffer,MAX_ACCESS_PATH,validator->accessPath); } -static void validateType(JsonValidator *validator, +/* The return types of these validators is bool to allow validator to signal whether to continue + to gather more validation exceptions in this part of the JSON tree/graph. Returning false says + this part is invalid enough such that further evaluation would probably confuse the user with + contradictory information. */ + +static bool validateType(JsonValidator *validator, int typeCode, JSValueSpec *valueSpec){ - printf("validateType.1 typeCode=%d vspecMask=0x%x\n",typeCode,valueSpec->typeMask);fflush(stdout); if (((1 << typeCode) & valueSpec->typeMask) == 0){ - validationThrow(validator,12,"type %d not permitted at %s",typeCode,validatorAccessPath(validator)); + noteValidityException(validator,12,"type %d not permitted at %s",typeCode,validatorAccessPath(validator)); + return false; + } else { + return true; } - printf("validateType.2\n");fflush(stdout); } /* for mutual recursion */ -static void validateJSON(JsonValidator *validator, Json *value, JSValueSpec *valueSpec); +static bool validateJSON(JsonValidator *validator, Json *value, JSValueSpec *valueSpec); -static void validateJSONObject(JsonValidator *validator, +static bool validateJSONObject(JsonValidator *validator, JsonObject *object, JSValueSpec *valueSpec){ AccessPath *accessPath = validator->accessPath; @@ -248,10 +274,9 @@ static void validateJSONObject(JsonValidator *validator, if (valueSpec->required){ for (int r=0; rrequiredCount; r++){ char *requiredProperty = valueSpec->required[r]; - printf("VJO requiredProp=%s\n",requiredProperty);fflush(stdout); if (jsonObjectGetPropertyValue(object,requiredProperty) == NULL){ - validationThrow(validator,12,"missing required property %s at %s", - requiredProperty,validatorAccessPath(validator)); + noteValidityException(validator,12,"missing required property %s at %s", + requiredProperty,validatorAccessPath(validator)); } } } @@ -272,9 +297,11 @@ static void validateJSONObject(JsonValidator *validator, if (propertySpec != NULL){ validateJSON(validator,propertyValue,propertySpec); } else { - printf("*WARNING* unspecified property seen, '%s', and checking code is not complete, vspec->props=0x%p\n", - propertyName,valueSpec->properties); - fflush(stdout); + if (validator->flags && VALIDATOR_WARN_ON_UNDEFINED_PROPERTIES){ + printf("*WARNING* unspecified property seen, '%s', and checking code is not complete, vspec->props=0x%p\n", + propertyName,valueSpec->properties); + fflush(stdout); + } if (valueSpec->properties){ /* htDump(valueSpec->properties); */ } @@ -292,21 +319,21 @@ static void validateJSONObject(JsonValidator *validator, if (valueSpec->validatorFlags & JS_VALIDATOR_MAX_PROPS){ int64_t lim = valueSpec->minProperties; if (propertyCount > lim){ - validationThrow(validator,12,"too many properties, %d > MAX=%d at %s", + noteValidityException(validator,12,"too many properties, %d > MAX=%d at %s", propertyCount,lim,validatorAccessPath(validator)); } } if (valueSpec->validatorFlags & JS_VALIDATOR_MIN_PROPS){ int64_t lim = valueSpec->minProperties; if (propertyCount < lim){ - validationThrow(validator,12,"too few properties, %d < MIN=%d at %s", - propertyCount,lim,validatorAccessPath(validator)); + noteValidityException(validator,12,"too few properties, %d < MIN=%d at %s", + propertyCount,lim,validatorAccessPath(validator)); } } - + return true; } -void validateJSONArray(JsonValidator *validator, +bool validateJSONArray(JsonValidator *validator, JsonArray *array, JSValueSpec *valueSpec){ AccessPath *accessPath = validator->accessPath; @@ -317,17 +344,17 @@ void validateJSONArray(JsonValidator *validator, if (valueSpec->validatorFlags & JS_VALIDATOR_MAX_ITEMS){ int64_t lim = valueSpec->maxItems; if (elementCount > lim){ - validationThrow(validator,12, - "too many array items, %d > MAX=%d at %s", - elementCount,lim,validatorAccessPath(validator)); + noteValidityException(validator,12, + "too many array items, %d > MAX=%d at %s", + elementCount,lim,validatorAccessPath(validator)); } } if (valueSpec->validatorFlags & JS_VALIDATOR_MIN_ITEMS){ int64_t lim = valueSpec->minItems; if (elementCount < lim){ - validationThrow(validator,12, - "too few array ites, %d < MIN=%d at %s", - elementCount,lim,validatorAccessPath(validator)); + noteValidityException(validator,12, + "too few array ites, %d < MIN=%d at %s", + elementCount,lim,validatorAccessPath(validator)); } } /* 'uniqueItems' is poorly specified regarding equality predicates. @@ -349,15 +376,16 @@ void validateJSONArray(JsonValidator *validator, if (valueSpec->uniqueItems){ long longHash = jsonLongHash(itemValue); if (lhtGet(uniquenessSet,longHash) != NULL){ - validationThrow(validator,12,"array uniqueItems violation %s is duplicate at %s", - itemValue,i,validatorAccessPath(validator)); + noteValidityException(validator,12,"array uniqueItems violation %s is duplicate at %s", + itemValue,i,validatorAccessPath(validator)); } } accessPathPop(accessPath); } + return true; } -static void validateJSONString(JsonValidator *validator, +static bool validateJSONString(JsonValidator *validator, Json *string, JSValueSpec *valueSpec){ char *s = jsonAsString(string); @@ -365,23 +393,45 @@ static void validateJSONString(JsonValidator *validator, if (valueSpec->validatorFlags & JS_VALIDATOR_MAX_LENGTH){ long lim = valueSpec->maxLength; if (len > lim){ - validationThrow(validator,12,"string too long, %d > MAX=%d at %s", - len,lim,validatorAccessPath(validator)); + noteValidityException(validator,12,"string too long, %d > MAX=%d at %s", + len,lim,validatorAccessPath(validator)); } } if (valueSpec->validatorFlags & JS_VALIDATOR_MIN_LENGTH){ long lim = valueSpec->minLength; if (len < lim){ - validationThrow(validator,12,"string too short, %d < MAX=%d at %s", - len,lim,validatorAccessPath(validator)); + noteValidityException(validator,12,"string too short, %d < MAX=%d at %s", + len,lim,validatorAccessPath(validator)); + } + } + if (valueSpec->pattern && (valueSpec->regexCompilationError == 0)){ + if (valueSpec->compiledPattern == NULL){ + regex_t *rx = regexAlloc(); + int compStatus = regexComp(rx,valueSpec->pattern,REG_EXTENDED); + if (compStatus){ + regexFree(rx); + valueSpec->regexCompilationError = compStatus; + printf("*** WARNING *** pattern '%s' failed to compile with status %d\n", + valueSpec->pattern,compStatus); + } else { + valueSpec->compiledPattern = rx; + } + } + if (valueSpec->compiledPattern){ + int regexStatus = regexExec(valueSpec->compiledPattern,s,0,NULL,0); + if (regexStatus != 0){ + noteValidityException(validator,12,"string pattern match fail s='%s', pat='%s', at %s", + s,valueSpec->pattern,validatorAccessPath(validator)); + } } } // String pattern; // ECMA-262 regex // prefix items - // additionalProperties + // additionalProperties + return true; } -static void validateJSONNumber(JsonValidator *validator, +static bool validateJSONNumber(JsonValidator *validator, Json *doubleOrLegacyInt, JSValueSpec *valueSpec){ // Number multipleOf; // can be a filled, and must be > 0 @@ -393,69 +443,73 @@ static void validateJSONNumber(JsonValidator *validator, if (valueSpec->validatorFlags & JS_VALIDATOR_MAX){ double lim = valueSpec->maximum; if (valueSpec->exclusiveMaximum ? (d >= lim) : (d > lim)){ - validationThrow(validator,12,"value too large, %f %s MAX=%f at %s", - d, - (valueSpec->exclusiveMaximum ? ">=" : ">"), - lim, - validatorAccessPath(validator)); + noteValidityException(validator,12,"value too large, %f %s MAX=%f at %s", + d, + (valueSpec->exclusiveMaximum ? ">=" : ">"), + lim, + validatorAccessPath(validator)); } } if (valueSpec->validatorFlags & JS_VALIDATOR_MIN){ double lim = valueSpec->minimum; if (valueSpec->exclusiveMinimum ? (d <= lim) : (d < lim)){ - validationThrow(validator,12,"value too small, %f %s MAX=%f at %s", - d, - (valueSpec->exclusiveMinimum ? "<=" : "<"), - lim, - validatorAccessPath(validator)); + noteValidityException(validator,12,"value too small, %f %s MAX=%f at %s", + d, + (valueSpec->exclusiveMinimum ? "<=" : "<"), + lim, + validatorAccessPath(validator)); } } + return true; } -static void validateJSONInteger(JsonValidator *validator, +static bool validateJSONInteger(JsonValidator *validator, Json *value64, JSValueSpec *valueSpec){ // Number multipleOf; // can be a filled, and must be > 0 // Number exclusiveMaximum; // Number exclusiveMinimum; int64_t i = jsonAsInt64(value64); + printf("JOE i=%lld valFlags=0x%llx\n",i,valueSpec->validatorFlags); if (valueSpec->validatorFlags & JS_VALIDATOR_MAX){ int64_t lim = (int64_t)valueSpec->maximum; if (valueSpec->exclusiveMaximum ? (i >= lim) : (i > lim)){ - validationThrow(validator,12,"value too large, %lld %s MAX=%lld at %s", - i, - (valueSpec->exclusiveMaximum ? ">=" : ">"), - lim, - validatorAccessPath(validator)); + noteValidityException(validator,12,"value too large, %lld %s MAX=%lld at %s", + i, + (valueSpec->exclusiveMaximum ? ">=" : ">"), + lim, + validatorAccessPath(validator)); } } if (valueSpec->validatorFlags & JS_VALIDATOR_MIN){ int64_t lim = (int64_t)valueSpec->minimum; if (valueSpec->exclusiveMinimum ? (i <= lim) : (i < lim)){ - validationThrow(validator,12,"value too small, %lld %s MAX=%lld at %s", - i, - (valueSpec->exclusiveMinimum ? "<=" : "<"), - lim, - validatorAccessPath(validator)); + noteValidityException(validator,12,"value too small, %lld %s MAX=%lld at %s", + i, + (valueSpec->exclusiveMinimum ? "<=" : "<"), + lim, + validatorAccessPath(validator)); } } + return true; } static JSValueSpec *resolveRef(JsonValidator *validator, JSValueSpec *valueSpec){ char *ref = valueSpec->ref; + printf("resolveRef ref=%s\n",valueSpec->ref); if (!memcmp(ref,"#/$defs/",8)){ /* local resolution */ int refLen = strlen(ref); int pos = 2; char *key = ref+8; JSValueSpec *topSchema = validator->schema->topValueSpec; if (topSchema->definitions == NULL){ - validationThrow(validator,12,"schema '%s' does not define shared '$defs'", - (topSchema->id ? topSchema->id : "")); + noteValidityException(validator,12,"schema '%s' does not define shared '$defs'", + (topSchema->id ? topSchema->id : "")); return NULL; /* for the compiler, bless its heart */ } JSValueSpec *resolvedSchema = htGet(topSchema->definitions,key); if (resolvedSchema == NULL){ - validationThrow(validator,12,"schema ref '%s' does not resolve against '$defs'",ref); + noteValidityException(validator,12,"schema ref '%s' does not resolve against '$defs'",ref); return NULL; } return resolvedSchema; @@ -464,43 +518,48 @@ static JSValueSpec *resolveRef(JsonValidator *validator, JSValueSpec *valueSpec) need to merge URL according W3 rules and pass to pluggable resolver that we hope this validator has. */ - validationThrow(validator,12,"external refs not yet implemented",ref); + noteValidityException(validator,12,"external refs not yet implemented '%s' at %s", + valueSpec->ref,validatorAccessPath(validator)); return NULL; } } -static void validateJSON(JsonValidator *validator, Json *value, JSValueSpec *valueSpec){ +static bool validateJSON(JsonValidator *validator, Json *value, JSValueSpec *valueSpec){ while (valueSpec->ref){ valueSpec = resolveRef(validator,valueSpec); + if (valueSpec == NULL){ + return false; + } } printf("validate JSON value->type=%d\n",value->type); fflush(stdout); // type is string or array, does this mean to collapse ValueSpecTypes // should we flag errors on out-of-bounds validation keywords for type keyword if (jsonIsNull(value)){ - validateType(validator,JSTYPE_NULL,valueSpec); + return validateType(validator,JSTYPE_NULL,valueSpec); } else if (jsonIsBoolean(value)){ - validateType(validator,JSTYPE_BOOLEAN,valueSpec); + return validateType(validator,JSTYPE_BOOLEAN,valueSpec); } else if (jsonIsObject(value)){ - validateType(validator,JSTYPE_OBJECT,valueSpec); - validateJSONObject(validator,jsonAsObject(value),valueSpec); + return (validateType(validator,JSTYPE_OBJECT,valueSpec) && + validateJSONObject(validator,jsonAsObject(value),valueSpec)); } else if (jsonIsArray(value)){ - validateType(validator,JSTYPE_ARRAY,valueSpec); - validateJSONArray(validator,jsonAsArray(value),valueSpec); + return (validateType(validator,JSTYPE_ARRAY,valueSpec) && + validateJSONArray(validator,jsonAsArray(value),valueSpec)); } else if (jsonIsString(value)){ - validateType(validator,JSTYPE_STRING,valueSpec); - validateJSONString(validator,value,valueSpec); + return (validateType(validator,JSTYPE_STRING,valueSpec) && + validateJSONString(validator,value,valueSpec)); } else if (jsonIsNumber(value)){ if (!jsonIsInt64(value)){ - validateType(validator,JSTYPE_NUMBER,valueSpec); - validateJSONInteger(validator,value,valueSpec); /* general, comparisons done as doubles */ + return (validateType(validator,JSTYPE_NUMBER,valueSpec) && + validateJSONNumber(validator,value,valueSpec)); /* general, comparisons done as doubles */ } else { - validateType(validator,JSTYPE_INTEGER,valueSpec); - validateJSONNumber(validator,value,valueSpec); + return (validateType(validator,JSTYPE_INTEGER,valueSpec) && + validateJSONInteger(validator,value,valueSpec)); } } else { printf("*** PANIC *** unhandled JS Value with type = %d\n",value->type); + return false; } } @@ -508,13 +567,17 @@ int jsonValidateSchema(JsonValidator *validator, Json *value, JsonSchema *schema if (setjmp(validator->recoveryData) == 0) { /* normal execution */ validator->schema = schema; validateJSON(validator,value,schema->topValueSpec); - printf("after validate without throw\n"); + printf("after validate without throw, should show validation exceptions\n"); fflush(stdout); - return 0; + if (validator->firstValidityException == NULL){ + return JSON_VALIDATOR_NO_EXCEPTIONS; + } else { + return JSON_VALIDATOR_HAS_EXCEPTIONS; + } } else { printf("validation failed '%s'\n",validator->errorMessage); fflush(stdout); - return validator->errorCode; + return JSON_VALIDATOR_INTERNAL_FAILURE; } } @@ -819,6 +882,20 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopL JSValueSpec *valueSpec = makeValueSpec(builder,getID(builder,object),description,title,typeArray,typeArrayCount); valueSpec->ref = getString(builder,object,"$ref",NULL); + if (valueSpec->ref){ + /* THe JSON Schema spec says that if a $ref is defined it will be the only thing looked at, so + we short-cicuit here. + + "In Draft 4-7, $ref behaves a little differently. When an object + contains a $ref property, the object is considered a reference, + not a schema. Therefore, any other properties you put in that + object will not be treated as JSON Schema keywords and will be + ignored by the validator. $ref can only be used where a schema + is expected." + + */ + return valueSpec; + } valueSpec->definitions = getDefinitions(builder,object); valueSpec->allOf = getComposite(builder,object,"allOf"); valueSpec->anyOf = getComposite(builder,object,"anyOf"); @@ -830,8 +907,10 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopL accessPathPop(accessPath); } /// valueSpec.not = getComposite(object,accessPath); + printf("valueSpec typeMask=0x%x\n",valueSpec->typeMask); for (int typeCode=0; typeCode<=JSTYPE_MAX; typeCode++){ if ((1 << typeCode) & valueSpec->typeMask){ + printf("switch typeCode = %d\n",typeCode); switch (typeCode){ case JSTYPE_OBJECT: { @@ -949,12 +1028,21 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopL { JSValueSpec *numericSpec = valueSpec; // new JSNumericSpec(description,title,(jsType == JSType.INTEGER)); + printf("JOE JSTYPE_INT/NUM: "); + printAccessPath(stdout,accessPath); numericSpec->isInteger = (typeCode == JSTYPE_INTEGER); numericSpec->multipleOf = getNumber(builder,object,"multipleOf",MISSING_FLOAT_VALIDATOR); numericSpec->maximum = getNumber(builder,object,"maximum",MISSING_FLOAT_VALIDATOR); numericSpec->exclusiveMaximum = getBooleanValue(builder,object,"exclusiveMaximum",false); numericSpec->minimum = getNumber(builder,object,"minimum",MISSING_FLOAT_VALIDATOR); + printf("max = %f min=%f\n",numericSpec->maximum,numericSpec->minimum); numericSpec->exclusiveMinimum = getBooleanValue(builder,object,"exclusiveMinimum",false); + if (numericSpec->maximum != MISSING_FLOAT_VALIDATOR){ + numericSpec->validatorFlags |= JS_VALIDATOR_MAX; + } + if (numericSpec->minimum != MISSING_FLOAT_VALIDATOR){ + numericSpec->validatorFlags |= JS_VALIDATOR_MIN; + } if (numericSpec->multipleOf != MISSING_FLOAT_VALIDATOR){ checkPositiveDouble(builder,numericSpec->multipleOf,"multipleOf"); numericSpec->validatorFlags |= JS_VALIDATOR_MULTOF; @@ -967,6 +1055,7 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopL } }// for (jsType... + printf("JOE end schema for loop\n"); return valueSpec; } else { schemaThrow(builder,12,"top level schema must be an object"); diff --git a/h/json.h b/h/json.h index c7f5e4a45..8d069bf99 100644 --- a/h/json.h +++ b/h/json.h @@ -595,6 +595,14 @@ Json *jsonBuildInt(JsonBuilder *b, int i, int *errorCode); +/* This builds a V2 compatible integer, jsonBuiltInt does the legacy 32 bit integer */ +Json *jsonBuildInt64(JsonBuilder *b, + Json *parent, + char *parentKey, + int64 i, + int *errorCode); + + Json *jsonBuildBool(JsonBuilder *b, Json *parent, char *parentKey, diff --git a/h/jsonschema.h b/h/jsonschema.h index 3d3a5223f..f2168f240 100644 --- a/h/jsonschema.h +++ b/h/jsonschema.h @@ -43,12 +43,25 @@ typedef struct JsonSchemaBuilder_tag { #define MAX_ACCESS_PATH 1024 +#define MAX_VALIDITY_EXCEPTION_MSG 1024 + +typedef struct ValidityException_tag { + int code; + struct ValidityException_tag *next; + char message[MAX_VALIDITY_EXCEPTION_MSG]; +} ValidityException; + +#define VALIDATOR_WARN_ON_UNDEFINED_PROPERTIES 0x0001 + typedef struct JsonValidator_tag { JsonSchema *schema; int errorCode; char *errorMessage; int errorMessageLength; char accessPathBuffer[MAX_ACCESS_PATH]; + ValidityException *firstValidityException; + ValidityException *lastValidityException; + int flags; AccessPath *accessPath; jmp_buf recoveryData; } JsonValidator; @@ -63,6 +76,11 @@ JsonSchema *jsonBuildSchema(JsonSchemaBuilder *builder, Json *jsValue); JsonValidator *makeJsonValidator(); void freeJsonValidator(JsonValidator *validator); + +#define JSON_VALIDATOR_NO_EXCEPTIONS 0 +#define JSON_VALIDATOR_HAS_EXCEPTIONS 4 +#define JSON_VALIDATOR_INTERNAL_FAILURE 8 + int jsonValidateSchema(JsonValidator *validator, Json *value, JsonSchema *schema); diff --git a/platform/README.md b/platform/README.md new file mode 100644 index 000000000..4237b96c2 --- /dev/null +++ b/platform/README.md @@ -0,0 +1,15 @@ +# Platform Notes + +The zowe-common-c library is primarily driven by the need to have C programming interfaces for +making ZOS programming on C less painful regarding OS aware features. However, the many parts +of this library can run on other platforms, which makes development and testing easier. Non-ZOS +platform support files go in sub-directories here. Since xlc and it's runtime libraries are +highly POSIX compliant, there is not too much needed for Linux and other Unix's. Windows needs +more support for quirky non-compatible IO, systems calls, time functions, regular-expressions, etc. + +## CPP or not to be + +C++ is not used on ZOS, because ZOWE-common-c targets both LE (Posix) runtime environents and Metal +(traditional MVS) environments. IBM's xlc C++ does not support Metal and probably never will. So C++ +is out in general. However, to provide platform support on Windows, C++ is needed because API's are +often *only* published in C++. diff --git a/platform/windows/cppregex.cpp b/platform/windows/cppregex.cpp new file mode 100644 index 000000000..02838287b --- /dev/null +++ b/platform/windows/cppregex.cpp @@ -0,0 +1,36 @@ +#include "cppregex.h" +#include + +CPPRegex::CPPRegex() +{ + /* value = start; */ +} + +int CPPRegex::compile(const char *pattern, int cflags) +{ + try { + rx.assign(pattern,std::regex::ECMAScript); + } catch (const std::regex_error& e){ + return e.code(); + } + return 0; +} + +enum reg_errcode_t { + REG_NOERROR = 0, REG_NOMATCH, REG_BADPAT, REG_ECOLLATE, + REG_ECTYPE, REG_EESCAPE, REG_ESUBREG, REG_EBRACK, + REG_EPAREN, REG_EBRACE, REG_BADBR, REG_ERANGE, + REG_ESPACE, REG_BADRPT, REG_EEND, REG_ESIZE, + REG_ERPAREN +}; + +bool CPPRegex::exec(const char *first, const char *last, std::cmatch& matchResults){ + try { + bool status = regex_search(first, last, matchResults, rx); + return status; + } catch (const std::regex_error& e){ + return e.code(); + } +} + + diff --git a/platform/windows/cppregex.h b/platform/windows/cppregex.h new file mode 100644 index 000000000..8212f98db --- /dev/null +++ b/platform/windows/cppregex.h @@ -0,0 +1,17 @@ +#ifndef __CPPREGEX_H__ +#define __CPPREGEX_H__ + +#include + +class CPPRegex +{ + public: + CPPRegex(); + int compile(const char *pat, int cflags); + bool exec(const char *first, const char *last, std::cmatch& results); + + private: + std::regex rx; +}; + +#endif // __CPPEGEX_H__ diff --git a/platform/windows/winregex.cpp b/platform/windows/winregex.cpp new file mode 100644 index 000000000..611efd09b --- /dev/null +++ b/platform/windows/winregex.cpp @@ -0,0 +1,65 @@ +#include +#include "winregex.h" +#include "cppregex.h" + + +regex_t *regexAlloc(){ + regex_t *m; + CPPRegex *obj; + + m = (decltype(m))malloc(sizeof(*m)); + obj = new CPPRegex(); + m->obj = obj; + + return m; +} + +void regexFree(regex_t *m){ + if (m == NULL) + return; + delete static_cast(m->obj); + free(m); +} + +int regexComp(regex_t *r, const char *pat, int cflags){ + CPPRegex *obj; + + if (r == NULL){ + return REG_E_UNKNOWN; + } + + obj = static_cast(r->obj); + return obj->compile(pat,cflags); +} + +int regexExec(regex_t *r, const char *str, size_t nmatch, + regmatch_t *pmatch, int eflags){ + CPPRegex *obj; + std::cmatch matchResults; + + if (r == NULL){ + return REG_E_UNKNOWN; + } + + obj = static_cast(r->obj); + // here + // 1 worry about additional arguments vs. linue DIE + // 2 get pmatch data structure to fill in. + bool matched = obj->exec(str,str+strlen(str),matchResults); + if (matched){ + for (int i=0; i + +/* regcomp() cflags */ + +#define REG_EXTENDED 0x001 /* Use Extended RE syntax rules */ +#define REG_ICASE 0x002 /* Ignore case in match */ +#define REG_NEWLINE 0x004 /* Convert to + */ +#define REG_NOSUB 0x008 /* regexec() not report + subexpressions */ + + +/* regexec() eflags */ + +#define REG_NOTBOL 0x100 /* First character not start of line */ +#define REG_NOTEOL 0x200 /* Last character not end of line */ + + +/* Regular Expression error codes */ + +#define REG_NOMATCH 1 /* RE pattern not found */ +#define REG_BADPAT 2 /* Invalid Regular Expression */ +#define REG_ECOLLATE 3 /* Invalid collating element */ +#define REG_ECTYPE 4 /* Invalid character class */ +#define REG_EESCAPE 5 /* Last character is \ */ +#define REG_ESUBREG 6 /* Invalid number in \digit */ +#define REG_EBRACK 7 /* imbalance */ +#define REG_EPAREN 8 /* \( \) or () imbalance */ +#define REG_EBRACE 9 /* \{ \} or { } imbalance */ +#define REG_BADBR 10 /* Invalid \{ \} range exp */ +#define REG_ERANGE 11 /* Invalid range exp endpoint */ +#define REG_ESPACE 12 /* Out of memory */ +#define REG_BADRPT 13 /* ?*+ not preceded by valid RE */ +#define REG_ECHAR 14 /* invalid multibyte character */ +#define REG_EBOL 15 /* ^ anchor and not BOL */ +#define REG_EEOL 16 /* $ anchor and not EOL */ +#define REG_EMPTY 17 +#define REG_E_MEMORY 15 +#define REG_E_UNKNOWN 21 + + /* + static const reg_error_t REG_BADPAT = 2; // Invalid pattern. + static const reg_error_t REG_BADPAT = 2; // Invalid pattern. + static const reg_error_t REG_ECOLLATE = 3; // Undefined collating element. + static const reg_error_t REG_ECTYPE = 4; // Invalid character class name. + static const reg_error_t REG_EESCAPE = 5; // Trailing backslash. + static const reg_error_t REG_ESUBREG = 6; // Invalid back reference. + static const reg_error_t REG_EBRACK = 7; // Unmatched left bracket. + static const reg_error_t REG_EPAREN = 8; // Parenthesis imbalance. + static const reg_error_t REG_EBRACE = 9; // Unmatched \{. + static const reg_error_t REG_BADBR = 10; // Invalid contents of \{\}. + static const reg_error_t REG_ERANGE = 11; // Invalid range end. + static const reg_error_t REG_ESPACE = 12; // Ran out of memory. + static const reg_error_t REG_BADRPT = 13; // No preceding re for repetition op. + static const reg_error_t REG_EEND = 14; // unexpected end of expression + static const reg_error_t REG_ESIZE = 15; // expression too big + static const reg_error_t REG_ERPAREN = 8; // = REG_EPAREN : unmatched right parenthesis + static const reg_error_t REG_EMPTY = 17; // empty expression + static const reg_error_t REG_E_MEMORY = 15; // = REG_ESIZE : out of memory + static const reg_error_t REG_ECOMPLEXITY = 18; // complexity too high + static const reg_error_t REG_ESTACK = 19; // out of stack space + static const reg_error_t REG_E_PERL = 20; // Perl (?...) error + static const reg_error_t REG_E_UNKNOWN = 21; // unknown error + static const reg_error_t REG_ENOSYS = 21; // = REG_E_UNKNOWN : Reserved. + */ + + +typedef struct winregex { + void *obj; +} regex_t; + +typedef int64_t regoff_t; + +typedef struct regmatch{ + regoff_t rm_so; /* should be regoff_t */ + regoff_t rm_eo; +} regmatch_t; + +regex_t *regexAlloc(); +void regexFree(regex_t *m); + +int regexComp(regex_t *r, const char *pattern, int cflags); +int regexExec(regex_t *r, const char *str, size_t nmatch, + regmatch_t *pmatch, int eflags); + +#ifdef __cplusplus +} +#endif + +#endif /* __WINREGEX_H__ */ From c39d7dcede4e1e0d603943b6c866c59deebe1486 Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 1 Feb 2022 11:01:35 -0500 Subject: [PATCH 04/42] space Signed-off-by: Joe --- platform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/README.md b/platform/README.md index 4237b96c2..847fe030c 100644 --- a/platform/README.md +++ b/platform/README.md @@ -12,4 +12,4 @@ more support for quirky non-compatible IO, systems calls, time functions, regula C++ is not used on ZOS, because ZOWE-common-c targets both LE (Posix) runtime environents and Metal (traditional MVS) environments. IBM's xlc C++ does not support Metal and probably never will. So C++ is out in general. However, to provide platform support on Windows, C++ is needed because API's are -often *only* published in C++. +often *only* published in C++. From 67962254eae40a862cc837c1f52746e93271a2f3 Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 1 Feb 2022 23:59:04 -0500 Subject: [PATCH 05/42] More work on validation and adding a unit test Signed-off-by: Joe --- c/jsonschema.c | 7 - c/yaml2json.c | 375 ++++++++++++++++++ h/yaml2json.h | 10 + tests/schemadata/example-zowe.yaml | 613 +++++++++++++++++++++++++++++ tests/schemadata/zoweyaml.schema | 72 ++++ tests/schematest.c | 161 ++++++++ 6 files changed, 1231 insertions(+), 7 deletions(-) create mode 100644 c/yaml2json.c create mode 100644 h/yaml2json.h create mode 100644 tests/schemadata/example-zowe.yaml create mode 100644 tests/schemadata/zoweyaml.schema create mode 100644 tests/schematest.c diff --git a/c/jsonschema.c b/c/jsonschema.c index 7998a12c6..e277ce4ae 100644 --- a/c/jsonschema.c +++ b/c/jsonschema.c @@ -470,7 +470,6 @@ static bool validateJSONInteger(JsonValidator *validator, // Number exclusiveMaximum; // Number exclusiveMinimum; int64_t i = jsonAsInt64(value64); - printf("JOE i=%lld valFlags=0x%llx\n",i,valueSpec->validatorFlags); if (valueSpec->validatorFlags & JS_VALIDATOR_MAX){ int64_t lim = (int64_t)valueSpec->maximum; if (valueSpec->exclusiveMaximum ? (i >= lim) : (i > lim)){ @@ -907,10 +906,8 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopL accessPathPop(accessPath); } /// valueSpec.not = getComposite(object,accessPath); - printf("valueSpec typeMask=0x%x\n",valueSpec->typeMask); for (int typeCode=0; typeCode<=JSTYPE_MAX; typeCode++){ if ((1 << typeCode) & valueSpec->typeMask){ - printf("switch typeCode = %d\n",typeCode); switch (typeCode){ case JSTYPE_OBJECT: { @@ -1028,14 +1025,11 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopL { JSValueSpec *numericSpec = valueSpec; // new JSNumericSpec(description,title,(jsType == JSType.INTEGER)); - printf("JOE JSTYPE_INT/NUM: "); - printAccessPath(stdout,accessPath); numericSpec->isInteger = (typeCode == JSTYPE_INTEGER); numericSpec->multipleOf = getNumber(builder,object,"multipleOf",MISSING_FLOAT_VALIDATOR); numericSpec->maximum = getNumber(builder,object,"maximum",MISSING_FLOAT_VALIDATOR); numericSpec->exclusiveMaximum = getBooleanValue(builder,object,"exclusiveMaximum",false); numericSpec->minimum = getNumber(builder,object,"minimum",MISSING_FLOAT_VALIDATOR); - printf("max = %f min=%f\n",numericSpec->maximum,numericSpec->minimum); numericSpec->exclusiveMinimum = getBooleanValue(builder,object,"exclusiveMinimum",false); if (numericSpec->maximum != MISSING_FLOAT_VALIDATOR){ numericSpec->validatorFlags |= JS_VALIDATOR_MAX; @@ -1055,7 +1049,6 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopL } }// for (jsType... - printf("JOE end schema for loop\n"); return valueSpec; } else { schemaThrow(builder,12,"top level schema must be an object"); diff --git a/c/yaml2json.c b/c/yaml2json.c new file mode 100644 index 000000000..29afcb1c7 --- /dev/null +++ b/c/yaml2json.c @@ -0,0 +1,375 @@ +#include + /* #include */ + +#include +#include +#include +#include + +#ifdef NDEBUG +#undef NDEBUG +#endif +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "json.h" +#include "jsonschema.h" + +static char *tokenTypeName(yaml_token_type_t type){ + switch (type){ + case YAML_NO_TOKEN: return "NO_TOKEN"; + case YAML_STREAM_START_TOKEN: return "STREAM_START"; + case YAML_STREAM_END_TOKEN: return "STREAM_END"; + case YAML_VERSION_DIRECTIVE_TOKEN: return "VERSION_DIRECTIVE"; + case YAML_TAG_DIRECTIVE_TOKEN: return "TAG_DIRECTIVE"; + case YAML_DOCUMENT_START_TOKEN: return "DOCUMENT_START"; + case YAML_DOCUMENT_END_TOKEN: return "DOCUMENT_END"; + case YAML_BLOCK_SEQUENCE_START_TOKEN: return "BLOCK_SEQUENCE_START"; + case YAML_BLOCK_MAPPING_START_TOKEN: return "BLOCK_MAPPING_START"; + case YAML_BLOCK_END_TOKEN: return "BLOCK_END"; + case YAML_FLOW_SEQUENCE_START_TOKEN: return "FLOW_SEQUENCE_START"; + case YAML_FLOW_SEQUENCE_END_TOKEN: return "FLOW_SEQUENCE_END"; + case YAML_FLOW_MAPPING_START_TOKEN: return "FLOW_MAPPING_START"; + case YAML_FLOW_MAPPING_END_TOKEN: return "FLOW_MAPPING_END"; + case YAML_BLOCK_ENTRY_TOKEN: return "BLOCK_ENTRY"; + case YAML_FLOW_ENTRY_TOKEN: return "FLOW_ENTRY"; + case YAML_KEY_TOKEN: return "KEY"; + case YAML_VALUE_TOKEN: return "VALUE"; + case YAML_ALIAS_TOKEN: return "ALIAS"; + case YAML_ANCHOR_TOKEN: return "ANCHOR"; + case YAML_TAG_TOKEN: return "TAG"; + case YAML_SCALAR_TOKEN: return "SCALAR"; + default: + { + printf("unknown token seen with type = %d\n",(int)type); + return "UNKNOWN_TOKEN"; + } + } +} + +static char *getScalarStyleName(yaml_scalar_style_t style){ + switch (style){ + case YAML_ANY_SCALAR_STYLE: return "any"; + case YAML_PLAIN_SCALAR_STYLE: return "plain"; + case YAML_SINGLE_QUOTED_SCALAR_STYLE: return "single"; + case YAML_DOUBLE_QUOTED_SCALAR_STYLE: return "double"; + case YAML_LITERAL_SCALAR_STYLE: return "literal"; + case YAML_FOLDED_SCALAR_STYLE: return "folded"; + default: return "UNKNOWN_STYLE"; + } +} + + +yaml_document_t *readYAML(char *filename){ + FILE *file; + yaml_parser_t parser; + yaml_document_t *document = (yaml_document_t*)safeMalloc(sizeof(yaml_document_t),"YAML Doc"); + int done = 0; + int count = 0; + int error = 0; + + file = fopen(filename, "rb"); + assert(file); + + assert(yaml_parser_initialize(&parser)); + + yaml_parser_set_input_file(&parser, file); + + printf("before parser_load\n");fflush(stdout); + + if (!yaml_parser_load(&parser, document)) { + + printf("bad yaml load\n");fflush(stdout); + error = 1; + } else if (yaml_document_get_root_node(document)){ + + } else { + error = 1; + } + + printf("before parser_delete\n");fflush(stdout); + yaml_parser_delete(&parser); + printf("after parser_delete\n");fflush(stdout); + assert(!fclose(file)); + + if (error){ + safeFree((char*)document,sizeof(yaml_document_t)); + document = NULL; + } + + return document; +} + +static void indent(int x){ + for (int i=0; itype){ + case YAML_NO_NODE: + { + indent(depth); + printf("NoNode\n"); + } + break; + case YAML_SCALAR_NODE: + { + indent(depth); + size_t dataLen = node->data.scalar.length; + int printLength = dataLen > SCALAR_SIZE_LIMIT ? 40 : (int)dataLen; + /* printf("printLength=%d dataLen=%lld\n",printLength,dataLen); */ + printf("Scalar: (len=%d) %*.*s\n",printLength,printLength,printLength,node->data.scalar.value); + } + break; + case YAML_SEQUENCE_NODE: + { + yaml_node_item_t *item; + indent(depth);printf("SequenceStart (%s)\n",getSequenceStyleName(node->data.sequence.style)); + for (item = node->data.sequence.items.start; item < node->data.sequence.items.top; item++) { + yaml_node_t *nextNode = yaml_document_get_node(doc, *item); + if (nextNode){ + pprintYAML1(doc,nextNode,depth+2); + } else { + indent(depth+4); + printf("dead end item\n"); + } + } + indent(depth);printf("SequenceEnd\n"); + } + break; + case YAML_MAPPING_NODE: + yaml_node_pair_t *pair; + indent(depth);printf("MapStart\n"); + for (pair = node->data.mapping.pairs.start; pair < node->data.mapping.pairs.top; pair++) { + yaml_node_t *keyNode = yaml_document_get_node(doc, pair->key); + /* printf("keyNode 0x%p\n",keyNode); */ + if (keyNode) { + if (keyNode->type != YAML_SCALAR_NODE){ + printf("*** UNEXPECTED NON SCALAR KEY ***\n"); + } else { + indent(depth+2); + size_t dataLen = keyNode->data.scalar.length; + int printLength = dataLen > SCALAR_SIZE_LIMIT ? 40 : (int)dataLen; + /* printf("printLength=%d dataLen=%lld\n",printLength,dataLen); */ + printf("%*.*s:\n",printLength,printLength,keyNode->data.scalar.value); + } + } else { + printf("dead end key\n"); + } + yaml_node_t *valueNode = yaml_document_get_node(doc, pair->value); + if (valueNode){ + pprintYAML1(doc,valueNode,depth+4); + } else{ + printf("dead end value\n"); + } + } + indent(depth);printf("MapEnd\n"); + break; + default: + printf("unexpected yaml node type %d\n",node->type); + break; + } +} + +void pprintYAML(yaml_document_t *document){ + pprintYAML1(document,yaml_document_get_root_node(document),0); +} + +#define MAX_JSON_KEY 256 +#define MAX_JSON_STRING 65536 + +/* this needs to be smarter some day */ + +static bool isSyntacticallyInteger(yaml_char_t *data, int length){ + if (length == 0){ + printf("empty string not integer\n"); + return false; + } + for (int i=0; i= '0' && c <= '9'){ + + } else { + printf("%s is NOT an integer\n",data); + return false; + } + } + printf("%s is an integer\n",data); + return true; +} + +static int64_t readInt(yaml_char_t *data, int length, bool *valid){ + int64_t val64 = 0; + bool allDecimal = true; + + for (int i=0; i= '0' && c <= '9'){ + val64 = (10 * val64) + (c-'0'); + } else { + allDecimal = false; + } + } + + *valid = allDecimal; + return val64; +} + +#define JSON_FAIL_NOT_HANDLED 100 +#define JSON_FAIL_BAD_INTEGER 104 + +static Json *yaml2JSON1(JsonBuilder *b, Json *parent, char *parentKey, + yaml_document_t *doc, yaml_node_t *node, int depth){ + int buildStatus = 0; + switch (node->type){ + case YAML_NO_NODE: + { + printf("*** WARNING *** NoNode\n"); + return NULL; + } + case YAML_SCALAR_NODE: + { + size_t dataLen = node->data.scalar.length; + int valueLength = (int)dataLen; + if (dataLen > MAX_JSON_STRING){ + printf("*** WARNING *** oversize JSON string!\n"); + valueLength = MAX_JSON_STRING; + } + // Negative numbers, hexadecimal + yaml_scalar_style_t style = node->data.scalar.style; + switch (style){ + case YAML_ANY_SCALAR_STYLE: + case YAML_PLAIN_SCALAR_STYLE: + case YAML_SINGLE_QUOTED_SCALAR_STYLE: + case YAML_DOUBLE_QUOTED_SCALAR_STYLE: + case YAML_LITERAL_SCALAR_STYLE: + case YAML_FOLDED_SCALAR_STYLE: + char *tag = (char*)node->tag; + printf("tag = %s scalarStyle=%s\n",tag,getScalarStyleName(node->data.scalar.style)); + Json *scalar = NULL; + // HERE, make test with float, int, bool, null, ddate + if (!strcmp(tag,YAML_NULL_TAG)){ + } else if (!strcmp(tag,YAML_NULL_TAG)){ + /* Json *scalar = jsonBuildNull(b,parent,parentKey,&buildStatus); */ + } else if (!strcmp(tag,YAML_BOOL_TAG)){ + /* Json *scalar = jsonBuildBool(b,parent,parentKey,"FOO",3,&buildStatus); */ + } else if (!strcmp(tag,YAML_INT_TAG) || + (!strcmp(tag,YAML_STR_TAG) && + (style == YAML_PLAIN_SCALAR_STYLE) && + isSyntacticallyInteger(node->data.scalar.value,valueLength))){ + bool valid; + int64_t x = readInt(node->data.scalar.value,valueLength,&valid); + if (valid){ + scalar = jsonBuildInt64(b,parent,parentKey,x,&buildStatus); + } else { + buildStatus = JSON_FAIL_BAD_INTEGER; + } + /* Json *scalar = jsonBuildInt(b,parent,parentKey,"FOO",3,&buildStatus); */ + } else if (!strcmp(tag,YAML_STR_TAG)){ + scalar = jsonBuildString(b,parent,parentKey,(char*)node->data.scalar.value,valueLength,&buildStatus); + } else if (!strcmp(tag,YAML_FLOAT_TAG)){ + printf("*** Warning don't know how to handle float yet\n"); + buildStatus = JSON_FAIL_NOT_HANDLED; + } else if (!strcmp(tag,YAML_TIMESTAMP_TAG)){ + printf("*** Warning don't know how to handle timestamp yet\n"); + buildStatus = JSON_FAIL_NOT_HANDLED; + } + if (buildStatus){ + printf("*** WARNING *** Failed to add property/scalar err=%d\n",buildStatus); + } + return scalar; + default: + printf("*** WARNING *** - unknown scalar style %d",node->data.scalar.style); + return NULL; + } + } + case YAML_SEQUENCE_NODE: + { + yaml_node_item_t *item; + Json *jsonArray = jsonBuildArray(b,parent,parentKey,&buildStatus); + if (jsonArray){ + for (item = node->data.sequence.items.start; item < node->data.sequence.items.top; item++) { + yaml_node_t *nextNode = yaml_document_get_node(doc, *item); + if (nextNode){ + yaml2JSON1(b,jsonArray,NULL,doc,nextNode,depth+2); + } else { + printf("*** WARNING *** dead end item\n"); + } + } + return jsonArray; + } else { + printf("*** WARNING *** Failed to add json array err=%d\n",buildStatus); + return NULL; + } + } + case YAML_MAPPING_NODE: + { + yaml_node_pair_t *pair; + Json *jsonObject = jsonBuildObject(b,parent,parentKey,&buildStatus); + if (jsonObject){ + for (pair = node->data.mapping.pairs.start; pair < node->data.mapping.pairs.top; pair++) { + yaml_node_t *keyNode = yaml_document_get_node(doc, pair->key); + /* printf("keyNode 0x%p\n",keyNode); */ + char *key = NULL; + int keyLength; + if (keyNode) { + if (keyNode->type != YAML_SCALAR_NODE){ + printf("*** WARNING *** Non Scalar key\n"); + } else { + size_t dataLen = keyNode->data.scalar.length; + keyLength = (int)dataLen; + if (keyLength > MAX_JSON_KEY){ + printf("*** WARNING *** key too long '%*.*s...'\n", + MAX_JSON_KEY,MAX_JSON_KEY,keyNode->data.scalar.value); + } else { + key = (char*)keyNode->data.scalar.value; + } + } + } else { + printf("*** WARNING *** dead end key\n"); + } + if (key){ + yaml_node_t *valueNode = yaml_document_get_node(doc, pair->value); + if (valueNode){ + yaml2JSON1(b,jsonObject,key,doc,valueNode,depth+2); + } else{ + printf("*** WARNING *** dead end value\n"); + } + } + } + return jsonObject; + } else { + printf("*** WARNING *** Failed to add json object err=%d\n",buildStatus); + return NULL; + } + } + break; + default: + printf("*** WARNING *** unexpected yaml node type %d\n",node->type); + return NULL; + } +} + +Json *yaml2JSON(yaml_document_t *document, ShortLivedHeap *slh){ + JsonBuilder *builder = makeJsonBuilder(slh); + Json *json = yaml2JSON1(builder,NULL,NULL,document,yaml_document_get_root_node(document),0); + freeJsonBuilder(builder,false); + return json; +} diff --git a/h/yaml2json.h b/h/yaml2json.h new file mode 100644 index 000000000..6b42c89e8 --- /dev/null +++ b/h/yaml2json.h @@ -0,0 +1,10 @@ +#ifndef __ZOWE_YAML2JSON__ +#define __ZOWE_YAML2JSON__ 1 + +#include "yaml.h" + +yaml_document_t *readYAML(char *filename); +void pprintYAML(yaml_document_t *document); +Json *yaml2JSON(yaml_document_t *document, ShortLivedHeap *slh); + +#endif diff --git a/tests/schemadata/example-zowe.yaml b/tests/schemadata/example-zowe.yaml new file mode 100644 index 000000000..ee6f585ac --- /dev/null +++ b/tests/schemadata/example-zowe.yaml @@ -0,0 +1,613 @@ +################################################################################ +# This program and the accompanying materials are made available under the terms of the +# Eclipse Public License v2.0 which accompanies this distribution, and is available at +# https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. +################################################################################ + +#=============================================================================== +# This is a YAML configuration file for Zowe instance. +# +# YAML is a human-friendly data serialization language for all programming languages. +# To learn more about YAML specifications, please check https://yaml.org/. +# +# To learn more details about the entries, please check https://docs.zowe.org/. +# +# For first time users, or for the most common use cases, please pay more +# attention on the entries marked with "**COMMONLY_CUSTOMIZED**". +# +# If you modify any settings listed in "zwe init --help" command, you may need to +# re-run "zwe init" command to make them take effect. +#=============================================================================== + +#------------------------------------------------------------------------------- +# Zowe global configurations +# +# This section includes Zowe setup information used by `zwe install` and +# `zwe init` command, as well as default configurations for Zowe runtime. +#------------------------------------------------------------------------------- +zowe: + + #------------------------------------------------------------------------------- + # These configurations are used by "zwe install" or "zwe init" commands. + #------------------------------------------------------------------------------- + setup: + # MVS data set related configurations + mvs: + # **COMMONLY_CUSTOMIZED** + # where Zowe MVS data sets will be installed + # hlq: IBMUSER.ZWEV2 + hlq: IBMUSER.F$$ + # **COMMONLY_CUSTOMIZED** + # PROCLIB where Zowe STCs will be copied over + proclib: USER.PROCLIB + # **COMMONLY_CUSTOMIZED** + # Zowe PARMLIB + parmlib: IBMUSER.ZWEV2.CUST.PARMLIB + # **COMMONLY_CUSTOMIZED** + # JCL library where Zowe will store temporary JCLs during initialization + jcllib: IBMUSER.ZWEV2.CUST.JCLLIB + # APF authorized LOADLIB for Zowe + # Optional. If it's empty, .SZWEAUTH will be APF authorized. + authLoadlib: + # **COMMONLY_CUSTOMIZED** + # APF authorized LOADLIB for Zowe ZIS Plugins + authPluginLib: IBMUSER.ZWEV2.CUST.ZWESAPL + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # # Security related configurations. This setup is optional. + # security: + # # security product name. Can be RACF, ACF2 or TSS + # product: RACF + # # security group name + # groups: + # # Zowe admin user group + # admin: ZWEADMIN + # # Zowe STC group + # stc: ZWEADMIN + # # Zowe SysProg group + # sysProg: ZWEADMIN + # # security user name + # users: + # # Zowe runtime user name of main service + # zowe: ZWESVUSR + # # Zowe runtime user name of Cross Memory Server + # xmem: ZWESIUSR + # # Zowe runtime user name of Auxilary Service + # aux: ZWESIUSR + # # STC names + # stcs: + # # STC name of Zowe main service + # zowe: ZWESLSTC + # # STC name of Zowe Cross Memory Server + # xmem: ZWESISTC + # # STC name of Zowe Auxilary Service + # aux: ZWESASTC + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # Certificate related configurations + # + # There are 5 configurations cases. Please choose one from below. + + # >>>> Certificate setup scenario 1 + # PKCS12 (keystore) with Zowe generate certificates. + certificate: + # Type of certificate storage. Valid values are: PKCS12 or JCERACFKS + type: PKCS12 + pkcs12: + # **COMMONLY_CUSTOMIZED** + # Keystore directory + directory: /var/zowe/keystore + # **COMMONLY_CUSTOMIZED** + # # Certificate alias name. Optional, default value is localhost. + # # Note: please use all lower cases as alias. + # name: localhost + # **COMMONLY_CUSTOMIZED** + # # Keystore password. Optional, default value is password. + # password: password + # **COMMONLY_CUSTOMIZED** + # # Alias name of self-signed certificate authority. Optional, default value is local_ca. + # # Note: please use all lower cases as alias. + # caAlias: local_ca + # **COMMONLY_CUSTOMIZED** + # # Password of keystore stored self-signed certificate authority. Optional, default value is local_ca_password. + # caPassword: local_ca_password + # # Distinguished name for Zowe generated certificates. All optional. + # dname: + # caCommonName: "" + # commonName: "" + # orgUnit: "" + # org: "" + # locality: "" + # state: "" + # country: "" + # # Validity days for Zowe generated certificates + # validity: 3650 + # # Domain names and IPs should be added into certificate SAN + # # If this field is not defined, `zwe init` command will use + # # `zowe.externalDomains`. + # san: + # - dvipa.my-company.com + # # sample IP address + # - 12.34.56.78 + + # # >>>> Certificate setup scenario 2 + # # PKCS12 (keystore) with importing certificate generated by other CA. + # certificate: + # # Type of certificate storage. Valid values are: PKCS12 or JCERACFKS + # type: PKCS12 + # pkcs12: + # # **COMMONLY_CUSTOMIZED** + # # Keystore directory + # directory: /var/zowe/keystore + # # # Certificate alias name. Optional, default value is localhost. + # # # Note: please use all lower cases as alias. + # # name: localhost + # # # Keystore password. Optional, default value is password. + # # password: password + # import: + # # **COMMONLY_CUSTOMIZED** + # # Existing PKCS12 keystore which holds the certificate issued by external CA. + # keystore: /var/zowe/extcerts/dummy_certs.keystore.p12 + # # **COMMONLY_CUSTOMIZED** + # # Password of the above keystore + # password: dummycert + # # **COMMONLY_CUSTOMIZED** + # # Certificate alias will be imported + # # Note: please use all lower cases as alias. + # alias: dummy_certs + # # **COMMONLY_CUSTOMIZED** + # # PEM format certificate authorities will also be imported and trusted. + # importCertificateAuthorities: + # # This should be the certificate authority signed the certificate will be imported. + # - /var/zowe/extcerts/dummy_ca.cer + + # # >>>> Certificate setup scenario 3 + # # JCERACFKS (z/OS Keyring) with Zowe generated certificates. + # certificate: + # # Type of certificate storage. Valid values are: PKCS12 or JCERACFKS + # type: JCERACFKS + # keyring: + # # **COMMONLY_CUSTOMIZED** + # # keyring name + # name: Zowe2Keyring + # # **COMMONLY_CUSTOMIZED** + # # # Label of Zowe certificate. Optional, default value is localhost. + # # label: Primary Zowe v2 Cert + # # **COMMONLY_CUSTOMIZED** + # # # label of Zowe CA certificate. Optional, default value is localca. + # # caLabel: Zowe v2 CA + # # # Distinguished name for Zowe generated certificates. All optional. + # # dname: + # # caCommonName: "" + # # commonName: "" + # # orgUnit: "" + # # org: "" + # # locality: "" + # # state: "" + # # country: "" + # # # Validity days for Zowe generated certificates + # # validity: 3650 + # # # Domain names and IPs should be added into certificate SAN + # # # If this field is not defined, `zwe init` command will use + # # # `zowe.externalDomains`. + # # # **NOTE**: due to the limitation of RACDCERT command, this field should + # # # contain exactly 2 entries with the domain name and IP address. + # # san: + # # - dvipa.my-company.com + # # - 12.34.56.78 + + # # >>>> Certificate setup scenario 4 + # # JCERACFKS (z/OS Keyring) and connect to existing certificate + # certificate: + # # Type of certificate storage. Valid values are: PKCS12 or JCERACFKS + # type: JCERACFKS + # keyring: + # # **COMMONLY_CUSTOMIZED** + # # keyring name + # name: Zowe2Keyring + # connect: + # # **COMMONLY_CUSTOMIZED** + # # Current owner of the existing certificate, can be SITE or an user ID. + # user: IBMUSER + # # **COMMONLY_CUSTOMIZED** + # # Label of the existing certificate will be connected to Zowe keyring. + # label: Dummy Cert + # # **COMMONLY_CUSTOMIZED** + # # If you have other certificate authorities want to be trusted in Zowe keyring, + # # list the certificate labels here. + # # **NOTE**, due to the limitation of RACDCERT command, this field should + # # contain maximum 2 entries. + # importCertificateAuthorities: + # - Dummy CA + + # # >>>> Certificate setup scenario 5 + # # JCERACFKS (z/OS Keyring) with importing certificate stored in data set + # certificate: + # # Type of certificate storage. Valid values are: PKCS12 or JCERACFKS + # type: JCERACFKS + # keyring: + # # **COMMONLY_CUSTOMIZED** + # # keyring name + # name: Zowe2Keyring + # # **COMMONLY_CUSTOMIZED** + # # # Label of Zowe certificate. Optional, default value is localhost. + # # label: Primary Zowe v2 Cert + # import: + # # **COMMONLY_CUSTOMIZED** + # # Name of the data set holds the certificate issued by other CA. + # # This data set should be in PKCS12 format and contain private key. + # dsName: IBMUSER.DUMMY.CERTP12 + # # **COMMONLY_CUSTOMIZED** + # # Password for the PKCS12 data set. + # password: dummycert + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # VSAM configurations if you are using VSAM as Caching Service storage + vsam: + # VSAM data set with Record-Level-Sharing enabled or not + # Valid values could be: NONRLS or RLS. + mode: NONRLS + # Volume name if you are using VSAM in NONRLS mode + volume: VOL123 + # Storage class name if you are using VSAM in RLS mode + storageClass: + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # **COMMONLY_CUSTOMIZED** + # Zowe runtime (root) directory + # + # **NOTE**: if it's not specified and you passed "--update-config" argument + # when you run "zwe init" command, this value will be updated with the Zowe + # runtime where the "zwe" command is located. + # + # This value is required by ZWESLSTC to know where is Zowe runtime. + runtimeDirectory: "" + + # **COMMONLY_CUSTOMIZED** + # Where to store runtime logs + logDirectory: /global/zowe/instance/logs + + # **COMMONLY_CUSTOMIZED** + # Zowe runtime workspace directory + workspaceDirectory: /global/zowe/instance/workspace + + # **COMMONLY_CUSTOMIZED** + # Where extensions are installed + extensionDirectory: /global/zowe/extensions + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # runtime z/OS job name + job: + # Zowe JES job name + name: ZWE1SV$ + # Prefix of component address space + prefix: ZWE1 + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # **COMMONLY_CUSTOMIZED** + # You can list your external domains how you want to access Zowe. + # This should be the domain list you would like to put into your web browser + # address bar. + externalDomains: + # this should be the domain name of you Dynamic VIP Address (DVIPA) + - dvipa.my-company.com + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # This is the port you use to access Zowe Gateway from your web browser. + # + # In many use cases, this should be same as `components.gateway.port`. But in + # some use cases, like containerization, this port could be different. + externalPort: 65236 + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # You can define any extra environment variables as key/value pairs here + environments: + # Example of a global environment variable for all components + # MY_ENV_VAR: my_env_val + + # Another example to customize SSH port for VT Terminal Desktop app + # ZWED_SSH_PORT: 22 + # ZWED_TN3270_PORT: 23 + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # Enable debug mode for zowe launch scripts + launchScript: + # set to "debug" or "trace" to display extra debug information + logLevel: "" + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # Default Zowe certificate + # + # **NOTE**: these fields can be updated automatically if you pass + # `--update-config` to `zwe init` command. The generated value will base on + # your setup in `zowe.setup.certificate` section. + certificate: + keystore: + type: PKCS12 + file: /global/zowe/keystore/localhost/localhost.keystore.p12 + password: password + alias: localhost + truststore: + type: PKCS12 + file: /global/zowe/keystore/localhost/localhost.truststore.p12 + password: password + pem: + key: /global/zowe/keystore/localhost/localhost.key + certificate: /global/zowe/keystore/localhost/localhost.cer + certificateAuthorities: /global/zowe/keystore/local_ca/local_ca.cer + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # How we want to verify SSL certificates of services. Valid values are: + # - STRICT: will validate if the certificate is trusted in our trust store and + # if the certificate Command Name and Subject Alternative Name (SAN) + # is validate. This is recommendate for the best security. + # - NONSTRICT: will validate if the certificate is trusted in our trust store. + # This mode does not validate certificate Common Name and Subject + # Alternative Name (SAN). + # - DISABLED: disable certificate validation. This is NOT recommendated for + # security + verifyCertificates: STRICT + + +#------------------------------------------------------------------------------- +# Java configuraiton +# +# Some Zowe components requires Java. Define the path where you have your Java +# is installed. +# +# **NOTE**: this field can be updated automatically if you pass `--update-config` +# to `zwe init` command. +#------------------------------------------------------------------------------- +java: + # **COMMONLY_CUSTOMIZED** + # Path to your Java home directory + home: "" + + +#------------------------------------------------------------------------------- +# node.js configuraiton +# +# Some Zowe components requires node.js. Define the path where you have your +# node.js is installed. +# +# **NOTE**: this field can be updated automatically if you pass `--update-config` +# to `zwe init` command. +#------------------------------------------------------------------------------- +node: + # **COMMONLY_CUSTOMIZED** + # Path to your node.js home directory + home: "" + + +#------------------------------------------------------------------------------- +# z/OSMF configuraiton +# +# If your Zowe instance is configured to use z/OSMF for authentication or other +# features. You need to define how to access your z/OSMF instance. +#------------------------------------------------------------------------------- +zOSMF: + # **COMMONLY_CUSTOMIZED** + # host name of your z/OSMF instance + host: dvipa.my-company.com + # **COMMONLY_CUSTOMIZED** + port: "443" + applId: IZUDFLT + + +#------------------------------------------------------------------------------- +# Zowe components default configurations +# +# This section includes default configurations for all Zowe components installed +# on the Zowe instance. +# +# Every component should define their own section under `components` with their +# component ID. +# +# For each component, they can always have "enabled" property and "certificate" +# property. More configurations for each component can be found in component +# manifest file. +#------------------------------------------------------------------------------- +components: + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + gateway: + enabled: true + port: 7554 + debug: false + + apiml: + security: + auth: + provider: zosmf + zosmf: + jwtAutoconfiguration: auto + serviceId: zosmf + authorization: + endpoint: + enabled: false + provider: + jwtInitializerTimeout: 5 + x509: + enabled: false + zosmf: + applid: IZUDFLT + service: + allowEncodedSlashes: true + corsEnabled: false + server: + internal: + # gateway supports internal connector + enabled: false + port: 7556 + ssl: + enabled: false + # internal connector can use different certificate + # certificates: + # alias: + + # If we customize this to use different external certificate, than should also + # define "server.internal.ssl.certificate" and enable "server.internal.ssl.enabled". + # certificates: + # alias: + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + metrics-service: + enabled: false + port: 7551 + debug: false + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + api-catalog: + enabled: true + port: 7552 + debug: false + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + discovery: + enabled: true + port: 7553 + debug: false + # Define this value to match your number of Discovery StatefulSet if you are running containerized Zowe + # replicas: 1 + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + caching-service: + enabled: true + port: 7555 + debug: false + + storage: + evictionStrategy: reject + # can be inMemory, VSAM + mode: VSAM + size: 10000 + vsam: + # your VSAM data set created by ZWECSVSM job + # this is required if storage mode is VSAM + name: + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + app-server: + enabled: true + port: 7556 + # we can customize any component with custom certificate + # the missing definitions will be picked from "zowe.certificate" + # certificate: + # keystore: + # alias: app-server + # pem: + # key: /global/zowe/keystore/localhost/localhost.keystore.app-server.key + # certificate: /global/zowe/keystore/localhost/localhost.keystore.app-server.cer-ebcdic + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + zss: + enabled: true + port: 7557 + crossMemoryServerName: ZWESIS_STD + tls: true + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + jobs-api: + enabled: false + port: 7558 + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + files-api: + enabled: false + port: 7559 + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + explorer-jes: + enabled: true + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + explorer-mvs: + enabled: true + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + explorer-uss: + enabled: true + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # Each extension can have dedicated definition similar to core components. + # my-extension: + # enabled: true + + +#------------------------------------------------------------------------------- +# Zowe high availability instances customizations +# +# This section includes customizations for each Zowe high availability instance. +# +# You can start each HA instance with `zwe start --ha-instance `. +#------------------------------------------------------------------------------- +# haInstances: +# +# # HA instance ID +# lpar1: +# # hostname where this instance will be started +# hostname: lpar1.my-company.com +# # Your &SYSNAME for this LPAR +# # This sysname will be used to route your JES command to target system. +# sysname: LPR1 +# # for this HA instance, we didn't customize "components", so it will use default value. + +# # HA instance ID, we will start 2 instances on LPAR2 +# # **NOTE**, we can only start one gateway in same LPAR. +# lpar2a: +# # hostname where this instance will be started +# hostname: lpar2.my-company.com +# # Your &SYSNAME for this LPAR +# # This sysname will be used to route your JES command to target system. +# sysname: LPR2 + +# # These configurations will overwrite highest level default "components" configuration +# components: +# discovery: +# # use customized port on this instance +# port: 17553 +# api-catalog: +# port: 17552 +# app-server: +# # no app-server in this instance +# enabled: false +# zss: +# # no app-server in this instance +# enabled: false +# jobs-api: +# port: 18545 +# enabled: true +# files-api: +# port: 18547 +# enabled: true +# caching-service: +# port: 17555 +# +# lpar2b: +# hostname: lpar2.my-company.com +# # your &SYSNAME for this LPAR +# sysname: LPR2 + +# # These configurations will overwrite highest level default "components" configuration +# components: +# gateway: +# enabled: false +# discovery: +# enabled: false +# api-catalog: +# enabled: false +# app-server: +# enabled: false +# port: 28544 +# zss: +# port: 28542 +# jobs-api: +# enabled: true +# files-api: +# enabled: false +# caching-service: +# enabled: false diff --git a/tests/schemadata/zoweyaml.schema b/tests/schemadata/zoweyaml.schema new file mode 100644 index 000000000..2dd3fc3e4 --- /dev/null +++ b/tests/schemadata/zoweyaml.schema @@ -0,0 +1,72 @@ +{ + "name": "zowe_yaml_schema", + "$id": "http//zowe.org/schemas/yaml-2.0.json", + "type": "object", + "required": [ "zowe" ], + "properties": { + "zowe": { + "type": "object", + + "description": "This is the top-level Zowe Server Configuration Object", + "properties": { + "setup": { + "type": "object", + "properties": { + "mvs": { + "type": "object", + "description": "MVS (ZOS) Dataset names and prefixes", + "properties": { + "proclib": { + "title": "PROCLIB", + "description": "JCL Procedure Library", + "examples": [ "FOO.BAR.BLETCH" ], + "$ref": "#/$defs/dataset" }, + "parmlib": { + "description": "Parameter Library (44-char ZOS dotted name)", + "$ref": "#/$defs/dataset" }, + "hlq": { "$ref": "#/$defs/dataset" } + } + } + } + }, + "job": { + "type": "object", + "required": [ "name", "prefix" ], + "properties": { + "name": { "$ref": "#/$defs/jobname" }, + "prefix": { + "type": "string" + } + } + }, + "externalPort": { + "description": "This must be a valid TCP Port or offset reference", + "$ref": "#/$defs/tcpPort" }, + "logDirectory": { + "description": "Must be a valid USS (Unix-style) path", + "type": "string" + } + } + } + }, + "$defs": { + "dataset": { + "type": "string", + "description": "A 44-char all caps dotted ZOS name", + "pattern": "^([A-Z0-9\\$\\#\\@]){1,8}(\\.([A-Z0-9\\$\\#\\@]){1,8}){0,11}$", + "minLength": 3, + "maxLength": 44 + }, + "jobname": { + "type": "string", + "pattern": "^([A-Z0-9\\$\\#\\@]){1,8}$", + "minLength": 3, + "maxLength": 8 + }, + "tcpPort": { + "type": "integer", + "minimum": 1024, + "maximum": 65535 + } + } +} diff --git a/tests/schematest.c b/tests/schematest.c new file mode 100644 index 000000000..b39b315ad --- /dev/null +++ b/tests/schematest.c @@ -0,0 +1,161 @@ +#include +/* #include */ + +#include +#include +#include +#include + +#ifdef NDEBUG +#undef NDEBUG +#endif +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "json.h" +#include "jsonschema.h" +#include "yaml2json.h" + +/* + Notes: + + (all work assumed to be done from shell in this directory) + + Windows Build ______________________________ + + Assuming clang is installed + + set YAML=c:\repos\libyaml ## Wherever you git clone'd libyaml + + clang++ -c ../platform/windows/cppregex.cpp ../platform/windows/winregex.cpp + + clang -I%YAML%/include -I./src -I../h -I ../platform/windows -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 -o schematest.exe schematest.c %YAML%/src/api.c %YAML%/src/reader.c %YAML%/src/scanner.c %YAML%/src/parser.c %YAML%/src/loader.c %YAML%/src/writer.c %YAML%/src/emitter.c %YAML%/src/dumper.c ../c/yaml2json.c ../c/jsonschema.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c cppregex.o winregex.o + + ZOS Build _________________________________ + + + + + Running the Test ________________________________ + + schematest ? + + schematest yaml schemadata/example-zowe.yaml utf8 schemadata/zoweyaml.schema + */ + +int main(int argc, char *argv[]) +{ + int number; + char *syntax = argv[1]; + char *filename = argv[2]; + char *charset = argv[3]; + char *schemaFilename = (argc >= 5 ? argv[4] : NULL); + int errorBufferSize = 1024; + printf("Sizeof(size_t) = %lld\n",sizeof(size_t)); +#ifdef __ZOWE_OS_WINDOWS + int stdoutFD = _fileno(stdout); +#else + int stdoutFD = STDOUT_FILENO; +#endif + char *errorBuffer = safeMalloc(errorBufferSize,"ErrorBuffer"); + + /* + Arg for charset + functionalize the char tests + materialize the char constants + see if we can parse on other alphabets + "soft" fork to JoeNemo + */ + + if (argc < 4) { + printf(" ? \n"); + return 0; + } + + Json *json = NULL; + ShortLivedHeap *slh = makeShortLivedHeap(0x10000, 100); + + if (!strcmp(syntax,"json")){ + memset(errorBuffer,0,errorBufferSize); + json = jsonParseFile2(slh,filename,errorBuffer,errorBufferSize); + printf("json parsed json=0x%p, with error '%s'\n",json,errorBuffer); + + } else if (!strcmp(syntax,"yaml")){ + printf("yaml syntax case\n"); + fflush(stdout); + yaml_document_t *doc = readYAML(filename); + printf("yaml doc at 0x%p\n",doc); + if (doc){ + pprintYAML(doc); + json = yaml2JSON(doc,slh); + } else { + printf("yaml probably had syntax issues\n"); + } + } else { + printf("unhandled syntax\n"); + } + + if (json){ + printf("Showing internal JSON whether from Yaml, JSON or PARMLIB\n"); + jsonPrinter *p = makeJsonPrinter(stdoutFD); + jsonEnablePrettyPrint(p); + jsonPrint(p,json); + + if (schemaFilename){ + memset(errorBuffer,0,errorBufferSize); + Json *schemaJSON = jsonParseFile2(slh,schemaFilename,errorBuffer,errorBufferSize); + printf("json parsed schema json=0x%p, with error '%s'\n",json,errorBuffer); + fflush(stdout); + if (schemaJSON){ + printf("Regurgitating JSON Schema before digesting it for real\n"); + jsonPrint(p,schemaJSON); + printf("\n"); + fflush(stdout); + JsonSchemaBuilder *builder = makeJsonSchemaBuilder(DEFAULT_JSON_SCHEMA_VERSION); + printf("ckpt.2\n");fflush(stdout); + JsonSchema *schema = jsonBuildSchema(builder,schemaJSON); + printf("ckpt.3 schema=0x%p\n",schema);fflush(stdout); + if (schema){ + JsonValidator *validator = makeJsonValidator(); + printf("Before Validate\n");fflush(stdout); + int validateStatus = jsonValidateSchema(validator,json,schema); + switch (validateStatus){ + case JSON_VALIDATOR_NO_EXCEPTIONS: + printf("No validity Exceptions\n"); + break; + case JSON_VALIDATOR_HAS_EXCEPTIONS: + { + printf("Validity Exceptions:\n"); + ValidityException *e = validator->firstValidityException; + while (e){ + printf(" %s\n",e->message); + e = e->next; + } + } + break; + case JSON_VALIDATOR_INTERNAL_FAILURE: + printf("validation internal failure"); + break; + } + freeJsonValidator(validator); + printf("Done with Validation Test\n"); + fflush(stdout); + } else { + printf("Schema Build Failed: %s\n",builder->errorMessage); + } + freeJsonSchemaBuilder(builder); + } else { + printf("Failed to read schema JSON\n"); + } + } + } else{ + printf("was not able to make JSON from source\n"); + + } + printf("JSON Test end\n"); + fflush(stdout); + return 0; +} + From 9ddfbe73f00e475d4416700be9ee35d663188eea Mon Sep 17 00:00:00 2001 From: Joe Date: Wed, 2 Feb 2022 01:21:49 -0500 Subject: [PATCH 06/42] posix regex support Signed-off-by: Joe --- platform/posix/psxregex.c | 10 ++++++++++ platform/posix/psxregex.h | 12 ++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 platform/posix/psxregex.c create mode 100644 platform/posix/psxregex.h diff --git a/platform/posix/psxregex.c b/platform/posix/psxregex.c new file mode 100644 index 000000000..10bedf2c9 --- /dev/null +++ b/platform/posix/psxregex.c @@ -0,0 +1,10 @@ + + +regex_t *regexAlloc(){ + return (regex_t*)safeMalloc(sizeof(regex_t),"regex_t"); +} + +void regexFree(regex_t *r){ + regfree(r); + safefree((char*)r,sizeof(regex_t)); +} diff --git a/platform/posix/psxregex.h b/platform/posix/psxregex.h new file mode 100644 index 000000000..69d027b33 --- /dev/null +++ b/platform/posix/psxregex.h @@ -0,0 +1,12 @@ +#ifndef __ZOWE_PSxREGEX__ +#define __ZOWE_PS + +#include + +regex_t *regexAlloc(); +void regexFree(regex_t *m); + +#define regexComp regcomp +#define regexExec regexec + +#endif From d02dbb10daef2883e96011859332894a3d402769 Mon Sep 17 00:00:00 2001 From: Leonty Chudinov Date: Thu, 3 Feb 2022 21:10:17 +0500 Subject: [PATCH 07/42] Make libyaml work on zos Signed-off-by: Leonty Chudinov --- c/json.c | 7 +++- c/jsonschema.c | 4 +- c/yaml2json.c | 84 ++++++++++++++++++++++++++++++--------- h/json.h | 3 +- platform/posix/psxregex.c | 4 +- tests/Makefile | 69 ++++++++++++++++++++++++++++++++ tests/schematest.c | 3 ++ 7 files changed, 149 insertions(+), 25 deletions(-) create mode 100644 tests/Makefile diff --git a/c/json.c b/c/json.c index eba3c1901..4d4f2b4f0 100644 --- a/c/json.c +++ b/c/json.c @@ -2044,7 +2044,12 @@ Json *jsonBuildNull(JsonBuilder *b, } } - +char* jsonBuildKey(JsonBuilder *b, const char *key, int len) { + JsonParser *parser = (JsonParser*)b; + char *keyCopy = jsonParserAlloc(parser, len + 1); + snprintf(keyCopy, len+1, "%.*s", len, key); + return keyCopy; +} int jsonIsObject(Json *json) { return json->type == JSON_TYPE_OBJECT; diff --git a/c/jsonschema.c b/c/jsonschema.c index e277ce4ae..f92abd562 100644 --- a/c/jsonschema.c +++ b/c/jsonschema.c @@ -141,7 +141,7 @@ typedef struct JSValueSpec_tag { } JSValueSpec; typedef struct AccessPathUnion_tag { - + int dummy; } AccessPathUnion; @@ -275,7 +275,7 @@ static bool validateJSONObject(JsonValidator *validator, for (int r=0; rrequiredCount; r++){ char *requiredProperty = valueSpec->required[r]; if (jsonObjectGetPropertyValue(object,requiredProperty) == NULL){ - noteValidityException(validator,12,"missing required property %s at %s", + noteValidityException(validator,12,"missing required property '%s' at '%s'", requiredProperty,validatorAccessPath(validator)); } } diff --git a/c/yaml2json.c b/c/yaml2json.c index 29afcb1c7..5faccccf3 100644 --- a/c/yaml2json.c +++ b/c/yaml2json.c @@ -5,6 +5,7 @@ #include #include #include +#include #ifdef NDEBUG #undef NDEBUG @@ -61,6 +62,26 @@ static char *getScalarStyleName(yaml_scalar_style_t style){ } } +static int yamlReadHandler(void *data, unsigned char *buffer, size_t size, size_t *size_read) { + FILE *fp = data; + int rc = 1; + size_t bytes_read = fread(buffer, 1, size, fp); + if (bytes_read > 0) { +#ifdef __ZOWE_OS_ZOS + if (__etoa_l((char *)buffer, bytes_read) == -1) { + fprintf (stderr, "failed to convert yaml input - %s\n", strerror(errno)); + rc = 0; + } +#endif + } + if (ferror(fp)) { + fprintf (stderr, "failed to read yaml input - %s\n", strerror(errno)); + rc = 0; + } + *size_read = bytes_read; + return rc; +} + yaml_document_t *readYAML(char *filename){ FILE *file; @@ -75,7 +96,7 @@ yaml_document_t *readYAML(char *filename){ assert(yaml_parser_initialize(&parser)); - yaml_parser_set_input_file(&parser, file); + yaml_parser_set_input(&parser, yamlReadHandler, file); printf("before parser_load\n");fflush(stdout); @@ -119,6 +140,16 @@ static char *getSequenceStyleName(yaml_sequence_style_t style){ #define SCALAR_SIZE_LIMIT 1024 +static void printYamlScalar(const yaml_node_t *node, bool eol) { + size_t dataLen = node->data.scalar.length; + int printLength = dataLen > SCALAR_SIZE_LIMIT ? 40 : (int)dataLen; + char val[printLength + 1]; + snprintf(val, printLength + 1, "%.*s", printLength, node->data.scalar.value); +#ifdef __ZOWE_OS_ZOS + __atoe(val); +#endif + printf("%s%c", val, eol ? '\n' : ''); +} static void pprintYAML1(yaml_document_t *doc, yaml_node_t *node, int depth){ @@ -133,9 +164,8 @@ static void pprintYAML1(yaml_document_t *doc, yaml_node_t *node, int depth){ { indent(depth); size_t dataLen = node->data.scalar.length; - int printLength = dataLen > SCALAR_SIZE_LIMIT ? 40 : (int)dataLen; - /* printf("printLength=%d dataLen=%lld\n",printLength,dataLen); */ - printf("Scalar: (len=%d) %*.*s\n",printLength,printLength,printLength,node->data.scalar.value); + printf("Scalar: (len=%d)", node->data.scalar.length); + printYamlScalar(node, true); } break; case YAML_SEQUENCE_NODE: @@ -155,6 +185,7 @@ static void pprintYAML1(yaml_document_t *doc, yaml_node_t *node, int depth){ } break; case YAML_MAPPING_NODE: + { yaml_node_pair_t *pair; indent(depth);printf("MapStart\n"); for (pair = node->data.mapping.pairs.start; pair < node->data.mapping.pairs.top; pair++) { @@ -168,7 +199,8 @@ static void pprintYAML1(yaml_document_t *doc, yaml_node_t *node, int depth){ size_t dataLen = keyNode->data.scalar.length; int printLength = dataLen > SCALAR_SIZE_LIMIT ? 40 : (int)dataLen; /* printf("printLength=%d dataLen=%lld\n",printLength,dataLen); */ - printf("%*.*s:\n",printLength,printLength,keyNode->data.scalar.value); + printYamlScalar(keyNode, false); + printf(":\n"); } } else { printf("dead end key\n"); @@ -181,6 +213,7 @@ static void pprintYAML1(yaml_document_t *doc, yaml_node_t *node, int depth){ } } indent(depth);printf("MapEnd\n"); + } break; default: printf("unexpected yaml node type %d\n",node->type); @@ -261,33 +294,43 @@ static Json *yaml2JSON1(JsonBuilder *b, Json *parent, char *parentKey, case YAML_DOUBLE_QUOTED_SCALAR_STYLE: case YAML_LITERAL_SCALAR_STYLE: case YAML_FOLDED_SCALAR_STYLE: + { + char val[valueLength+1]; char *tag = (char*)node->tag; - printf("tag = %s scalarStyle=%s\n",tag,getScalarStyleName(node->data.scalar.style)); + int tagLen = strlen(tag); + char tagBuf[tagLen + 1]; + snprintf(val, valueLength+1, "%.*s", valueLength, (const char *)node->data.scalar.value); + snprintf(tagBuf, tagLen + 1, "%.*s", tagLen, tag); +#ifdef __ZOWE_OS_ZOS + __atoe(val); + __atoe(tagBuf); +#endif + printf("tag = %s scalarStyle=%s\n",tagBuf,getScalarStyleName(node->data.scalar.style)); Json *scalar = NULL; // HERE, make test with float, int, bool, null, ddate - if (!strcmp(tag,YAML_NULL_TAG)){ - } else if (!strcmp(tag,YAML_NULL_TAG)){ + if (!strcmp(tagBuf,YAML_NULL_TAG)){ + } else if (!strcmp(tagBuf,YAML_NULL_TAG)){ /* Json *scalar = jsonBuildNull(b,parent,parentKey,&buildStatus); */ - } else if (!strcmp(tag,YAML_BOOL_TAG)){ + } else if (!strcmp(tagBuf,YAML_BOOL_TAG)){ /* Json *scalar = jsonBuildBool(b,parent,parentKey,"FOO",3,&buildStatus); */ - } else if (!strcmp(tag,YAML_INT_TAG) || - (!strcmp(tag,YAML_STR_TAG) && + } else if (!strcmp(tagBuf,YAML_INT_TAG) || + (!strcmp(tagBuf,YAML_STR_TAG) && (style == YAML_PLAIN_SCALAR_STYLE) && - isSyntacticallyInteger(node->data.scalar.value,valueLength))){ + isSyntacticallyInteger(val,valueLength))){ bool valid; - int64_t x = readInt(node->data.scalar.value,valueLength,&valid); + int64_t x = readInt(val,valueLength,&valid); if (valid){ scalar = jsonBuildInt64(b,parent,parentKey,x,&buildStatus); } else { buildStatus = JSON_FAIL_BAD_INTEGER; } /* Json *scalar = jsonBuildInt(b,parent,parentKey,"FOO",3,&buildStatus); */ - } else if (!strcmp(tag,YAML_STR_TAG)){ - scalar = jsonBuildString(b,parent,parentKey,(char*)node->data.scalar.value,valueLength,&buildStatus); - } else if (!strcmp(tag,YAML_FLOAT_TAG)){ + } else if (!strcmp(tagBuf,YAML_STR_TAG)){ + scalar = jsonBuildString(b,parent,parentKey,val,valueLength,&buildStatus); + } else if (!strcmp(tagBuf,YAML_FLOAT_TAG)){ printf("*** Warning don't know how to handle float yet\n"); buildStatus = JSON_FAIL_NOT_HANDLED; - } else if (!strcmp(tag,YAML_TIMESTAMP_TAG)){ + } else if (!strcmp(tagBuf,YAML_TIMESTAMP_TAG)){ printf("*** Warning don't know how to handle timestamp yet\n"); buildStatus = JSON_FAIL_NOT_HANDLED; } @@ -295,6 +338,7 @@ static Json *yaml2JSON1(JsonBuilder *b, Json *parent, char *parentKey, printf("*** WARNING *** Failed to add property/scalar err=%d\n",buildStatus); } return scalar; + } default: printf("*** WARNING *** - unknown scalar style %d",node->data.scalar.style); return NULL; @@ -346,9 +390,13 @@ static Json *yaml2JSON1(JsonBuilder *b, Json *parent, char *parentKey, printf("*** WARNING *** dead end key\n"); } if (key){ + char *keyBuf = jsonBuildKey(b, key, keyLength); +#ifdef __ZOWE_OS_ZOS + __atoe(keyBuf); +#endif yaml_node_t *valueNode = yaml_document_get_node(doc, pair->value); if (valueNode){ - yaml2JSON1(b,jsonObject,key,doc,valueNode,depth+2); + yaml2JSON1(b,jsonObject,keyBuf,doc,valueNode,depth+2); } else{ printf("*** WARNING *** dead end value\n"); } diff --git a/h/json.h b/h/json.h index 8d069bf99..620c13425 100644 --- a/h/json.h +++ b/h/json.h @@ -614,8 +614,7 @@ Json *jsonBuildNull(JsonBuilder *b, char *parentKey, int *errorCode); - - +char* jsonBuildKey(JsonBuilder *b, const char *key, int len); #endif /* __JSON__ */ diff --git a/platform/posix/psxregex.c b/platform/posix/psxregex.c index 10bedf2c9..ee33bbabc 100644 --- a/platform/posix/psxregex.c +++ b/platform/posix/psxregex.c @@ -1,4 +1,4 @@ - +#include "psxregex.h" regex_t *regexAlloc(){ return (regex_t*)safeMalloc(sizeof(regex_t),"regex_t"); @@ -6,5 +6,5 @@ regex_t *regexAlloc(){ void regexFree(regex_t *r){ regfree(r); - safefree((char*)r,sizeof(regex_t)); + safeFree((char*)r,sizeof(regex_t)); } diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 000000000..53d165b5c --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,69 @@ +################################################################################ +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. +################################################################################ + +# Makefile for z/OS +# make prepare && make +CC:=c89 +MAJOR:=0 +MINOR:=2 +PATCH:=5 +ENHANCED_ASCII:=-Wc,ASCII -D_ENHANCED_ASCII_EXT=0xFFFFFFFF +VERSION:=\"$(MAJOR).$(MINOR).$(PATCH)\" + +MAJOR:=0 +MINOR:=2 +PATCH:=5 +VERSION:=\"$(MAJOR).$(MINOR).$(PATCH)\" +DEFINESYAML:=-DYAML_VERSION_MAJOR=$(MAJOR) -DYAML_VERSION_MINOR=$(MINOR) -DYAML_VERSION_PATCH=$(PATCH) -DYAML_VERSION_STRING=$(VERSION) +DEFINES:=-D_XOPEN_SOURCE=600 -D_OPEN_THREADS=1 +LP64:=lp64 + +CC_FLAGS:=-Ilibyaml/include -I../h -I../platform/posix $(DEFINES) -Wc,dll,expo,langlvl\(extc99\),gonum,goff,hgpr,roconst,ASM,asmlib\('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN'\) -Wc,agg,list\(\),so\(\),off,xref,$(LP64) +LD_FLAGS:=-Wl,$(LP64) + +LIBYAMLOBJS:=api.o reader.o scanner.o parser.o loader.o writer.o emitter.o dumper.o +OBJS:=schematest.o yaml2json.o jsonschema.o json.o xlate.o charsets.o bpxskt.o logging.o collections.o timeutls.o timeutls.o utils.o alloc.o zosfile.o zos.o le.o scheduling.o recovery.o psxregex.o + +.PHONY: clean all prepare + +all: schematest + +schematest: $(OBJS) $(LIBYAMLOBJS) + $(CC) $(LD_FLAGS) -o $@ $^ + +schematest.o: schematest.c + $(CC) $(CC_FLAGS) -c schematest.c + +%.o: ../c/%.c + $(CC) $(CC_FLAGS) -c $< + +%.o: ../platform/posix/%.c + $(CC) $(CC_FLAGS) -c $< + +%.o: libyaml/src/%.c + $(CC) $(CC_FLAGS) $(DEFINESYAML) $(ENHANCED_ASCII) -c $< + +libyaml: + git clone git@github.com:yaml/libyaml.git + +prepare: libyaml + +clean: + rm -f schematest *.o + +################################################################################ +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. +################################################################################ \ No newline at end of file diff --git a/tests/schematest.c b/tests/schematest.c index b39b315ad..79fbec6bd 100644 --- a/tests/schematest.c +++ b/tests/schematest.c @@ -5,6 +5,9 @@ #include #include #include +#ifndef __ZOWE_OS_WINDOWS +#include +#endif #ifdef NDEBUG #undef NDEBUG From 78cda6535fe41a1a99c9ddc8c2eafa0da5b6b754 Mon Sep 17 00:00:00 2001 From: Leonty Chudinov Date: Fri, 4 Feb 2022 17:14:02 +0500 Subject: [PATCH 08/42] Update Makefile for 31-bit mode Signed-off-by: Leonty Chudinov --- tests/Makefile | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 53d165b5c..56536ac57 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -11,11 +11,14 @@ # Makefile for z/OS # make prepare && make CC:=c89 -MAJOR:=0 -MINOR:=2 -PATCH:=5 -ENHANCED_ASCII:=-Wc,ASCII -D_ENHANCED_ASCII_EXT=0xFFFFFFFF -VERSION:=\"$(MAJOR).$(MINOR).$(PATCH)\" +BITS:= +#BITS:=lp64 + +ifeq ($(BITS),lp64) + ENHANCED_ASCII:=-D_ENHANCED_ASCII_EXT=0xFFFFFFFF -Wc,ASCII +else + ENHANCED_ASCII:=-D_ENHANCED_ASCII_EXT=0xFFFFFFFF -Wc,ASCII,NOXPLINK +endif MAJOR:=0 MINOR:=2 @@ -23,10 +26,9 @@ PATCH:=5 VERSION:=\"$(MAJOR).$(MINOR).$(PATCH)\" DEFINESYAML:=-DYAML_VERSION_MAJOR=$(MAJOR) -DYAML_VERSION_MINOR=$(MINOR) -DYAML_VERSION_PATCH=$(PATCH) -DYAML_VERSION_STRING=$(VERSION) DEFINES:=-D_XOPEN_SOURCE=600 -D_OPEN_THREADS=1 -LP64:=lp64 -CC_FLAGS:=-Ilibyaml/include -I../h -I../platform/posix $(DEFINES) -Wc,dll,expo,langlvl\(extc99\),gonum,goff,hgpr,roconst,ASM,asmlib\('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN'\) -Wc,agg,list\(\),so\(\),off,xref,$(LP64) -LD_FLAGS:=-Wl,$(LP64) +CC_FLAGS:=-Ilibyaml/include -I../h -I../platform/posix $(DEFINES) -Wc,dll,expo,langlvl\(extc99\),gonum,goff,hgpr,roconst,ASM,asmlib\('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN'\) -Wc,agg,list\(\),so\(\),off,xref,$(BITS) +LD_FLAGS:=-Wl,$(BITS) LIBYAMLOBJS:=api.o reader.o scanner.o parser.o loader.o writer.o emitter.o dumper.o OBJS:=schematest.o yaml2json.o jsonschema.o json.o xlate.o charsets.o bpxskt.o logging.o collections.o timeutls.o timeutls.o utils.o alloc.o zosfile.o zos.o le.o scheduling.o recovery.o psxregex.o From b736f70ecabe06f0d56bd3c6fbb5949a94d9ea72 Mon Sep 17 00:00:00 2001 From: Leonty Chudinov Date: Fri, 4 Feb 2022 17:15:16 +0500 Subject: [PATCH 09/42] Remove printf Signed-off-by: Leonty Chudinov --- tests/schematest.c | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/schematest.c b/tests/schematest.c index 79fbec6bd..0fd1dcd71 100644 --- a/tests/schematest.c +++ b/tests/schematest.c @@ -56,7 +56,6 @@ int main(int argc, char *argv[]) char *charset = argv[3]; char *schemaFilename = (argc >= 5 ? argv[4] : NULL); int errorBufferSize = 1024; - printf("Sizeof(size_t) = %lld\n",sizeof(size_t)); #ifdef __ZOWE_OS_WINDOWS int stdoutFD = _fileno(stdout); #else From 226f2d01ae9d9d65e33a33cddd5382d40738ffad Mon Sep 17 00:00:00 2001 From: Leonty Chudinov Date: Mon, 7 Feb 2022 10:45:19 +0500 Subject: [PATCH 10/42] Remove printf Signed-off-by: Leonty Chudinov --- c/json.c | 1 - 1 file changed, 1 deletion(-) diff --git a/c/json.c b/c/json.c index 4d4f2b4f0..ffba0d3ad 100644 --- a/c/json.c +++ b/c/json.c @@ -1082,7 +1082,6 @@ void freeJsonParser(JsonParser *parser) { static void jsonParseFail(JsonParser *parser, char *formatString, ...) { - printf("JSON PARSE FAIL!!!!!\n"); fflush(stdout); if (parser->jsonError == NULL) { int size = 1024; From e5d1cb26f42deaf6b97007d0808504de3f358463 Mon Sep 17 00:00:00 2001 From: Leonty Chudinov Date: Mon, 7 Feb 2022 11:01:18 +0500 Subject: [PATCH 11/42] Refactoring Signed-off-by: Leonty Chudinov --- c/yaml2json.c | 76 +++++++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/c/yaml2json.c b/c/yaml2json.c index 5faccccf3..4d5ad7926 100644 --- a/c/yaml2json.c +++ b/c/yaml2json.c @@ -18,6 +18,20 @@ #include "json.h" #include "jsonschema.h" +static int convertToNative(char *buf, size_t size) { +#ifdef __ZOWE_OS_ZOS + return __atoe_l(buf, size); +#endif + return 0; +} + +static int convertFromNative(char *buf, size_t size) { +#ifdef __ZOWE_OS_ZOS + return __etoa_l(buf, size); +#endif + return 0; +} + static char *tokenTypeName(yaml_token_type_t type){ switch (type){ case YAML_NO_TOKEN: return "NO_TOKEN"; @@ -62,23 +76,21 @@ static char *getScalarStyleName(yaml_scalar_style_t style){ } } -static int yamlReadHandler(void *data, unsigned char *buffer, size_t size, size_t *size_read) { +static int yamlReadHandler(void *data, unsigned char *buffer, size_t size, size_t *sizeRead) { FILE *fp = data; int rc = 1; - size_t bytes_read = fread(buffer, 1, size, fp); - if (bytes_read > 0) { -#ifdef __ZOWE_OS_ZOS - if (__etoa_l((char *)buffer, bytes_read) == -1) { + size_t bytesRead = fread(buffer, 1, size, fp); + if (bytesRead > 0) { + if (convertFromNative((char *)buffer, bytesRead) == -1) { fprintf (stderr, "failed to convert yaml input - %s\n", strerror(errno)); rc = 0; } -#endif } if (ferror(fp)) { fprintf (stderr, "failed to read yaml input - %s\n", strerror(errno)); rc = 0; } - *size_read = bytes_read; + *sizeRead = bytesRead; return rc; } @@ -145,9 +157,7 @@ static void printYamlScalar(const yaml_node_t *node, bool eol) { int printLength = dataLen > SCALAR_SIZE_LIMIT ? 40 : (int)dataLen; char val[printLength + 1]; snprintf(val, printLength + 1, "%.*s", printLength, node->data.scalar.value); -#ifdef __ZOWE_OS_ZOS - __atoe(val); -#endif + convertToNative(val, printLength); printf("%s%c", val, eol ? '\n' : ''); } @@ -295,42 +305,40 @@ static Json *yaml2JSON1(JsonBuilder *b, Json *parent, char *parentKey, case YAML_LITERAL_SCALAR_STYLE: case YAML_FOLDED_SCALAR_STYLE: { - char val[valueLength+1]; + char nativeValue[valueLength+1]; char *tag = (char*)node->tag; int tagLen = strlen(tag); - char tagBuf[tagLen + 1]; - snprintf(val, valueLength+1, "%.*s", valueLength, (const char *)node->data.scalar.value); - snprintf(tagBuf, tagLen + 1, "%.*s", tagLen, tag); -#ifdef __ZOWE_OS_ZOS - __atoe(val); - __atoe(tagBuf); -#endif - printf("tag = %s scalarStyle=%s\n",tagBuf,getScalarStyleName(node->data.scalar.style)); + char nativeTag[tagLen + 1]; + snprintf(nativeValue, valueLength+1, "%.*s", valueLength, (const char *)node->data.scalar.value); + snprintf(nativeTag, tagLen + 1, "%.*s", tagLen, tag); + convertToNative(nativeValue, valueLength); + convertToNative(nativeTag, tagLen); + printf("tag = %s scalarStyle=%s\n",nativeTag,getScalarStyleName(node->data.scalar.style)); Json *scalar = NULL; // HERE, make test with float, int, bool, null, ddate - if (!strcmp(tagBuf,YAML_NULL_TAG)){ - } else if (!strcmp(tagBuf,YAML_NULL_TAG)){ + if (!strcmp(nativeTag,YAML_NULL_TAG)){ + } else if (!strcmp(nativeTag,YAML_NULL_TAG)){ /* Json *scalar = jsonBuildNull(b,parent,parentKey,&buildStatus); */ - } else if (!strcmp(tagBuf,YAML_BOOL_TAG)){ + } else if (!strcmp(nativeTag,YAML_BOOL_TAG)){ /* Json *scalar = jsonBuildBool(b,parent,parentKey,"FOO",3,&buildStatus); */ - } else if (!strcmp(tagBuf,YAML_INT_TAG) || - (!strcmp(tagBuf,YAML_STR_TAG) && + } else if (!strcmp(nativeTag,YAML_INT_TAG) || + (!strcmp(nativeTag,YAML_STR_TAG) && (style == YAML_PLAIN_SCALAR_STYLE) && - isSyntacticallyInteger(val,valueLength))){ + isSyntacticallyInteger(nativeValue,valueLength))){ bool valid; - int64_t x = readInt(val,valueLength,&valid); + int64_t x = readInt(nativeValue,valueLength,&valid); if (valid){ scalar = jsonBuildInt64(b,parent,parentKey,x,&buildStatus); } else { buildStatus = JSON_FAIL_BAD_INTEGER; } /* Json *scalar = jsonBuildInt(b,parent,parentKey,"FOO",3,&buildStatus); */ - } else if (!strcmp(tagBuf,YAML_STR_TAG)){ - scalar = jsonBuildString(b,parent,parentKey,val,valueLength,&buildStatus); - } else if (!strcmp(tagBuf,YAML_FLOAT_TAG)){ + } else if (!strcmp(nativeTag,YAML_STR_TAG)){ + scalar = jsonBuildString(b,parent,parentKey,nativeValue,valueLength,&buildStatus); + } else if (!strcmp(nativeTag,YAML_FLOAT_TAG)){ printf("*** Warning don't know how to handle float yet\n"); buildStatus = JSON_FAIL_NOT_HANDLED; - } else if (!strcmp(tagBuf,YAML_TIMESTAMP_TAG)){ + } else if (!strcmp(nativeTag,YAML_TIMESTAMP_TAG)){ printf("*** Warning don't know how to handle timestamp yet\n"); buildStatus = JSON_FAIL_NOT_HANDLED; } @@ -390,13 +398,11 @@ static Json *yaml2JSON1(JsonBuilder *b, Json *parent, char *parentKey, printf("*** WARNING *** dead end key\n"); } if (key){ - char *keyBuf = jsonBuildKey(b, key, keyLength); -#ifdef __ZOWE_OS_ZOS - __atoe(keyBuf); -#endif + char *keyNative = jsonBuildKey(b, key, keyLength); + convertToNative(keyNative, keyLength); yaml_node_t *valueNode = yaml_document_get_node(doc, pair->value); if (valueNode){ - yaml2JSON1(b,jsonObject,keyBuf,doc,valueNode,depth+2); + yaml2JSON1(b,jsonObject,keyNative,doc,valueNode,depth+2); } else{ printf("*** WARNING *** dead end value\n"); } From f37a06f5a408ba62e156344e33d71f40f2249e85 Mon Sep 17 00:00:00 2001 From: Leonty Chudinov Date: Mon, 7 Feb 2022 20:06:33 +0500 Subject: [PATCH 12/42] Improve error handling Signed-off-by: Leonty Chudinov --- c/yaml2json.c | 117 ++++++++++++++++++++++++++++++++------------- h/yaml2json.h | 2 +- tests/schematest.c | 4 +- 3 files changed, 87 insertions(+), 36 deletions(-) diff --git a/c/yaml2json.c b/c/yaml2json.c index 4d5ad7926..f3be701e2 100644 --- a/c/yaml2json.c +++ b/c/yaml2json.c @@ -94,44 +94,95 @@ static int yamlReadHandler(void *data, unsigned char *buffer, size_t size, size_ return rc; } +static void decodeParserError(yaml_parser_t *parser, char *errorBuf, size_t errorBufSize) { + switch (parser->error) { + case YAML_MEMORY_ERROR: + snprintf(errorBuf, errorBufSize, "YAML memory error: not enough memory for parsing"); + break; + case YAML_READER_ERROR: { + size_t problemLen = strlen(parser->problem); + char problemNative[problemLen + 1]; + snprintf (problemNative, problemLen + 1, "%s", parser->problem); + convertToNative(problemNative, problemLen); + if (parser->problem_value != -1) { + snprintf(errorBuf, errorBufSize, "YAML reader error: %s: #%X at %ld", problemNative, parser->problem_value, (long)parser->problem_offset); + } else { + snprintf(errorBuf, errorBufSize, "YAML reader error: %s at %ld", problemNative, (long)parser->problem_offset); + } + break; + } + case YAML_SCANNER_ERROR: + if (parser->context) { + snprintf(errorBuf, errorBufSize, "YAML scanner error: %s at line %d, column %d" + "%s at line %d, column %d\n", parser->context, + (int)parser->context_mark.line+1, (int)parser->context_mark.column+1, + parser->problem, (int)parser->problem_mark.line+1, + (int)parser->problem_mark.column+1); + } else { + snprintf(errorBuf, errorBufSize, "YAML scanner error: %s at line %d, column %d", + parser->problem, (int)parser->problem_mark.line+1, + (int)parser->problem_mark.column+1); + } + break; + case YAML_PARSER_ERROR: + if (parser->context) { + snprintf(errorBuf, errorBufSize, "YAML parser error: %s at line %d, column %d\n" + "%s at line %d, column %d", parser->context, + (int)parser->context_mark.line+1, (int)parser->context_mark.column+1, + parser->problem, (int)parser->problem_mark.line+1, + (int)parser->problem_mark.column+1); + } else { + snprintf(errorBuf, errorBufSize, "YAML parser error: %s at line %d, column %d", + parser->problem, (int)parser->problem_mark.line+1, + (int)parser->problem_mark.column+1); + } + break; + default: + snprintf(errorBuf, errorBufSize, "YAML parser: unknown error"); + } +} -yaml_document_t *readYAML(char *filename){ - FILE *file; - yaml_parser_t parser; - yaml_document_t *document = (yaml_document_t*)safeMalloc(sizeof(yaml_document_t),"YAML Doc"); - int done = 0; - int count = 0; - int error = 0; - - file = fopen(filename, "rb"); - assert(file); - - assert(yaml_parser_initialize(&parser)); - - yaml_parser_set_input(&parser, yamlReadHandler, file); - - printf("before parser_load\n");fflush(stdout); - - if (!yaml_parser_load(&parser, document)) { - - printf("bad yaml load\n");fflush(stdout); - error = 1; - } else if (yaml_document_get_root_node(document)){ +yaml_document_t *readYAML(const char *filename, char *errorBuf, size_t errorBufSize) { + FILE *file = NULL; + yaml_document_t *document = NULL; + yaml_parser_t parser = {0}; + bool done = false; - } else { - error = 1; - } + do { + if (!(document = (yaml_document_t*)safeMalloc(sizeof(yaml_document_t), "YAML Doc"))) { + snprintf(errorBuf, errorBufSize, "failed to alloc memory for YAML doc"); + break; + } + memset(document, 0, sizeof(document)); + if (!(file = fopen(filename, "rb"))) { + snprintf(errorBuf, errorBufSize, "failed to read '%s' - %s", filename, strerror(errno)); + break; + } + if (!yaml_parser_initialize(&parser)) { + snprintf(errorBuf, errorBufSize, "failed to initialize YAML parser"); + break; + } + yaml_parser_set_input(&parser, yamlReadHandler, file); + if (!yaml_parser_load(&parser, document)) { + decodeParserError(&parser, errorBuf, errorBufSize); + break; + } + if (!yaml_document_get_root_node(document)){ + snprintf(errorBuf, errorBufSize, "failed to get root node in YAML '%s'", filename); + break; + } + done = true; + } while (0); - printf("before parser_delete\n");fflush(stdout); - yaml_parser_delete(&parser); - printf("after parser_delete\n");fflush(stdout); - assert(!fclose(file)); - - if (error){ - safeFree((char*)document,sizeof(yaml_document_t)); + if (!done && document) { + yaml_document_delete(document); + safeFree((char*)document, sizeof(yaml_document_t)); document = NULL; } - + if (file) { + fclose(file); + } + yaml_parser_delete(&parser); return document; } diff --git a/h/yaml2json.h b/h/yaml2json.h index 6b42c89e8..7af751201 100644 --- a/h/yaml2json.h +++ b/h/yaml2json.h @@ -3,7 +3,7 @@ #include "yaml.h" -yaml_document_t *readYAML(char *filename); +yaml_document_t *readYAML(const char *filename, char *errorBuf, size_t errorBufSize); void pprintYAML(yaml_document_t *document); Json *yaml2JSON(yaml_document_t *document, ShortLivedHeap *slh); diff --git a/tests/schematest.c b/tests/schematest.c index 0fd1dcd71..e87c9ee32 100644 --- a/tests/schematest.c +++ b/tests/schematest.c @@ -87,13 +87,13 @@ int main(int argc, char *argv[]) } else if (!strcmp(syntax,"yaml")){ printf("yaml syntax case\n"); fflush(stdout); - yaml_document_t *doc = readYAML(filename); + yaml_document_t *doc = readYAML(filename, errorBuffer, errorBufferSize); printf("yaml doc at 0x%p\n",doc); if (doc){ pprintYAML(doc); json = yaml2JSON(doc,slh); } else { - printf("yaml probably had syntax issues\n"); + printf("%s\n", errorBuffer); } } else { printf("unhandled syntax\n"); From cd35cb307662b92f9f80b8ec37dde7b129e4b119 Mon Sep 17 00:00:00 2001 From: Joe Date: Sun, 13 Feb 2022 12:39:41 -0500 Subject: [PATCH 13/42] Fix a few bugs in fileGetChar and directoryReading Signed-off-by: Joe --- c/winskt.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/c/winskt.c b/c/winskt.c index 095cc1e8f..ec30c30dd 100644 --- a/c/winskt.c +++ b/c/winskt.c @@ -920,7 +920,7 @@ int fileGetChar(UnixFile *file, int *returnCode, int *reasonCode){ *reasonCode = 0xBFF; return -1; } else if (file->bufferPos < file->bufferFill){ - return (int)(file->buffer[file->bufferPos++]); + return (int)(file->buffer[file->bufferPos++])&0xFF; } else if (file->eofKnown){ return -1; } else{ @@ -932,7 +932,7 @@ int fileGetChar(UnixFile *file, int *returnCode, int *reasonCode){ } file->bufferFill = bytesRead; file->bufferPos = 1; - return file->buffer[0]; + return file->buffer[0]&0xFF; } else{ return -1; } @@ -1040,7 +1040,10 @@ int directoryRead(UnixFile *directory, char *entryBuffer, int entryBufferLength, if (directory->hasMoreEntries){ int filenameLength = strlen(findData->cFileName); - printf("found %s\n",findData->cFileName); + DirectoryEntry *entry = (DirectoryEntry*)entryBuffer; + entry->entryLength = filenameLength+2; + entry->nameLength = filenameLength; + memcpy(entry->name,findData->cFileName,filenameLength); if (FindNextFile(hFind, findData)){ directory->hasMoreEntries = TRUE; } else { From dc80cdd746476fd437ca71260f4c2e5e8e5baea1 Mon Sep 17 00:00:00 2001 From: Joe Date: Sun, 13 Feb 2022 12:41:36 -0500 Subject: [PATCH 14/42] Add convenience 'ArrayList' data structure for many non-performance-criticial uses Signed-off-by: Joe --- c/collections.c | 78 +++++++++++++++++++++++++++++++++++++++++++++++++ h/collections.h | 25 +++++++++++++++- 2 files changed, 102 insertions(+), 1 deletion(-) diff --git a/c/collections.c b/c/collections.c index 608d70219..4abc9bdd1 100644 --- a/c/collections.c +++ b/c/collections.c @@ -1417,6 +1417,84 @@ void *qRemove(Queue *q){ #endif /* END OF OS-VARIANT Queue stuff */ +/* The Array List (flexible Array thing that smells like Java and javascript) */ + +static void *arrayListAlloc(ArrayList *list, uint32_t size, char *location){ + if (list->slh){ + return (void*)SLHAlloc(list->slh,size); + } else{ + return safeMalloc(size,location); + } +} + +ArrayList *makeArrayList(){ + ArrayList *list = (ArrayList*)safeMalloc(sizeof(ArrayList),"ArrayList"); + list->capacity = 8; + list->size = 0; + list->array = (void**)safeMalloc(list->capacity*sizeof(void*),"ArrayListArray"); + list->slh = NULL; + return list; +} + +void arrayListFree(ArrayList *list){ + if (list->slh == NULL){ + safeFree((char*)list->array,list->capacity*sizeof(void*)); + safeFree((char*)list,sizeof(ArrayList)); + } +} + +void initEmbeddedArrayList(ArrayList *list, + ShortLivedHeap *slh){ /* can be null for use safeMalloc - standard heap alloc */ + list->capacity = 8; + list->size = 0; + list->slh = slh; + list->array = (void**)arrayListAlloc(list,list->capacity*sizeof(void*),"ArrayListArray"); +} + +void arrayListAdd(ArrayList *list, void *thing){ + if (list->size == list->capacity){ + int newCapacity = 2*list->capacity; + void** newArray = (void**)arrayListAlloc(list,newCapacity*sizeof(void*),"ArrayListExtend"); + memcpy(newArray,list->array,list->capacity*sizeof(void*)); + if (list->slh == NULL){ + safeFree((char*)list->array,list->capacity*sizeof(void*)); + } + list->array = newArray; + list->capacity = newCapacity; + } + list->array[list->size++] = thing; +} + +void arrayListSort(ArrayList *list, int (*comparator)(const void *a, const void *b)){ + qsort(list->array,list->size,sizeof(void*),comparator); +} + +bool arrayListContains(ArrayList *list, void *element){ + for (int i=0; isize; i++){ + if (list->array[i] == element){ + return true; + } + } + return false; +} + +void *arrayListElement(ArrayList *list, int i){ + if (isize){ + return list->array[i]; + } else{ + return NULL; + } +} + +void *arrayListShallowCopy(ArrayList *source, ArrayList *target){ + target->capacity = source->capacity; + target->size = source->size; + target->slh = source->slh; + target->array = (void**)arrayListAlloc(target,target->capacity*sizeof(void*),"ArrayListArray"); + memcpy(target->array,source->array,target->capacity*sizeof(void**)); + return target; +} + /* This program and the accompanying materials are diff --git a/h/collections.h b/h/collections.h index 406b3be0b..83be11c77 100644 --- a/h/collections.h +++ b/h/collections.h @@ -20,6 +20,7 @@ */ #include "zowetypes.h" +#include "utils.h" #ifndef __ZOWE_OS_ZOS #include "openprims.h" @@ -50,6 +51,13 @@ #define makeQueue MAKELCFQ #define destroyQueue DSTRLCFQ +#define makeArrayList ALSTMAKE +#define arrayListAdd ALSTADD +#define arrayListElement ALSTELMT +#define arrayListSort ALSTSORT +#define arrayListShallowCopy ALSHLCPY +#define initEmbbededArrayList ALINEMAR + #endif typedef struct fixedBlockMgr_tag{ @@ -307,8 +315,23 @@ void qInsert(Queue *q, void *newData) QueueAmode64; */ void *qRemove(Queue *q) QueueAmode64; -#endif +typedef struct ArrayList_tag{ + int capacity; + int size; + void **array; + ShortLivedHeap *slh; +} ArrayList; + +ArrayList *makeArrayList(); +void arrayListFree(ArrayList *list); +void arrayListAdd(ArrayList *list, void *thing); +void *arrayListElement(ArrayList *list, int i); +void initEmbeddedArrayList(ArrayList *list, + ShortLivedHeap *slh); +void *arrayListShallowCopy(ArrayList *source, ArrayList *target); +void arrayListSort(ArrayList *list, int (*comparator)(const void *a, const void *b)); +#endif /* This program and the accompanying materials are From deeaf4e131ab46724644e4f4378bf810aa6026f7 Mon Sep 17 00:00:00 2001 From: Joe Date: Sun, 13 Feb 2022 12:43:54 -0500 Subject: [PATCH 15/42] Adding facility to xlate called CharsetOracle that allows callers to guess the charset of an untagged text file Signed-off-by: Joe --- c/xlate.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ h/xlate.h | 15 +++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/c/xlate.c b/c/xlate.c index 6c4b2d8f7..16021118f 100644 --- a/c/xlate.c +++ b/c/xlate.c @@ -20,6 +20,8 @@ #include #endif +#include "zowetypes.h" +#include "alloc.h" #include "xlate.h" typedef struct { @@ -78,6 +80,48 @@ char* a2e(char *buffer, int len) } +CharsetOracle *makeCharsetOracle(){ + CharsetOracle *oracle = (CharsetOracle*)safeMalloc(sizeof(CharsetOracle),"CharsetOracle"); + memset(oracle,0,sizeof(CharsetOracle)); + return oracle; +} + +void freeCharsetOracle(CharsetOracle *oracle){ + safeFree((char*)oracle,sizeof(CharsetOracle)); +} + +void charsetOracleDigest(CharsetOracle *oracle, char *s, int len){ + for (int i=0; irangeBuckets[(c&0xF0)>>4]++; + oracle->charBuckets[c&0xff]++; + } + oracle->total += len; +} + +#define min(a,b) (((a) < (b)) ? (a) : (b)) +#define max(a,b) (((a) > (b)) ? (a) : (b)) + +int guessCharset(CharsetOracle *oracle, double *confidence){ + /* should also take variety of character */ + int *buckets = oracle->rangeBuckets; + int sub20 = buckets[0] + buckets[1]; + int twenties = buckets[2]; + int thirties = buckets[3]; + int range40To7F = buckets[4] + buckets[5] + buckets[6] + buckets[7]; + int low = sub20 + twenties + thirties + range40To7F; + int total = oracle->total; + int range80ToFF = total-low; + + double lengthConfidence = min(1.0,((double)(oracle->total-50))/50); + *confidence = lengthConfidence; + if (range40To7F > range80ToFF){ + return CHARSET_ORACLE_UTF_FAMILY; + } else { + return CHARSET_ORACLE_EBCDIC_FAMILY; + } +} + /* This program and the accompanying materials are made available under the terms of the Eclipse Public License v2.0 which accompanies diff --git a/h/xlate.h b/h/xlate.h index 291d28b1d..7199aa3c4 100644 --- a/h/xlate.h +++ b/h/xlate.h @@ -70,6 +70,21 @@ extern "C" char* e2a(char *buffer, int len); char* a2e(char *buffer, int len); + +typedef struct CharsetOracle_tag { + int total; + int charBuckets[0x100]; + int rangeBuckets[0x10]; +} CharsetOracle; + +#define CHARSET_ORACLE_EBCDIC_FAMILY 0x10000 +#define CHARSET_ORACLE_UTF_FAMILY 0x20000 + +CharsetOracle *makeCharsetOracle(); + +void freeCharsetOracle(CharsetOracle *oracle); +void charsetOracleDigest(CharsetOracle *oracle, char *s, int len); +int guessCharset(CharsetOracle *oracle, double *confidence); #ifdef __cplusplus } From 074d0bd14dbbadf61b61fa638280ee927c4e232f Mon Sep 17 00:00:00 2001 From: Joe Date: Sun, 13 Feb 2022 12:49:37 -0500 Subject: [PATCH 16/42] Adding a unit tester for the CharsetOracle in xlate.c Signed-off-by: Joe --- tests/charsetguesser.c | 78 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 tests/charsetguesser.c diff --git a/tests/charsetguesser.c b/tests/charsetguesser.c new file mode 100644 index 000000000..e1d2b0bff --- /dev/null +++ b/tests/charsetguesser.c @@ -0,0 +1,78 @@ +#include +/* #include */ + +#include +#include +#include +#include + +#ifdef NDEBUG +#undef NDEBUG +#endif +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "unixfile.h" +#include "json.h" +#include "xlate.h" + + +/* + Notes: + + Windows Build ______________________________ + + Assuming clang is installed + + clang -I%YAML%/include -I./src -I../h -I ../platform/windows -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -o charsetguesser.exe charsetguesser.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c + + ZOS Build _________________________________ + + + + Running the Test ________________________________ + + charsetguesser + + */ + +int main(int argc, char *argv[]) +{ + char *filename = argv[1]; + UnixFile *in = NULL; + int maxChars = 1000; + int returnCode = 0; + int reasonCode = 0; + + + in = fileOpen(filename, FILE_OPTION_READ_ONLY, 0, 1024, &returnCode, &reasonCode); + /* int status = (in ? fileInfo(filename, &info, &returnCode, &reasonCode) : 12); */ + if (in != NULL){ /* && status == 0) { */ + CharsetOracle *oracle = makeCharsetOracle(); + + for (int i=0; i maxChars){ + break; + } + char c = (char)(b&0xff); + charsetOracleDigest(oracle,&c,1); + } + fileClose(in,&returnCode,&reasonCode); + double confidence; + int charset = guessCharset(oracle,&confidence); + printf("charset guess=0x%08x, confidence=%f\n",charset,confidence); + for (int i=0; i<16; i++){ + printf(" 0x%02X: %d\n",i<<4,oracle->rangeBuckets[i]); + } + freeCharsetOracle(oracle); + } else { + printf("could not open %s ret=%d reason=0x%x\n",filename,returnCode,reasonCode); + } + + return 0; +} + From 9ae0716e1b8ebf902f5e9f37d566f3d21cf8f664 Mon Sep 17 00:00:00 2001 From: Joe Date: Sun, 13 Feb 2022 12:52:27 -0500 Subject: [PATCH 17/42] Adding much implementation to config manager including a CLI and fixing bugs in json in yaml exposed by the new code. An interim commit. This feature branch is not quite done. Signed-off-by: Joe --- c/configmgr.c | 786 ++++++++++++++++++++++++++++++++++++++++++++++++- c/jsonschema.c | 28 +- c/yaml2json.c | 32 +- h/json.h | 1 + h/jsonschema.h | 1 + 5 files changed, 816 insertions(+), 32 deletions(-) diff --git a/c/configmgr.c b/c/configmgr.c index d38fa8049..e9f9cc377 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -50,12 +50,20 @@ #include "bpxnet.h" #include "unixfile.h" #include "json.h" +#include "jsonschema.h" +#include "yaml.h" +#include "yaml2json.h" #include "charsets.h" +#include "collections.h" #ifdef __ZOWE_OS_WINDOWS typedef int64_t ssize_t; +#include "winregex.h" +#else +#include "psxregex.h" /* works on ZOS and Linux */ #endif + /* A configuration manager is a facility providing access to a set of configuration source (most likely textual) in many formats in one or more locations whose data definition is provided by a set of JSON Schema Definions. @@ -77,26 +85,496 @@ typedef int64_t ssize_t; https://datatracker.ietf.org/doc/html/draft-bhutton-json-schema-validation) + set YAML=c:\repos\libyaml ## Wherever you git clone'd libyaml + + clang++ -c ../platform/windows/cppregex.cpp ../platform/windows/winregex.cpp + + clang -I%YAML%/include -I./src -I../h -I ../platform/windows -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 -o configmgr.exe configmgr.c %YAML%/src/api.c %YAML%/src/reader.c %YAML%/src/scanner.c %YAML%/src/parser.c %YAML%/src/loader.c %YAML%/src/writer.c %YAML%/src/emitter.c %YAML%/src/dumper.c ../c/yaml2json.c ../c/jsonschema.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c cppregex.o winregex.o + + configmgr "../tests/schemadata" "" "LIBRARY(FOO):DIR(BAR)" yak + + configmgr -s "../tests/schemadata" -p "LIBRARY(FOO):DIR(BAR)" extract "/foo/bar" + + */ -#define CONFIG_PATH_OMVS_DIR 0x0001 -#define CONFIG_PATH_MVS_LIBRAY 0x0002 +#define CONFIG_PATH_OMVS_FILE 0x0001 +#define CONFIG_PATH_OMVS_DIR 0x0002 +#define CONFIG_PATH_OMVS_LIBRARY 0x0004 + typedef struct ConfigPathElement_tag { int flags; char *name; + char *data; struct ConfigPathElement_tag *next; } ConfigPathElement; typedef struct ConfigManager_tag { ShortLivedHeap *slh; - ConfigPathElement *schemaPath; - configPathElement *configPath; + char *rootSchemaDirectory; + ConfigPathElement *schemaPath; /* maybe */ + ConfigPathElement *configPath; + JsonSchema *topSchema; + Json *config; hashtable *schemaCache; hashtable *configCache; + int traceLevel; + FILE *traceOut; } ConfigManager; +#define INFO 0 +#define DEBUG 1 +#define DEBUG2 2 + +static void trace(ConfigManager *mgr, int level, char *formatString, ...){ + if (mgr->traceLevel >= level){ + va_list argPointer; + va_start(argPointer,formatString); + vfprintf(mgr->traceOut,formatString,argPointer); + va_end(argPointer); + } +} + +/* configlet semantcs *IS* javascript semantics (and-or-not scalars) + Configs live in product read-only directories + + configrunner is the new ZWE + + */ + +typedef struct ConfigletParm_tag { + int type; // int,string + char *name; +} ConfigletParm; + +typedef struct ConfigVersion_tag { + int major; + int minor; + int patch; +} ConfigVersion; + + + +#define CONFIG_ACTION_MKDIR 0x0001 +#define CONFIG_ACTION_CHMOD 0x0002 +#define CONFIG_ACTION_RMDIR 0x0003 +#define CONFIG_ACTION_RM 0x0004 + +#define CONFIG_ACTION_SCRIPT_STEP 0x0005 + +#define CONFIG_ACTION_SET_LOGICAL_DIR 0x0100 + +#define CONFIG_SERIAL_COMPOUND_ACTION +#define CONFIG_PARALLEL_COMPOUND_ACTION + +#define CONFIG_ACTION_UNPAX 0x0200 + +/* An action is an expression that + 1) returns status information + 2) logs the fact that it ran and status with args + + + Only Actions change the status of the system. + + Expressions are exactly what they sound like + 1) Functional, stateless + 2) Arith, logic, + +*/ + + +typedef struct ConfigAction_tag { + int type; + ArrayList parameters; /* ConfigletParms's. Formal Parameters, not values */ +} ConfigAction; + +typedef struct ConfigPrerequisite_tag { + char *name; + ConfigVersion *minVersion; +} ConfigPrerequisite; + +typedef struct Configlet_tag { + char *fullyQualifiedName; // dotted namespace, like org.zowe... + char *displayName; + char *schemaFilename; + ConfigVersion *version; + ArrayList prerequisites; +} Configlet; + + + +/* + zwe install doesn't set up to clone well HA + + targets: + zowe_early + zowe 1.X_to_2.0 migrate + zss_install + + Scenario 1 + + Optionally Get Pax into install (unpax) dir + + build instance dir if not there + copy files into instance directory + build instance yaml + + Saturday + What do these do + Zwe install carrying stuff into the right places (node pre-req-no-code) + Zwe init need these post-install or post-SMPE + + Data first, for config + + QuickJS + - Native Packed Decimal + +*/ + +static char *substring(ConfigManager *mgr, char *s, int start, int end){ + int len = end-start; + char *sub = (mgr ? SLHAlloc(mgr->slh,len+1) : safeMalloc(len+1,"ConfigMgr.substring")); + memcpy(sub,s+start,len); + sub[len] = 0; + return sub; +} + +static char *makeFullPath(ConfigManager *mgr, char *path, char *simpleFilename, bool useSLH){ + int pathLen = strlen(path); + int fileLen = strlen(simpleFilename); + int len = pathLen + fileLen + 4; + bool needsSlash = path[pathLen-1] != '/'; + char *result = (useSLH ? SLHAlloc(mgr->slh, len) : malloc(len)); + snprintf(result,len,"%s%s%s", + path,(needsSlash ? "/" : ""),simpleFilename); + return result; +} + +static char *extractMatch(ConfigManager *mgr, char *s, regmatch_t *match){ + return substring(mgr,s,(int)match->rm_so,(int)match->rm_eo); +} + +ConfigPathElement *makePathElement(ConfigManager *mgr, int flags, char *name, char *data){ + ConfigPathElement *element = (ConfigPathElement*)SLHAlloc(mgr->slh,sizeof(ConfigPathElement)); + memset(element,0,sizeof(ConfigPathElement)); + element->flags = flags; + element->name = name; + element->data = data; + if (mgr->configPath == NULL){ + mgr->configPath = element; + } else { + ConfigPathElement *tail = mgr->configPath; + while (tail->next){ + tail = tail->next; + } + tail->next = element; + } + return element; +} + +static bool addPathElement(ConfigManager *mgr, char *pathElementArg){ + ConfigPathElement *element = (ConfigPathElement*)SLHAlloc(mgr->slh,sizeof(ConfigPathElement)); + memset(element,0,sizeof(ConfigPathElement)); + + regmatch_t matches[10]; + regex_t *argPattern = regexAlloc(); + /* Nice Regex test site */ + char *pat = "^(LIBRARY|DIR|FILE)\\((\\S+)\\)$"; + int compStatus = regexComp(argPattern,pat,REG_EXTENDED); + if (compStatus != 0){ + trace(mgr,INFO,"Internal error, pattern compilation failed\n"); + return false; + } + if (!strcmp(pathElementArg,"PARMLIBS")){ + /* need ZOS impl */ + printf("implement me\n"); + return false; + } else if (regexExec(argPattern,pathElementArg,10,matches,0) == 0){ + char *elementTypeName = extractMatch(mgr,pathElementArg,&matches[1]); + char *elementData = extractMatch(mgr,pathElementArg,&matches[2]); + int flags = 0; + if (!strcmp(elementTypeName,"LIBRARY")){ + flags = CONFIG_PATH_OMVS_LIBRARY; + } else if (!strcmp(elementTypeName,"DIR")){ + flags = CONFIG_PATH_OMVS_DIR; + } else if (!strcmp(elementTypeName,"FILE")){ + flags = CONFIG_PATH_OMVS_FILE; + + } else { + return false; /* internal logic error */ + } + makePathElement(mgr,flags,pathElementArg,elementData); + return true; + } else { + trace(mgr,DEBUG,"unhandled path element '%s'\n",pathElementArg); + return false; + } + +} + +static int buildConfigPath(ConfigManager *mgr, char *configPathArg){ + int pos = 0; + int len = strlen(configPathArg); + while (pos < len){ + int nextColon = indexOf(configPathArg,len,':',pos); + int nextPos; + int pathElementLength = 0; + trace(mgr,DEBUG2,"nextColon = %d\n",nextColon); + + if (nextColon == -1){ + nextPos = len; + pathElementLength = len - pos; + } else { + nextPos = nextColon+1; + pathElementLength = nextColon - pos; + } + + trace(mgr,DEBUG2,"ckpt.2 nextPos=%d\n",nextPos); + + char *pathElementArg = substring(mgr,configPathArg,pos,pos+pathElementLength); + + trace(mgr,DEBUG2,"ckpt.3\n"); + + bool status = addPathElement(mgr,pathElementArg); + + if (status == false){ + trace(mgr,INFO,"path building failed\n"); + return 12; + } + pos = nextPos; + } + return 0; +} + +static void printConfigPath(ConfigManager *mgr){ + /* Thurs AM */ + trace(mgr,INFO,"Path Elements: mgr=0x%p\n",mgr); + ConfigPathElement *element = mgr->configPath; + while (element){ + trace(mgr,INFO," %04X %s\n",element->flags,element->name); + element = element->next; + } +} + +#define ZOWE_SCHEMA_FILE "zoweyaml.schema" + +void freeConfigManager(ConfigManager *mgr); + +ConfigManager *makeConfigManager(char *configPathArg, char *rootSchemaDirectory, + int traceLevel, FILE *traceOut){ + ConfigManager *mgr = (ConfigManager*)safeMalloc(sizeof(ConfigManager),"ConfigManager"); +#ifdef __ZOWE_OS_WINDOWS + int stdoutFD = _fileno(stdout); + int stderrFD = _fileno(stdout); +#else + int stdoutFD = STDOUT_FILENO; + int stderrFD = STDERR_FILENO; +#endif + + memset(mgr,0,sizeof(ConfigManager)); + mgr->traceLevel = traceLevel; + mgr->traceOut = traceOut; + mgr->slh = makeShortLivedHeap(0x10000,0x100); + if (buildConfigPath(mgr,configPathArg)){ + printf("built config path failed\n");fflush(stdout); + safeFree((char*)mgr,sizeof(ConfigManager)); + return NULL; + } + trace(mgr,DEBUG,"built config path\n"); + + mgr->rootSchemaDirectory = rootSchemaDirectory; + char *zoweYamlPath = makeFullPath(mgr,rootSchemaDirectory,ZOWE_SCHEMA_FILE,true); + int returnCode = 0; + int reasonCode = 0; + FileInfo yamlFileInfo; + int infoStatus = fileInfo(zoweYamlPath,&yamlFileInfo,&returnCode,&reasonCode); + if (infoStatus){ + freeConfigManager(mgr); + return NULL; + } + char errorBuffer[1024]; + Json *jsonWithSchema = jsonParseFile2(mgr->slh,zoweYamlPath,errorBuffer,1024); + if (jsonWithSchema == NULL){ + trace(mgr,INFO,"failed to read JSON with base schema: %s\n",errorBuffer); + freeConfigManager(mgr); + return NULL; + } + if (mgr->traceLevel >= 1){ + jsonPrinter *p = makeJsonPrinter(mgr->traceOut == stdout ? stdoutFD : stderrFD); + jsonEnablePrettyPrint(p); + jsonPrint(p,jsonWithSchema); + trace(mgr,INFO,"\n"); + } + JsonSchemaBuilder *builder = makeJsonSchemaBuilder(DEFAULT_JSON_SCHEMA_VERSION); + JsonSchema *schema = jsonBuildSchema(builder,jsonWithSchema); + if (schema == NULL){ + printf("Schema Build Failed: %s\n",builder->errorMessage); + freeJsonSchemaBuilder(builder); + freeConfigManager(mgr); + return NULL; + } else { + trace(mgr,INFO,"JSON Schema built successfully\n"); + mgr->topSchema = schema; + } + freeJsonSchemaBuilder(builder); + /* + Thurs read and merge from multiple sources + */ + return mgr; +} + +static Json *readJson(ConfigManager *mgr, ConfigPathElement *pathElement){ + if (pathElement->flags & CONFIG_PATH_OMVS_FILE){ + trace(mgr,DEBUG,"before read YAML file=%s\n",pathElement->data); + yaml_document_t *doc = readYAML(pathElement->data); + trace(mgr,DEBUG,"yaml doc at 0x%p\n",doc); + if (doc){ + if (mgr->traceLevel >= 1){ + pprintYAML(doc); + } + return yaml2JSON(doc,mgr->slh); + } else { + trace(mgr,INFO,"WARNING, yaml read failed\n"); + return NULL; + } + + } else { + trace(mgr,INFO,"WARNING, only simple file case yet implemented\n"); + return NULL; + } +} + +/* need to collect violations as this goes */ +static void overloadConfiguration(ConfigManager *mgr, + ConfigPathElement *pathElement, + ConfigPathElement *pathTail){ + if (pathTail == NULL){ + printf("at end of config path \n");fflush(stdout); + mgr->config = readJson(mgr,pathElement); + printf("mgr->config = 0x%p\n",mgr->config); + } else { + Json *overlay = NULL; + /* overloadPathElement(pathElement); */ + } +} + +void loadConfigurations(ConfigManager *mgr){ + ConfigPathElement *pathElement = mgr->configPath; + overloadConfiguration(mgr,pathElement,pathElement->next); +} + +void freeConfigManager(ConfigManager *mgr){ + SLHFree(mgr->slh); + safeFree((char*)mgr,sizeof(ConfigManager)); +} + +#define JSON_POINTER_NORMAL_KEY 1 +#define JSON_POINTER_INTEGER 2 + +typedef struct JsonPointerElement_tag { + char *string; + int type; +} JsonPointerElement; + +typedef struct JsonPointer_tag { + ArrayList elements; +} JsonPointer; + +static JsonPointerElement *makePointerElement(char *token, int type){ + JsonPointerElement *element = (JsonPointerElement*)safeMalloc(sizeof(JsonPointerElement),"JsonPointerElement"); + element->string = token; + element->type = type; + return element; +} + +/* See RFC 6901 */ +JsonPointer *parseJsonPointer(char *s){ + int len = strlen(s); + /* char *reduced = safeMalloc(len+1); */ + JsonPointer *jp = (JsonPointer*)safeMalloc(sizeof(JsonPointer),"JSONPointer"); + ArrayList *list = &(jp->elements); + initEmbeddedArrayList(list,NULL); + if (len == 0){ + return jp; + } + int pos = 1; + while (pos < len){ + int nextSlash = indexOf(s,len,'/',pos); + int end = (nextSlash == -1 ? len : nextSlash); + int tokenLen = end-pos+1; + char *token = safeMalloc(tokenLen,"JSON Ptr Token"); + memset(token,0,tokenLen); + /* Step 2: reduce the following digraphs + ~0 -> '~' + ~1 -> '/' + */ + bool pendingTilde = false; + int tPos = 0; + bool allDigits = true; + for (int i=pos; i '9'){ + allDigits = false; + } + if (pendingTilde){ + if (c == '0'){ + token[tPos++] = '~'; + } else if (c == '1'){ + token[tPos++] = '/'; + } else { + token[tPos++] = '~'; + token[tPos++] = c; + } + } else { + if (i == '~'){ + pendingTilde = true; + } else { + token[tPos++] = c; + } + } + } + if (pendingTilde){ + token[tPos++] = '~'; + } + + arrayListAdd(list, + makePointerElement(token, + (allDigits ? JSON_POINTER_INTEGER : JSON_POINTER_NORMAL_KEY))); + + if (nextSlash == -1){ + break; + } else { + pos = nextSlash + 1; + } + } + return jp; +} + +void printJsonPointer(FILE *out, JsonPointer *jp){ + ArrayList *list = &jp->elements; + fprintf(out,"JSON Pointer:\n"); + for (int i=0; isize; i++){ + JsonPointerElement *element = (JsonPointerElement*)arrayListElement(list,i); + fprintf(out," 0x%04X: %s\n",element->type,element->string); + } +} + +#define JSON_POINTER_TOO_DEEP 101 +#define JSON_POINTER_ARRAY_INDEX_NOT_INTEGER 102 +#define JSON_POINTER_ARRAY_INDEX_OUT_OF_BOUNDS 103 + +#define ZCFG_ALLOC_HEAP 1 +#define ZCFG_ALLOC_SLH 2 +#define ZCFG_ALLOC_MEMCPY 3 + +/* These eventually need ZWE unique messages */ #define ZCFG_SUCCESS 0 +#define ZCFG_TYPE_MISMATCH 1 +#define ZCFG_POINTER_TOO_DEEP JSON_POINTER_TOO_DEEP +#define ZCFG_POINTER_ARRAY_INDEX_NOT_INTEGER JSON_POINTER_ARRAY_INDEX_NOT_INTEGER +#define ZCFG_POINTER_ARRAY_INDEX_OUT_OF_BOUNDS JSON_POINTER_ARRAY_INDEX_OUT_OF_BOUNDS + /* Merging notes defaulting and merging requires the same data shape at all levels and sources @@ -107,27 +585,317 @@ typedef struct ConfigManager_tag { /* zowe.setup.mvs.proclib */ int cfgGetInt(ConfigManager *mgr, int *value, ...){ /* path elements which are strings and ints for array indexing */ - + return 0; } int cfgGetInt64(ConfigManager *mgr, int64 *value, ...){ + return 0; +} + +static Json *jsonPointerDereference(Json *json, JsonPointer *jsonPointer, int *errorReason, int traceLevel){ + ArrayList *elements = &(jsonPointer->elements); + Json *value = json; + for (int i=0; isize; i++){ + JsonPointerElement *element = (JsonPointerElement*)arrayListElement(elements,i); + if (traceLevel >= 1){ + printf("deref elt=0x%p, i=%d value=0x%p\n",element,i,value);fflush(stdout); + } + if (jsonIsArray(value)){ + printf("AAA\n");fflush(stdout); + JsonArray *array = jsonAsArray(value); + printf("array case = 0x%p\n",array);fflush(stdout); + if (element->type == JSON_POINTER_INTEGER){ + int index = atoi(element->string); + int arraySize = jsonArrayGetCount(array); + if (index >= arraySize){ + *errorReason = JSON_POINTER_ARRAY_INDEX_OUT_OF_BOUNDS; + return NULL; + } + value = jsonArrayGetItem(array,index); + } else { + *errorReason = JSON_POINTER_ARRAY_INDEX_NOT_INTEGER; + return NULL; + } + } else if (jsonIsObject(value)){ + printf("OOO\n");fflush(stdout); + JsonObject *object = jsonAsObject(value); + printf("object case = 0x%p\n",object);fflush(stdout); + value = jsonObjectGetPropertyValue(object,element->string); + printf("value for key='%s' is 0x%p\n",element->string,value); + fflush(stdout); + if (value == NULL){ + *errorReason = JSON_POINTER_TOO_DEEP; + return NULL; + } + } else { + printf("SSS\n");fflush(stdout); + printf("cannot dereference scalar\n"); + fflush(stdout); + return NULL; + } + } + return value; } -#define ZCFG_ALLOC_HEAP 1 -#define ZCFG_ALLOC_SLH 2 -#define ZCFG_ALLOC_MEMCPY 3 +int cfgGetStringJ(ConfigManager *mgr, Json **result, JsonPointer *jp){ + int errorReason = 0; + Json *value = jsonPointerDereference(mgr->config,jp,&errorReason,mgr->traceLevel); + if (value){ + if (jsonIsString(value)){ + *result = value; + return ZCFG_SUCCESS; + } else { + *result = NULL; + return ZCFG_TYPE_MISMATCH; + } + } else { + *result = NULL; + return errorReason; + } +} + +int vcfgGetStringC(ConfigManager *mgr, char **value, int allocOptions, void *mem, va_list args){ + return ZCFG_SUCCESS; +} + +int cfgGetStringC(ConfigManager *mgr, char **value, int allocOptions, void *mem, ...){ + va_list argPointer; + va_start(argPointer,mem); + vcfgGetStringC(NULL,value,allocOptions,mem,argPointer); + va_end(argPointer); -int cfgGetString(ConfigManager *mgr, char *value, int allocOptions, void *mem, ...){ return ZCFG_SUCCESS; } int cfgGetInstanceType(ConfigManager *mgr, int *type, int *subType, ...){ + return 0; +} +/* like get string, but doesn't care about data types */ + +int cfgGetAnyJ(ConfigManager *mgr, Json **result, JsonPointer *jp){ + int errorReason = 0; + Json *value = jsonPointerDereference(mgr->config,jp,&errorReason,mgr->traceLevel); + if (mgr->traceLevel >= 1){ + printf("cfgGetAny: value=0x%p error=%d\n",value,errorReason); + } + if (value){ + *result = value; + return ZCFG_SUCCESS; + } else { + *result = NULL; + return errorReason; + } } +int cfgGetAny(ConfigManager *mgr, char *value, int allocOptions, void *mem, ...){ + return 0; +} +static void extractText(ConfigManager *mgr, JsonPointer *jp, FILE *out){ + Json *value = NULL; + printf("extract ckpt.1\n");fflush(stdout); + int status = cfgGetAnyJ(mgr,&value,jp); + printf("extract ckpt.2\n");fflush(stdout); + if (status){ + fprintf(out,"error not found, reason=%d",status); + } else { + if (jsonIsObject(value) || + jsonIsArray(value)){ + fprintf(out,"error: cannot access whole objects or arrays"); + } else if (jsonIsString(value)){ + fprintf(out,"%s",jsonAsString(value)); + } else if (jsonIsInt64(value)){ + fprintf(out,"%lld",jsonAsInt64(value)); + } else if (jsonIsDouble(value)){ + fprintf(out,"%f",jsonAsDouble(value)); + } else if (jsonIsBoolean(value)){ + fprintf(out,"%s",jsonAsBoolean(value) ? "true" : "false"); + } else if (jsonIsNull(value)){ + fprintf(out,"null"); + } else { + fprintf(out,"error: unhandled type"); + } + } +} +#define MAX_PATH_NAME 1024 +#define DATA_BUFFER_SIZE 2048 + +static void showDirectory(const char *dirname){ + int returnCode = 0; + int reasonCode = 0; + int stemLen = strlen(dirname); + int needsSlash = dirname[stemLen-1] != '/'; + char path[MAX_PATH_NAME]; + char directoryDataBuffer[DATA_BUFFER_SIZE]; + bool includeDotted = false; + UnixFile *directory = NULL; + printf("in show directory\n"); + fflush(stdout); + if ((directory = directoryOpen(dirname,&returnCode,&reasonCode)) == NULL){ + printf("directory open (%s) failure rc=%d reason=0x%x\n",dirname,returnCode,reasonCode); + } else { + + int entriesRead = 0; + while ((entriesRead = directoryRead(directory,directoryDataBuffer,DATA_BUFFER_SIZE,&returnCode,&reasonCode)) > 0){ + char *entryStart = directoryDataBuffer; + for (int e = 0; e \n"); + fprintf(out," options\n"); + fprintf(out," -h : show help\n"); + fprintf(out," -t : enable tracing with level from 1-3\n"); + fprintf(out," -o : OUT|ERR , ERR is default\n"); + fprintf(out," -s : root schema directory\n"); + fprintf(out," -w : workspace directory\n"); + fprintf(out," -p : list of colon-separated configPathElements - see below\n"); + fprintf(out," commands:\n"); + fprintf(out," extract : prints value to stdout\n"); + fprintf(out," configPathElement: \n"); + fprintf(out," LIB(datasetName) - a library that can contain config data\n"); + fprintf(out," FILE(filename) - the name of a file containing Yaml\n"); + fprintf(out," PARMLIBS - all PARMLIBS that are defined to this running Program in ZOS, nothing if not on ZOS\n"); +} +int main(int argc, char **argv){ + char *rootSchemaDirectory = NULL; // Read-only ZOWE runtime_directory (maybe + char *zoweWorkspaceHome = NULL; // Read-write is there always a zowe.yaml in there + char *configPath = NULL; + char *command = NULL; + char *traceArg = NULL; + int argx = 1; + int traceLevel = 0; + FILE *traceOut = stderr; + if (argc == 1){ + showHelp(traceOut); + return 0; + } + while (argx < argc){ + char *optionValue = NULL; + if (getStringOption(argc,argv,&argx,"-h")){ + showHelp(traceOut); + return 0; + } else if ((optionValue = getStringOption(argc,argv,&argx,"-s")) != NULL){ + rootSchemaDirectory = optionValue; + } else if ((optionValue = getStringOption(argc,argv,&argx,"-t")) != NULL){ + traceArg = optionValue; + } else if ((optionValue = getStringOption(argc,argv,&argx,"-o")) != NULL){ + if (!strcmp(optionValue,"OUT")){ + traceOut = stdout; + } + } else if ((optionValue = getStringOption(argc,argv,&argx,"-w")) != NULL){ + zoweWorkspaceHome = optionValue; + } else if ((optionValue = getStringOption(argc,argv,&argx,"-p")) != NULL){ + configPath = optionValue; + } else { + char *nextArg = argv[argx]; + if (strlen(nextArg) && nextArg[0] == '-'){ + fprintf(traceOut,"\n *** unknown option *** '%s'\n",nextArg); + showHelp(traceOut); + return 0; + } else { + break; + } + } + } + + if (traceArg){ + traceLevel = atoi(traceArg); + if (traceLevel >= 1){ + printf("ConfigMgr tracelevel set to %d\n",traceLevel); + } + } + + if (rootSchemaDirectory == NULL){ + fprintf(traceOut,"Must specify root schema directory\n"); + showHelp(traceOut); + return 0; + } + if (configPath == NULL){ + fprintf(traceOut,"Must specify config path\n"); + showHelp(traceOut); + return 0; + } + + ConfigManager *mgr = makeConfigManager(configPath,rootSchemaDirectory,traceLevel,traceOut); + if (mgr == NULL){ + trace(mgr,INFO,"Failed to build configmgr\n"); + return 0; + } + trace(mgr,DEBUG,"ConfigMgr built at 0x%p\n",mgr); + + + if (argx >= argc){ + printf("\n *** No Command Seen ***\n\n"); + showHelp(traceOut); + return 0; + } + command = argv[argx++]; + trace(mgr,DEBUG,"command = %s\n",command); + if (mgr->traceLevel >= 1){ + printConfigPath(mgr); + } + loadConfigurations(mgr); + trace(mgr,DEBUG,"configuration parms are loaded\n"); + if (!strcmp(command,"validate")){ /* just a testing mode */ + + } else if (!strcmp(command,"extract")){ + if (argx >= argc){ + trace(mgr,INFO,"extract command requires a json path\n"); + } else { + char *path = argv[argx++]; + JsonPointer *jp = parseJsonPointer(path); + if (jp == NULL){ + trace(mgr,INFO,"Could not parse JSON pointer '%s'\n",path); + return 0; + } + if (mgr->traceLevel >= 1){ + printJsonPointer(mgr->traceOut,jp); + fflush(mgr->traceOut); + } + /* Friday, extract some text and show Jack */ + extractText(mgr,jp,stdout); + printf("\n"); + fflush(stdout); + } + } + return 0; +} diff --git a/c/jsonschema.c b/c/jsonschema.c index e277ce4ae..e4677fc5d 100644 --- a/c/jsonschema.c +++ b/c/jsonschema.c @@ -850,9 +850,11 @@ static char *allJSTypeNames[ALL_TYPES_COUNT] = { "null", "boolean", "object", "a /* This function is recursive for all places that can be filled by a sub schema within a schema */ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopLevel){ AccessPath *accessPath = builder->accessPath; - printf("JSONSchema build\n"); - printAccessPath(stdout,accessPath); - fflush(stdout); + if (builder->traceLevel >= 2){ + printf("JSONSchema build\n"); + printAccessPath(stdout,accessPath); + fflush(stdout); + } if (jsonIsObject(jsValue)){ JsonObject *object = jsonAsObject(jsValue); ensureUniqueKeys(builder,object); @@ -860,8 +862,10 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopL char **typeArray = NULL; int typeArrayCount = 0; if (typeValue == NULL){ - printf("*** INFO *** untyped value at path:\n"); - printAccessPath(stdout,accessPath); + if (builder->traceLevel >= 1){ + printf("*** INFO *** untyped value at path:\n"); + printAccessPath(stdout,accessPath); + } typeArray = allJSTypeNames; typeArrayCount = ALL_TYPES_COUNT; } else { @@ -924,11 +928,9 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopL objectSpec->validatorFlags |= JS_VALIDATOR_MIN_PROPS; } JsonArray *required = getArrayValue(builder,object,"required"); - printf("builder required array = 0x%p\n",required); if (required != NULL){ int count = 0; objectSpec->required = getStringArrayOrFail(builder,required,&count); - printf("during build, required is 0x%p\n",objectSpec->required); objectSpec->requiredCount = count; } if (properties != NULL){ @@ -1076,8 +1078,10 @@ void freeJsonSchemaBuilder(JsonSchemaBuilder *builder){ JsonSchema *jsonBuildSchema(JsonSchemaBuilder *builder, Json *jsValue){ if (setjmp(builder->recoveryData) == 0) { /* normal execution */ - printf("after setjmp normal\n"); - fflush(stdout); + if (builder->traceLevel >= 2){ + printf("after setjmp normal\n"); + fflush(stdout); + } JSValueSpec *topValueSpec = build(builder,jsValue,true); JsonSchema *schema = (JsonSchema*)SLHAlloc(builder->slh,sizeof(JsonSchema)); schema->topValueSpec = topValueSpec; @@ -1087,8 +1091,10 @@ JsonSchema *jsonBuildSchema(JsonSchemaBuilder *builder, Json *jsValue){ builder->slh = NULL; return schema; } else { /* throw handling */ - printf("schema build fail %s\n",builder->errorMessage); - fflush(stdout); + if (builder->traceLevel >= 2){ + printf("schema build fail %s\n",builder->errorMessage); + fflush(stdout); + } return NULL; } } diff --git a/c/yaml2json.c b/c/yaml2json.c index 29afcb1c7..176f6fa0f 100644 --- a/c/yaml2json.c +++ b/c/yaml2json.c @@ -63,6 +63,7 @@ static char *getScalarStyleName(yaml_scalar_style_t style){ yaml_document_t *readYAML(char *filename){ + int traceLevel = 0; FILE *file; yaml_parser_t parser; yaml_document_t *document = (yaml_document_t*)safeMalloc(sizeof(yaml_document_t),"YAML Doc"); @@ -70,17 +71,18 @@ yaml_document_t *readYAML(char *filename){ int count = 0; int error = 0; - file = fopen(filename, "rb"); + file = fopen(filename, "rb"); assert(file); assert(yaml_parser_initialize(&parser)); yaml_parser_set_input_file(&parser, file); - - printf("before parser_load\n");fflush(stdout); + + if (traceLevel >= 1){ + printf("before parser_load\n");fflush(stdout); + } if (!yaml_parser_load(&parser, document)) { - printf("bad yaml load\n");fflush(stdout); error = 1; } else if (yaml_document_get_root_node(document)){ @@ -88,10 +90,14 @@ yaml_document_t *readYAML(char *filename){ } else { error = 1; } - - printf("before parser_delete\n");fflush(stdout); + + if (traceLevel >= 1){ + printf("before parser_delete\n");fflush(stdout); + } yaml_parser_delete(&parser); - printf("after parser_delete\n");fflush(stdout); + if (traceLevel >= 1){ + printf("after parser_delete\n");fflush(stdout); + } assert(!fclose(file)); if (error){ @@ -199,7 +205,7 @@ void pprintYAML(yaml_document_t *document){ static bool isSyntacticallyInteger(yaml_char_t *data, int length){ if (length == 0){ - printf("empty string not integer\n"); + /* printf("empty string not integer\n"); */ return false; } for (int i=0; i= '0' && c <= '9'){ } else { - printf("%s is NOT an integer\n",data); + /* printf("%s is NOT an integer\n",data); */ return false; } } - printf("%s is an integer\n",data); + /* printf("%s is an integer\n",data); */ return true; } @@ -236,7 +242,7 @@ static int64_t readInt(yaml_char_t *data, int length, bool *valid){ #define JSON_FAIL_BAD_INTEGER 104 static Json *yaml2JSON1(JsonBuilder *b, Json *parent, char *parentKey, - yaml_document_t *doc, yaml_node_t *node, int depth){ + yaml_document_t *doc, yaml_node_t *node, int depth){ int buildStatus = 0; switch (node->type){ case YAML_NO_NODE: @@ -262,7 +268,9 @@ static Json *yaml2JSON1(JsonBuilder *b, Json *parent, char *parentKey, case YAML_LITERAL_SCALAR_STYLE: case YAML_FOLDED_SCALAR_STYLE: char *tag = (char*)node->tag; - printf("tag = %s scalarStyle=%s\n",tag,getScalarStyleName(node->data.scalar.style)); + if (b->traceLevel >= 2){ + printf("tag = %s scalarStyle=%s\n",tag,getScalarStyleName(node->data.scalar.style)); + } Json *scalar = NULL; // HERE, make test with float, int, bool, null, ddate if (!strcmp(tag,YAML_NULL_TAG)){ diff --git a/h/json.h b/h/json.h index 8d069bf99..41c41e3fe 100644 --- a/h/json.h +++ b/h/json.h @@ -561,6 +561,7 @@ struct JsonParser_tag { typedef struct JsonBuilder_tag { JsonParser parser; Json *root; + int traceLevel; } JsonBuilder; JsonBuilder *makeJsonBuilder(ShortLivedHeap *slh); diff --git a/h/jsonschema.h b/h/jsonschema.h index f2168f240..f383aba9e 100644 --- a/h/jsonschema.h +++ b/h/jsonschema.h @@ -30,6 +30,7 @@ typedef struct JsonSchemaBuilder_tag { JsonSchema *schema; ShortLivedHeap *slh; int version; + int traceLevel; AccessPath *accessPath; int errorCode; char *errorMessage; From 6afc1716c31978366595dde03f28abfe25455baa Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 15 Feb 2022 01:23:54 -0500 Subject: [PATCH 18/42] Lots of merge fixes for ZOS branch merge and config merging and unit tests Signed-off-by: Joe --- c/configmgr.c | 206 +++++++++------------ c/json.c | 270 +++++++++++++++++++++++++++- c/yaml2json.c | 10 +- h/json.h | 41 ++++- tests/jsonmergetest.c | 101 +++++++++++ tests/schemadata/zowebase.yaml | 39 ++++ tests/schemadata/zoweoverrides.yaml | 4 + 7 files changed, 544 insertions(+), 127 deletions(-) create mode 100644 tests/jsonmergetest.c create mode 100644 tests/schemadata/zowebase.yaml create mode 100644 tests/schemadata/zoweoverrides.yaml diff --git a/c/configmgr.c b/c/configmgr.c index e9f9cc377..91359e54b 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -36,9 +36,9 @@ #include #include #include -#include /* JOE 1/20/22 */ -#include /* JOE 1/20/22 */ -#include /* JOE 1/20/22 */ +#include +#include +#include #include #endif @@ -95,6 +95,8 @@ typedef int64_t ssize_t; configmgr -s "../tests/schemadata" -p "LIBRARY(FOO):DIR(BAR)" extract "/foo/bar" + configmgr -s "../tests/schemadata" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowebase.yaml)" validate + */ @@ -123,6 +125,25 @@ typedef struct ConfigManager_tag { FILE *traceOut; } ConfigManager; +#ifdef __ZOWE_OS_WINDOWS +static int stdoutFD(){ + return _fileno(stdout); +} + +static int stderrFD(){ + return _fileno(stdout); +} +#else +static int stdoutFD(){ + return STDOUT_FILENO; +} + +static int stderrFD(){ + return STDERR_FILENO; +} +#endif + + #define INFO 0 #define DEBUG 1 #define DEBUG2 2 @@ -139,7 +160,7 @@ static void trace(ConfigManager *mgr, int level, char *formatString, ...){ /* configlet semantcs *IS* javascript semantics (and-or-not scalars) Configs live in product read-only directories - configrunner is the new ZWE + configrunner handles all the yaml/json/parmlib that the ZWE scripts need */ @@ -357,6 +378,14 @@ static void printConfigPath(ConfigManager *mgr){ } } +static void jsonPrettyPrint(ConfigManager *mgr, Json *json){ + jsonPrinter *p = makeJsonPrinter(mgr->traceOut == stdout ? stdoutFD() : stderrFD()); + jsonEnablePrettyPrint(p); + jsonPrint(p,json); + trace(mgr,INFO,"\n"); + +} + #define ZOWE_SCHEMA_FILE "zoweyaml.schema" void freeConfigManager(ConfigManager *mgr); @@ -364,13 +393,6 @@ void freeConfigManager(ConfigManager *mgr); ConfigManager *makeConfigManager(char *configPathArg, char *rootSchemaDirectory, int traceLevel, FILE *traceOut){ ConfigManager *mgr = (ConfigManager*)safeMalloc(sizeof(ConfigManager),"ConfigManager"); -#ifdef __ZOWE_OS_WINDOWS - int stdoutFD = _fileno(stdout); - int stderrFD = _fileno(stdout); -#else - int stdoutFD = STDOUT_FILENO; - int stderrFD = STDERR_FILENO; -#endif memset(mgr,0,sizeof(ConfigManager)); mgr->traceLevel = traceLevel; @@ -401,10 +423,8 @@ ConfigManager *makeConfigManager(char *configPathArg, char *rootSchemaDirectory, return NULL; } if (mgr->traceLevel >= 1){ - jsonPrinter *p = makeJsonPrinter(mgr->traceOut == stdout ? stdoutFD : stderrFD); - jsonEnablePrettyPrint(p); - jsonPrint(p,jsonWithSchema); - trace(mgr,INFO,"\n"); + jsonPrettyPrint(mgr,jsonWithSchema); + } JsonSchemaBuilder *builder = makeJsonSchemaBuilder(DEFAULT_JSON_SCHEMA_VERSION); JsonSchema *schema = jsonBuildSchema(builder,jsonWithSchema); @@ -418,16 +438,16 @@ ConfigManager *makeConfigManager(char *configPathArg, char *rootSchemaDirectory, mgr->topSchema = schema; } freeJsonSchemaBuilder(builder); - /* - Thurs read and merge from multiple sources - */ return mgr; } +#define YAML_ERROR_MAX 1024 + static Json *readJson(ConfigManager *mgr, ConfigPathElement *pathElement){ + char errorBuffer[YAML_ERROR_MAX]; if (pathElement->flags & CONFIG_PATH_OMVS_FILE){ trace(mgr,DEBUG,"before read YAML file=%s\n",pathElement->data); - yaml_document_t *doc = readYAML(pathElement->data); + yaml_document_t *doc = readYAML(pathElement->data,errorBuffer,YAML_ERROR_MAX); trace(mgr,DEBUG,"yaml doc at 0x%p\n",doc); if (doc){ if (mgr->traceLevel >= 1){ @@ -445,23 +465,35 @@ static Json *readJson(ConfigManager *mgr, ConfigPathElement *pathElement){ } } + /* need to collect violations as this goes */ -static void overloadConfiguration(ConfigManager *mgr, - ConfigPathElement *pathElement, - ConfigPathElement *pathTail){ +static int overloadConfiguration(ConfigManager *mgr, + ConfigPathElement *pathElement, + ConfigPathElement *pathTail){ if (pathTail == NULL){ printf("at end of config path \n");fflush(stdout); mgr->config = readJson(mgr,pathElement); printf("mgr->config = 0x%p\n",mgr->config); + return 0; /* success */ } else { - Json *overlay = NULL; - /* overloadPathElement(pathElement); */ + Json *overlay = readJson(mgr,pathElement); + int rhsStatus = overloadConfiguration(mgr,pathTail,pathTail->next); + printf("read the overlay with json=0x%p and status=%d\n",overlay,rhsStatus); + fflush(stdout); + if (rhsStatus){ + return rhsStatus; /* don't merge if we couldn't load what's to the right in the list */ + } + int mergeStatus = 0; + mgr->config = jsonMerge(mgr->slh,overlay,mgr->config, + JSON_MERGE_FLAG_CONCATENATE_ARRAYS, + &mergeStatus); + return mergeStatus; } } -void loadConfigurations(ConfigManager *mgr){ +static int loadConfigurations(ConfigManager *mgr){ ConfigPathElement *pathElement = mgr->configPath; - overloadConfiguration(mgr,pathElement,pathElement->next); + return overloadConfiguration(mgr,pathElement,pathElement->next); } void freeConfigManager(ConfigManager *mgr){ @@ -469,96 +501,6 @@ void freeConfigManager(ConfigManager *mgr){ safeFree((char*)mgr,sizeof(ConfigManager)); } -#define JSON_POINTER_NORMAL_KEY 1 -#define JSON_POINTER_INTEGER 2 - -typedef struct JsonPointerElement_tag { - char *string; - int type; -} JsonPointerElement; - -typedef struct JsonPointer_tag { - ArrayList elements; -} JsonPointer; - -static JsonPointerElement *makePointerElement(char *token, int type){ - JsonPointerElement *element = (JsonPointerElement*)safeMalloc(sizeof(JsonPointerElement),"JsonPointerElement"); - element->string = token; - element->type = type; - return element; -} - -/* See RFC 6901 */ -JsonPointer *parseJsonPointer(char *s){ - int len = strlen(s); - /* char *reduced = safeMalloc(len+1); */ - JsonPointer *jp = (JsonPointer*)safeMalloc(sizeof(JsonPointer),"JSONPointer"); - ArrayList *list = &(jp->elements); - initEmbeddedArrayList(list,NULL); - if (len == 0){ - return jp; - } - int pos = 1; - while (pos < len){ - int nextSlash = indexOf(s,len,'/',pos); - int end = (nextSlash == -1 ? len : nextSlash); - int tokenLen = end-pos+1; - char *token = safeMalloc(tokenLen,"JSON Ptr Token"); - memset(token,0,tokenLen); - /* Step 2: reduce the following digraphs - ~0 -> '~' - ~1 -> '/' - */ - bool pendingTilde = false; - int tPos = 0; - bool allDigits = true; - for (int i=pos; i '9'){ - allDigits = false; - } - if (pendingTilde){ - if (c == '0'){ - token[tPos++] = '~'; - } else if (c == '1'){ - token[tPos++] = '/'; - } else { - token[tPos++] = '~'; - token[tPos++] = c; - } - } else { - if (i == '~'){ - pendingTilde = true; - } else { - token[tPos++] = c; - } - } - } - if (pendingTilde){ - token[tPos++] = '~'; - } - - arrayListAdd(list, - makePointerElement(token, - (allDigits ? JSON_POINTER_INTEGER : JSON_POINTER_NORMAL_KEY))); - - if (nextSlash == -1){ - break; - } else { - pos = nextSlash + 1; - } - } - return jp; -} - -void printJsonPointer(FILE *out, JsonPointer *jp){ - ArrayList *list = &jp->elements; - fprintf(out,"JSON Pointer:\n"); - for (int i=0; isize; i++){ - JsonPointerElement *element = (JsonPointerElement*)arrayListElement(list,i); - fprintf(out," 0x%04X: %s\n",element->type,element->string); - } -} #define JSON_POINTER_TOO_DEEP 101 #define JSON_POINTER_ARRAY_INDEX_NOT_INTEGER 102 @@ -789,6 +731,7 @@ static void showHelp(FILE *out){ fprintf(out," -p : list of colon-separated configPathElements - see below\n"); fprintf(out," commands:\n"); fprintf(out," extract : prints value to stdout\n"); + fprintf(out," validate : just loads and validates merged configuration\n"); fprintf(out," configPathElement: \n"); fprintf(out," LIB(datasetName) - a library that can contain config data\n"); fprintf(out," FILE(filename) - the name of a file containing Yaml\n"); @@ -873,10 +816,37 @@ int main(int argc, char **argv){ if (mgr->traceLevel >= 1){ printConfigPath(mgr); } - loadConfigurations(mgr); + int loadStatus = loadConfigurations(mgr); + if (loadStatus){ + trace(mgr,INFO,"Failed to load configuration, element may be bad, or less likey a bad merge\n"); + } trace(mgr,DEBUG,"configuration parms are loaded\n"); if (!strcmp(command,"validate")){ /* just a testing mode */ - + trace(mgr,INFO,"about to validate merged yamls as\n"); + jsonPrettyPrint(mgr,mgr->config); + JsonValidator *validator = makeJsonValidator(); + trace(mgr,DEBUG,"Before Validate\n"); + int validateStatus = jsonValidateSchema(validator,mgr->config,mgr->topSchema); + trace(mgr,INFO,"validate status = %d\n",validateStatus); + switch (validateStatus){ + case JSON_VALIDATOR_NO_EXCEPTIONS: + trace(mgr,INFO,"No validity Exceptions\n"); + break; + case JSON_VALIDATOR_HAS_EXCEPTIONS: + { + trace(mgr,INFO,"Validity Exceptions:\n"); + ValidityException *e = validator->firstValidityException; + while (e){ + trace(mgr,INFO," %s\n",e->message); + e = e->next; + } + } + break; + case JSON_VALIDATOR_INTERNAL_FAILURE: + trace(mgr,INFO,"validation internal failure"); + break; + } + freeJsonValidator(validator); } else if (!strcmp(command,"extract")){ if (argx >= argc){ trace(mgr,INFO,"extract command requires a json path\n"); diff --git a/c/json.c b/c/json.c index ffba0d3ad..a836fce31 100644 --- a/c/json.c +++ b/c/json.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #include #include #include +#include #include #include #include /* JOE 1/20/22 */ @@ -2005,10 +2007,30 @@ Json *jsonBuildInt64(JsonBuilder *b, } } +Json *jsonBuildDouble(JsonBuilder *b, + Json *parent, + char *parentKey, + double d, + int *errorCode){ + JsonParser *parser = (JsonParser*)b; + int lvalueStatus = checkParentLValue(parent,parentKey); + if (lvalueStatus < 0){ + Json *json = (Json*) jsonParserAlloc(parser, sizeof (Json)); + json->type = JSON_TYPE_DOUBLE; + json->data.floatValue = d; + *errorCode = 0; + addToParent(b,lvalueStatus,parent,parentKey,json); + return json; + } else { + *errorCode = lvalueStatus; + return NULL; + } +} + Json *jsonBuildBool(JsonBuilder *b, Json *parent, char *parentKey, - bool *truthValue, + bool truthValue, int *errorCode){ JsonParser *parser = (JsonParser*)b; int lvalueStatus = checkParentLValue(parent,parentKey); @@ -2509,6 +2531,252 @@ JsonObject *jsonObjectProperty(JsonObject *object, char *propertyName, int *stat } } +/* JSON Merging */ + +typedef struct JsonMerger_tag { + JsonBuilder builder; /* must be first member to support casts */ + int flags; +} JsonMerger; + +static void copyJson(JsonBuilder *builder, Json *parent, char *parentKey, Json *json){ + int errorCode = 0; /* consumed and discarded because existing tree is valid */ + if (jsonIsObject(json)){ + JsonObject *jsonObject = jsonAsObject(json); + JsonProperty *prop = jsonObjectGetFirstProperty(jsonObject); + Json *copyObject = jsonBuildObject(builder,parent,parentKey,&errorCode); + while (prop){ + copyJson(builder,copyObject,prop->key,prop->value); + prop = jsonObjectGetNextProperty(prop); + } + } else if (jsonIsArray(json)){ + JsonArray *jsonArray = jsonAsArray(json); + Json *copyArray = jsonBuildArray(builder,parent,parentKey,&errorCode); + int size = jsonArrayGetCount(jsonArray); + for (int i=0; itype); + return; + } +} + +static int mergeJson1(JsonMerger *merger, Json *parent, char *parentKey, Json *overrides, Json *base){ + int errorCode = 0; + JsonBuilder *builder = &merger->builder; + /* + printf("jsonMerge1 typeO=%d typeB=%d builder=0x%p\n",overrides->type,base->type,builder); + fflush(stdout); + */ + if (jsonIsObject(base)){ + JsonObject *baseObject = jsonAsObject(base); + if (jsonIsObject(overrides)){ + JsonObject *overridesObject = jsonAsObject(overrides); + Json *merged = jsonBuildObject(builder,parent,parentKey,&errorCode); + JsonProperty *baseProp = jsonObjectGetFirstProperty(baseObject); + int status = JSON_MERGE_STATUS_SUCCESS; + while (baseProp && !status ){ + /* printf(" base loop %s\n",baseProp->key);fflush(stdout); */ + Json *baseValue = baseProp->value; + Json *overrideValue = jsonObjectGetPropertyValue(overridesObject,baseProp->key); + if (overrideValue == NULL){ /* NOT JSON NULL, really null */ + copyJson(builder,merged,baseProp->key,baseValue); + status = 0; + } else { + status = mergeJson1(merger,merged,baseProp->key,overrideValue,baseValue); + } + /* need to set result */ + baseProp = jsonObjectGetNextProperty(baseProp); + } + JsonProperty *overrideProp = jsonObjectGetFirstProperty(overridesObject); + while (overrideProp && !status){ + /* printf(" override loop %s\n",overrideProp->key);fflush(stdout); */ + if (!jsonObjectHasKey(baseObject,overrideProp->key)){ + copyJson(builder,merged,overrideProp->key,overrideProp->value); + } + overrideProp = jsonObjectGetNextProperty(overrideProp); + } + return status; + } else { + copyJson(builder,parent,parentKey,overrides); + return JSON_MERGE_STATUS_SUCCESS; + } + } else if (jsonIsArray(base)){ + JsonArray *baseArray = jsonAsArray(base); + if (jsonIsArray(overrides)){ + JsonArray *overridesArray = jsonAsArray(overrides); + int arrayPolicy = merger->flags & JSON_MERGE_FLAG_ARRAY_POLICY_MASK; + Json *merged = jsonBuildArray(builder,parent,parentKey,&errorCode); + int sizeB = jsonArrayGetCount(baseArray); + int sizeO = jsonArrayGetCount(overridesArray); + int i; + switch (arrayPolicy){ + case JSON_MERGE_FLAG_CONCATENATE_ARRAYS: /* len(merge) = len(a)+len(b) */ + for (i=0; i sizeO ? sizeB : sizeO); + for (i=0; i")); + fflush(stdout); + */ + copyJson(builder,parent,parentKey,overrides); + return JSON_MERGE_STATUS_SUCCESS; + } +} + +Json *jsonMerge(ShortLivedHeap *slh, Json *overrides, Json *base, int flags, int *statusPtr){ + JsonMerger merger; + memset(&merger,0,sizeof(JsonMerger)); + merger.flags = flags; + merger.builder.parser.slh = slh; + + int status = mergeJson1(&merger,NULL,NULL,overrides,base); + *statusPtr = status; + if (status){ + return NULL; + } else { + return merger.builder.root; + } +} + +/****** JSON Pointers ******************/ + +static JsonPointerElement *makePointerElement(char *token, int type){ + JsonPointerElement *element = (JsonPointerElement*)safeMalloc(sizeof(JsonPointerElement),"JsonPointerElement"); + element->string = token; + element->type = type; + return element; +} + +/* See RFC 6901 - absolute pointes only, so far */ +JsonPointer *parseJsonPointer(char *s){ + int len = strlen(s); + /* char *reduced = safeMalloc(len+1); */ + JsonPointer *jp = (JsonPointer*)safeMalloc(sizeof(JsonPointer),"JSONPointer"); + ArrayList *list = &(jp->elements); + initEmbeddedArrayList(list,NULL); + if (len == 0){ + return jp; + } + int pos = 1; + while (pos < len){ + int nextSlash = indexOf(s,len,'/',pos); + int end = (nextSlash == -1 ? len : nextSlash); + int tokenLen = end-pos+1; + char *token = safeMalloc(tokenLen,"JSON Ptr Token"); + memset(token,0,tokenLen); + /* Step 2: reduce the following digraphs + ~0 -> '~' + ~1 -> '/' + */ + bool pendingTilde = false; + int tPos = 0; + bool allDigits = true; + for (int i=pos; i '9'){ + allDigits = false; + } + if (pendingTilde){ + if (c == '0'){ + token[tPos++] = '~'; + } else if (c == '1'){ + token[tPos++] = '/'; + } else { + token[tPos++] = '~'; + token[tPos++] = c; + } + } else { + if (i == '~'){ + pendingTilde = true; + } else { + token[tPos++] = c; + } + } + } + if (pendingTilde){ + token[tPos++] = '~'; + } + + arrayListAdd(list, + makePointerElement(token, + (allDigits ? JSON_POINTER_INTEGER : JSON_POINTER_NORMAL_KEY))); + + if (nextSlash == -1){ + break; + } else { + pos = nextSlash + 1; + } + } + return jp; +} + +void printJsonPointer(FILE *out, JsonPointer *jp){ + ArrayList *list = &jp->elements; + fprintf(out,"JSON Pointer:\n"); + for (int i=0; isize; i++){ + JsonPointerElement *element = (JsonPointerElement*)arrayListElement(list,i); + fprintf(out," 0x%04X: %s\n",element->type,element->string); + } +} + + + + + void reportJSONDataProblem(void *jsonObject, int status, char *propertyName){ switch (status){ case JSON_PROPERTY_NOT_FOUND: diff --git a/c/yaml2json.c b/c/yaml2json.c index 95cda4da9..b89c0ec23 100644 --- a/c/yaml2json.c +++ b/c/yaml2json.c @@ -153,7 +153,7 @@ yaml_document_t *readYAML(const char *filename, char *errorBuf, size_t errorBufS snprintf(errorBuf, errorBufSize, "failed to alloc memory for YAML doc"); break; } - memset(document, 0, sizeof(document)); + memset(document, 0, sizeof(yaml_document_t)); if (!(file = fopen(filename, "rb"))) { snprintf(errorBuf, errorBufSize, "failed to read '%s' - %s", filename, strerror(errno)); break; @@ -209,7 +209,7 @@ static void printYamlScalar(const yaml_node_t *node, bool eol) { char val[printLength + 1]; snprintf(val, printLength + 1, "%.*s", printLength, node->data.scalar.value); convertToNative(val, printLength); - printf("%s%c", val, eol ? '\n' : ''); + printf("%s%s", val, eol ? "\n" : ""); } @@ -225,7 +225,7 @@ static void pprintYAML1(yaml_document_t *doc, yaml_node_t *node, int depth){ { indent(depth); size_t dataLen = node->data.scalar.length; - printf("Scalar: (len=%d)", node->data.scalar.length); + printf("Scalar: (len=%d)", (int)node->data.scalar.length); printYamlScalar(node, true); } break; @@ -377,9 +377,9 @@ static Json *yaml2JSON1(JsonBuilder *b, Json *parent, char *parentKey, } else if (!strcmp(nativeTag,YAML_INT_TAG) || (!strcmp(nativeTag,YAML_STR_TAG) && (style == YAML_PLAIN_SCALAR_STYLE) && - isSyntacticallyInteger(nativeValue,valueLength))){ + isSyntacticallyInteger((yaml_char_t*)nativeValue,valueLength))){ bool valid; - int64_t x = readInt(nativeValue,valueLength,&valid); + int64_t x = readInt((yaml_char_t*)nativeValue,valueLength,&valid); if (valid){ scalar = jsonBuildInt64(b,parent,parentKey,x,&buildStatus); } else { diff --git a/h/json.h b/h/json.h index 4be79cba0..36a3cc13b 100644 --- a/h/json.h +++ b/h/json.h @@ -23,6 +23,7 @@ #endif /* METTLE */ #include "zowetypes.h" #include "utils.h" +#include "collections.h" /** \file * \brief json.h is an implementation of an efficient low-level JSON writer and parser. @@ -559,7 +560,7 @@ struct JsonParser_tag { }; typedef struct JsonBuilder_tag { - JsonParser parser; + JsonParser parser; /* must be first member to support casts */ Json *root; int traceLevel; } JsonBuilder; @@ -607,16 +608,50 @@ Json *jsonBuildInt64(JsonBuilder *b, Json *jsonBuildBool(JsonBuilder *b, Json *parent, char *parentKey, - bool *truthValue, - int *errorCode); + bool truthValue, + int *errorCode); Json *jsonBuildNull(JsonBuilder *b, Json *parent, char *parentKey, int *errorCode); + char* jsonBuildKey(JsonBuilder *b, const char *key, int len); +#define JSON_MERGE_STATUS_SUCCESS 0 +#define JSON_MERGE_STATUS_UNMERGEABLE_TYPES 1 +#define JSON_MERGE_STATUS_UNIMPLEMENTED 2 + +/* Merging occurs when arrays match up to arays, or objects to objects */ + +#define JSON_MERGE_FLAG_ARRAY_POLICY_MASK 0x000F + +#define JSON_MERGE_FLAG_CONCATENATE_ARRAYS 0x0001 /* len(merge) = len(a)+len(b) */ +#define JSON_MERGE_FLAG_MERGE_IN_PLACE 0x0002 /* len(merge) = max(len(a),len(b)), a overrides corresponding elements of b */ +#define JSON_MERGE_FLAG_TAKE_BASE 0x0003 /* len(merge) = len(b) */ +#define JSON_MERGE_FLAG_TAKE_OVERRIDES 0x0004 /* len(merge) = len(a) */ + +Json *jsonMerge(ShortLivedHeap *slh, Json *overrides, Json *base, int flags, int *statusPtr); + +/* JSON Pointers */ + +#define JSON_POINTER_NORMAL_KEY 1 +#define JSON_POINTER_INTEGER 2 + +typedef struct JsonPointerElement_tag { + char *string; + int type; +} JsonPointerElement; + +/* These JSON pointers are *ABSOLUTE*, supporting relative is more work */ + +typedef struct JsonPointer_tag { + ArrayList elements; +} JsonPointer; + +JsonPointer *parseJsonPointer(char *s); +void printJsonPointer(FILE *out, JsonPointer *jp); #endif /* __JSON__ */ diff --git a/tests/jsonmergetest.c b/tests/jsonmergetest.c new file mode 100644 index 000000000..dc26dfd57 --- /dev/null +++ b/tests/jsonmergetest.c @@ -0,0 +1,101 @@ +#include +#include +#include +#include +#include + +#ifdef NDEBUG +#undef NDEBUG +#endif +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "json.h" + +/* + Notes: + + (all work assumed to be done from shell in this directory) + + Windows Build ______________________________ + + + clang -I./src -I../h -I ../platform/windows -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -o jsonmergetest.exe jsonmergetest.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc + + + ZOS Build _________________________________ + + + + + Running the Test ________________________________ + + jsonschematest + + */ + +int main(int argc, char *argv[]) +{ +#ifdef __ZOWE_OS_WINDOWS + int stdoutFD = _fileno(stdout); +#else + int stdoutFD = STDOUT_FILENO; +#endif + + if (argc < 4){ + printf("Usage jsonmergetest \n"); + printf(" ArrayMergePolicy: \n"); + printf(" concat: take all base elements followed by all overrides - len(merge) = len(o)+len(b)\n"); + printf(" merge: take from both arrays favoring overrides - len(merge) = max(len(o),len(b))\n"); + printf(" base: take base array only - len(merge) = len(b)\n"); + printf(" overrides: take overrides array only - len(merge) = len(a)\n"); + } + char *arrayPolicy = argv[1]; + char *overridesFile = argv[2]; + char *baseFile = argv[3]; + int flags = 0; + if (!strcmp(arrayPolicy,"concat")){ + flags = JSON_MERGE_FLAG_CONCATENATE_ARRAYS; + } else if (!strcmp(arrayPolicy,"merge")){ + flags = JSON_MERGE_FLAG_MERGE_IN_PLACE; + } else if (!strcmp(arrayPolicy,"base")){ + flags = JSON_MERGE_FLAG_TAKE_BASE; + } else if (!strcmp(arrayPolicy,"overrides")){ + flags = JSON_MERGE_FLAG_TAKE_OVERRIDES; + } else { + printf("unknown array merge policy, '%s'\n",arrayPolicy); + } + int errorBufferSize = 1024; + char *errorBuffer = safeMalloc(errorBufferSize,"ErrorBuffer"); + memset(errorBuffer,0,errorBufferSize); + + ShortLivedHeap *slh = makeShortLivedHeap(0x10000, 100); + + Json *base = jsonParseFile2(slh,baseFile,errorBuffer,errorBufferSize); + if (base == NULL){ + printf("Json parse fail in %s, with error '%s'\n",baseFile,errorBuffer); + return 12; + } + printf("read base\n"); fflush(stdout); + Json *overrides = jsonParseFile2(slh,overridesFile,errorBuffer,errorBufferSize); + if (overrides == NULL){ + printf("Json parse fail in %s, with error '%s'\n",overridesFile,errorBuffer); + return 12; + } + printf("read overrides\n"); fflush(stdout); + jsonPrinter *p = makeJsonPrinter(stdoutFD); + jsonEnablePrettyPrint(p); + int status = 0; + Json *merged = jsonMerge(slh,overrides,base,flags,&status); + printf("merged\n"); fflush(stdout); + if (merged){ + jsonPrint(p,merged); + fflush(stdout); + } else { + printf("failed to merge, status = %d\n",status); + } + return 0; +} + diff --git a/tests/schemadata/zowebase.yaml b/tests/schemadata/zowebase.yaml new file mode 100644 index 000000000..d8f7360e3 --- /dev/null +++ b/tests/schemadata/zowebase.yaml @@ -0,0 +1,39 @@ +zowe: + + #------------------------------------------------------------------------------- + # These configurations are used by "zwe install" or "zwe init" commands. + #------------------------------------------------------------------------------- + setup: + # MVS data set related configurations + mvs: + # **COMMONLY_CUSTOMIZED** + # where Zowe MVS data sets will be installed + # hlq: IBMUSER.ZWEV2 + hlq: IBMUSER.FOO + # **COMMONLY_CUSTOMIZED** + # PROCLIB where Zowe STCs will be copied over + proclib: VENDOR.PROCLIB + # **COMMONLY_CUSTOMIZED** + # Zowe PARMLIB + parmlib: IBMUSER.ZWEV2.CUST.PARMLIB + # **COMMONLY_CUSTOMIZED** + # JCL library where Zowe will store temporary JCLs during initialization + jcllib: IBMUSER.ZWEV2.CUST.JCLLIB + # APF authorized LOADLIB for Zowe + # Optional. If it's empty, .SZWEAUTH will be APF authorized. + authLoadlib: + # **COMMONLY_CUSTOMIZED** + # APF authorized LOADLIB for Zowe ZIS Plugins + authPluginLib: IBMUSER.ZWEV2.CUST.ZWESAPL + + # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> + # Certificate related configurations + # + # There are 5 configurations cases. Please choose one from below. + + # >>>> Certificate setup scenario 1 + # PKCS12 (keystore) with Zowe generate certificates. + certificate: + # Type of certificate storage. Valid values are: PKCS12 or JCERACFKS + type: PKCS12 + pkcs12: diff --git a/tests/schemadata/zoweoverrides.yaml b/tests/schemadata/zoweoverrides.yaml new file mode 100644 index 000000000..e3524ddc1 --- /dev/null +++ b/tests/schemadata/zoweoverrides.yaml @@ -0,0 +1,4 @@ +zowe: + setup: + mvs: + hlq: SOMELIB.BAR From 4668e0f9acdcf839c8ab72ac5d35ac47330c57a8 Mon Sep 17 00:00:00 2001 From: Leonty Chudinov Date: Tue, 15 Feb 2022 16:51:59 +0500 Subject: [PATCH 19/42] Make configmgr work on zos Signed-off-by: Leonty Chudinov --- c/configmgr.c | 12 ++++++++++-- tests/Makefile | 18 +++++++++++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/c/configmgr.c b/c/configmgr.c index 91359e54b..97a3b4802 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -135,11 +135,19 @@ static int stderrFD(){ } #else static int stdoutFD(){ +#ifdef STDOUT_FILENO return STDOUT_FILENO; +#else + return 1; +#endif } static int stderrFD(){ +#ifdef STDERR_FILENO return STDERR_FILENO; +#else + return 2; +#endif } #endif @@ -149,7 +157,7 @@ static int stderrFD(){ #define DEBUG2 2 static void trace(ConfigManager *mgr, int level, char *formatString, ...){ - if (mgr->traceLevel >= level){ + if (mgr && mgr->traceLevel >= level){ va_list argPointer; va_start(argPointer,formatString); vfprintf(mgr->traceOut,formatString,argPointer); @@ -301,7 +309,7 @@ static bool addPathElement(ConfigManager *mgr, char *pathElementArg){ regmatch_t matches[10]; regex_t *argPattern = regexAlloc(); /* Nice Regex test site */ - char *pat = "^(LIBRARY|DIR|FILE)\\((\\S+)\\)$"; + char *pat = "^(LIBRARY|DIR|FILE)\\(([^)]+)\\)$"; int compStatus = regexComp(argPattern,pat,REG_EXTENDED); if (compStatus != 0){ trace(mgr,INFO,"Internal error, pattern compilation failed\n"); diff --git a/tests/Makefile b/tests/Makefile index 56536ac57..c45b84bb0 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -31,13 +31,17 @@ CC_FLAGS:=-Ilibyaml/include -I../h -I../platform/posix $(DEFINES) -Wc,dll,expo,l LD_FLAGS:=-Wl,$(BITS) LIBYAMLOBJS:=api.o reader.o scanner.o parser.o loader.o writer.o emitter.o dumper.o -OBJS:=schematest.o yaml2json.o jsonschema.o json.o xlate.o charsets.o bpxskt.o logging.o collections.o timeutls.o timeutls.o utils.o alloc.o zosfile.o zos.o le.o scheduling.o recovery.o psxregex.o +SCHEMATESTOBJS:=schematest.o yaml2json.o jsonschema.o json.o xlate.o charsets.o bpxskt.o logging.o collections.o timeutls.o timeutls.o utils.o alloc.o zosfile.o zos.o le.o scheduling.o recovery.o psxregex.o +CONFIGMGROBJ:=configmgr.o yaml2json.o jsonschema.o json.o xlate.o charsets.o bpxskt.o logging.o collections.o timeutls.o timeutls.o utils.o alloc.o zosfile.o zos.o le.o scheduling.o recovery.o psxregex.o -.PHONY: clean all prepare +.PHONY: clean all prepare test_configmgr -all: schematest +all: configmgr schematest -schematest: $(OBJS) $(LIBYAMLOBJS) +configmgr: $(CONFIGMGROBJ) $(LIBYAMLOBJS) + $(CC) $(LD_FLAGS) -o $@ $^ + +schematest: $(SCHEMATESTOBJS) $(LIBYAMLOBJS) $(CC) $(LD_FLAGS) -o $@ $^ schematest.o: schematest.c @@ -58,7 +62,11 @@ libyaml: prepare: libyaml clean: - rm -f schematest *.o + rm -f configmgr schematest *.o + +test_configmgr: configmgr + ./configmgr -t 3 -s ./schemadata -p 'FILE(./schemadata/zoweoverrides.yaml):FILE(./schemadata/zowebase.yaml)' validate + ################################################################################ # This program and the accompanying materials are From 2e3c37eafb8426d50a169dda13b0bac66d827786 Mon Sep 17 00:00:00 2001 From: Leonty Chudinov Date: Wed, 16 Feb 2022 11:05:32 +0500 Subject: [PATCH 20/42] Implement `env` command for configmgr Signed-off-by: Leonty Chudinov --- c/configmgr.c | 133 +++++++++++++++++++++++++++++++++++++++++++++++-- tests/Makefile | 3 +- 2 files changed, 131 insertions(+), 5 deletions(-) diff --git a/c/configmgr.c b/c/configmgr.c index 97a3b4802..c0f5ff80f 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -40,6 +40,7 @@ #include #include #include +#include #endif @@ -479,15 +480,14 @@ static int overloadConfiguration(ConfigManager *mgr, ConfigPathElement *pathElement, ConfigPathElement *pathTail){ if (pathTail == NULL){ - printf("at end of config path \n");fflush(stdout); + trace(mgr, DEBUG2, "at end of config path\n"); mgr->config = readJson(mgr,pathElement); - printf("mgr->config = 0x%p\n",mgr->config); + trace(mgr, DEBUG2, "mgr->config = 0x%p\n", mgr->config); return 0; /* success */ } else { Json *overlay = readJson(mgr,pathElement); int rhsStatus = overloadConfiguration(mgr,pathTail,pathTail->next); - printf("read the overlay with json=0x%p and status=%d\n",overlay,rhsStatus); - fflush(stdout); + trace(mgr, DEBUG2, "read the overlay with json=0x%p and status=%d\n",overlay,rhsStatus); if (rhsStatus){ return rhsStatus; /* don't merge if we couldn't load what's to the right in the list */ } @@ -740,12 +740,123 @@ static void showHelp(FILE *out){ fprintf(out," commands:\n"); fprintf(out," extract : prints value to stdout\n"); fprintf(out," validate : just loads and validates merged configuration\n"); + fprintf(out," env : prints merged configuration to a file as a list of environment vars\n"); fprintf(out," configPathElement: \n"); fprintf(out," LIB(datasetName) - a library that can contain config data\n"); fprintf(out," FILE(filename) - the name of a file containing Yaml\n"); fprintf(out," PARMLIBS - all PARMLIBS that are defined to this running Program in ZOS, nothing if not on ZOS\n"); } +#define ZWE_PREFIX "ZWE" +#define ZOWE_ENV_PATH ZWE_PREFIX "_zowe_environments" + +static void convertJsonToEnv(FILE *fp, const char *path, Json *value); +static void convertJsonObjectToEnv(FILE *fp, const char *path, JsonObject *object); +static void convertJsonArrayToEnv(FILE *fp, const char *path, JsonArray *array); + +static int numberLen(int number) { + if (number == 0) { + return 1; + } + if (number < 0) { + return 1 + numberLen(-number); + } + return (int)(log10((double)number) + 1); +} + +static size_t escapeForEnv(const char *s, bool isKey, char *buffer, size_t bufferSize) { + size_t pos = 0; + int len = strlen(s); + memset(buffer, 0, bufferSize); + for (int i = 0; i < len; i++) { + char c = s[i]; + if (c == '\"' || c == '\"') { + pos += snprintf (buffer + pos, bufferSize - pos, "\\"); + } + if (isKey && c == '-') { + c = '_'; + } + pos += snprintf (buffer + pos, bufferSize - pos, "%c", c); + } + return pos; +} + +static outputEnvKey(FILE * out, const char *str) { + size_t escapedSize = strlen(str) * 2 + 1; + char escaped[escapedSize]; + escapeForEnv(str, true, escaped, escapedSize); + fprintf(out, "%s=", escaped); +} + +static outputEnvString(FILE * out, const char *str) { + size_t escapedSize = strlen(str) * 2 + 1; + char escaped[escapedSize]; + escapeForEnv(str, false, escaped, escapedSize); + fprintf(out, "\"%s\"\n", escaped); +} + +static outputEnvInt64(FILE * out, int64_t num) { + fprintf(out, "%lld\n", num); +} + +static outputEnvDouble(FILE * out, double num) { + fprintf(out, "%f\n", num); +} + +static outputEnvBoolean(FILE * out, bool b) { + fprintf(out, "%s\n", b ? "true": "false"); +} + +static void convertJsonToEnv(FILE *out, const char *path, Json *value) { + if (jsonIsObject(value)) { + convertJsonObjectToEnv(out, path, jsonAsObject(value)); + } else if (jsonIsArray(value)) { + convertJsonArrayToEnv(out, path, jsonAsArray(value)); + } else { + if (jsonIsString(value)) { + outputEnvKey(out, path); + outputEnvString(out, jsonAsString(value)); + } else if (jsonIsInt64(value)) { + outputEnvKey(out, path); + outputEnvInt64(out, jsonAsInt64(value)); + } else if (jsonIsDouble(value)) { + outputEnvKey(out, path); + outputEnvDouble(out, jsonAsDouble(value)); + } else if (jsonIsBoolean(value)) { + outputEnvKey(out, path); + outputEnvBoolean(out, jsonAsBoolean(value)); + } else if (jsonIsNull(value)) { + /* omit null */ + } + } +} + + +static void convertJsonObjectToEnv(FILE *out, const char *path, JsonObject *object) { + bool isEnvVar = (0 == strcmp(path, ZOWE_ENV_PATH)); + for (JsonProperty *prop = jsonObjectGetFirstProperty(object); prop != NULL; prop = jsonObjectGetNextProperty(prop)) { + const char *key = jsonPropertyGetKey(prop); + size_t pathSize = strlen(path) + strlen(key) + 2; + char currentPath[pathSize]; + snprintf (currentPath, pathSize, "%s_%s", path, key); + Json *value = jsonPropertyGetValue(prop); + convertJsonToEnv(out, isEnvVar ? key : currentPath, value); + } +} + +static void convertJsonArrayToEnv(FILE *out, const char *path, JsonArray *array) { + int count = jsonArrayGetCount(array); + for (int i = 0; i < count; i ++) { + int len = numberLen(i); + size_t pathSize = strlen(path) + len + 2; + char currentPath[pathSize]; + snprintf (currentPath, pathSize, "%s_%d", path, i); + Json *value = jsonArrayGetItem(array, i); + convertJsonToEnv(out, currentPath, value); + } +} + + int main(int argc, char **argv){ char *rootSchemaDirectory = NULL; // Read-only ZOWE runtime_directory (maybe char *zoweWorkspaceHome = NULL; // Read-write is there always a zowe.yaml in there @@ -874,6 +985,20 @@ int main(int argc, char **argv){ printf("\n"); fflush(stdout); } + } else if (!strcmp(command, "env")) { + if (argx >= argc){ + trace(mgr, INFO, "env command requires an env file path\n"); + } else { + char *outputPath = argv[argx++]; + Json *config = mgr->config; + FILE *out = fopen(outputPath, "w"); + if (out) { + convertJsonToEnv(out, ZWE_PREFIX, config); + fclose(out); + } else { + trace (mgr, INFO, "failed to open output file '%s' - %s\n", outputPath, strerror(errno)); + } + } } return 0; } diff --git a/tests/Makefile b/tests/Makefile index c45b84bb0..4f43d7a9b 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -65,7 +65,8 @@ clean: rm -f configmgr schematest *.o test_configmgr: configmgr - ./configmgr -t 3 -s ./schemadata -p 'FILE(./schemadata/zoweoverrides.yaml):FILE(./schemadata/zowebase.yaml)' validate + ./configmgr -s ./schemadata -p 'FILE(./schemadata/zoweoverrides.yaml):FILE(./schemadata/zowebase.yaml)' validate + ./configmgr -s ./schemadata -p 'FILE(./schemadata/zoweoverrides.yaml):FILE(./schemadata/zowebase.yaml)' env out.env && cat out.env ################################################################################ From ffe1d21f6a673071773948f1fbef75b0387ef469 Mon Sep 17 00:00:00 2001 From: Leonty Chudinov Date: Sat, 19 Feb 2022 12:16:57 +0500 Subject: [PATCH 21/42] Add build script for configmgr Signed-off-by: Leonty Chudinov --- bin/.gitignore | 18 +++++++ build/.gitignore | 17 ++++++ build/build_configmgr.sh | 112 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 bin/.gitignore create mode 100644 build/.gitignore create mode 100755 build/build_configmgr.sh diff --git a/bin/.gitignore b/bin/.gitignore new file mode 100644 index 000000000..16a3e70e1 --- /dev/null +++ b/bin/.gitignore @@ -0,0 +1,18 @@ +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. + +* +!.gitignore + +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. \ No newline at end of file diff --git a/build/.gitignore b/build/.gitignore new file mode 100644 index 000000000..1f13f011a --- /dev/null +++ b/build/.gitignore @@ -0,0 +1,17 @@ +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. + +tmp* + +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. \ No newline at end of file diff --git a/build/build_configmgr.sh b/build/build_configmgr.sh new file mode 100755 index 000000000..3a3e8ca40 --- /dev/null +++ b/build/build_configmgr.sh @@ -0,0 +1,112 @@ +#!/bin/sh + +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. + +WORKING_DIR=$(dirname "$0") + + +echo "********************************************************************************" +echo "Building configmgr..." + +rm -f "${COMMON}/bin/configmgr" + +mkdir -p "${WORKING_DIR}/tmp-configmgr" && cd "$_" +COMMON="../.." + +LIBYAML="libyaml" +MAJOR=0 +MINOR=2 +PATCH=5 +VERSION="\"${MAJOR}.${MINOR}.${PATCH}\"" + +if [ ! -d "${LIBYAML}" ]; then + git clone git@github.com:yaml/libyaml.git +fi + +export _C89_ACCEPTABLE_RC=0 + +if ! c89 \ + -c \ + -Wc,dll,expo,langlvl\(extc99\),gonum,goff,hgpr,roconst,ASM,asmlib\('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN'\) \ + -Wc,agg,list\(\),so\(\),off,xref \ + -Wc,ascii,noxplink \ + -D_ENHANCED_ASCII_EXT=0xFFFFFFFF \ + -DYAML_VERSION_MAJOR="${MAJOR}" \ + -DYAML_VERSION_MINOR="${MINOR}" \ + -DYAML_VERSION_PATCH="${PATCH}" \ + -DYAML_VERSION_STRING="${VERSION}" \ + -I "${LIBYAML}/include" \ + ${LIBYAML}/src/api.c \ + ${LIBYAML}/src/reader.c \ + ${LIBYAML}/src/scanner.c \ + ${LIBYAML}/src/parser.c \ + ${LIBYAML}/src/loader.c \ + ${LIBYAML}/src/writer.c \ + ${LIBYAML}/src/emitter.c \ + ${LIBYAML}/src/dumper.c +then + echo "Build failed" + exit 8 +fi + +if c89 \ + -D_XOPEN_SOURCE=600 \ + -DNOIBMHTTP=1 \ + -D_OPEN_THREADS=1 \ + -Wc,dll,expo,langlvl\(extc99\),gonum,goff,hgpr,roconst,ASM,asmlib\('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN'\) \ + -Wc,agg,exp,list\(\),so\(\),off,xref \ + -Wl,dll \ + -I "${COMMON}/h" \ + -I "${COMMON}/platform/posix" \ + -I "${LIBYAML}/include" \ + -o "${COMMON}/bin/configmgr" \ + api.o \ + reader.o \ + scanner.o \ + parser.o \ + loader.o \ + writer.o \ + emitter.o \ + dumper.o \ + ${COMMON}/c/alloc.c \ + ${COMMON}/c/bpxskt.c \ + ${COMMON}/c/charsets.c \ + ${COMMON}/c/collections.c \ + ${COMMON}/c/configmgr.c \ + ${COMMON}/c/json.c \ + ${COMMON}/c/jsonschema.c \ + ${COMMON}/c/le.c \ + ${COMMON}/c/logging.c \ + ${COMMON}/platform/posix/psxregex.c \ + ${COMMON}/c/recovery.c \ + ${COMMON}/c/scheduling.c \ + ${COMMON}/c/timeutls.c \ + ${COMMON}/c/utils.c \ + ${COMMON}/c/xlate.c \ + ${COMMON}/c/yaml2json.c \ + ${COMMON}/c/zos.c \ + ${COMMON}/c/zosfile.c +then + echo "Build successful" + exit 0 +else + # remove configmgr in case the linker had RC=4 and produced the binary + rm -f "${COMMON}/bin/configmgr" + echo "Build failed" + exit 8 +fi + + +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. \ No newline at end of file From 22c9bbd1aaaa82d9df1414673271f04d3e372ff9 Mon Sep 17 00:00:00 2001 From: Joe Date: Wed, 23 Feb 2022 01:56:32 -0500 Subject: [PATCH 22/42] Added support for embedded javascript in embeddedjs.c, an built a unit test jstest.c Signed-off-by: Joe --- c/configmgr.c | 21 +- c/embeddedjs.c | 659 ++++++++++++++++++++++++++++++++ c/json.c | 25 ++ c/yaml2json.c | 401 ++++++++++++++++++- h/embeddedjs.h | 58 +++ h/json.h | 28 +- tests/jstest.c | 151 ++++++++ tests/schemadata/yamltypes.yaml | 22 ++ tests/schematest.c | 8 +- 9 files changed, 1356 insertions(+), 17 deletions(-) create mode 100644 c/embeddedjs.c create mode 100644 h/embeddedjs.h create mode 100644 tests/jstest.c create mode 100644 tests/schemadata/yamltypes.yaml diff --git a/c/configmgr.c b/c/configmgr.c index c0f5ff80f..b6accfcb2 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -80,17 +80,22 @@ typedef int64_t ssize_t; (PDS/PDSE's). - References + -- References ---------- http://json-schema.org https://datatracker.ietf.org/doc/html/draft-bhutton-json-schema-validation) + + + -- Compilation ---------- + + set QJS=c:\repos\quickjs set YAML=c:\repos\libyaml ## Wherever you git clone'd libyaml clang++ -c ../platform/windows/cppregex.cpp ../platform/windows/winregex.cpp - clang -I%YAML%/include -I./src -I../h -I ../platform/windows -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 -o configmgr.exe configmgr.c %YAML%/src/api.c %YAML%/src/reader.c %YAML%/src/scanner.c %YAML%/src/parser.c %YAML%/src/loader.c %YAML%/src/writer.c %YAML%/src/emitter.c %YAML%/src/dumper.c ../c/yaml2json.c ../c/jsonschema.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c cppregex.o winregex.o + clang -I%YAML%/include -I %QJS% -I./src -I../h -I ../platform/windows -DCONFIG_VERSION=\"2021-03-27\" -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 --rtlib=compiler-rt -o configmgr.exe configmgr.c %QJS%\quickjs.c %QJS%\cutils.c %QJS%\quickjs-libc.c %QJS%\libbf.c %QJS%\libregexp.c %QJS%\libunicode.c %QJS%\winpthread.c %QJS%\wintime.c %QJS%\windirent.c %QJS%\winunistd.c %YAML%/src/api.c %YAML%/src/reader.c %YAML%/src/scanner.c %YAML%/src/parser.c %YAML%/src/loader.c %YAML%/src/writer.c %YAML%/src/emitter.c %YAML%/src/dumper.c ../c/yaml2json.c ../c/jsonschema.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c cppregex.o winregex.o configmgr "../tests/schemadata" "" "LIBRARY(FOO):DIR(BAR)" yak @@ -98,7 +103,17 @@ typedef int64_t ssize_t; configmgr -s "../tests/schemadata" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowebase.yaml)" validate - + HERE + - Embeddedjs.c - new file + - JSON/wUneval -> source + - buildEmbJS + - customize with more roots + - run + - embedding wrapper + QJS JSValue->Json + - forks/pushes portable-quickjs clang -> libquickjs.a + + */ #define CONFIG_PATH_OMVS_FILE 0x0001 diff --git a/c/embeddedjs.c b/c/embeddedjs.c new file mode 100644 index 000000000..d1eca7ca7 --- /dev/null +++ b/c/embeddedjs.c @@ -0,0 +1,659 @@ + + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#ifdef METTLE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "metalio.h" +#include "qsam.h" + +#else + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 600 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include /* JOE 1/20/22 */ +#include /* JOE 1/20/22 */ +#include /* JOE 1/20/22 */ +#include + +#endif + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "openprims.h" +#include "bpxnet.h" +#include "unixfile.h" +#include "json.h" +#include "charsets.h" +#include "embeddedjs.h" + +#ifdef __ZOWE_OS_WINDOWS +typedef int64_t ssize_t; +#endif + +JSValue ejsEvalBuffer(EmbeddedJS *ejs, + const void *buffer, int bufferLength, + const char *filename, int eval_flags, + int *statusPtr){ + JSContext *ctx = ejs->ctx; + JSValue val; + int ret; + + if ((eval_flags & JS_EVAL_TYPE_MASK) == JS_EVAL_TYPE_MODULE) { + /* for the modules, we compile then run to be able to set + import.meta */ + val = JS_Eval(ctx, buffer, bufferLength, filename, + eval_flags | JS_EVAL_FLAG_COMPILE_ONLY); + if (!JS_IsException(val)) { + js_module_set_import_meta(ctx, val, TRUE, TRUE); + val = JS_EvalFunction(ctx, val); + } + } else { + val = JS_Eval(ctx, buffer, bufferLength, filename, eval_flags); + } + if (JS_IsException(val)) { + js_std_dump_error(ctx); + ret = -1; + } else { + ret = 0; + } + *statusPtr = ret; + return val; +} + +void ejsFreeJSValue(EmbeddedJS *ejs, JSValue value){ + JS_FreeValue(ejs->ctx, value); +} + +int ejsSetGlobalProperty(EmbeddedJS *ejs, const char *propertyName, JSValue value){ + JSContext *ctx = ejs->ctx; + JSValue theGlobal = JS_GetGlobalObject(ctx); + return JS_SetPropertyStr(ctx,theGlobal,propertyName,value); +} + +int ejsEvalFile(EmbeddedJS *ejs, const char *filename, int loadMode){ + uint8_t *buf; + int ret, eval_flags; + size_t bufferLength; + + buf = js_load_file(ejs->ctx, &bufferLength, filename); + if (!buf) { + perror(filename); + exit(1); + } + + /* Beware of overly-expedient 3-valued logic here! */ + if (loadMode < 0) { + loadMode = (has_suffix(filename, ".mjs") || + JS_DetectModule((const char *)buf, bufferLength)); + } + if (loadMode) + eval_flags = JS_EVAL_TYPE_MODULE; + else + eval_flags = JS_EVAL_TYPE_GLOBAL; + JSValue evalResult = ejsEvalBuffer(ejs, buf, bufferLength, filename, eval_flags, &ret); + js_free(ejs->ctx, buf); + JS_FreeValue(ejs->ctx, evalResult); + return ret; +} + +static JSClassID js_joe_thingy_class_id; + +static void js_joe_thingy_finalizer(JSRuntime *rt, JSValue val){ + /* JSSTDFile *s = JS_GetOpaque(val, js_std_file_class_id); */ + printf("Joe Thingy Finalizer running\n"); +} + + +static JSClassDef js_joe_thingy_class = { + "THINGY", + .finalizer = js_joe_thingy_finalizer, +}; + +/* + #define JS_CFUNC_DEF(name, length, func1) + + name is JS name + + #define JS_CFUNC_MAGIC_DEF(name, length, func1, magic) + +*/ + +JSValue js_joe_thingy_three(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv){ + return JS_NewInt32(ctx, 3); +} + +static const JSCFunctionListEntry js_joe_thingy_proto_funcs[] = { + JS_CFUNC_DEF("three", 0, js_joe_thingy_three ), +}; + +static JSValue js_joe_boop(JSContext *ctx, JSValueConst this_val, + int argc, JSValueConst *argv) +{ + printf("boop boop boop!!\n"); + return JS_UNDEFINED; +} + +static const JSCFunctionListEntry js_joe_funcs[] = { + JS_CFUNC_DEF("boop", 0, js_joe_boop ), +}; + + + +static int js_joe_init(JSContext *ctx, JSModuleDef *m) +{ + JSValue proto; + + /* FILE class */ + /* the class ID is created once */ + JS_NewClassID(&js_joe_thingy_class_id); + /* the class is created once per runtime */ + JS_NewClass(JS_GetRuntime(ctx), js_joe_thingy_class_id, &js_joe_thingy_class); + proto = JS_NewObject(ctx); + JS_SetPropertyFunctionList(ctx, + proto, + js_joe_thingy_proto_funcs, + countof(js_joe_thingy_proto_funcs)); + JS_SetClassProto(ctx, + js_joe_thingy_class_id, + proto); + + JS_SetModuleExportList(ctx, m, js_joe_funcs, + countof(js_joe_funcs)); + return 0; +} + + +JSModuleDef *js_init_module_joe(JSContext *ctx, const char *module_name) +{ + JSModuleDef *m; + m = JS_NewCModule(ctx, module_name, js_joe_init); + if (!m){ + return NULL; + } + JS_AddModuleExportList(ctx, m, js_joe_funcs, countof(js_joe_funcs)); + return m; +} + +/* also used to initialize the worker context */ +static JSContext *makeEmbeddedJSContext(JSRuntime *rt) +{ + JSContext *ctx; + ctx = JS_NewContext(rt); + if (!ctx) + return NULL; +#ifdef CONFIG_BIGNUM + if (bignum_ext) { + JS_AddIntrinsicBigFloat(ctx); + JS_AddIntrinsicBigDecimal(ctx); + JS_AddIntrinsicOperators(ctx); + JS_EnableBignumExt(ctx, TRUE); + } +#endif + /* system modules */ + js_init_module_std(ctx, "std"); + js_init_module_os(ctx, "os"); + js_init_module_joe(ctx, "joe"); + return ctx; +} + +/* + + How to build JSValues + + many are static inlines in quickjs.h + + int's + static js_force_inline JSValue JS_NewInt64(JSContext *ctx, int64_t val) + + floats + static js_force_inline JSValue JS_NewFloat64(JSContext *ctx, double d) + + bool + static js_force_inline JSValue JS_NewBool(JSContext *ctx, JS_BOOL val) + + string + JSValue JS_NewStringLen(JSContext *ctx, const char *str1, size_t len1); + JSValue JS_NewString(JSContext *ctx, const char *str); + + Array + JSValue JS_NewArray(JSContext *ctx); + + setting values + int JS_SetPropertyUint32(JSContext *ctx, + JSValueConst this_obj, <--- JSValue of array ok here + uint32_t idx, JSValue val); + + + JS_SetPropertyUint32( + + Object + JSValue JS_NewObject(JSContext *ctx); + + setting props + int JS_SetPropertyStr(JSContext *ctx, JSValueConst this_obj, + const char *prop, JSValue val); + + testing + static inline JS_BOOL JS_IsNumber(JSValueConst v) + static inline JS_BOOL JS_IsBool(JSValueConst v) + static inline JS_BOOL JS_IsNull(JSValueConst v) + static inline JS_BOOL JS_IsUndefined(JSValueConst v) + static inline JS_BOOL JS_IsString(JSValueConst v) + static inline JS_BOOL JS_IsSymbol(JSValueConst v) + static inline JS_BOOL JS_IsObject(JSValueConst v) + int JS_IsArray(JSContext *ctx, JSValueConst val); + + Accessors: + Object: + JSValue JS_GetPropertyStr(JSContext *ctx, JSValueConst this_obj, + const char *prop); + Array: + JSValue JS_GetPropertyUint32(JSContext *ctx, JSValueConst this_obj, + uint32_t idx); + + Here + Convert with deletions/filtering + ejsEvalBuffer needs to return value and not free the JSValue + JSTest can show values + simple print + optional pretty print + expression can be as-is + function( -the 2nd dimensions) { <-- if object + return + + */ + +static Json *jsToJson1(EmbeddedJS *ejs, + JsonBuilder *b, Json *parent, char *parentKey, + JSValue value, + int depth){ + int buildStatus = 0; + int32_t tag = JS_VALUE_GET_TAG(value); + JSContext *ctx = ejs->ctx; + /* printf("jsToJSON1 tag=%d\n",tag);*/ + if (JS_IsUndefined(value)){ + fprintf(stderr,"*** Panic *** Undefined not handled\n"); + fflush(stderr); + return NULL; + } else if (JS_IsSymbol(value)){ + fprintf(stderr,"*** Panic *** Symbol not handled\n"); + fflush(stderr); + return NULL; + } else if (JS_IsNumber(value)){ + /* printf("number case\n"); */ + // Int64 vs, 32 vs float + if (JS_TAG_IS_FLOAT64(tag)){ + return jsonBuildDouble(b,parent,parentKey,JS_VALUE_GET_FLOAT64(value),&buildStatus); + } else { + int intValue = 0; + int intStatus = JS_ToInt32(ctx,&intValue,value); + /* printf("non float %d, status=%d\n",intValue,intStatus); */ + return jsonBuildInt64(b,parent,parentKey,(int64_t)intValue,&buildStatus); + } + } else if (JS_IsBool(value)){ + return jsonBuildBool(b,parent,parentKey,(bool)JS_VALUE_GET_BOOL(value),&buildStatus); + } else if (JS_IsString(value)){ + const char *str = JS_ToCString(ctx, value); + if (str == NULL){ + fprintf(stderr,"*** Panic *** Could not extract CString from Value\n"); + fflush(stderr); + return NULL; + } else { + Json *jsonString = jsonBuildString(b,parent,parentKey,(char*)str,strlen(str),&buildStatus); + JS_FreeCString(ctx, str); + return jsonString; + } + } else if (JS_IsNull(value)){ + return jsonBuildNull(b,parent,parentKey,&buildStatus); + } else if (JS_IsObject(value)){ + Json *jsonObject = jsonBuildObject(b,parent,parentKey,&buildStatus); + /* iterate properties */ + JSPropertyEnum *properties = NULL; /* I assume this will get fresh storage */ + uint32_t propertyCount = 0; + if (JS_GetOwnPropertyNames(ctx,&properties,&propertyCount,value,JS_GPN_STRING_MASK /* flags */)){ + printf("getOwnProperties failed\n"); + return NULL; + } else { + for (uint32_t i=0; iatom); + JSPropertyDescriptor descriptor; + /* BEWARE!!! + JS_GetOwnProperty has *NASTY* 3-valued logic. < 0 error, 0 not found, 1 found */ + int valueStatus = JS_GetOwnProperty(ctx,&descriptor,value,property->atom); + const char *cPropertyName = JS_ToCString(ctx,propertyName); + /* printf("jsToJson property i=%d %s valStatus=%d\n",i,cPropertyName,valueStatus); */ + if (valueStatus > 0){ + jsToJson1(ejs,b,jsonObject,(char*)cPropertyName,descriptor.value,depth+1); + } else { + printf("*** WARNING *** could not get value for property '%s', status=%d\n",cPropertyName,valueStatus); + } + + } + } + + return jsonObject; + } else if (JS_IsArray(ctx,value)){ + Json *jsonArray = jsonBuildArray(b,parent,parentKey,&buildStatus); + JSValue aLenJS = JS_GetPropertyStr(ctx,value,"length"); + int aLen = 0; + buildStatus = JS_ToInt32(ctx,&aLen,aLenJS); + printf("JS array len = %d\n",aLen); + for (int i=0; ictx; + switch (json->type) { + case JSON_TYPE_NUMBER: + case JSON_TYPE_INT64: + return JS_NewInt64(ctx,(int64_t)jsonAsInt64(json)); + case JSON_TYPE_DOUBLE: + return JS_NewFloat64(ctx,jsonAsDouble(json)); + case JSON_TYPE_STRING: + return JS_NewString(ctx,jsonAsString(json)); + case JSON_TYPE_BOOLEAN: + return JS_NewBool(ctx,jsonAsBoolean(json)); + case JSON_TYPE_NULL: + return JS_NULL; + case JSON_TYPE_OBJECT: + { + if (hideUnevaluated && isZoweUnevaluated(json)){ + return JS_UNDEFINED; + } else { + JsonObject *jsonObject = jsonAsObject(json); + JSValue object = JS_NewObject(ctx); + JsonProperty *property; + + for (property = jsonObjectGetFirstProperty(jsonObject); + property != NULL; + property = jsonObjectGetNextProperty(property)) { + JS_SetPropertyStr(ctx, + object, + jsonPropertyGetKey(property), + jsonToJS1(ejs,jsonPropertyGetValue(property),hideUnevaluated)); + } + return object; + } + } + case JSON_TYPE_ARRAY: + { + JsonArray *jsonArray = jsonAsArray(json); + JSValue array = JS_NewArray(ctx); + int count = jsonArrayGetCount(jsonArray); + + for (uint32_t i = 0; i < count; i++) { + JS_SetPropertyUint32(ctx, + array, + i, + jsonToJS1(ejs,jsonArrayGetItem(jsonArray,i),hideUnevaluated)); + } + return array; + } + default: + printf("*** PANIC *** unknown JSON type %d\n",json->type); + /* The next line is cheesy, but ... */ + return JS_NULL; + } +} + + + + +JSValue ejsJsonToJS(EmbeddedJS *ejs, Json *json){ + return jsonToJS1(ejs,json,false); +} + +char *json2JS(Json *json){ + JsonBuffer *buffer = makeJsonBuffer(); + /* jsonPrinter *p = makeBufferJsonPrinter(sourceCCSID,buffer); */ +#ifdef __ZOWE_OS_WINDOWS + int stdoutFD = _fileno(stdout); +#else + int stdoutFD = STDOUT_FILENO; +#endif + jsonPrinter *p = makeJsonPrinter(stdoutFD); + jsonEnablePrettyPrint(p); + + return NULL; +} + +static void visitJSON(Json *json, Json *parent, char *key, int index, + bool (*visitor)(void *context, Json *value, Json *parent, char *key, int index), + void *context){ + switch (json->type) { + case JSON_TYPE_NUMBER: + case JSON_TYPE_INT64: + case JSON_TYPE_DOUBLE: + case JSON_TYPE_STRING: + case JSON_TYPE_BOOLEAN: + case JSON_TYPE_NULL: + visitor(context,json,parent,key,index); + break; + case JSON_TYPE_OBJECT: + { + JsonObject *jsonObject = jsonAsObject(json); + JsonProperty *property; + for (property = jsonObjectGetFirstProperty(jsonObject); + property != NULL; + property = jsonObjectGetNextProperty(property)) { + char *key = jsonPropertyGetKey(property); + Json *value = jsonPropertyGetValue(property); + bool shouldVisitChildren = visitor(context,value,json,key,-1); + if (shouldVisitChildren){ + visitJSON(value,json,key,-1,visitor,context); + } + } + } + break; + case JSON_TYPE_ARRAY: + { + JsonArray *jsonArray = jsonAsArray(json); + int count = jsonArrayGetCount(jsonArray); + for (uint32_t i = 0; i < count; i++) { + Json *element = jsonArrayGetItem(jsonArray,i); + bool shouldVisitChildren = visitor(context,element,json,NULL,i); + if (shouldVisitChildren){ + visitJSON(element,json,NULL,i,visitor,context); + } + } + } + break; + default: + printf("*** PANIC *** unknown JSON type %d\n",json->type); + /* The next line is cheesy, but ... */ + break; + } +} + +static void setJsonProperty(JsonObject *object, char *key, Json *value){ + JsonProperty *property; + for (property = jsonObjectGetFirstProperty(object); + property != NULL; + property = jsonObjectGetNextProperty(property)) { + char *propertyKey = jsonPropertyGetKey(property); + if (!strcmp(propertyKey,key)){ + property->value = value; + break; + } + } +} + +static void setJsonArrayElement(JsonArray *array, int index, Json *value){ + array->elements[index] = value; +} + +typedef struct TemplateEvaluationContext_tag { + EmbeddedJS *ejs; + Json *topJson; + ShortLivedHeap *slh; +} TemplateEvaluationContext; + +static bool evaluationVisitor(void *context, Json *json, Json *parent, char *keyInParent, int indexInParent){ + if (isZoweUnevaluated(json)){ + JsonObject *object = jsonAsObject(json); + JsonProperty *property; + TemplateEvaluationContext *evalContext = (TemplateEvaluationContext*)context; + EmbeddedJS *ejs = evalContext->ejs; + JsonObject *topObject = jsonAsObject(evalContext->topJson); + for (property = jsonObjectGetFirstProperty(topObject); + property != NULL; + property = jsonObjectGetNextProperty(property)) { + char *key = jsonPropertyGetKey(property); + Json *value = jsonPropertyGetValue(property); + ejsSetGlobalProperty(ejs,key,ejsJsonToJS(ejs,value)); + } + Json *sourceValue = jsonObjectGetPropertyValue(object,"source"); + if (sourceValue){ + char *source = jsonAsString(sourceValue); + printf("should evaluate: %s\n",source); + int evalStatus = 0; + JSValue output = ejsEvalBuffer(ejs,source,strlen(source),"",0,&evalStatus); + if (evalStatus){ + printf("failed to evaluate '%s', status=%d\n",source,evalStatus); + } else { + printf("evaluation succeeded\n"); + dumpbuffer((char*)&output,sizeof(JSValue)); + Json *evaluationResult = ejsJSToJson(ejs,output,evalContext->slh); + printf("evaluationResult back in Json 0x%p\n",evaluationResult); + fflush(stdout); + if (keyInParent){ + setJsonProperty(jsonAsObject(parent),keyInParent,evaluationResult); + } else { + setJsonArrayElement(jsonAsArray(parent),indexInParent,evaluationResult); + } + } + } + return false; + } else { + return true; + } +} + +Json *evaluateJsonTemplates(EmbeddedJS *ejs, ShortLivedHeap *slh, Json *json){ + /* jsonToJS1(ejs,json,false); */ + TemplateEvaluationContext evalContext; + if (jsonIsObject(json)){ + JsonObject *topObject = jsonAsObject(json); + + evalContext.ejs = ejs; + evalContext.slh = slh; + evalContext.topJson = json; + visitJSON(json,NULL,NULL,-1,evaluationVisitor,&evalContext); + return json; + } else { + printf("top json is not an object\n"); + return NULL; + } +} + + + + +EmbeddedJS *makeEmbeddedJS(EmbeddedJS *sharedRuntimeEJS){ /* can be NULL */ + EmbeddedJS *embeddedJS = (EmbeddedJS*)safeMalloc(sizeof(EmbeddedJS),"EmbeddedJS"); + memset(embeddedJS,0,sizeof(EmbeddedJS)); + if (sharedRuntimeEJS){ + embeddedJS->rt = sharedRuntimeEJS->rt; + } else { + JSRuntime *rt = JS_NewRuntime(); + embeddedJS->rt = rt; + } + /* + JS_SetMemoryLimit(rt, memory_limit); + JS_SetMaxStackSize(rt, stack_size); + */ + js_std_set_worker_new_context_func(makeEmbeddedJSContext); + js_std_init_handlers(embeddedJS->rt); + JSContext *ctx = makeEmbeddedJSContext(embeddedJS->rt); + embeddedJS->ctx = ctx; + if (!ctx) { + fprintf(stderr, "qjs: cannot allocate JS context\n"); + exit(2); + } + + printf("JSTest Made Runtime and context \n"); + fflush(stdout); + + + /* loader for ES6 modules */ + JS_SetModuleLoaderFunc(embeddedJS->rt, NULL, js_module_loader, NULL); + + if (false){ /* dump_unhandled_promise_rejection) { - JOE doe this later */ + JS_SetHostPromiseRejectionTracker(embeddedJS->rt, js_std_promise_rejection_tracker, + NULL); + } + js_std_add_helpers(ctx, 0, NULL); /* argc - optind, argv + optind); */ + + int evalStatus = 0; + /* make 'std' and 'os' visible to non module code */ + if (true){ /* load_std) {*/ + const char *str = "import * as std from 'std';\n" + "import * as os from 'os';\n" + "import * as joe from 'joe';\n" + "globalThis.std = std;\n" + "globalThis.os = os;\n" + "globalThis.joe = joe;\n"; + JSValue throwaway = ejsEvalBuffer(embeddedJS, str, strlen(str), "", JS_EVAL_TYPE_MODULE, &evalStatus); + } + + + return embeddedJS; +} + diff --git a/c/json.c b/c/json.c index a836fce31..d0a4d5058 100644 --- a/c/json.c +++ b/c/json.c @@ -421,6 +421,17 @@ void jsonWrite(jsonPrinter *p, char *text, bool escape, int inputCCSID) { jsonConvertAndWriteBuffer(p, text, strlen(text), escape, inputCCSID); } +void jsonWriteParseably(jsonPrinter *p, char *text, int len, bool quote, bool escape, int inputCCSID){ + if (quote){ + jsonWrite(p, "\"", false, inputCCSID); + } + jsonConvertAndWriteBuffer(p, text, len, escape, inputCCSID); + if (quote){ + jsonWrite(p, "\"", false, inputCCSID); + } +} + + static void jsonIndent(jsonPrinter *p) { int depth = p->depth; @@ -835,6 +846,12 @@ void freeJsonBuffer(JsonBuffer *buf) { safeFree((void *)buf, sizeof (*buf)); } +char *jsonBufferCopy(JsonBuffer *buf){ + char *data = safeMalloc(buf->len,"copyJsonBuffer"); + memcpy(data,buf->data,buf->len); + return data; +} + void jsonBufferRewind(JsonBuffer *buf) { buf->len = 0; memset(buf->data, '0', buf->size); @@ -2318,6 +2335,14 @@ void jsonPrintProperty(jsonPrinter* printer, JsonProperty *property) { jsonPrintInternal(printer, jsonPropertyGetKey(property), jsonPropertyGetValue(property)); } +static bool filterFromPrinting(jsonPrinter *printer, char *keyOrNull, Json *value){ + if (printer->filter){ + return printer->filter(printer->filterContext,keyOrNull,value); + } else { + return false; + } +} + void jsonPrintObject(jsonPrinter* printer, JsonObject *object) { JsonProperty *property; diff --git a/c/yaml2json.c b/c/yaml2json.c index b89c0ec23..3e9638bc6 100644 --- a/c/yaml2json.c +++ b/c/yaml2json.c @@ -15,9 +15,18 @@ #include "zowetypes.h" #include "alloc.h" #include "utils.h" +#include "charsets.h" #include "json.h" #include "jsonschema.h" + +#if defined(__ZOWE_OS_ZOS) +# define SOURCE_CODE_CHARSET CCSID_IBM1047 +#else +# define SOURCE_CODE_CHARSET CCSID_UTF_8 +#endif + + static int convertToNative(char *buf, size_t size) { #ifdef __ZOWE_OS_ZOS return __atoe_l(buf, size); @@ -289,7 +298,9 @@ void pprintYAML(yaml_document_t *document){ #define MAX_JSON_KEY 256 #define MAX_JSON_STRING 65536 -/* this needs to be smarter some day */ +/* this needs to be smarter some day + Refer to Yaml spec http://yaml.org/spec/1.2-old/spec.html#id2805071 + */ static bool isSyntacticallyInteger(yaml_char_t *data, int length){ if (length == 0){ @@ -309,6 +320,44 @@ static bool isSyntacticallyInteger(yaml_char_t *data, int length){ return true; } +static bool isSyntacticallyBool(yaml_char_t *data, int length){ + if ((length == 4) && + (!memcmp(data,"true",4) || + !memcmp(data,"True",4) || + !memcmp(data,"TRUE",4))){ + return true; + } else if ((length == 5) && + (!memcmp(data,"false",5) || + !memcmp(data,"False",5) || + !memcmp(data,"FALSE",5))){ + return true; + } else { + return false; + } +} + +static bool isSyntacticallyNull(yaml_char_t *data, int length){ + if ((length == 1) && + !memcmp(data,"~",1)){ + return true; + } else if ((length == 4) && + (!memcmp(data,"null",4) || + !memcmp(data,"Null",4) || + !memcmp(data,"NULL",4))){ + return true; + } else { + return false; + } +} + +static bool isSyntacticallyTemplate(yaml_char_t *data, int length){ + if (strstr((char*)data,"${{")){ + return true; + } else { + return false; + } +} + static int64_t readInt(yaml_char_t *data, int length, bool *valid){ int64_t val64 = 0; bool allDecimal = true; @@ -326,8 +375,104 @@ static int64_t readInt(yaml_char_t *data, int length, bool *valid){ return val64; } -#define JSON_FAIL_NOT_HANDLED 100 -#define JSON_FAIL_BAD_INTEGER 104 +static bool readBool(yaml_char_t *data, int length, bool *valid){ + if ((length == 4) && + (!memcmp(data,"true",4) || + !memcmp(data,"True",4) || + !memcmp(data,"TRUE",4))){ + *valid = true; + return true; + } else if ((length == 5) && + (!memcmp(data,"false",5) || + !memcmp(data,"False",5) || + !memcmp(data,"FALSE",5))){ + *valid = true; + return false; + } else { + *valid = false; + return false; + } +} + + +#define JSON_FAIL_NOT_HANDLED 100 +#define JSON_FAIL_BAD_INTEGER 104 +#define JSON_FAIL_BAD_BOOL 108 +#define JSON_FAIL_BAD_NULL 112 +#define JSON_FAIL_BAD_TEMPLATE 116 + +static char *extractString(JsonBuilder *b, char *s, char *e){ + int len = e-s; + char *copy = SLHAlloc(b->parser.slh,len+1); + memcpy(copy,s,len); + copy[len] = 0; + return copy; +} + +static void addPlusIfNecessary(jsonPrinter *p, int sourceCCSID, bool *firstPtr){ + bool first = *firstPtr; + if (!first){ + jsonWriteParseably(p," + ",3,false,false,sourceCCSID); + } + *firstPtr = false; +} + +static int buildTemplateJSON(JsonBuilder *b, Json *parent, char *parentKey, + char *nativeValue, int valueLength){ + char *tail = nativeValue; + JsonBuffer *buffer = makeJsonBuffer(); + int sourceCCSID = SOURCE_CODE_CHARSET; + jsonPrinter *p = makeBufferJsonPrinter(sourceCCSID,buffer); + int status = 0; + bool first = true; + while (true){ + char *nextExpr = strstr(tail,"${{"); + + if (nextExpr){ + if (nextExpr > tail){ + char *frag = extractString(b,tail,nextExpr); + printf("frag = '%s'\n",frag); + addPlusIfNecessary(p,sourceCCSID,&first); + jsonWriteParseably(p,frag,strlen(frag),true,false,sourceCCSID); + } + char *end = strstr(nextExpr,"}}"); + if (end){ + char *exprText = extractString(b,nextExpr+3,end); + tail = end+2; + addPlusIfNecessary(p,sourceCCSID,&first); + jsonWriteParseably(p,exprText,strlen(exprText),false,false,sourceCCSID); + } else { + status = JSON_FAIL_BAD_TEMPLATE; + break; + } + } else { + char *lastFrag = extractString(b,tail,nativeValue+valueLength); + printf("lastFrag = '%s'\n",lastFrag); + if (strlen(lastFrag) > 0){ + addPlusIfNecessary(p,sourceCCSID,&first); + jsonWriteParseably(p,lastFrag,strlen(lastFrag),true,false,sourceCCSID); + } + break; + } + } + if (status == 0){ + Json *object = jsonBuildObject(b,parent,parentKey,&status); + if (!status){ + jsonBuildString(b,object,ZOWE_INTERNAL_TYPE,ZOWE_UNEVALUATED,strlen(ZOWE_UNEVALUATED),&status); + if (!status){ + jsonBufferTerminateString(buffer); + char *sourceCode = jsonBufferCopy(buffer); + printf("source code is: %s\n",sourceCode); + dumpbuffer(sourceCode,strlen(sourceCode)); + jsonBuildString(b,object,"source",sourceCode,strlen(sourceCode),&status); + } + } + } + freeJsonPrinter(p); + freeJsonBuffer(buffer); + return status; +} + static Json *yaml2JSON1(JsonBuilder *b, Json *parent, char *parentKey, yaml_document_t *doc, yaml_node_t *node, int depth){ @@ -369,11 +514,11 @@ static Json *yaml2JSON1(JsonBuilder *b, Json *parent, char *parentKey, } Json *scalar = NULL; // HERE, make test with float, int, bool, null, ddate - if (!strcmp(nativeTag,YAML_NULL_TAG)){ - } else if (!strcmp(nativeTag,YAML_NULL_TAG)){ - /* Json *scalar = jsonBuildNull(b,parent,parentKey,&buildStatus); */ - } else if (!strcmp(nativeTag,YAML_BOOL_TAG)){ - /* Json *scalar = jsonBuildBool(b,parent,parentKey,"FOO",3,&buildStatus); */ + if (!strcmp(nativeTag,YAML_NULL_TAG) || + (!strcmp(nativeTag,YAML_STR_TAG) && + (style == YAML_PLAIN_SCALAR_STYLE) && + isSyntacticallyNull((yaml_char_t*)nativeValue,valueLength))){ + scalar = jsonBuildNull(b,parent,parentKey,&buildStatus); } else if (!strcmp(nativeTag,YAML_INT_TAG) || (!strcmp(nativeTag,YAML_STR_TAG) && (style == YAML_PLAIN_SCALAR_STYLE) && @@ -385,9 +530,35 @@ static Json *yaml2JSON1(JsonBuilder *b, Json *parent, char *parentKey, } else { buildStatus = JSON_FAIL_BAD_INTEGER; } - /* Json *scalar = jsonBuildInt(b,parent,parentKey,"FOO",3,&buildStatus); */ + } else if (!strcmp(nativeTag,YAML_BOOL_TAG) || + (!strcmp(nativeTag,YAML_STR_TAG) && + (style == YAML_PLAIN_SCALAR_STYLE) && + isSyntacticallyBool((yaml_char_t*)nativeValue,valueLength))){ + bool valid; + bool x = readBool((yaml_char_t*)nativeValue,valueLength,&valid); + if (valid){ + scalar = jsonBuildBool(b,parent,parentKey,x,&buildStatus); + } else { + buildStatus = JSON_FAIL_BAD_BOOL; + } } else if (!strcmp(nativeTag,YAML_STR_TAG)){ - scalar = jsonBuildString(b,parent,parentKey,nativeValue,valueLength,&buildStatus); + /* + Must be plain + ${{ size = 0; + bos->capacity = chunkSize; + bos->chunkSize = chunkSize; + bos->data = safeMalloc(chunkSize,"ByteOutputStream Chunk"); + return bos; +} + +int bosWrite(ByteOutputStream *bos, char *data, int dataSize){ + if (bos->size+dataSize > bos->capacity){ + int extendSize = (bos->chunkSize > dataSize) ? bos->chunkSize : dataSize; + printf("bos extend currSize=0x%x dataSize=0x%x chunk=0x%x extend=0x%x\n", + bos->size,dataSize,bos->chunkSize,extendSize); + int newCapacity = bos->capacity + extendSize; + char *newData = safeMalloc(newCapacity,"BOS extend"); + memcpy(newData,bos->data,bos->size); + safeFree(bos->data,bos->capacity); + bos->data = newData; + bos->capacity = newCapacity; + } + memcpy(bos->data+bos->size,data,dataSize); + bos->size += dataSize; + return bos->size; +} + +int bosAppendString(ByteOutputStream *bos, char *s){ + return bosWrite(bos,s,strlen(s)); +} + +int bosAppendChar(ByteOutputStream *bos, char c){ + return bosWrite(bos,&c,1); +} + +char *bosNullTerminateAndUse(ByteOutputStream *bos){ + char c = 0; + bosWrite(bos,&c,1); + return bos->data; +} + +char *bosUse(ByteOutputStream *bos){ + return bos->data; +} + +void bosReset(ByteOutputStream *bos){ + bos->size = 0; +} + +void bosFree(ByteOutputStream *bos, bool freeBuffer){ + if (freeBuffer){ + safeFree(bos->data,bos->capacity); + } + safeFree((char*)bos,sizeof(ByteOutputStream)); +} + +/* Friday, expand it */ + Json *yaml2JSON(yaml_document_t *document, ShortLivedHeap *slh){ JsonBuilder *builder = makeJsonBuilder(slh); Json *json = yaml2JSON1(builder,NULL,NULL,document,yaml_document_get_root_node(document),0); freeJsonBuilder(builder,false); return json; } + + +static void yaml2JS1(ByteOutputStream *bos, + yaml_document_t *doc, yaml_node_t *node, int depth, + int traceLevel){ + int buildStatus = 0; + switch (node->type){ + case YAML_NO_NODE: + { + printf("*** WARNING *** NoNode\n"); + return; + } + case YAML_SCALAR_NODE: + { + size_t dataLen = node->data.scalar.length; + int valueLength = (int)dataLen; + if (dataLen > MAX_JSON_STRING){ + printf("*** WARNING *** oversize JSON string!\n"); + valueLength = MAX_JSON_STRING; + } + // Negative numbers, hexadecimal + yaml_scalar_style_t style = node->data.scalar.style; + switch (style){ + case YAML_ANY_SCALAR_STYLE: + case YAML_PLAIN_SCALAR_STYLE: + case YAML_SINGLE_QUOTED_SCALAR_STYLE: + case YAML_DOUBLE_QUOTED_SCALAR_STYLE: + case YAML_LITERAL_SCALAR_STYLE: + case YAML_FOLDED_SCALAR_STYLE: + { + char nativeValue[valueLength+1]; + char *tag = (char*)node->tag; + int tagLen = strlen(tag); + char nativeTag[tagLen + 1]; + snprintf(nativeValue, valueLength+1, "%.*s", valueLength, (const char *)node->data.scalar.value); + snprintf(nativeTag, tagLen + 1, "%.*s", tagLen, tag); + convertToNative(nativeValue, valueLength); + convertToNative(nativeTag, tagLen); + if (traceLevel >= 2){ + printf("tag = %s scalarStyle=%s\n",nativeTag,getScalarStyleName(node->data.scalar.style)); + } + Json *scalar = NULL; + // HERE, make test with float, int, bool, null, ddate + if (!strcmp(nativeTag,YAML_NULL_TAG)){ + } else if (!strcmp(nativeTag,YAML_NULL_TAG)){ + /* Json *scalar = jsonBuildNull(b,parent,parentKey,&buildStatus); */ + } else if (!strcmp(nativeTag,YAML_BOOL_TAG)){ + /* Json *scalar = jsonBuildBool(b,parent,parentKey,"FOO",3,&buildStatus); */ + } else if (!strcmp(nativeTag,YAML_INT_TAG) || + (!strcmp(nativeTag,YAML_STR_TAG) && + (style == YAML_PLAIN_SCALAR_STYLE) && + isSyntacticallyInteger((yaml_char_t*)nativeValue,valueLength))){ + bool valid; + int64_t x = readInt((yaml_char_t*)nativeValue,valueLength,&valid); + if (valid){ + printf("%lld\n",x); + } else { + buildStatus = JSON_FAIL_BAD_INTEGER; + } + /* Json *scalar = jsonBuildInt(b,parent,parentKey,"FOO",3,&buildStatus); */ + } else if (!strcmp(nativeTag,YAML_STR_TAG)){ + printf("%*.*s\n",valueLength,valueLength,nativeValue); + } else if (!strcmp(nativeTag,YAML_FLOAT_TAG)){ + printf("*** Warning don't know how to handle float yet\n"); + buildStatus = JSON_FAIL_NOT_HANDLED; + } else if (!strcmp(nativeTag,YAML_TIMESTAMP_TAG)){ + printf("*** Warning don't know how to handle timestamp yet\n"); + buildStatus = JSON_FAIL_NOT_HANDLED; + } + if (buildStatus){ + printf("*** WARNING *** Failed to add property/scalar err=%d\n",buildStatus); + } + return; + } + default: + printf("*** WARNING *** - unknown scalar style %d",node->data.scalar.style); + return; + } + } + case YAML_SEQUENCE_NODE: + { + yaml_node_item_t *item; + printf("[\n"); + for (item = node->data.sequence.items.start; item < node->data.sequence.items.top; item++) { + yaml_node_t *nextNode = yaml_document_get_node(doc, *item); + if (nextNode){ + yaml2JS1(bos,doc,nextNode,depth+2,traceLevel); + } else { + printf("*** WARNING *** dead end item\n"); + } + } + printf("]\n"); + } + case YAML_MAPPING_NODE: + { + yaml_node_pair_t *pair; + /* Json *jsonObject = jsonBuildObject(b,parent,parentKey,&buildStatus); */ + for (pair = node->data.mapping.pairs.start; pair < node->data.mapping.pairs.top; pair++) { + yaml_node_t *keyNode = yaml_document_get_node(doc, pair->key); + /* printf("keyNode 0x%p\n",keyNode); */ + char *key = NULL; + int keyLength; + if (keyNode) { + if (keyNode->type != YAML_SCALAR_NODE){ + printf("*** WARNING *** Non Scalar key\n"); + } else { + size_t dataLen = keyNode->data.scalar.length; + keyLength = (int)dataLen; + if (keyLength > MAX_JSON_KEY){ + printf("*** WARNING *** key too long '%*.*s...'\n", + MAX_JSON_KEY,MAX_JSON_KEY,keyNode->data.scalar.value); + } else { + key = (char*)keyNode->data.scalar.value; + } + } + } else { + printf("*** WARNING *** dead end key\n"); + } + if (key){ + char keyNative[keyLength+1]; + memcpy(keyNative,key,keyLength); + keyNative[keyLength] = 0; + convertToNative(keyNative, keyLength); + yaml_node_t *valueNode = yaml_document_get_node(doc, pair->value); + if (valueNode){ + printf("%s\n",keyNative); + yaml2JS1(bos,doc,valueNode,depth+2,traceLevel); + } else{ + printf("*** WARNING *** dead end value\n"); + } + } + } + return; + } + break; + default: + printf("*** WARNING *** unexpected yaml node type %d\n",node->type); + return; + } +} + +/* this is not fully done, and might not be needed */ +static void yaml2JS(yaml_document_t *document, ShortLivedHeap *slh){ + ByteOutputStream *bos = makeByteOutputStream(0x1000); + yaml2JS1(bos,document,yaml_document_get_root_node(document),0,1); +} diff --git a/h/embeddedjs.h b/h/embeddedjs.h new file mode 100644 index 000000000..499cb1997 --- /dev/null +++ b/h/embeddedjs.h @@ -0,0 +1,58 @@ +#ifndef __ZOWE_EMBEDDEDJS__ +#define __ZOWE_EMBEDDEDJS__ 1 + +#include "cutils.h" +#include "quickjs-libc.h" + +#include "zowetypes.h" + +struct trace_malloc_data { + uint8_t *base; +}; + +#define FILE_LOAD_AUTODETECT -1 +#define FILE_LOAD_GLOBAL 0 /* not module */ +#define FILE_LOAD_MODULE 1 + +typedef struct EmbeddedJS_tag { + JSRuntime *rt; + JSContext *ctx; + struct trace_malloc_data trace_data; /* = { NULL }; */ + int optind; + char *expr; /* = NULL; */ + int interactivel; /* = 0; */ + int dump_memory; /* = 0; */ + int trace_memory; /* = 0;*/ + int empty_run; /* = 0; */ + int loadMode; /* FILE_LOAD_AUTODETECT */ + int load_std; /* = 0; */ + int dump_unhandled_promise_rejection; /* = 0; */ + size_t memory_limit; /* = 0; */ + char *include_list[32]; +#ifdef CONFIG_BIGNUM + int load_jscalc; +#endif + size_t stack_size; /* = 0; */ +} EmbeddedJS; + +JSValue ejsEvalBuffer(EmbeddedJS *ejs, + const void *buffer, int bufferLength, + const char *filename, int eval_flags, + int *statusPtr); + +int ejsEvalFile(EmbeddedJS *ejs, const char *filename, int loadMode); + +void ejsFreeJSValue(EmbeddedJS *ejs, JSValue value); + + +/* usually pass NULL as arg unless trying to build complex embedded JS + application that can run multiple threads or other multitasking. +*/ +EmbeddedJS *makeEmbeddedJS(EmbeddedJS *sharedRuntimeEJS); + +Json *ejsJSToJson(EmbeddedJS *ejs, JSValue value, ShortLivedHeap *slh); +JSValue ejsJsonToJS(EmbeddedJS *ejs, Json *json); +int ejsSetGlobalProperty(EmbeddedJS *ejs, const char *propetyName, JSValue value); +Json *evaluateJsonTemplates(EmbeddedJS *ejs, ShortLivedHeap *slh, Json *json); + +#endif diff --git a/h/json.h b/h/json.h index 36a3cc13b..d0c8980ea 100644 --- a/h/json.h +++ b/h/json.h @@ -32,6 +32,8 @@ * and chunked input and output. */ +typedef struct Json_tag Json; + /** * \brief jsonPrinter represents a high-level stream to write JSON. * @@ -60,6 +62,10 @@ typedef struct jsonPrinter_tag { char *_conversionBuffer; int ioErrorFlag; int isInMultipartString; + bool (*filter)(void *filterContext, + char *keyOrNull, + Json *value); + void *filterContext; } jsonPrinter; typedef struct jsonBuffer_tag { @@ -145,6 +151,13 @@ void jsonEndArray(jsonPrinter *p); void jsonAddString(jsonPrinter *p, char *keyOrNull, char *value); +/** + * + * A special method for writing javascript data that goes beyond standard JSON writing. + * + */ +void jsonWriteParseably(jsonPrinter *p, char *text, int len, bool quote, bool escape, int inputCCSID); + /** * \brief Allows the caller to place a string representing well-formed JSON * content in the output stream into an enclosing object or array @@ -335,8 +348,8 @@ JsonBuffer *makeJsonBuffer(void); void jsonBufferTerminateString(JsonBuffer *buffer); void jsonBufferRewind(JsonBuffer *buffer); void freeJsonBuffer(JsonBuffer *buffer); +char *jsonBufferCopy(JsonBuffer *buffer); -typedef struct Json_tag Json; typedef struct JsonObject_tag JsonObject; typedef struct JsonArray_tag JsonArray; typedef struct JsonProperty_tag JsonProperty; @@ -565,6 +578,9 @@ typedef struct JsonBuilder_tag { int traceLevel; } JsonBuilder; +#define ZOWE_INTERNAL_TYPE "__zowe_internal_type__" +#define ZOWE_UNEVALUATED "unevaluated" + JsonBuilder *makeJsonBuilder(ShortLivedHeap *slh); #define JSON_BUILD_FAIL_PARENT_IS_SCALAR 12 @@ -584,6 +600,10 @@ Json *jsonBuildArray(JsonBuilder *b, char *parentKey, int *errorCode); +/** + This copies the string into memory owned by the builder/parser. That is + the input string arg "s" is not incorporated into the result. +*/ Json *jsonBuildString(JsonBuilder *b, Json *parent, char *parentKey, @@ -604,6 +624,12 @@ Json *jsonBuildInt64(JsonBuilder *b, int64 i, int *errorCode); +Json *jsonBuildDouble(JsonBuilder *b, + Json *parent, + char *parentKey, + double d, + int *errorCode); + Json *jsonBuildBool(JsonBuilder *b, Json *parent, diff --git a/tests/jstest.c b/tests/jstest.c new file mode 100644 index 000000000..ff1f7ffcd --- /dev/null +++ b/tests/jstest.c @@ -0,0 +1,151 @@ +#include +#include +#include +#include +#include + +#include "cutils.h" +#include "quickjs-libc.h" + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "json.h" +#include "yaml2json.h" +#include "embeddedjs.h" + +/* + + set QJS=c:\repos\quickjs + + set YAML=c:\repos\libyaml ## Wherever you git clone'd libyaml + + clang++ -c ../platform/windows/cppregex.cpp ../platform/windows/winregex.cpp + + clang -I %QJS%/porting -I%YAML%/include -I../platform/windows -I %QJS% -I ..\h -DCONFIG_VERSION=\"2021-03-27\" -D_CRT_SECURE_NO_WARNINGS -Dstrdup=_strdup -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 -Wdeprecated-declarations --rtlib=compiler-rt -o jstest.exe jstest.c %QJS%\quickjs.c %QJS%\cutils.c %QJS%\quickjs-libc.c %QJS%\libbf.c %QJS%\libregexp.c %QJS%\libunicode.c %QJS%\porting\winpthread.c %QJS%\porting\wintime.c %QJS%\porting\windirent.c %QJS%\porting\winunistd.c %QJS%\repl.c %YAML%/src/api.c %YAML%/src/reader.c %YAML%/src/scanner.c %YAML%/src/parser.c %YAML%/src/loader.c %YAML%/src/writer.c %YAML%/src/emitter.c %YAML%/src/dumper.c ../c/yaml2json.c ../c/embeddedjs.c ../c/jsonschema.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c cppregex.o winregex.o + + + A software configuration is a virtual hierarchical document + - representable as JSON in memory (Javascript data structures, more precisely) + - not necessarily in one source document + - can be cloned with modifications for HA or Provisioning Needs + - can be semantically validated as a pre-flight mechanism to launch components, or from config tools + - not all things can be validate, but most can + + The ConfigManager is the single interface to the software configuation + - it is callable/embeddabl from all points + - Windows, Linux, ZOS at minimum + - JavaScript,C Java Bindings + - output to STDOUT for scripting + - it is embodied in a staticly linked executable that is part of the base software install + - like JDBC client. You get the data and query semantics w/o exactly knowing where every + - it is bootstrapped from a minimal set of schemas and file system paths + - it self-augments through property values that bring in more schemas and path elements + - it provides output of its state as environment variables for components that are + not highly configuration + - is THE ONLY means of ZOWE component learning its parameters, outside of legacy components that read envvars + + A configuration source is a user (or UI) created document + - is owned by the user + - is text + - can be place under Version Control, eg Git + - is not modified by installers, launchers, etc. + - can be syntactically validated and prompted for syntax in editors + - configuration sources have template capabilities, ie are not static data structures, + - can always co-reference other parts of same source + - can reference after merge/overlay any part of schema + - can be evaluated with additional args/globals defined at different times in the InstallationTarget's lifecycle + - SSL may be an exception to these rules - + + An RunTarget is the unit software instance request + - RunTargets can require other IT's by name and version. + - RunTargets have sections of the SoftwareConfiguration that probably come from separate ConfigSource + - RunTargets form a directed acyclic graph (like Make targets, OSGI Modules, NPM Packages) to be instantiale + - RunTargets have a lifecycle - + initialInstall + upgrade + clone() + start + kill (maybe, for optional cleanup) + uninstall + */ + + +int main(int argc, char **argv){ + int i; + int include_count = 0; + char *command = argv[1]; + int loadMode = FILE_LOAD_AUTODETECT; + int errorBufferSize = 1024; +#ifdef __ZOWE_OS_WINDOWS + int stdoutFD = _fileno(stdout); +#else + int stdoutFD = STDOUT_FILENO; +#endif + char *errorBuffer = safeMalloc(errorBufferSize,"ErrorBuffer"); + + printf("JSTest Start \n"); + fflush(stdout); + + EmbeddedJS *ejs = makeEmbeddedJS(NULL); + + + /* no includes yet + for(i = 0; i < include_count; i++) { + if (eval_file(ctx, include_list[i], module)) + goto fail; + } + */ + int evalStatus = 0; + + /* JOE, here, eval buffer, or file, or go interactive */ + if (!strcmp(command,"expr")){ + char *sourceCode = argv[2]; + JSValue output = ejsEvalBuffer(ejs, sourceCode, strlen(sourceCode), "", 0, &evalStatus); + if (evalStatus){ + printf("evaluation failed, status=%d\n",evalStatus); + } else { + ejsFreeJSValue(ejs,output); + } + } else if (!strcmp(command,"file")){ + char *filename = argv[2]; + if (ejsEvalFile(ejs, filename, loadMode)){ + printf("file eval failed\n"); + } + } else if (!strcmp(command,"yamlcopy") || + !strcmp(command,"yamleval")){ + char *filename = argv[2]; + ShortLivedHeap *slh = makeShortLivedHeap(0x10000, 100); + yaml_document_t *doc = readYAML(filename, errorBuffer, errorBufferSize); + if (doc == NULL){ + printf("yaml file is bad: %s\n",errorBuffer); + return 0; + } + Json *json = yaml2JSON(doc,slh); + jsonPrinter *p = makeJsonPrinter(stdoutFD); + jsonEnablePrettyPrint(p); + printf("Yaml file as json\n"); + jsonPrint(p,json); + + if (!strcmp(command,"yamlcopy")){ + JSValue jsValue = ejsJsonToJS(ejs,json); + printf("made jsValue\n"); + fflush(stdout); + ejsSetGlobalProperty(ejs,"magic1",jsValue); + char *s1 = "console.log(\"theThing=\"+JSON.stringify(magic1))"; + JSValue throwAway = ejsEvalBuffer(ejs, s1, strlen(s1), "", 0, &evalStatus); + printf("evalStatus=%d\n",evalStatus); + Json *json2 = ejsJSToJson(ejs,jsValue,slh); + printf("JSON translated back to zowe-common-c.json\n"); + jsonPrint(p,json2); + } else { + Json *resultantJSON = evaluateJsonTemplates(ejs,slh,json); + printf("JSON templates evaluated\n"); + jsonPrint(p,resultantJSON); + } + } + + printf("JSTest done\n"); + fflush(stdout); + return 0; +} diff --git a/tests/schemadata/yamltypes.yaml b/tests/schemadata/yamltypes.yaml new file mode 100644 index 000000000..85c6c6cad --- /dev/null +++ b/tests/schemadata/yamltypes.yaml @@ -0,0 +1,22 @@ +--- +zowe: + default: + port: 3000 + hlq: "ZWE.PROD" + setup: + certificate: + pkcs12: ~ + type: PKCS12 + mvs: + authLoadlib: ~ + authPluginLib: IBMUSER.ZWEV2.CUST.ZWESAPL + foo: 3 + goforit: true + hay: "Hay \" " + hlq: "${{ zowe.default.hlq+\".BAZ\" }}" + hlq2: ${{ zowe.default.hlq+".BAZ" }} + port: ${{ zowe.default.port+1 }} + jcllib: IBMUSER.ZWEV2.CUST.JCLLIB + parmlib: IBMUSER.ZWEV2.CUST.PARMLIB + proclib: VENDOR.PROCLIB + \ No newline at end of file diff --git a/tests/schematest.c b/tests/schematest.c index e87c9ee32..d7efd3a2a 100644 --- a/tests/schematest.c +++ b/tests/schematest.c @@ -5,9 +5,6 @@ #include #include #include -#ifndef __ZOWE_OS_WINDOWS -#include -#endif #ifdef NDEBUG #undef NDEBUG @@ -15,6 +12,11 @@ #include #include "zowetypes.h" + +#ifndef __ZOWE_OS_WINDOWS +#include +#endif + #include "alloc.h" #include "utils.h" #include "json.h" From 764a728bced8a0c84bf6bff00fd5c410a69998c5 Mon Sep 17 00:00:00 2001 From: Joe Date: Wed, 23 Feb 2022 11:06:03 -0500 Subject: [PATCH 23/42] Integrate JS eval to configmgr Signed-off-by: Joe --- c/configmgr.c | 106 ++++++++++++++-------------- c/embeddedjs.c | 18 +++-- c/yaml2json.c | 8 ++- tests/jstest.c | 2 + tests/schemadata/zowebase.yaml | 18 +---- tests/schemadata/zoweoverrides.yaml | 2 +- 6 files changed, 74 insertions(+), 80 deletions(-) diff --git a/c/configmgr.c b/c/configmgr.c index b6accfcb2..932cf97bd 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -54,6 +54,7 @@ #include "jsonschema.h" #include "yaml.h" #include "yaml2json.h" +#include "embeddedjs.h" #include "charsets.h" #include "collections.h" @@ -89,29 +90,21 @@ typedef int64_t ssize_t; -- Compilation ---------- - set QJS=c:\repos\quickjs - + set QJS=c:\repos\quickjs-portable ## wherever you git cloned quickjs-portable + set YAML=c:\repos\libyaml ## Wherever you git clone'd libyaml clang++ -c ../platform/windows/cppregex.cpp ../platform/windows/winregex.cpp - clang -I%YAML%/include -I %QJS% -I./src -I../h -I ../platform/windows -DCONFIG_VERSION=\"2021-03-27\" -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 --rtlib=compiler-rt -o configmgr.exe configmgr.c %QJS%\quickjs.c %QJS%\cutils.c %QJS%\quickjs-libc.c %QJS%\libbf.c %QJS%\libregexp.c %QJS%\libunicode.c %QJS%\winpthread.c %QJS%\wintime.c %QJS%\windirent.c %QJS%\winunistd.c %YAML%/src/api.c %YAML%/src/reader.c %YAML%/src/scanner.c %YAML%/src/parser.c %YAML%/src/loader.c %YAML%/src/writer.c %YAML%/src/emitter.c %YAML%/src/dumper.c ../c/yaml2json.c ../c/jsonschema.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c cppregex.o winregex.o + clang -I %QJS%\porting -I%YAML%/include -I %QJS% -I./src -I../h -I ../platform/windows -DCONFIG_VERSION=\"2021-03-27\" -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 --rtlib=compiler-rt -o configmgr.exe configmgr.c embeddedjs.c %QJS%\quickjs.c %QJS%\cutils.c %QJS%\quickjs-libc.c %QJS%\libbf.c %QJS%\libregexp.c %QJS%\libunicode.c %QJS%\porting\winpthread.c %QJS%\porting\wintime.c %QJS%\porting\windirent.c %QJS%\porting\winunistd.c %YAML%/src/api.c %YAML%/src/reader.c %YAML%/src/scanner.c %YAML%/src/parser.c %YAML%/src/loader.c %YAML%/src/writer.c %YAML%/src/emitter.c %YAML%/src/dumper.c ../c/yaml2json.c ../c/jsonschema.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c cppregex.o winregex.o configmgr "../tests/schemadata" "" "LIBRARY(FOO):DIR(BAR)" yak - configmgr -s "../tests/schemadata" -p "LIBRARY(FOO):DIR(BAR)" extract "/foo/bar" + configmgr -s "../tests/schemadata" -p "LIBRARY(FOO):DIR(BAR)" extract "/zowe/setup/" configmgr -s "../tests/schemadata" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowebase.yaml)" validate - HERE - - Embeddedjs.c - new file - - JSON/wUneval -> source - - buildEmbJS - - customize with more roots - - run - - embedding wrapper - QJS JSValue->Json - - forks/pushes portable-quickjs clang -> libquickjs.a + configmgr -s "../tests/schemadata" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowebase.yaml)" extract "/zowe/setup/mvs/proclib" */ @@ -120,7 +113,6 @@ typedef int64_t ssize_t; #define CONFIG_PATH_OMVS_DIR 0x0002 #define CONFIG_PATH_OMVS_LIBRARY 0x0004 - typedef struct ConfigPathElement_tag { int flags; char *name; @@ -135,10 +127,11 @@ typedef struct ConfigManager_tag { ConfigPathElement *configPath; JsonSchema *topSchema; Json *config; - hashtable *schemaCache; - hashtable *configCache; - int traceLevel; - FILE *traceOut; + hashtable *schemaCache; + hashtable *configCache; + int traceLevel; + FILE *traceOut; + EmbeddedJS *ejs; } ConfigManager; #ifdef __ZOWE_OS_WINDOWS @@ -422,6 +415,8 @@ ConfigManager *makeConfigManager(char *configPathArg, char *rootSchemaDirectory, mgr->traceLevel = traceLevel; mgr->traceOut = traceOut; mgr->slh = makeShortLivedHeap(0x10000,0x100); + EmbeddedJS *ejs = makeEmbeddedJS(NULL); + mgr->ejs = ejs; if (buildConfigPath(mgr,configPathArg)){ printf("built config path failed\n");fflush(stdout); safeFree((char*)mgr,sizeof(ConfigManager)); @@ -489,6 +484,27 @@ static Json *readJson(ConfigManager *mgr, ConfigPathElement *pathElement){ } } +void freeConfigManager(ConfigManager *mgr){ + SLHFree(mgr->slh); + safeFree((char*)mgr,sizeof(ConfigManager)); +} + +#define JSON_POINTER_TOO_DEEP 101 +#define JSON_POINTER_ARRAY_INDEX_NOT_INTEGER 102 +#define JSON_POINTER_ARRAY_INDEX_OUT_OF_BOUNDS 103 + +#define ZCFG_ALLOC_HEAP 1 +#define ZCFG_ALLOC_SLH 2 +#define ZCFG_ALLOC_MEMCPY 3 + +/* These eventually need ZWE unique messages */ +#define ZCFG_SUCCESS 0 +#define ZCFG_TYPE_MISMATCH 1 +#define ZCFG_EVAL_FAILURE 2 +#define ZCFG_POINTER_TOO_DEEP JSON_POINTER_TOO_DEEP +#define ZCFG_POINTER_ARRAY_INDEX_NOT_INTEGER JSON_POINTER_ARRAY_INDEX_NOT_INTEGER +#define ZCFG_POINTER_ARRAY_INDEX_OUT_OF_BOUNDS JSON_POINTER_ARRAY_INDEX_OUT_OF_BOUNDS + /* need to collect violations as this goes */ static int overloadConfiguration(ConfigManager *mgr, @@ -516,30 +532,20 @@ static int overloadConfiguration(ConfigManager *mgr, static int loadConfigurations(ConfigManager *mgr){ ConfigPathElement *pathElement = mgr->configPath; - return overloadConfiguration(mgr,pathElement,pathElement->next); -} - -void freeConfigManager(ConfigManager *mgr){ - SLHFree(mgr->slh); - safeFree((char*)mgr,sizeof(ConfigManager)); + int overloadStatus = overloadConfiguration(mgr,pathElement,pathElement->next); + if (overloadStatus){ + return overloadStatus; + } else { + Json *evaluatedConfig = evaluateJsonTemplates(mgr->ejs,mgr->slh,mgr->config); + if (evaluatedConfig){ + mgr->config = evaluatedConfig; + return ZCFG_SUCCESS; + } else { + return ZCFG_EVAL_FAILURE; + } + } } - - -#define JSON_POINTER_TOO_DEEP 101 -#define JSON_POINTER_ARRAY_INDEX_NOT_INTEGER 102 -#define JSON_POINTER_ARRAY_INDEX_OUT_OF_BOUNDS 103 - -#define ZCFG_ALLOC_HEAP 1 -#define ZCFG_ALLOC_SLH 2 -#define ZCFG_ALLOC_MEMCPY 3 - -/* These eventually need ZWE unique messages */ -#define ZCFG_SUCCESS 0 -#define ZCFG_TYPE_MISMATCH 1 -#define ZCFG_POINTER_TOO_DEEP JSON_POINTER_TOO_DEEP -#define ZCFG_POINTER_ARRAY_INDEX_NOT_INTEGER JSON_POINTER_ARRAY_INDEX_NOT_INTEGER -#define ZCFG_POINTER_ARRAY_INDEX_OUT_OF_BOUNDS JSON_POINTER_ARRAY_INDEX_OUT_OF_BOUNDS - + /* Merging notes defaulting and merging requires the same data shape at all levels and sources @@ -567,9 +573,7 @@ static Json *jsonPointerDereference(Json *json, JsonPointer *jsonPointer, int *e printf("deref elt=0x%p, i=%d value=0x%p\n",element,i,value);fflush(stdout); } if (jsonIsArray(value)){ - printf("AAA\n");fflush(stdout); JsonArray *array = jsonAsArray(value); - printf("array case = 0x%p\n",array);fflush(stdout); if (element->type == JSON_POINTER_INTEGER){ int index = atoi(element->string); int arraySize = jsonArrayGetCount(array); @@ -583,12 +587,8 @@ static Json *jsonPointerDereference(Json *json, JsonPointer *jsonPointer, int *e return NULL; } } else if (jsonIsObject(value)){ - printf("OOO\n");fflush(stdout); JsonObject *object = jsonAsObject(value); - printf("object case = 0x%p\n",object);fflush(stdout); value = jsonObjectGetPropertyValue(object,element->string); - printf("value for key='%s' is 0x%p\n",element->string,value); - fflush(stdout); if (value == NULL){ *errorReason = JSON_POINTER_TOO_DEEP; return NULL; @@ -660,9 +660,7 @@ int cfgGetAny(ConfigManager *mgr, char *value, int allocOptions, void *mem, ...) static void extractText(ConfigManager *mgr, JsonPointer *jp, FILE *out){ Json *value = NULL; - printf("extract ckpt.1\n");fflush(stdout); int status = cfgGetAnyJ(mgr,&value,jp); - printf("extract ckpt.2\n");fflush(stdout); if (status){ fprintf(out,"error not found, reason=%d",status); } else { @@ -796,29 +794,29 @@ static size_t escapeForEnv(const char *s, bool isKey, char *buffer, size_t buffe return pos; } -static outputEnvKey(FILE * out, const char *str) { +static void outputEnvKey(FILE * out, const char *str) { size_t escapedSize = strlen(str) * 2 + 1; char escaped[escapedSize]; escapeForEnv(str, true, escaped, escapedSize); fprintf(out, "%s=", escaped); } -static outputEnvString(FILE * out, const char *str) { +static void outputEnvString(FILE * out, const char *str) { size_t escapedSize = strlen(str) * 2 + 1; char escaped[escapedSize]; escapeForEnv(str, false, escaped, escapedSize); fprintf(out, "\"%s\"\n", escaped); } -static outputEnvInt64(FILE * out, int64_t num) { +static void outputEnvInt64(FILE * out, int64_t num) { fprintf(out, "%lld\n", num); } -static outputEnvDouble(FILE * out, double num) { +static void outputEnvDouble(FILE * out, double num) { fprintf(out, "%f\n", num); } -static outputEnvBoolean(FILE * out, bool b) { +static void outputEnvBoolean(FILE * out, bool b) { fprintf(out, "%s\n", b ? "true": "false"); } diff --git a/c/embeddedjs.c b/c/embeddedjs.c index d1eca7ca7..4689f3588 100644 --- a/c/embeddedjs.c +++ b/c/embeddedjs.c @@ -568,15 +568,19 @@ static bool evaluationVisitor(void *context, Json *json, Json *parent, char *key if (evalStatus){ printf("failed to evaluate '%s', status=%d\n",source,evalStatus); } else { - printf("evaluation succeeded\n"); - dumpbuffer((char*)&output,sizeof(JSValue)); + /* + printf("evaluation succeeded\n"); + dumpbuffer((char*)&output,sizeof(JSValue)); + */ Json *evaluationResult = ejsJSToJson(ejs,output,evalContext->slh); - printf("evaluationResult back in Json 0x%p\n",evaluationResult); - fflush(stdout); - if (keyInParent){ - setJsonProperty(jsonAsObject(parent),keyInParent,evaluationResult); + if (evaluationResult){ + if (keyInParent){ + setJsonProperty(jsonAsObject(parent),keyInParent,evaluationResult); + } else { + setJsonArrayElement(jsonAsArray(parent),indexInParent,evaluationResult); + } } else { - setJsonArrayElement(jsonAsArray(parent),indexInParent,evaluationResult); + printf("Warning EJS failed to translate eval result JSValue to Json\n"); } } } diff --git a/c/yaml2json.c b/c/yaml2json.c index 3e9638bc6..46ff54fe8 100644 --- a/c/yaml2json.c +++ b/c/yaml2json.c @@ -447,7 +447,7 @@ static int buildTemplateJSON(JsonBuilder *b, Json *parent, char *parentKey, } } else { char *lastFrag = extractString(b,tail,nativeValue+valueLength); - printf("lastFrag = '%s'\n",lastFrag); + /* printf("lastFrag = '%s'\n",lastFrag); */ if (strlen(lastFrag) > 0){ addPlusIfNecessary(p,sourceCCSID,&first); jsonWriteParseably(p,lastFrag,strlen(lastFrag),true,false,sourceCCSID); @@ -462,8 +462,10 @@ static int buildTemplateJSON(JsonBuilder *b, Json *parent, char *parentKey, if (!status){ jsonBufferTerminateString(buffer); char *sourceCode = jsonBufferCopy(buffer); - printf("source code is: %s\n",sourceCode); - dumpbuffer(sourceCode,strlen(sourceCode)); + if (b->traceLevel >= 1){ + printf("embedded source code is (tl=%d): %s\n",b->traceLevel,sourceCode); + dumpbuffer(sourceCode,strlen(sourceCode)); + } jsonBuildString(b,object,"source",sourceCode,strlen(sourceCode),&status); } } diff --git a/tests/jstest.c b/tests/jstest.c index ff1f7ffcd..f42fef944 100644 --- a/tests/jstest.c +++ b/tests/jstest.c @@ -48,6 +48,8 @@ A configuration source is a user (or UI) created document - is owned by the user - is text + - "No Dark Matter" - all source/paths must be known + - Zowe-wide Changes to sources/path must be reviewed - can be place under Version Control, eg Git - is not modified by installers, launchers, etc. - can be syntactically validated and prompted for syntax in editors diff --git a/tests/schemadata/zowebase.yaml b/tests/schemadata/zowebase.yaml index d8f7360e3..8ac2c0ea1 100644 --- a/tests/schemadata/zowebase.yaml +++ b/tests/schemadata/zowebase.yaml @@ -6,24 +6,12 @@ zowe: setup: # MVS data set related configurations mvs: - # **COMMONLY_CUSTOMIZED** - # where Zowe MVS data sets will be installed - # hlq: IBMUSER.ZWEV2 - hlq: IBMUSER.FOO - # **COMMONLY_CUSTOMIZED** - # PROCLIB where Zowe STCs will be copied over - proclib: VENDOR.PROCLIB - # **COMMONLY_CUSTOMIZED** - # Zowe PARMLIB + xxx: "FOOBAR" + hlq: ${{ zowe.setup.mvs.xxx+".ZWEV2" }} + proclib: ${{ zowe.setup.mvs.hlq+".PROCLIB" }} parmlib: IBMUSER.ZWEV2.CUST.PARMLIB - # **COMMONLY_CUSTOMIZED** - # JCL library where Zowe will store temporary JCLs during initialization jcllib: IBMUSER.ZWEV2.CUST.JCLLIB - # APF authorized LOADLIB for Zowe - # Optional. If it's empty, .SZWEAUTH will be APF authorized. authLoadlib: - # **COMMONLY_CUSTOMIZED** - # APF authorized LOADLIB for Zowe ZIS Plugins authPluginLib: IBMUSER.ZWEV2.CUST.ZWESAPL # >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> diff --git a/tests/schemadata/zoweoverrides.yaml b/tests/schemadata/zoweoverrides.yaml index e3524ddc1..51b1f1a88 100644 --- a/tests/schemadata/zoweoverrides.yaml +++ b/tests/schemadata/zoweoverrides.yaml @@ -1,4 +1,4 @@ zowe: setup: mvs: - hlq: SOMELIB.BAR + jcllib: ${{ zowe.setup.mvs.hlq + ".FRED" }} From 447cea88f0404e03ffa95f4397db349c97495e43 Mon Sep 17 00:00:00 2001 From: Joe Devlin Date: Tue, 1 Mar 2022 00:08:57 -0600 Subject: [PATCH 24/42] Many fixes to allow XLClang to be used with zowe-common-c. It is a more accurate compiler and had about 100 warnings to fix Signed-off-by: Joe Devlin --- build/build_cmgr_xlclang.sh | 137 ++++++++++++++++++++++++++++++++++++ c/bpxskt.c | 35 +++++++-- c/charsets.c | 16 ++++- c/collections.c | 40 +++++------ c/configmgr.c | 34 +++++++++ c/embeddedjs.c | 6 +- c/httpserver.c | 13 ++++ c/json.c | 14 ++-- c/le.c | 4 +- c/scheduling.c | 3 + c/timeutls.c | 16 ++--- c/utils.c | 47 +++++-------- c/zos.c | 67 +++++++++++------- c/zosfile.c | 29 +++++++- h/qsam.h | 32 ++++----- h/zowetypes.h | 29 +++++++- platform/posix/psxregex.c | 2 + platform/posix/psxregex.h | 4 +- 18 files changed, 410 insertions(+), 118 deletions(-) create mode 100755 build/build_cmgr_xlclang.sh diff --git a/build/build_cmgr_xlclang.sh b/build/build_cmgr_xlclang.sh new file mode 100755 index 000000000..de88175f1 --- /dev/null +++ b/build/build_cmgr_xlclang.sh @@ -0,0 +1,137 @@ +#!/bin/sh + +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. + +WORKING_DIR=$(dirname "$0") + +# set -v + +echo "********************************************************************************" +echo "Building configmgr..." + +rm -f "${COMMON}/bin/configmgr" + +mkdir -p "${WORKING_DIR}/tmp-configmgr" && cd "$_" +COMMON="../.." + +QUICKJS="/u/zossteam/jdevlin/git2022/quickjs" + +LIBYAML="/u/zossteam/jdevlin/git2022/libyaml" +MAJOR=0 +MINOR=2 +PATCH=5 +VERSION="\"${MAJOR}.${MINOR}.${PATCH}\"" + +#if [ ! -d "${LIBYAML}" ]; then +# git clone git@github.com:yaml/libyaml.git +#fi + +# export _C89_ACCEPTABLE_RC=4 + +xlclang \ + -c \ + -q64 \ + -qascii \ + "-Wc,float(ieee),longname,langlvl(extc99),gonum,goff,ASM,asmlib('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN')" \ + -DYAML_VERSION_MAJOR=${MAJOR} \ + -DYAML_VERSION_MINOR=${MINOR} \ + -DYAML_VERSION_PATCH=${PATCH} \ + -DYAML_VERSION_STRING="${VERSION}" \ + -DYAML_DECLARE_STATIC=1 \ + -D_OPEN_SYS_FILE_EXT=1 \ + -D_XOPEN_SOURCE=600 \ + -D_OPEN_THREADS=1 \ + -DCONFIG_VERSION=\"2021-03-27\" \ + -I "${LIBYAML}/include" \ + -I "${QUICKJS}" \ + ${LIBYAML}/src/api.c \ + ${LIBYAML}/src/reader.c \ + ${LIBYAML}/src/scanner.c \ + ${LIBYAML}/src/parser.c \ + ${LIBYAML}/src/loader.c \ + ${LIBYAML}/src/writer.c \ + ${LIBYAML}/src/emitter.c \ + ${LIBYAML}/src/dumper.c \ + ${QUICKJS}/cutils.c \ + ${QUICKJS}/quickjs.c \ + ${QUICKJS}/quickjs-libc.c \ + ${QUICKJS}/libunicode.c \ + ${QUICKJS}/libregexp.c \ + ${QUICKJS}/porting/polyfill.c +#then +# echo "Done with qascii-compiled open-source parts" +#else +# echo "Build failed" +# exit 8 +#fi + +xlclang \ + -q64 \ + "-Wc,float(ieee),longname,langlvl(extc99),gonum,goff,ASM,asmlib('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN')" \ + -D_OPEN_SYS_FILE_EXT=1 \ + -D_XOPEN_SOURCE=600 \ + -D_OPEN_THREADS=1 \ + -DNOIBMHTTP=1 \ + -I "${COMMON}/h" \ + -I "${COMMON}/platform/posix" \ + -I "${LIBYAML}/include" \ + -I "${QUICKJS}" \ + -o "${COMMON}/bin/configmgr" \ + api.o \ + reader.o \ + scanner.o \ + parser.o \ + loader.o \ + writer.o \ + emitter.o \ + dumper.o \ + cutils.o \ + quickjs.o \ + quickjs-libc.o \ + libunicode.o \ + libregexp.o \ + polyfill.o \ + ${COMMON}/c/alloc.c \ + ${COMMON}/c/bpxskt.c \ + ${COMMON}/c/charsets.c \ + ${COMMON}/c/collections.c \ + ${COMMON}/c/configmgr.c \ + ${COMMON}/c/embeddedjs.c \ + ${COMMON}/c/json.c \ + ${COMMON}/c/jsonschema.c \ + ${COMMON}/c/le.c \ + ${COMMON}/c/logging.c \ + ${COMMON}/platform/posix/psxregex.c \ + ${COMMON}/c/recovery.c \ + ${COMMON}/c/scheduling.c \ + ${COMMON}/c/timeutls.c \ + ${COMMON}/c/utils.c \ + ${COMMON}/c/xlate.c \ + ${COMMON}/c/yaml2json.c \ + ${COMMON}/c/zos.c \ + ${COMMON}/c/zosfile.c +#then +# echo "Build successful" +# ls -l "${COMMON}/bin" +# exit 0 +#else +# # remove configmgr in case the linker had RC=4 and produced the binary +# rm -f "${COMMON}/bin/configmgr" +# echo "Build failed" +# exit 8 +#fi + + +# This program and the accompanying materials are +# made available under the terms of the Eclipse Public License v2.0 which accompanies +# this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html +# +# SPDX-License-Identifier: EPL-2.0 +# +# Copyright Contributors to the Zowe Project. diff --git a/c/bpxskt.c b/c/bpxskt.c index fa8d74d8a..89d960d2b 100644 --- a/c/bpxskt.c +++ b/c/bpxskt.c @@ -107,6 +107,28 @@ #endif +/* xlclang and clang what prototypes, these are incomplete, but quiet the compiler */ +int BPXSOC(); +int BPXCON(); +int BPXGHN(); +int BPXSLP(); +int BPXCHR(); +int BPXBND(); +int BPXLSN(); +int BPXACP(); +int BPXSEL(); +int BPXOPT(); +int BPXGNM(); +int BPXSTO(); +int BPXRFM(); +int BPXHST(); +int BPXIOC(); +int BPXRED(); +int BPXWRT(); +int BPXFCT(); +int BPXCLO(); + + #define SOCK_SO_REUSEADDR 0x00000004 #define SOCK_SO_SNDBUF 0x00001001 #define SOCK_SO_RCVBUF 0x00001002 @@ -151,6 +173,7 @@ void bpxSleep(int seconds) int socketInit(char *uniqueName){ /* do nothing for now */ + return 0; } SocketAddress *makeSocketAddr(InetAddr *addr, @@ -172,7 +195,7 @@ SocketAddress *makeSocketAddrIPv6(InetAddr *addr, unsigned short port){ SocketAddress *address = (SocketAddress*)safeMalloc31(sizeof(SocketAddress),"BPX SocketAddress"); memset(address,0,sizeof(SocketAddress)); if (socketTrace){ - printf("socket address at 0x%x\n",address); + printf("socket address at 0x%p\n",address); } address->length = 26; address->family = AF_INET6; @@ -181,7 +204,7 @@ SocketAddress *makeSocketAddrIPv6(InetAddr *addr, unsigned short port){ address->data6 = addr->data.data6; } if (socketTrace){ - printf("about to return socket address at 0x%x\n",address); + printf("about to return socket address at 0x%p\n",address); } return address; } @@ -744,7 +767,7 @@ int getV4HostByName(char *string){ /* dumpbuffer((char*)hostent,20); */ for (i=0; ilength; i++){ if (socketTrace){ - printf(" addr[%d] = %x\n",i,hostent->addrList[i]); + printf(" addr[%d] = 0x%p\n",i,hostent->addrList[i]); } if (hostent->addrList[i]){ numericAddress = *(hostent->addrList[i]); @@ -1276,7 +1299,7 @@ int udpReceiveFrom(Socket *socket, int socketAddressSize = SOCKET_ADDRESS_SIZE_IPV4; if (socketTrace > 2){ - printf("receiveFrom into buffer=0x%x bufLen=%d retVal=%d retCode=%d reasonCode=%d\n", + printf("receiveFrom into buffer=0x%p bufLen=%d retVal=%d retCode=%d reasonCode=%d\n", buffer,bufferLength,returnValue,*returnCode,*reasonCode); } @@ -1300,7 +1323,7 @@ int udpReceiveFrom(Socket *socket, return -1; } else { if (socketTrace > 2){ - printf("recvFrom into buffer=0x%x %d bytes\n",buffer,returnValue);fflush(stdout); + printf("recvFrom into buffer=0x%p %d bytes\n",buffer,returnValue);fflush(stdout); } *returnCode = 0; *reasonCode = 0; @@ -1680,7 +1703,7 @@ int not_main(int argc, char **argv){ sleep(1); } } - + return 0; } diff --git a/c/charsets.c b/c/charsets.c index 1a6e6ddcc..6db31041b 100644 --- a/c/charsets.c +++ b/c/charsets.c @@ -118,7 +118,7 @@ int convertCharset(char *input, return 0; } -#elif defined(__ZOWE_OS_ZOS) +#elif defined(__ZOWE_OS_ZOS) && !defined(__ZOWE_COMP_XLCLANG) /* @@ -167,7 +167,11 @@ static const union { }; +#ifdef __ZOWE_COMP_XLCLANG +#include "CUNHC.h" +#else #include "//'SYS1.SCUNHF(CUNHC)'" +#endif int convertCharset(char *input, int inputLength, @@ -223,6 +227,7 @@ int convertCharset(char *input, case CHARSET_OUTPUT_USE_SLH: outputBuffer = SLHAlloc(slh,outputAllocLength); outputLength = outputAllocLength; + *output = outputBuffer; break; } parms.Targ_Buf_Ptr = outputBuffer; @@ -253,11 +258,15 @@ int convertCharset(char *input, #endif *conversionOutputLength = (((char*)parms.Targ_Buf_Ptr) - outputBuffer); + printf("inputLen=%d reasonCode = %d src=%d targ=%d\n",inputLength,parms.Reason_Code,parms.Src_CCSID,parms.Targ_CCSID); + fflush(stdout); + if (parms.Return_Code){ if (outputMode == CHARSET_OUTPUT_SAFE_MALLOC){ safeFree(parms.Targ_Buf_Ptr,outputAllocLength); } *reasonCode = parms.Return_Code; + return CHARSET_CONVERSION_ROUTINE_FAILURE; } else { if (outputMode == CHARSET_OUTPUT_SAFE_MALLOC){ @@ -310,7 +319,9 @@ int getCharsetCode(const char *charsetName) { } } -#elif defined(__ZOWE_OS_LINUX) || defined(__ZOWE_OS_AIX) /* end of ZOWE_OS_ZOS */ +/* End of Traditional METAL and XLC cases, since linkage(OS64_NOSTACK) doesn't work in xlclang and clang + some C code goes through here, too. We should short circuit easy special cases here some day */ +#elif defined(__ZOWE_OS_LINUX) || defined(__ZOWE_OS_AIX) || (defined(__ZOWE_OS_ZOS) && defined(__ZOWE_COMP_XLCLANG)) #include #include @@ -391,6 +402,7 @@ int convertCharset(char *input, case CHARSET_OUTPUT_USE_SLH: outputBuffer = SLHAlloc(slh,outputAllocLength); outputSize = outputAllocLength; + *output = outputBuffer; break; default: return CHARSET_INTERNAL_ERROR; diff --git a/c/collections.c b/c/collections.c index 4abc9bdd1..65724bcb4 100644 --- a/c/collections.c +++ b/c/collections.c @@ -1101,7 +1101,7 @@ static int compareAndLoad(long *oldCounter, long *counterAddress, long *sourceAd : "r"(counterAddress), "r"(sourceAddress), "r"(resultAddress), "i"(sizeof(oldCounter)) : - "r0 r1 r14 r15"); + "r0","r1","r14","r15"); return status; } @@ -1133,7 +1133,7 @@ static int compareAndSwapTriple(long *oldCounter, long newCounter, long *counter : "r"(counterAddress), "r"(parms), "r"(newCounter), "i"(sizeof(oldCounter)) : - "r0 r1 r14 r15"); + "r0","r1","r14","r15"); return status; } @@ -1142,7 +1142,7 @@ void qEnqueue(Queue *q, QueueElement *newElement) { union { long long alignit; CSTSTParms parms; - }; + } vars; newElement->next = NULL; @@ -1197,7 +1197,7 @@ void qEnqueue(Queue *q, QueueElement *newElement) { } else { - memset(&parms,0,sizeof(CSTSTParms)); + memset(&vars.parms,0,sizeof(CSTSTParms)); /* Insert a queue element at the tail of a queue using PLO. @@ -1216,17 +1216,17 @@ void qEnqueue(Queue *q, QueueElement *newElement) { void *desiredQHead = (q->head ? q->head : newElement); /* HEAD */ - parms.thing1 = (long)desiredQHead; - parms.thing1Addr = &(q->head); + vars.parms.thing1 = (long)desiredQHead; + vars.parms.thing1Addr = &(q->head); /* TAIL */ - parms.thing2 = (long)newElement; - parms.thing2Addr = &(q->tail); + vars.parms.thing2 = (long)newElement; + vars.parms.thing2Addr = &(q->tail); /* PENULTIMATE */ - parms.thing3 = (long)newElement; + vars.parms.thing3 = (long)newElement; QueueElement *last = q->tail; - parms.thing3Addr = (q->tail ? &(last->next) : &(q->tail)); + vars.parms.thing3Addr = (q->tail ? &(last->next) : &(q->tail)); - if (compareAndSwapTriple(&lockCounter,newCounter,&q->counter,&parms)){ + if (compareAndSwapTriple(&lockCounter,newCounter,&q->counter,&vars.parms)){ break; } } @@ -1249,7 +1249,7 @@ QueueElement *qDequeue(Queue *q) { union { long long alignit; CSTSTParms parms; - }; + } vars; QueueElement *currentHead; @@ -1312,7 +1312,7 @@ QueueElement *qDequeue(Queue *q) { } else { - memset(&parms,0,sizeof(CSTSTParms)); + memset(&vars.parms,0,sizeof(CSTSTParms)); /* Remove a queue element from the head of a queue using PLO. @@ -1335,18 +1335,18 @@ QueueElement *qDequeue(Queue *q) { void *desiredQTail = (desiredQHead ? q->tail : NULL); /* HEAD */ - parms.thing1 = (long)desiredQHead; - parms.thing1Addr = &(q->head); + vars.parms.thing1 = (long)desiredQHead; + vars.parms.thing1Addr = &(q->head); /* TAIL */ - parms.thing2 = (long)desiredQTail; - parms.thing2Addr = &(q->tail); + vars.parms.thing2 = (long)desiredQTail; + vars.parms.thing2Addr = &(q->tail); /* CURRENT_HEAD */ - /* parms.thing3 = NULL -- was set to NULL when parmData was initialized */ + /* vars.parms.thing3 = NULL -- was set to NULL when parmData was initialized */ QueueElement *last = NULL; - parms.thing3Addr = &(currentHead->next); + vars.parms.thing3Addr = &(currentHead->next); - if (compareAndSwapTriple(&lockCounter,newCounter,&q->counter,&parms)) + if (compareAndSwapTriple(&lockCounter,newCounter,&q->counter,&vars.parms)) break; } } diff --git a/c/configmgr.c b/c/configmgr.c index 932cf97bd..94e403ad9 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -104,9 +104,26 @@ typedef int64_t ssize_t; configmgr -s "../tests/schemadata" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowebase.yaml)" validate + configmgr -t 2 -s "../tests/schemadata" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowebase.yaml)" validate + configmgr -s "../tests/schemadata" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowebase.yaml)" extract "/zowe/setup/mvs/proclib" + + -- Compilation with XLCLang on ZOS ------------------------------------- + + export YAML="/u/zossteam/jdevlin/git2022/libyaml" + export QJS="/u/zossteam/jdevlin/git2022/quickjs" + + NOTE!! charsets.c requires a path that is NOT unix path. xlclang and clang will not support this + + Easy, maybe temporary workaround is to copy this H file somewhere + cp "//'SYS1.SCUNHF(CUNHC)'" ../h/CUNHC.h + + xlclang -q64 -qascii -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 -DCONFIG_VERSION=\"2021-03-27\" -D_OPEN_SYS_FILE_EXT=1 -D_XOPEN_SOURCE=600 -D_OPEN_THREADS=1 -DSUBPOOL=132 "-Wc,float(ieee),longname,langlvl(extc99),gonum,goff,ASM,asmlib('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN')" -I ../h -I ../platform/posix -I ${YAML}/include -I ${QJS} -Wbitwise-op-parentheses -o configmgr configmgr.c yaml2json.c embeddedjs.c jsonschema.c json.c xlate.c charsets.c zosfile.c logging.c recovery.c scheduling.c zos.c le.c collections.c timeutls.c utils.c alloc.c ../platform/posix/psxregex.c ${QJS}/quickjs.c ${QJS}/cutils.c ${QJS}/quickjs-libc.c ${QJS}/libregexp.c ${QJS}/libunicode.c ${QJS}/porting/polyfill.c ${YAML}/src/api.c ${YAML}/src/reader.c ${YAML}/src/scanner.c ${YAML}/src/parser.c ${YAML}/src/loader.c ${YAML}/src/writer.c ${YAML}/src/emitter.c ${YAML}/src/dumper.c + + + */ #define CONFIG_PATH_OMVS_FILE 0x0001 @@ -352,6 +369,8 @@ static bool addPathElement(ConfigManager *mgr, char *pathElementArg){ } static int buildConfigPath(ConfigManager *mgr, char *configPathArg){ + printf("JOE buildConfigPath\n"); + fflush(stdout); int pos = 0; int len = strlen(configPathArg); while (pos < len){ @@ -416,7 +435,10 @@ ConfigManager *makeConfigManager(char *configPathArg, char *rootSchemaDirectory, mgr->traceOut = traceOut; mgr->slh = makeShortLivedHeap(0x10000,0x100); EmbeddedJS *ejs = makeEmbeddedJS(NULL); + printf("really\n"); + fflush(stdout); mgr->ejs = ejs; + trace(mgr,DEBUG,"before build config path\n"); if (buildConfigPath(mgr,configPathArg)){ printf("built config path failed\n");fflush(stdout); safeFree((char*)mgr,sizeof(ConfigManager)); @@ -429,12 +451,15 @@ ConfigManager *makeConfigManager(char *configPathArg, char *rootSchemaDirectory, int returnCode = 0; int reasonCode = 0; FileInfo yamlFileInfo; + trace(mgr,DEBUG,"before file info\n"); int infoStatus = fileInfo(zoweYamlPath,&yamlFileInfo,&returnCode,&reasonCode); if (infoStatus){ + trace(mgr,INFO,"failed to get fileInfo of '%s', infoStatus=%d\n",zoweYamlPath,infoStatus); freeConfigManager(mgr); return NULL; } char errorBuffer[1024]; + trace(mgr,DEBUG,"before jsonParseFile info\n"); Json *jsonWithSchema = jsonParseFile2(mgr->slh,zoweYamlPath,errorBuffer,1024); if (jsonWithSchema == NULL){ trace(mgr,INFO,"failed to read JSON with base schema: %s\n",errorBuffer); @@ -869,6 +894,15 @@ static void convertJsonArrayToEnv(FILE *out, const char *path, JsonArray *array) } } +/* + Here: + 1) Detect qascii or not + 2) store in code + 3) BPX routines do NOT tolerate qASCII like c-library + 4) fileInfo tag must be used on ZOS target to potentially preprocess input + 5) maybe Leonty has done this for yaml (or not) + */ + int main(int argc, char **argv){ char *rootSchemaDirectory = NULL; // Read-only ZOWE runtime_directory (maybe diff --git a/c/embeddedjs.c b/c/embeddedjs.c index 4689f3588..d42900d87 100644 --- a/c/embeddedjs.c +++ b/c/embeddedjs.c @@ -465,8 +465,10 @@ char *json2JS(Json *json){ /* jsonPrinter *p = makeBufferJsonPrinter(sourceCCSID,buffer); */ #ifdef __ZOWE_OS_WINDOWS int stdoutFD = _fileno(stdout); -#else +#elif defined(STDOUT_FILENO) int stdoutFD = STDOUT_FILENO; +#else + int stdoutFD = 1; /* this looks hacky, but it's been true for about 50 years */ #endif jsonPrinter *p = makeJsonPrinter(stdoutFD); jsonEnablePrettyPrint(p); @@ -657,7 +659,7 @@ EmbeddedJS *makeEmbeddedJS(EmbeddedJS *sharedRuntimeEJS){ /* can be NULL */ JSValue throwaway = ejsEvalBuffer(embeddedJS, str, strlen(str), "", JS_EVAL_TYPE_MODULE, &evalStatus); } - + printf("returning embeddedJS 0x%p\n",embeddedJS); return embeddedJS; } diff --git a/c/httpserver.c b/c/httpserver.c index e304fcfcb..94958221e 100644 --- a/c/httpserver.c +++ b/c/httpserver.c @@ -2518,6 +2518,19 @@ static char *getCookieValue(HttpRequest *request, char *cookieName){ return NULL; } +#ifdef __ZOWE_OS_ZOS +static int isLowerCasePasswordAllowed(){ + RCVT* rcvt = getCVT()->cvtrac; + return (RCVTFLG3_BIT_RCVTPLC & (rcvt->rcvtflg3)); + /* if lower-case pw allowed */ +} +#else +static int isLowerCasePasswordAllowed(){ + return TRUE; +} +#endif + + #ifdef __ZOWE_OS_ZOS static int safAuthenticate(HttpService *service, HttpRequest *request, AuthResponse *authResponse){ int safStatus = 0, racfStatus = 0, racfReason = 0; diff --git a/c/json.c b/c/json.c index d0a4d5058..9873fbe31 100644 --- a/c/json.c +++ b/c/json.c @@ -60,9 +60,13 @@ typedef int64_t ssize_t; /* - * - c89 -DTEST_JSON_PARSER "-Wc,langlvl(extc99),gonum,goff,hgpr,roconst,ASM,asmlib('SYS1.MACLIB')" -o parser -I ../h json.c utils.c alloc.c bpxskt.c charsets.c - c89 -DTEST_JSON_PRINTER "-Wc,langlvl(extc99),gonum,goff,hgpr,roconst,ASM,asmlib('SYS1.MACLIB')" -o printer -I ../h json.c utils.c alloc.c bpxskt.c charsets.c + + Some unit tests for parsing and printing. Updated 2022 for current filenames and modularity, and 64-bitness. + + xlc -q64 -DTEST_JSON_PARSER -D_OPEN_SYS_FILE_EXT=1 -D_XOPEN_SOURCE=600 -D_OPEN_THREADS=1 -DSUBPOOL=132 "-Wc,float(ieee),longname,langlvl(extc99),gonum,goff,ASM,asmlib('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN')" -I ../h -o jsonparser json.c zosfile.c collections.c charsets.c logging.c zos.c recovery.c scheduling.c le.c timeutls.c utils.c alloc.c + + xlc -q64 -DTEST_JSON_PRINTER -D_OPEN_SYS_FILE_EXT=1 -D_XOPEN_SOURCE=600 -D_OPEN_THREADS=1 -DSUBPOOL=132 "-Wc,float(ieee),longname,langlvl(extc99),gonum,goff,ASM,asmlib('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN')" -I ../h -o jsonprinter json.c zosfile.c collections.c charsets.c logging.c zos.c recovery.c scheduling.c le.c timeutls.c utils.c alloc.c + */ #define CONVERSION_BUFFER_DEFAULT_SIZE 8192 @@ -266,7 +270,7 @@ convertToUtf8(jsonPrinter *p, size_t len, char text[len], int inputCCSID) { newBuf = safeRealloc(p->_conversionBuffer, newSize, p->_conversionBufferSize, "JSON conversion buffer"); if (newBuf == NULL) { - //the old buffer will be free'd by freeJsonPrinter() if allocated + /* the old buffer will be free'd by freeJsonPrinter() if allocated */ JSONERROR("JSON: error, not enough memory to convert\n"); return -1; } @@ -2485,6 +2489,8 @@ Json *jsonParseFile(ShortLivedHeap *slh, const char *filename, char* errorBuffer } Json *jsonParseFile2(ShortLivedHeap *slh, const char *filename, char* errorBufferOrNull, int errorBufferSize){ + printf("JOE: jsonParseFile2\n"); + fflush(stdout); return jsonParseFileInternal(slh,filename,errorBufferOrNull,errorBufferSize,JSON_PARSE_VERSION_2); } diff --git a/c/le.c b/c/le.c index d4e243560..7a639bfcb 100644 --- a/c/le.c +++ b/c/le.c @@ -165,12 +165,12 @@ static LibraryFunction *findLibraryFunction(int rtlVectorOffset){ void showRTL(){ CAA *caa = (CAA*)getCAA(); void **rtlVector = caa->runtimeLibraryVectorTable; - printf("RTL Vector at 0x%x\n",rtlVector); + printf("RTL Vector at 0x%p\n",rtlVector); dumpbuffer((char*)rtlVector,ESTIMATED_RTL_VECTOR_SIZE); int estimatedEntries = ESTIMATED_RTL_VECTOR_SIZE / 4; for (int i=2; idateFlags = 0x03; @@ -266,10 +266,10 @@ int stckToTimestamp(int64 stck, char *output) int timestampToSTCK(char *todText, int64 *stck, int64 offset){ int *value = (int*)stck; - int *CVTPTR = (int*)((int*)16); - int *CVT = (int*)CVTPTR[0]; - int *SFT = (int*)(CVT[772/4]); - int serviceRoutine = SFT[352/4]; + int *mem = (int*)0; + void *cvt = INT2PTR(mem[0x10/4]); + int *sft = (int*)INT2PTR(((int*)cvt)[772/4]); + int serviceRoutine = sft[352/4]; int res = 0; CONVTODPlist plist; diff --git a/c/utils.c b/c/utils.c index 60c8c931f..050e5a5c6 100644 --- a/c/utils.c +++ b/c/utils.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include "qsam.h" #include "metalio.h" @@ -24,6 +25,7 @@ #include #include #include +#include #include #endif @@ -1078,8 +1080,8 @@ void encodeBase64NoAlloc(const char buf[], int size, char result[], int *resultS int byte1 = data[inCursor++] & 0xff; int byte2 = data[inCursor++] & 0xff; *resPtr++ = (char)(translation[byte0 >> 2]); - *resPtr++ = (char)(translation[(byte0 << 4) & 0x3f | (byte1 >> 4)]); - *resPtr++ = (char)(translation[(byte1 << 2) & 0x3f | (byte2 >> 6)]); + *resPtr++ = (char)(translation[((byte0 << 4) & 0x3f) | (byte1 >> 4)]); + *resPtr++ = (char)(translation[((byte1 << 2) & 0x3f) | (byte2 >> 6)]); *resPtr++ = (char)(translation[byte2 & 0x3f]); } @@ -1093,7 +1095,7 @@ void encodeBase64NoAlloc(const char buf[], int size, char result[], int *resultS } else{ int byte1 = data[inCursor++] & 0xff; - *resPtr++ = (char)(translation[(byte0 << 4) & 0x3f | (byte1 >> 4)]); + *resPtr++ = (char)(translation[((byte0 << 4) & 0x3f) | (byte1 >> 4)]); *resPtr++ = (char)(translation[(byte1 << 2) & 0x3f]); *resPtr++ = equalsChar; } @@ -1297,19 +1299,19 @@ int base32Decode (int alphabet, if (ch2 == -2) { break; } - output [outputIndex++] = (char) ((ch1 << 6) | (ch2 << 1) | (ch3 >> 4) & 0xff); + output [outputIndex++] = (char) ((ch1 << 6) | (ch2 << 1) | ((ch3 >> 4) & 0xff)); if (ch4 == -2) { break; } - output [outputIndex++] = (char) ((ch3 << 4) | (ch4 >> 1) & 0xff); + output [outputIndex++] = (char) ((ch3 << 4) | ((ch4 >> 1) & 0xff)); if (ch5 == -2) { break; } - output [outputIndex++] = (char) ((ch4 << 7) | (ch5 << 2) | (ch6 >> 3) & 0xff); + output [outputIndex++] = (char) ((ch4 << 7) | (ch5 << 2) | ((ch6 >> 3) & 0xff)); if (ch7 == -2) { break; } - output [outputIndex++] = (char) ((ch6 << 5) | (ch7) & 0xff); + output [outputIndex++] = (char) ((ch6 << 5) | ((ch7) & 0xff)); } *outputLength = outputIndex; return 0; @@ -1356,12 +1358,12 @@ int base32Encode (int alphabet, byte3 = *input++ & 0xff; byte4 = *input++ & 0xff; output [outputIndex++] = (char)(encodeTable [(byte0 >> 3) & 0x1f]); - output [outputIndex++] = (char)(encodeTable [(byte0 << 2) & 0x1f | (byte1 >> 6)]); + output [outputIndex++] = (char)(encodeTable [((byte0 << 2) & 0x1f) | (byte1 >> 6)]); output [outputIndex++] = (char)(encodeTable [(byte1 >> 1) & 0x1f]); - output [outputIndex++] = (char)(encodeTable [(byte1 << 4) & 0x1f | (byte2 >> 4)]); - output [outputIndex++] = (char)(encodeTable [(byte2 << 1) & 0x1f | (byte3 >> 7)]); + output [outputIndex++] = (char)(encodeTable [((byte1 << 4) & 0x1f) | (byte2 >> 4)]); + output [outputIndex++] = (char)(encodeTable [((byte2 << 1) & 0x1f) | (byte3 >> 7)]); output [outputIndex++] = (char)(encodeTable [(byte3 >> 2) & 0x1f]); - output [outputIndex++] = (char)(encodeTable [(byte3 << 3) & 0x1f | (byte4 >> 5)]); + output [outputIndex++] = (char)(encodeTable [((byte3 << 3) & 0x1f) | (byte4 >> 5)]); output [outputIndex++] = (char)(encodeTable [(byte4 & 0x1f)]); } if (numCharsLeft) { @@ -1379,7 +1381,7 @@ int base32Encode (int alphabet, } else if (numCharsLeft == 2) { int byte1 = *input++; - output [outputIndex++] = (char)(encodeTable [(byte0 << 2) & 0x1f | (byte1 >> 6)]); + output [outputIndex++] = (char)(encodeTable [((byte0 << 2) & 0x1f) | (byte1 >> 6)]); output [outputIndex++] = (char)(encodeTable [(byte1 >> 1) & 0x1f]); output [outputIndex++] = (char)(encodeTable [(byte1 << 4) & 0x1f]); output [outputIndex++] = equals; @@ -1390,9 +1392,9 @@ int base32Encode (int alphabet, else if (numCharsLeft == 3) { int byte1 = *input++; int byte2 = *input++; - output [outputIndex++] = (char)(encodeTable [(byte0 << 2) & 0x1f | (byte1 >> 6)]); + output [outputIndex++] = (char)(encodeTable [((byte0 << 2) & 0x1f) | (byte1 >> 6)]); output [outputIndex++] = (char)(encodeTable [(byte1 >> 1) & 0x1f]); - output [outputIndex++] = (char)(encodeTable [(byte1 << 4) & 0x1f | (byte2 >> 4)]); + output [outputIndex++] = (char)(encodeTable [((byte1 << 4) & 0x1f) | (byte2 >> 4)]); output [outputIndex++] = (char)(encodeTable [(byte2 << 1) & 0x1f]); output [outputIndex++] = equals; output [outputIndex++] = equals; @@ -1402,10 +1404,10 @@ int base32Encode (int alphabet, int byte1 = *input++; int byte2 = *input++; int byte3 = *input++; - output [outputIndex++] = (char)(encodeTable [(byte0 << 2) & 0x1f | (byte1 >> 6)]); + output [outputIndex++] = (char)(encodeTable [((byte0 << 2) & 0x1f) | (byte1 >> 6)]); output [outputIndex++] = (char)(encodeTable [(byte1 >> 1) & 0x1f]); - output [outputIndex++] = (char)(encodeTable [(byte1 << 4) & 0x1f | (byte2 >> 4)]); - output [outputIndex++] = (char)(encodeTable [(byte2 << 1) & 0x1f | (byte3 >> 7)]); + output [outputIndex++] = (char)(encodeTable [((byte1 << 4) & 0x1f) | (byte2 >> 4)]); + output [outputIndex++] = (char)(encodeTable [((byte2 << 1) & 0x1f) | (byte3 >> 7)]); output [outputIndex++] = (char)(encodeTable [(byte3 >> 2) & 0x1f]); output [outputIndex++] = (char)(encodeTable [(byte3 << 3) & 0x1f]); output [outputIndex++] = equals; @@ -2217,17 +2219,6 @@ void trimRight(char *str, int length) { } } -#ifdef __ZOWE_OS_ZOS -int isLowerCasePasswordAllowed(){ - RCVT* rcvt = getCVT()->cvtrac; - return (RCVTFLG3_BIT_RCVTPLC & (rcvt->rcvtflg3)); /* if lower-case pw allowed */ -} -#else -int isLowerCasePasswordAllowed(){ - return TRUE; -} -#endif - bool isPassPhrase(const char *password) { return strlen(password) > 8; } diff --git a/c/zos.c b/c/zos.c index 08e0c4d2d..e3b9c03a4 100644 --- a/c/zos.c +++ b/c/zos.c @@ -108,6 +108,7 @@ int supervisorMode(int enable){ } int setKey(int key){ +#ifndef __ZOWE_COMP_XLCLANG /* temp hack until Joe figures out what "&r" means */ int oldKey; __asm(" XR 2,2 \n" " SLL %1,4 \n" @@ -118,6 +119,9 @@ int setKey(int key){ :"&r"(key) :"r2"); return oldKey; +#else + return 8; +#endif } int64 getR12(void) { @@ -222,15 +226,14 @@ ExternalSecurityManager getExternalSecurityManager(void) { } CVT *getCVT(void) { - int * __ptr32 mem = (int * __ptr32) 0; - int * __ptr32 theCVT = (int * __ptr32)(*(mem+(0x10/4))); - return (CVT*)theCVT; + int *mem = (int*)0; + return (CVT*)INT2PTR(mem[0x10/4]); } Addr31 getATCVT(void) { - int * __ptr32 mem = (int * __ptr32) 0; - int * __ptr32 theATCVT = (int * __ptr32)(*(mem+(ATCVT_ADDRESS/4))); - return theATCVT; + int *mem = (int*)0; + /* int * __ptr32 theATCVT = (int * __ptr32)(*(mem+(ATCVT_ADDRESS/4))); */ + return (Addr31)INT2PTR(mem[ATCVT_ADDRESS/4]); } void *getIEACSTBL(void) { @@ -260,7 +263,7 @@ char *getSystemName (void) { TCB *getTCB(void) { int *mem = (int*)0; - return (TCB*)mem[CURRENT_TCB/sizeof(int)]; + return (TCB*)INT2PTR(mem[CURRENT_TCB/4]); } STCB *getSTCB(void) { @@ -275,7 +278,7 @@ OTCB *getOTCB(void) { ASCB *getASCB(void) { int *mem = (int*)0; - return (ASCB*)(mem[CURRENT_ASCB/sizeof(int)]&0x7FFFFFFF); + return (ASCB*)INT2PTR(mem[CURRENT_ASCB/4]); } ASXB *getASXB(void) { @@ -780,6 +783,20 @@ static char *makeCountedString(char *name, return result; } +/* forward decl */ +static int safVerifyInternal(int options, + char *userid, + char *password, + char *newPassword, + ACEE **aceeHandle, + void **messageAreaPtr, + int subpool, + char *applicationName, + int sessionType, + int *racfStatus, + int *racfReason, + IDTA *idta); + int safVerify(int options, char *userid, char *password, ACEE **aceeHandle, int *racfStatus, int *racfReason){ @@ -1019,8 +1036,8 @@ static int safVerifyInternal(int options, flags 1 is zero */ *ACEEPtr = *aceeHandle; if (safTrace){ - fprintf(safTraceFile,"setting acee anchor to %0.8X', %0.8X'\n",aceeHandle, *aceeHandle); - fprintf(safTraceFile,"setting real acee anchor to %0.8X', %0.8X'\n",ACEEPtr, *ACEEPtr); + fprintf(safTraceFile,"setting acee anchor to 0x%p', 0x%p'\n",aceeHandle, *aceeHandle); + fprintf(safTraceFile,"setting real acee anchor to 0x%p', 0x%p'\n",ACEEPtr, *ACEEPtr); } verifyRequest->aceeAnchor = ACEEPtr; /* placeholder for system-created ACEE */ } @@ -1029,12 +1046,12 @@ static int safVerifyInternal(int options, verifyRequest->initflg3 = verifyFlags3; if (safTrace){ - fprintf(safTraceFile,"countedUserid %0.8X' password %0.8X' passphrase %0.8X'\n", + fprintf(safTraceFile,"countedUserid 0x%p' password 0x%p' passphrase 0x%p'\n", countedUserid, countedPassword, countedPassphrase); - fprintf(safTraceFile,"about to go call saf wrapper at %0.8X' verify block at %0.8X' acee %0.8X'\n",safWrapper,verifyRequest, + fprintf(safTraceFile,"about to go call saf wrapper at 0x%p' verify block at 0x%p' acee 0x%p'\n",safWrapper,verifyRequest, (aceeHandle != NULL ? *aceeHandle : NULL)); dumpbuffer((char*)safWrapper,sizeof(safp)); - fprintf(safTraceFile,"verify:\n",0); + fprintf(safTraceFile,"verify:\n"); dumpbuffer((char*)verifyRequest,sizeof(safVerifyRequest)); } safStatus = SAF(safWrapper,useSupervisorMode); @@ -1138,21 +1155,21 @@ static int safAuth_internal(int options, char *safClass, char *entity, int acces } if (safTrace){ - fprintf(safTraceFile,"before calling SAF, wrapper=%0.8X':\n",safWrapper); + fprintf(safTraceFile,"before calling SAF, wrapper=0x%p':\n",safWrapper); dumpbuffer((char*)safWrapper,sizeof(safp)); - fprintf(safTraceFile,"before calling SAF, auth=%0.8X':\n", authRequest); + fprintf(safTraceFile,"before calling SAF, auth=0x%p':\n", authRequest); dumpbuffer((char*)authRequest,sizeof(safAuthRequest)); - fprintf(safTraceFile,"before calling SAF, class=%0.8X':\n", countedClass); + fprintf(safTraceFile,"before calling SAF, class=0x%p':\n", countedClass); dumpbuffer((char*)countedClass,countedClass[0]+1); - fprintf(safTraceFile,"before calling SAF, entity=%0.8X':\n", countedEntity); + fprintf(safTraceFile,"before calling SAF, entity=0x%p':\n", countedEntity); dumpbuffer((char*)countedEntity,countedEntity[3]+4); } safStatus = SAF(safWrapper, useSupervisorMode); if (safTrace){ fprintf(safTraceFile,"after calling SAF, safStatus=%d\n",safStatus); - fprintf(safTraceFile,"after calling SAF, wrapper=%0.8X':\n",safWrapper); + fprintf(safTraceFile,"after calling SAF, wrapper=0x%p':\n",safWrapper); dumpbuffer((char*)safWrapper,sizeof(safp)); - fprintf(safTraceFile,"after calling SAF, auth=%0.8X':\n", authRequest); + fprintf(safTraceFile,"after calling SAF, auth=0x%p':\n", authRequest); dumpbuffer((char*)authRequest,sizeof(safAuthRequest)); } *racfStatus = safWrapper->safprret; @@ -1210,15 +1227,15 @@ int safStat(int options, char *safClass, char *copy, int copyLength, int *racfSt statRequest->statLength = sizeof(safStatRequest); if (safTrace){ - fprintf(safTraceFile,"about to call saf, wrapper %0.8X:\n",safWrapper); + fprintf(safTraceFile,"about to call saf, wrapper 0x%p:\n",safWrapper); dumpbuffer((char*)safWrapper,sizeof(safp)); - fprintf(safTraceFile,"stat %0.8X:\n",statRequest); + fprintf(safTraceFile,"stat 0x%p:\n",statRequest); dumpbuffer((char*)statRequest,sizeof(safStatRequest)); - fprintf(safTraceFile,"className %0.8X:\n",classBuffer); + fprintf(safTraceFile,"className 0x%p:\n",classBuffer); dumpbuffer((char*)classBuffer,12); - fprintf(safTraceFile,"classCopy %0.8X:\n",copy); + fprintf(safTraceFile,"classCopy 0x%p:\n",copy); dumpbuffer((char*)copy,copyLength); - fprintf(safTraceFile,"workArea %0.8X\n",workArea); + fprintf(safTraceFile,"workArea 0x%p\n",workArea); fflush(safTraceFile); } safStatus = SAF(safWrapper, useSupervisorMode); @@ -1291,7 +1308,7 @@ int locate(char *dsn, int *volserCount, char *firstVolser){ memcpy(below2G->dsn44,dsn,dsnLength); /* - printf("dsn44 %x len %d\n",dsn44,dsnLength); + printf("dsn44 0x%p len %d\n",dsn44,dsnLength); dumpbuffer(dsn44,44); printf("arg %x:\n",svc26Arg); dumpbuffer(svc26Arg,16); diff --git a/c/zosfile.c b/c/zosfile.c index fb61745f3..16b4819b7 100644 --- a/c/zosfile.c +++ b/c/zosfile.c @@ -120,6 +120,28 @@ #define BPXGPN BPX1GPN #endif +/* Better compilers need these symbols to be declared as functions */ +int BPXRED(); +int BPXOPN(); +int BPXWRT(); +int BPXREN(); +int BPXCHR(); +int BPXCHM(); +int BPXCLO(); +int BPXLCO(); +int BPXSTA(); +int BPXUNL(); +int BPXOPD(); +int BPXMKD(); +int BPXRDD(); +int BPXRMD(); +int BPXCLD(); +int BPXUMK(); +int BPXFCT(); +int BPXLST(); +int BPXGGN(); +int BPXGPN(); + #define MAX_ENTRY_BUFFER_SIZE 2550 #define MAX_NUM_ENTRIES 1000 @@ -464,8 +486,8 @@ int fileChangeTagPure(const char *fileName, int *returnCode, int *reasonCode, int fileChangeTag(const char *fileName, int *returnCode, int *reasonCode, int ccsid) { bool pure = true; - fileChangeTagPure(fileName, returnCode, reasonCode, ccsid, pure); - } + return fileChangeTagPure(fileName, returnCode, reasonCode, ccsid, pure); +} int fileChangeTagPure(const char *fileName, int *returnCode, int *reasonCode, int ccsid, bool pure) { @@ -763,6 +785,8 @@ int fileInfo(const char *filename, BPXYSTAT *stats, int *returnCode, int *reason reasonCodePtr = reasonCode; #endif + printf("BPXSTA on filename\n"); + dumpbuffer(filename,strlen(filename)); BPXSTA(&nameLength, filename, &statsLength, @@ -770,6 +794,7 @@ int fileInfo(const char *filename, BPXYSTAT *stats, int *returnCode, int *reason &returnValue, returnCode, reasonCodePtr); + printf("return ret=%d reason=%d\n",*returnCode,*reasonCode); if (fileTrace) { if(returnValue != 0) { diff --git a/h/qsam.h b/h/qsam.h index 221c7370e..19c9634d2 100644 --- a/h/qsam.h +++ b/h/qsam.h @@ -117,15 +117,15 @@ #define DCB_RECFM_FBS 0x98 #define DCB_RECFM_VBS 0x58 -typedef __packed struct dcbExtension{ /* Aka, the DCBE */ +typedef _Packed struct dcbExtension{ /* Aka, the DCBE */ char id[4]; short int len; char rsvd006[2]; void * __ptr32 dcb; unsigned int rela; - __packed union { + _Packed union { char flg1; /* Set by OPEN processing */ - __packed struct { + _Packed struct { int open : 1; int md31 : 1; int slbi : 1; @@ -133,9 +133,9 @@ typedef __packed struct dcbExtension{ /* Aka, the DCBE */ int rsvd010_4 : 4; }; }; - __packed union { + _Packed union { char flg2; /* Set by requestor prior to OPEN */ - __packed struct { + _Packed struct { int rmode31 : 1; int pasteod : 1; int rsvd011_2 : 1; @@ -147,9 +147,9 @@ typedef __packed struct dcbExtension{ /* Aka, the DCBE */ }; }; short int nstr; - __packed union { + _Packed union { char flg3; /* Set by requestor prior to OPEN */ - __packed struct { + _Packed struct { int large : 1; int fixedbuf: 1; int eadscb : 1; @@ -170,8 +170,8 @@ typedef __packed struct dcbExtension{ /* Aka, the DCBE */ } DCBExtension; /* map dcbDevice at 0 size is 20 */ -typedef __packed struct dcbDevice{ - __packed struct dcbExtension * __ptr32 dcbe; +typedef _Packed struct dcbDevice{ + _Packed struct dcbExtension * __ptr32 dcbe; char unmapped[16]; /* A complex, device type sensitive, portion of the DCB that we do not need */ } DCBDevice; @@ -179,7 +179,7 @@ typedef __packed struct dcbDevice{ #define DCB_COMMON_OFFSET 0x14 #define VB_RECORD_OFFSET 4 -typedef __packed struct dcbCommon{ +typedef _Packed struct dcbCommon{ /* OFFSET 0x14 */ char bufno; /* 0 if no pool */ int bufcb:24; /* buffer pool control block = 000001 if no pool */ @@ -221,7 +221,7 @@ typedef __packed struct dcbCommon{ /* map dcbBBQCommon at 52 (0x34) BSAM/BPAM/QSAM common - size 20 */ #define DCB_BBQ_COMMON_OFFSET 0x34 -typedef __packed struct dcbBBQCommon { +typedef _Packed struct dcbBBQCommon { char optionCodes; /* OPTCD */ int check:24; /* or internal QSAM ERROR RTN */ int synad; /* synchns err rtn addr, =1 if not set */ @@ -237,7 +237,7 @@ typedef __packed struct dcbBBQCommon { /* map dcbBsamBpamInterface at 72 (0x48) */ #define DCB_BSAM_BPAM_OFFSET 0x48 -typedef __packed struct dcbBsamBpamInterface{ +typedef _Packed struct dcbBsamBpamInterface{ char ncp; /* MAX NUM OF OUTSTANDING READ/WRITES */ int eobr:24; /* internal AM use */ int eobw; /* internal AM use */ @@ -246,7 +246,7 @@ typedef __packed struct dcbBsamBpamInterface{ int cnp; /* control, note, point */ } DCBBBQInterface; -typedef __packed struct { +typedef _Packed struct { DCBDevice Device; DCBCommon Common; DCBBBQCommon BBQCommon; @@ -256,7 +256,7 @@ typedef __packed struct { /* There is lots of code that adds 4 bytes to skip past the DCB open list that preceeds the DCB. Do not declare additional variables prior to the DCB. */ -typedef __packed struct { +typedef _Packed struct { union { DCBSAM * __ptr32 OpenCloseList; unsigned int OpenCloseListOptions; @@ -279,7 +279,7 @@ typedef __packed struct { #define OPEN_CLOSE_EXTEND 0x0E000000 #define OPEN_CLOSE_OUTINX 0x06000000 -typedef __packed struct { +typedef _Packed struct { char flag1; char flag2; char sens0; @@ -295,7 +295,7 @@ typedef __packed struct { char seek[8]; } IOB; -typedef __packed struct DECB_tag{ +typedef _Packed struct DECB_tag{ int ecb; char type1; /* 0x80 - "S" coded for length */ char type2; /* 0x80 is a suggestion */ diff --git a/h/zowetypes.h b/h/zowetypes.h index d98bdac4a..1019ef83d 100644 --- a/h/zowetypes.h +++ b/h/zowetypes.h @@ -84,6 +84,7 @@ __ZOWE_COMP_GCC __ZOWE_COMP_VCPP __ZOWE_COMP_XLC __ZOWE_COMP_IXLC +__ZOWE_COMP_CLANG - new in 2022! __ZOWE_64BIT @@ -144,6 +145,10 @@ no ifdef means XLC LE on ZOS, and everything else. This is effectively our "def #define __ZOWE_OS_ZOS #define __ZOWE_ARCH_Z +#define __ZOWE_COMP_XLC +#ifdef __clang__ +#define __ZOWE_COMP_XLCLANG /* new in 2022, xlclang defines *BOTH* __IBMC__ *AND* __clang__ */ +#endif #ifdef _LP64 #define __ZOWE_64 @@ -155,6 +160,26 @@ no ifdef means XLC LE on ZOS, and everything else. This is effectively our "def #endif /* __MVS__ __IBMC__ */ + +/* new clang on ZOS cases */ +#if defined(__MVS__) && defined(__clang__) && (defined (_LP64) || defined (_ILP32)) + +#define __ZOWE_OS_ZOS +#define __ZOWE_ARCH_Z +#define __ZOWE_COMP_CLANG + +#ifdef _LP64 +#define __ZOWE_64 +#else +#define __ZOWE_32 +#endif + +/* Structure packing */ +#define ZOWE_PRAGMA_PACK _Pragma ( "pack(packed)" ) +#define ZOWE_PRAGMA_PACK_RESET _Pragma ( "pack(reset)" ) + +#endif /* __MVS__ __clang__ */ + #if defined(_AIX) && (defined (__IBMC__) || defined (__IBMCPP__)) #define __ZOWE_OS_AIX 1 #define __ZOWE_COMP_XLC 1 @@ -171,7 +196,7 @@ no ifdef means XLC LE on ZOS, and everything else. This is effectively our "def #endif /* AIX */ /* Base Macros for GCC, which is usually Linux or Unix, but Z,P and X architectures can use GCC so analysis is more complex */ -#ifdef __GNUC__ +#if defined(__GNUC__) && !defined(__clang__) #ifdef _LP64 #define __ZOWE_64 @@ -255,6 +280,7 @@ typedef uint64_t uint64; #define INT64_INT(x) ((int)x) #define INT_INT64(x) ((int64)x) #define CHARPTR_INT64(x) ((int64)x) +#define INT2PTR(x) ((void*)((int64_t)(x))) #else @@ -264,6 +290,7 @@ typedef unsigned long long uint64; #define INT64_INT(x) ((int)x) #define INT_INT64(x) ((int64)x) #define CHARPTR_INT64(x) ((int64)((int)x)) +#define INT2PTR(x) ((void*)(x)) #endif diff --git a/platform/posix/psxregex.c b/platform/posix/psxregex.c index ee33bbabc..0493eca06 100644 --- a/platform/posix/psxregex.c +++ b/platform/posix/psxregex.c @@ -1,4 +1,6 @@ #include "psxregex.h" +#include "zowetypes.h" +#include "alloc.h" regex_t *regexAlloc(){ return (regex_t*)safeMalloc(sizeof(regex_t),"regex_t"); diff --git a/platform/posix/psxregex.h b/platform/posix/psxregex.h index 69d027b33..8a1053690 100644 --- a/platform/posix/psxregex.h +++ b/platform/posix/psxregex.h @@ -1,5 +1,5 @@ -#ifndef __ZOWE_PSxREGEX__ -#define __ZOWE_PS +#ifndef __ZOWE_PSXREGEX__ +#define __ZOWE_PSXREGEX__ #include From a394acca112b645e738881b676dfb66eb28d7ca1 Mon Sep 17 00:00:00 2001 From: Leonty Chudinov Date: Tue, 1 Mar 2022 21:17:58 +0500 Subject: [PATCH 25/42] Quickjs zos Signed-off-by: Leonty Chudinov --- c/configmgr.c | 1 + c/embeddedjs.c | 56 +++++++++++++++++++++++++++++++++++++++++++++----- c/yaml2json.c | 3 ++- tests/Makefile | 32 +++++++++++++++++++---------- 4 files changed, 75 insertions(+), 17 deletions(-) diff --git a/c/configmgr.c b/c/configmgr.c index 94e403ad9..6ea5adb37 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -561,6 +561,7 @@ static int loadConfigurations(ConfigManager *mgr){ if (overloadStatus){ return overloadStatus; } else { + jsonPrettyPrint(mgr, mgr->config); Json *evaluatedConfig = evaluateJsonTemplates(mgr->ejs,mgr->slh,mgr->config); if (evaluatedConfig){ mgr->config = evaluatedConfig; diff --git a/c/embeddedjs.c b/c/embeddedjs.c index d42900d87..88faea627 100644 --- a/c/embeddedjs.c +++ b/c/embeddedjs.c @@ -59,6 +59,20 @@ typedef int64_t ssize_t; #endif +static int convertToNative(char *buf, size_t size) { +#ifdef __ZOWE_OS_ZOS + return __atoe_l(buf, size); +#endif + return 0; +} + +static int convertFromNative(char *buf, size_t size) { +#ifdef __ZOWE_OS_ZOS + return __etoa_l(buf, size); +#endif + return 0; +} + JSValue ejsEvalBuffer(EmbeddedJS *ejs, const void *buffer, int bufferLength, const char *filename, int eval_flags, @@ -96,6 +110,7 @@ void ejsFreeJSValue(EmbeddedJS *ejs, JSValue value){ int ejsSetGlobalProperty(EmbeddedJS *ejs, const char *propertyName, JSValue value){ JSContext *ctx = ejs->ctx; JSValue theGlobal = JS_GetGlobalObject(ctx); + return JS_SetPropertyStr(ctx,theGlobal,propertyName,value); } @@ -328,7 +343,11 @@ static Json *jsToJson1(EmbeddedJS *ejs, fflush(stderr); return NULL; } else { - Json *jsonString = jsonBuildString(b,parent,parentKey,(char*)str,strlen(str),&buildStatus); + size_t strLen = strlen(str); + char nativeStr[strLen+1]; + snprintf (nativeStr, strLen + 1, "%.*s", strLen, str); + convertToNative(nativeStr, strLen); + Json *jsonString = jsonBuildString(b,parent,parentKey,(char*)nativeStr,strlen(nativeStr),&buildStatus); JS_FreeCString(ctx, str); return jsonString; } @@ -407,7 +426,15 @@ static JSValue jsonToJS1(EmbeddedJS *ejs, Json *json, bool hideUnevaluated){ case JSON_TYPE_DOUBLE: return JS_NewFloat64(ctx,jsonAsDouble(json)); case JSON_TYPE_STRING: - return JS_NewString(ctx,jsonAsString(json)); + { + char *str = jsonAsString(json); + size_t strLen = strlen(str); + char convertedStr[strLen+1]; + snprintf (convertedStr, strLen + 1, "%.*s", strLen, str); + printf ("about to convert string '%s'\n", convertedStr); + convertFromNative(convertedStr, strLen); + return JS_NewString(ctx, convertedStr); + } case JSON_TYPE_BOOLEAN: return JS_NewBool(ctx,jsonAsBoolean(json)); case JSON_TYPE_NULL: @@ -424,10 +451,18 @@ static JSValue jsonToJS1(EmbeddedJS *ejs, Json *json, bool hideUnevaluated){ for (property = jsonObjectGetFirstProperty(jsonObject); property != NULL; property = jsonObjectGetNextProperty(property)) { + { + char *key = jsonPropertyGetKey(property); + size_t keyLen = strlen(key); + char convertedKey[keyLen+1]; + snprintf (convertedKey, keyLen + 1, "%.*s", keyLen, key); + printf ("about to convert key '%s'\n", convertedKey); + convertFromNative(convertedKey, keyLen); JS_SetPropertyStr(ctx, object, - jsonPropertyGetKey(property), + convertedKey, jsonToJS1(ejs,jsonPropertyGetValue(property),hideUnevaluated)); + } } return object; } @@ -559,14 +594,25 @@ static bool evaluationVisitor(void *context, Json *json, Json *parent, char *key property = jsonObjectGetNextProperty(property)) { char *key = jsonPropertyGetKey(property); Json *value = jsonPropertyGetValue(property); - ejsSetGlobalProperty(ejs,key,ejsJsonToJS(ejs,value)); + printf ("global object key '%s'\n", key); + size_t keyLen = strlen(key); + char convertedKey[keyLen+1]; + snprintf (convertedKey, keyLen + 1, "%.*s", keyLen, key); + convertFromNative(convertedKey, keyLen); + ejsSetGlobalProperty(ejs,convertedKey,ejsJsonToJS(ejs,value)); } Json *sourceValue = jsonObjectGetPropertyValue(object,"source"); if (sourceValue){ char *source = jsonAsString(sourceValue); + size_t sourceLen = strlen(source); + char asciiSource[sourceLen + 1]; + snprintf (asciiSource, sourceLen + 1, "%.*s", sourceLen, source); + convertFromNative(asciiSource, sourceLen); printf("should evaluate: %s\n",source); int evalStatus = 0; - JSValue output = ejsEvalBuffer(ejs,source,strlen(source),"",0,&evalStatus); + char embedded[] = ""; + convertFromNative(embedded, sizeof(embedded)); + JSValue output = ejsEvalBuffer(ejs,asciiSource,strlen(asciiSource),embedded,0,&evalStatus); if (evalStatus){ printf("failed to evaluate '%s', status=%d\n",source,evalStatus); } else { diff --git a/c/yaml2json.c b/c/yaml2json.c index 46ff54fe8..a2db1a8d3 100644 --- a/c/yaml2json.c +++ b/c/yaml2json.c @@ -421,7 +421,8 @@ static int buildTemplateJSON(JsonBuilder *b, Json *parent, char *parentKey, char *nativeValue, int valueLength){ char *tail = nativeValue; JsonBuffer *buffer = makeJsonBuffer(); - int sourceCCSID = SOURCE_CODE_CHARSET; + int sourceCCSID = CCSID_UTF_8; + printf ("buildTemplateJSON nativeValue '%.*s' sourceCodeCharset %d\n", valueLength, nativeValue, sourceCCSID); jsonPrinter *p = makeBufferJsonPrinter(sourceCCSID,buffer); int status = 0; bool first = true; diff --git a/tests/Makefile b/tests/Makefile index 4f43d7a9b..26fd55f6a 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -10,38 +10,39 @@ # Makefile for z/OS # make prepare && make -CC:=c89 +CC:=xlclang BITS:= #BITS:=lp64 ifeq ($(BITS),lp64) - ENHANCED_ASCII:=-D_ENHANCED_ASCII_EXT=0xFFFFFFFF -Wc,ASCII + ENHANCED_ASCII:=-qascii -D_ENHANCED_ASCII_EXT=0xFFFFFFFF else - ENHANCED_ASCII:=-D_ENHANCED_ASCII_EXT=0xFFFFFFFF -Wc,ASCII,NOXPLINK + ENHANCED_ASCII:=-qascii -D_ENHANCED_ASCII_EXT=0xFFFFFFFF -Wc,NOXPLINK endif MAJOR:=0 MINOR:=2 PATCH:=5 VERSION:=\"$(MAJOR).$(MINOR).$(PATCH)\" -DEFINESYAML:=-DYAML_VERSION_MAJOR=$(MAJOR) -DYAML_VERSION_MINOR=$(MINOR) -DYAML_VERSION_PATCH=$(PATCH) -DYAML_VERSION_STRING=$(VERSION) -DEFINES:=-D_XOPEN_SOURCE=600 -D_OPEN_THREADS=1 +DEFINESYAML:=-DYAML_VERSION_MAJOR=$(MAJOR) -DYAML_VERSION_MINOR=$(MINOR) -DYAML_VERSION_PATCH=$(PATCH) -DYAML_VERSION_STRING=$(VERSION) -DYAML_DECLARE_STATIC=1 +DEFINES:=-D_XOPEN_SOURCE=600 -D_OPEN_THREADS=1 -D_OPEN_SYS_FILE_EXT=1 -DCONFIG_VERSION=\"2021-03-27\" -CC_FLAGS:=-Ilibyaml/include -I../h -I../platform/posix $(DEFINES) -Wc,dll,expo,langlvl\(extc99\),gonum,goff,hgpr,roconst,ASM,asmlib\('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN'\) -Wc,agg,list\(\),so\(\),off,xref,$(BITS) +CC_FLAGS:=-Iquickjs-portable -Ilibyaml/include -I../h -I../platform/posix $(DEFINES) -Wc,dll,expo,langlvl\(extc99\),gonum,goff,hgpr,roconst,ASM,asmlib\('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN'\) -Wc,agg,list\(\),so\(\),off,xref,$(BITS) LD_FLAGS:=-Wl,$(BITS) LIBYAMLOBJS:=api.o reader.o scanner.o parser.o loader.o writer.o emitter.o dumper.o -SCHEMATESTOBJS:=schematest.o yaml2json.o jsonschema.o json.o xlate.o charsets.o bpxskt.o logging.o collections.o timeutls.o timeutls.o utils.o alloc.o zosfile.o zos.o le.o scheduling.o recovery.o psxregex.o -CONFIGMGROBJ:=configmgr.o yaml2json.o jsonschema.o json.o xlate.o charsets.o bpxskt.o logging.o collections.o timeutls.o timeutls.o utils.o alloc.o zosfile.o zos.o le.o scheduling.o recovery.o psxregex.o +SCHEMATESTOBJS:=schematest.o yaml2json.o jsonschema.o json.o xlate.o charsets.o bpxskt.o logging.o collections.o timeutls.o timeutls.o utils.o alloc.o embeddedjs.o zosfile.o zos.o le.o scheduling.o recovery.o psxregex.o +QUICKJSOBS:=cutils.o quickjs.o quickjs-libc.o libunicode.o libregexp.o polyfill.o +CONFIGMGROBJ:=configmgr.o yaml2json.o jsonschema.o json.o xlate.o charsets.o bpxskt.o logging.o collections.o timeutls.o timeutls.o utils.o alloc.o embeddedjs.o zosfile.o zos.o le.o scheduling.o recovery.o psxregex.o .PHONY: clean all prepare test_configmgr all: configmgr schematest -configmgr: $(CONFIGMGROBJ) $(LIBYAMLOBJS) +configmgr: $(CONFIGMGROBJ) $(LIBYAMLOBJS) $(QUICKJSOBS) $(CC) $(LD_FLAGS) -o $@ $^ -schematest: $(SCHEMATESTOBJS) $(LIBYAMLOBJS) +schematest: $(SCHEMATESTOBJS) $(LIBYAMLOBJS) $(QUICKJSOBS) $(CC) $(LD_FLAGS) -o $@ $^ schematest.o: schematest.c @@ -56,10 +57,19 @@ schematest.o: schematest.c %.o: libyaml/src/%.c $(CC) $(CC_FLAGS) $(DEFINESYAML) $(ENHANCED_ASCII) -c $< +%.o: quickjs-portable/%.c + $(CC) $(CC_FLAGS) $(DEFINESYAML) $(ENHANCED_ASCII) -c $< + +%.o: quickjs-portable/porting/%.c + $(CC) $(CC_FLAGS) $(DEFINESYAML) $(ENHANCED_ASCII) -c $< + libyaml: git clone git@github.com:yaml/libyaml.git -prepare: libyaml +quickjs-portable: + git clone git@github.com:JoeNemo/quickjs-portable.git + +prepare: libyaml quickjs-portable clean: rm -f configmgr schematest *.o From a358a20fdc466d6b0ca68c7b3c6e981075c2d448 Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 8 Mar 2022 00:46:11 -0500 Subject: [PATCH 26/42] parsing infrastruture to support jq and parmlib syntaxes Signed-off-by: Joe --- c/configmgr.c | 24 +- c/embeddedjs.c | 10 +- c/microjq.c | 392 ++++++++++++++++++++++++++ c/parsetools.c | 691 ++++++++++++++++++++++++++++++++++++++++++++++ c/utils.c | 6 +- h/parsetools.h | 445 +++++++++++++++++++++++++++++ tests/parsetest.c | 116 ++++++++ 7 files changed, 1676 insertions(+), 8 deletions(-) create mode 100644 c/microjq.c create mode 100644 c/parsetools.c create mode 100644 h/parsetools.h create mode 100644 tests/parsetest.c diff --git a/c/configmgr.c b/c/configmgr.c index 6ea5adb37..bb9f5ba51 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -108,7 +108,6 @@ typedef int64_t ssize_t; configmgr -s "../tests/schemadata" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowebase.yaml)" extract "/zowe/setup/mvs/proclib" - -- Compilation with XLCLang on ZOS ------------------------------------- export YAML="/u/zossteam/jdevlin/git2022/libyaml" @@ -123,7 +122,23 @@ typedef int64_t ssize_t; xlclang -q64 -qascii -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 -DCONFIG_VERSION=\"2021-03-27\" -D_OPEN_SYS_FILE_EXT=1 -D_XOPEN_SOURCE=600 -D_OPEN_THREADS=1 -DSUBPOOL=132 "-Wc,float(ieee),longname,langlvl(extc99),gonum,goff,ASM,asmlib('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN')" -I ../h -I ../platform/posix -I ${YAML}/include -I ${QJS} -Wbitwise-op-parentheses -o configmgr configmgr.c yaml2json.c embeddedjs.c jsonschema.c json.c xlate.c charsets.c zosfile.c logging.c recovery.c scheduling.c zos.c le.c collections.c timeutls.c utils.c alloc.c ../platform/posix/psxregex.c ${QJS}/quickjs.c ${QJS}/cutils.c ${QJS}/quickjs-libc.c ${QJS}/libregexp.c ${QJS}/libunicode.c ${QJS}/porting/polyfill.c ${YAML}/src/api.c ${YAML}/src/reader.c ${YAML}/src/scanner.c ${YAML}/src/parser.c ${YAML}/src/loader.c ${YAML}/src/writer.c ${YAML}/src/emitter.c ${YAML}/src/dumper.c + ------- + + Supporting a subset of jq syntax for extraction: + + JQ available at + + https://stedolan.github.io/jq/ + + Examples: + + jq ".properties.zowe.type" ../tests/schemadata/zoweyaml.schema + + Good examples in https://stedolan.github.io/jq/tutorial/ + curl https://api.github.com/repos/stedolan/jq/commits?per_page=5 > commits.json + + */ #define CONFIG_PATH_OMVS_FILE 0x0001 @@ -1014,6 +1029,13 @@ int main(int argc, char **argv){ break; } freeJsonValidator(validator); + } else if (!strcmp(command,"jq")){ + if (argx >= argc){ + trace(mgr,INFO,"jq requires at least one filter argument"); + } else { + char *jqArg = argv[argx++]; + + } } else if (!strcmp(command,"extract")){ if (argx >= argc){ trace(mgr,INFO,"extract command requires a json path\n"); diff --git a/c/embeddedjs.c b/c/embeddedjs.c index 88faea627..a8023f22b 100644 --- a/c/embeddedjs.c +++ b/c/embeddedjs.c @@ -345,7 +345,7 @@ static Json *jsToJson1(EmbeddedJS *ejs, } else { size_t strLen = strlen(str); char nativeStr[strLen+1]; - snprintf (nativeStr, strLen + 1, "%.*s", strLen, str); + snprintf (nativeStr, strLen + 1, "%.*s", (int)strLen, str); convertToNative(nativeStr, strLen); Json *jsonString = jsonBuildString(b,parent,parentKey,(char*)nativeStr,strlen(nativeStr),&buildStatus); JS_FreeCString(ctx, str); @@ -430,7 +430,7 @@ static JSValue jsonToJS1(EmbeddedJS *ejs, Json *json, bool hideUnevaluated){ char *str = jsonAsString(json); size_t strLen = strlen(str); char convertedStr[strLen+1]; - snprintf (convertedStr, strLen + 1, "%.*s", strLen, str); + snprintf (convertedStr, strLen + 1, "%.*s", (int)strLen, str); printf ("about to convert string '%s'\n", convertedStr); convertFromNative(convertedStr, strLen); return JS_NewString(ctx, convertedStr); @@ -455,7 +455,7 @@ static JSValue jsonToJS1(EmbeddedJS *ejs, Json *json, bool hideUnevaluated){ char *key = jsonPropertyGetKey(property); size_t keyLen = strlen(key); char convertedKey[keyLen+1]; - snprintf (convertedKey, keyLen + 1, "%.*s", keyLen, key); + snprintf (convertedKey, keyLen + 1, "%.*s", (int)keyLen, key); printf ("about to convert key '%s'\n", convertedKey); convertFromNative(convertedKey, keyLen); JS_SetPropertyStr(ctx, @@ -597,7 +597,7 @@ static bool evaluationVisitor(void *context, Json *json, Json *parent, char *key printf ("global object key '%s'\n", key); size_t keyLen = strlen(key); char convertedKey[keyLen+1]; - snprintf (convertedKey, keyLen + 1, "%.*s", keyLen, key); + snprintf (convertedKey, keyLen + 1, "%.*s", (int)keyLen, key); convertFromNative(convertedKey, keyLen); ejsSetGlobalProperty(ejs,convertedKey,ejsJsonToJS(ejs,value)); } @@ -606,7 +606,7 @@ static bool evaluationVisitor(void *context, Json *json, Json *parent, char *key char *source = jsonAsString(sourceValue); size_t sourceLen = strlen(source); char asciiSource[sourceLen + 1]; - snprintf (asciiSource, sourceLen + 1, "%.*s", sourceLen, source); + snprintf (asciiSource, sourceLen + 1, "%.*s", (int)sourceLen, source); convertFromNative(asciiSource, sourceLen); printf("should evaluate: %s\n",source); int evalStatus = 0; diff --git a/c/microjq.c b/c/microjq.c new file mode 100644 index 000000000..4216167ca --- /dev/null +++ b/c/microjq.c @@ -0,0 +1,392 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#ifdef METTLE + +#include +#include +#include +#include +#include +#include +#include +#include +#include "metalio.h" +#include "qsam.h" + +#else + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 600 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#endif + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "openprims.h" +#include "bpxnet.h" +#include "unixfile.h" +#include "json.h" +#include "parsetools.h" + +/* + clang -I %QJS%/porting -I%YAML%/include -I../platform/windows -I %QJS% -I ..\h -Wdeprecated-declarations -D_CRT_SECURE_NO_WARNINGS -o microjq.exe microjq.c parsetools.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c + */ + +#define STATE_INITIAL 0 +#define STATE_INTEGER 1 +#define STATE_IDENTIFIER 2 +#define STATE_SQUOTE 3 +#define STATE_DQUOTE 4 + +#define TOKEN_IDENTIFIER FIRST_REAL_TOKEN_ID +#define TOKEN_INTEGER (FIRST_REAL_TOKEN_ID+1) +#define TOKEN_DQUOTE_STRING FIRST_REAL_TOKEN_ID+2 +#define TOKEN_SQUOTE_STRING FIRST_REAL_TOKEN_ID+3 +#define TOKEN_LPAREN FIRST_REAL_TOKEN_ID+4 +#define TOKEN_RPAREN FIRST_REAL_TOKEN_ID+5 +#define TOKEN_LBRACK FIRST_REAL_TOKEN_ID+6 +#define TOKEN_RBRACK FIRST_REAL_TOKEN_ID+7 +#define TOKEN_LBRACE FIRST_REAL_TOKEN_ID+8 +#define TOKEN_RBRACE FIRST_REAL_TOKEN_ID+9 +#define TOKEN_DOT FIRST_REAL_TOKEN_ID+10 +#define TOKEN_COMMA FIRST_REAL_TOKEN_ID+11 +#define TOKEN_VBAR FIRST_REAL_TOKEN_ID+12 +#define TOKEN_COLON FIRST_REAL_TOKEN_ID+13 +#define TOKEN_QMARK FIRST_REAL_TOKEN_ID+14 +#define TOKEN_PLUS FIRST_REAL_TOKEN_ID+15 +#define TOKEN_DASH FIRST_REAL_TOKEN_ID+16 +#define TOKEN_STAR FIRST_REAL_TOKEN_ID+17 +#define TOKEN_SLASH FIRST_REAL_TOKEN_ID+18 +#define TOKEN_PERCENT FIRST_REAL_TOKEN_ID+19 + +#define LAST_TOKEN_ID TOKEN_PERCENT /* KEEP ME UPDATED!!! - whenever a new token ID is added*/ + + +typedef struct JQParser_tag { + AbstractTokenizer tokenizer; /* low token, high token */ + int ccsid; + int tokenState; /* = STATE_INITIAL; */ + char *data; + int length; + int pos; +} JQParser; + +static char *getTokenName(int id){ + switch (id){ + case EOF_TOKEN: return "EOF"; + case TOKEN_IDENTIFIER: return "ID"; + case TOKEN_INTEGER: return "INTEGER"; + case TOKEN_DQUOTE_STRING: return "DQUOTE_STRING"; + case TOKEN_SQUOTE_STRING: return "SQUOTE_STRING"; + case TOKEN_LPAREN: return "LPAREN"; + case TOKEN_RPAREN: return "RPAREN"; + case TOKEN_LBRACK: return "LBRACK"; + case TOKEN_RBRACK: return "RBRACK"; + case TOKEN_LBRACE: return "LBRACE"; + case TOKEN_RBRACE: return "RBRACE"; + case TOKEN_DOT: return "DOT"; + case TOKEN_COMMA: return "COMMA"; + case TOKEN_VBAR: return "VBAR"; + case TOKEN_COLON: return "COLON"; + case TOKEN_QMARK: return "QMARK"; + case TOKEN_PLUS: return "PLUS"; + case TOKEN_DASH: return "DASH"; + case TOKEN_STAR: return "STAR"; + case TOKEN_SLASH: return "SLASH"; + case TOKEN_PERCENT: return "PERCENT"; + default:return "BAD_TOKEN"; + } +} + +static bool isLetter(JQParser *jqp, int c){ + return testCharProp(jqp->ccsid,c,CPROP_LETTER); +} + +static bool isDigit(JQParser *jqp, int c){ + return testCharProp(jqp->ccsid,c,CPROP_DIGIT); +} + +static bool isWhite(JQParser *jqp, int c){ + return testCharProp(jqp->ccsid,c,CPROP_WHITE); +} + +static int jqpToken(JQParser *jqp){ + /* belt *AND* suspenders */ + AbstractTokenizer *tokenizer = (AbstractTokenizer*)jqp; + if (jqp->pos >= jqp->length){ + tokenizer->lastTokenStart = jqp->pos; /* 0-length token */ + tokenizer->lastTokenEnd = jqp->pos; + return EOF_TOKEN; + } + int tokenID = NO_VALID_TOKEN; + while(tokenID == NO_VALID_TOKEN){ + int c = -1; // EOF by defaultx + int charPos = jqp->pos; + if (jqp->pos < jqp->length){ + c = (int)jqp->data[jqp->pos++]; + } + /* printf("at pos=%d, c=0x%x state=%d\n",jqp->pos,c,jqp->tokenState); */ + switch (jqp->tokenState){ + case STATE_INITIAL: + if (c == -1){ + tokenID = EOF_TOKEN; + tokenizer->lastTokenStart = jqp->length; + } else if (isLetter(jqp,c) || + (c == UNI_DOLLAR) || + (c == UNI_UNDER)){ + jqp->tokenState = STATE_IDENTIFIER; + tokenizer->lastTokenStart = charPos; + } else if (isDigit(jqp,c)){ + jqp->tokenState = STATE_INTEGER; + tokenizer->lastTokenStart = charPos; + } else if (isWhite(jqp,c)){ + // just roll forward + } else if (c == UNI_DQUOTE){ + jqp->tokenState = STATE_DQUOTE; + tokenizer->lastTokenStart = charPos; + } else if (c == UNI_SQUOTE){ + jqp->tokenState = STATE_SQUOTE; + tokenizer->lastTokenStart = charPos; + } else { + /* Single-char-punc tokens */ + switch (c){ + case UNI_LPAREN: tokenID = TOKEN_LPAREN; break; + case UNI_RPAREN: tokenID = TOKEN_RPAREN; break; + case UNI_LBRACK: tokenID = TOKEN_LBRACK; break; + case UNI_RBRACK: tokenID = TOKEN_RBRACK; break; + case UNI_LBRACE: tokenID = TOKEN_LBRACE; break; + case UNI_RBRACE: tokenID = TOKEN_RBRACE; break; + case UNI_DOT: tokenID = TOKEN_DOT; break; + case UNI_COMMA: tokenID = TOKEN_COMMA; break; + case UNI_COLON: tokenID = TOKEN_COLON; break; + case UNI_VBAR: tokenID = TOKEN_VBAR; break; + case UNI_QMARK: tokenID = TOKEN_QMARK; break; + case UNI_PLUS: tokenID = TOKEN_PLUS; break; + case UNI_DASH: tokenID = TOKEN_DASH; break; + case UNI_STAR: tokenID = TOKEN_STAR; break; + case UNI_SLASH: tokenID = TOKEN_SLASH; break; + case UNI_PERCENT: tokenID = TOKEN_PERCENT; break; + default: + /* unhandled char, kill the parse */ + return NO_VALID_TOKEN; + } + tokenizer->lastTokenStart = charPos; + } + break; + case STATE_INTEGER: + if (!isDigit(jqp,c)){ + if (c != -1) jqp->pos--; /* nothing to push back */ + tokenID = TOKEN_INTEGER; + } + break; + case STATE_IDENTIFIER: + if (isLetter(jqp,c) || + (c == UNI_DOLLAR) || + (c == UNI_UNDER)){ + // accumulate + } else { + if (c != -1) jqp->pos--; /* nothing to push back */ + tokenID = TOKEN_IDENTIFIER; + } + break; + case STATE_DQUOTE: + if (c == -1){ /* EOF in string */ + return NO_VALID_TOKEN; + } else if (c == UNI_DQUOTE){ + tokenID = TOKEN_DQUOTE_STRING; + } + break; + case STATE_SQUOTE: + if (c == -1){ /* EOF in string */ + return NO_VALID_TOKEN; + } else if (c == UNI_SQUOTE){ + tokenID = TOKEN_SQUOTE_STRING; + } + default: + printf("*** PANIC *** internal error unknown tokenizer state = %d\n",jqp->tokenState); + return NO_VALID_TOKEN; + // unknow state, kill the parse + } + if (tokenID != NO_VALID_TOKEN){ + tokenizer->lastTokenEnd = jqp->pos; + jqp->tokenState = STATE_INITIAL; + return tokenID; + } + } + return 0; +} + +int getNextToken(AbstractTokenizer *tokenizer, char *s, int len, int pos, int *nextPos){ + JQParser *jqp = (JQParser*)tokenizer; + jqp->pos = pos; + int id = jqpToken(jqp); + *nextPos = jqp->pos; + return id; +} + +/* + Grammar + + Expr = Operation (PIPE* Operation)* + + Opertion = + Literal + CreateArray + CreateObject + Filter + + Literal + Number + String dquote and squote'd + + CreateArray + LBRACK Expr (COMMA Expr) RBRACK + + CreateObject + LPAREN KeyValue (COMMA KeyValue) RPAREN + + KeyValue + Key COLON Expression + + Key Identifier + + + Test Filters + '.results[] | {name, age}' + + Real JQ + + set JQ="c:\temp" + %JQ%\jq-win64 ... + + */ + +static void tokenTest(JQParser *jqp){ + AbstractTokenizer *tokenizer = (AbstractTokenizer*)jqp; + printf("length = %d\n",jqp->length); + while (true){ + int tokenID = jqpToken(jqp); + printf("token id=%d, %s pos is now=%d\n",tokenID,getTokenName(tokenID),jqp->pos); + printf(" from %d to %d\n",tokenizer->lastTokenStart,tokenizer->lastTokenEnd); + if (tokenID == NO_VALID_TOKEN){ + printf("bad token seen near %d\n",jqp->pos); + break; + } else if (tokenID == EOF_TOKEN){ + printf("EOF Seen\n"); + break; + } + } +} + +#define TOP (FIRST_GRULE_ID+0) +#define EXPR (FIRST_GRULE_ID+1) +#define EXPR_TAIL (FIRST_GRULE_ID+2) +#define EXPR_ADD_SUB (FIRST_GRULE_ID+3) +#define PLUS_MINUS (FIRST_GRULE_ID+4) +#define TERM (FIRST_GRULE_ID+5) +#define TERM_TAIL (FIRST_GRULE_ID+6) +#define TERM_MULT_DIV (FIRST_GRULE_ID+7) +#define STAR_SLASH (FIRST_GRULE_ID+8) +#define FACTOR (FIRST_GRULE_ID+9) + +static char *getRuleName(int id){ + switch (id){ + case TOP: return "TOP"; + case EXPR: return "EXPR"; + case EXPR_TAIL: return "EXPR_TAIL"; + case EXPR_ADD_SUB: return "EXPR_ADD_SUB"; + case PLUS_MINUS: return "PLUS_MINUS"; + case TERM: return "TERM"; + case TERM_TAIL: return "TERM_TAIL"; + case TERM_MULT_DIV: return "TERM_MULT_DIV"; + case STAR_SLASH: return "STAR_SLASH"; + case FACTOR: return "FACTOR"; + default: + return "UNKNOWN_RULE"; + } +} + +static GRuleSpec expressionGrammar[] = { + { G_SEQ, TOP, .sequence = { EXPR, EOF_TOKEN, G_END} }, + { G_SEQ, EXPR, .sequence = { TERM, EXPR_TAIL, G_END }}, + { G_STAR, EXPR_TAIL, .star = EXPR_ADD_SUB }, + { G_SEQ, EXPR_ADD_SUB, .sequence = { PLUS_MINUS, TERM, G_END }}, + { G_ALT, PLUS_MINUS, .alternates = { TOKEN_PLUS, TOKEN_DASH, G_END }}, + { G_SEQ, TERM, .sequence = { FACTOR, TERM_TAIL, G_END }}, + { G_STAR, TERM_TAIL, .star = TERM_MULT_DIV }, + { G_SEQ, TERM_MULT_DIV, .sequence = { STAR_SLASH, FACTOR, G_END}}, + { G_ALT, STAR_SLASH, .alternates = { TOKEN_STAR, TOKEN_SLASH, G_END }}, + { G_ALT, FACTOR, .alternates = { TOKEN_IDENTIFIER, TOKEN_INTEGER, G_END }}, /* add expression recursion later */ + { G_END } +}; + + +static void parseTest1(JQParser *jqp){ + AbstractTokenizer *tokenizer = (AbstractTokenizer*)jqp; + tokenizer->lowTokenID = FIRST_REAL_TOKEN_ID; + tokenizer->highTokenID = LAST_TOKEN_ID; + tokenizer->nextToken = getNextToken; + tokenizer->getTokenIDName = getTokenName; + GParseContext *ctx = gParse(expressionGrammar,TOP,tokenizer,jqp->data,jqp->length,getRuleName); + printf("parse ctx = 0x%p\n",ctx); + if (ctx->status > 0){ + #ifdef __ZOWE_OS_WINDOWS + int stdoutFD = _fileno(stdout); +#else + int stdoutFD = STDOUT_FILENO; +#endif + ShortLivedHeap *slh = makeShortLivedHeap(0x10000, 100); + Json *tree = gBuildJSON(ctx,slh); + if (tree){ + jsonPrinter *p = makeJsonPrinter(stdoutFD); + jsonEnablePrettyPrint(p); + printf("parse result as json\n"); + jsonPrint(p,tree); + } + } +} + + +int main(int argc, char **argv){ + char *command = argv[1]; + char *filter = argv[2]; + char *filename = argv[3]; + JQParser jqp; + memset(&jqp,0,sizeof(JQParser)); + jqp.data = argv[2]; + jqp.length = strlen(jqp.data); + jqp.ccsid = 1208; + if (!strcmp(command,"tokenize")){ + printf("tokenize...\n"); + tokenTest(&jqp); + } else if (!strcmp(command,"parse")){ + printf("parse...\n"); + parseTest1(&jqp); + } else { + printf("bad command: %s\n",command); + } + return 0; +} diff --git a/c/parsetools.c b/c/parsetools.c new file mode 100644 index 000000000..acdb1e147 --- /dev/null +++ b/c/parsetools.c @@ -0,0 +1,691 @@ +#include +#include +#include +#include +#include +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "parsetools.h" +#include "json.h" + +/* + + clang -I../h -I../platform/windows -D_CRT_SECURE_NO_WARNINGS -Dstrdup=_strdup -DYAML_DECLARE_STATIC=1 -Wdeprecated-declarations --rtlib=compiler-rt -o parsetools.exe parsetools.c timeutls.c utils.c alloc.c + + Grammar + (PHRASEREF n) + (TOKENREF n) + (ALT n) + (SEQ n) + (STAR ) + (PLUS n) + + */ + +static int cp1047to1208[256] = { + 0x00, 0x01, 0x02, 0x03, 0x9c, 0x09, 0x86, 0x7f, 0x97, 0x8d, 0x8e, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x9d, 0x0a, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8f, 0x1c, 0x1d, 0x1e, 0x1f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1b, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x05, 0x06, 0x07, + 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9a, 0x9b, 0x14, 0x15, 0x9e, 0x1a, + 0x20, 0xa0, 0xe2, 0xe4, 0xe0, 0xe1, 0xe3, 0xe5, 0xe7, 0xf1, 0xa2, 0x2e, 0x3c, 0x28, 0x2b, 0x7c, + 0x26, 0xe9, 0xea, 0xeb, 0xe8, 0xed, 0xee, 0xef, 0xec, 0xdf, 0x21, 0x24, 0x2a, 0x29, 0x3b, 0x5e, + 0x2d, 0x2f, 0xc2, 0xc4, 0xc0, 0xc1, 0xc3, 0xc5, 0xc7, 0xd1, 0xa6, 0x2c, 0x25, 0x5f, 0x3e, 0x3f, + 0xf8, 0xc9, 0xca, 0xcb, 0xc8, 0xcd, 0xce, 0xcf, 0xcc, 0x60, 0x3a, 0x23, 0x40, 0x27, 0x3d, 0x22, + 0xd8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xab, 0xbb, 0xf0, 0xfd, 0xfe, 0xb1, + 0xb0, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0xaa, 0xba, 0xe6, 0xb8, 0xc6, 0xa4, + 0xb5, 0x7e, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7a, 0xa1, 0xbf, 0xd0, 0x5b, 0xde, 0xae, + 0xac, 0xa3, 0xa5, 0xb7, 0xa9, 0xa7, 0xb6, 0xbc, 0xbd, 0xbe, 0xdd, 0xa8, 0xaf, 0x5d, 0xb4, 0xd7, + 0x7b, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xad, 0xf4, 0xf6, 0xf2, 0xf3, 0xf5, + 0x7d, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0xb9, 0xfb, 0xfc, 0xf9, 0xfa, 0xff, + 0x5c, 0xf7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0xb2, 0xd4, 0xd6, 0xd2, 0xd3, 0xd5, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xb3, 0xdb, 0xdc, 0xd9, 0xda, 0x9f +}; + +bool testCharProp(int ccsid, int c, int prop){ + switch(ccsid){ + case IBM1047: + return (charProps[cp1047to1208[c]] & prop) != 0; + case UTF8: + return (charProps[c] & prop) != 0; + default: + printf("*** PANIC *** Unsupported CCSID = %d\n",ccsid); + return false; + } +} + +#define MAX_DEPTH 200 + + +typedef struct SavedState_tag { + GRule *rule; + int index; + int pos; + GRule *parent; + int indexInParent; + /* AST building */ + SyntaxNode *node; + int nodeChildCount; + +} SavedState; + +static void setParseState(SavedState *state, + GRule *rule, + int index, + int pos, + GRule *parent, + int indexInParent){ + state->rule = rule; + state->index = index; + state->pos = pos; + state->parent = parent; + state->indexInParent = indexInParent; +} + +static int countRuleSet(GRule *rules){ + int count = 0; + while (rules[count].type != G_END){ + count++; + } + return count; +} + +static bool matchToken(AbstractTokenizer *tokenizer, + char *s, int len, + int pos, int *nextPos, + int desiredTokenID, + int *tokenIDPtr){ + int tokenID = tokenizer->nextToken(tokenizer,s,len,pos,nextPos); + *tokenIDPtr = tokenID; + return (tokenID == desiredTokenID); +} + +static bool isTokenReference(AbstractTokenizer *tokenizer, int id){ + return ((id == EOF_TOKEN) || + (id >= tokenizer->lowTokenID && id <= tokenizer->highTokenID)); +} + +static GRule *getRule(GRule *ruleSet, int count, int id){ + /* printf("get rule %d\n",id); */ + fflush(stdout); + if (id < FIRST_GRULE_ID){ + printf("rule id %d lower than min %d\n",id,FIRST_GRULE_ID); + return NULL; + } + for (int i=0; irefs = (GRuleRef*)SLHAlloc(slh,rule->subCount*sizeof(GRuleRef)); + for (int i=0; isubCount; i++){ + int index = indices[i]; + /* printf("setRRA i=%d\n",i); */ + GRuleRef *ref = &rule->refs[i]; + ref->isTokenRef = isTokenReference(tokenizer,index); + if (ref->isTokenRef){ + ref->tokenID = index; + } else { + GRule *referent = getRule(ruleSet,ruleCount,index); + if (referent == NULL){ + return false; + } + ref->parent = rule; + ref->rule = referent; + ref->indexInParent = index; + } + } + return true; +} + +static GRule *makeGRules(GRuleSpec *ruleSpecs, AbstractTokenizer *tokenizer, ShortLivedHeap *slh, int *ruleCountPtr){ + int ruleCount = 0; + while (ruleSpecs[ruleCount].type != G_END){ + ruleCount++; + } + *ruleCountPtr = ruleCount; + GRule *rules = (GRule*)SLHAlloc(slh,ruleCount*sizeof(GRule)); + int r; + for (r=0; rtype = spec->type; + rule->id = spec->id; + /* printf("rule at 0x%p with id=%d\n",rule,rule->id); */ + } + for (r=0; rtype){ + case G_SEQ: + { + int subCount = 0; + while (spec->sequence.subs[subCount] != G_END) subCount++; + rule->subCount = subCount; + if (!setRuleRefArray(slh,tokenizer,rule,rules,ruleCount,&spec->sequence.subs[0])){ + return NULL; + } + } + break; + case G_ALT: + { + int subCount = 0; + while (spec->alternates.subs[subCount] != G_END) subCount++; + rule->subCount = subCount; + if (!setRuleRefArray(slh,tokenizer,rule,rules,ruleCount,&spec->alternates.subs[0])){ + return NULL; + } + } + break; + case G_STAR: + { + int subCount = 1; + rule->subCount = subCount; + if (!setRuleRefArray(slh,tokenizer,rule,rules,ruleCount,&spec->star.sub)){ + return NULL; + } + } + break; + default: + printf("*** PANIC unhandled rule type in compilation %d\n",rule->type); + } + } + return rules; +} + +/* + + HERE + 0) specs could be arrays of strings for simpler parsing + maybe + 0.5) factor to parse test and jqtest + 0.6) separate out the JLExer + 1) write microJQ grammar + 2) start pushing builder operations corresponding to loops, trees + 3) tree construction comes from temporarily associating text ranges and token ID's with this node tree + can tree construction go direct to json + can all steps be a set of JSON Build calls that are kept in a list, + and then backtracked to on earlier places in list, and then finally played forward to make the JSON Tree + 4) JSON program can be written to take this output and test it in Node.js + 5) bring it into QuickJS as evaluator + 6) or just interpret it straight up + 7) Trace improvements: + backtrack uniqueID and establishment trace + backtrack printf enhance to be specific + + + build steps + + SEQ + kv pairs - (uniquified names of subs -> json) + + ALT + { + indicator: + value: + } + + STAR + [ ] + + TOKEN + jsonValuizer + makeString ( various escaping rules, like handling \r \u, etc ) + makeInteger + makeBoolean + makeNull + but can make anything!! + + +*/ + +typedef struct GContinuation_tag { + struct GContinuation_tag *previous; +#ifdef METTLE +#error "need setjmp" +#else + jmp_buf resumeData; +#endif + int indexInRule; + int position; + int depth; + int buildStepHWM; +} GContinuation; + +static char *getBuildStepTypeName(int type){ + switch (type){ + case BUILD_OBJECT: return "Object"; + case BUILD_ARRAY: return "Array"; + case BUILD_STRING: return "String"; + case BUILD_INT: return "int"; + case BUILD_INT64: return "int64"; + case BUILD_DOUBLE: return "double"; + case BUILD_BOOL: return "boolean"; + case BUILD_NULL: return "null"; + case BUILD_KEY: return "key"; + case BUILD_POP: return "pop()"; + case BUILD_ALT: return "altWrap()"; + default: return "UnknownBuildStep"; + } + +} + +#define GPARSE_SUCCESS(x) ((x)>=0) +#define GPARSE_FAIL (-1) +#define setResumePoint(cPtr) setjmp((cPtr)->resumeData) + +static void setStepRef(GBuildStep *step, GRuleRef *ref){ + step->tokenID = (ref->isTokenRef ? ref->tokenID : NO_VALID_TOKEN); + step->rule = ref->rule; +} + +static void setAltChoice(GBuildStep *step, GRuleRef *ref){ + /* printf("setAltChoice isTokenRef=%d, tokenID=%d\n",ref->isTokenRef,ref->tokenID); */ + step->altChoice = ref; +} + +static GBuildStep *makeBuildStep(GParseContext *ctx, int type, GRuleRef *ref, GRule *rule){ + if (ctx->buildStepCount >= ctx->buildStepsSize){ + printf("**** PANIC **** out of build steps\n"); + printf("**** PANIC **** out of build steps\n"); + printf("**** PANIC **** out of build steps\n"); + printf("**** PANIC **** out of build steps\n"); + return NULL; + } + GBuildStep *step = &ctx->buildSteps[ctx->buildStepCount++]; + memset(step,0,sizeof(GBuildStep)); + step->type = type; + if (ref){ + setStepRef(step,ref); + } else if (rule){ + step->tokenID = NO_VALID_TOKEN; + step->rule = rule; + } else { + step->tokenID = NO_VALID_TOKEN; + step->rule = NULL; + } + step->valueStart = STEP_UNDEFINED_POSITION; + step->valueEnd = STEP_UNDEFINED_POSITION; + return step; +} + +static void buildTokenValue(GParseContext *ctx, GRuleRef *ref, int lastTokenStart, int lastTokenEnd){ + GBuildStep *step = makeBuildStep(ctx,BUILD_STRING,ref,NULL); + step->valueStart = lastTokenStart; + step->valueEnd = lastTokenEnd; +} + +static void buildSeqElement(GParseContext *ctx, GRuleRef *ref){ + makeBuildStep(ctx,BUILD_KEY,ref,NULL); +} + +static GBuildStep *buildAltValue(GParseContext *ctx, GRule *rule){ + return makeBuildStep(ctx,BUILD_ALT,NULL,rule); +} + +static void buildObject(GParseContext *ctx, GRule *rule){ + makeBuildStep(ctx,BUILD_OBJECT,NULL,rule); +} + +static void buildArray(GParseContext *ctx, GRule *rule){ + makeBuildStep(ctx,BUILD_ARRAY,NULL,rule); +} + +static void buildPop(GParseContext *ctx){ + makeBuildStep(ctx,BUILD_POP,NULL,NULL); +} + +static void indent(int depth){ + for (int i=0; i Backtracking... build step back from %d to %d\n", + ctx->buildStepCount, + continuation->buildStepHWM); + fflush(stdout); + longjmp(continuation->resumeData,1); +} + +static int runTokenMatch(GParseContext *ctx, GRuleRef *ref, int pos, int depth){ + AbstractTokenizer *tokenizer = ctx->tokenizer; + int nextPos = 0; + int tokenID = 0; + indent(depth); + printf("runTokenMatch trying id=0x%x, (%s) pos=%d\n",ref->tokenID,tokenizer->getTokenIDName(ref->tokenID),pos); + fflush(stdout); + bool matched = matchToken(tokenizer,ctx->s,ctx->len,pos,&nextPos,ref->tokenID,&tokenID); + if (matched){ + buildTokenValue(ctx,ref,tokenizer->lastTokenStart,tokenizer->lastTokenEnd); + return nextPos; + } else if (tokenID == NO_VALID_TOKEN){ + printf("*** Tokenizer failed, parse must stop\n"); + printf("*** Tokenizer failed, parse must stop\n"); + printf("*** Tokenizer failed, parse must stop\n"); + printf("*** Tokenizer failed, parse must stop\n"); + /* should throw to outer error handler */ + return 0x7FFFFFFF; + } else { + return GPARSE_FAIL; + } +} + +static char *ruleName(GParseContext *ctx, GRule *rule){ + return ctx->ruleNamer(rule->id); +} + +static int runSeq(GParseContext *ctx, GRule *rule, int startPos, int depth, GContinuation *resumePoint){ + indent(depth); + printf("runSeq (%s) at pos=%d, resume=0x%p\n",ruleName(ctx,rule),startPos,resumePoint); + fflush(stdout); + int pos = startPos; + for (int index=0; indexsubCount; index++){ + GRuleRef *ref = &rule->refs[index]; + buildSeqElement(ctx,ref); + int matchResult = + (ref->isTokenRef ? + runTokenMatch(ctx,ref,pos,depth) : + parseDispatch(ctx,ref->rule,pos,depth,resumePoint)); + indent(depth+1); + printf("seq submatch index=%d result=%d\n",index,matchResult); + if (GPARSE_SUCCESS(matchResult)){ + pos = matchResult; + } else if (resumePoint){ + backtrack(ctx,resumePoint); + } else { + return GPARSE_FAIL; + } + } + return pos; +} + +static int runAlt(GParseContext *ctx, GRule *rule, int startPos, int depth, GContinuation *resumePoint){ + indent(depth); + printf("runAlt (%s) at pos=%d, resume=0x%p\n",ruleName(ctx,rule),startPos,resumePoint); + fflush(stdout); + GBuildStep *altStep = buildAltValue(ctx,rule); + GContinuation continuation; + continuation.previous = resumePoint; + continuation.indexInRule = 0; + continuation.position = startPos; + continuation.depth = depth; + continuation.buildStepHWM = ctx->buildStepCount; + int isResumption = setResumePoint(&continuation); + int index = (isResumption ? continuation.indexInRule : 0); + int pos = (isResumption ? continuation.position : startPos); + ctx->buildStepCount = continuation.buildStepHWM; + for (; indexsubCount; index++){ + GRuleRef *ref = &rule->refs[index]; + setAltChoice(altStep,ref); + continuation.indexInRule = index+1; + if (ref->isTokenRef){ + int matchResult = runTokenMatch(ctx,ref,pos,depth); + if (GPARSE_SUCCESS(matchResult)){ + indent(depth+1); + printf("alt token success matchResult=%d\n",matchResult); + return matchResult; + } else if (index+1 < rule->subCount){ + // just roll forward + indent(depth+1); + printf("alt token roll forward\n"); + } else if (resumePoint){ + indent(depth+1); + printf("alt token backtrack\n"); + backtrack(ctx,resumePoint); + } else { + indent(depth+1); + printf("alt token fail\n"); + return GPARSE_FAIL; + } + } else { + GContinuation *failContinuation = ((index + 1 < rule->subCount) ? + &continuation : + resumePoint); + int parseResult = parseDispatch(ctx,ref->rule,pos,depth,failContinuation); + indent(depth+1); + printf("alt subrule res = %d\n",parseResult); + return parseResult; // we never loop except when hitting a token + } + } + indent(depth+1); + printf("out of ALT's\n"); + return GPARSE_FAIL; +} + +// can easily by plus or repeat, or opt +static int runStar(GParseContext *ctx, GRule *rule, int startPos, int depth, GContinuation *resumePoint){ + indent(depth); + printf("runStar (%s) at pos=%d, resume=0x%p\n",ruleName(ctx,rule),startPos,resumePoint); + fflush(stdout); + GContinuation continuation; + continuation.previous = resumePoint; + continuation.indexInRule = 0; + continuation.position = startPos; + continuation.depth = depth; + continuation.buildStepHWM = ctx->buildStepCount; + GRuleRef *ref = &rule->refs[0]; + int isResumption = setResumePoint(&continuation); + if (isResumption){ + ctx->buildStepCount = continuation.buildStepHWM; + return continuation.position; // still always a success + } else { + /* this is a "greedy" star */ + while (true){ + int matchResult = + (ref->isTokenRef ? + runTokenMatch(ctx,ref,continuation.position,depth) : + parseDispatch(ctx,ref->rule,continuation.position,depth,&continuation)); + if (GPARSE_SUCCESS(matchResult)){ + continuation.position = matchResult; + continuation.buildStepHWM = ctx->buildStepCount; + continuation.indexInRule++; + // keep looping + } else { + return continuation.position; // it's always a success, even if zero matched + } + } + } +} + +static int parseDispatch(GParseContext *ctx, GRule *rule, int startPos, int depth, GContinuation *resumePoint){ + switch (rule->type){ + case G_SEQ: + { + buildObject(ctx,rule); + int res = runSeq(ctx,rule,startPos,depth+1,resumePoint); + buildPop(ctx); + return res; + } + case G_ALT: + { + int res = runAlt(ctx,rule,startPos,depth+1,resumePoint); + buildPop(ctx); + return res; + } + case G_STAR: + { + buildArray(ctx,rule); + int res = runStar(ctx,rule,startPos,depth+1,resumePoint); + buildPop(ctx); + return res; + } + default: + printf("*** PANIC *** Unhandled ruleType = %d\n",rule->type); + return GPARSE_FAIL; /* should bounce to "total failure setjmp" */ + } +} + +void showBuildSteps(GParseContext *ctx){ + for (int i=0; ibuildStepCount; i++){ + GBuildStep *step = &ctx->buildSteps[i]; + if (step->type == BUILD_ALT){ + bool isToken = (step->tokenID != NO_VALID_TOKEN); + int markingID = (step->tokenID != NO_VALID_TOKEN) ? step->tokenID : step->rule->id; + printf("Step(%d): ALT choice byToken=%d id=%d\n",i,isToken,markingID); + } else if (step->tokenID != NO_VALID_TOKEN){ + printf("Step(%d) %s: token %s text from %d to %d\n", + i, + getBuildStepTypeName(step->type), + ctx->tokenizer->getTokenIDName(step->tokenID), + step->valueStart,step->valueEnd); + } else if (step->rule){ + printf("Step(%d) %s: %s\n",i,getBuildStepTypeName(step->type),ctx->ruleNamer(step->rule->id)); + } else { + printf("Step(%d) %s\n",i,getBuildStepTypeName(step->type)); + } + } +} + +Json *gBuildJSON(GParseContext *ctx, ShortLivedHeap *slh){ + if (slh == NULL){ + slh = ctx->slh; + } + JsonBuilder *b = makeJsonBuilder(slh); + int sp = 0; + Json **parentStack = (Json**)SLHAlloc(ctx->slh,MAX_DEPTH*sizeof(Json*)); + Json *currentParent = NULL; + char *currentKey = NULL; + for (int i=0; ibuildStepCount; i++){ + printf("build step %d\n",i); + GBuildStep *step = &ctx->buildSteps[i]; + int errorCode = 0; + switch (step->type){ + case BUILD_OBJECT: + { + Json *o = jsonBuildObject(b,currentParent,currentKey,&errorCode); + parentStack[sp++] = currentParent; + currentParent = o; + currentKey = NULL; + } + break; + case BUILD_ARRAY: + { + Json *a = jsonBuildArray(b,currentParent,currentKey,&errorCode); + parentStack[sp++] = currentParent; + currentParent = a; + currentKey = NULL; + } + break; + case BUILD_POP: + { + currentParent = parentStack[--sp]; + currentKey = NULL; + } + break; + case BUILD_KEY: + { + /* unless EOF */ + switch (step->tokenID){ + case NO_VALID_TOKEN: + currentKey = ctx->ruleNamer(step->rule->id); + break; + case EOF_TOKEN: + currentKey = ""; + break; + default: + ctx->tokenizer->getTokenIDName(step->tokenID); + break; + } + } + break; + case BUILD_ALT: + { + Json *alt = jsonBuildObject(b,currentParent,currentKey,&errorCode); + parentStack[sp++] = currentParent; + currentParent = alt; + GRuleRef *choiceRef = step->altChoice; + int choiceID = (choiceRef->isTokenRef ? choiceRef->tokenID : choiceRef->rule->id); + jsonBuildInt(b,alt,"altID",choiceID,&errorCode); + currentKey = "value"; + } + break; + case BUILD_STRING: + if (currentKey == NULL){ + printf("no parent for string!!!\n"); + } else { + int len = step->valueEnd - step->valueStart; + jsonBuildString(b,currentParent,currentKey,ctx->s+step->valueStart,len,&errorCode); + currentKey = NULL; /* because was consumed */ + } + break; + case BUILD_INT64: + case BUILD_INT: + case BUILD_DOUBLE: + case BUILD_BOOL: + case BUILD_NULL: + printf("unhandled JSON value type for build type %d\n",step->type); + break; + default: + printf("*** PANIC *** unhandled JSON builder step type %d\n",step->type); + break; + } + /* + } else if (step->type + } else if (step->tokenID != NO_VALID_TOKEN){ + } else if (step->rule){ + printf("Step %s: %s\n",getBuildStepTypeName(step->type),ctx->ruleNamer(step->rule->id)); + } else { + printf("Step %s\n",getBuildStepTypeName(step->type)); + } + */ + if (errorCode){ + printf("last build step failed with code = %d\n",errorCode); + } + } + printf("final sp = %d\n",sp); + Json *result = b->root; + freeJsonBuilder(b,false); + return result; +} + +GParseContext *gParse(GRuleSpec *ruleSet, int topRuleID, AbstractTokenizer *tokenizer, char *s, int len, + char *(*ruleNamer)(int id)){ + ShortLivedHeap *slh = makeShortLivedHeap(0x10000, 100); + GParseContext *ctx = (GParseContext *)SLHAlloc(slh,sizeof(GParseContext)); + memset(ctx,0,sizeof(GParseContext)); + ctx->tokenizer = tokenizer; + ctx->s = s; + ctx->len = len; + ctx->ruleNamer = ruleNamer; + ctx->slh = slh; + ctx->buildStepsSize = MAX_BUILD_STEPS; + ctx->buildSteps = (GBuildStep*)SLHAlloc(slh,ctx->buildStepsSize*sizeof(GBuildStep)); + int ruleSetSize = 0; + GRule *rules = makeGRules(ruleSet,tokenizer,slh,&ruleSetSize); + ctx->rules = rules; + GRule *topRule = getRule(rules,ruleSetSize,topRuleID); + if (topRule->type == G_SEQ){ + int parseResult = parseDispatch(ctx,topRule,0,0,NULL); + printf("parseResult=%d\n",parseResult); + ctx->status = parseResult; + showBuildSteps(ctx); + return ctx; + } else { + printf("top rule must be G_SEQ\n"); + ctx->status = -1; + return ctx; + } +} diff --git a/c/utils.c b/c/utils.c index 050e5a5c6..cce3802cd 100644 --- a/c/utils.c +++ b/c/utils.c @@ -24,8 +24,10 @@ #else #include #include -#include -#include +#include +#ifndef _MSC_VER /* Windows always has to be the oddball */ +#include +#endif #include #endif diff --git a/h/parsetools.h b/h/parsetools.h new file mode 100644 index 000000000..632a89abf --- /dev/null +++ b/h/parsetools.h @@ -0,0 +1,445 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#ifndef __ZOWE_PARSETOOLS__ +#define __ZOWE_PARSETOOLS__ 1 + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "json.h" + +/* Common Tokenization/Lexing Infrastucture */ + +#define EOF_TOKEN -1 +#define NO_VALID_TOKEN 0 +#define FIRST_REAL_TOKEN_ID 1 + + +#define CPROP_ALPHA 0x0001 +#define CPROP_DIGIT 0x0002 +#define CPROP_HEX 0x0004 +#define CPROP_CONTROL 0x0008 +#define CPROP_LETTER 0x0010 +#define CPROP_LOWER 0x0020 +#define CPROP_WHITE 0x0040 + +static int charProps[256] = { + CPROP_CONTROL, /* 0x0 */ + CPROP_CONTROL, /* 0x1 */ + CPROP_CONTROL, /* 0x2 */ + CPROP_CONTROL, /* 0x3 */ + CPROP_CONTROL, /* 0x4 */ + CPROP_CONTROL, /* 0x5 */ + CPROP_CONTROL, /* 0x6 */ + CPROP_CONTROL, /* 0x7 */ + CPROP_CONTROL, /* 0x8 */ + CPROP_CONTROL |CPROP_WHITE, /* 0x9 */ + CPROP_CONTROL |CPROP_WHITE, /* 0xa */ + CPROP_CONTROL |CPROP_WHITE, /* 0xb */ + CPROP_CONTROL |CPROP_WHITE, /* 0xc */ + CPROP_CONTROL |CPROP_WHITE, /* 0xd */ + CPROP_CONTROL, /* 0xe */ + CPROP_CONTROL, /* 0xf */ + CPROP_CONTROL, /* 0x10 */ + CPROP_CONTROL, /* 0x11 */ + CPROP_CONTROL, /* 0x12 */ + CPROP_CONTROL, /* 0x13 */ + CPROP_CONTROL, /* 0x14 */ + CPROP_CONTROL, /* 0x15 */ + CPROP_CONTROL, /* 0x16 */ + CPROP_CONTROL, /* 0x17 */ + CPROP_CONTROL, /* 0x18 */ + CPROP_CONTROL, /* 0x19 */ + CPROP_CONTROL, /* 0x1a */ + CPROP_CONTROL, /* 0x1b */ + CPROP_CONTROL |CPROP_WHITE, /* 0x1c */ + CPROP_CONTROL |CPROP_WHITE, /* 0x1d */ + CPROP_CONTROL |CPROP_WHITE, /* 0x1e */ + CPROP_CONTROL |CPROP_WHITE, /* 0x1f */ + CPROP_WHITE, /* 0x20 */ + 0, /* 0x21 */ + 0, /* 0x22 */ + 0, /* 0x23 */ + 0, /* 0x24 */ + 0, /* 0x25 */ + 0, /* 0x26 */ + 0, /* 0x27 */ + 0, /* 0x28 */ + 0, /* 0x29 */ + 0, /* 0x2a */ + 0, /* 0x2b */ + 0, /* 0x2c */ + 0, /* 0x2d */ + 0, /* 0x2e */ + 0, /* 0x2f */ + CPROP_DIGIT |CPROP_HEX, /* 0x30 */ + CPROP_DIGIT |CPROP_HEX, /* 0x31 */ + CPROP_DIGIT |CPROP_HEX, /* 0x32 */ + CPROP_DIGIT |CPROP_HEX, /* 0x33 */ + CPROP_DIGIT |CPROP_HEX, /* 0x34 */ + CPROP_DIGIT |CPROP_HEX, /* 0x35 */ + CPROP_DIGIT |CPROP_HEX, /* 0x36 */ + CPROP_DIGIT |CPROP_HEX, /* 0x37 */ + CPROP_DIGIT |CPROP_HEX, /* 0x38 */ + CPROP_DIGIT |CPROP_HEX, /* 0x39 */ + 0, /* 0x3a */ + 0, /* 0x3b */ + 0, /* 0x3c */ + 0, /* 0x3d */ + 0, /* 0x3e */ + 0, /* 0x3f */ + 0, /* 0x40 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER, /* 0x41 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER, /* 0x42 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER, /* 0x43 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER, /* 0x44 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER, /* 0x45 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER, /* 0x46 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x47 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x48 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x49 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x4a */ + CPROP_ALPHA |CPROP_LETTER, /* 0x4b */ + CPROP_ALPHA |CPROP_LETTER, /* 0x4c */ + CPROP_ALPHA |CPROP_LETTER, /* 0x4d */ + CPROP_ALPHA |CPROP_LETTER, /* 0x4e */ + CPROP_ALPHA |CPROP_LETTER, /* 0x4f */ + CPROP_ALPHA |CPROP_LETTER, /* 0x50 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x51 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x52 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x53 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x54 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x55 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x56 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x57 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x58 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x59 */ + CPROP_ALPHA |CPROP_LETTER, /* 0x5a */ + 0, /* 0x5b */ + 0, /* 0x5c */ + 0, /* 0x5d */ + 0, /* 0x5e */ + 0, /* 0x5f */ + 0, /* 0x60 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER |CPROP_LOWER, /* 0x61 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER |CPROP_LOWER, /* 0x62 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER |CPROP_LOWER, /* 0x63 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER |CPROP_LOWER, /* 0x64 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER |CPROP_LOWER, /* 0x65 */ + CPROP_ALPHA |CPROP_HEX |CPROP_LETTER |CPROP_LOWER, /* 0x66 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x67 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x68 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x69 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x6a */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x6b */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x6c */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x6d */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x6e */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x6f */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x70 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x71 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x72 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x73 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x74 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x75 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x76 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x77 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x78 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x79 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0x7a */ + 0, /* 0x7b */ + 0, /* 0x7c */ + 0, /* 0x7d */ + 0, /* 0x7e */ + CPROP_CONTROL, /* 0x7f */ + CPROP_CONTROL, /* 0x80 */ + CPROP_CONTROL, /* 0x81 */ + CPROP_CONTROL, /* 0x82 */ + CPROP_CONTROL, /* 0x83 */ + CPROP_CONTROL, /* 0x84 */ + CPROP_CONTROL, /* 0x85 */ + CPROP_CONTROL, /* 0x86 */ + CPROP_CONTROL, /* 0x87 */ + CPROP_CONTROL, /* 0x88 */ + CPROP_CONTROL, /* 0x89 */ + CPROP_CONTROL, /* 0x8a */ + CPROP_CONTROL, /* 0x8b */ + CPROP_CONTROL, /* 0x8c */ + CPROP_CONTROL, /* 0x8d */ + CPROP_CONTROL, /* 0x8e */ + CPROP_CONTROL, /* 0x8f */ + CPROP_CONTROL, /* 0x90 */ + CPROP_CONTROL, /* 0x91 */ + CPROP_CONTROL, /* 0x92 */ + CPROP_CONTROL, /* 0x93 */ + CPROP_CONTROL, /* 0x94 */ + CPROP_CONTROL, /* 0x95 */ + CPROP_CONTROL, /* 0x96 */ + CPROP_CONTROL, /* 0x97 */ + CPROP_CONTROL, /* 0x98 */ + CPROP_CONTROL, /* 0x99 */ + CPROP_CONTROL, /* 0x9a */ + CPROP_CONTROL, /* 0x9b */ + CPROP_CONTROL, /* 0x9c */ + CPROP_CONTROL, /* 0x9d */ + CPROP_CONTROL, /* 0x9e */ + CPROP_CONTROL, /* 0x9f */ + 0, /* 0xa0 */ + 0, /* 0xa1 */ + 0, /* 0xa2 */ + 0, /* 0xa3 */ + 0, /* 0xa4 */ + 0, /* 0xa5 */ + 0, /* 0xa6 */ + 0, /* 0xa7 */ + 0, /* 0xa8 */ + 0, /* 0xa9 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xaa */ + 0, /* 0xab */ + 0, /* 0xac */ + 0, /* 0xad */ + 0, /* 0xae */ + 0, /* 0xaf */ + 0, /* 0xb0 */ + 0, /* 0xb1 */ + 0, /* 0xb2 */ + 0, /* 0xb3 */ + 0, /* 0xb4 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xb5 */ + 0, /* 0xb6 */ + 0, /* 0xb7 */ + 0, /* 0xb8 */ + 0, /* 0xb9 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xba */ + 0, /* 0xbb */ + 0, /* 0xbc */ + 0, /* 0xbd */ + 0, /* 0xbe */ + 0, /* 0xbf */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc0 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc1 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc2 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc3 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc4 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc5 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc6 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc7 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc8 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xc9 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xca */ + CPROP_ALPHA |CPROP_LETTER, /* 0xcb */ + CPROP_ALPHA |CPROP_LETTER, /* 0xcc */ + CPROP_ALPHA |CPROP_LETTER, /* 0xcd */ + CPROP_ALPHA |CPROP_LETTER, /* 0xce */ + CPROP_ALPHA |CPROP_LETTER, /* 0xcf */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd0 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd1 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd2 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd3 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd4 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd5 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd6 */ + 0, /* 0xd7 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd8 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xd9 */ + CPROP_ALPHA |CPROP_LETTER, /* 0xda */ + CPROP_ALPHA |CPROP_LETTER, /* 0xdb */ + CPROP_ALPHA |CPROP_LETTER, /* 0xdc */ + CPROP_ALPHA |CPROP_LETTER, /* 0xdd */ + CPROP_ALPHA |CPROP_LETTER, /* 0xde */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xdf */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe0 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe1 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe2 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe3 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe4 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe5 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe6 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe7 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe8 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xe9 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xea */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xeb */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xec */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xed */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xee */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xef */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf0 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf1 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf2 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf3 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf4 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf5 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf6 */ + 0, /* 0xf7 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf8 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xf9 */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xfa */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xfb */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xfc */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xfd */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER, /* 0xfe */ + CPROP_ALPHA |CPROP_LETTER |CPROP_LOWER /* 0xff */ +}; + +#define IBM1047 1047 +#define UTF8 1208 + +#define UNI_TAB 0x09 +#define UNI_LF 0x0A +#define UNI_CR 0X0D +#define UNI_SPACE 0x20 +#define UNI_BANG 0x21 +#define UNI_DQUOTE 0x22 +#define UNI_HASH 0x23 +#define UNI_DOLLAR 0x24 +#define UNI_PERCENT 0x25 +#define UNI_AMP 0x26 +#define UNI_SQUOTE 0x27 +#define UNI_LPAREN 0x28 +#define UNI_RPAREN 0x29 +#define UNI_STAR 0x2A +#define UNI_PLUS 0x2B +#define UNI_COMMA 0x2C +#define UNI_DASH 0x2D +#define UNI_DOT 0x2E +#define UNI_SLASH 0x2F +#define UNI_COLON 0x3A +#define UNI_SEMI 0x3B +#define UNI_LESS 0x3C +#define UNI_EQUAL 0x3D +#define UNI_GREATER 0x3E +#define UNI_QMARK 0x3F +#define UNI_AT 0x40 +#define UNI_LBRACK 0x5B +#define UNI_BSLASH 0x5C +#define UNI_RBRACK 0x5D +#define UNI_CARET 0x5E +#define UNI_UNDER 0x5F +#define UNI_BQUOTE 0x60 +#define UNI_LBRACE 0x7B +#define UNI_VBAR 0x7C +#define UNI_RBRACE 0x7D +#define UNI_TILDE 0x7E + +typedef struct AbstractTokenizer_tag { + int lowTokenID; + int highTokenID; + int lastTokenStart; + int lastTokenEnd; + int (*nextToken)(struct AbstractTokenizer_tag *tokenizer, char *s, int len, int pos, int *nextPos); + char *(*getTokenIDName)(int tokenID); +} AbstractTokenizer; + +bool testCharProp(int ccsid, int c, int prop); + +/* Common Parsing infrastructure */ + +#define FIRST_GRULE_ID 10000 + +#define G_END 0 +#define G_SEQ 1 +#define G_ALT 2 +#define G_STAR 3 +#define G_PLUS 4 + +#define G_MAX_SUBS 12 + +typedef struct GRuleSpec_tag { + int type; + int id; + union { + struct { int subs[G_MAX_SUBS]; } sequence; + struct { int subs[G_MAX_SUBS]; } alternates; + struct { int sub; } star; + struct { int sub; } plus; + }; +} GRuleSpec; + +typedef struct GRuleRef_tag { + struct GRule_tag *parent; + struct GRule_tag *rule; + int indexInParent; + int tokenID; + bool isTokenRef; +} GRuleRef; + +typedef struct GRule_tag { + int type; + int id; + int subCount; + GRuleRef *refs; +} GRule; + + + +#define BUILD_OBJECT 1 +#define BUILD_ARRAY 2 +#define BUILD_STRING 3 +#define BUILD_INT 4 +#define BUILD_INT64 5 +#define BUILD_DOUBLE 6 +#define BUILD_BOOL 7 +#define BUILD_NULL 8 +#define BUILD_KEY 9 +#define BUILD_POP 10 +#define BUILD_ALT 11 + +#define MAX_BUILD_STEPS 10000 /* who in hell knows? JK, actually should be a parameter to GParse() */ +#define STEP_UNDEFINED_POSITION (-1) + +typedef struct GBuildStep_tag { + int type; /* BUILD_xxx above */ + GRule *rule; + int tokenID; + char *keyInParent; + int valueStart; /* index into parse source */ + int valueEnd; /* index into parse source */ + GRuleRef *altChoice; +} GBuildStep; + +typedef struct GParseContext_tag { + AbstractTokenizer *tokenizer; + GRule *rules; + char *s; + int len; + ShortLivedHeap *slh; + char *(*ruleNamer)(int id); + int buildStepsSize; + int buildStepCount; + GBuildStep *buildSteps; + int status; + int errorPosition; + char *errorMessage; +} GParseContext; + +typedef struct SyntaxNode_tag { + int type; /* owned by the user of this facility */ + int childCapacity; + int childCount; + struct SyntaxNode_tag **children; +} SyntaxNode; + +typedef struct SyntaxTree_tag { + ShortLivedHeap slh; + SyntaxNode *topNode; +} SyntaxTree; + +typedef struct GParseResult_tag { + SyntaxTree *tree; +} GParseResult; + +GParseContext *gParse(GRuleSpec *ruleSet, int topRuleID, AbstractTokenizer *tokenizer, char *s, int len, + char *(*ruleName)(int id)); + +Json *gBuildJSON(GParseContext *ctx, ShortLivedHeap *slh); + +#endif diff --git a/tests/parsetest.c b/tests/parsetest.c new file mode 100644 index 000000000..e146f2d8c --- /dev/null +++ b/tests/parsetest.c @@ -0,0 +1,116 @@ +#include +#include +#include +#include +#include +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "json.h" +#include "parsetools.h" + +/* + + clang -I../h -I../platform/windows -D_CRT_SECURE_NO_WARNINGS -Dstrdup=_strdup -DYAML_DECLARE_STATIC=1 -Wdeprecated-declarations --rtlib=compiler-rt -o parsetest.exe parsetest.c ../c/parsetools.c ../c/json.c ../c/winskt.c ../c/xlate.c ../c/charsets.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c + + Grammar + (PHRASEREF n) + (TOKENREF n) + (ALT n) + (SEQ n) + (STAR ) + (PLUS n) + + */ + +static int myNextToken(AbstractTokenizer *tokenizer, char *s, int len, int pos, int *nextPos){ + if (pos < len){ + char c = s[pos]; + *nextPos = (pos+1); + return c&0xFF; + } else { + *nextPos = pos; + return EOF_TOKEN; + } +} + +static char *myTokenIDName(int id){ + // leaky + switch(id){ + case EOF_TOKEN: return ""; + case NO_VALID_TOKEN: return ""; + default: + { + char *buffer = safeMalloc(8,"Token ID"); + snprintf(buffer,8,"<%c>",id); + return buffer; + } + } +} + +#define TOP (FIRST_GRULE_ID+0) +#define R1 (FIRST_GRULE_ID+1) +#define R2 (FIRST_GRULE_ID+2) +#define EE (FIRST_GRULE_ID+3) +#define EEE (FIRST_GRULE_ID+4) +#define BBB (FIRST_GRULE_ID+5) + +static GRuleSpec testSet1[] = { + { G_SEQ, TOP, .sequence = { (int)'A', R1, (int)'D', (int)EOF_TOKEN, G_END} }, + { G_ALT, R1, .alternates = { (int)'B', (int)'C', G_END}}, + { G_END } +}; + + +static GRuleSpec testSet2[] = { + { G_SEQ, TOP, .sequence = { (int)'A', R1, (int)'D', (int)EOF_TOKEN, G_END} }, + { G_ALT, R1, .alternates = { EEE, EE, G_END}}, + { G_SEQ, EEE, .sequence = { (int)'E', (int)'E', (int)'E', G_END}}, + { G_SEQ, EE, .sequence = { (int)'E', (int)'E', G_END}}, + { G_END } +}; + +static GRuleSpec testSet3[] = { + { G_SEQ, TOP, .sequence = { (int)'A', R1, (int)'D', (int)EOF_TOKEN, G_END} }, + { G_STAR, R1, .star = R2 }, + { G_ALT, R2, .alternates = { BBB, EE, G_END}}, + { G_SEQ, BBB, .sequence = { (int)'B', (int)'B', (int)'B', G_END}}, + { G_SEQ, EE, .sequence = { (int)'E', (int)'E', G_END}}, + { G_END } +}; + +static char *getTestSetRuleName(int id){ + switch (id){ + case TOP: return "TOP"; + case R1: return "R1"; + case R2: return "R2"; + case EE: return "EE"; + case EEE: return "EEE"; + case BBB: return "BBB"; + default: + return "UNKNOWN_RULE"; + } +} + + + +int main(int argc, char **argv){ + AbstractTokenizer tokenizer; + tokenizer.lowTokenID = 1; + tokenizer.highTokenID = 127; + tokenizer.nextToken = myNextToken; + tokenizer.getTokenIDName = myTokenIDName; + GParseContext *ctx = gParse(testSet3,TOP,&tokenizer,argv[1],strlen(argv[1]),getTestSetRuleName); + return 0; +} + + +/* + seq( A E* D ) + + A, eps, D=E, fail + E* bears fruit until the first time it doesn't match one more +*/ + From dc6c2c28e147eaa93573ad925b596b18a50301a2 Mon Sep 17 00:00:00 2001 From: Joe Date: Thu, 10 Mar 2022 23:43:10 -0500 Subject: [PATCH 27/42] microjq does many things now and is integrated to configmgr Signed-off-by: Joe --- c/configmgr.c | 47 +++- c/embeddedjs.c | 12 +- c/json.c | 21 +- c/microjq.c | 658 +++++++++++++++++++++++++++------------------- c/parsetools.c | 344 ++++++++++++++++++------ c/yaml2json.c | 4 +- h/json.h | 5 + h/microjq.h | 13 + h/parsetools.h | 66 ++++- tests/jqtest.c | 130 +++++++++ tests/parsetest.c | 113 +++++++- 11 files changed, 1002 insertions(+), 411 deletions(-) create mode 100644 h/microjq.h create mode 100644 tests/jqtest.c diff --git a/c/configmgr.c b/c/configmgr.c index bb9f5ba51..986e9aa8a 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -50,13 +50,15 @@ #include "openprims.h" #include "bpxnet.h" #include "unixfile.h" +#include "charsets.h" +#include "collections.h" #include "json.h" #include "jsonschema.h" #include "yaml.h" #include "yaml2json.h" #include "embeddedjs.h" -#include "charsets.h" -#include "collections.h" +#include "parsetools.h" +#include "microjq.h" #ifdef __ZOWE_OS_WINDOWS typedef int64_t ssize_t; @@ -96,7 +98,7 @@ typedef int64_t ssize_t; clang++ -c ../platform/windows/cppregex.cpp ../platform/windows/winregex.cpp - clang -I %QJS%\porting -I%YAML%/include -I %QJS% -I./src -I../h -I ../platform/windows -DCONFIG_VERSION=\"2021-03-27\" -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 --rtlib=compiler-rt -o configmgr.exe configmgr.c embeddedjs.c %QJS%\quickjs.c %QJS%\cutils.c %QJS%\quickjs-libc.c %QJS%\libbf.c %QJS%\libregexp.c %QJS%\libunicode.c %QJS%\porting\winpthread.c %QJS%\porting\wintime.c %QJS%\porting\windirent.c %QJS%\porting\winunistd.c %YAML%/src/api.c %YAML%/src/reader.c %YAML%/src/scanner.c %YAML%/src/parser.c %YAML%/src/loader.c %YAML%/src/writer.c %YAML%/src/emitter.c %YAML%/src/dumper.c ../c/yaml2json.c ../c/jsonschema.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c cppregex.o winregex.o + clang -I %QJS%\porting -I%YAML%/include -I %QJS% -I./src -I../h -I ../platform/windows -DCONFIG_VERSION=\"2021-03-27\" -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 --rtlib=compiler-rt -o configmgr.exe configmgr.c embeddedjs.c %QJS%\quickjs.c %QJS%\cutils.c %QJS%\quickjs-libc.c %QJS%\libbf.c %QJS%\libregexp.c %QJS%\libunicode.c %QJS%\porting\winpthread.c %QJS%\porting\wintime.c %QJS%\porting\windirent.c %QJS%\porting\winunistd.c %YAML%/src/api.c %YAML%/src/reader.c %YAML%/src/scanner.c %YAML%/src/parser.c %YAML%/src/loader.c %YAML%/src/writer.c %YAML%/src/emitter.c %YAML%/src/dumper.c ../c/yaml2json.c ../c/microjq.c ../c/parsetools.c ../c/jsonschema.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c cppregex.o winregex.o configmgr "../tests/schemadata" "" "LIBRARY(FOO):DIR(BAR)" yak @@ -384,8 +386,6 @@ static bool addPathElement(ConfigManager *mgr, char *pathElementArg){ } static int buildConfigPath(ConfigManager *mgr, char *configPathArg){ - printf("JOE buildConfigPath\n"); - fflush(stdout); int pos = 0; int len = strlen(configPathArg); while (pos < len){ @@ -450,8 +450,6 @@ ConfigManager *makeConfigManager(char *configPathArg, char *rootSchemaDirectory, mgr->traceOut = traceOut; mgr->slh = makeShortLivedHeap(0x10000,0x100); EmbeddedJS *ejs = makeEmbeddedJS(NULL); - printf("really\n"); - fflush(stdout); mgr->ejs = ejs; trace(mgr,DEBUG,"before build config path\n"); if (buildConfigPath(mgr,configPathArg)){ @@ -493,7 +491,7 @@ ConfigManager *makeConfigManager(char *configPathArg, char *rootSchemaDirectory, freeConfigManager(mgr); return NULL; } else { - trace(mgr,INFO,"JSON Schema built successfully\n"); + trace(mgr,DEBUG,"JSON Schema built successfully\n"); mgr->topSchema = schema; } freeJsonSchemaBuilder(builder); @@ -576,7 +574,10 @@ static int loadConfigurations(ConfigManager *mgr){ if (overloadStatus){ return overloadStatus; } else { - jsonPrettyPrint(mgr, mgr->config); + if (mgr->traceLevel >= 1){ + printf("config before template eval:\n"); + jsonPrettyPrint(mgr, mgr->config); + } Json *evaluatedConfig = evaluateJsonTemplates(mgr->ejs,mgr->slh,mgr->config); if (evaluatedConfig){ mgr->config = evaluatedConfig; @@ -790,6 +791,8 @@ static void showHelp(FILE *out){ fprintf(out," -o : OUT|ERR , ERR is default\n"); fprintf(out," -s : root schema directory\n"); fprintf(out," -w : workspace directory\n"); + fprintf(out," -c : compact output for jq and extract commands\n"); + fprintf(out," -r : raw string output for jq and extract commands\n"); fprintf(out," -p : list of colon-separated configPathElements - see below\n"); fprintf(out," commands:\n"); fprintf(out," extract : prints value to stdout\n"); @@ -929,6 +932,8 @@ int main(int argc, char **argv){ int argx = 1; int traceLevel = 0; FILE *traceOut = stderr; + bool jqCompact = false; + bool jqRaw = false; if (argc == 1){ showHelp(traceOut); return 0; @@ -950,6 +955,10 @@ int main(int argc, char **argv){ zoweWorkspaceHome = optionValue; } else if ((optionValue = getStringOption(argc,argv,&argx,"-p")) != NULL){ configPath = optionValue; + } else if ((optionValue = getStringOption(argc,argv,&argx,"-c")) != NULL){ + jqCompact = true; + } else if ((optionValue = getStringOption(argc,argv,&argx,"-c")) != NULL){ + jqRaw = true; } else { char *nextArg = argv[argx]; if (strlen(nextArg) && nextArg[0] == '-'){ @@ -1033,8 +1042,23 @@ int main(int argc, char **argv){ if (argx >= argc){ trace(mgr,INFO,"jq requires at least one filter argument"); } else { - char *jqArg = argv[argx++]; - + JQTokenizer jqt; + memset(&jqt,0,sizeof(JQTokenizer)); + jqt.data = argv[argx++]; + jqt.length = strlen(jqt.data); + jqt.ccsid = 1208; + + Json *jqTree = parseJQ(&jqt,mgr->slh,0); + if (jqTree){ + int flags = ((jqCompact ? 0 : JQ_FLAG_PRINT_PRETTY)| + (jqRaw ? JQ_FLAG_RAW_STRINGS : 0)); + int evalStatus = evalJQ(mgr->config,jqTree,stdout,flags,mgr->traceLevel); + if (evalStatus != 0){ + trace(mgr, INFO,"micro jq eval problem %d\n",evalStatus); + } + } else { + trace(mgr, INFO, "Failed to parse jq expression\n"); + } } } else if (!strcmp(command,"extract")){ if (argx >= argc){ @@ -1050,7 +1074,6 @@ int main(int argc, char **argv){ printJsonPointer(mgr->traceOut,jp); fflush(mgr->traceOut); } - /* Friday, extract some text and show Jack */ extractText(mgr,jp,stdout); printf("\n"); fflush(stdout); diff --git a/c/embeddedjs.c b/c/embeddedjs.c index a8023f22b..67bcb461b 100644 --- a/c/embeddedjs.c +++ b/c/embeddedjs.c @@ -431,7 +431,7 @@ static JSValue jsonToJS1(EmbeddedJS *ejs, Json *json, bool hideUnevaluated){ size_t strLen = strlen(str); char convertedStr[strLen+1]; snprintf (convertedStr, strLen + 1, "%.*s", (int)strLen, str); - printf ("about to convert string '%s'\n", convertedStr); + /* printf ("about to convert string '%s'\n", convertedStr); */ convertFromNative(convertedStr, strLen); return JS_NewString(ctx, convertedStr); } @@ -456,7 +456,7 @@ static JSValue jsonToJS1(EmbeddedJS *ejs, Json *json, bool hideUnevaluated){ size_t keyLen = strlen(key); char convertedKey[keyLen+1]; snprintf (convertedKey, keyLen + 1, "%.*s", (int)keyLen, key); - printf ("about to convert key '%s'\n", convertedKey); + /* printf ("about to convert key '%s'\n", convertedKey); */ convertFromNative(convertedKey, keyLen); JS_SetPropertyStr(ctx, object, @@ -594,7 +594,7 @@ static bool evaluationVisitor(void *context, Json *json, Json *parent, char *key property = jsonObjectGetNextProperty(property)) { char *key = jsonPropertyGetKey(property); Json *value = jsonPropertyGetValue(property); - printf ("global object key '%s'\n", key); + /* printf ("global object key '%s'\n", key); */ size_t keyLen = strlen(key); char convertedKey[keyLen+1]; snprintf (convertedKey, keyLen + 1, "%.*s", (int)keyLen, key); @@ -608,7 +608,7 @@ static bool evaluationVisitor(void *context, Json *json, Json *parent, char *key char asciiSource[sourceLen + 1]; snprintf (asciiSource, sourceLen + 1, "%.*s", (int)sourceLen, source); convertFromNative(asciiSource, sourceLen); - printf("should evaluate: %s\n",source); + /* printf("should evaluate: %s\n",source); */ int evalStatus = 0; char embedded[] = ""; convertFromNative(embedded, sizeof(embedded)); @@ -680,9 +680,6 @@ EmbeddedJS *makeEmbeddedJS(EmbeddedJS *sharedRuntimeEJS){ /* can be NULL */ exit(2); } - printf("JSTest Made Runtime and context \n"); - fflush(stdout); - /* loader for ES6 modules */ JS_SetModuleLoaderFunc(embeddedJS->rt, NULL, js_module_loader, NULL); @@ -705,7 +702,6 @@ EmbeddedJS *makeEmbeddedJS(EmbeddedJS *sharedRuntimeEJS){ /* can be NULL */ JSValue throwaway = ejsEvalBuffer(embeddedJS, str, strlen(str), "", JS_EVAL_TYPE_MODULE, &evalStatus); } - printf("returning embeddedJS 0x%p\n",embeddedJS); return embeddedJS; } diff --git a/c/json.c b/c/json.c index 9873fbe31..466d0a4f5 100644 --- a/c/json.c +++ b/c/json.c @@ -105,10 +105,7 @@ int jsonIsError(Json *json) { static void writeToBuffer(struct jsonPrinter_tag *p, char *text, int len); -jsonPrinter *makeJsonPrinter(int fd) { - jsonPrinter *p = (jsonPrinter*) safeMalloc(sizeof (jsonPrinter), "JSON Printer"); - - p->fd = fd; +void jsonPrinterReset(jsonPrinter *p){ p->depth = 0; p->indentString = " "; p->isStart = TRUE; @@ -116,6 +113,12 @@ jsonPrinter *makeJsonPrinter(int fd) { p->_conversionBufferSize = 0; p->_conversionBuffer = NULL; p->mode = JSON_MODE_NATIVE_CHARSET; +} + +jsonPrinter *makeJsonPrinter(int fd) { + jsonPrinter *p = (jsonPrinter*) safeMalloc(sizeof (jsonPrinter), "JSON Printer"); + p->fd = fd; + jsonPrinterReset(p); return p; } @@ -132,13 +135,7 @@ jsonPrinter *makeCustomJsonPrinter(void (*writeMethod)(jsonPrinter *, char *, in p->isCustom = TRUE; p->customWrite = writeMethod; p->customObject = object; - p->depth = 0; - p->indentString = " "; - p->isStart = TRUE; - p->isFirstLine = TRUE; - p->_conversionBufferSize = 0; - p->_conversionBuffer = NULL; - p->mode = JSON_MODE_NATIVE_CHARSET; + jsonPrinterReset(p); return p; } @@ -2489,8 +2486,6 @@ Json *jsonParseFile(ShortLivedHeap *slh, const char *filename, char* errorBuffer } Json *jsonParseFile2(ShortLivedHeap *slh, const char *filename, char* errorBufferOrNull, int errorBufferSize){ - printf("JOE: jsonParseFile2\n"); - fflush(stdout); return jsonParseFileInternal(slh,filename,errorBufferOrNull,errorBufferSize,JSON_PARSE_VERSION_2); } diff --git a/c/microjq.c b/c/microjq.c index 4216167ca..fac64ca61 100644 --- a/c/microjq.c +++ b/c/microjq.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #endif @@ -50,202 +51,8 @@ #include "unixfile.h" #include "json.h" #include "parsetools.h" +#include "microjq.h" -/* - clang -I %QJS%/porting -I%YAML%/include -I../platform/windows -I %QJS% -I ..\h -Wdeprecated-declarations -D_CRT_SECURE_NO_WARNINGS -o microjq.exe microjq.c parsetools.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c - */ - -#define STATE_INITIAL 0 -#define STATE_INTEGER 1 -#define STATE_IDENTIFIER 2 -#define STATE_SQUOTE 3 -#define STATE_DQUOTE 4 - -#define TOKEN_IDENTIFIER FIRST_REAL_TOKEN_ID -#define TOKEN_INTEGER (FIRST_REAL_TOKEN_ID+1) -#define TOKEN_DQUOTE_STRING FIRST_REAL_TOKEN_ID+2 -#define TOKEN_SQUOTE_STRING FIRST_REAL_TOKEN_ID+3 -#define TOKEN_LPAREN FIRST_REAL_TOKEN_ID+4 -#define TOKEN_RPAREN FIRST_REAL_TOKEN_ID+5 -#define TOKEN_LBRACK FIRST_REAL_TOKEN_ID+6 -#define TOKEN_RBRACK FIRST_REAL_TOKEN_ID+7 -#define TOKEN_LBRACE FIRST_REAL_TOKEN_ID+8 -#define TOKEN_RBRACE FIRST_REAL_TOKEN_ID+9 -#define TOKEN_DOT FIRST_REAL_TOKEN_ID+10 -#define TOKEN_COMMA FIRST_REAL_TOKEN_ID+11 -#define TOKEN_VBAR FIRST_REAL_TOKEN_ID+12 -#define TOKEN_COLON FIRST_REAL_TOKEN_ID+13 -#define TOKEN_QMARK FIRST_REAL_TOKEN_ID+14 -#define TOKEN_PLUS FIRST_REAL_TOKEN_ID+15 -#define TOKEN_DASH FIRST_REAL_TOKEN_ID+16 -#define TOKEN_STAR FIRST_REAL_TOKEN_ID+17 -#define TOKEN_SLASH FIRST_REAL_TOKEN_ID+18 -#define TOKEN_PERCENT FIRST_REAL_TOKEN_ID+19 - -#define LAST_TOKEN_ID TOKEN_PERCENT /* KEEP ME UPDATED!!! - whenever a new token ID is added*/ - - -typedef struct JQParser_tag { - AbstractTokenizer tokenizer; /* low token, high token */ - int ccsid; - int tokenState; /* = STATE_INITIAL; */ - char *data; - int length; - int pos; -} JQParser; - -static char *getTokenName(int id){ - switch (id){ - case EOF_TOKEN: return "EOF"; - case TOKEN_IDENTIFIER: return "ID"; - case TOKEN_INTEGER: return "INTEGER"; - case TOKEN_DQUOTE_STRING: return "DQUOTE_STRING"; - case TOKEN_SQUOTE_STRING: return "SQUOTE_STRING"; - case TOKEN_LPAREN: return "LPAREN"; - case TOKEN_RPAREN: return "RPAREN"; - case TOKEN_LBRACK: return "LBRACK"; - case TOKEN_RBRACK: return "RBRACK"; - case TOKEN_LBRACE: return "LBRACE"; - case TOKEN_RBRACE: return "RBRACE"; - case TOKEN_DOT: return "DOT"; - case TOKEN_COMMA: return "COMMA"; - case TOKEN_VBAR: return "VBAR"; - case TOKEN_COLON: return "COLON"; - case TOKEN_QMARK: return "QMARK"; - case TOKEN_PLUS: return "PLUS"; - case TOKEN_DASH: return "DASH"; - case TOKEN_STAR: return "STAR"; - case TOKEN_SLASH: return "SLASH"; - case TOKEN_PERCENT: return "PERCENT"; - default:return "BAD_TOKEN"; - } -} - -static bool isLetter(JQParser *jqp, int c){ - return testCharProp(jqp->ccsid,c,CPROP_LETTER); -} - -static bool isDigit(JQParser *jqp, int c){ - return testCharProp(jqp->ccsid,c,CPROP_DIGIT); -} - -static bool isWhite(JQParser *jqp, int c){ - return testCharProp(jqp->ccsid,c,CPROP_WHITE); -} - -static int jqpToken(JQParser *jqp){ - /* belt *AND* suspenders */ - AbstractTokenizer *tokenizer = (AbstractTokenizer*)jqp; - if (jqp->pos >= jqp->length){ - tokenizer->lastTokenStart = jqp->pos; /* 0-length token */ - tokenizer->lastTokenEnd = jqp->pos; - return EOF_TOKEN; - } - int tokenID = NO_VALID_TOKEN; - while(tokenID == NO_VALID_TOKEN){ - int c = -1; // EOF by defaultx - int charPos = jqp->pos; - if (jqp->pos < jqp->length){ - c = (int)jqp->data[jqp->pos++]; - } - /* printf("at pos=%d, c=0x%x state=%d\n",jqp->pos,c,jqp->tokenState); */ - switch (jqp->tokenState){ - case STATE_INITIAL: - if (c == -1){ - tokenID = EOF_TOKEN; - tokenizer->lastTokenStart = jqp->length; - } else if (isLetter(jqp,c) || - (c == UNI_DOLLAR) || - (c == UNI_UNDER)){ - jqp->tokenState = STATE_IDENTIFIER; - tokenizer->lastTokenStart = charPos; - } else if (isDigit(jqp,c)){ - jqp->tokenState = STATE_INTEGER; - tokenizer->lastTokenStart = charPos; - } else if (isWhite(jqp,c)){ - // just roll forward - } else if (c == UNI_DQUOTE){ - jqp->tokenState = STATE_DQUOTE; - tokenizer->lastTokenStart = charPos; - } else if (c == UNI_SQUOTE){ - jqp->tokenState = STATE_SQUOTE; - tokenizer->lastTokenStart = charPos; - } else { - /* Single-char-punc tokens */ - switch (c){ - case UNI_LPAREN: tokenID = TOKEN_LPAREN; break; - case UNI_RPAREN: tokenID = TOKEN_RPAREN; break; - case UNI_LBRACK: tokenID = TOKEN_LBRACK; break; - case UNI_RBRACK: tokenID = TOKEN_RBRACK; break; - case UNI_LBRACE: tokenID = TOKEN_LBRACE; break; - case UNI_RBRACE: tokenID = TOKEN_RBRACE; break; - case UNI_DOT: tokenID = TOKEN_DOT; break; - case UNI_COMMA: tokenID = TOKEN_COMMA; break; - case UNI_COLON: tokenID = TOKEN_COLON; break; - case UNI_VBAR: tokenID = TOKEN_VBAR; break; - case UNI_QMARK: tokenID = TOKEN_QMARK; break; - case UNI_PLUS: tokenID = TOKEN_PLUS; break; - case UNI_DASH: tokenID = TOKEN_DASH; break; - case UNI_STAR: tokenID = TOKEN_STAR; break; - case UNI_SLASH: tokenID = TOKEN_SLASH; break; - case UNI_PERCENT: tokenID = TOKEN_PERCENT; break; - default: - /* unhandled char, kill the parse */ - return NO_VALID_TOKEN; - } - tokenizer->lastTokenStart = charPos; - } - break; - case STATE_INTEGER: - if (!isDigit(jqp,c)){ - if (c != -1) jqp->pos--; /* nothing to push back */ - tokenID = TOKEN_INTEGER; - } - break; - case STATE_IDENTIFIER: - if (isLetter(jqp,c) || - (c == UNI_DOLLAR) || - (c == UNI_UNDER)){ - // accumulate - } else { - if (c != -1) jqp->pos--; /* nothing to push back */ - tokenID = TOKEN_IDENTIFIER; - } - break; - case STATE_DQUOTE: - if (c == -1){ /* EOF in string */ - return NO_VALID_TOKEN; - } else if (c == UNI_DQUOTE){ - tokenID = TOKEN_DQUOTE_STRING; - } - break; - case STATE_SQUOTE: - if (c == -1){ /* EOF in string */ - return NO_VALID_TOKEN; - } else if (c == UNI_SQUOTE){ - tokenID = TOKEN_SQUOTE_STRING; - } - default: - printf("*** PANIC *** internal error unknown tokenizer state = %d\n",jqp->tokenState); - return NO_VALID_TOKEN; - // unknow state, kill the parse - } - if (tokenID != NO_VALID_TOKEN){ - tokenizer->lastTokenEnd = jqp->pos; - jqp->tokenState = STATE_INITIAL; - return tokenID; - } - } - return 0; -} - -int getNextToken(AbstractTokenizer *tokenizer, char *s, int len, int pos, int *nextPos){ - JQParser *jqp = (JQParser*)tokenizer; - jqp->pos = pos; - int id = jqpToken(jqp); - *nextPos = jqp->pos; - return id; -} /* Grammar @@ -284,109 +91,406 @@ int getNextToken(AbstractTokenizer *tokenizer, char *s, int len, int pos, int *n */ -static void tokenTest(JQParser *jqp){ - AbstractTokenizer *tokenizer = (AbstractTokenizer*)jqp; - printf("length = %d\n",jqp->length); - while (true){ - int tokenID = jqpToken(jqp); - printf("token id=%d, %s pos is now=%d\n",tokenID,getTokenName(tokenID),jqp->pos); - printf(" from %d to %d\n",tokenizer->lastTokenStart,tokenizer->lastTokenEnd); - if (tokenID == NO_VALID_TOKEN){ - printf("bad token seen near %d\n",jqp->pos); - break; - } else if (tokenID == EOF_TOKEN){ - printf("EOF Seen\n"); - break; - } - } -} -#define TOP (FIRST_GRULE_ID+0) -#define EXPR (FIRST_GRULE_ID+1) -#define EXPR_TAIL (FIRST_GRULE_ID+2) -#define EXPR_ADD_SUB (FIRST_GRULE_ID+3) -#define PLUS_MINUS (FIRST_GRULE_ID+4) -#define TERM (FIRST_GRULE_ID+5) -#define TERM_TAIL (FIRST_GRULE_ID+6) -#define TERM_MULT_DIV (FIRST_GRULE_ID+7) -#define STAR_SLASH (FIRST_GRULE_ID+8) -#define FACTOR (FIRST_GRULE_ID+9) - -static char *getRuleName(int id){ +#define TOP (FIRST_GRULE_ID+0) +#define EXPR (FIRST_GRULE_ID+1) +#define EXPR_TAIL (FIRST_GRULE_ID+2) +#define EXPR_PIPE_OP (FIRST_GRULE_ID+3) +#define OPERATION (FIRST_GRULE_ID+4) +#define CREATE_ARRAY (FIRST_GRULE_ID+5) +#define CREATE_ARRAY_TAIL (FIRST_GRULE_ID+6) +#define COMMA_EXPR (FIRST_GRULE_ID+7) +#define CREATE_OBJECT (FIRST_GRULE_ID+8) +#define CREATE_OBJECT_TAIL (FIRST_GRULE_ID+9) +#define COMMA_KEY_VALUE (FIRST_GRULE_ID+10) +#define KEY_VALUE (FIRST_GRULE_ID+11) +#define FILTER (FIRST_GRULE_ID+12) +#define TRAVERSAL (FIRST_GRULE_ID+13) +#define MORE_TRAVERSALS (FIRST_GRULE_ID+14) +#define INDEX (FIRST_GRULE_ID+15) +#define PICK (FIRST_GRULE_ID+16) +#define EXPLODE (FIRST_GRULE_ID+17) +#define CONTEXT (FIRST_GRULE_ID+18) + +static char *getJQRuleName(int id){ switch (id){ case TOP: return "TOP"; case EXPR: return "EXPR"; case EXPR_TAIL: return "EXPR_TAIL"; - case EXPR_ADD_SUB: return "EXPR_ADD_SUB"; - case PLUS_MINUS: return "PLUS_MINUS"; - case TERM: return "TERM"; - case TERM_TAIL: return "TERM_TAIL"; - case TERM_MULT_DIV: return "TERM_MULT_DIV"; - case STAR_SLASH: return "STAR_SLASH"; - case FACTOR: return "FACTOR"; + case EXPR_PIPE_OP: return "EXPR_PIPE_OP"; + case OPERATION: return "OPERATION"; + case CREATE_ARRAY: return "CREATE_ARRAY"; + case CREATE_ARRAY_TAIL: return "CREATE_ARRAY_TAIL"; + case COMMA_EXPR: return "COMMA_EXPR"; + case CREATE_OBJECT: return "CREATE_OBJECT"; + case CREATE_OBJECT_TAIL: return "CREATE_OBJECT_TAIL"; + case COMMA_KEY_VALUE: return "COMMA_KEY_VALUE"; + case KEY_VALUE: return "KEY_VALUE"; + case FILTER: return "FILTER"; + case TRAVERSAL: return "TRAVERSAL"; + case MORE_TRAVERSALS: return "MORE_TRAVERSALS"; + case INDEX: return "INDEX"; + case PICK: return "PICK"; + case EXPLODE: return "EXPLODE"; + case CONTEXT: return "CONTEXT"; default: return "UNKNOWN_RULE"; } } -static GRuleSpec expressionGrammar[] = { - { G_SEQ, TOP, .sequence = { EXPR, EOF_TOKEN, G_END} }, - { G_SEQ, EXPR, .sequence = { TERM, EXPR_TAIL, G_END }}, - { G_STAR, EXPR_TAIL, .star = EXPR_ADD_SUB }, - { G_SEQ, EXPR_ADD_SUB, .sequence = { PLUS_MINUS, TERM, G_END }}, - { G_ALT, PLUS_MINUS, .alternates = { TOKEN_PLUS, TOKEN_DASH, G_END }}, - { G_SEQ, TERM, .sequence = { FACTOR, TERM_TAIL, G_END }}, - { G_STAR, TERM_TAIL, .star = TERM_MULT_DIV }, - { G_SEQ, TERM_MULT_DIV, .sequence = { STAR_SLASH, FACTOR, G_END}}, - { G_ALT, STAR_SLASH, .alternates = { TOKEN_STAR, TOKEN_SLASH, G_END }}, - { G_ALT, FACTOR, .alternates = { TOKEN_IDENTIFIER, TOKEN_INTEGER, G_END }}, /* add expression recursion later */ +static GRuleSpec jqGrammar[] = { + { G_SEQ, TOP, .sequence = { EXPR, EOF_TOKEN, G_END} }, + { G_SEQ, EXPR, .sequence = { OPERATION, EXPR_TAIL, G_END }}, + { G_STAR, EXPR_TAIL, .star = EXPR_PIPE_OP }, + { G_SEQ, EXPR_PIPE_OP, .sequence = { JTOKEN_VBAR, OPERATION, G_END }}, + { G_ALT, OPERATION, .alternates = { JTOKEN_INTEGER, + /* JTOKEN_DQUOTE_STRING, + JTOKEN_SQUOTE_STRING, + CREATE_ARRAY, + CREATE_OBJECT, --> how are these distinguishable from traversals?? */ + FILTER, + G_END }}, + { G_SEQ, CREATE_ARRAY, .sequence = { JTOKEN_LBRACK, EXPR, CREATE_ARRAY_TAIL, JTOKEN_RBRACK, G_END }}, + { G_STAR, CREATE_ARRAY_TAIL, .star = COMMA_EXPR }, + { G_SEQ, COMMA_EXPR, .sequence = { JTOKEN_COMMA, EXPR, G_END }}, + { G_SEQ, CREATE_OBJECT, .sequence = { JTOKEN_LBRACE, KEY_VALUE, CREATE_OBJECT_TAIL, JTOKEN_RBRACE, G_END }}, + { G_STAR, CREATE_OBJECT_TAIL, .star = COMMA_KEY_VALUE }, + { G_SEQ, COMMA_KEY_VALUE, .sequence = {JTOKEN_COMMA, KEY_VALUE, G_END }}, + { G_SEQ, KEY_VALUE, .sequence = { JTOKEN_IDENTIFIER, JTOKEN_COLON, EXPR, G_END }}, + { G_SEQ, FILTER, .sequence = { TRAVERSAL, MORE_TRAVERSALS, G_END }}, + { G_ALT, TRAVERSAL, .alternates = { INDEX, PICK, EXPLODE, CONTEXT, G_END }}, + { G_STAR, MORE_TRAVERSALS, .star = TRAVERSAL }, + { G_SEQ, INDEX, .sequence = { JTOKEN_LBRACK, JTOKEN_INTEGER, JTOKEN_RBRACK, G_END}}, + { G_SEQ, PICK, .sequence = { JTOKEN_DOT, JTOKEN_IDENTIFIER, G_END }}, + { G_SEQ, EXPLODE, .sequence = { JTOKEN_LBRACK, JTOKEN_RBRACK, G_END }}, + { G_SEQ, CONTEXT, .sequence = { JTOKEN_DOT, G_END}}, { G_END } }; -static void parseTest1(JQParser *jqp){ - AbstractTokenizer *tokenizer = (AbstractTokenizer*)jqp; +Json *parseJQ(JQTokenizer *jqt, ShortLivedHeap *slh, int traceLevel){ + AbstractTokenizer *tokenizer = (AbstractTokenizer*)jqt; tokenizer->lowTokenID = FIRST_REAL_TOKEN_ID; - tokenizer->highTokenID = LAST_TOKEN_ID; - tokenizer->nextToken = getNextToken; - tokenizer->getTokenIDName = getTokenName; - GParseContext *ctx = gParse(expressionGrammar,TOP,tokenizer,jqp->data,jqp->length,getRuleName); - printf("parse ctx = 0x%p\n",ctx); + tokenizer->highTokenID = LAST_JTOKEN_ID; + tokenizer->nextToken = getNextJToken; + tokenizer->getTokenIDName = getJTokenName; + tokenizer->getTokenJsonType = getJTokenJsonType; + GParseContext *ctx = gParse(jqGrammar,TOP,tokenizer,jqt->data,jqt->length,getJQRuleName,traceLevel); if (ctx->status > 0){ - #ifdef __ZOWE_OS_WINDOWS - int stdoutFD = _fileno(stdout); -#else - int stdoutFD = STDOUT_FILENO; -#endif - ShortLivedHeap *slh = makeShortLivedHeap(0x10000, 100); - Json *tree = gBuildJSON(ctx,slh); - if (tree){ - jsonPrinter *p = makeJsonPrinter(stdoutFD); - jsonEnablePrettyPrint(p); - printf("parse result as json\n"); - jsonPrint(p,tree); + return gBuildJSON(ctx,slh); + } else { + return NULL; + } +} + +#define JQ_ERROR_MAX 1024 + + + +typedef struct JQEvalContext_tag { + FILE *out; + jmp_buf recoveryData; + int flags; + int traceLevel; + int errorCode; + char *errorMessage; + int errorMessageLength; + jsonPrinter *printer; + FILE *traceOut; +} JQEvalContext; + +#define JQ_SUCCESS 0 +#define JQ_MISSING_PROPERTY 8 +#define JQ_BAD_PROPERTY_TYPE 12 +#define JQ_NOT_AN_OBJECT 16 +#define JQ_NOT_AN_ARRAY 20 +#define JQ_UNEXPECTED_OPERATION 24 +#define JQ_INTERNAL_NULL_POINTER 28 +#define JQ_NON_INTEGER_ARRAY_INDEX_NOT_YET_SUPPORTED 32 +#define JQ_ARRAY_INDEX_OUT_OF_BOUNDS 36 + +static void jqEvalThrow(JQEvalContext *ctx, int errorCode, char *formatString, ...){ + va_list argPointer; + char *text = safeMalloc(JQ_ERROR_MAX,"ErrorBuffer"); + va_start(argPointer,formatString); + vsnprintf(text,JQ_ERROR_MAX,formatString,argPointer); + va_end(argPointer); + ctx->errorCode = errorCode; + ctx->errorMessage = text; + ctx->errorMessageLength = JQ_ERROR_MAX; + longjmp(ctx->recoveryData,1); +} + +static Json *getJQProperty(JQEvalContext *ctx, JsonObject *value, char *key, bool isRequired){ + if (value == NULL){ + jqEvalThrow(ctx,JQ_INTERNAL_NULL_POINTER,"NULL pointer seen when looking for key = '%s'",key); + return NULL; + } + Json *propertyValue = jsonObjectGetPropertyValue(value,key); + if (isRequired && (propertyValue == NULL)){ + jqEvalThrow(ctx,JQ_MISSING_PROPERTY,"Missing required property %s",key); + } + return propertyValue; +} + +static int getJQInt(JQEvalContext *ctx, JsonObject *value, char *key, bool isRequired, int defaultValue){ + Json *intProperty = getJQProperty(ctx,value,key,isRequired); + if (jsonIsNumber(intProperty)){ + return jsonAsNumber(intProperty); + } else if (isRequired){ + jqEvalThrow(ctx,JQ_BAD_PROPERTY_TYPE,"Property '%s' is not int",key); + return 0; /* unreachable */ + } else { + return defaultValue; + } +} + +static JsonObject *getJQObject(JQEvalContext *ctx, JsonObject *value, char *key, bool isRequired){ + Json *p = getJQProperty(ctx,value,key,isRequired); + if (jsonIsObject(p)){ + return jsonAsObject(p); + } else if (isRequired){ + jqEvalThrow(ctx,JQ_BAD_PROPERTY_TYPE,"Property '%s' is not object",key); + return NULL; /* unreachable */ + } else { + return NULL; + } +} + +static char *getJQString(JQEvalContext *ctx, JsonObject *value, char *key, bool isRequired){ + Json *p = getJQProperty(ctx,value,key,isRequired); + if (jsonIsString(p)){ + return jsonAsString(p); + } else if (isRequired){ + jqEvalThrow(ctx,JQ_BAD_PROPERTY_TYPE,"Property '%s' is not string",key); + return NULL; /* unreachable */ + } else { + return NULL; + } +} + +static JsonArray *getJQArray(JQEvalContext *ctx, JsonObject *value, char *key, bool isRequired){ + Json *p = getJQProperty(ctx,value,key,isRequired); + if (jsonIsArray(p)){ + return jsonAsArray(p); + } else if (isRequired){ + jqEvalThrow(ctx,JQ_BAD_PROPERTY_TYPE,"Property '%s' is not array",key); + return NULL; /* unreachable */ + } else { + return NULL; + } +} + +static JsonObject *jqCastToObject(JQEvalContext *ctx, Json *json){ + if (json == NULL){ + jqEvalThrow(ctx,JQ_INTERNAL_NULL_POINTER,"NULL pointer seen when casting to JsonObject"); + return NULL; + } + if (jsonIsObject(json)){ + return jsonAsObject(json); + } else { + jqEvalThrow(ctx,JQ_NOT_AN_OBJECT,"Attempt to use value as object that is not an object"); + return NULL; /* unreachable */ + } +} + +static JsonArray *jqCastToArray(JQEvalContext *ctx, Json *json){ + if (jsonIsArray(json)){ + return jsonAsArray(json); + } else { + jqEvalThrow(ctx,JQ_NOT_AN_ARRAY,"Attempt to use value as array that is not an array"); + return NULL; /* unreachable */ + } +} + +static void jqPrint(JQEvalContext *ctx, Json *json){ + if (jsonIsString(json) && ctx->flags & JQ_FLAG_RAW_STRINGS){ + char *s = jsonAsString(json); + fprintf(ctx->out,"%s",s); + } else { + jsonPrint(ctx->printer,json); + } +} + +static void evalTraversal(JQEvalContext *ctx, Json *value, Json *filter, int index, int moreCount){ + JsonObject *filterObject = jqCastToObject(ctx,filter); + if (ctx->traceLevel >= 1){ + fprintf(ctx->traceOut,"evalTraversal, index=%d mcount=%d\n",index,moreCount); + } + Json *firstTraversal = getJQProperty(ctx,filterObject,"TRAVERSAL",true); + JsonArray *moreTraversals = getJQArray(ctx,filterObject,"MORE_TRAVERSALS",true); + Json *traversal = (index == 0 ) ? firstTraversal : jsonArrayGetItem(moreTraversals,index-1); + JsonObject *traversalObject = jqCastToObject(ctx,traversal); + int traversalType = getJQInt(ctx,traversalObject,"altID",true,0); + Json *traversalDetails = getJQProperty(ctx,traversalObject,"value",true); + switch (traversalType){ + case INDEX: + { + JsonObject *detailsObject = jqCastToObject(ctx,traversalDetails); + int arrayIndex = getJQInt(ctx,detailsObject,"INTEGER",false,-1); + if (arrayIndex == -1){ + jqEvalThrow(ctx,JQ_NON_INTEGER_ARRAY_INDEX_NOT_YET_SUPPORTED,"Index must be integer"); + } else { + JsonArray *valueArray = jqCastToArray(ctx,value); + if (arrayIndex >= 0 || arrayIndex < jsonArrayGetCount(valueArray)){ + Json *valueForIndex = jsonArrayGetItem(valueArray,arrayIndex); + if (index+1 <= moreCount){ + evalTraversal(ctx,valueForIndex,filter,index+1,moreCount); + } else { + jqPrint(ctx,valueForIndex); + } + } else { + jqEvalThrow(ctx,JQ_ARRAY_INDEX_OUT_OF_BOUNDS,"% is not in size of array",arrayIndex); + } + } + } + break; + case PICK: + { + JsonObject *detailsObject = jqCastToObject(ctx,traversalDetails); + char *identifier = getJQString(ctx,detailsObject,"ID",true); + JsonObject *valueObject = jqCastToObject(ctx,value); + Json *valueForID = getJQProperty(ctx,valueObject,identifier,true); + if (index+1 <= moreCount){ + evalTraversal(ctx,valueForID,filter,index+1,moreCount); + } else { + jqPrint(ctx,valueForID); + } + } + break; + case EXPLODE: + { + JsonArray *valueArray = jqCastToArray(ctx,value); + int valueCount = jsonArrayGetCount(valueArray); + for (int v=0; vprinter); + Json *element = jsonArrayGetItem(valueArray,v); + if (index+1 <= moreCount){ + evalTraversal(ctx,element,filter,index+1,moreCount); + } else { + jqPrint(ctx,element); + } + fprintf(ctx->out,"\n"); + } + } + break; + case CONTEXT: + if (index+1 <= moreCount){ + evalTraversal(ctx,value,filter,index+1,moreCount); + } else { + jqPrint(ctx,value); } + break; + default: + jqEvalThrow(ctx,JQ_UNEXPECTED_OPERATION,"unexpeted operation %d",traversalType); } } +static void evalFilter(JQEvalContext *ctx, Json *value, Json *filter){ + JsonObject *filterObject = jqCastToObject(ctx,filter); + JsonArray *moreTraversals = getJQArray(ctx,filterObject,"MORE_TRAVERSALS",true); + int moreCount = jsonArrayGetCount(moreTraversals); + evalTraversal(ctx,value,filter,0,moreCount); +} + +static void evalOperation(JQEvalContext *ctx, Json *value, Json *operation){ + JsonObject *operationObject = jqCastToObject(ctx,operation); + int altID = getJQInt(ctx,operationObject,"altID",true,-1); + switch (altID){ + case FILTER: + evalFilter(ctx,value,getJQProperty(ctx,operationObject,"value",true)); + break; + default: + jqEvalThrow(ctx,JQ_UNEXPECTED_OPERATION,"unexpected operation %d",altID); + } +} + +static void evalJQExpr(JQEvalContext *ctx, Json *value, Json *expr){ + JsonObject *exprObject = jqCastToObject(ctx,expr); + Json *operation = getJQProperty(ctx,exprObject,"OPERATION",true); + Json *exprTail = getJQProperty(ctx,exprObject,"EXPR_TAIL",true); + evalOperation(ctx,value,operation); + /* ignoring tail for now */ +} -int main(int argc, char **argv){ - char *command = argv[1]; - char *filter = argv[2]; - char *filename = argv[3]; - JQParser jqp; - memset(&jqp,0,sizeof(JQParser)); - jqp.data = argv[2]; - jqp.length = strlen(jqp.data); - jqp.ccsid = 1208; - if (!strcmp(command,"tokenize")){ - printf("tokenize...\n"); - tokenTest(&jqp); - } else if (!strcmp(command,"parse")){ - printf("parse...\n"); - parseTest1(&jqp); +int evalJQ(Json *value, Json *jqTree, FILE *out, int flags, int traceLevel){ + JQEvalContext ctx; + memset(&ctx,0,sizeof(JQEvalContext)); + ctx.out = out; +#ifdef __ZOWE_OS_WINDOWS + int fd = _fileno(out); +#else + int fd = fileno(out); +#endif + ctx.printer = makeJsonPrinter(fd); + if (flags & JQ_FLAG_PRINT_PRETTY){ + jsonEnablePrettyPrint(ctx.printer); + } + ctx.traceLevel = traceLevel; + ctx.traceOut = stderr; + ctx.flags = flags; + if (setjmp(ctx.recoveryData) == 0) { /* normal execution */ + evalJQExpr(&ctx,value,getJQProperty(&ctx,jqCastToObject(&ctx,jqTree),"EXPR",true)); + return JQ_SUCCESS; } else { - printf("bad command: %s\n",command); + fprintf(stderr,"uJQ error message: %s\n",ctx.errorMessage); + return ctx.errorCode; } - return 0; } + +/* + Known examples from zowe-install-plugins + +arguments: "-r .apimlServices.static.file" +arguments: "-r .apimlServices.static[].file" +arguments: "-r .appfwPlugins[0].path" +arguments: "-r .commands.configure" +arguments: "-r .commands.preConfigure" +arguments: "-r .commands.start" +arguments: "-r .commands.validate" +arguments: "-r .components.caching-service.storage.mode" +arguments: "-r .gatewaySharedLibs[0]" +arguments: "-r .zOSMF.host" +arguments: "-r .zOSMF.port" +arguments: "-r .zowe.launchScript.logLevel" +arguments: "-r .zowe.runtimeDirectory" +arguments: "-r .zowe.setup.certificate.dname.caCommonName" +arguments: "-r .zowe.setup.certificate.dname.commonName" +arguments: "-r .zowe.setup.certificate.dname.country" +arguments: "-r .zowe.setup.certificate.dname.locality" +arguments: "-r .zowe.setup.certificate.dname.org" +arguments: "-r .zowe.setup.certificate.dname.orgUnit" +arguments: "-r .zowe.setup.certificate.dname.state" +arguments: "-r .zowe.setup.certificate.importCertificateAuthorities" +arguments: "-r .zowe.setup.certificate.pkcs12.caAlias" +arguments: "-r .zowe.setup.certificate.pkcs12.caPassword" +arguments: "-r .zowe.setup.certificate.pkcs12.directory" +arguments: "-r .zowe.setup.certificate.pkcs12.import.alias" +arguments: "-r .zowe.setup.certificate.pkcs12.import.keystore" +arguments: "-r .zowe.setup.certificate.pkcs12.import.password" +arguments: "-r .zowe.setup.certificate.pkcs12.name" +arguments: "-r .zowe.setup.certificate.pkcs12.password" +arguments: "-r .zowe.setup.certificate.san" +arguments: "-r .zowe.setup.certificate.type" +arguments: "-r .zowe.setup.certificate.validity" +arguments: "-r .zowe.setup.certificate.zOSMF.ca" +arguments: "-r .zowe.setup.certificate.zOSMF.user" +arguments: "-r .zowe.setup.mvs.authLoadlib" +arguments: "-r .zowe.setup.mvs.authPluginLib" +arguments: "-r .zowe.setup.mvs.hlq" +arguments: "-r .zowe.setup.mvs.jcllib" +arguments: "-r .zowe.setup.mvs.parmlib" +arguments: "-r .zowe.setup.mvs.proclib" +arguments: "-r .zowe.setup.security.groups.admin" +arguments: "-r .zowe.setup.security.groups.stc" +arguments: "-r .zowe.setup.security.groups.sysProg" +arguments: "-r .zowe.setup.security.product" +arguments: "-r .zowe.setup.security.stcs.aux" +arguments: "-r .zowe.setup.security.stcs.xmem" +arguments: "-r .zowe.setup.security.stcs.zowe" +arguments: "-r .zowe.setup.security.users.aux" +arguments: "-r .zowe.setup.security.users.xmem" +arguments: "-r .zowe.setup.security.users.zowe" +arguments: "-r .zowe.verifyCertificates" +arguments: "-r .zowe.workspaceDirectory" + */ diff --git a/c/parsetools.c b/c/parsetools.c index acdb1e147..fc580af3f 100644 --- a/c/parsetools.c +++ b/c/parsetools.c @@ -214,51 +214,6 @@ static GRule *makeGRules(GRuleSpec *ruleSpecs, AbstractTokenizer *tokenizer, Sho return rules; } -/* - - HERE - 0) specs could be arrays of strings for simpler parsing - maybe - 0.5) factor to parse test and jqtest - 0.6) separate out the JLExer - 1) write microJQ grammar - 2) start pushing builder operations corresponding to loops, trees - 3) tree construction comes from temporarily associating text ranges and token ID's with this node tree - can tree construction go direct to json - can all steps be a set of JSON Build calls that are kept in a list, - and then backtracked to on earlier places in list, and then finally played forward to make the JSON Tree - 4) JSON program can be written to take this output and test it in Node.js - 5) bring it into QuickJS as evaluator - 6) or just interpret it straight up - 7) Trace improvements: - backtrack uniqueID and establishment trace - backtrack printf enhance to be specific - - - build steps - - SEQ - kv pairs - (uniquified names of subs -> json) - - ALT - { - indicator: - value: - } - - STAR - [ ] - - TOKEN - jsonValuizer - makeString ( various escaping rules, like handling \r \u, etc ) - makeInteger - makeBoolean - makeNull - but can make anything!! - - -*/ typedef struct GContinuation_tag { struct GContinuation_tag *previous; @@ -331,7 +286,17 @@ static GBuildStep *makeBuildStep(GParseContext *ctx, int type, GRuleRef *ref, GR } static void buildTokenValue(GParseContext *ctx, GRuleRef *ref, int lastTokenStart, int lastTokenEnd){ - GBuildStep *step = makeBuildStep(ctx,BUILD_STRING,ref,NULL); + int jsonType = JSON_TYPE_STRING; + if (ctx->tokenizer->getTokenJsonType){ + jsonType = ctx->tokenizer->getTokenJsonType(ref->tokenID); + } + int buildType = BUILD_STRING; + switch (jsonType){ + case JSON_TYPE_NUMBER: + buildType = BUILD_NUMBER; + break; + } + GBuildStep *step = makeBuildStep(ctx,buildType,ref,NULL); step->valueStart = lastTokenStart; step->valueEnd = lastTokenEnd; } @@ -363,10 +328,12 @@ static void indent(int depth){ static int parseDispatch(GParseContext *ctx, GRule *rule, int startPos, int depth, GContinuation *resumePoint); static void backtrack(GParseContext *ctx, GContinuation *continuation){ - printf("-------> Backtracking... build step back from %d to %d\n", - ctx->buildStepCount, - continuation->buildStepHWM); - fflush(stdout); + if (ctx->traceLevel >= 1){ + printf("-------> Backtracking... build step back from %d to %d\n", + ctx->buildStepCount, + continuation->buildStepHWM); + fflush(stdout); + } longjmp(continuation->resumeData,1); } @@ -374,9 +341,11 @@ static int runTokenMatch(GParseContext *ctx, GRuleRef *ref, int pos, int depth){ AbstractTokenizer *tokenizer = ctx->tokenizer; int nextPos = 0; int tokenID = 0; - indent(depth); - printf("runTokenMatch trying id=0x%x, (%s) pos=%d\n",ref->tokenID,tokenizer->getTokenIDName(ref->tokenID),pos); - fflush(stdout); + if (ctx->traceLevel >= 1){ + indent(depth); + printf("runTokenMatch trying id=0x%x, (%s) pos=%d\n",ref->tokenID,tokenizer->getTokenIDName(ref->tokenID),pos); + fflush(stdout); + } bool matched = matchToken(tokenizer,ctx->s,ctx->len,pos,&nextPos,ref->tokenID,&tokenID); if (matched){ buildTokenValue(ctx,ref,tokenizer->lastTokenStart,tokenizer->lastTokenEnd); @@ -398,9 +367,11 @@ static char *ruleName(GParseContext *ctx, GRule *rule){ } static int runSeq(GParseContext *ctx, GRule *rule, int startPos, int depth, GContinuation *resumePoint){ - indent(depth); - printf("runSeq (%s) at pos=%d, resume=0x%p\n",ruleName(ctx,rule),startPos,resumePoint); - fflush(stdout); + if (ctx->traceLevel >= 1){ + indent(depth); + printf("runSeq (%s) at pos=%d, resume=0x%p\n",ruleName(ctx,rule),startPos,resumePoint); + fflush(stdout); + } int pos = startPos; for (int index=0; indexsubCount; index++){ GRuleRef *ref = &rule->refs[index]; @@ -409,8 +380,10 @@ static int runSeq(GParseContext *ctx, GRule *rule, int startPos, int depth, GCon (ref->isTokenRef ? runTokenMatch(ctx,ref,pos,depth) : parseDispatch(ctx,ref->rule,pos,depth,resumePoint)); - indent(depth+1); - printf("seq submatch index=%d result=%d\n",index,matchResult); + if (ctx->traceLevel >= 1){ + indent(depth+1); + printf("seq submatch index=%d result=%d\n",index,matchResult); + } if (GPARSE_SUCCESS(matchResult)){ pos = matchResult; } else if (resumePoint){ @@ -423,9 +396,11 @@ static int runSeq(GParseContext *ctx, GRule *rule, int startPos, int depth, GCon } static int runAlt(GParseContext *ctx, GRule *rule, int startPos, int depth, GContinuation *resumePoint){ - indent(depth); - printf("runAlt (%s) at pos=%d, resume=0x%p\n",ruleName(ctx,rule),startPos,resumePoint); - fflush(stdout); + if (ctx->traceLevel >= 1){ + indent(depth); + printf("runAlt (%s) at pos=%d, resume=0x%p\n",ruleName(ctx,rule),startPos,resumePoint); + fflush(stdout); + } GBuildStep *altStep = buildAltValue(ctx,rule); GContinuation continuation; continuation.previous = resumePoint; @@ -444,20 +419,28 @@ static int runAlt(GParseContext *ctx, GRule *rule, int startPos, int depth, GCon if (ref->isTokenRef){ int matchResult = runTokenMatch(ctx,ref,pos,depth); if (GPARSE_SUCCESS(matchResult)){ - indent(depth+1); - printf("alt token success matchResult=%d\n",matchResult); + if (ctx->traceLevel >= 1){ + indent(depth+1); + printf("alt token success matchResult=%d\n",matchResult); + } return matchResult; } else if (index+1 < rule->subCount){ // just roll forward - indent(depth+1); - printf("alt token roll forward\n"); + if (ctx->traceLevel >= 1){ + indent(depth+1); + printf("alt token roll forward\n"); + } } else if (resumePoint){ - indent(depth+1); - printf("alt token backtrack\n"); + if (ctx->traceLevel >= 1){ + indent(depth+1); + printf("alt token backtrack\n"); + } backtrack(ctx,resumePoint); } else { - indent(depth+1); - printf("alt token fail\n"); + if (ctx->traceLevel >= 1){ + indent(depth+1); + printf("alt token fail\n"); + } return GPARSE_FAIL; } } else { @@ -465,21 +448,27 @@ static int runAlt(GParseContext *ctx, GRule *rule, int startPos, int depth, GCon &continuation : resumePoint); int parseResult = parseDispatch(ctx,ref->rule,pos,depth,failContinuation); - indent(depth+1); - printf("alt subrule res = %d\n",parseResult); + if (ctx->traceLevel >= 1){ + indent(depth+1); + printf("alt subrule res = %d\n",parseResult); + } return parseResult; // we never loop except when hitting a token } } - indent(depth+1); - printf("out of ALT's\n"); + if (ctx->traceLevel >= 1){ + indent(depth+1); + printf("out of ALT's\n"); + } return GPARSE_FAIL; } // can easily by plus or repeat, or opt static int runStar(GParseContext *ctx, GRule *rule, int startPos, int depth, GContinuation *resumePoint){ - indent(depth); - printf("runStar (%s) at pos=%d, resume=0x%p\n",ruleName(ctx,rule),startPos,resumePoint); - fflush(stdout); + if (ctx->traceLevel >= 1){ + indent(depth); + printf("runStar (%s) at pos=%d, resume=0x%p\n",ruleName(ctx,rule),startPos,resumePoint); + fflush(stdout); + } GContinuation continuation; continuation.previous = resumePoint; continuation.indexInRule = 0; @@ -569,7 +558,9 @@ Json *gBuildJSON(GParseContext *ctx, ShortLivedHeap *slh){ Json *currentParent = NULL; char *currentKey = NULL; for (int i=0; ibuildStepCount; i++){ - printf("build step %d\n",i); + if (ctx->traceLevel >= 1){ + printf("build step %d, currKey=%s\n",i,currentKey); + } GBuildStep *step = &ctx->buildSteps[i]; int errorCode = 0; switch (step->type){ @@ -606,7 +597,7 @@ Json *gBuildJSON(GParseContext *ctx, ShortLivedHeap *slh){ currentKey = ""; break; default: - ctx->tokenizer->getTokenIDName(step->tokenID); + currentKey = ctx->tokenizer->getTokenIDName(step->tokenID); break; } } @@ -624,13 +615,27 @@ Json *gBuildJSON(GParseContext *ctx, ShortLivedHeap *slh){ break; case BUILD_STRING: if (currentKey == NULL){ - printf("no parent for string!!!\n"); + printf("*** WARNING *** no parent for string!!!\n"); } else { int len = step->valueEnd - step->valueStart; jsonBuildString(b,currentParent,currentKey,ctx->s+step->valueStart,len,&errorCode); currentKey = NULL; /* because was consumed */ } break; + case BUILD_NUMBER: + { + int len = step->valueEnd - step->valueStart; + char *numberText = SLHAlloc(ctx->slh,len+1); + memcpy(numberText,ctx->s+step->valueStart,len); + numberText[len] = 0; + int64_t val = strtoll(numberText,NULL,10); + if (val < 0x7FFFFFFF){ + jsonBuildInt(b,currentParent,currentKey,(int)val,&errorCode); + } else { + jsonBuildInt64(b,currentParent,currentKey,val,&errorCode); + } + } + break; case BUILD_INT64: case BUILD_INT: case BUILD_DOUBLE: @@ -655,14 +660,14 @@ Json *gBuildJSON(GParseContext *ctx, ShortLivedHeap *slh){ printf("last build step failed with code = %d\n",errorCode); } } - printf("final sp = %d\n",sp); Json *result = b->root; freeJsonBuilder(b,false); return result; } GParseContext *gParse(GRuleSpec *ruleSet, int topRuleID, AbstractTokenizer *tokenizer, char *s, int len, - char *(*ruleNamer)(int id)){ + char *(*ruleNamer)(int id), + int traceLevel){ ShortLivedHeap *slh = makeShortLivedHeap(0x10000, 100); GParseContext *ctx = (GParseContext *)SLHAlloc(slh,sizeof(GParseContext)); memset(ctx,0,sizeof(GParseContext)); @@ -673,15 +678,18 @@ GParseContext *gParse(GRuleSpec *ruleSet, int topRuleID, AbstractTokenizer *toke ctx->slh = slh; ctx->buildStepsSize = MAX_BUILD_STEPS; ctx->buildSteps = (GBuildStep*)SLHAlloc(slh,ctx->buildStepsSize*sizeof(GBuildStep)); + ctx->traceLevel = traceLevel; int ruleSetSize = 0; GRule *rules = makeGRules(ruleSet,tokenizer,slh,&ruleSetSize); ctx->rules = rules; GRule *topRule = getRule(rules,ruleSetSize,topRuleID); if (topRule->type == G_SEQ){ int parseResult = parseDispatch(ctx,topRule,0,0,NULL); - printf("parseResult=%d\n",parseResult); ctx->status = parseResult; - showBuildSteps(ctx); + if (ctx->traceLevel >= 1){ + printf("parseResult=%d\n",parseResult); + showBuildSteps(ctx); + } return ctx; } else { printf("top rule must be G_SEQ\n"); @@ -689,3 +697,173 @@ GParseContext *gParse(GRuleSpec *ruleSet, int topRuleID, AbstractTokenizer *toke return ctx; } } + +/* JLexer (Flexible Tokenizer for java-ish languages) */ + +#define STATE_INITIAL 0 +#define STATE_INTEGER 1 +#define STATE_IDENTIFIER 2 +#define STATE_SQUOTE 3 +#define STATE_DQUOTE 4 + +char *getJTokenName(int id){ + switch (id){ + case EOF_TOKEN: return "EOF"; + case JTOKEN_IDENTIFIER: return "ID"; + case JTOKEN_INTEGER: return "INTEGER"; + case JTOKEN_DQUOTE_STRING: return "DQUOTE_STRING"; + case JTOKEN_SQUOTE_STRING: return "SQUOTE_STRING"; + case JTOKEN_LPAREN: return "LPAREN"; + case JTOKEN_RPAREN: return "RPAREN"; + case JTOKEN_LBRACK: return "LBRACK"; + case JTOKEN_RBRACK: return "RBRACK"; + case JTOKEN_LBRACE: return "LBRACE"; + case JTOKEN_RBRACE: return "RBRACE"; + case JTOKEN_DOT: return "DOT"; + case JTOKEN_COMMA: return "COMMA"; + case JTOKEN_VBAR: return "VBAR"; + case JTOKEN_COLON: return "COLON"; + case JTOKEN_QMARK: return "QMARK"; + case JTOKEN_PLUS: return "PLUS"; + case JTOKEN_DASH: return "DASH"; + case JTOKEN_STAR: return "STAR"; + case JTOKEN_SLASH: return "SLASH"; + case JTOKEN_PERCENT: return "PERCENT"; + default:return "BAD_TOKEN"; + } +} + +static bool isLetter(JQTokenizer *jqt, int c){ + return testCharProp(jqt->ccsid,c,CPROP_LETTER); +} + +static bool isDigit(JQTokenizer *jqt, int c){ + return testCharProp(jqt->ccsid,c,CPROP_DIGIT); +} + +static bool isWhite(JQTokenizer *jqt, int c){ + return testCharProp(jqt->ccsid,c,CPROP_WHITE); +} + +int jqtToken(JQTokenizer *jqt){ + /* belt *AND* suspenders */ + AbstractTokenizer *tokenizer = (AbstractTokenizer*)jqt; + if (jqt->pos >= jqt->length){ + tokenizer->lastTokenStart = jqt->pos; /* 0-length token */ + tokenizer->lastTokenEnd = jqt->pos; + return EOF_TOKEN; + } + int tokenID = NO_VALID_TOKEN; + while(tokenID == NO_VALID_TOKEN){ + int c = -1; // EOF by defaultx + int charPos = jqt->pos; + if (jqt->pos < jqt->length){ + c = (int)jqt->data[jqt->pos++]; + } + /* printf("at pos=%d, c=0x%x state=%d\n",jqt->pos,c,jqt->tokenState); */ + switch (jqt->tokenState){ + case STATE_INITIAL: + if (c == -1){ + tokenID = EOF_TOKEN; + tokenizer->lastTokenStart = jqt->length; + } else if (isLetter(jqt,c) || + (c == UNI_DOLLAR) || + (c == UNI_UNDER)){ + jqt->tokenState = STATE_IDENTIFIER; + tokenizer->lastTokenStart = charPos; + } else if (isDigit(jqt,c)){ + jqt->tokenState = STATE_INTEGER; + tokenizer->lastTokenStart = charPos; + } else if (isWhite(jqt,c)){ + // just roll forward + } else if (c == UNI_DQUOTE){ + jqt->tokenState = STATE_DQUOTE; + tokenizer->lastTokenStart = charPos; + } else if (c == UNI_SQUOTE){ + jqt->tokenState = STATE_SQUOTE; + tokenizer->lastTokenStart = charPos; + } else { + /* Single-char-punc tokens */ + switch (c){ + case UNI_LPAREN: tokenID = JTOKEN_LPAREN; break; + case UNI_RPAREN: tokenID = JTOKEN_RPAREN; break; + case UNI_LBRACK: tokenID = JTOKEN_LBRACK; break; + case UNI_RBRACK: tokenID = JTOKEN_RBRACK; break; + case UNI_LBRACE: tokenID = JTOKEN_LBRACE; break; + case UNI_RBRACE: tokenID = JTOKEN_RBRACE; break; + case UNI_DOT: tokenID = JTOKEN_DOT; break; + case UNI_COMMA: tokenID = JTOKEN_COMMA; break; + case UNI_COLON: tokenID = JTOKEN_COLON; break; + case UNI_VBAR: tokenID = JTOKEN_VBAR; break; + case UNI_QMARK: tokenID = JTOKEN_QMARK; break; + case UNI_PLUS: tokenID = JTOKEN_PLUS; break; + case UNI_DASH: tokenID = JTOKEN_DASH; break; + case UNI_STAR: tokenID = JTOKEN_STAR; break; + case UNI_SLASH: tokenID = JTOKEN_SLASH; break; + case UNI_PERCENT: tokenID = JTOKEN_PERCENT; break; + default: + /* unhandled char, kill the parse */ + return NO_VALID_TOKEN; + } + tokenizer->lastTokenStart = charPos; + } + break; + case STATE_INTEGER: + if (!isDigit(jqt,c)){ + if (c != -1) jqt->pos--; /* nothing to push back */ + tokenID = JTOKEN_INTEGER; + } + break; + case STATE_IDENTIFIER: + if (isLetter(jqt,c) || + (c == UNI_DOLLAR) || + (c == UNI_UNDER)){ + // accumulate + } else { + if (c != -1) jqt->pos--; /* nothing to push back */ + tokenID = JTOKEN_IDENTIFIER; + } + break; + case STATE_DQUOTE: + if (c == -1){ /* EOF in string */ + return NO_VALID_TOKEN; + } else if (c == UNI_DQUOTE){ + tokenID = JTOKEN_DQUOTE_STRING; + } + break; + case STATE_SQUOTE: + if (c == -1){ /* EOF in string */ + return NO_VALID_TOKEN; + } else if (c == UNI_SQUOTE){ + tokenID = JTOKEN_SQUOTE_STRING; + } + default: + printf("*** PANIC *** internal error unknown tokenizer state = %d\n",jqt->tokenState); + return NO_VALID_TOKEN; + // unknow state, kill the parse + } + if (tokenID != NO_VALID_TOKEN){ + tokenizer->lastTokenEnd = jqt->pos; + jqt->tokenState = STATE_INITIAL; + return tokenID; + } + } + return 0; +} + +int getNextJToken(AbstractTokenizer *tokenizer, char *s, int len, int pos, int *nextPos){ + JQTokenizer *jqt = (JQTokenizer*)tokenizer; + jqt->pos = pos; + int id = jqtToken(jqt); + *nextPos = jqt->pos; + return id; +} + +int getJTokenJsonType(int tokenID){ + switch (tokenID){ + case JTOKEN_INTEGER: + return JSON_TYPE_NUMBER; /* see json.h */ + default: + return JSON_TYPE_STRING; + } +} diff --git a/c/yaml2json.c b/c/yaml2json.c index a2db1a8d3..4f48ced4e 100644 --- a/c/yaml2json.c +++ b/c/yaml2json.c @@ -422,7 +422,7 @@ static int buildTemplateJSON(JsonBuilder *b, Json *parent, char *parentKey, char *tail = nativeValue; JsonBuffer *buffer = makeJsonBuffer(); int sourceCCSID = CCSID_UTF_8; - printf ("buildTemplateJSON nativeValue '%.*s' sourceCodeCharset %d\n", valueLength, nativeValue, sourceCCSID); + /* printf ("buildTemplateJSON nativeValue '%.*s' sourceCodeCharset %d\n", valueLength, nativeValue, sourceCCSID);*/ jsonPrinter *p = makeBufferJsonPrinter(sourceCCSID,buffer); int status = 0; bool first = true; @@ -432,7 +432,7 @@ static int buildTemplateJSON(JsonBuilder *b, Json *parent, char *parentKey, if (nextExpr){ if (nextExpr > tail){ char *frag = extractString(b,tail,nextExpr); - printf("frag = '%s'\n",frag); + /* printf("frag = '%s'\n",frag); */ addPlusIfNecessary(p,sourceCCSID,&first); jsonWriteParseably(p,frag,strlen(frag),true,false,sourceCCSID); } diff --git a/h/json.h b/h/json.h index d0c8980ea..15d1d5009 100644 --- a/h/json.h +++ b/h/json.h @@ -90,6 +90,11 @@ jsonPrinter *makeCustomUtf8JsonPrinter( void (*writeMethod)(jsonPrinter *, char *, int), void *object, int inputCCSID); +/** + * \brief Reset a printer to its starting state. + */ +void jsonPrinterReset(jsonPrinter *printer); + jsonPrinter *makeBufferJsonPrinter(int inputCCSID, JsonBuffer *buf); /** diff --git a/h/microjq.h b/h/microjq.h new file mode 100644 index 000000000..a99934576 --- /dev/null +++ b/h/microjq.h @@ -0,0 +1,13 @@ +#ifndef __ZOWE_MICROJQ__ +#define __ZOWE_MICROJQ__ 1 + +#define JQ_FLAG_RAW_STRINGS 0x0001 +#define JQ_FLAG_PRINT_PRETTY 0x0002 + +#include "json.h" +#include "parsetools.h" + +Json *parseJQ(JQTokenizer *jqt, ShortLivedHeap *slh, int traceLevel); +int evalJQ(Json *value, Json *jqTree, FILE *out, int flags, int traceLevel); + +#endif diff --git a/h/parsetools.h b/h/parsetools.h index 632a89abf..80fbb0933 100644 --- a/h/parsetools.h +++ b/h/parsetools.h @@ -337,6 +337,7 @@ typedef struct AbstractTokenizer_tag { int lastTokenEnd; int (*nextToken)(struct AbstractTokenizer_tag *tokenizer, char *s, int len, int pos, int *nextPos); char *(*getTokenIDName)(int tokenID); + int (*getTokenJsonType)(int tokenID); /* if not specified, you get a string! */ } AbstractTokenizer; bool testCharProp(int ccsid, int c, int prop); @@ -384,14 +385,16 @@ typedef struct GRule_tag { #define BUILD_OBJECT 1 #define BUILD_ARRAY 2 #define BUILD_STRING 3 -#define BUILD_INT 4 -#define BUILD_INT64 5 -#define BUILD_DOUBLE 6 -#define BUILD_BOOL 7 -#define BUILD_NULL 8 -#define BUILD_KEY 9 -#define BUILD_POP 10 -#define BUILD_ALT 11 +#define BUILD_NUMBER 4 +#define BUILD_INT 5 +#define BUILD_INT64 6 +#define BUILD_DOUBLE 7 +#define BUILD_BOOL 8 +#define BUILD_NULL 9 +#define BUILD_KEY 10 +#define BUILD_POP 11 +#define BUILD_ALT 12 + #define MAX_BUILD_STEPS 10000 /* who in hell knows? JK, actually should be a parameter to GParse() */ #define STEP_UNDEFINED_POSITION (-1) @@ -417,6 +420,7 @@ typedef struct GParseContext_tag { int buildStepCount; GBuildStep *buildSteps; int status; + int traceLevel; int errorPosition; char *errorMessage; } GParseContext; @@ -438,8 +442,52 @@ typedef struct GParseResult_tag { } GParseResult; GParseContext *gParse(GRuleSpec *ruleSet, int topRuleID, AbstractTokenizer *tokenizer, char *s, int len, - char *(*ruleName)(int id)); + char *(*ruleName)(int id), + int traceLevel); Json *gBuildJSON(GParseContext *ctx, ShortLivedHeap *slh); + +/* These definitions are for JLexer, a flexible tokenizer for java/javascript-ish syntaxes */ + +typedef struct JQTokenizer_tag { + AbstractTokenizer tokenizer; /* cheesy "subclassing" idiom */ + int ccsid; + int tokenState; /* = STATE_INITIAL; */ + char *data; + int length; + int pos; +} JQTokenizer; + +#define JTOKEN_IDENTIFIER FIRST_REAL_TOKEN_ID +#define JTOKEN_INTEGER (FIRST_REAL_TOKEN_ID+1) +#define JTOKEN_DQUOTE_STRING FIRST_REAL_TOKEN_ID+2 +#define JTOKEN_SQUOTE_STRING FIRST_REAL_TOKEN_ID+3 +#define JTOKEN_LPAREN FIRST_REAL_TOKEN_ID+4 +#define JTOKEN_RPAREN FIRST_REAL_TOKEN_ID+5 +#define JTOKEN_LBRACK FIRST_REAL_TOKEN_ID+6 +#define JTOKEN_RBRACK FIRST_REAL_TOKEN_ID+7 +#define JTOKEN_LBRACE FIRST_REAL_TOKEN_ID+8 +#define JTOKEN_RBRACE FIRST_REAL_TOKEN_ID+9 +#define JTOKEN_DOT FIRST_REAL_TOKEN_ID+10 +#define JTOKEN_COMMA FIRST_REAL_TOKEN_ID+11 +#define JTOKEN_VBAR FIRST_REAL_TOKEN_ID+12 +#define JTOKEN_COLON FIRST_REAL_TOKEN_ID+13 +#define JTOKEN_QMARK FIRST_REAL_TOKEN_ID+14 +#define JTOKEN_PLUS FIRST_REAL_TOKEN_ID+15 +#define JTOKEN_DASH FIRST_REAL_TOKEN_ID+16 +#define JTOKEN_STAR FIRST_REAL_TOKEN_ID+17 +#define JTOKEN_SLASH FIRST_REAL_TOKEN_ID+18 +#define JTOKEN_PERCENT FIRST_REAL_TOKEN_ID+19 + +#define LAST_JTOKEN_ID JTOKEN_PERCENT /* KEEP ME UPDATED!!! - whenever a new token ID is added*/ + +char *getJTokenName(int id); +/* testing interface */ +int jqtToken(JQTokenizer *jqt); +/* proper interface for use in GParser */ +int getNextJToken(AbstractTokenizer *tokenizer, char *s, int len, int pos, int *nextPos); +int getJTokenJsonType(int tokenID); + + #endif diff --git a/tests/jqtest.c b/tests/jqtest.c new file mode 100644 index 000000000..d311c0ca9 --- /dev/null +++ b/tests/jqtest.c @@ -0,0 +1,130 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + +#ifdef METTLE + +#include +#include +#include +#include +#include +#include +#include +#include +#include "metalio.h" +#include "qsam.h" + +#else + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE 600 +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#endif + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "openprims.h" +#include "bpxnet.h" +#include "unixfile.h" +#include "json.h" +#include "parsetools.h" +#include "microjq.h" + +/* + _Compiling on Windows_ + + clang -I../platform/windows -I ..\h -Wdeprecated-declarations -D_CRT_SECURE_NO_WARNINGS -o jqtest.exe jqtest.c ../c/microjq.c ../c/parsetools.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c + + */ + +static void pprintJSON(Json *tree){ +#ifdef __ZOWE_OS_WINDOWS + int stdoutFD = _fileno(stdout); +#else + int stdoutFD = STDOUT_FILENO; +#endif + if (tree){ + jsonPrinter *p = makeJsonPrinter(stdoutFD); + jsonEnablePrettyPrint(p); + printf("parse result as json\n"); + jsonPrint(p,tree); + } +} + +int main(int argc, char **argv){ + char *command = argv[1]; + char *filter = argv[2]; + char *filename = argv[3]; + JQTokenizer jqt; + memset(&jqt,0,sizeof(JQTokenizer)); + jqt.data = argv[2]; + jqt.length = strlen(jqt.data); + jqt.ccsid = 1208; + ShortLivedHeap *slh = makeShortLivedHeap(0x10000, 100); + if (!strcmp(command,"parse")){ + printf("parse...\n"); + Json *tree = parseJQ(&jqt,slh,1); + if (tree){ + pprintJSON(tree); + } else { + printf("Parse failed\n"); + } + } else if (!strcmp(command,"eval") || + !strcmp(command,"evalTrace") || + !strcmp(command,"evalCompact")){ + Json *jqTree = parseJQ(&jqt,slh,0); + if (jqTree){ + int errorBufferSize = 1024; + char *errorBuffer = safeMalloc(errorBufferSize,"ErrorBuffer"); + memset(errorBuffer,0,errorBufferSize); + Json *json = jsonParseFile2(slh,filename,errorBuffer,errorBufferSize); + if (json == NULL){ + printf("json parsed json=0x%p, with error '%s'\n",json,errorBuffer); + return 8; + } + int traceLevel = 0; + int flags = JQ_FLAG_PRINT_PRETTY; + if (!strcmp(command,"evalTrace")){ + traceLevel = 1; + } + if (!strcmp(command,"evalRaw")){ + flags |= JQ_FLAG_RAW_STRINGS; + } + if (!strcmp(command,"evalCompact")){ + flags ^= JQ_FLAG_PRINT_PRETTY; + } + int evalStatus = evalJQ(json,jqTree,stdout,flags,traceLevel); + if (evalStatus != 0){ + fprintf(stderr,"micro jq eval problem %d\n",evalStatus); + } + } else { + printf("Parse failed\n"); + } + } else { + printf("bad command: %s\n",command); + } + return 0; +} diff --git a/tests/parsetest.c b/tests/parsetest.c index e146f2d8c..517d9ad21 100644 --- a/tests/parsetest.c +++ b/tests/parsetest.c @@ -96,7 +96,7 @@ static char *getTestSetRuleName(int id){ -int main(int argc, char **argv){ +static int old_main(int argc, char **argv){ AbstractTokenizer tokenizer; tokenizer.lowTokenID = 1; tokenizer.highTokenID = 127; @@ -106,11 +106,110 @@ int main(int argc, char **argv){ return 0; } +static void tokenTest(JQTokenizer *jqt){ + AbstractTokenizer *tokenizer = (AbstractTokenizer*)jqt; + printf("length = %d\n",jqt->length); + while (true){ + int tokenID = jqtToken(jqt); + printf("token id=%d, %s pos is now=%d\n",tokenID,getJTokenName(tokenID),jqt->pos); + printf(" from %d to %d\n",tokenizer->lastTokenStart,tokenizer->lastTokenEnd); + if (tokenID == NO_VALID_TOKEN){ + printf("bad token seen near %d\n",jqt->pos); + break; + } else if (tokenID == EOF_TOKEN){ + printf("EOF Seen\n"); + break; + } + } +} + +#define TOP (FIRST_GRULE_ID+0) +#define EXPR (FIRST_GRULE_ID+1) +#define EXPR_TAIL (FIRST_GRULE_ID+2) +#define EXPR_ADD_SUB (FIRST_GRULE_ID+3) +#define PLUS_MINUS (FIRST_GRULE_ID+4) +#define TERM (FIRST_GRULE_ID+5) +#define TERM_TAIL (FIRST_GRULE_ID+6) +#define TERM_MULT_DIV (FIRST_GRULE_ID+7) +#define STAR_SLASH (FIRST_GRULE_ID+8) +#define FACTOR (FIRST_GRULE_ID+9) + +static char *getExprRuleName(int id){ + switch (id){ + case TOP: return "TOP"; + case EXPR: return "EXPR"; + case EXPR_TAIL: return "EXPR_TAIL"; + case EXPR_ADD_SUB: return "EXPR_ADD_SUB"; + case PLUS_MINUS: return "PLUS_MINUS"; + case TERM: return "TERM"; + case TERM_TAIL: return "TERM_TAIL"; + case TERM_MULT_DIV: return "TERM_MULT_DIV"; + case STAR_SLASH: return "STAR_SLASH"; + case FACTOR: return "FACTOR"; + default: + return "UNKNOWN_RULE"; + } +} + +static GRuleSpec expressionGrammar[] = { + { G_SEQ, TOP, .sequence = { EXPR, EOF_TOKEN, G_END} }, + { G_SEQ, EXPR, .sequence = { TERM, EXPR_TAIL, G_END }}, + { G_STAR, EXPR_TAIL, .star = EXPR_ADD_SUB }, + { G_SEQ, EXPR_ADD_SUB, .sequence = { PLUS_MINUS, TERM, G_END }}, + { G_ALT, PLUS_MINUS, .alternates = { JTOKEN_PLUS, JTOKEN_DASH, G_END }}, + { G_SEQ, TERM, .sequence = { FACTOR, TERM_TAIL, G_END }}, + { G_STAR, TERM_TAIL, .star = TERM_MULT_DIV }, + { G_SEQ, TERM_MULT_DIV, .sequence = { STAR_SLASH, FACTOR, G_END}}, + { G_ALT, STAR_SLASH, .alternates = { JTOKEN_STAR, JTOKEN_SLASH, G_END }}, + { G_ALT, FACTOR, .alternates = { JTOKEN_IDENTIFIER, JTOKEN_INTEGER, G_END }}, /* add expression recursion later */ + { G_END } +}; + + +static void parseTest1(JQTokenizer *jqt){ + AbstractTokenizer *tokenizer = (AbstractTokenizer*)jqt; + tokenizer->lowTokenID = FIRST_REAL_TOKEN_ID; + tokenizer->highTokenID = LAST_JTOKEN_ID; + tokenizer->nextToken = getNextJToken; + tokenizer->getTokenIDName = getJTokenName; + GParseContext *ctx = gParse(expressionGrammar,TOP,tokenizer,jqt->data,jqt->length,getExprRuleName); + printf("parse ctx = 0x%p\n",ctx); + if (ctx->status > 0){ + #ifdef __ZOWE_OS_WINDOWS + int stdoutFD = _fileno(stdout); +#else + int stdoutFD = STDOUT_FILENO; +#endif + ShortLivedHeap *slh = makeShortLivedHeap(0x10000, 100); + Json *tree = gBuildJSON(ctx,slh); + if (tree){ + jsonPrinter *p = makeJsonPrinter(stdoutFD); + jsonEnablePrettyPrint(p); + printf("parse result as json\n"); + jsonPrint(p,tree); + } + } +} + +int main(int argc, char **argv){ + char *command = argv[1]; + char *filter = argv[2]; + char *filename = argv[3]; + JQTokenizer jqt; + memset(&jqt,0,sizeof(JQTokenizer)); + jqt.data = argv[2]; + jqt.length = strlen(jqt.data); + jqt.ccsid = 1208; + if (!strcmp(command,"tokenize")){ + printf("tokenize...\n"); + tokenTest(&jqt); + } else if (!strcmp(command,"parse")){ + printf("parse...\n"); + parseTest1(&jqt); + } else { + printf("bad command: %s\n",command); + } + return 0; +} -/* - seq( A E* D ) - - A, eps, D=E, fail - E* bears fruit until the first time it doesn't match one more -*/ From 64e9bc40cb8704cd62205b4f1b9c25365f93d871 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 14 Mar 2022 19:10:48 -0400 Subject: [PATCH 28/42] minor bug in jq support Signed-off-by: Joe --- c/configmgr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c/configmgr.c b/c/configmgr.c index 986e9aa8a..c11829dbc 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -957,7 +957,7 @@ int main(int argc, char **argv){ configPath = optionValue; } else if ((optionValue = getStringOption(argc,argv,&argx,"-c")) != NULL){ jqCompact = true; - } else if ((optionValue = getStringOption(argc,argv,&argx,"-c")) != NULL){ + } else if ((optionValue = getStringOption(argc,argv,&argx,"-r")) != NULL){ jqRaw = true; } else { char *nextArg = argv[argx]; From 3e82a81d46767ea07b2571037662407dd5c7ea23 Mon Sep 17 00:00:00 2001 From: Joe Devlin Date: Tue, 15 Mar 2022 13:17:57 -0500 Subject: [PATCH 29/42] Making isLowerCasePasswordAllowed exported Signed-off-by: Joe Devlin --- c/httpserver.c | 4 ++-- h/httpserver.h | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/c/httpserver.c b/c/httpserver.c index 94958221e..c1335bb49 100644 --- a/c/httpserver.c +++ b/c/httpserver.c @@ -2519,13 +2519,13 @@ static char *getCookieValue(HttpRequest *request, char *cookieName){ } #ifdef __ZOWE_OS_ZOS -static int isLowerCasePasswordAllowed(){ +int isLowerCasePasswordAllowed(){ RCVT* rcvt = getCVT()->cvtrac; return (RCVTFLG3_BIT_RCVTPLC & (rcvt->rcvtflg3)); /* if lower-case pw allowed */ } #else -static int isLowerCasePasswordAllowed(){ +int isLowerCasePasswordAllowed(){ return TRUE; } #endif diff --git a/h/httpserver.h b/h/httpserver.h index 116710686..2cddc8631 100644 --- a/h/httpserver.h +++ b/h/httpserver.h @@ -590,6 +590,8 @@ int setHttpCloseConversationTrace(int toWhat); int setHttpAuthTrace(int toWhat); #endif +int isLowerCasePasswordAllowed(); + int httpServerInitJwtContext(HttpServer *self, bool legacyFallback, const char *pkcs11TokenName, From 6eee8b65936460034d706aa028071cb05cea717e Mon Sep 17 00:00:00 2001 From: Joe Devlin Date: Tue, 15 Mar 2022 13:28:47 -0500 Subject: [PATCH 30/42] minor METAL compile issues Signed-off-by: Joe Devlin --- c/utils.c | 1 - c/zos.c | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/c/utils.c b/c/utils.c index cce3802cd..f727af5a1 100644 --- a/c/utils.c +++ b/c/utils.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include "qsam.h" #include "metalio.h" diff --git a/c/zos.c b/c/zos.c index e3b9c03a4..de65088d1 100644 --- a/c/zos.c +++ b/c/zos.c @@ -1051,7 +1051,9 @@ static int safVerifyInternal(int options, fprintf(safTraceFile,"about to go call saf wrapper at 0x%p' verify block at 0x%p' acee 0x%p'\n",safWrapper,verifyRequest, (aceeHandle != NULL ? *aceeHandle : NULL)); dumpbuffer((char*)safWrapper,sizeof(safp)); +#ifndef METTLE fprintf(safTraceFile,"verify:\n"); +#endif dumpbuffer((char*)verifyRequest,sizeof(safVerifyRequest)); } safStatus = SAF(safWrapper,useSupervisorMode); From e2eecbcb73949a56d491e371334788df17f848f5 Mon Sep 17 00:00:00 2001 From: Joe Devlin Date: Fri, 18 Mar 2022 00:31:55 -0500 Subject: [PATCH 31/42] many fixes seen from code review of PR Signed-off-by: Joe Devlin --- build/build_cmgr_xlclang.sh | 12 +++++++++--- c/collections.c | 4 ++-- c/configmgr.c | 12 ++++++------ c/embeddedjs.c | 17 ++++++++++------- c/json.c | 11 +++++------ c/jsonschema.c | 2 +- c/le.c | 4 ++-- c/recovery.c | 4 ++-- c/xlate.c | 2 +- c/yaml2json.c | 11 +++++++++-- c/zos.c | 2 +- h/le.h | 4 ++-- h/xlate.h | 2 +- h/zowetypes.h | 4 ++++ 14 files changed, 55 insertions(+), 36 deletions(-) diff --git a/build/build_cmgr_xlclang.sh b/build/build_cmgr_xlclang.sh index de88175f1..b8f7f4d1f 100755 --- a/build/build_cmgr_xlclang.sh +++ b/build/build_cmgr_xlclang.sh @@ -15,14 +15,18 @@ WORKING_DIR=$(dirname "$0") echo "********************************************************************************" echo "Building configmgr..." -rm -f "${COMMON}/bin/configmgr" +# These paths assume that the build is run from /zss/deps/zowe-common-c/builds + + mkdir -p "${WORKING_DIR}/tmp-configmgr" && cd "$_" + COMMON="../.." +QUICKJS="../../../../../quickjs" +LIBYAML="../../../../../libyaml" -QUICKJS="/u/zossteam/jdevlin/git2022/quickjs" +rm -f "${COMMON}/bin/configmgr" -LIBYAML="/u/zossteam/jdevlin/git2022/libyaml" MAJOR=0 MINOR=2 PATCH=5 @@ -107,6 +111,8 @@ xlclang \ ${COMMON}/c/jsonschema.c \ ${COMMON}/c/le.c \ ${COMMON}/c/logging.c \ + ${COMMON}/c/microjq.c \ + ${COMMON}/c/parsetools.c \ ${COMMON}/platform/posix/psxregex.c \ ${COMMON}/c/recovery.c \ ${COMMON}/c/scheduling.c \ diff --git a/c/collections.c b/c/collections.c index 65724bcb4..ce9a5fb97 100644 --- a/c/collections.c +++ b/c/collections.c @@ -1427,7 +1427,7 @@ static void *arrayListAlloc(ArrayList *list, uint32_t size, char *location){ } } -ArrayList *makeArrayList(){ +ArrayList *makeArrayList(void){ ArrayList *list = (ArrayList*)safeMalloc(sizeof(ArrayList),"ArrayList"); list->capacity = 8; list->size = 0; @@ -1491,7 +1491,7 @@ void *arrayListShallowCopy(ArrayList *source, ArrayList *target){ target->size = source->size; target->slh = source->slh; target->array = (void**)arrayListAlloc(target,target->capacity*sizeof(void*),"ArrayListArray"); - memcpy(target->array,source->array,target->capacity*sizeof(void**)); + memcpy(target->array,source->array,target->size*sizeof(void**)); return target; } diff --git a/c/configmgr.c b/c/configmgr.c index c11829dbc..692bd87d3 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -169,15 +169,15 @@ typedef struct ConfigManager_tag { } ConfigManager; #ifdef __ZOWE_OS_WINDOWS -static int stdoutFD(){ +static int stdoutFD(void){ return _fileno(stdout); } -static int stderrFD(){ +static int stderrFD(void){ return _fileno(stdout); } #else -static int stdoutFD(){ +static int stdoutFD(void){ #ifdef STDOUT_FILENO return STDOUT_FILENO; #else @@ -185,7 +185,7 @@ static int stdoutFD(){ #endif } -static int stderrFD(){ +static int stderrFD(void){ #ifdef STDERR_FILENO return STDERR_FILENO; #else @@ -712,7 +712,7 @@ static void extractText(ConfigManager *mgr, JsonPointer *jp, FILE *out){ } else if (jsonIsString(value)){ fprintf(out,"%s",jsonAsString(value)); } else if (jsonIsInt64(value)){ - fprintf(out,"%lld",jsonAsInt64(value)); + fprintf(out,"%lld",INT64_LL(jsonAsInt64(value))); } else if (jsonIsDouble(value)){ fprintf(out,"%f",jsonAsDouble(value)); } else if (jsonIsBoolean(value)){ @@ -853,7 +853,7 @@ static void outputEnvString(FILE * out, const char *str) { } static void outputEnvInt64(FILE * out, int64_t num) { - fprintf(out, "%lld\n", num); + fprintf(out, "%lld\n", INT64_LL(num)); } static void outputEnvDouble(FILE * out, double num) { diff --git a/c/embeddedjs.c b/c/embeddedjs.c index 67bcb461b..e02d64157 100644 --- a/c/embeddedjs.c +++ b/c/embeddedjs.c @@ -26,10 +26,6 @@ #else -#ifndef _XOPEN_SOURCE -#define _XOPEN_SOURCE 600 -#endif - #include #include #include @@ -38,9 +34,9 @@ #include #include #include -#include /* JOE 1/20/22 */ -#include /* JOE 1/20/22 */ -#include /* JOE 1/20/22 */ +#include +#include +#include #include #endif @@ -59,6 +55,13 @@ typedef int64_t ssize_t; #endif +#ifdef __ZOWE_OS_ZOS + +int __atoe_l(char *bufferptr, int leng); +int __etoa_l(char *bufferptr, int leng); + +#endif + static int convertToNative(char *buf, size_t size) { #ifdef __ZOWE_OS_ZOS return __atoe_l(buf, size); diff --git a/c/json.c b/c/json.c index 466d0a4f5..03516f4a8 100644 --- a/c/json.c +++ b/c/json.c @@ -38,9 +38,8 @@ #include #include #include -#include /* JOE 1/20/22 */ -#include /* JOE 1/20/22 */ -#include /* JOE 1/20/22 */ +#include +#include #include #endif @@ -185,7 +184,8 @@ void jsonWriteBufferInternal(jsonPrinter *p, char *text, int len) { } else if (p->fd == _fileno(stderr)){ newWriteReturn = fwrite(text, 1, len, stderr); } else { - printf("JOE WINDOWS FAKE SOCKET WRITE bad case\n"); + JSONERROR("JSON: can only write to stderr or stdout on Windows, not p->fd=%d\n",p->fd); + jsonSetIOErrorFlag(p); } #elif !defined(METTLE) int newWriteReturn = write(p->fd,text+bytesWritten,len-bytesWritten); @@ -482,7 +482,7 @@ void jsonWriteUInt(jsonPrinter *p, unsigned int value) { static void jsonWriteInt64(jsonPrinter *p, int64 value) { char buffer[64]; - snprintf(buffer, sizeof (buffer), "%lld", value); + snprintf(buffer, sizeof (buffer), "%lld", INT64_LL(value)); jsonWrite(p, buffer, false, SOURCE_CODE_CHARSET); } @@ -1102,7 +1102,6 @@ void freeJsonParser(JsonParser *parser) { static void jsonParseFail(JsonParser *parser, char *formatString, ...) { - fflush(stdout); if (parser->jsonError == NULL) { int size = 1024; char *buffer = jsonParserAlloc(parser, size); diff --git a/c/jsonschema.c b/c/jsonschema.c index dd00507d9..e344a202e 100644 --- a/c/jsonschema.c +++ b/c/jsonschema.c @@ -228,7 +228,7 @@ static ValidityException *noteValidityException(JsonValidator *validator, int in return exception; } -JsonValidator *makeJsonValidator(){ +JsonValidator *makeJsonValidator(void){ JsonValidator *validator = (JsonValidator*)safeMalloc(sizeof(JsonValidator),"JsonValidator"); memset(validator,0,sizeof(JsonValidator)); validator->accessPath = (AccessPath*)safeMalloc(sizeof(AccessPath),"JsonValidatorAccessPath"); diff --git a/c/le.c b/c/le.c index 7a639bfcb..2da29fe01 100644 --- a/c/le.c +++ b/c/le.c @@ -86,7 +86,7 @@ LibraryFunction libraryFunctionTable[LIBRARY_FUNCTION_COUNT] #endif }; -char *getCAA(){ +char *getCAA(void){ char *realCAA = NULL; #if !defined(METTLE) && defined(_LP64) @@ -162,7 +162,7 @@ static LibraryFunction *findLibraryFunction(int rtlVectorOffset){ #define ESTIMATED_RTL_VECTOR_SIZE 0xB00 -void showRTL(){ +void showRTL(void){ CAA *caa = (CAA*)getCAA(); void **rtlVector = caa->runtimeLibraryVectorTable; printf("RTL Vector at 0x%p\n",rtlVector); diff --git a/c/recovery.c b/c/recovery.c index 06710d299..18d5b2b46 100644 --- a/c/recovery.c +++ b/c/recovery.c @@ -2004,9 +2004,9 @@ void recoveryGetABENDCode(SDWA *sdwa, int *completionCode, int *reasonCode) { if (flag & 0x04) { char *sdwadata = (char*)sdwa; - SDWAPTRS *sdwaptrs = (SDWAPTRS *)(sdwa->sdwaxpad); + SDWAPTRS *sdwaptrs = (SDWAPTRS *)INT2PTR(sdwa->sdwaxpad); if (sdwaptrs != NULL) { - char *sdwarc1 = (char *)sdwaptrs->sdwasrvp; + char *sdwarc1 = (char *)INT2PTR(sdwaptrs->sdwasrvp); if (sdwarc1 != NULL) { rsn = *(int * __ptr32)(sdwarc1 + 44); } diff --git a/c/xlate.c b/c/xlate.c index 16021118f..1bc449aff 100644 --- a/c/xlate.c +++ b/c/xlate.c @@ -80,7 +80,7 @@ char* a2e(char *buffer, int len) } -CharsetOracle *makeCharsetOracle(){ +CharsetOracle *makeCharsetOracle(void){ CharsetOracle *oracle = (CharsetOracle*)safeMalloc(sizeof(CharsetOracle),"CharsetOracle"); memset(oracle,0,sizeof(CharsetOracle)); return oracle; diff --git a/c/yaml2json.c b/c/yaml2json.c index 4f48ced4e..de5cc20d2 100644 --- a/c/yaml2json.c +++ b/c/yaml2json.c @@ -19,6 +19,13 @@ #include "json.h" #include "jsonschema.h" +#ifdef __ZOWE_OS_ZOS + +int __atoe_l(char *bufferptr, int leng); +int __etoa_l(char *bufferptr, int leng); + +#endif + #if defined(__ZOWE_OS_ZOS) # define SOURCE_CODE_CHARSET CCSID_IBM1047 @@ -760,7 +767,7 @@ static void yaml2JS1(ByteOutputStream *bos, printf("tag = %s scalarStyle=%s\n",nativeTag,getScalarStyleName(node->data.scalar.style)); } Json *scalar = NULL; - // HERE, make test with float, int, bool, null, ddate + /* HERE, make test with float, int, bool, null, date */ if (!strcmp(nativeTag,YAML_NULL_TAG)){ } else if (!strcmp(nativeTag,YAML_NULL_TAG)){ /* Json *scalar = jsonBuildNull(b,parent,parentKey,&buildStatus); */ @@ -773,7 +780,7 @@ static void yaml2JS1(ByteOutputStream *bos, bool valid; int64_t x = readInt((yaml_char_t*)nativeValue,valueLength,&valid); if (valid){ - printf("%lld\n",x); + /* this is not yet done */ } else { buildStatus = JSON_FAIL_BAD_INTEGER; } diff --git a/c/zos.c b/c/zos.c index de65088d1..0e1e25081 100644 --- a/c/zos.c +++ b/c/zos.c @@ -1413,7 +1413,7 @@ static void *loadByNameInternal(char *moduleName, int *statusPtr, if (status){ return 0; /* entry point not found */ } else{ - return (void*)entryPoint; + return INT2PTR(entryPoint); } } diff --git a/h/le.h b/h/le.h index 964fed720..224f6c779 100644 --- a/h/le.h +++ b/h/le.h @@ -245,8 +245,8 @@ ZOWE_PRAGMA_PACK_RESET #endif -char *getCAA(); -void showRTL(); +char *getCAA(void); +void showRTL(void); #else /* not __ZOWE_OS_ZOS - */ diff --git a/h/xlate.h b/h/xlate.h index 7199aa3c4..3a0d15a36 100644 --- a/h/xlate.h +++ b/h/xlate.h @@ -80,7 +80,7 @@ typedef struct CharsetOracle_tag { #define CHARSET_ORACLE_EBCDIC_FAMILY 0x10000 #define CHARSET_ORACLE_UTF_FAMILY 0x20000 -CharsetOracle *makeCharsetOracle(); +CharsetOracle *makeCharsetOracle(void); void freeCharsetOracle(CharsetOracle *oracle); void charsetOracleDigest(CharsetOracle *oracle, char *s, int len); diff --git a/h/zowetypes.h b/h/zowetypes.h index 1019ef83d..52df8f73c 100644 --- a/h/zowetypes.h +++ b/h/zowetypes.h @@ -281,6 +281,8 @@ typedef uint64_t uint64; #define INT_INT64(x) ((int64)x) #define CHARPTR_INT64(x) ((int64)x) #define INT2PTR(x) ((void*)((int64_t)(x))) +#define INT64_LL(x) ((long long)(x)) +#define UINT64_ULL(x) ((unsigned long long)(x)) #else @@ -291,6 +293,8 @@ typedef unsigned long long uint64; #define INT_INT64(x) ((int64)x) #define CHARPTR_INT64(x) ((int64)((int)x)) #define INT2PTR(x) ((void*)(x)) +#define INT64_LL(x) ((long long)(x)) +#define UINT64_ULL(x) ((unsigned long long)(x)) #endif From af5555bfd317d55313df7cab58c8ffbf0ec91385 Mon Sep 17 00:00:00 2001 From: Joe Devlin Date: Fri, 18 Mar 2022 21:14:32 -0500 Subject: [PATCH 32/42] more xlclang and static analysis based fixes Signed-off-by: Joe Devlin --- c/bpxskt.c | 20 +++++++--- c/json.c | 37 +++++++----------- c/zos.c | 104 ++++++++++++++++++++++++++++----------------------- h/recovery.h | 8 ++-- h/unixfile.h | 4 +- 5 files changed, 91 insertions(+), 82 deletions(-) diff --git a/c/bpxskt.c b/c/bpxskt.c index 89d960d2b..2893ac6bd 100644 --- a/c/bpxskt.c +++ b/c/bpxskt.c @@ -447,6 +447,9 @@ Socket *udpPeer(SocketAddress *socketAddress, } } +/* Forward decl */ +static int setSocketReuseAddr(int sd, int *returnCode, int *reasonCode); + Socket *tcpServer2(InetAddr *addr, int port, int tlsFlags, @@ -521,7 +524,7 @@ Socket *tcpServer2(InetAddr *addr, dumpbuffer((char*)socketAddress,sizeof(SocketAddress)); dumpbuffer((char*)addr,sizeof(InetAddr)); } - setSocketReuseAddr(&sd, + setSocketReuseAddr(sd, returnCode, reasonCodePtr); if (returnValue != 0){ @@ -1230,10 +1233,9 @@ int getSocketOption(Socket *socket, int optionName, int *optionDataLength, char } } -int setSocketOption(Socket *socket, int level, int optionName, int optionDataLength, char *optionData, - int *returnCode, int *reasonCode){ +static int setSDOption(int sd, int level, int optionName, int optionDataLength, char *optionData, + int *returnCode, int *reasonCode){ int status = 0; - int sd = socket->sd; int returnValue = 0; *returnCode = *reasonCode = 0; int *reasonCodePtr; @@ -1266,6 +1268,12 @@ int setSocketOption(Socket *socket, int level, int optionName, int optionDataLen } } +int setSocketOption(Socket *socket, int level, int optionName, int optionDataLength, char *optionData, + int *returnCode, int *reasonCode){ + return setSDOption(socket->sd,level,optionName,optionDataLength,optionData,returnCode,reasonCode); +} + + int setSocketNoDelay(Socket *socket, int noDelay, int *returnCode, int *reasonCode){ return setSocketOption(socket,IPPROTO_TCP,TCP_NODELAY,sizeof(int),(char*)&noDelay,returnCode,reasonCode); } @@ -1278,9 +1286,9 @@ int setSocketReadBufferSize(Socket *socket, int bufferSize, int *returnCode, int return setSocketOption(socket,SOL_SOCKET,SOCK_SO_RCVBUF,sizeof(int),(char*)&bufferSize,returnCode,reasonCode); } -int setSocketReuseAddr(Socket *socket, int *returnCode, int *reasonCode){ +static int setSocketReuseAddr(int sd, int *returnCode, int *reasonCode){ int on = 1; - return setSocketOption(socket,SOL_SOCKET,SOCK_SO_REUSEADDR,sizeof(int),(char*)&on,returnCode,reasonCode); + return setSDOption(sd,SOL_SOCKET,SOCK_SO_REUSEADDR,sizeof(int),(char*)&on,returnCode,reasonCode); } int udpReceiveFrom(Socket *socket, diff --git a/c/json.c b/c/json.c index 03516f4a8..3bf02996c 100644 --- a/c/json.c +++ b/c/json.c @@ -26,10 +26,6 @@ #else -#ifndef _XOPEN_SOURCE -#define _XOPEN_SOURCE 600 -#endif - #include #include #include @@ -57,6 +53,11 @@ typedef int64_t ssize_t; #endif +/* Having lots of problems including unistd, so here's a hack */ +#ifdef __ZOWE_OS_ZOS +ssize_t write(int fd, const void *buf, size_t count); +#endif + /* @@ -217,16 +218,10 @@ void jsonWriteBufferInternal(jsonPrinter *p, char *text, int len) { #define ESCAPE_LEN 6 /* \u0123 */ #define ESCAPE_NATIVE_BUF_SIZE 12 /* some bytes have been added for safety */ -#if defined(__ZOWE_OS_ZOS) -# pragma convert("UTF8") -#endif -static char UTF8_QUOTE = '"'; -static char UTF8_BACKSLASH = '\\'; -static char UTF8_ESCAPED_QUOTE[2] = "\\\""; -static char UTF8_ESCAPED_BACKSLASH[2]= "\\\\"; -#if defined(__ZOWE_OS_ZOS) -# pragma convert(pop) -#endif +static char UTF8_QUOTE = 0x22; +static char UTF8_BACKSLASH = 0x5C; +static char UTF8_ESCAPED_QUOTE[2] = { 0x5C, 0x22 }; +static char UTF8_ESCAPED_BACKSLASH[2]={ 0x5C, 0x5C}; #define UTF8_GET_NEXT_CHAR($size, $buf, $idx, $outChar, $err) do { \ if ((($buf)[*($idx)] & 0x80) == 0) { \ @@ -318,8 +313,9 @@ writeBufferWithEscaping(jsonPrinter *p, size_t len, char text[len]) { return -1; } DEBUG("character at %p + %d, len %d, char %x\n", text, i, len, utf8Char); - if (((utf8Char >= 0) && (utf8Char <= effectiveControlCharBoundary)) - || (utf8Char == UTF8_BACKSLASH) || (utf8Char == UTF8_QUOTE)) { + if ((utf8Char <= effectiveControlCharBoundary) || + (utf8Char == UTF8_BACKSLASH) || + (utf8Char == UTF8_QUOTE)) { chunkLen = i - 1 - currentChunkOffset; if (chunkLen > 0) { @@ -395,14 +391,7 @@ void jsonConvertAndWriteBuffer(jsonPrinter *p, char *text, size_t len, DUMPBUF(text, len); } if (escape) { - size_t bytesWritten; - - bytesWritten = writeBufferWithEscaping(p, len, text); - if (bytesWritten < 0) { - JSONERROR("jsonConvertAndWriteBuffer() error: bytesWritten = %d\n", - (int)bytesWritten); - return; - } + writeBufferWithEscaping(p, len, text); } else { jsonWriteBufferInternal(p, text, len); } diff --git a/c/zos.c b/c/zos.c index 0e1e25081..ba01062ba 100644 --- a/c/zos.c +++ b/c/zos.c @@ -461,6 +461,8 @@ typedef struct safp_tag{ #define SAF_VERIFY_NESTED_COPY 0x20 #define SAF_VERIFY_NO_MFA 0x10 +#pragma pack(packed) + typedef struct safVerifyRequest_tag{ char initlen; char initsubp; @@ -507,6 +509,13 @@ typedef struct safVerifyRequest_tag{ void * __ptr32 idta; } safVerifyRequest; +/* Helper function to workaround xlclang FE and backend not agreeing on packed ptr32 size */ + +static int sizeofSafVerifyRequest(void){ + return sizeof(safVerifyRequest); +} + + /* first flag set */ #define SAF_AUTH_RACFIND_SPECIFIED 0x80 #define SAF_AUTH_RACFIND_YES 0x40 @@ -533,15 +542,59 @@ typedef struct safVerifyRequest_tag{ #define SAF_AUTH_STATUS_WRITEONLY 0x20 #define SAF_AUTH_STATUS_ACCESS 0x10 +typedef struct safAuthRequest_tag{ + char achkleng; + char old_installationDataAddress[3]; + char achkflg1; + char old_entityNameAddress[3]; + char achkflg2; + char old_classNameAddress[3]; + char achkflg3; + char old_volser3ByteAddress[3]; + void * __ptr32 old_volserAddress; + void * __ptr32 applName; + void * __ptr32 acee; + void * __ptr32 owner; /* ? */ + void * __ptr32 installationData; + void * __ptr32 entityName; + void * __ptr32 className; + void * __ptr32 volser; + void * __ptr32 acclvl1; + void * __ptr32 acclvl2; + short fileSequenceNumber; + char tapeFlags; + char achkflg4; + void * __ptr32 userid; + void * __ptr32 groupName; + void * __ptr32 ddname; + void * __ptr32 reserved1; + void * __ptr32 utoken; + void * __ptr32 rtoken; + void * __ptr32 logstr; + void * __ptr32 recvr; +} safAuthRequest; + +typedef struct safStatRequest_tag{ + void * __ptr32 className; + void * __ptr32 CDTentry; + short statLength; + char reserved[2]; + void * __ptr32 classCopy; + int classCopyLength; + void * __ptr32 statNext; +} safStatRequest; + +#pragma pack(reset) + static int SAF(safp * __ptr32 safwrapper, int useSupervisorMode) { int returnCode = 0; int supervisorState = 0; - char * __ptr32 cvt = * (char * __ptr32 * __ptr32 ) 16; - char * __ptr32 safVectorTable = (char * __ptr32) *(int *)(cvt + 248); - char * __ptr32 safRouter = (char * __ptr32) *(int *)(safVectorTable + 12); + CVT *cvt = getCVT(); + int *safVectorTable = (int*)INT2PTR(cvt->cvtsaf); + int safRouter = safVectorTable[3]; /* not using ptr type becuz embedded ASM does not care */ ALLOC_STRUCT31( STRUCT31_NAME(below2G), @@ -595,47 +648,6 @@ void setSafTrace(int traceLevel, void *traceFile){ safTraceFile = traceFile; } -typedef struct safAuthRequest_tag{ - char achkleng; - char old_installationDataAddress[3]; - char achkflg1; - char old_entityNameAddress[3]; - char achkflg2; - char old_classNameAddress[3]; - char achkflg3; - char old_volser3ByteAddress[3]; - void * __ptr32 old_volserAddress; - void * __ptr32 applName; - void * __ptr32 acee; - void * __ptr32 owner; /* ? */ - void * __ptr32 installationData; - void * __ptr32 entityName; - void * __ptr32 className; - void * __ptr32 volser; - void * __ptr32 acclvl1; - void * __ptr32 acclvl2; - short fileSequenceNumber; - char tapeFlags; - char achkflg4; - void * __ptr32 userid; - void * __ptr32 groupName; - void * __ptr32 ddname; - void * __ptr32 reserved1; - void * __ptr32 utoken; - void * __ptr32 rtoken; - void * __ptr32 logstr; - void * __ptr32 recvr; -} safAuthRequest; - -typedef struct safStatRequest_tag{ - void * __ptr32 className; - void * __ptr32 CDTentry; - short statLength; - char reserved[2]; - void * __ptr32 classCopy; - int classCopyLength; - void * __ptr32 statNext; -} safStatRequest; static safp *makeSAFCallData(int requestNumber, int useSupervisorMode, @@ -652,7 +664,7 @@ static safp *makeSAFCallData(int requestNumber, version = SAFPR192; switch (requestNumber){ case SAFPVER: - specificDataSize = sizeof(safVerifyRequest); + specificDataSize = sizeofSafVerifyRequest(); flags = SAFP_FLAG_R18; if (useSupervisorMode){ flags |= SAFP_FLAG_SYST; @@ -995,7 +1007,7 @@ static int safVerifyInternal(int options, } } - verifyRequest->initlen = sizeof(safVerifyRequest); + verifyRequest->initlen = sizeofSafVerifyRequest(); if (options & VERIFY_CREATE){ verifyFlags1 |= SAF_VERIFY_CREATE; } else if (options & VERIFY_DELETE){ diff --git a/h/recovery.h b/h/recovery.h index ec7d91850..fc131e241 100644 --- a/h/recovery.h +++ b/h/recovery.h @@ -295,7 +295,7 @@ typedef struct RecoveryContext_tag { char sdumpxSaveArea[72]; } RecoveryContext; -typedef struct RecoveryStatePool_tag RecoveryStatePool; +struct RecoveryStatePool_tag; ZOWE_PRAGMA_PACK_RESET @@ -401,7 +401,7 @@ int recoveryEstablishRouter(int flags); * of the RC_RCV_xxxx error codes. *****************************************************************************/ int recoveryEstablishRouter2(RecoveryContext *userContext, - RecoveryStatePool *userStatePool, + struct RecoveryStatePool_tag *userStatePool, int flags); /***************************************************************************** @@ -415,7 +415,7 @@ int recoveryEstablishRouter2(RecoveryContext *userContext, * Return value: * State pool structure on success or NULL on failure. *****************************************************************************/ -RecoveryStatePool *recoveryMakeStatePool(unsigned int stateCount); +struct RecoveryStatePool_tag *recoveryMakeStatePool(unsigned int stateCount); /***************************************************************************** * Remove a user state pool. @@ -428,7 +428,7 @@ RecoveryStatePool *recoveryMakeStatePool(unsigned int stateCount); * Return value: * N/A *****************************************************************************/ -void recoveryRemoveStatePool(RecoveryStatePool *statePool); +void recoveryRemoveStatePool(struct RecoveryStatePool_tag *statePool); #endif /* RCVR_CPOOL_STATES */ diff --git a/h/unixfile.h b/h/unixfile.h index 58d8560ae..0cef65145 100644 --- a/h/unixfile.h +++ b/h/unixfile.h @@ -269,8 +269,8 @@ typedef struct BPXYATT_tag { /* End of Version 1 */ int fileFormat:8; int reserved2:24; - int fileTagCCSID:16; - int fileTagFlags:16; + unsigned int fileTagCCSID:16; + unsigned int fileTagFlags:16; char reserved3[8]; /* End of Version 2 */ int64 accessTime2; From 508c6c156536481056f6cac65274f2033494e492 Mon Sep 17 00:00:00 2001 From: Joe Devlin Date: Fri, 18 Mar 2022 23:19:48 -0500 Subject: [PATCH 33/42] missed a change to Recovery struct Signed-off-by: Joe Devlin --- h/crossmemory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/h/crossmemory.h b/h/crossmemory.h index e8fcc6978..07ae1890c 100644 --- a/h/crossmemory.h +++ b/h/crossmemory.h @@ -231,7 +231,7 @@ typedef struct CrossMemoryServerGlobalArea_tag { int pcLogLevel; - PAD_LONG(0, RecoveryStatePool *pcssRecoveryPool); + PAD_LONG(0, struct RecoveryStatePool_tag *pcssRecoveryPool); CPID pcssStackPool; char reserved3[492]; From 515481ad3c9a77532cdb9f847590591ee8e20879 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 21 Mar 2022 16:39:03 -0400 Subject: [PATCH 34/42] Lot's of testing and filling in gaps in the schemavalidator, particularly allOf, anyOf, oneOf, and not Signed-off-by: Joe --- c/configmgr.c | 7 + c/json.c | 13 +- c/jsonschema.c | 426 +++++++++++++++++++++++++++-------- h/json.h | 1 + h/jsonschema.h | 1 + tests/schemadata/food.schema | 39 ++++ tests/schemadata/food1.json | 8 + tests/schematest.c | 2 - 8 files changed, 395 insertions(+), 102 deletions(-) create mode 100644 tests/schemadata/food.schema create mode 100644 tests/schemadata/food1.json diff --git a/c/configmgr.c b/c/configmgr.c index 692bd87d3..c34573eae 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -140,6 +140,13 @@ typedef int64_t ssize_t; curl https://api.github.com/repos/stedolan/jq/commits?per_page=5 > commits.json + --- + 1) top level call + 2) The "zowe" global, eg. zowe.validate(json,schema) -> null | [ ] + 3) configmgr - make schema arg explicit and path + 4) build file references "jdevlin" + 5) semantic validation + 6) $ {{ zowe.runtimeDirectory }} */ diff --git a/c/json.c b/c/json.c index 3bf02996c..395c8103c 100644 --- a/c/json.c +++ b/c/json.c @@ -2683,8 +2683,9 @@ static int mergeJson1(JsonMerger *merger, Json *parent, char *parentKey, Json *o return JSON_MERGE_STATUS_SUCCESS; } } else { - /* printf("merge scalar case parentKey=%s\n",(parentKey ? parentKey : "")); - fflush(stdout); + /* + printf("merge scalar case parentKey=%s\n",(parentKey ? parentKey : "")); + fflush(stdout); */ copyJson(builder,parent,parentKey,overrides); return JSON_MERGE_STATUS_SUCCESS; @@ -2706,6 +2707,14 @@ Json *jsonMerge(ShortLivedHeap *slh, Json *overrides, Json *base, int flags, int } } +Json *jsonCopy(ShortLivedHeap *slh, Json *value){ + JsonBuilder builder; + memset(&builder,0,sizeof(JsonBuilder)); + builder.parser.slh = slh; + copyJson(&builder,NULL,NULL,value); + return builder.root; +} + /****** JSON Pointers ******************/ static JsonPointerElement *makePointerElement(char *token, int type){ diff --git a/c/jsonschema.c b/c/jsonschema.c index e344a202e..93b6c6362 100644 --- a/c/jsonschema.c +++ b/c/jsonschema.c @@ -90,6 +90,7 @@ typedef int64_t ssize_t; typedef struct JSValueSpec_tag { int typeMask; + int enumeratedValuesCount; Json **enumeratedValues; Json *constValue; char *id; @@ -100,9 +101,12 @@ typedef struct JSValueSpec_tag { bool nullable; /* shared sub-schema definitions go here */ hashtable *definitions; - /* Compositing */ + /* Quantifying/Compositing */ + int allOfCount; struct JSValueSpec_tag **allOf; + int anyOfCount; struct JSValueSpec_tag **anyOf; + int oneOfCount; struct JSValueSpec_tag **oneOf; struct JSValueSpec_tag *not; @@ -126,7 +130,7 @@ typedef struct JSValueSpec_tag { bool exclusiveMaximum; bool exclusiveMinimum; double multipleOf; - bool isInteger; + /* bool isInteger; */ /* for arrays */ struct JSValueSpec_tag *itemSpec; @@ -144,8 +148,6 @@ typedef struct AccessPathUnion_tag { int dummy; } AccessPathUnion; - - static void printAccessPath(FILE *out, AccessPath *path){ for (int i=0; icurrentSize; i++){ if (path->elements[i].isName){ @@ -210,7 +212,13 @@ static void validationThrow(JsonValidator *validator, int errorCode, char *forma longjmp(validator->recoveryData,1); } -static ValidityException *noteValidityException(JsonValidator *validator, int invalidityCode, char *formatString, ...){ +static ValidityException *noteValidityException(JsonValidator *validator, + bool isSpeculating, + int invalidityCode, + char *formatString, ...){ + if (isSpeculating){ + return NULL; + } va_list argPointer; ValidityException *exception = (ValidityException*)safeMalloc(sizeof(ValidityException),"ValidityException"); exception->code = invalidityCode; @@ -250,11 +258,30 @@ static char *validatorAccessPath(JsonValidator *validator){ this part is invalid enough such that further evaluation would probably confuse the user with contradictory information. */ +typedef enum VResult_tag { + InvalidStop = 0, + ValidStop = 1, + InvalidContinue = 2, + ValidContinue = 3 +} VResult; + +static bool vResultValid(VResult r){ + return (r == ValidStop) || (r == ValidContinue); +} + +static VResult invertValidity(VResult r){ + return (VResult)(((int)r)^0x1); +} + static bool validateType(JsonValidator *validator, + bool isSpeculating, int typeCode, JSValueSpec *valueSpec){ + if (validator->traceLevel >= 1){ + printf("typeCode=%d shifted=0x%x mask=0x%x\n",typeCode,(1 << typeCode),valueSpec->typeMask); + } if (((1 << typeCode) & valueSpec->typeMask) == 0){ - noteValidityException(validator,12,"type %d not permitted at %s",typeCode,validatorAccessPath(validator)); + noteValidityException(validator,isSpeculating,12,"type %d not permitted at %s",typeCode,validatorAccessPath(validator)); return false; } else { return true; @@ -262,20 +289,24 @@ static bool validateType(JsonValidator *validator, } /* for mutual recursion */ -static bool validateJSON(JsonValidator *validator, Json *value, JSValueSpec *valueSpec); +static VResult validateJSON(JsonValidator *validator, bool isSpeculating, Json *value, JSValueSpec *valueSpec); -static bool validateJSONObject(JsonValidator *validator, - JsonObject *object, - JSValueSpec *valueSpec){ +static VResult validateJSONObject(JsonValidator *validator, + bool isSpeculating, + JsonObject *object, + JSValueSpec *valueSpec){ + int invalidCount = 0; AccessPath *accessPath = validator->accessPath; - printf("validateJSONObject required=0x%p\n",valueSpec->required); - printAccessPath(stdout,accessPath); - fflush(stdout); + if (validator->traceLevel >= 1){ + printf("validateJSONObject required=0x%p\n",valueSpec->required); + printAccessPath(stdout,accessPath); + fflush(stdout); + } if (valueSpec->required){ for (int r=0; rrequiredCount; r++){ char *requiredProperty = valueSpec->required[r]; if (jsonObjectGetPropertyValue(object,requiredProperty) == NULL){ - noteValidityException(validator,12,"missing required property '%s' at '%s'", + noteValidityException(validator,isSpeculating,12,"missing required property '%s' at '%s'", requiredProperty,validatorAccessPath(validator)); } } @@ -286,8 +317,10 @@ static bool validateJSONObject(JsonValidator *validator, while (property){ propertyCount++; char *propertyName = jsonPropertyGetKey(property); - printf("validate object pname=%s\n",propertyName); - fflush(stdout); + if (validator->traceLevel >= 1){ + printf("validate object pname=%s\n",propertyName); + fflush(stdout); + } Json *propertyValue = jsonPropertyGetValue(property); JSValueSpec *propertySpec = (valueSpec->properties ? (JSValueSpec*)htGet(valueSpec->properties,propertyName) : @@ -295,7 +328,10 @@ static bool validateJSONObject(JsonValidator *validator, accessPathPushName(accessPath,propertyName); if (propertySpec != NULL){ - validateJSON(validator,propertyValue,propertySpec); + VResult propResult = validateJSON(validator,isSpeculating,propertyValue,propertySpec); + if (!vResultValid(propResult)){ + invalidCount++; + } } else { if (validator->flags && VALIDATOR_WARN_ON_UNDEFINED_PROPERTIES){ printf("*WARNING* unspecified property seen, '%s', and checking code is not complete, vspec->props=0x%p\n", @@ -319,24 +355,28 @@ static bool validateJSONObject(JsonValidator *validator, if (valueSpec->validatorFlags & JS_VALIDATOR_MAX_PROPS){ int64_t lim = valueSpec->minProperties; if (propertyCount > lim){ - noteValidityException(validator,12,"too many properties, %d > MAX=%d at %s", + invalidCount++; + noteValidityException(validator,isSpeculating,12,"too many properties, %d > MAX=%d at %s", propertyCount,lim,validatorAccessPath(validator)); } } if (valueSpec->validatorFlags & JS_VALIDATOR_MIN_PROPS){ int64_t lim = valueSpec->minProperties; if (propertyCount < lim){ - noteValidityException(validator,12,"too few properties, %d < MIN=%d at %s", + invalidCount++; + noteValidityException(validator,isSpeculating,12,"too few properties, %d < MIN=%d at %s", propertyCount,lim,validatorAccessPath(validator)); } } - return true; + return (invalidCount > 0 ? InvalidContinue : ValidContinue); } -bool validateJSONArray(JsonValidator *validator, - JsonArray *array, - JSValueSpec *valueSpec){ +static VResult validateJSONArray(JsonValidator *validator, + bool isSpeculating, + JsonArray *array, + JSValueSpec *valueSpec){ AccessPath *accessPath = validator->accessPath; + int invalidCount = 0; //boolean uniqueItems; // Long maxContains; // Long minContains; @@ -344,7 +384,8 @@ bool validateJSONArray(JsonValidator *validator, if (valueSpec->validatorFlags & JS_VALIDATOR_MAX_ITEMS){ int64_t lim = valueSpec->maxItems; if (elementCount > lim){ - noteValidityException(validator,12, + invalidCount++; + noteValidityException(validator,isSpeculating,12, "too many array items, %d > MAX=%d at %s", elementCount,lim,validatorAccessPath(validator)); } @@ -352,7 +393,8 @@ bool validateJSONArray(JsonValidator *validator, if (valueSpec->validatorFlags & JS_VALIDATOR_MIN_ITEMS){ int64_t lim = valueSpec->minItems; if (elementCount < lim){ - noteValidityException(validator,12, + invalidCount++; + noteValidityException(validator,isSpeculating,12, "too few array ites, %d < MIN=%d at %s", elementCount,lim,validatorAccessPath(validator)); } @@ -371,36 +413,45 @@ bool validateJSONArray(JsonValidator *validator, Json *itemValue = jsonArrayGetItem(array,i); accessPathPushIndex(accessPath,i); if (valueSpec->itemSpec != NULL){ - validateJSON(validator,itemValue,valueSpec->itemSpec); + VResult elementStatus = validateJSON(validator,isSpeculating,itemValue,valueSpec->itemSpec); + if (!vResultValid(elementStatus)){ + invalidCount++; + } + } if (valueSpec->uniqueItems){ long longHash = jsonLongHash(itemValue); if (lhtGet(uniquenessSet,longHash) != NULL){ - noteValidityException(validator,12,"array uniqueItems violation %s is duplicate at %s", + invalidCount++; + noteValidityException(validator,isSpeculating,12,"array uniqueItems violation %s is duplicate at %s", itemValue,i,validatorAccessPath(validator)); } } accessPathPop(accessPath); } - return true; + return (invalidCount > 0 ? InvalidContinue : ValidContinue); } -static bool validateJSONString(JsonValidator *validator, - Json *string, - JSValueSpec *valueSpec){ +static VResult validateJSONString(JsonValidator *validator, + bool isSpeculating, + Json *string, + JSValueSpec *valueSpec){ + int invalidCount = 0; char *s = jsonAsString(string); int len = strlen(s); if (valueSpec->validatorFlags & JS_VALIDATOR_MAX_LENGTH){ long lim = valueSpec->maxLength; if (len > lim){ - noteValidityException(validator,12,"string too long, %d > MAX=%d at %s", + invalidCount++; + noteValidityException(validator,isSpeculating,12,"string too long, %d > MAX=%d at %s", len,lim,validatorAccessPath(validator)); } } if (valueSpec->validatorFlags & JS_VALIDATOR_MIN_LENGTH){ long lim = valueSpec->minLength; if (len < lim){ - noteValidityException(validator,12,"string too short, %d < MAX=%d at %s", + invalidCount++; + noteValidityException(validator,isSpeculating,12,"string too short, %d < MAX=%d at %s", len,lim,validatorAccessPath(validator)); } } @@ -420,7 +471,8 @@ static bool validateJSONString(JsonValidator *validator, if (valueSpec->compiledPattern){ int regexStatus = regexExec(valueSpec->compiledPattern,s,0,NULL,0); if (regexStatus != 0){ - noteValidityException(validator,12,"string pattern match fail s='%s', pat='%s', at %s", + invalidCount++; + noteValidityException(validator,isSpeculating,12,"string pattern match fail s='%s', pat='%s', at %s", s,valueSpec->pattern,validatorAccessPath(validator)); } } @@ -428,22 +480,25 @@ static bool validateJSONString(JsonValidator *validator, // String pattern; // ECMA-262 regex // prefix items // additionalProperties - return true; + return (invalidCount > 0 ? InvalidContinue : ValidContinue); } -static bool validateJSONNumber(JsonValidator *validator, - Json *doubleOrLegacyInt, - JSValueSpec *valueSpec){ +static VResult validateJSONNumber(JsonValidator *validator, + bool isSpeculating, + Json *doubleOrLegacyInt, + JSValueSpec *valueSpec){ // Number multipleOf; // can be a filled, and must be > 0 // Number exclusiveMaximum; // Number exclusiveMinimum; + int invalidCount = 0; double d = (doubleOrLegacyInt->type == JSON_TYPE_NUMBER ? (double)jsonAsNumber(doubleOrLegacyInt) : jsonAsDouble(doubleOrLegacyInt)); if (valueSpec->validatorFlags & JS_VALIDATOR_MAX){ double lim = valueSpec->maximum; if (valueSpec->exclusiveMaximum ? (d >= lim) : (d > lim)){ - noteValidityException(validator,12,"value too large, %f %s MAX=%f at %s", + invalidCount++; + noteValidityException(validator,isSpeculating,12,"value too large, %f %s MAX=%f at %s", d, (valueSpec->exclusiveMaximum ? ">=" : ">"), lim, @@ -453,27 +508,31 @@ static bool validateJSONNumber(JsonValidator *validator, if (valueSpec->validatorFlags & JS_VALIDATOR_MIN){ double lim = valueSpec->minimum; if (valueSpec->exclusiveMinimum ? (d <= lim) : (d < lim)){ - noteValidityException(validator,12,"value too small, %f %s MAX=%f at %s", + invalidCount++; + noteValidityException(validator,isSpeculating,12,"value too small, %f %s MAX=%f at %s", d, (valueSpec->exclusiveMinimum ? "<=" : "<"), lim, validatorAccessPath(validator)); } } - return true; + return (invalidCount > 0 ? InvalidContinue : ValidContinue); } -static bool validateJSONInteger(JsonValidator *validator, - Json *value64, - JSValueSpec *valueSpec){ +static VResult validateJSONInteger(JsonValidator *validator, + bool isSpeculating, + Json *value64, + JSValueSpec *valueSpec){ // Number multipleOf; // can be a filled, and must be > 0 // Number exclusiveMaximum; // Number exclusiveMinimum; + int invalidCount = 0; int64_t i = jsonAsInt64(value64); if (valueSpec->validatorFlags & JS_VALIDATOR_MAX){ int64_t lim = (int64_t)valueSpec->maximum; if (valueSpec->exclusiveMaximum ? (i >= lim) : (i > lim)){ - noteValidityException(validator,12,"value too large, %lld %s MAX=%lld at %s", + invalidCount++; + noteValidityException(validator,isSpeculating,12,"value too large, %lld %s MAX=%lld at %s", i, (valueSpec->exclusiveMaximum ? ">=" : ">"), lim, @@ -483,32 +542,35 @@ static bool validateJSONInteger(JsonValidator *validator, if (valueSpec->validatorFlags & JS_VALIDATOR_MIN){ int64_t lim = (int64_t)valueSpec->minimum; if (valueSpec->exclusiveMinimum ? (i <= lim) : (i < lim)){ - noteValidityException(validator,12,"value too small, %lld %s MAX=%lld at %s", + invalidCount++; + noteValidityException(validator,isSpeculating,12,"value too small, %lld %s MAX=%lld at %s", i, (valueSpec->exclusiveMinimum ? "<=" : "<"), lim, validatorAccessPath(validator)); } } - return true; + return (invalidCount > 0 ? InvalidContinue : ValidContinue); } -static JSValueSpec *resolveRef(JsonValidator *validator, JSValueSpec *valueSpec){ +static JSValueSpec *resolveRef(JsonValidator *validator, bool isSpeculating, JSValueSpec *valueSpec){ char *ref = valueSpec->ref; - printf("resolveRef ref=%s\n",valueSpec->ref); + if (validator->traceLevel >= 1){ + printf("resolveRef ref=%s\n",valueSpec->ref); + } if (!memcmp(ref,"#/$defs/",8)){ /* local resolution */ int refLen = strlen(ref); int pos = 2; char *key = ref+8; JSValueSpec *topSchema = validator->schema->topValueSpec; if (topSchema->definitions == NULL){ - noteValidityException(validator,12,"schema '%s' does not define shared '$defs'", + noteValidityException(validator,isSpeculating,12,"schema '%s' does not define shared '$defs'", (topSchema->id ? topSchema->id : "")); return NULL; /* for the compiler, bless its heart */ } JSValueSpec *resolvedSchema = htGet(topSchema->definitions,key); if (resolvedSchema == NULL){ - noteValidityException(validator,12,"schema ref '%s' does not resolve against '$defs'",ref); + noteValidityException(validator,isSpeculating,12,"schema ref '%s' does not resolve against '$defs'",ref); return NULL; } return resolvedSchema; @@ -517,55 +579,200 @@ static JSValueSpec *resolveRef(JsonValidator *validator, JSValueSpec *valueSpec) need to merge URL according W3 rules and pass to pluggable resolver that we hope this validator has. */ - noteValidityException(validator,12,"external refs not yet implemented '%s' at %s", + noteValidityException(validator,isSpeculating,12,"external refs not yet implemented '%s' at %s", valueSpec->ref,validatorAccessPath(validator)); return NULL; } } -static bool validateJSON(JsonValidator *validator, Json *value, JSValueSpec *valueSpec){ - while (valueSpec->ref){ - valueSpec = resolveRef(validator,valueSpec); - if (valueSpec == NULL){ - return false; - } - } - printf("validate JSON value->type=%d\n",value->type); - fflush(stdout); - // type is string or array, does this mean to collapse ValueSpecTypes - // should we flag errors on out-of-bounds validation keywords for type keyword +static VResult validateJSONSimple(JsonValidator *validator, bool isSpeculating, Json *value, JSValueSpec *valueSpec){ if (jsonIsNull(value)){ - return validateType(validator,JSTYPE_NULL,valueSpec); + return (validateType(validator,isSpeculating,JSTYPE_NULL,valueSpec) ? + ValidContinue : InvalidStop); } else if (jsonIsBoolean(value)){ - return validateType(validator,JSTYPE_BOOLEAN,valueSpec); + printf("bool case\n"); + return (validateType(validator,isSpeculating,JSTYPE_BOOLEAN,valueSpec) ? + ValidContinue : InvalidStop); } else if (jsonIsObject(value)){ - return (validateType(validator,JSTYPE_OBJECT,valueSpec) && - validateJSONObject(validator,jsonAsObject(value),valueSpec)); + return (validateType(validator,isSpeculating,JSTYPE_OBJECT,valueSpec) ? + validateJSONObject(validator,isSpeculating,jsonAsObject(value),valueSpec) : + InvalidStop); } else if (jsonIsArray(value)){ - return (validateType(validator,JSTYPE_ARRAY,valueSpec) && - validateJSONArray(validator,jsonAsArray(value),valueSpec)); + return (validateType(validator,isSpeculating,JSTYPE_ARRAY,valueSpec) ? + validateJSONArray(validator,isSpeculating,jsonAsArray(value),valueSpec) : + InvalidStop); } else if (jsonIsString(value)){ - return (validateType(validator,JSTYPE_STRING,valueSpec) && - validateJSONString(validator,value,valueSpec)); + return (validateType(validator,isSpeculating,JSTYPE_STRING,valueSpec) ? + validateJSONString(validator,isSpeculating,value,valueSpec) : + InvalidStop); } else if (jsonIsNumber(value)){ if (!jsonIsInt64(value)){ - return (validateType(validator,JSTYPE_NUMBER,valueSpec) && - validateJSONNumber(validator,value,valueSpec)); /* general, comparisons done as doubles */ + return (validateType(validator,isSpeculating,JSTYPE_NUMBER,valueSpec) ? + validateJSONNumber(validator,isSpeculating,value,valueSpec) : /* general, comparisons done as doubles */ + InvalidStop); } else { - return (validateType(validator,JSTYPE_INTEGER,valueSpec) && - validateJSONInteger(validator,value,valueSpec)); + return (validateType(validator,isSpeculating,JSTYPE_INTEGER,valueSpec) ? + validateJSONInteger(validator,isSpeculating,value,valueSpec) : + InvalidStop); } } else { printf("*** PANIC *** unhandled JS Value with type = %d\n",value->type); + return InvalidStop; + } +} + +#define QUANT_ALL 1 +#define QUANT_ANY 2 +#define QUANT_ONE 3 + +static VResult validateQuantifiedSpecs(JsonValidator *validator, + bool isSpeculating, + Json *value, JSValueSpec **valueSpecs, + int specCount, int quantType){ + int validCount = 0; + + for (int i=0; itraceLevel >= 1){ + printf("vQS i=%d type=%d valid=%d vCount=%d\n",i,quantType,eltValid,validCount); + } + switch (quantType){ + case QUANT_ALL: + if (!eltValid){ + noteValidityException(validator,isSpeculating,12,"not allOf schemas at %s are valid", + validatorAccessPath(validator)); + return InvalidStop; + } + break; + case QUANT_ANY: + if (eltValid){ + return ValidStop; + } + case QUANT_ONE: + if (validCount > 1){ + noteValidityException(validator,isSpeculating,12,"more than oneOf schemas at %s are valid", + validatorAccessPath(validator)); + return InvalidStop; + } + break; + } + } + switch (quantType){ + case QUANT_ALL: + return ValidContinue; + case QUANT_ANY: + noteValidityException(validator,isSpeculating,12,"not anyOf schemas at %s are valid", + validatorAccessPath(validator)); + + return ValidContinue; + case QUANT_ONE: + /* printf("quantOne vCount=%d\n",validCount); */ + return (validCount == 1) ? ValidContinue : InvalidContinue; + default: + /* can't really get here */ + return InvalidStop; + } +} + +/* here, + 1) patProps + 2) add'l props + commit +*/ + +static bool jsonEquals(Json *a, Json *b){ + if (jsonIsArray(a) || jsonIsArray(b)){ + return false; + } else if (jsonIsObject(a) || jsonIsObject(b)){ return false; + } else if (jsonIsString(a)){ + if (jsonIsString(b)){ + char *aStr = jsonAsString(a); + char *bStr = jsonAsString(b); + return !strcmp(aStr,bStr); + } else { + return false; + } + } else { + return !memcmp(a,b,sizeof(Json)); + } +} + +static VResult validateJSON(JsonValidator *validator, bool isSpeculating,Json *value, JSValueSpec *valueSpec){ + while (valueSpec->ref){ + valueSpec = resolveRef(validator,isSpeculating,valueSpec); + if (valueSpec == NULL){ + return InvalidStop; + } + } + if (validator->traceLevel >= 1){ + printf("validate JSON value->type=%d specTypeMask=0x%x\n",value->type,valueSpec->typeMask); + fflush(stdout); + } + if (valueSpec->constValue){ + bool eq = jsonEquals(value,valueSpec->constValue); + if (!eq){ + noteValidityException(validator,isSpeculating,12,"unequal constant value at %s", + validatorAccessPath(validator)); + return InvalidStop; + } else { + return ValidContinue; + } + } else if (valueSpec->enumeratedValues){ + bool matched = false; + for (int ee=0; eeenumeratedValuesCount; ee++){ + Json *enumValue = valueSpec->enumeratedValues[ee]; + if (jsonEquals(value,enumValue)){ + matched = true; + break; + } + } + if (!matched){ + noteValidityException(validator,isSpeculating,12,"no matching enum value at %s", + validatorAccessPath(validator)); + return InvalidStop; + } else { + return ValidContinue; + } + } else { + /* type is string or array, does this mean to collapse ValueSpecTypes + should we flag errors on out-of-bounds validation keywords for type keyword */ + VResult validity = validateJSONSimple(validator,isSpeculating,value,valueSpec); + /* here, mix in the quantified and compound */ + if (!vResultValid(validity)) return validity; + if (valueSpec->allOf){ + validity = validateQuantifiedSpecs(validator,isSpeculating,value,valueSpec->allOf,valueSpec->allOfCount,QUANT_ALL); + } + if (!vResultValid(validity)) return InvalidStop; + if (valueSpec->anyOf){ + validity = validateQuantifiedSpecs(validator,isSpeculating,value,valueSpec->anyOf,valueSpec->anyOfCount,QUANT_ANY); + } + if (!vResultValid(validity)) return InvalidStop; + if (valueSpec->oneOf){ + validity = validateQuantifiedSpecs(validator,isSpeculating,value,valueSpec->oneOf,valueSpec->oneOfCount,QUANT_ONE); + } + if (!vResultValid(validity)) return InvalidStop; + if (valueSpec->not){ + validity = validateJSON(validator,true,value,valueSpec->not); + if (vResultValid(validity)){ + noteValidityException(validator,isSpeculating,12,"negated schema at %s is valid", + validatorAccessPath(validator)); + } + validity = invertValidity(validity); + } + return validity; } } int jsonValidateSchema(JsonValidator *validator, Json *value, JsonSchema *schema){ if (setjmp(validator->recoveryData) == 0) { /* normal execution */ validator->schema = schema; - validateJSON(validator,value,schema->topValueSpec); + VResult validity = validateJSON(validator,false,value,schema->topValueSpec); printf("after validate without throw, should show validation exceptions\n"); fflush(stdout); if (validator->firstValidityException == NULL){ @@ -789,7 +996,11 @@ static JSValueSpec *makeValueSpec(JsonSchemaBuilder *builder, spec->title = title; for (int i=0; itypeMask |= (1 << typeCode); + spec->typeMask |= (1 << typeCode); + if (typeCode == JSTYPE_NUMBER){ + /* numbers allow integers, but not the converse */ + spec->typeMask |= (1 << JSTYPE_INTEGER); + } } return spec; } @@ -822,11 +1033,12 @@ static hashtable *getDefinitions(JsonSchemaBuilder *builder, JsonObject *object) } } -static JSValueSpec **getComposite(JsonSchemaBuilder *builder, JsonObject *object, char *key){ +static JSValueSpec **getComposite(JsonSchemaBuilder *builder, JsonObject *object, char *key, int *countPtr){ AccessPath *accessPath = builder->accessPath; JsonArray *array = jsonObjectGetArray(object,key); if (array != NULL){ int len = jsonArrayGetCount(array); + *countPtr = len; JSValueSpec **schemata = (JSValueSpec**)SLHAlloc(builder->slh,len*sizeof(JSValueSpec*)); accessPathPushName(accessPath,key); for (int i=0; idefinitions = getDefinitions(builder,object); - valueSpec->allOf = getComposite(builder,object,"allOf"); - valueSpec->anyOf = getComposite(builder,object,"anyOf"); - valueSpec->oneOf = getComposite(builder,object,"oneOf"); + valueSpec->allOf = getComposite(builder,object,"allOf",&valueSpec->allOfCount); + valueSpec->anyOf = getComposite(builder,object,"anyOf",&valueSpec->anyOfCount); + valueSpec->oneOf = getComposite(builder,object,"oneOf",&valueSpec->oneOfCount); Json *notSpec = jsonObjectGetPropertyValue(object,"not"); if (notSpec != NULL){ accessPathPushName(accessPath,"not"); valueSpec->not = build(builder,notSpec,false); accessPathPop(accessPath); } + Json *constValue = jsonObjectGetPropertyValue(object,"const"); + JsonArray *enumValues = getArrayValue(builder,object,"enum"); + if (constValue && enumValues){ + schemaThrow(builder,12,"cannot specify both 'const' and 'enum' for the same schema"); + } + if (constValue){ + valueSpec->constValue = jsonCopy(builder->slh,constValue); + } + if (enumValues){ + valueSpec->enumeratedValuesCount = jsonArrayGetCount(enumValues); + valueSpec->enumeratedValues = (Json**)SLHAlloc(builder->slh,valueSpec->enumeratedValuesCount*sizeof(Json*)); + for (int ee=0; eeenumeratedValuesCount; ee++){ + valueSpec->enumeratedValues[ee] = jsonCopy(builder->slh,jsonArrayGetItem(enumValues,ee)); + } + } /// valueSpec.not = getComposite(object,accessPath); + bool handledNumeric = false; for (int typeCode=0; typeCode<=JSTYPE_MAX; typeCode++){ if ((1 << typeCode) & valueSpec->typeMask){ switch (typeCode){ @@ -1025,23 +1253,25 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopL case JSTYPE_NUMBER: case JSTYPE_INTEGER: { - JSValueSpec *numericSpec = valueSpec; - // new JSNumericSpec(description,title,(jsType == JSType.INTEGER)); - numericSpec->isInteger = (typeCode == JSTYPE_INTEGER); - numericSpec->multipleOf = getNumber(builder,object,"multipleOf",MISSING_FLOAT_VALIDATOR); - numericSpec->maximum = getNumber(builder,object,"maximum",MISSING_FLOAT_VALIDATOR); - numericSpec->exclusiveMaximum = getBooleanValue(builder,object,"exclusiveMaximum",false); - numericSpec->minimum = getNumber(builder,object,"minimum",MISSING_FLOAT_VALIDATOR); - numericSpec->exclusiveMinimum = getBooleanValue(builder,object,"exclusiveMinimum",false); - if (numericSpec->maximum != MISSING_FLOAT_VALIDATOR){ - numericSpec->validatorFlags |= JS_VALIDATOR_MAX; - } - if (numericSpec->minimum != MISSING_FLOAT_VALIDATOR){ - numericSpec->validatorFlags |= JS_VALIDATOR_MIN; - } - if (numericSpec->multipleOf != MISSING_FLOAT_VALIDATOR){ - checkPositiveDouble(builder,numericSpec->multipleOf,"multipleOf"); - numericSpec->validatorFlags |= JS_VALIDATOR_MULTOF; + if (!handledNumeric){ + handledNumeric = true; + JSValueSpec *numericSpec = valueSpec; + // new JSNumericSpec(description,title,(jsType == JSType.INTEGER)); + numericSpec->multipleOf = getNumber(builder,object,"multipleOf",MISSING_FLOAT_VALIDATOR); + numericSpec->maximum = getNumber(builder,object,"maximum",MISSING_FLOAT_VALIDATOR); + numericSpec->exclusiveMaximum = getBooleanValue(builder,object,"exclusiveMaximum",false); + numericSpec->minimum = getNumber(builder,object,"minimum",MISSING_FLOAT_VALIDATOR); + numericSpec->exclusiveMinimum = getBooleanValue(builder,object,"exclusiveMinimum",false); + if (numericSpec->maximum != MISSING_FLOAT_VALIDATOR){ + numericSpec->validatorFlags |= JS_VALIDATOR_MAX; + } + if (numericSpec->minimum != MISSING_FLOAT_VALIDATOR){ + numericSpec->validatorFlags |= JS_VALIDATOR_MIN; + } + if (numericSpec->multipleOf != MISSING_FLOAT_VALIDATOR){ + checkPositiveDouble(builder,numericSpec->multipleOf,"multipleOf"); + numericSpec->validatorFlags |= JS_VALIDATOR_MULTOF; + } } } break; diff --git a/h/json.h b/h/json.h index 15d1d5009..f17a76a61 100644 --- a/h/json.h +++ b/h/json.h @@ -664,6 +664,7 @@ char* jsonBuildKey(JsonBuilder *b, const char *key, int len); #define JSON_MERGE_FLAG_TAKE_OVERRIDES 0x0004 /* len(merge) = len(a) */ Json *jsonMerge(ShortLivedHeap *slh, Json *overrides, Json *base, int flags, int *statusPtr); +Json *jsonCopy(ShortLivedHeap *slh, Json *value); /* JSON Pointers */ diff --git a/h/jsonschema.h b/h/jsonschema.h index f383aba9e..db583a008 100644 --- a/h/jsonschema.h +++ b/h/jsonschema.h @@ -63,6 +63,7 @@ typedef struct JsonValidator_tag { ValidityException *firstValidityException; ValidityException *lastValidityException; int flags; + int traceLevel; AccessPath *accessPath; jmp_buf recoveryData; } JsonValidator; diff --git a/tests/schemadata/food.schema b/tests/schemadata/food.schema new file mode 100644 index 000000000..c5fb90ba5 --- /dev/null +++ b/tests/schemadata/food.schema @@ -0,0 +1,39 @@ +{ + "name": "zowe_yaml_schema", + "$id": "http//zowe.org/schemas/yaml-2.0.json", + "type": "object", + "required": [ "fruit", "beverage" ], + "properties": { + "fruit": { + "allOf": [ + { "type": "string" }, + { "maxLength": 5 } + ] + }, + "vegetable": { + "anyOf": [ + { "type": "number", + "minimum": 13 }, + { "type": "string", + "pattern": "^c.*$"} + ] + }, + "entree": { + "const": "beef", + }, + "carbs": { + "enum": [ "potato", "rice" ], + }, + "beverage": { + "oneOf": [ + { "type": "string", + "pattern": "^.*k$"}, + { "type": "string", + "pattern": "^m.*$"} + ] + }, + "dessert": { + "not": { "type": "boolean" } + } + } +} diff --git a/tests/schemadata/food1.json b/tests/schemadata/food1.json new file mode 100644 index 000000000..60adfe1a8 --- /dev/null +++ b/tests/schemadata/food1.json @@ -0,0 +1,8 @@ +{ + "fruit": "grape", + "vegetable": 14, + "beverage": "merlot", + "entree": "beef", + "carbs": "rice", + "dessert": 3 +} diff --git a/tests/schematest.c b/tests/schematest.c index d7efd3a2a..7967628e5 100644 --- a/tests/schematest.c +++ b/tests/schematest.c @@ -118,9 +118,7 @@ int main(int argc, char *argv[]) printf("\n"); fflush(stdout); JsonSchemaBuilder *builder = makeJsonSchemaBuilder(DEFAULT_JSON_SCHEMA_VERSION); - printf("ckpt.2\n");fflush(stdout); JsonSchema *schema = jsonBuildSchema(builder,schemaJSON); - printf("ckpt.3 schema=0x%p\n",schema);fflush(stdout); if (schema){ JsonValidator *validator = makeJsonValidator(); printf("Before Validate\n");fflush(stdout); From 4cacdc7882b34fec8d8a2a5c28b65a0883ff71f1 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 21 Mar 2022 23:48:34 -0400 Subject: [PATCH 35/42] more jsonschema tests, fixes and feature gaps Signed-off-by: Joe --- c/jsonschema.c | 135 ++++++++++++++++++++++++++++------- tests/schemadata/food.schema | 3 + tests/schemadata/food1.json | 10 +-- 3 files changed, 119 insertions(+), 29 deletions(-) diff --git a/c/jsonschema.c b/c/jsonschema.c index 93b6c6362..1acdf4649 100644 --- a/c/jsonschema.c +++ b/c/jsonschema.c @@ -76,6 +76,19 @@ typedef int64_t ssize_t; #define JSTYPE_INTEGER 6 #define JSTYPE_MAX 6 +static char *getJSTypeName(int type){ + switch (type){ + case JSTYPE_NULL: return "null"; + case JSTYPE_BOOLEAN: return "boolean"; + case JSTYPE_OBJECT: return "objct"; + case JSTYPE_ARRAY: return "array"; + case JSTYPE_STRING: return "string"; + case JSTYPE_NUMBER: return "number"; + case JSTYPE_INTEGER: return "integer"; + default: return "unknown"; + } +} + #define JS_VALIDATOR_MULTOF 0x00000001 #define JS_VALIDATOR_MAX 0x00000002 #define JS_VALIDATOR_MIN 0x00000004 @@ -88,6 +101,14 @@ typedef int64_t ssize_t; #define JS_VALIDATOR_MAX_LENGTH 0x00000200 #define JS_VALIDATOR_MIN_LENGTH 0x00000400 +typedef struct PatternProperty_tag { + char *pattern; + regex_t *compiledPattern; + int compilationError; + struct JSValueSpec_tag *valueSpec; + struct PatternProperty_tag *next; +} PatternProperty; + typedef struct JSValueSpec_tag { int typeMask; int enumeratedValuesCount; @@ -119,12 +140,17 @@ typedef struct JSValueSpec_tag { int64_t maxContains; int64_t minItems; int64_t maxItems; + + /* for Objects */ int64_t minProperties; int64_t maxProperties; int requiredCount; char **required; hashtable *properties; hashtable *dependentRequired; + bool additionalProperties; + PatternProperty *firstPatternProperty; + PatternProperty *lastPatternProperty; /* for numbers */ bool exclusiveMaximum; @@ -281,13 +307,55 @@ static bool validateType(JsonValidator *validator, printf("typeCode=%d shifted=0x%x mask=0x%x\n",typeCode,(1 << typeCode),valueSpec->typeMask); } if (((1 << typeCode) & valueSpec->typeMask) == 0){ - noteValidityException(validator,isSpeculating,12,"type %d not permitted at %s",typeCode,validatorAccessPath(validator)); + noteValidityException(validator,isSpeculating,12,"type '%s' not permitted at %s", + getJSTypeName(typeCode),validatorAccessPath(validator)); return false; } else { return true; } } +static regex_t *compileRegex(char *pattern, int *regexStatus){ + regex_t *rx = regexAlloc(); + int compStatus = regexComp(rx,pattern,REG_EXTENDED); + if (compStatus){ + regexFree(rx); + *regexStatus = compStatus; + printf("*** WARNING *** pattern '%s' failed to compile with status %d\n", + pattern,compStatus); + return NULL; + } else { + *regexStatus = 0; + return rx; + } +} + +static JSValueSpec *getPropertyValueSpec(JSValueSpec *valueSpec, char *propertyName){ + JSValueSpec *propertyValueSpec = NULL; + if (valueSpec->properties){ + propertyValueSpec = (JSValueSpec*)htGet(valueSpec->properties,propertyName); + } + if (propertyValueSpec != NULL){ + return propertyValueSpec; + } + PatternProperty *patternProperty = valueSpec->firstPatternProperty; + while (patternProperty){ + if (patternProperty->compilationError == 0){ + if (patternProperty->compiledPattern == NULL){ + patternProperty->compiledPattern = compileRegex(patternProperty->pattern,&patternProperty->compilationError); + } + if (patternProperty->compiledPattern){ + int regexStatus = regexExec(patternProperty->compiledPattern,propertyName,0,NULL,0); + if (regexStatus == 0){ + return patternProperty->valueSpec; + } + } + } + patternProperty = patternProperty->next; + } + return NULL; +} + /* for mutual recursion */ static VResult validateJSON(JsonValidator *validator, bool isSpeculating, Json *value, JSValueSpec *valueSpec); @@ -322,9 +390,7 @@ static VResult validateJSONObject(JsonValidator *validator, fflush(stdout); } Json *propertyValue = jsonPropertyGetValue(property); - JSValueSpec *propertySpec = (valueSpec->properties ? - (JSValueSpec*)htGet(valueSpec->properties,propertyName) : - NULL); + JSValueSpec *propertySpec = getPropertyValueSpec(valueSpec,propertyName); accessPathPushName(accessPath,propertyName); if (propertySpec != NULL){ @@ -333,7 +399,10 @@ static VResult validateJSONObject(JsonValidator *validator, invalidCount++; } } else { - if (validator->flags && VALIDATOR_WARN_ON_UNDEFINED_PROPERTIES){ + if (valueSpec->additionalProperties == false){ + noteValidityException(validator,isSpeculating,12,"unspecified additional property not allowed: '%s' at '%s'", + propertyName,validatorAccessPath(validator)); + } else if (validator->flags && VALIDATOR_WARN_ON_UNDEFINED_PROPERTIES){ printf("*WARNING* unspecified property seen, '%s', and checking code is not complete, vspec->props=0x%p\n", propertyName,valueSpec->properties); fflush(stdout); @@ -432,6 +501,7 @@ static VResult validateJSONArray(JsonValidator *validator, return (invalidCount > 0 ? InvalidContinue : ValidContinue); } + static VResult validateJSONString(JsonValidator *validator, bool isSpeculating, Json *string, @@ -457,16 +527,7 @@ static VResult validateJSONString(JsonValidator *validator, } if (valueSpec->pattern && (valueSpec->regexCompilationError == 0)){ if (valueSpec->compiledPattern == NULL){ - regex_t *rx = regexAlloc(); - int compStatus = regexComp(rx,valueSpec->pattern,REG_EXTENDED); - if (compStatus){ - regexFree(rx); - valueSpec->regexCompilationError = compStatus; - printf("*** WARNING *** pattern '%s' failed to compile with status %d\n", - valueSpec->pattern,compStatus); - } else { - valueSpec->compiledPattern = rx; - } + valueSpec->compiledPattern = compileRegex(valueSpec->pattern,&valueSpec->regexCompilationError); } if (valueSpec->compiledPattern){ int regexStatus = regexExec(valueSpec->compiledPattern,s,0,NULL,0); @@ -477,9 +538,6 @@ static VResult validateJSONString(JsonValidator *validator, } } } - // String pattern; // ECMA-262 regex - // prefix items - // additionalProperties return (invalidCount > 0 ? InvalidContinue : ValidContinue); } @@ -590,7 +648,6 @@ static VResult validateJSONSimple(JsonValidator *validator, bool isSpeculating, return (validateType(validator,isSpeculating,JSTYPE_NULL,valueSpec) ? ValidContinue : InvalidStop); } else if (jsonIsBoolean(value)){ - printf("bool case\n"); return (validateType(validator,isSpeculating,JSTYPE_BOOLEAN,valueSpec) ? ValidContinue : InvalidStop); } else if (jsonIsObject(value)){ @@ -679,12 +736,6 @@ static VResult validateQuantifiedSpecs(JsonValidator *validator, } } -/* here, - 1) patProps - 2) add'l props - commit -*/ - static bool jsonEquals(Json *a, Json *b){ if (jsonIsArray(a) || jsonIsArray(b)){ return false; @@ -1033,6 +1084,38 @@ static hashtable *getDefinitions(JsonSchemaBuilder *builder, JsonObject *object) } } +static void addPatternProperties(JsonSchemaBuilder *builder, JSValueSpec *valueSpec, JsonObject *object){ + JsonObject *patternObject = getObjectValue(builder,object,"patternProperties"); + if (patternObject != NULL){ + AccessPath *accessPath = builder->accessPath; + accessPathPushName(accessPath,"patternProperties"); + JsonProperty *property = jsonObjectGetFirstProperty(patternObject); + while (property){ + char *propertyName = jsonPropertyGetKey(property); + Json *propertyValue = jsonPropertyGetValue(property); + if (propertyValue == NULL){ + schemaThrow(builder,12,"patternProperty %s is NULL, which is not allowed",propertyName); + return; + } + accessPathPushName(accessPath,propertyName); + PatternProperty *patternProperty = (PatternProperty*)SLHAlloc(builder->slh,sizeof(PatternProperty)); + if (valueSpec->firstPatternProperty == NULL){ + valueSpec->firstPatternProperty = patternProperty; + valueSpec->lastPatternProperty = patternProperty; + } else { + valueSpec->lastPatternProperty->next = patternProperty; + valueSpec->lastPatternProperty = patternProperty; + } + memset(patternProperty,0,sizeof(PatternProperty)); + patternProperty->pattern = propertyName; + patternProperty->valueSpec = build(builder,propertyValue,false); + accessPathPop(accessPath); + property = jsonObjectGetNextProperty(property); + } + accessPathPop(accessPath); + } +} + static JSValueSpec **getComposite(JsonSchemaBuilder *builder, JsonObject *object, char *key, int *countPtr){ AccessPath *accessPath = builder->accessPath; JsonArray *array = jsonObjectGetArray(object,key); @@ -1112,6 +1195,8 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopL return valueSpec; } valueSpec->definitions = getDefinitions(builder,object); + valueSpec->additionalProperties = getBooleanValue(builder,object,"additionalProperties",true); + addPatternProperties(builder,valueSpec,object); valueSpec->allOf = getComposite(builder,object,"allOf",&valueSpec->allOfCount); valueSpec->anyOf = getComposite(builder,object,"anyOf",&valueSpec->anyOfCount); valueSpec->oneOf = getComposite(builder,object,"oneOf",&valueSpec->oneOfCount); diff --git a/tests/schemadata/food.schema b/tests/schemadata/food.schema index c5fb90ba5..1fb95f443 100644 --- a/tests/schemadata/food.schema +++ b/tests/schemadata/food.schema @@ -3,6 +3,9 @@ "$id": "http//zowe.org/schemas/yaml-2.0.json", "type": "object", "required": [ "fruit", "beverage" ], + "additionalProperties": false, + "patternProperties": { + "^course.$": { "type": "number"}}, "properties": { "fruit": { "allOf": [ diff --git a/tests/schemadata/food1.json b/tests/schemadata/food1.json index 60adfe1a8..b033f7bc9 100644 --- a/tests/schemadata/food1.json +++ b/tests/schemadata/food1.json @@ -1,8 +1,10 @@ { "fruit": "grape", - "vegetable": 14, - "beverage": "merlot", - "entree": "beef", + "vegetable": 33, + "beverage": "millstone", "carbs": "rice", - "dessert": 3 + "dessert": 3, + "entree": "beef", + "course1": "cheese", + } From 412bc6f2d8884bf1bf5dc5bdc55572072039a097 Mon Sep 17 00:00:00 2001 From: Joe Date: Tue, 22 Mar 2022 12:26:31 -0400 Subject: [PATCH 36/42] More fixes driven by Irek's PR review Signed-off-by: Joe --- c/json.c | 33 +++++++++++++++++++++++++++------ c/xlate.c | 4 ++++ c/zosfile.c | 2 -- h/json.h | 1 + h/xlate.h | 4 ++++ 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/c/json.c b/c/json.c index 395c8103c..ce5b2cf7b 100644 --- a/c/json.c +++ b/c/json.c @@ -2671,7 +2671,7 @@ static int mergeJson1(JsonMerger *merger, Json *parent, char *parentKey, Json *o copyJson(builder,merged,NULL,jsonArrayGetItem(baseArray,i)); } return JSON_MERGE_STATUS_SUCCESS; - case JSON_MERGE_FLAG_TAKE_OVERRIDES: /* len(merge) = len(a) */ + case JSON_MERGE_FLAG_TAKE_OVERRIDES: /* len(merge) = len(o) */ default: for (i=0; ielements; + for (int i=0; isize; i++){ + JsonPointerElement *element = (JsonPointerElement*)arrayListElement(list,i); + safeFree((char*)element,sizeof(JsonPointerElement)); + } + safeFree((char*)jp,sizeof(JsonPointerElement)); } void printJsonPointer(FILE *out, JsonPointer *jp){ diff --git a/c/xlate.c b/c/xlate.c index 1bc449aff..068c6a267 100644 --- a/c/xlate.c +++ b/c/xlate.c @@ -79,6 +79,10 @@ char* a2e(char *buffer, int len) return buffer; } +/** A Charset oracle is a string digester that can guess charsets + based upon what it is fed. It's heuristic, so YMMV. +*/ + CharsetOracle *makeCharsetOracle(void){ CharsetOracle *oracle = (CharsetOracle*)safeMalloc(sizeof(CharsetOracle),"CharsetOracle"); diff --git a/c/zosfile.c b/c/zosfile.c index 16b4819b7..44bb0467a 100644 --- a/c/zosfile.c +++ b/c/zosfile.c @@ -785,7 +785,6 @@ int fileInfo(const char *filename, BPXYSTAT *stats, int *returnCode, int *reason reasonCodePtr = reasonCode; #endif - printf("BPXSTA on filename\n"); dumpbuffer(filename,strlen(filename)); BPXSTA(&nameLength, filename, @@ -794,7 +793,6 @@ int fileInfo(const char *filename, BPXYSTAT *stats, int *returnCode, int *reason &returnValue, returnCode, reasonCodePtr); - printf("return ret=%d reason=%d\n",*returnCode,*reasonCode); if (fileTrace) { if(returnValue != 0) { diff --git a/h/json.h b/h/json.h index f17a76a61..1e413f251 100644 --- a/h/json.h +++ b/h/json.h @@ -683,6 +683,7 @@ typedef struct JsonPointer_tag { } JsonPointer; JsonPointer *parseJsonPointer(char *s); +void freeJsonPointer(JsonPointer *jp); void printJsonPointer(FILE *out, JsonPointer *jp); #endif /* __JSON__ */ diff --git a/h/xlate.h b/h/xlate.h index 3a0d15a36..0adab9ce7 100644 --- a/h/xlate.h +++ b/h/xlate.h @@ -80,6 +80,10 @@ typedef struct CharsetOracle_tag { #define CHARSET_ORACLE_EBCDIC_FAMILY 0x10000 #define CHARSET_ORACLE_UTF_FAMILY 0x20000 +/** A Charset oracle is a string digester that can guess charsets + based upon what it is fed. It's heuristic, so YMMV. +*/ + CharsetOracle *makeCharsetOracle(void); void freeCharsetOracle(CharsetOracle *oracle); From c2222bd2cc4d5499b7d4fc32b074c0dc32ab87c3 Mon Sep 17 00:00:00 2001 From: Joe Date: Thu, 24 Mar 2022 00:28:43 -0400 Subject: [PATCH 37/42] Adding support for inter-schema references and anchors Signed-off-by: Joe --- c/jsonschema.c | 260 ++++++++++++++++++++++++++++--- h/jsonschema.h | 15 ++ tests/schemadata/bundle1.json | 6 + tests/schemadata/food1.json | 2 +- tests/schemadata/zowebundle.json | 66 ++++++++ tests/schematest.c | 10 +- 6 files changed, 331 insertions(+), 28 deletions(-) create mode 100644 tests/schemadata/bundle1.json create mode 100644 tests/schemadata/zowebundle.json diff --git a/c/jsonschema.c b/c/jsonschema.c index 1acdf4649..5d5afc4a8 100644 --- a/c/jsonschema.c +++ b/c/jsonschema.c @@ -115,8 +115,11 @@ typedef struct JSValueSpec_tag { Json **enumeratedValues; Json *constValue; char *id; + char *hostName; /* from ID if set */ + char *fileName; /* from ID if set */ char *description; char *title; + char *anchor; char *ref; bool deprecated; bool nullable; @@ -133,6 +136,10 @@ typedef struct JSValueSpec_tag { /* optional validators have flag for presence */ uint64_t validatorFlags; + /* only filled for top level JSValueSpecs, ie no parent, or $id is present */ + hashtable *anchorTable; + /* everything with an ID under the top level */ + hashtable *idTable; double minimum; double maximum; @@ -168,6 +175,9 @@ typedef struct JSValueSpec_tag { char *pattern; regex_t *compiledPattern; int regexCompilationError; + + /* back/up pointer */ + struct JSValueSpec_tag *parent; } JSValueSpec; typedef struct AccessPathUnion_tag { @@ -279,6 +289,26 @@ static char *validatorAccessPath(JsonValidator *validator){ return makeAccessPathString(validator->accessPathBuffer,MAX_ACCESS_PATH,validator->accessPath); } +static JSValueSpec *getTopLevelAncestor(JSValueSpec *valueSpec){ + JSValueSpec *ancestor = valueSpec; + while (ancestor->parent){ + if (ancestor->id != NULL){ + return ancestor; + } + ancestor = ancestor->parent; + } + return ancestor; +} + +static JSValueSpec *getOutermostAncestor(JSValueSpec *valueSpec){ + JSValueSpec *ancestor = valueSpec; + while (ancestor->parent){ + ancestor = ancestor->parent; + } + return ancestor; +} + + /* The return types of these validators is bool to allow validator to signal whether to continue to gather more validation exceptions in this part of the JSON tree/graph. Returning false says this part is invalid enough such that further evaluation would probably confuse the user with @@ -611,12 +641,93 @@ static VResult validateJSONInteger(JsonValidator *validator, return (invalidCount > 0 ? InvalidContinue : ValidContinue); } +static char *httpPattern = "^(https?://[^/]+)/([^#]*)(#.*)?$"; +static char *filePattern = "^/([^#]*)(#.*)?$"; + +static int httpMatch(JsonValidator *v, char *s){ + if (v->httpRegexError){ + return v->httpRegexError; + } else if (v->httpRegex == NULL){ + v->httpRegex = compileRegex(httpPattern,&v->httpRegexError); + } + if (v->httpRegex){ + return regexExec(v->httpRegex,s,MAX_VALIDATOR_MATCHES,v->matches,0); + } else { + return v->httpRegexError; + } +} + +static int fileMatch(JsonValidator *v, char *s){ + if (v->fileRegexError){ + return v->fileRegexError; + } else if (v->fileRegex == NULL){ + v->fileRegex = compileRegex(filePattern,&v->fileRegexError); + } + if (v->fileRegex){ + return regexExec(v->fileRegex,s,MAX_VALIDATOR_MATCHES,v->matches,0); + } else { + return v->fileRegexError; + } +} + +static JSValueSpec *getTopSchemaByName(JsonValidator *validator, + char *hostName, int hostNameLength, + char *fileName, int fileNameLength){ + int len = hostNameLength+fileNameLength+8; + char *key = safeMalloc(len,"topSchemaKey"); + snprintf(key,len,"%*.*s/%*.*s", + hostNameLength,hostNameLength,hostName, + fileNameLength,fileNameLength,fileName); + JSValueSpec *schema = htGet(validator->schema->topValueSpec->definitions,key); + if (validator->traceLevel >= 1){ + printf("getTopSchema for key='%s' yields 0x%p\n",key,schema); + } + safeFree(key,len); + return schema; +} + +static int matchLen(regmatch_t *match){ + return (int)(match->rm_eo - match->rm_so); +} + +static JSValueSpec *resolveCrossSchemaRef(JsonValidator *validator, bool isSpeculating, + JSValueSpec *referredTopSchema, char *ref, regmatch_t *anchorMatch){ + if (anchorMatch->rm_so == -1){ + return referredTopSchema; + } else if (referredTopSchema->anchorTable == NULL){ + noteValidityException(validator,isSpeculating,12,"schema '%s' does not define any anchors for ref '%s'", + (referredTopSchema->id ? referredTopSchema->id : ""), + ref); + return NULL; + } else { + char *anchor = ref+anchorMatch->rm_so+1; /* +1 to skip the '#' char */ + JSValueSpec *spec = (JSValueSpec*)htGet(referredTopSchema->anchorTable,anchor); + if (validator->traceLevel >= 1){ + printf("anchor '%s' resolves to 0x%p\n",anchor,spec); + } + if (spec == NULL){ + noteValidityException(validator,isSpeculating,12,"anchor schema ref '%s' does not resolve in referred schema",ref); + } + return spec; + } +} + + static JSValueSpec *resolveRef(JsonValidator *validator, bool isSpeculating, JSValueSpec *valueSpec){ char *ref = valueSpec->ref; + int refLen = strlen(ref); + int matchStatus = 0; if (validator->traceLevel >= 1){ printf("resolveRef ref=%s\n",valueSpec->ref); } - if (!memcmp(ref,"#/$defs/",8)){ /* local resolution */ + JSValueSpec *containingSpec = getTopLevelAncestor(valueSpec); + if (containingSpec == NULL){ /* this code is wrong if topLevel (parent chain) is no good */ + printf("*** PANIC *** no top level\n"); + return NULL; + } + + /* Case 1 local reference w/o anchor */ + if (refLen >= 8 && !memcmp(ref,"#/$defs/",8)){ /* local resolution */ int refLen = strlen(ref); int pos = 2; char *key = ref+8; @@ -628,16 +739,60 @@ static JSValueSpec *resolveRef(JsonValidator *validator, bool isSpeculating, JSV } JSValueSpec *resolvedSchema = htGet(topSchema->definitions,key); if (resolvedSchema == NULL){ - noteValidityException(validator,isSpeculating,12,"schema ref '%s' does not resolve against '$defs'",ref); + noteValidityException(validator,isSpeculating,12,"path schema ref '%s' does not resolve against '$defs'",ref); return NULL; + } else { + return resolvedSchema; } - return resolvedSchema; + /* Case 2, global URL reference */ + } else if (refLen >= 10 && ((matchStatus = httpMatch(validator,ref)) == 0)){ + regmatch_t domainMatch = validator->matches[1]; + regmatch_t fileMatch = validator->matches[2]; + regmatch_t anchorMatch = validator->matches[3]; + JSValueSpec *referredTopSchema = getTopSchemaByName(validator, + ref+domainMatch.rm_so,matchLen(&domainMatch), + ref+fileMatch.rm_so,matchLen(&fileMatch)); + if (referredTopSchema == NULL){ + noteValidityException(validator,isSpeculating,12,"cross-domain schema not found for ref '%s'",ref); + return NULL; + } else { + return resolveCrossSchemaRef(validator,isSpeculating,referredTopSchema,ref,&anchorMatch); + } + /* Case 3, same-domain URL fragment (ie file) reference */ + } else if (refLen >= 1 && ((matchStatus = fileMatch(validator,ref)) == 0)){ + regmatch_t fileMatch = validator->matches[1]; + regmatch_t anchorMatch = validator->matches[2]; + JSValueSpec *referredTopSchema = getTopSchemaByName(validator, + containingSpec->hostName,strlen(containingSpec->hostName), + ref+fileMatch.rm_so,matchLen(&fileMatch)); + if (referredTopSchema == NULL){ + noteValidityException(validator,isSpeculating,12,"same-domain schema not found for ref '%s'",ref); + return NULL; + } else { + return resolveCrossSchemaRef(validator,isSpeculating,referredTopSchema,ref,&anchorMatch); + } + /* Case 4, just an anchor in same schema */ + } else if (refLen >= 1 && ref[0] == '#'){ + if (containingSpec->anchorTable == NULL){ + noteValidityException(validator,isSpeculating,12,"schema '%s' does not define any anchors for ref '%s'", + (containingSpec->id ? containingSpec->id : ""), + ref); + return NULL; + } + JSValueSpec *resolvedSchema = (JSValueSpec*)htGet(containingSpec->anchorTable,ref+1); + if (resolvedSchema == NULL){ + noteValidityException(validator,isSpeculating,12,"anchor schema ref '%s' does not resolve in containing schema",ref); + return NULL; + } else { + return resolvedSchema; + } + /* Other cases not known or supported */ } else { /* notes, need to merge URL according W3 rules and pass to pluggable resolver that we hope this validator has. */ - noteValidityException(validator,isSpeculating,12,"external refs not yet implemented '%s' at %s", + noteValidityException(validator,isSpeculating,12,"bad or unimplemented ref at '%s' at %s", valueSpec->ref,validatorAccessPath(validator)); return NULL; } @@ -701,7 +856,7 @@ static VResult validateQuantifiedSpecs(JsonValidator *validator, switch (quantType){ case QUANT_ALL: if (!eltValid){ - noteValidityException(validator,isSpeculating,12,"not allOf schemas at %s are valid", + noteValidityException(validator,isSpeculating,12,"not allOf schemas at '%s' are valid", validatorAccessPath(validator)); return InvalidStop; } @@ -712,7 +867,7 @@ static VResult validateQuantifiedSpecs(JsonValidator *validator, } case QUANT_ONE: if (validCount > 1){ - noteValidityException(validator,isSpeculating,12,"more than oneOf schemas at %s are valid", + noteValidityException(validator,isSpeculating,12,"more than oneOf schemas at '%s' are valid", validatorAccessPath(validator)); return InvalidStop; } @@ -723,7 +878,7 @@ static VResult validateQuantifiedSpecs(JsonValidator *validator, case QUANT_ALL: return ValidContinue; case QUANT_ANY: - noteValidityException(validator,isSpeculating,12,"not anyOf schemas at %s are valid", + noteValidityException(validator,isSpeculating,12,"not anyOf schemas at '%s' are valid", validatorAccessPath(validator)); return ValidContinue; @@ -1032,9 +1187,39 @@ static char *getID(JsonSchemaBuilder *builder, JsonObject *o) { } /* forwarding for complex recursion */ -static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopLevel); +static JSValueSpec *build(JsonSchemaBuilder *builder, JSValueSpec *parent, Json *jsValue, bool isTopLevel); + +static void indexByAnchor(JSValueSpec *valueSpec){ + JSValueSpec *topLevelAncestor = getTopLevelAncestor(valueSpec); + if (topLevelAncestor->anchorTable == NULL){ + topLevelAncestor->anchorTable = htCreate(257,stringHash,stringCompare,NULL,NULL); + } + htPut(topLevelAncestor->anchorTable,valueSpec->anchor,valueSpec); +} + +static void indexByID(JSValueSpec *valueSpec){ + JSValueSpec *outermostAncestor = getOutermostAncestor(valueSpec); + if (outermostAncestor->idTable == NULL){ + outermostAncestor->idTable = htCreate(257,stringHash,stringCompare,NULL,NULL); + } + htPut(outermostAncestor->idTable,valueSpec->id,valueSpec); +} + +static char *extractString(JsonSchemaBuilder *b, char *s, regmatch_t *match){ + if (match->rm_so != -1){ + int len = (int)(match->rm_eo - match->rm_so); + char *copy = SLHAlloc(b->slh,len+1); + memcpy(copy,s+match->rm_so,len); + copy[len] = 0; + return copy; + } else { + return NULL; + } +} + static JSValueSpec *makeValueSpec(JsonSchemaBuilder *builder, + JSValueSpec *parent, char *id, char *description, char *title, @@ -1045,6 +1230,28 @@ static JSValueSpec *makeValueSpec(JsonSchemaBuilder *builder, spec->id = id; spec->description = description; spec->title = title; + spec->parent = parent; + /* parent must be set */ + if (id){ + indexByID(spec); + regmatch_t matches[10]; + + int regexStatus = 0; + regex_t *regex = compileRegex(httpPattern,®exStatus); + if (regex){ + int matchStatus = regexExec(regex,id,10,matches,0); + if (matchStatus){ + fprintf(stderr,"id does not look like a URL: '%s', regexMatchStatus=%d\n",id,matchStatus); + } else { + spec->hostName = extractString(builder,id,&matches[1]); + spec->fileName = extractString(builder,id,&matches[2]); + printf("top schema found with ID=%s, host='%s' file='%s'\n", + id,spec->hostName,spec->fileName); + } + regexFree(regex); + } + + } for (int i=0; itypeMask |= (1 << typeCode); @@ -1058,7 +1265,7 @@ static JSValueSpec *makeValueSpec(JsonSchemaBuilder *builder, /* Map */ -static hashtable *getDefinitions(JsonSchemaBuilder *builder, JsonObject *object){ +static hashtable *getDefinitions(JsonSchemaBuilder *builder, JSValueSpec *parent, JsonObject *object){ AccessPath *accessPath = builder->accessPath; JsonObject *definitionsObject = getObjectValue(builder,object,"$defs"); if (definitionsObject != NULL){ @@ -1073,7 +1280,7 @@ static hashtable *getDefinitions(JsonSchemaBuilder *builder, JsonObject *object) return NULL; } accessPathPushName(accessPath,propertyName); - htPut(definitionMap,propertyName,build(builder,propertyValue,false)); + htPut(definitionMap,propertyName,build(builder,parent,propertyValue,false)); accessPathPop(accessPath); property = jsonObjectGetNextProperty(property); } @@ -1108,7 +1315,7 @@ static void addPatternProperties(JsonSchemaBuilder *builder, JSValueSpec *valueS } memset(patternProperty,0,sizeof(PatternProperty)); patternProperty->pattern = propertyName; - patternProperty->valueSpec = build(builder,propertyValue,false); + patternProperty->valueSpec = build(builder,valueSpec,propertyValue,false); accessPathPop(accessPath); property = jsonObjectGetNextProperty(property); } @@ -1116,7 +1323,8 @@ static void addPatternProperties(JsonSchemaBuilder *builder, JSValueSpec *valueS } } -static JSValueSpec **getComposite(JsonSchemaBuilder *builder, JsonObject *object, char *key, int *countPtr){ +static JSValueSpec **getComposite(JsonSchemaBuilder *builder, JSValueSpec *parent, + JsonObject *object, char *key, int *countPtr){ AccessPath *accessPath = builder->accessPath; JsonArray *array = jsonObjectGetArray(object,key); if (array != NULL){ @@ -1126,7 +1334,7 @@ static JSValueSpec **getComposite(JsonSchemaBuilder *builder, JsonObject *object accessPathPushName(accessPath,key); for (int i=0; iaccessPath; if (builder->traceLevel >= 2){ printf("JSONSchema build\n"); @@ -1177,9 +1385,13 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopL char *description = getString(builder,object,"description",NULL); char *title = getString(builder,object,"title",NULL); - - JSValueSpec *valueSpec = makeValueSpec(builder,getID(builder,object),description,title,typeArray,typeArrayCount); + char *id = getID(builder,object); + JSValueSpec *valueSpec = makeValueSpec(builder,parent,id,description,title,typeArray,typeArrayCount); valueSpec->ref = getString(builder,object,"$ref",NULL); + valueSpec->anchor = getString(builder,object,"$anchor",NULL); + if (valueSpec->anchor){ + indexByAnchor(valueSpec); + } if (valueSpec->ref){ /* THe JSON Schema spec says that if a $ref is defined it will be the only thing looked at, so we short-cicuit here. @@ -1194,16 +1406,16 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopL */ return valueSpec; } - valueSpec->definitions = getDefinitions(builder,object); + valueSpec->definitions = getDefinitions(builder,valueSpec,object); valueSpec->additionalProperties = getBooleanValue(builder,object,"additionalProperties",true); addPatternProperties(builder,valueSpec,object); - valueSpec->allOf = getComposite(builder,object,"allOf",&valueSpec->allOfCount); - valueSpec->anyOf = getComposite(builder,object,"anyOf",&valueSpec->anyOfCount); - valueSpec->oneOf = getComposite(builder,object,"oneOf",&valueSpec->oneOfCount); + valueSpec->allOf = getComposite(builder,valueSpec,object,"allOf",&valueSpec->allOfCount); + valueSpec->anyOf = getComposite(builder,valueSpec,object,"anyOf",&valueSpec->anyOfCount); + valueSpec->oneOf = getComposite(builder,valueSpec,object,"oneOf",&valueSpec->oneOfCount); Json *notSpec = jsonObjectGetPropertyValue(object,"not"); if (notSpec != NULL){ accessPathPushName(accessPath,"not"); - valueSpec->not = build(builder,notSpec,false); + valueSpec->not = build(builder,valueSpec,notSpec,false); accessPathPop(accessPath); } Json *constValue = jsonObjectGetPropertyValue(object,"const"); @@ -1257,7 +1469,7 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopL if (valueSpec->properties == NULL){ valueSpec->properties = htCreate(101,stringHash,stringCompare,NULL,NULL); } - htPut(valueSpec->properties,propertyName,build(builder,propertyValue,false)); + htPut(valueSpec->properties,propertyName,build(builder,valueSpec,propertyValue,false)); accessPathPop(accessPath); property = jsonObjectGetNextProperty(property); } @@ -1302,7 +1514,7 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, Json *jsValue, bool isTopL Json *items = jsonObjectGetPropertyValue(object,"items"); if (items != NULL){ accessPathPushName(accessPath,"items"); - arraySpec->itemSpec = build(builder,items,false); + arraySpec->itemSpec = build(builder,valueSpec,items,false); accessPathPop(accessPath); } else { // what does this default to or error @@ -1397,7 +1609,7 @@ JsonSchema *jsonBuildSchema(JsonSchemaBuilder *builder, Json *jsValue){ printf("after setjmp normal\n"); fflush(stdout); } - JSValueSpec *topValueSpec = build(builder,jsValue,true); + JSValueSpec *topValueSpec = build(builder,NULL,jsValue,true); JsonSchema *schema = (JsonSchema*)SLHAlloc(builder->slh,sizeof(JsonSchema)); schema->topValueSpec = topValueSpec; schema->version = builder->version; diff --git a/h/jsonschema.h b/h/jsonschema.h index db583a008..198cd2623 100644 --- a/h/jsonschema.h +++ b/h/jsonschema.h @@ -7,6 +7,13 @@ #include #endif +#ifdef __ZOWE_OS_WINDOWS +#include "winregex.h" +#else +#include "psxregex.h" +#endif + + typedef struct JsonSchema_tag { ShortLivedHeap *slh; int version; @@ -54,6 +61,8 @@ typedef struct ValidityException_tag { #define VALIDATOR_WARN_ON_UNDEFINED_PROPERTIES 0x0001 +#define MAX_VALIDATOR_MATCHES 8 + typedef struct JsonValidator_tag { JsonSchema *schema; int errorCode; @@ -65,6 +74,12 @@ typedef struct JsonValidator_tag { int flags; int traceLevel; AccessPath *accessPath; + /* we use these pattern many times and manage the resources from here */ + regmatch_t matches[MAX_VALIDATOR_MATCHES]; + regex_t *httpRegex; + int httpRegexError; + regex_t *fileRegex; + int fileRegexError; jmp_buf recoveryData; } JsonValidator; diff --git a/tests/schemadata/bundle1.json b/tests/schemadata/bundle1.json new file mode 100644 index 000000000..0c593f681 --- /dev/null +++ b/tests/schemadata/bundle1.json @@ -0,0 +1,6 @@ +{ + "A": "apple", + "listenerPort": 34444, + "BB": "FOO", + "Z": "60609" +} diff --git a/tests/schemadata/food1.json b/tests/schemadata/food1.json index b033f7bc9..a11ce5d57 100644 --- a/tests/schemadata/food1.json +++ b/tests/schemadata/food1.json @@ -5,6 +5,6 @@ "carbs": "rice", "dessert": 3, "entree": "beef", - "course1": "cheese", + "course1": 333, } diff --git a/tests/schemadata/zowebundle.json b/tests/schemadata/zowebundle.json new file mode 100644 index 000000000..a84d3e320 --- /dev/null +++ b/tests/schemadata/zowebundle.json @@ -0,0 +1,66 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09", + "$id": "https://zarf.org/schema/appserver", + "allOf": [ + { + "$ref": "https://zowe.org/schema/base" + }, + { + "$ref": "https://zowe.org/schema/common#zoweComponent" + }, + { + "type": "object", + "required": [ + "BB" + ], + "properties": { + "BB": { + "const": "FOO" + } + } + } + ], + "$defs": { + "https://zowe.org/schema/base": { + "$schema": "https://json-schema.org/draft/2019-09", + "$id": "https://zowe.org/schema/base", + "type": "object", + "properties": { + "A": { + "type": "string" + }, + "Z": { + "$ref": "/schema/common#zoweZip" + } + } + }, + "https://zowe.org/schema/common": { + "$schema": "https://json-schema.org/draft/2019-09", + "$id": "https://zowe.org/schema/common", + "$defs": { + "component": { + "type": "object", + "$anchor": "zoweComponent", + "required": [ + "listenerPort" + ], + "properties": { + "listenerPort": { + "$ref": "#zowePort" + } + } + }, + "port": { + "$anchor": "zowePort", + "type": "integer", + "maximum": 65535 + }, + "zipCode": { + "$anchor": "zoweZip", + "type": "string", + "pattern": "^[0-9]{5}$" + } + } + } + } +} diff --git a/tests/schematest.c b/tests/schematest.c index 7967628e5..518126d3c 100644 --- a/tests/schematest.c +++ b/tests/schematest.c @@ -84,7 +84,9 @@ int main(int argc, char *argv[]) if (!strcmp(syntax,"json")){ memset(errorBuffer,0,errorBufferSize); json = jsonParseFile2(slh,filename,errorBuffer,errorBufferSize); - printf("json parsed json=0x%p, with error '%s'\n",json,errorBuffer); + if (json == NULL){ + printf("json parsed json=0x%p, with error '%s'\n",json,errorBuffer); + } } else if (!strcmp(syntax,"yaml")){ printf("yaml syntax case\n"); @@ -110,8 +112,10 @@ int main(int argc, char *argv[]) if (schemaFilename){ memset(errorBuffer,0,errorBufferSize); Json *schemaJSON = jsonParseFile2(slh,schemaFilename,errorBuffer,errorBufferSize); - printf("json parsed schema json=0x%p, with error '%s'\n",json,errorBuffer); - fflush(stdout); + if (schemaJSON == NULL){ + printf("json parsed schema json=0x%p, with error '%s'\n",json,errorBuffer); + fflush(stdout); + } if (schemaJSON){ printf("Regurgitating JSON Schema before digesting it for real\n"); jsonPrint(p,schemaJSON); From 8d62f033ba3e6508063cbcc0512b372824f7ce1d Mon Sep 17 00:00:00 2001 From: Joe Date: Fri, 25 Mar 2022 17:41:30 -0400 Subject: [PATCH 38/42] Updating configmgr to use multiple schemas, and fixing a bug or two that was seen, and adding another test case Signed-off-by: Joe --- c/configmgr.c | 135 ++++++++++++------ c/jsonschema.c | 40 ++++-- h/jsonschema.h | 19 ++- tests/schemadata/bundle1.json | 4 +- tests/schemadata/zoweappserver.json | 23 +++ tests/schemadata/zowebase.json | 14 ++ tests/schemadata/zowecommon.json | 28 ++++ .../{zowebase.yaml => zowedefault.yaml} | 0 tests/schematest.c | 5 +- 9 files changed, 200 insertions(+), 68 deletions(-) create mode 100644 tests/schemadata/zoweappserver.json create mode 100644 tests/schemadata/zowebase.json create mode 100644 tests/schemadata/zowecommon.json rename tests/schemadata/{zowebase.yaml => zowedefault.yaml} (100%) diff --git a/c/configmgr.c b/c/configmgr.c index c34573eae..4c6f226a5 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -106,7 +106,9 @@ typedef int64_t ssize_t; configmgr -s "../tests/schemadata" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowebase.yaml)" validate - configmgr -t 2 -s "../tests/schemadata" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowebase.yaml)" validate + configmgr -t 2 -s "../tests/schemadata/zoweyaml.schema" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowebase.yaml)" validate + + configmgr -t 1 -s "../tests/schemadata/zoweappserver.json:../tests/schemadata/zowebase.json:../tests/schemadata/zowecommon.json" -p "FILE(../tests/schemadata/bundle1.json)" validate configmgr -s "../tests/schemadata" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowebase.yaml)" extract "/zowe/setup/mvs/proclib" @@ -166,7 +168,9 @@ typedef struct ConfigManager_tag { char *rootSchemaDirectory; ConfigPathElement *schemaPath; /* maybe */ ConfigPathElement *configPath; - JsonSchema *topSchema; + JsonSchema *topSchema; + JsonSchema **otherSchemas; + int otherSchemasCount; Json *config; hashtable *schemaCache; hashtable *configCache; @@ -444,43 +448,24 @@ static void jsonPrettyPrint(ConfigManager *mgr, Json *json){ } -#define ZOWE_SCHEMA_FILE "zoweyaml.schema" +/* #define ZOWE_SCHEMA_FILE "zoweyaml.schema" */ void freeConfigManager(ConfigManager *mgr); -ConfigManager *makeConfigManager(char *configPathArg, char *rootSchemaDirectory, - int traceLevel, FILE *traceOut){ - ConfigManager *mgr = (ConfigManager*)safeMalloc(sizeof(ConfigManager),"ConfigManager"); - - memset(mgr,0,sizeof(ConfigManager)); - mgr->traceLevel = traceLevel; - mgr->traceOut = traceOut; - mgr->slh = makeShortLivedHeap(0x10000,0x100); - EmbeddedJS *ejs = makeEmbeddedJS(NULL); - mgr->ejs = ejs; - trace(mgr,DEBUG,"before build config path\n"); - if (buildConfigPath(mgr,configPathArg)){ - printf("built config path failed\n");fflush(stdout); - safeFree((char*)mgr,sizeof(ConfigManager)); - return NULL; - } - trace(mgr,DEBUG,"built config path\n"); - - mgr->rootSchemaDirectory = rootSchemaDirectory; - char *zoweYamlPath = makeFullPath(mgr,rootSchemaDirectory,ZOWE_SCHEMA_FILE,true); +static JsonSchema *loadOneSchema(ConfigManager *mgr, char *schemaFilePath){ int returnCode = 0; int reasonCode = 0; FileInfo yamlFileInfo; trace(mgr,DEBUG,"before file info\n"); - int infoStatus = fileInfo(zoweYamlPath,&yamlFileInfo,&returnCode,&reasonCode); + int infoStatus = fileInfo(schemaFilePath,&yamlFileInfo,&returnCode,&reasonCode); if (infoStatus){ - trace(mgr,INFO,"failed to get fileInfo of '%s', infoStatus=%d\n",zoweYamlPath,infoStatus); + trace(mgr,INFO,"failed to get fileInfo of '%s', infoStatus=%d\n",schemaFilePath,infoStatus); freeConfigManager(mgr); return NULL; } char errorBuffer[1024]; trace(mgr,DEBUG,"before jsonParseFile info\n"); - Json *jsonWithSchema = jsonParseFile2(mgr->slh,zoweYamlPath,errorBuffer,1024); + Json *jsonWithSchema = jsonParseFile2(mgr->slh,schemaFilePath,errorBuffer,1024); if (jsonWithSchema == NULL){ trace(mgr,INFO,"failed to read JSON with base schema: %s\n",errorBuffer); freeConfigManager(mgr); @@ -491,17 +476,76 @@ ConfigManager *makeConfigManager(char *configPathArg, char *rootSchemaDirectory, } JsonSchemaBuilder *builder = makeJsonSchemaBuilder(DEFAULT_JSON_SCHEMA_VERSION); + if (mgr->traceLevel >= 1){ + printf("about to build schema for '%s'\n",schemaFilePath);fflush(stdout); + } JsonSchema *schema = jsonBuildSchema(builder,jsonWithSchema); if (schema == NULL){ - printf("Schema Build Failed: %s\n",builder->errorMessage); + printf("Schema Build for '%s' Failed: %s\n",schemaFilePath,builder->errorMessage); freeJsonSchemaBuilder(builder); freeConfigManager(mgr); return NULL; } else { trace(mgr,DEBUG,"JSON Schema built successfully\n"); - mgr->topSchema = schema; } - freeJsonSchemaBuilder(builder); + freeJsonSchemaBuilder(builder); + return schema; +} + +#define MAX_SCHEMAS 100 + +ConfigManager *makeConfigManager(char *configPathArg, char *schemaPath, + int traceLevel, FILE *traceOut){ + printf("makeConfigMgr\n");fflush(stdout); + ConfigManager *mgr = (ConfigManager*)safeMalloc(sizeof(ConfigManager),"ConfigManager"); + + memset(mgr,0,sizeof(ConfigManager)); + mgr->traceLevel = traceLevel; + mgr->traceOut = traceOut; + mgr->slh = makeShortLivedHeap(0x10000,0x100); + EmbeddedJS *ejs = makeEmbeddedJS(NULL); + mgr->ejs = ejs; + trace(mgr,DEBUG,"before build config path\n"); + if (buildConfigPath(mgr,configPathArg)){ + printf("built config path failed\n");fflush(stdout); + safeFree((char*)mgr,sizeof(ConfigManager)); + return NULL; + } + trace(mgr,DEBUG,"built config path\n"); + + mgr->otherSchemas = (JsonSchema**)SLHAlloc(mgr->slh,MAX_SCHEMAS*sizeof(JsonSchema)); + mgr->rootSchemaDirectory = schemaPath; /* rootSchemaDirectory; */ + + int pos = 0; + int len = strlen(schemaPath); + int schemaCount = 0; + while (pos < len){ + int nextColon = indexOf(schemaPath,len,':',pos); + int nextPos; + int pathElementLength = 0; + + if (nextColon == -1){ + nextPos = len; + pathElementLength = len - pos; + } else { + nextPos = nextColon+1; + pathElementLength = nextColon - pos; + } + + char *schemaFilePath = substring(mgr,schemaPath,pos,pos+pathElementLength); + + /* char *schemaFilePath = makeFullPath(mgr,rootSchemaDirectory,ZOWE_SCHEMA_FILE,true); */ + + if (schemaCount == 0){ + mgr->topSchema = loadOneSchema(mgr,schemaFilePath); + } else { + mgr->otherSchemas[schemaCount-1] = loadOneSchema(mgr,schemaFilePath); + } + + pos = nextPos; + schemaCount++; + } + mgr->otherSchemasCount = schemaCount-1; return mgr; } @@ -521,8 +565,7 @@ static Json *readJson(ConfigManager *mgr, ConfigPathElement *pathElement){ } else { trace(mgr,INFO,"WARNING, yaml read failed\n"); return NULL; - } - + } } else { trace(mgr,INFO,"WARNING, only simple file case yet implemented\n"); return NULL; @@ -793,14 +836,14 @@ static void showHelp(FILE *out){ fprintf(out,"Usage:\n"); fprintf(out," configmgr [options] \n"); fprintf(out," options\n"); - fprintf(out," -h : show help\n"); - fprintf(out," -t : enable tracing with level from 1-3\n"); - fprintf(out," -o : OUT|ERR , ERR is default\n"); - fprintf(out," -s : root schema directory\n"); - fprintf(out," -w : workspace directory\n"); - fprintf(out," -c : compact output for jq and extract commands\n"); - fprintf(out," -r : raw string output for jq and extract commands\n"); - fprintf(out," -p : list of colon-separated configPathElements - see below\n"); + fprintf(out," -h : show help\n"); + fprintf(out," -t : enable tracing with level from 1-3\n"); + fprintf(out," -o : OUT|ERR , ERR is default\n"); + fprintf(out," -s : (: : workspace directory\n"); + fprintf(out," -c : compact output for jq and extract commands\n"); + fprintf(out," -r : raw string output for jq and extract commands\n"); + fprintf(out," -p : list of colon-separated configPathElements - see below\n"); fprintf(out," commands:\n"); fprintf(out," extract : prints value to stdout\n"); fprintf(out," validate : just loads and validates merged configuration\n"); @@ -931,7 +974,7 @@ static void convertJsonArrayToEnv(FILE *out, const char *path, JsonArray *array) int main(int argc, char **argv){ - char *rootSchemaDirectory = NULL; // Read-only ZOWE runtime_directory (maybe + char *schemaPath = NULL; char *zoweWorkspaceHome = NULL; // Read-write is there always a zowe.yaml in there char *configPath = NULL; char *command = NULL; @@ -951,7 +994,7 @@ int main(int argc, char **argv){ showHelp(traceOut); return 0; } else if ((optionValue = getStringOption(argc,argv,&argx,"-s")) != NULL){ - rootSchemaDirectory = optionValue; + schemaPath = optionValue; } else if ((optionValue = getStringOption(argc,argv,&argx,"-t")) != NULL){ traceArg = optionValue; } else if ((optionValue = getStringOption(argc,argv,&argx,"-o")) != NULL){ @@ -985,8 +1028,8 @@ int main(int argc, char **argv){ } } - if (rootSchemaDirectory == NULL){ - fprintf(traceOut,"Must specify root schema directory\n"); + if (schemaPath == NULL){ + fprintf(traceOut,"Must specify schema path with at least one schema"); showHelp(traceOut); return 0; } @@ -996,7 +1039,7 @@ int main(int argc, char **argv){ return 0; } - ConfigManager *mgr = makeConfigManager(configPath,rootSchemaDirectory,traceLevel,traceOut); + ConfigManager *mgr = makeConfigManager(configPath,schemaPath,traceLevel,traceOut); if (mgr == NULL){ trace(mgr,INFO,"Failed to build configmgr\n"); return 0; @@ -1023,8 +1066,10 @@ int main(int argc, char **argv){ trace(mgr,INFO,"about to validate merged yamls as\n"); jsonPrettyPrint(mgr,mgr->config); JsonValidator *validator = makeJsonValidator(); + validator->traceLevel = mgr->traceLevel; trace(mgr,DEBUG,"Before Validate\n"); - int validateStatus = jsonValidateSchema(validator,mgr->config,mgr->topSchema); + int validateStatus = jsonValidateSchema(validator,mgr->config, + mgr->topSchema,mgr->otherSchemas,mgr->otherSchemasCount); trace(mgr,INFO,"validate status = %d\n",validateStatus); switch (validateStatus){ case JSON_VALIDATOR_NO_EXCEPTIONS: diff --git a/c/jsonschema.c b/c/jsonschema.c index 5d5afc4a8..40dfa5bf3 100644 --- a/c/jsonschema.c +++ b/c/jsonschema.c @@ -678,9 +678,25 @@ static JSValueSpec *getTopSchemaByName(JsonValidator *validator, snprintf(key,len,"%*.*s/%*.*s", hostNameLength,hostNameLength,hostName, fileNameLength,fileNameLength,fileName); - JSValueSpec *schema = htGet(validator->schema->topValueSpec->definitions,key); + hashtable *definitionTable = validator->topSchema->topValueSpec->definitions; + JSValueSpec *schema = (definitionTable ? htGet(definitionTable,key) : NULL); if (validator->traceLevel >= 1){ - printf("getTopSchema for key='%s' yields 0x%p\n",key,schema); + printf("getTopSchema for key='%s' yields 0x%p (from HT)\n",key,schema); + } + if (schema == NULL){ + for (int i=0; iotherSchemaCount; i++){ + JsonSchema *otherSchema = validator->otherSchemas[i]; + JSValueSpec *otherTopSpec = otherSchema->topValueSpec; + + if (otherTopSpec->id && !strcmp(otherTopSpec->id,key)){ + schema = otherTopSpec; + if (validator->traceLevel >= 1){ + printf("getTopSchema for key='%s' yields 0x%p (from Other Schemas)\n",key,schema); + } + break; + } + } + } safeFree(key,len); return schema; @@ -731,7 +747,7 @@ static JSValueSpec *resolveRef(JsonValidator *validator, bool isSpeculating, JSV int refLen = strlen(ref); int pos = 2; char *key = ref+8; - JSValueSpec *topSchema = validator->schema->topValueSpec; + JSValueSpec *topSchema = validator->topSchema->topValueSpec; if (topSchema->definitions == NULL){ noteValidityException(validator,isSpeculating,12,"schema '%s' does not define shared '$defs'", (topSchema->id ? topSchema->id : "")); @@ -975,10 +991,13 @@ static VResult validateJSON(JsonValidator *validator, bool isSpeculating,Json *v } } -int jsonValidateSchema(JsonValidator *validator, Json *value, JsonSchema *schema){ +int jsonValidateSchema(JsonValidator *validator, Json *value, JsonSchema *topSchema, + JsonSchema **otherSchemas, int otherSchemaCount){ if (setjmp(validator->recoveryData) == 0) { /* normal execution */ - validator->schema = schema; - VResult validity = validateJSON(validator,false,value,schema->topValueSpec); + validator->topSchema = topSchema; + validator->otherSchemas = otherSchemas; + validator->otherSchemaCount = otherSchemaCount; + VResult validity = validateJSON(validator,false,value,topSchema->topValueSpec); printf("after validate without throw, should show validation exceptions\n"); fflush(stdout); if (validator->firstValidityException == NULL){ @@ -1245,8 +1264,6 @@ static JSValueSpec *makeValueSpec(JsonSchemaBuilder *builder, } else { spec->hostName = extractString(builder,id,&matches[1]); spec->fileName = extractString(builder,id,&matches[2]); - printf("top schema found with ID=%s, host='%s' file='%s'\n", - id,spec->hostName,spec->fileName); } regexFree(regex); } @@ -1577,7 +1594,7 @@ static JSValueSpec *build(JsonSchemaBuilder *builder, JSValueSpec *parent, Json } } - }// for (jsType... + }/* for (jsType... */ return valueSpec; } else { schemaThrow(builder,12,"top level schema must be an object"); @@ -1603,13 +1620,14 @@ void freeJsonSchemaBuilder(JsonSchemaBuilder *builder){ safeFree((char*)builder,sizeof(JsonSchemaBuilder)); } -JsonSchema *jsonBuildSchema(JsonSchemaBuilder *builder, Json *jsValue){ +JsonSchema *jsonBuildSchema(JsonSchemaBuilder *builder, + Json *schemaJson){ if (setjmp(builder->recoveryData) == 0) { /* normal execution */ if (builder->traceLevel >= 2){ printf("after setjmp normal\n"); fflush(stdout); } - JSValueSpec *topValueSpec = build(builder,NULL,jsValue,true); + JSValueSpec *topValueSpec = build(builder,NULL,schemaJson,true); JsonSchema *schema = (JsonSchema*)SLHAlloc(builder->slh,sizeof(JsonSchema)); schema->topValueSpec = topValueSpec; schema->version = builder->version; diff --git a/h/jsonschema.h b/h/jsonschema.h index 198cd2623..291d2d9a3 100644 --- a/h/jsonschema.h +++ b/h/jsonschema.h @@ -64,11 +64,15 @@ typedef struct ValidityException_tag { #define MAX_VALIDATOR_MATCHES 8 typedef struct JsonValidator_tag { - JsonSchema *schema; - int errorCode; - char *errorMessage; - int errorMessageLength; - char accessPathBuffer[MAX_ACCESS_PATH]; + JsonSchema *topSchema; + /* Other schemas that are referred to by the top and each other can be loaded into + one validator */ + JsonSchema **otherSchemas; + int otherSchemaCount; + int errorCode; + char *errorMessage; + int errorMessageLength; + char accessPathBuffer[MAX_ACCESS_PATH]; ValidityException *firstValidityException; ValidityException *lastValidityException; int flags; @@ -89,7 +93,7 @@ typedef struct JsonValidator_tag { JsonSchemaBuilder *makeJsonSchemaBuilder(int version); void freeJsonSchemaBuilder(JsonSchemaBuilder *builder); -JsonSchema *jsonBuildSchema(JsonSchemaBuilder *builder, Json *jsValue); +JsonSchema *jsonBuildSchema(JsonSchemaBuilder *builder, Json *schemaJson); JsonValidator *makeJsonValidator(); void freeJsonValidator(JsonValidator *validator); @@ -98,7 +102,8 @@ void freeJsonValidator(JsonValidator *validator); #define JSON_VALIDATOR_HAS_EXCEPTIONS 4 #define JSON_VALIDATOR_INTERNAL_FAILURE 8 -int jsonValidateSchema(JsonValidator *validator, Json *value, JsonSchema *schema); +int jsonValidateSchema(JsonValidator *validator, Json *value, JsonSchema *topSchema, + JsonSchema **otherSchemas, int otherSchemaCount); #endif diff --git a/tests/schemadata/bundle1.json b/tests/schemadata/bundle1.json index 0c593f681..81247f3ad 100644 --- a/tests/schemadata/bundle1.json +++ b/tests/schemadata/bundle1.json @@ -1,6 +1,6 @@ { "A": "apple", - "listenerPort": 34444, + "listenerPort": 344444, "BB": "FOO", - "Z": "60609" + "Z": "606094" } diff --git a/tests/schemadata/zoweappserver.json b/tests/schemadata/zoweappserver.json new file mode 100644 index 000000000..d22f5aeaa --- /dev/null +++ b/tests/schemadata/zoweappserver.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09", + "$id": "https://zarf.org/schema/appserver", + "allOf": [ + { + "$ref": "https://zowe.org/schema/base" + }, + { + "$ref": "https://zowe.org/schema/common#zoweComponent" + }, + { + "type": "object", + "required": [ + "BB" + ], + "properties": { + "BB": { + "const": "FOO" + } + } + } + ], +} diff --git a/tests/schemadata/zowebase.json b/tests/schemadata/zowebase.json new file mode 100644 index 000000000..b4c2277f8 --- /dev/null +++ b/tests/schemadata/zowebase.json @@ -0,0 +1,14 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09", + "$id": "https://zowe.org/schema/base", + "type": "object", + "properties": { + "A": { + "type": "string" + }, + "Z": { + "$ref": "/schema/common#zoweZip" + } + } +} + diff --git a/tests/schemadata/zowecommon.json b/tests/schemadata/zowecommon.json new file mode 100644 index 000000000..f8f51ff7a --- /dev/null +++ b/tests/schemadata/zowecommon.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09", + "$id": "https://zowe.org/schema/common", + "$defs": { + "component": { + "type": "object", + "$anchor": "zoweComponent", + "required": [ + "listenerPort" + ], + "properties": { + "listenerPort": { + "$ref": "#zowePort" + } + } + }, + "port": { + "$anchor": "zowePort", + "type": "integer", + "maximum": 65535 + }, + "zipCode": { + "$anchor": "zoweZip", + "type": "string", + "pattern": "^[0-9]{5}$" + } + } +} diff --git a/tests/schemadata/zowebase.yaml b/tests/schemadata/zowedefault.yaml similarity index 100% rename from tests/schemadata/zowebase.yaml rename to tests/schemadata/zowedefault.yaml diff --git a/tests/schematest.c b/tests/schematest.c index 518126d3c..70e046652 100644 --- a/tests/schematest.c +++ b/tests/schematest.c @@ -50,8 +50,7 @@ schematest yaml schemadata/example-zowe.yaml utf8 schemadata/zoweyaml.schema */ -int main(int argc, char *argv[]) -{ +int main(int argc, char *argv[]){ int number; char *syntax = argv[1]; char *filename = argv[2]; @@ -126,7 +125,7 @@ int main(int argc, char *argv[]) if (schema){ JsonValidator *validator = makeJsonValidator(); printf("Before Validate\n");fflush(stdout); - int validateStatus = jsonValidateSchema(validator,json,schema); + int validateStatus = jsonValidateSchema(validator,json,schema,NULL,0); switch (validateStatus){ case JSON_VALIDATOR_NO_EXCEPTIONS: printf("No validity Exceptions\n"); From 28fd30edb30a68a3fa453d5b95606cd7f76a0311 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 28 Mar 2022 13:16:28 -0400 Subject: [PATCH 39/42] More small fixes from Irek's PR review Signed-off-by: Joe --- c/json.c | 8 ++++++-- c/timeutls.c | 10 ++++++++-- c/zos.c | 13 +++---------- c/zosfile.c | 1 - 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/c/json.c b/c/json.c index 732936191..5a1b12b27 100644 --- a/c/json.c +++ b/c/json.c @@ -392,7 +392,12 @@ void jsonConvertAndWriteBuffer(jsonPrinter *p, char *text, size_t len, DUMPBUF(text, len); } if (escape) { - writeBufferWithEscaping(p, len, text); + ssize_t bytesWritten = writeBufferWithEscaping(p, len, text); + if (bytesWritten < 0){ + JSONERROR("jsonConvertAndWriteBuffer() error: bytesWritten = %zd\n", + bytesWritten); + return; + } } else { jsonWriteBufferInternal(p, text, len); } @@ -2728,7 +2733,6 @@ static JsonPointerElement *makePointerElement(char *token, int type){ /* See RFC 6901 - absolute pointes only, so far */ JsonPointer *parseJsonPointer(char *s){ int len = strlen(s); - /* char *reduced = safeMalloc(len+1); */ JsonPointer *jp = (JsonPointer*)safeMalloc(sizeof(JsonPointer),"JSONPointer"); ArrayList *list = &(jp->elements); initEmbeddedArrayList(list,NULL); diff --git a/c/timeutls.c b/c/timeutls.c index df748b736..a0f22a700 100644 --- a/c/timeutls.c +++ b/c/timeutls.c @@ -464,10 +464,16 @@ int timeZoneDifferenceFor(int64 theTime) { /* There has to be a better way... */ time_t base = (time_t) theTime; - struct tm local; + struct tm local = { 0 }; #ifdef __ZOWE_OS_WINDOWS struct tm *tempTM = localtime(&base); - memcpy(&local,tempTM,sizeof(struct tm)); + if (tempTM){ + memcpy(&local,tempTM,sizeof(struct tm)); + } else { + /* Nothing good to do here, and return value will be v.bad. + Garbage In, Garbage Out! + */ + } #else localtime_r(&base, &local); #endif diff --git a/c/zos.c b/c/zos.c index ba01062ba..402294493 100644 --- a/c/zos.c +++ b/c/zos.c @@ -108,20 +108,15 @@ int supervisorMode(int enable){ } int setKey(int key){ -#ifndef __ZOWE_COMP_XLCLANG /* temp hack until Joe figures out what "&r" means */ - int oldKey; + int shiftedKey = key << 4; __asm(" XR 2,2 \n" - " SLL %1,4 \n" " MODESET KEYREG=(%1),SAVEKEY=(2) \n" " SRL 2,4 \n" " ST 2,%0 \n" :"=m"(oldKey) - :"&r"(key) + :"r"(shiftedKey) :"r2"); return oldKey; -#else - return 8; -#endif } int64 getR12(void) { @@ -278,7 +273,7 @@ OTCB *getOTCB(void) { ASCB *getASCB(void) { int *mem = (int*)0; - return (ASCB*)INT2PTR(mem[CURRENT_ASCB/4]); + return (ASCB*)INT2PTR((mem[CURRENT_ASCB/4])&0x7FFFFFFF); } ASXB *getASXB(void) { @@ -1063,9 +1058,7 @@ static int safVerifyInternal(int options, fprintf(safTraceFile,"about to go call saf wrapper at 0x%p' verify block at 0x%p' acee 0x%p'\n",safWrapper,verifyRequest, (aceeHandle != NULL ? *aceeHandle : NULL)); dumpbuffer((char*)safWrapper,sizeof(safp)); -#ifndef METTLE fprintf(safTraceFile,"verify:\n"); -#endif dumpbuffer((char*)verifyRequest,sizeof(safVerifyRequest)); } safStatus = SAF(safWrapper,useSupervisorMode); diff --git a/c/zosfile.c b/c/zosfile.c index 44bb0467a..8ec0307ee 100644 --- a/c/zosfile.c +++ b/c/zosfile.c @@ -785,7 +785,6 @@ int fileInfo(const char *filename, BPXYSTAT *stats, int *returnCode, int *reason reasonCodePtr = reasonCode; #endif - dumpbuffer(filename,strlen(filename)); BPXSTA(&nameLength, filename, &statsLength, From eb915b9f6d655b648d0c6f852c2f78c98c57ca90 Mon Sep 17 00:00:00 2001 From: Joe Devlin Date: Mon, 28 Mar 2022 21:29:41 -0500 Subject: [PATCH 40/42] More PR review fixes including missing copyrights, setKey(), bad function name, and non-standard '__packed' annotation Signed-off-by: Joe Devlin --- c/as.c | 4 +-- c/configmgr.c | 10 ++++++ c/embeddedjs.c | 63 +++++++++++++++++++---------------- c/jsonschema.c | 10 ++++++ c/microjq.c | 10 ++++++ c/parsetools.c | 20 +++++++++++ c/psxskt.c | 10 ++++++ c/winskt.c | 10 ++++++ c/yaml2json.c | 21 ++++++++++++ c/zos.c | 1 + h/embeddedjs.h | 20 +++++++++++ h/jsonschema.h | 20 +++++++++++ h/parsetools.h | 10 ++++++ platform/posix/psxregex.c | 20 +++++++++++ platform/posix/psxregex.h | 20 +++++++++++ platform/windows/cppregex.cpp | 20 ++++++++++- platform/windows/cppregex.h | 20 +++++++++++ platform/windows/winregex.cpp | 21 ++++++++++-- platform/windows/winregex.h | 20 +++++++++++ tests/setkeytest.c | 41 +++++++++++++++++++++++ 20 files changed, 338 insertions(+), 33 deletions(-) create mode 100644 tests/setkeytest.c diff --git a/c/as.c b/c/as.c index bc1585c25..3ab42e77f 100644 --- a/c/as.c +++ b/c/as.c @@ -260,9 +260,9 @@ int addressSpaceCreateWithTerm(const ASParmString *startParmString, void * __ptr32 termCallbackParm, int *reasonCode) { - __packed union { + _Packed union { ASUserToken tokenValue; - __packed struct { + _Packed struct { ASCRETermCallback * __ptr32 termCallback; void * __ptr32 termCallbackParm; }; diff --git a/c/configmgr.c b/c/configmgr.c index 4c6f226a5..18f1acbe9 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -1147,3 +1147,13 @@ int main(int argc, char **argv){ } return 0; } + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/embeddedjs.c b/c/embeddedjs.c index e02d64157..d7cc09694 100644 --- a/c/embeddedjs.c +++ b/c/embeddedjs.c @@ -1,5 +1,3 @@ - - /* This program and the accompanying materials are made available under the terms of the Eclipse Public License v2.0 which accompanies @@ -143,17 +141,17 @@ int ejsEvalFile(EmbeddedJS *ejs, const char *filename, int loadMode){ return ret; } -static JSClassID js_joe_thingy_class_id; +static JSClassID js_experiment_thingy_class_id; -static void js_joe_thingy_finalizer(JSRuntime *rt, JSValue val){ +static void js_experiment_thingy_finalizer(JSRuntime *rt, JSValue val){ /* JSSTDFile *s = JS_GetOpaque(val, js_std_file_class_id); */ - printf("Joe Thingy Finalizer running\n"); + printf("Experiment Thingy Finalizer running\n"); } -static JSClassDef js_joe_thingy_class = { +static JSClassDef js_experiment_thingy_class = { "THINGY", - .finalizer = js_joe_thingy_finalizer, + .finalizer = js_experiment_thingy_finalizer, }; /* @@ -165,59 +163,59 @@ static JSClassDef js_joe_thingy_class = { */ -JSValue js_joe_thingy_three(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv){ +JSValue js_experiment_thingy_three(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv){ return JS_NewInt32(ctx, 3); } -static const JSCFunctionListEntry js_joe_thingy_proto_funcs[] = { - JS_CFUNC_DEF("three", 0, js_joe_thingy_three ), +static const JSCFunctionListEntry js_experiment_thingy_proto_funcs[] = { + JS_CFUNC_DEF("three", 0, js_experiment_thingy_three ), }; -static JSValue js_joe_boop(JSContext *ctx, JSValueConst this_val, +static JSValue js_experiment_boop(JSContext *ctx, JSValueConst this_val, int argc, JSValueConst *argv) { printf("boop boop boop!!\n"); return JS_UNDEFINED; } -static const JSCFunctionListEntry js_joe_funcs[] = { - JS_CFUNC_DEF("boop", 0, js_joe_boop ), +static const JSCFunctionListEntry js_experiment_funcs[] = { + JS_CFUNC_DEF("boop", 0, js_experiment_boop ), }; -static int js_joe_init(JSContext *ctx, JSModuleDef *m) +static int js_experiment_init(JSContext *ctx, JSModuleDef *m) { JSValue proto; /* FILE class */ /* the class ID is created once */ - JS_NewClassID(&js_joe_thingy_class_id); + JS_NewClassID(&js_experiment_thingy_class_id); /* the class is created once per runtime */ - JS_NewClass(JS_GetRuntime(ctx), js_joe_thingy_class_id, &js_joe_thingy_class); + JS_NewClass(JS_GetRuntime(ctx), js_experiment_thingy_class_id, &js_experiment_thingy_class); proto = JS_NewObject(ctx); JS_SetPropertyFunctionList(ctx, proto, - js_joe_thingy_proto_funcs, - countof(js_joe_thingy_proto_funcs)); + js_experiment_thingy_proto_funcs, + countof(js_experiment_thingy_proto_funcs)); JS_SetClassProto(ctx, - js_joe_thingy_class_id, + js_experiment_thingy_class_id, proto); - JS_SetModuleExportList(ctx, m, js_joe_funcs, - countof(js_joe_funcs)); + JS_SetModuleExportList(ctx, m, js_experiment_funcs, + countof(js_experiment_funcs)); return 0; } -JSModuleDef *js_init_module_joe(JSContext *ctx, const char *module_name) +JSModuleDef *js_init_module_experiment(JSContext *ctx, const char *module_name) { JSModuleDef *m; - m = JS_NewCModule(ctx, module_name, js_joe_init); + m = JS_NewCModule(ctx, module_name, js_experiment_init); if (!m){ return NULL; } - JS_AddModuleExportList(ctx, m, js_joe_funcs, countof(js_joe_funcs)); + JS_AddModuleExportList(ctx, m, js_experiment_funcs, countof(js_experiment_funcs)); return m; } @@ -239,7 +237,7 @@ static JSContext *makeEmbeddedJSContext(JSRuntime *rt) /* system modules */ js_init_module_std(ctx, "std"); js_init_module_os(ctx, "os"); - js_init_module_joe(ctx, "joe"); + js_init_module_experiment(ctx, "experiment"); return ctx; } @@ -687,7 +685,7 @@ EmbeddedJS *makeEmbeddedJS(EmbeddedJS *sharedRuntimeEJS){ /* can be NULL */ /* loader for ES6 modules */ JS_SetModuleLoaderFunc(embeddedJS->rt, NULL, js_module_loader, NULL); - if (false){ /* dump_unhandled_promise_rejection) { - JOE doe this later */ + if (false){ /* dump_unhandled_promise_rejection) { - do this later */ JS_SetHostPromiseRejectionTracker(embeddedJS->rt, js_std_promise_rejection_tracker, NULL); } @@ -698,13 +696,22 @@ EmbeddedJS *makeEmbeddedJS(EmbeddedJS *sharedRuntimeEJS){ /* can be NULL */ if (true){ /* load_std) {*/ const char *str = "import * as std from 'std';\n" "import * as os from 'os';\n" - "import * as joe from 'joe';\n" + "import * as experiment from 'experiment';\n" "globalThis.std = std;\n" "globalThis.os = os;\n" - "globalThis.joe = joe;\n"; + "globalThis.experiment = experiment;\n"; JSValue throwaway = ejsEvalBuffer(embeddedJS, str, strlen(str), "", JS_EVAL_TYPE_MODULE, &evalStatus); } return embeddedJS; } +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/jsonschema.c b/c/jsonschema.c index 40dfa5bf3..e65e27836 100644 --- a/c/jsonschema.c +++ b/c/jsonschema.c @@ -1643,3 +1643,13 @@ JsonSchema *jsonBuildSchema(JsonSchemaBuilder *builder, return NULL; } } + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/microjq.c b/c/microjq.c index fac64ca61..8462b5dc8 100644 --- a/c/microjq.c +++ b/c/microjq.c @@ -494,3 +494,13 @@ arguments: "-r .zowe.setup.security.users.zowe" arguments: "-r .zowe.verifyCertificates" arguments: "-r .zowe.workspaceDirectory" */ + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/parsetools.c b/c/parsetools.c index fc580af3f..8568b35e0 100644 --- a/c/parsetools.c +++ b/c/parsetools.c @@ -1,3 +1,13 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + #include #include #include @@ -867,3 +877,13 @@ int getJTokenJsonType(int tokenID){ return JSON_TYPE_STRING; } } + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/psxskt.c b/c/psxskt.c index 2e50f879f..f3f299650 100644 --- a/c/psxskt.c +++ b/c/psxskt.c @@ -1480,3 +1480,13 @@ int setSocketWriteBufferSize(Socket *socket, int bufferSize, int *returnCode, in int setSocketReadBufferSize(Socket *socket, int bufferSize, int *returnCode, int *reasonCode){ return setSocketOptionEx(socket,SOL_SOCKET, " (SOL_SOCKET)",SO_RCVBUF, " (SO_RCVBUF)", sizeof(int),(char*)&bufferSize,returnCode,reasonCode); } + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/winskt.c b/c/winskt.c index ec30c30dd..d7894627d 100644 --- a/c/winskt.c +++ b/c/winskt.c @@ -1124,3 +1124,13 @@ int socketSetRemove(SocketSet *set, Socket *socket){ set->revisionNumber++; return 0; } + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/yaml2json.c b/c/yaml2json.c index de5cc20d2..c62189b53 100644 --- a/c/yaml2json.c +++ b/c/yaml2json.c @@ -1,3 +1,14 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + + #include /* #include */ @@ -871,3 +882,13 @@ static void yaml2JS(yaml_document_t *document, ShortLivedHeap *slh){ ByteOutputStream *bos = makeByteOutputStream(0x1000); yaml2JS1(bos,document,yaml_document_get_root_node(document),0,1); } + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/c/zos.c b/c/zos.c index 402294493..75b23dce5 100644 --- a/c/zos.c +++ b/c/zos.c @@ -109,6 +109,7 @@ int supervisorMode(int enable){ int setKey(int key){ int shiftedKey = key << 4; + int oldKey; __asm(" XR 2,2 \n" " MODESET KEYREG=(%1),SAVEKEY=(2) \n" " SRL 2,4 \n" diff --git a/h/embeddedjs.h b/h/embeddedjs.h index 499cb1997..3e6b6ff9b 100644 --- a/h/embeddedjs.h +++ b/h/embeddedjs.h @@ -1,3 +1,13 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + #ifndef __ZOWE_EMBEDDEDJS__ #define __ZOWE_EMBEDDEDJS__ 1 @@ -56,3 +66,13 @@ int ejsSetGlobalProperty(EmbeddedJS *ejs, const char *propetyName, JSValue value Json *evaluateJsonTemplates(EmbeddedJS *ejs, ShortLivedHeap *slh, Json *json); #endif + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/h/jsonschema.h b/h/jsonschema.h index 291d2d9a3..ce5ff0176 100644 --- a/h/jsonschema.h +++ b/h/jsonschema.h @@ -1,3 +1,13 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + #ifndef __ZOWE_JSONSCHEMA__ #define __ZOWE_JSONSCHEMA__ 1 @@ -107,3 +117,13 @@ int jsonValidateSchema(JsonValidator *validator, Json *value, JsonSchema *topSch #endif + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/h/parsetools.h b/h/parsetools.h index 80fbb0933..129f0202e 100644 --- a/h/parsetools.h +++ b/h/parsetools.h @@ -491,3 +491,13 @@ int getJTokenJsonType(int tokenID); #endif + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/platform/posix/psxregex.c b/platform/posix/psxregex.c index 0493eca06..a9a8a4578 100644 --- a/platform/posix/psxregex.c +++ b/platform/posix/psxregex.c @@ -1,3 +1,13 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + #include "psxregex.h" #include "zowetypes.h" #include "alloc.h" @@ -10,3 +20,13 @@ void regexFree(regex_t *r){ regfree(r); safeFree((char*)r,sizeof(regex_t)); } + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/platform/posix/psxregex.h b/platform/posix/psxregex.h index 8a1053690..bc0cbdeaf 100644 --- a/platform/posix/psxregex.h +++ b/platform/posix/psxregex.h @@ -1,3 +1,13 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + #ifndef __ZOWE_PSXREGEX__ #define __ZOWE_PSXREGEX__ @@ -10,3 +20,13 @@ void regexFree(regex_t *m); #define regexExec regexec #endif + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/platform/windows/cppregex.cpp b/platform/windows/cppregex.cpp index 02838287b..fb1efaa33 100644 --- a/platform/windows/cppregex.cpp +++ b/platform/windows/cppregex.cpp @@ -1,3 +1,13 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + #include "cppregex.h" #include @@ -33,4 +43,12 @@ bool CPPRegex::exec(const char *first, const char *last, std::cmatch& matchResul } } - +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/platform/windows/cppregex.h b/platform/windows/cppregex.h index 8212f98db..7106a903a 100644 --- a/platform/windows/cppregex.h +++ b/platform/windows/cppregex.h @@ -1,3 +1,13 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + #ifndef __CPPREGEX_H__ #define __CPPREGEX_H__ @@ -15,3 +25,13 @@ class CPPRegex }; #endif // __CPPEGEX_H__ + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/platform/windows/winregex.cpp b/platform/windows/winregex.cpp index 611efd09b..7eadf5e69 100644 --- a/platform/windows/winregex.cpp +++ b/platform/windows/winregex.cpp @@ -1,3 +1,13 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + #include #include "winregex.h" #include "cppregex.h" @@ -61,5 +71,12 @@ int regexExec(regex_t *r, const char *str, size_t nmatch, return (matched ? 0 : REG_NOMATCH); } - - +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/platform/windows/winregex.h b/platform/windows/winregex.h index 458f0c558..5e5ceaa9c 100644 --- a/platform/windows/winregex.h +++ b/platform/windows/winregex.h @@ -1,3 +1,13 @@ +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ + #ifndef __WINREGEX_H__ #define __WINREGEX_H__ @@ -95,3 +105,13 @@ int regexExec(regex_t *r, const char *str, size_t nmatch, #endif #endif /* __WINREGEX_H__ */ + +/* + This program and the accompanying materials are + made available under the terms of the Eclipse Public License v2.0 which accompanies + this distribution, and is available at https://www.eclipse.org/legal/epl-v20.html + + SPDX-License-Identifier: EPL-2.0 + + Copyright Contributors to the Zowe Project. +*/ diff --git a/tests/setkeytest.c b/tests/setkeytest.c new file mode 100644 index 000000000..1446dc248 --- /dev/null +++ b/tests/setkeytest.c @@ -0,0 +1,41 @@ +#include +#include +#include +#include +#include + +#include "zowetypes.h" +#include "alloc.h" +#include "utils.h" +#include "zos.h" + + +/* + xlclang -q64 -D_OPEN_SYS_FILE_EXT=1 -D_XOPEN_SOURCE=600 -D_OPEN_THREADS=1 -DSUBPOOL=132 "-Wc,float(ieee),longname,langlvl(extc99),gonum,goff,ASM,asmlib('CEE.SCEEMAC','SYS1.MACLIB','SYS1.MODGEN')" "-Wl,ac=1" -I ../h -I ../platform/posix -Wbitwise-op-parentheses -o setkeytest setkeytest.c ../c/zos.c ../c/timeutls.c ../c/utils.c ../c/alloc.c + + */ + +static int getPrefix(){ + int32_t prefix = 0; + __asm(" STPX %0 " + : + :"m"(prefix) + :"r15"); + return prefix; +} + + +int main(int argc, char **argv){ + if (argc >= 2 && !strcmp(argv[1],"true")){ + supervisorMode(true); + int oldKey = setKey(0); + int prefix = getPrefix(); + printf("WITH PRIV: this ZOS has prefix 0x%x\n",prefix); + setKey(oldKey); + supervisorMode(false); + } else{ + int prefix = getPrefix(); + printf("NO PRIV: this ZOS has prefix 0x%x\n",prefix); + } + return 0; +} From 2299294d8bf15203985d6a1c6f214be0e71cd905 Mon Sep 17 00:00:00 2001 From: Joe Devlin Date: Mon, 28 Mar 2022 21:43:38 -0500 Subject: [PATCH 41/42] minor METAL C bug and a slight change to a test Signed-off-by: Joe Devlin --- c/zos.c | 2 +- tests/setkeytest.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/c/zos.c b/c/zos.c index 75b23dce5..0b54395aa 100644 --- a/c/zos.c +++ b/c/zos.c @@ -1059,7 +1059,7 @@ static int safVerifyInternal(int options, fprintf(safTraceFile,"about to go call saf wrapper at 0x%p' verify block at 0x%p' acee 0x%p'\n",safWrapper,verifyRequest, (aceeHandle != NULL ? *aceeHandle : NULL)); dumpbuffer((char*)safWrapper,sizeof(safp)); - fprintf(safTraceFile,"verify:\n"); + fprintf(safTraceFile,"verify: request at 0x%p\n",verifyRequest); dumpbuffer((char*)verifyRequest,sizeof(safVerifyRequest)); } safStatus = SAF(safWrapper,useSupervisorMode); diff --git a/tests/setkeytest.c b/tests/setkeytest.c index 1446dc248..6b293192b 100644 --- a/tests/setkeytest.c +++ b/tests/setkeytest.c @@ -33,6 +33,7 @@ int main(int argc, char **argv){ printf("WITH PRIV: this ZOS has prefix 0x%x\n",prefix); setKey(oldKey); supervisorMode(false); + printf("oldKey was = %d\n",oldKey); } else{ int prefix = getPrefix(); printf("NO PRIV: this ZOS has prefix 0x%x\n",prefix); From 1dd111c9cc396a4206a5a253c519f8fbae72cabf Mon Sep 17 00:00:00 2001 From: Joe Date: Thu, 31 Mar 2022 16:52:28 -0400 Subject: [PATCH 42/42] Hiding QuickJS h files from other zowe-common-c and clang/xlclang warnings Signed-off-by: Joe --- c/cmutils.c | 5 +-- c/configmgr.c | 53 ++++++++---------------------- c/crossmemory.c | 4 +-- c/discovery.c | 14 ++++---- c/embeddedjs.c | 85 +++++++++++++++++++++++++++++++++++++++++-------- c/fdpoll.c | 6 +++- h/crossmemory.h | 4 +-- h/discovery.h | 1 + h/dynalloc.h | 4 +-- h/embeddedjs.h | 52 ++++++++---------------------- 10 files changed, 121 insertions(+), 107 deletions(-) diff --git a/c/cmutils.c b/c/cmutils.c index 48c68a6b6..dd3b5b46c 100644 --- a/c/cmutils.c +++ b/c/cmutils.c @@ -26,6 +26,7 @@ #include "cmutils.h" #include "zos.h" + static int getCallersKey() { int key = 0; @@ -44,9 +45,9 @@ static int getCallersKey() { return key; } -#define IS_LE64 (!defined(METTLE) && defined(_LP64)) +/* #define IS_LE64 (!defined(METTLE) && defined(_LP64)) */ -#if !IS_LE64 +#if !(!defined(METTLE) && defined(_LP64)) /* Disable these for LE 64-bit for now, the compile complains about GPR 4 */ diff --git a/c/configmgr.c b/c/configmgr.c index 18f1acbe9..0e2956a1c 100644 --- a/c/configmgr.c +++ b/c/configmgr.c @@ -59,6 +59,7 @@ #include "embeddedjs.h" #include "parsetools.h" #include "microjq.h" +#include "configmgr.h" #ifdef __ZOWE_OS_WINDOWS typedef int64_t ssize_t; @@ -98,7 +99,7 @@ typedef int64_t ssize_t; clang++ -c ../platform/windows/cppregex.cpp ../platform/windows/winregex.cpp - clang -I %QJS%\porting -I%YAML%/include -I %QJS% -I./src -I../h -I ../platform/windows -DCONFIG_VERSION=\"2021-03-27\" -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 --rtlib=compiler-rt -o configmgr.exe configmgr.c embeddedjs.c %QJS%\quickjs.c %QJS%\cutils.c %QJS%\quickjs-libc.c %QJS%\libbf.c %QJS%\libregexp.c %QJS%\libunicode.c %QJS%\porting\winpthread.c %QJS%\porting\wintime.c %QJS%\porting\windirent.c %QJS%\porting\winunistd.c %YAML%/src/api.c %YAML%/src/reader.c %YAML%/src/scanner.c %YAML%/src/parser.c %YAML%/src/loader.c %YAML%/src/writer.c %YAML%/src/emitter.c %YAML%/src/dumper.c ../c/yaml2json.c ../c/microjq.c ../c/parsetools.c ../c/jsonschema.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c cppregex.o winregex.o + clang -I %QJS%\porting -I%YAML%/include -I %QJS% -I./src -I../h -I ../platform/windows -DCMGRTEST=1 -DCONFIG_VERSION=\"2021-03-27\" -Dstrdup=_strdup -D_CRT_SECURE_NO_WARNINGS -DYAML_VERSION_MAJOR=0 -DYAML_VERSION_MINOR=2 -DYAML_VERSION_PATCH=5 -DYAML_VERSION_STRING=\"0.2.5\" -DYAML_DECLARE_STATIC=1 --rtlib=compiler-rt -o configmgr.exe configmgr.c embeddedjs.c %QJS%\quickjs.c %QJS%\cutils.c %QJS%\quickjs-libc.c %QJS%\libbf.c %QJS%\libregexp.c %QJS%\libunicode.c %QJS%\porting\winpthread.c %QJS%\porting\wintime.c %QJS%\porting\windirent.c %QJS%\porting\winunistd.c %YAML%/src/api.c %YAML%/src/reader.c %YAML%/src/scanner.c %YAML%/src/parser.c %YAML%/src/loader.c %YAML%/src/writer.c %YAML%/src/emitter.c %YAML%/src/dumper.c ../c/yaml2json.c ../c/microjq.c ../c/parsetools.c ../c/jsonschema.c ../c/json.c ../c/xlate.c ../c/charsets.c ../c/winskt.c ../c/logging.c ../c/collections.c ../c/timeutls.c ../c/utils.c ../c/alloc.c cppregex.o winregex.o configmgr "../tests/schemadata" "" "LIBRARY(FOO):DIR(BAR)" yak @@ -110,7 +111,7 @@ typedef int64_t ssize_t; configmgr -t 1 -s "../tests/schemadata/zoweappserver.json:../tests/schemadata/zowebase.json:../tests/schemadata/zowecommon.json" -p "FILE(../tests/schemadata/bundle1.json)" validate - configmgr -s "../tests/schemadata" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowebase.yaml)" extract "/zowe/setup/mvs/proclib" + configmgr -s "../tests/schemadata/zoweappserver.json:../tests/schemadata/zowebase.json:../tests/schemadata/zowecommon.json" -p "FILE(../tests/schemadata/zoweoverrides.yaml):FILE(../tests/schemadata/zowedefault.yaml)" extract "/zowe/setup/mvs/proclib" -- Compilation with XLCLang on ZOS ------------------------------------- @@ -156,29 +157,6 @@ typedef int64_t ssize_t; #define CONFIG_PATH_OMVS_DIR 0x0002 #define CONFIG_PATH_OMVS_LIBRARY 0x0004 -typedef struct ConfigPathElement_tag { - int flags; - char *name; - char *data; - struct ConfigPathElement_tag *next; -} ConfigPathElement; - -typedef struct ConfigManager_tag { - ShortLivedHeap *slh; - char *rootSchemaDirectory; - ConfigPathElement *schemaPath; /* maybe */ - ConfigPathElement *configPath; - JsonSchema *topSchema; - JsonSchema **otherSchemas; - int otherSchemasCount; - Json *config; - hashtable *schemaCache; - hashtable *configCache; - int traceLevel; - FILE *traceOut; - EmbeddedJS *ejs; -} ConfigManager; - #ifdef __ZOWE_OS_WINDOWS static int stdoutFD(void){ return _fileno(stdout); @@ -496,7 +474,7 @@ static JsonSchema *loadOneSchema(ConfigManager *mgr, char *schemaFilePath){ ConfigManager *makeConfigManager(char *configPathArg, char *schemaPath, int traceLevel, FILE *traceOut){ - printf("makeConfigMgr\n");fflush(stdout); + fprintf(traceOut,"makeConfigMgr\n");fflush(traceOut); ConfigManager *mgr = (ConfigManager*)safeMalloc(sizeof(ConfigManager),"ConfigManager"); memset(mgr,0,sizeof(ConfigManager)); @@ -618,7 +596,7 @@ static int overloadConfiguration(ConfigManager *mgr, } } -static int loadConfigurations(ConfigManager *mgr){ +int loadConfigurations(ConfigManager *mgr){ ConfigPathElement *pathElement = mgr->configPath; int overloadStatus = overloadConfiguration(mgr,pathElement,pathElement->next); if (overloadStatus){ @@ -686,9 +664,10 @@ static Json *jsonPointerDereference(Json *json, JsonPointer *jsonPointer, int *e return NULL; } } else { - printf("SSS\n");fflush(stdout); - printf("cannot dereference scalar\n"); - fflush(stdout); + if (traceLevel >= 1){ + printf("cannot dereference scalar\n"); + fflush(stdout); + } return NULL; } } @@ -963,17 +942,11 @@ static void convertJsonArrayToEnv(FILE *out, const char *path, JsonArray *array) } } -/* - Here: - 1) Detect qascii or not - 2) store in code - 3) BPX routines do NOT tolerate qASCII like c-library - 4) fileInfo tag must be used on ZOS target to potentially preprocess input - 5) maybe Leonty has done this for yaml (or not) - */ - - +#ifdef CMGRTEST int main(int argc, char **argv){ +#else +static int not_main(int argc, char **argv){ +#endif char *schemaPath = NULL; char *zoweWorkspaceHome = NULL; // Read-write is there always a zowe.yaml in there char *configPath = NULL; diff --git a/c/crossmemory.c b/c/crossmemory.c index 9ef6bc4fa..bd15f1ca3 100644 --- a/c/crossmemory.c +++ b/c/crossmemory.c @@ -602,7 +602,7 @@ static void initLogMessagePrefix(LogMessagePrefix *prefix) { ASCB *ascb = getASCB(); char *jobName = getASCBJobname(ascb); TCB *tcb = getTCB(); - snprintf(prefix->text, sizeof(prefix->text), "%22.22s %8.8s %08X(%04X) %08X ", currentTime.text, jobName, ascb, ascb->ascbasid, tcb); + snprintf(prefix->text, sizeof(prefix->text), "%22.22s %8.8s %8p(%04X) %8p ", currentTime.text, jobName, ascb, ascb->ascbasid, tcb); prefix->text[sizeof(prefix->text) - 1] = ' '; } @@ -632,7 +632,7 @@ static void printWithPrefix(LoggingContext *context, LoggingComponent *component if (lineIdx == 0) { initLogMessagePrefix(&prefix); } - printf("%.*s%.*s\n", sizeof(prefix.text), prefix.text, nextLineLength, nextLine); + printf("%.*s%.*s\n", (int)sizeof(prefix.text), prefix.text, (int)nextLineLength, nextLine); nextLine += (nextLineLength + 1); } diff --git a/c/discovery.c b/c/discovery.c index 2f2306837..a8c04834e 100644 --- a/c/discovery.c +++ b/c/discovery.c @@ -259,7 +259,7 @@ static ASCB *getASCBByASID(int asid){ CVT *cvt = getCVT(); ASVT *asvt = (ASVT*)cvt->cvtasvt; - ASCB *ascb = (ASCB*)asvt->asvtenty; + ASCB *ascb = (ASCB*)INT2PTR(asvt->asvtenty); while (ascb){ if (ascb->ascbasid == asid){ return ascb; @@ -334,6 +334,7 @@ int walkTCBs(DiscoveryContext *context, TCB *firstTCB = (TCB*)getStructCopy(context,ascb,0,asxb->asxbftcb,sizeof(TCB)); walkTCBs1(context,ascb,firstTCB,(TCB*)ANY_TCB,0,visitor,visitorContext,NULL); } + return 0; } @@ -344,7 +345,7 @@ static void visitSSCTEntry(DiscoveryContext *context, if (context->ssctTraceLevel >= 1){ dumpbuffer((char*)ssctChain,sizeof(SSCT)); } - void *usr1 = (void*)ssctChain->ssctsuse; + void *usr1 = (void*)INT2PTR(ssctChain->ssctsuse); zowelog(NULL, LOG_COMP_DISCOVERY, ZOWE_LOG_DEBUG, "user pointer at 0x%x COMMON?=%s\n",usr1,isPointerCommon(gda,usr1) ? "YES" : "NO"); if (isPointerCommon(gda,usr1)){ if (context->ssctTraceLevel >= 1){ @@ -449,7 +450,7 @@ static void ispfAnchorVisitor(DiscoveryContext *context, */ if (tcbfsa[10] == *((int*)"ISPF")){ /* printf("WOO HOO\n"); */ - int *tldHandle = (int*)tcbfsa[6]; + int *tldHandle = (int*)INT2PTR(tcbfsa[6]); int *tldPtr = (int*)((int*)getStructCopy(context,ascb,0,tldHandle,4))[0]; /* printf("tldPtr=0x%x\n",tldPtr); */ if (tldPtr){ @@ -738,6 +739,7 @@ int findSessions(DiscoveryContext *context, break; /* if we ever get a good result, leave the while loop */ } } + return 0; } /************************** ZOS Model Maintenance ********************************/ @@ -759,7 +761,7 @@ static void gatherStartedTasks(DiscoveryContext *context, ZOSModel *model){ StartedTaskVisitor *userVisitor = model ? model->startedTaskVisitor : NULL; void* userVisitorData = model ? model->visitorsData : NULL; - ASCB *ascb = (ASCB*)asvt->asvtenty; + ASCB *ascb = (ASCB*)INT2PTR(asvt->asvtenty); while (ascb){ char *jobname = getASCBJobname(ascb); char *jobnameCopy = SLHAlloc(context->slh,12); @@ -818,13 +820,13 @@ static void scanTSBs(ZOSModel *model){ zowedump(NULL, LOG_COMP_DISCOVERY, ZOWE_LOG_DEBUG2,(char*)tsbxCopy,sizeof(IKTTSBX)); } TSBInfo *tsbInfo = (TSBInfo*)SLHAlloc(model->tsbScanSLH,sizeof(TSBInfo)); - tsbInfo->ascb = (ASCB*)(tsbCopy->tcbstatAndASCB & 0x00FFFFFF); + tsbInfo->ascb = (ASCB*)INT2PTR(tsbCopy->tcbstatAndASCB & 0x00FFFFFF); tsbInfo->tsb = tsbCopy; tsbInfo->tsbx = tsbxCopy; char *luname = &(tsbCopy->tsbtrmid[0]); htPut(tsbTable,luname,tsbInfo); - tsb = (IKJTSB*)(tsbxCopy->flagAndFwdPointer&0xFFFFFF); + tsb = (IKJTSB*)INT2PTR(tsbxCopy->flagAndFwdPointer&0xFFFFFF); } else{ tsb = NULL; } diff --git a/c/embeddedjs.c b/c/embeddedjs.c index d7cc09694..b5ef4490a 100644 --- a/c/embeddedjs.c +++ b/c/embeddedjs.c @@ -49,6 +49,9 @@ #include "charsets.h" #include "embeddedjs.h" +#include "cutils.h" +#include "quickjs-libc.h" + #ifdef __ZOWE_OS_WINDOWS typedef int64_t ssize_t; #endif @@ -74,10 +77,43 @@ static int convertFromNative(char *buf, size_t size) { return 0; } -JSValue ejsEvalBuffer(EmbeddedJS *ejs, - const void *buffer, int bufferLength, - const char *filename, int eval_flags, - int *statusPtr){ +struct trace_malloc_data { + uint8_t *base; +}; + +#define FILE_LOAD_AUTODETECT -1 +#define FILE_LOAD_GLOBAL 0 /* not module */ +#define FILE_LOAD_MODULE 1 + +struct EmbeddedJS_tag { + JSRuntime *rt; + JSContext *ctx; + struct trace_malloc_data trace_data; /* = { NULL }; */ + int optind; + char *expr; /* = NULL; */ + int interactivel; /* = 0; */ + int dump_memory; /* = 0; */ + int trace_memory; /* = 0;*/ + int empty_run; /* = 0; */ + int loadMode; /* FILE_LOAD_AUTODETECT */ + int load_std; /* = 0; */ + int dump_unhandled_promise_rejection; /* = 0; */ + size_t memory_limit; /* = 0; */ + char *include_list[32]; +#ifdef CONFIG_BIGNUM + int load_jscalc; +#endif + size_t stack_size; /* = 0; */ +}; + +struct JSValueBox_tag { + JSValue value; +}; + +static JSValue ejsEvalBuffer1(EmbeddedJS *ejs, + const void *buffer, int bufferLength, + const char *filename, int eval_flags, + int *statusPtr){ JSContext *ctx = ejs->ctx; JSValue val; int ret; @@ -104,17 +140,29 @@ JSValue ejsEvalBuffer(EmbeddedJS *ejs, return val; } -void ejsFreeJSValue(EmbeddedJS *ejs, JSValue value){ - JS_FreeValue(ejs->ctx, value); +JSValueBox ejsEvalBuffer(EmbeddedJS *ejs, + const void *buffer, int bufferLength, + const char *filename, int eval_flags, + int *statusPtr){ + JSValue value = ejsEvalBuffer1(ejs,buffer,bufferLength,filename,eval_flags,statusPtr); + return *(JSValueBox*)(&value); } -int ejsSetGlobalProperty(EmbeddedJS *ejs, const char *propertyName, JSValue value){ +void ejsFreeJSValue(EmbeddedJS *ejs, JSValueBox valueBox){ + JS_FreeValue(ejs->ctx, *((JSValue*)&valueBox)); +} + +static int ejsSetGlobalProperty_internal(EmbeddedJS *ejs, const char *propertyName, JSValue value){ JSContext *ctx = ejs->ctx; JSValue theGlobal = JS_GetGlobalObject(ctx); return JS_SetPropertyStr(ctx,theGlobal,propertyName,value); } +int ejsSetGlobalProperty(EmbeddedJS *ejs, const char *propertyName, JSValueBox valueBox){ + return ejsSetGlobalProperty_internal(ejs,propertyName,valueBox.value); +} + int ejsEvalFile(EmbeddedJS *ejs, const char *filename, int loadMode){ uint8_t *buf; int ret, eval_flags; @@ -135,7 +183,7 @@ int ejsEvalFile(EmbeddedJS *ejs, const char *filename, int loadMode){ eval_flags = JS_EVAL_TYPE_MODULE; else eval_flags = JS_EVAL_TYPE_GLOBAL; - JSValue evalResult = ejsEvalBuffer(ejs, buf, bufferLength, filename, eval_flags, &ret); + JSValue evalResult = ejsEvalBuffer1(ejs, buf, bufferLength, filename, eval_flags, &ret); js_free(ejs->ctx, buf); JS_FreeValue(ejs->ctx, evalResult); return ret; @@ -411,13 +459,17 @@ static bool isZoweUnevaluated(Json *json){ return false; } -Json *ejsJSToJson(EmbeddedJS *ejs, JSValue value, ShortLivedHeap *slh){ +static Json *ejsJSToJson_internal(EmbeddedJS *ejs, JSValue value, ShortLivedHeap *slh){ JsonBuilder *builder = makeJsonBuilder(slh); Json *json = jsToJson1(ejs,builder,NULL,NULL,value,0); freeJsonBuilder(builder,false); return json; } +Json *ejsJSToJson(EmbeddedJS *ejs, JSValueBox valueBox, ShortLivedHeap *slh){ + return ejsJSToJson_internal(ejs,valueBox.value,slh); +} + static JSValue jsonToJS1(EmbeddedJS *ejs, Json *json, bool hideUnevaluated){ JSContext *ctx = ejs->ctx; switch (json->type) { @@ -492,10 +544,15 @@ static JSValue jsonToJS1(EmbeddedJS *ejs, Json *json, bool hideUnevaluated){ -JSValue ejsJsonToJS(EmbeddedJS *ejs, Json *json){ +static JSValue ejsJsonToJS_internal(EmbeddedJS *ejs, Json *json){ return jsonToJS1(ejs,json,false); } +JSValueBox ejsJsonToJS(EmbeddedJS *ejs, Json *json){ + JSValue value = ejsJsonToJS_internal(ejs,json); + return *((JSValueBox*)&value); +} + char *json2JS(Json *json){ JsonBuffer *buffer = makeJsonBuffer(); /* jsonPrinter *p = makeBufferJsonPrinter(sourceCCSID,buffer); */ @@ -600,7 +657,7 @@ static bool evaluationVisitor(void *context, Json *json, Json *parent, char *key char convertedKey[keyLen+1]; snprintf (convertedKey, keyLen + 1, "%.*s", (int)keyLen, key); convertFromNative(convertedKey, keyLen); - ejsSetGlobalProperty(ejs,convertedKey,ejsJsonToJS(ejs,value)); + ejsSetGlobalProperty_internal(ejs,convertedKey,ejsJsonToJS_internal(ejs,value)); } Json *sourceValue = jsonObjectGetPropertyValue(object,"source"); if (sourceValue){ @@ -613,7 +670,7 @@ static bool evaluationVisitor(void *context, Json *json, Json *parent, char *key int evalStatus = 0; char embedded[] = ""; convertFromNative(embedded, sizeof(embedded)); - JSValue output = ejsEvalBuffer(ejs,asciiSource,strlen(asciiSource),embedded,0,&evalStatus); + JSValue output = ejsEvalBuffer1(ejs,asciiSource,strlen(asciiSource),embedded,0,&evalStatus); if (evalStatus){ printf("failed to evaluate '%s', status=%d\n",source,evalStatus); } else { @@ -621,7 +678,7 @@ static bool evaluationVisitor(void *context, Json *json, Json *parent, char *key printf("evaluation succeeded\n"); dumpbuffer((char*)&output,sizeof(JSValue)); */ - Json *evaluationResult = ejsJSToJson(ejs,output,evalContext->slh); + Json *evaluationResult = ejsJSToJson_internal(ejs,output,evalContext->slh); if (evaluationResult){ if (keyInParent){ setJsonProperty(jsonAsObject(parent),keyInParent,evaluationResult); @@ -700,7 +757,7 @@ EmbeddedJS *makeEmbeddedJS(EmbeddedJS *sharedRuntimeEJS){ /* can be NULL */ "globalThis.std = std;\n" "globalThis.os = os;\n" "globalThis.experiment = experiment;\n"; - JSValue throwaway = ejsEvalBuffer(embeddedJS, str, strlen(str), "", JS_EVAL_TYPE_MODULE, &evalStatus); + JSValue throwaway = ejsEvalBuffer1(embeddedJS, str, strlen(str), "", JS_EVAL_TYPE_MODULE, &evalStatus); } return embeddedJS; diff --git a/c/fdpoll.c b/c/fdpoll.c index 2ea3da6e4..9c8064297 100755 --- a/c/fdpoll.c +++ b/c/fdpoll.c @@ -13,6 +13,10 @@ #endif +/* xlclang and clang demand a prototype */ + +int BPXPOL(); + int fdPoll(PollItem* fds, short nmqs, short nfds, int timeout, int *returnCode, int *reasonCode) { int returnValue; int *reasonCodePtr; @@ -45,4 +49,4 @@ int fdPoll(PollItem* fds, short nmqs, short nfds, int timeout, int *returnCode, #else #error No implemention for fdpoll.h -#endif \ No newline at end of file +#endif diff --git a/h/crossmemory.h b/h/crossmemory.h index 07ae1890c..f80360813 100644 --- a/h/crossmemory.h +++ b/h/crossmemory.h @@ -174,8 +174,8 @@ typedef struct ELXLIST_tag { #define CROSS_MEMORY_SERVER_MAX_CMD_TASK_NUM 30 -typedef struct CrossMemoryServerGlobalArea_tag; -typedef struct CrossMemoryService_tag; +struct CrossMemoryServerGlobalArea_tag; +struct CrossMemoryService_tag; typedef int (CrossMemoryServiceFunction)(struct CrossMemoryServerGlobalArea_tag *globalArea, struct CrossMemoryService_tag *service, void *parm); diff --git a/h/discovery.h b/h/discovery.h index 22d7f8326..50672888c 100644 --- a/h/discovery.h +++ b/h/discovery.h @@ -164,6 +164,7 @@ typedef struct TSBInfo_tag{ IKTTSBX *tsbx; } TSBInfo; +struct DiscoveryContext_tag; typedef void SubsystemVisitor(struct DiscoveryContext_tag *context, SSCT *ssctChain, diff --git a/h/dynalloc.h b/h/dynalloc.h index b75987233..7cebd8e4a 100644 --- a/h/dynalloc.h +++ b/h/dynalloc.h @@ -25,8 +25,8 @@ #error z/OS targets are supported only #endif -#define turn_on_HOB(x) x = (TextUnit* __ptr32) ((int) x | 0x80000000) -#define turn_off_HOB(x) x = (TextUnit* __ptr32) ((int) x & 0x7FFFFFFF) +#define turn_on_HOB(x) x = (TextUnit* __ptr32) INT2PTR((int) x | 0x80000000) +#define turn_off_HOB(x) x = (TextUnit* __ptr32) INT2PTR((int) x & 0x7FFFFFFF) #define S99VRBAL 0x01 /* Allocation */ #define S99VRBUN 0x02 /* Unallocation */ diff --git a/h/embeddedjs.h b/h/embeddedjs.h index 3e6b6ff9b..1d6bb613c 100644 --- a/h/embeddedjs.h +++ b/h/embeddedjs.h @@ -11,58 +11,34 @@ #ifndef __ZOWE_EMBEDDEDJS__ #define __ZOWE_EMBEDDEDJS__ 1 -#include "cutils.h" -#include "quickjs-libc.h" - #include "zowetypes.h" -struct trace_malloc_data { - uint8_t *base; -}; +struct EmbeddedJS_tag; -#define FILE_LOAD_AUTODETECT -1 -#define FILE_LOAD_GLOBAL 0 /* not module */ -#define FILE_LOAD_MODULE 1 +typedef struct EmbeddedJS_tag EmbeddedJS; -typedef struct EmbeddedJS_tag { - JSRuntime *rt; - JSContext *ctx; - struct trace_malloc_data trace_data; /* = { NULL }; */ - int optind; - char *expr; /* = NULL; */ - int interactivel; /* = 0; */ - int dump_memory; /* = 0; */ - int trace_memory; /* = 0;*/ - int empty_run; /* = 0; */ - int loadMode; /* FILE_LOAD_AUTODETECT */ - int load_std; /* = 0; */ - int dump_unhandled_promise_rejection; /* = 0; */ - size_t memory_limit; /* = 0; */ - char *include_list[32]; -#ifdef CONFIG_BIGNUM - int load_jscalc; -#endif - size_t stack_size; /* = 0; */ -} EmbeddedJS; +struct JSValueBox_tag; + +typedef struct JSValueBox_tag JSValueBox; -JSValue ejsEvalBuffer(EmbeddedJS *ejs, - const void *buffer, int bufferLength, - const char *filename, int eval_flags, - int *statusPtr); +JSValueBox ejsEvalBuffer(EmbeddedJS *ejs, + const void *buffer, int bufferLength, + const char *filename, int eval_flags, + int *statusPtr); int ejsEvalFile(EmbeddedJS *ejs, const char *filename, int loadMode); -void ejsFreeJSValue(EmbeddedJS *ejs, JSValue value); +void ejsFreeJSValue(EmbeddedJS *ejs, JSValueBox valueBox); /* usually pass NULL as arg unless trying to build complex embedded JS application that can run multiple threads or other multitasking. */ -EmbeddedJS *makeEmbeddedJS(EmbeddedJS *sharedRuntimeEJS); +EmbeddedJS *makeEmbeddedJS(EmbeddedJS *sharedEJS); -Json *ejsJSToJson(EmbeddedJS *ejs, JSValue value, ShortLivedHeap *slh); -JSValue ejsJsonToJS(EmbeddedJS *ejs, Json *json); -int ejsSetGlobalProperty(EmbeddedJS *ejs, const char *propetyName, JSValue value); +Json *ejsJSToJson(EmbeddedJS *ejs, JSValueBox value, ShortLivedHeap *slh); +JSValueBox ejsJsonToJS(EmbeddedJS *ejs, Json *json); +int ejsSetGlobalProperty(EmbeddedJS *ejs, const char *propetyName, JSValueBox valueBox); Json *evaluateJsonTemplates(EmbeddedJS *ejs, ShortLivedHeap *slh, Json *json); #endif