From 798648931ffa40bfd1e5299a7199f8affc1990a4 Mon Sep 17 00:00:00 2001 From: Orest Zborowski Date: Mon, 16 Aug 2021 11:48:42 -0400 Subject: [PATCH 1/3] Compile with /W3 on Windows Enable /WX (treat warnings as errors). Fix some basic errors that were found. Enable /FC (emit full path on diagnostics) to make it easier for IDE. Add (commented out) entries for enabling debug builds for dev. --- src/SDSFilePython.cpp | 22 ++++---- src/SharedMemory.cpp | 116 +++++++++++++++++++++--------------------- src/setup.py | 4 +- 3 files changed, 72 insertions(+), 70 deletions(-) diff --git a/src/SDSFilePython.cpp b/src/SDSFilePython.cpp index 8f5b33a1..fd8e8760 100644 --- a/src/SDSFilePython.cpp +++ b/src/SDSFilePython.cpp @@ -425,7 +425,7 @@ StringListToVector(PyObject* listFilenames) { //---------------------------------------------------- // Input: Python List of Tuples(asciiz string,int) -// Output: Write to pListNames. +// Output: Write to pListNames. // ASCIIZ strings follow by UINT8 enum (string1, 0, enum1, string2, 0, enum2, etc.) // Returns: Length of data in pListNames // @@ -724,7 +724,7 @@ const char* FindBang(const char *pString) { //---------------------------------------------------- // Returns true if included bool IsIncluded(PyObject* pInclusionList, const char* pArrayName) { - + // If there is no inclusion list, assume all are included if (pInclusionList) { @@ -794,7 +794,7 @@ void* ReadFromSharedMemory(SDS_SHARED_MEMORY_CALLBACK* pSMCB) { PyObject* pystring = NULL; PyObject* pListName = NULL; - LOGGING("Reading from shared memory\n"); + LOGGING("Reading from shared memory\n"); //----------- LOAD ARRAY NAMES ------------------------- int64_t nameSize = pFileHeader->NameBlockSize; @@ -1033,7 +1033,7 @@ PyObject* ReadFinalWrap(SDS_FINAL_CALLBACK* pSDSFinalCallback) { // CALLBACK2 - can wrap more than one file // finalCount is how many info sections to return // if there are sections inside a single file, the finalCount > 1 -void* ReadFinal(SDS_FINAL_CALLBACK* pSDSFinalCallback, int64_t finalCount) { +void* ReadFinal(SDS_FINAL_CALLBACK* pSDSFinalCallback, int64_t finalCount) { PyObject* returnItem = NULL; @@ -1065,9 +1065,9 @@ void* ReadFinal(SDS_FINAL_CALLBACK* pSDSFinalCallback, int64_t finalCount) { //---------------------------------------- // CALLBACK2 - all files were stacked into one column void* ReadFinalStack( - SDS_STACK_CALLBACK* pSDSFinalCallback, - int64_t finalCount, - SDS_STACK_CALLBACK_FILES *pSDSFileInfo, + SDS_STACK_CALLBACK* pSDSFinalCallback, + int64_t finalCount, + SDS_STACK_CALLBACK_FILES *pSDSFileInfo, SDS_FILTER* pSDSFilter, int64_t fileCount) { @@ -1396,7 +1396,7 @@ PyObject *CompressFile(PyObject* self, PyObject *args, PyObject *kwargs) SDSWriteFile( fileName, shareName, // can be NULL - folderName, + folderName, &SDSWriteInfo, &SDSWriteCallbacks ); @@ -1526,7 +1526,7 @@ GetFilters(PyObject* kwargs, SDS_READ_CALLBACKS* pRCB) { PyObject* maskItem = PyDict_GetItemString(kwargs, "mask"); if (maskItem && PyArray_Check(maskItem)) { - if (PyArray_TYPE((PyArrayObject*)maskItem) == NPY_BOOL) { + if (PyArray_TYPE((PyArrayObject*)maskItem) == NPY_BOOL) { pRCB->Filter.BoolMaskLength = ArrayLength((PyArrayObject*)maskItem); pRCB->Filter.pBoolMask = (bool*)PyArray_GETPTR1((PyArrayObject*)maskItem, 0); // Needed @@ -1662,7 +1662,7 @@ PyObject *DecompressFile(PyObject* self, PyObject *args, PyObject *kwargs) { // PyObject* InternalDecompressFiles( - PyObject* self, + PyObject* self, PyObject* args, PyObject* kwargs, int multiMode) { @@ -1880,7 +1880,7 @@ PyObject *SetLustreGateway(PyObject* self, PyObject *args) { //printf("hint: %s\n", fileName); - g_gatewaylist.empty(); + g_gatewaylist.clear(); int64_t tupleLength = PyTuple_GET_SIZE(tuple); g_gatewaylist.reserve(tupleLength); diff --git a/src/SharedMemory.cpp b/src/SharedMemory.cpp index 0f65a8ad..94fc9f30 100644 --- a/src/SharedMemory.cpp +++ b/src/SharedMemory.cpp @@ -30,7 +30,7 @@ void LogWarningLE(HRESULT error) { //------------------------------------------------------------------------------------------ // Checks Windows policy settings if the current process has // the privilege enabled. -// +// bool CheckWindowsPrivilege(const char* pPrivilegeName) { @@ -92,7 +92,7 @@ bool SetPrivilege( if (!LookupPrivilegeValue( NULL, // lookup privilege on local system - lpszPrivilege, // privilege to lookup + lpszPrivilege, // privilege to lookup &luid)) // receives LUID of privilege { printf("LookupPrivilegeValue error: %u\n", GetLastError()); @@ -369,7 +369,7 @@ UtilSharedNumaMemoryBegin( // The pReturnStruct will be valid if the call succeeded // if bTest = true, it will not complain if it cannot find shared memory // if returns S_OK you are mapped -// returns S_FALSE if it does not exist yet +// returns S_FALSE if it does not exist yet HRESULT UtilSharedMemoryCopy( const char* pMappingName, @@ -488,17 +488,17 @@ UtilMappedViewReadBegin( PULONG pulFile; PMAPPED_VIEW_STRUCT pMappedViewStruct; - const char* pMappingName; + //UNUSED? const char* pMappingName; // // NULL indicates failure - default to that. // *pReturnStruct = NULL; - if (!CheckWindowsSharedMemoryPrerequisites(pMappingName)) - { - return -(S_FALSE); - } + //UNUSED? if (!CheckWindowsSharedMemoryPrerequisites(pMappingName)) + //UNUSED? { + //UNUSED? return -(S_FALSE); + //UNUSED? } // // Allocate fixed, zero inited memory @@ -543,27 +543,27 @@ UtilMappedViewReadBegin( return(HRESULT_FROM_WIN32(GetLastError())); } - // - // Break off any \ or : or / in the file - // - // Search for the last one - // - { - const char* pTemp = pszFilename; - pMappingName = pTemp; - - while (*pTemp != 0) { - - if (*pTemp == '\\' || - *pTemp == ':' || - *pTemp == '/') { - - pMappingName = pTemp + 1; - } - - pTemp++; - } - } + //UNUSED? // + //UNUSED? // Break off any \ or : or / in the file + //UNUSED? // + //UNUSED? // Search for the last one + //UNUSED? // + //UNUSED? { + //UNUSED? const char* pTemp = pszFilename; + //UNUSED? pMappingName = pTemp; + //UNUSED? + //UNUSED? while (*pTemp != 0) { + //UNUSED? + //UNUSED? if (*pTemp == '\\' || + //UNUSED? *pTemp == ':' || + //UNUSED? *pTemp == '/') { + //UNUSED? + //UNUSED? pMappingName = pTemp + 1; + //UNUSED? } + //UNUSED? + //UNUSED? pTemp++; + //UNUSED? } + //UNUSED? } // // We create a file mapping in order to map the file @@ -674,17 +674,17 @@ UtilMappedViewWriteBegin( PULONG pulFile; PMAPPED_VIEW_STRUCT pMappedViewStruct; - const char* pMappingName; + //UNUSED? const char* pMappingName; // // NULL indicates failure - default to that. // *pReturnStruct = NULL; - if (!CheckWindowsSharedMemoryPrerequisites(pMappingName)) - { - return -(S_FALSE); - } + //UNUSED? if (!CheckWindowsSharedMemoryPrerequisites(pMappingName)) + //UNUSED? { + //UNUSED? return -(S_FALSE); + //UNUSED? } // // Allocate fixed, zero inited memory @@ -729,27 +729,27 @@ UtilMappedViewWriteBegin( return (HRESULT_FROM_WIN32(GetLastError())); } - // - // Break off any \ or : or / in the file - // - // Search for the last one - // - { - const char* pTemp = pszFilename; - pMappingName = pTemp; - - while (*pTemp != 0) { - - if (*pTemp == '\\' || - *pTemp == ':' || - *pTemp == '/') { - - pMappingName = pTemp + 1; - } - - pTemp++; - } - } + //UNUSED? // + //UNUSED? // Break off any \ or : or / in the file + //UNUSED? // + //UNUSED? // Search for the last one + //UNUSED? // + //UNUSED? { + //UNUSED? const char* pTemp = pszFilename; + //UNUSED? pMappingName = pTemp; + //UNUSED? + //UNUSED? while (*pTemp != 0) { + //UNUSED? + //UNUSED? if (*pTemp == '\\' || + //UNUSED? *pTemp == ':' || + //UNUSED? *pTemp == '/') { + //UNUSED? + //UNUSED? pMappingName = pTemp + 1; + //UNUSED? } + //UNUSED? + //UNUSED? pTemp++; + //UNUSED? } + //UNUSED? } // // We create a file mapping inorder to map the file @@ -853,7 +853,7 @@ UtilSharedMemoryBegin( // Our memory buffer will be readable and writable: int protection = PROT_READ | PROT_WRITE; - + //MAP_HUGETLB(since Linux 2.6.32) // Allocate the mapping using "huge pages." See the Linux kernel // source file Documentation / vm / hugetlbpage.txt for further @@ -941,7 +941,7 @@ UtilSharedMemoryCopy( // Our memory buffer will be readable and writable: int protection = PROT_READ; - + if (!bCanOnlyRead) { protection |= PROT_WRITE; } @@ -969,7 +969,7 @@ UtilSharedMemoryCopy( // Allocate memory // PMAPPED_VIEW_STRUCT pMappedViewStruct; - + pMappedViewStruct= static_cast(WORKSPACE_ALLOC(sizeof(MAPPED_VIEW_STRUCT))); diff --git a/src/setup.py b/src/setup.py index d2cc8779..05801529 100644 --- a/src/setup.py +++ b/src/setup.py @@ -75,7 +75,9 @@ #extra_compile_args = ['/MT /Ox /Ob2 /Oi /Ot'], # For MSVC windows compiler 2019 it has the new __CxxFrameHandler4 which is found in vcrntime140_1.dll which is not on all systems # We use /dsFH4- to disable this frame handler - extra_compile_args = ['/Ox','/Ob2','/Oi','/Ot','/d2FH4-'], + extra_compile_args = ['/Ox','/Ob2','/Oi','/Ot','/d2FH4-','/W3','/WX','/FC'], + #extra_compile_args = ['/Od','/Z7','/d2FH4-','/W3','/WX','/FC'], + #extra_link_args = ['/debug'] ) setuptools.setup( From e250e18154658b3305fe29bcf8f5a2cb601bfc52 Mon Sep 17 00:00:00 2001 From: Orest Zborowski Date: Mon, 16 Aug 2021 15:15:28 -0400 Subject: [PATCH 2/3] Replace deprecated PyUnicode_GET_SIZE() --- src/RipTide.cpp | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/RipTide.cpp b/src/RipTide.cpp index e371aaf8..07d70c53 100644 --- a/src/RipTide.cpp +++ b/src/RipTide.cpp @@ -756,7 +756,7 @@ PyArrayObject* AllocateNumpyArrayForData(int ndim, npy_intp* dims, int32_t numpy // nullify string for special matlab runt case *pData = 0; } - + if (!returnObject) { printf("!!!out of memory allocating numpy array size:%llu dims:%d dtype:%d itemsize:%lld flags:%d dim0:%lld\n", len, ndim, numpyType, itemsize, array_flags, (int64_t)dims[0]); @@ -961,12 +961,12 @@ bool ConvertSingleItemArray(void* pInput, int16_t numpyInType, _m256all* pDest, fvalue = (double)value; break; CASE_NPY_UINT64: - + value = (int64_t)*(uint64_t*)pInput; fvalue = (double)value; break; CASE_NPY_INT64: - + value = (int64_t)*(int64_t*)pInput; fvalue = (double)value; break; @@ -998,9 +998,9 @@ bool ConvertSingleItemArray(void* pInput, int16_t numpyInType, _m256all* pDest, pDest->i = _mm256_set1_epi32((int32_t)value); break; CASE_NPY_UINT64: - + CASE_NPY_INT64: - + pDest->ci.i1 = _mm_set1_epi64x(value); pDest->ci.i2 = _mm_set1_epi64x(value); break; @@ -1071,7 +1071,7 @@ bool ConvertScalarObject(PyObject* inObject1, _m256all* pDest, int16_t numpyOutT pDest->i = _mm256_set1_epi32((int32_t)value); break; CASE_NPY_UINT64: - + CASE_NPY_INT64: pDest->ci.i1 = _mm_set1_epi64x(value); @@ -1151,12 +1151,12 @@ bool ConvertScalarObject(PyObject* inObject1, _m256all* pDest, int16_t numpyOutT pDest->i = _mm256_set1_epi32((uint32_t)value2); break; CASE_NPY_INT64: - + pDest->ci.i1 = _mm_set1_epi64x(value); pDest->ci.i2 = _mm_set1_epi64x(value); break; CASE_NPY_UINT64: - + pDest->ci.i1 = _mm_set1_epi64x(value2); pDest->ci.i2 = _mm_set1_epi64x(value2); break; @@ -1196,12 +1196,12 @@ bool ConvertScalarObject(PyObject* inObject1, _m256all* pDest, int16_t numpyOutT pDest->i = _mm256_set1_epi32((int32_t)value); break; CASE_NPY_UINT64: - + pDest->ci.i1 = _mm_set1_epi64x((uint64_t)value); pDest->ci.i2 = _mm_set1_epi64x((uint64_t)value); break; CASE_NPY_INT64: - + pDest->ci.i1 = _mm_set1_epi64x((int64_t)value); pDest->ci.i2 = _mm_set1_epi64x((int64_t)value); break; @@ -1228,7 +1228,11 @@ bool ConvertScalarObject(PyObject* inObject1, _m256all* pDest, int16_t numpyOutT } else if (PyUnicode_Check(inObject1)) { // happens when pass in 'test' - *pItemSize = PyUnicode_GET_SIZE(inObject1) * 4; + if (PyUnicode_READY(inObject1) < 0) { + printf("!!unable to make UNICODE object ready"); + return false; + } + *pItemSize = PyUnicode_GET_LENGTH(inObject1) * 4; // memory leak needs to be deleted *ppDataIn = PyUnicode_AsUCS4Copy(inObject1); return true; @@ -1944,7 +1948,7 @@ PyMODINIT_FUNC PyInit_riptide_cpp() { LOGGING("An error occurred when creating/registering SDS Python types."); return NULL; } - + // start up the worker threads now in case we use them g_cMathWorker->StartWorkerThreads(0); @@ -1991,12 +1995,12 @@ void* GetDefaultForType(int numpyInType) { case NPY_INT16: pgDefault = &gDefaultInt16; break; - CASE_NPY_INT32: + CASE_NPY_INT32: // case NPY_INT: This is the same numeric value as NPY_INT32 above pgDefault = &gDefaultInt32; break; CASE_NPY_INT64: - + pgDefault = &gDefaultInt64; break; case NPY_UINT8: @@ -2010,7 +2014,7 @@ void* GetDefaultForType(int numpyInType) { pgDefault = &gDefaultUInt32; break; CASE_NPY_UINT64: - + pgDefault = &gDefaultUInt64; break; case NPY_STRING: From ce0d26d234c5f5332c4b4f2635376306bd4a8533 Mon Sep 17 00:00:00 2001 From: Orest Zborowski Date: Mon, 16 Aug 2021 15:16:41 -0400 Subject: [PATCH 3/3] Fix crash double-closing SDS handle --- src/SDSFile.cpp | 161 ++++++++++++++++++++++++------------------------ 1 file changed, 81 insertions(+), 80 deletions(-) diff --git a/src/SDSFile.cpp b/src/SDSFile.cpp index 4e1f829b..f9d772a5 100644 --- a/src/SDSFile.cpp +++ b/src/SDSFile.cpp @@ -29,7 +29,7 @@ extern int64_t SumBooleanMask(const int8_t* pIn, int64_t length); //----------------------------------------------- // SDS File Format // First 512 bytes - SDS_FILE_HEADER -// +// // --- SDS_FILE_HEADER // - NameBlock ptr (list of array names and types) (variable lengths) // - MetaBlock ptr (meta data json string often compressed) (one string block often compressed) @@ -401,7 +401,7 @@ int64_t SDSFileSize(const char* fileName) { if (GetFileAttributesEx(fileName, GetFileExInfoStandard, &fileInfo)) { LARGE_INTEGER li; - + li.LowPart = fileInfo.nFileSizeLow; li.HighPart = fileInfo.nFileSizeHigh; return li.QuadPart; @@ -429,7 +429,7 @@ SDS_FILE_HANDLE SDSFileOpen(const char* fileName, bool writeOption, bool overlap } } - // open the existing file for reading + // open the existing file for reading SDS_FILE_HANDLE Handle = CreateFile ( fileName, @@ -1041,7 +1041,7 @@ class SDSIncludeExclude { //------------------------------------------------------------- // Tries to find the '!' char in the string // if it finds the ! it returns the location - // + // const char* FindSep(const char *pString, char SEPARATOR) { while (*pString) { if (*pString == SEPARATOR) return pString; @@ -1184,7 +1184,7 @@ class SDSIncludeExclude { }; //------------------------------------------------------- -// Returns bytesPerRow +// Returns bytesPerRow int64_t GetBytesPerRow(SDS_ARRAY_BLOCK* pBlockInfo) { // calculate how many rows @@ -1234,7 +1234,7 @@ GetBytesPerBand(SDSArrayInfo* pArrayInfo, int64_t bandSize, int64_t* bandCount=N //------------------------------------ // -static size_t +static size_t DecompressWithFilter( int64_t compressedSize, // pBlockInfo->ArrayCompressedSize int64_t uncompressedSize, @@ -1244,7 +1244,7 @@ DecompressWithFilter( SDS_FILE_HANDLE fileHandle, void* tempBuffer, // used to decompress void* destBuffer, // the array buffer (final destination of data) - SDS_FILTER* pFilter, + SDS_FILTER* pFilter, int64_t rowOffset, // the original row offset int64_t stackIndex, // when stacking, the stack # int32_t core, // thread # we are on @@ -1463,7 +1463,7 @@ ReadAndDecompressBandWithFilter( int64_t runningTrueCount = 0; LOGGING("[%lld] seek to %lld compsz:%lld uncompsz:%lld stackIndex:%lld bytesPerRow:%lld <-- ReadAndDecompressBandWithFilter\n", rowOffset, pBlockInfo->ArrayDataOffset, pBlockInfo->ArrayCompressedSize, pBlockInfo->ArrayUncompressedSize, stackIndex, bytesPerRow); - + for (int32_t i = 0; i < pBlockInfo->ArrayBandCount; i++) { int64_t compressedSize = pBands[i] - previousSize; previousSize = pBands[i]; @@ -1548,7 +1548,7 @@ ReadAndDecompressBandWithFilter( // // On Entry tempBuffer is guaranteed to be at least the compressedSize // The mask must always be a boolean mask -// If a mask exists, data +// If a mask exists, data // return size of bytes read or -1 on failure static size_t ReadAndDecompressArrayBlockWithFilter( @@ -1619,8 +1619,8 @@ ReadAndDecompressArrayBlockWithFilter( GetBytesPerRow(pBlockInfo), eventHandle, fileHandle, - tempBuffer, - destBuffer, + tempBuffer, + destBuffer, pFilter, rowOffset, stackIndex, @@ -1634,8 +1634,8 @@ ReadAndDecompressArrayBlockWithFilter( if (didAlloc && tempBuffer) { WORKSPACE_FREE(tempBuffer); } - - + + return result; } @@ -1646,13 +1646,13 @@ ReadAndDecompressArrayBlockWithFilter( // // On Entry tempBuffer is guaranteed to be at least the compressedSize // The mask must always be a boolean mask -// If a mask exists, data +// If a mask exists, data // return size of bytes read or -1 on failure static size_t ReadAndDecompressArrayBlock( SDS_ARRAY_BLOCK *pBlockInfo, // may contain banding information SDS_EVENT_HANDLE eventHandle, - SDS_FILE_HANDLE fileHandle, + SDS_FILE_HANDLE fileHandle, void* tempBuffer, // used to decompress void* destBuffer, // the array buffer (final destination of data) int64_t arrayIndex, // the array # we are on @@ -1678,7 +1678,7 @@ ReadAndDecompressArrayBlock( } } else - + if (tempBuffer) { if (pBlockInfo->ArrayBandCount > 0) { @@ -1838,13 +1838,13 @@ void FillFileHeader( // For version 4.4 pFileHeader->SectionBlockSize =0; pFileHeader->SectionBlockOffset =0; // points to section directory if it exists - pFileHeader->SectionBlockCount = 0; + pFileHeader->SectionBlockCount = 0; pFileHeader->SectionBlockReservedSize = 0; pFileHeader->FileOffset = fileOffset; pFileHeader->TimeStampUTCNanos = 0; for (uint64_t i = 0; i < sizeof(pFileHeader->Reserved); i++) { - pFileHeader->Reserved[i] = 0; + pFileHeader->Reserved[i] = 0; } } @@ -2180,10 +2180,10 @@ SDS_FILE_HANDLE StartCompressedFile( // Fill FileHeader with LOGGING("Using fileOffset %lld\n", fileOffset); - FillFileHeader(pFileHeader, fileOffset, COMPRESSION_MODE_COMPRESS_FILE, + FillFileHeader(pFileHeader, fileOffset, COMPRESSION_MODE_COMPRESS_FILE, compType, compLevel, - fileType, stackType, - authorId, + fileType, stackType, + authorId, listNameLength, listNameCount, cSize, arrayCount, bandSize); @@ -2208,7 +2208,7 @@ SDS_FILE_HANDLE StartCompressedFile( //diff = diff - cSize; //if (diff > 0) { // fwrite(filler, diff, 1, fileHandle); - //} + //} //WORKSPACE_FREE(filler); // Check if we allocated when compressing meta @@ -2241,7 +2241,7 @@ void EndCompressedFile( if (pWriteInfo->sectionName) { SDSSectionName cSDSSectionName; - // This is the first section name + // This is the first section name char* pListNames = NULL; // Are we the first entry? @@ -2325,7 +2325,7 @@ void EndCompressedFile( } if (pListNames) WORKSPACE_FREE(pListNames); - + } //Timestamp @@ -2466,10 +2466,10 @@ bool CompressFileArray(void* pstCompressArraysV, int32_t core, int64_t t) { // Calculate how much to allocate int64_t dest_size = source_size; int64_t wantedSize = source_size; - + if (pstCompressArrays->compType == COMPRESSION_TYPE_ZSTD) { - // TODO: subroutine + // TODO: subroutine if (bandSize > 0) { // calculate how many bands bytesPerBand = GetBytesPerBand(pArrayInfo, bandSize, &bandCount); @@ -2630,7 +2630,7 @@ bool CompressFileArray(void* pstCompressArraysV, int32_t core, int64_t t) { LOGGING("[%lld][%lld] seek to fileOffset %lld sz: %lld\n", t, arrayNumber, fileOffset, pArrayBlock->ArrayCompressedSize); //=========================== - // Write compressed chunk + // Write compressed chunk int64_t result = DefaultFileIO.FileWriteChunk( pstCompressArrays->eventHandles[core], @@ -2713,7 +2713,7 @@ void sigbus_termination_handler(int32_t signum) { // totalItemSize -?? // // metaData -- block of bytes to store as metadata -// metaDataSize -- +// metaDataSize -- // bool SDSWriteFileInternal( const char * fileName, @@ -3198,7 +3198,7 @@ bool IsNameIncluded( if (pFolderName->IsIncluded(nameToCheck)) return true; } } - + } else { // just a name -- no folder -- probably not allowed @@ -3314,9 +3314,9 @@ class SDSDecompressFile { // pInclude = NULL is allowed // shareName NULL is allowed SDSDecompressFile( - const char* fileName, + const char* fileName, SDSIncludeExclude* pInclude = NULL, - int64_t instanceIndex=0, + int64_t instanceIndex=0, const char* shareName=NULL, SDSIncludeExclude* pFolderName=NULL, SDSIncludeExclude* pSectionsName = NULL, @@ -3698,7 +3698,7 @@ class SDSDecompressFile { // // Returns: // true or false. if true and NULL passed in then can call GetMetaData() - // + // // bool DecompressMetaData( SDS_FILE_HEADER* pFileHeader, @@ -3794,7 +3794,7 @@ class SDSDecompressFile { //------------------------------------------------ // Returns false on failure // Returns true on success - // + // // Reads from File and decompresses into shared memory // Call EndDecompressedFile() when done bool @@ -3827,7 +3827,7 @@ class SDSDecompressFile { HRESULT hr = DefaultMemoryIO.Begin(totalSize); if (hr >= 0) { - // + // // Fileheader for shared memory SDS_FILE_HEADER* pMemoryFileHeader = DefaultMemoryIO.GetFileHeader(); @@ -3908,7 +3908,7 @@ class SDSDecompressFile { fileType == SDS_FILE_TYPE_TABLE || fileType == SDS_FILE_TYPE_ARRAY); - // MORE WORK TO DO + // MORE WORK TO DO SDS_READ_DECOMPRESS_ARRAYS* pstCompressArrays = AllocDecompressArrays(pReadCallbacks, arrayCount, true, oneFile, fileTypeStackable); @@ -3928,7 +3928,7 @@ class SDSDecompressFile { pstCompressArrays->pFilter = &pReadCallbacks->Filter; - // TODO: + // TODO: void* saveState = pReadCallbacks->BeginAllowThreads(); g_cMathWorker->DoMultiThreadedWork((int)arrayCount, DecompressFileArray, pstCompressArrays); pReadCallbacks->EndAllowThreads(saveState); @@ -3958,7 +3958,7 @@ class SDSDecompressFile { //------------------------------------------------ // Returns false on failure // Returns true on success - // + // // Calls EndDecompressedFile() when done bool CopyIntoSharedMemory(SDS_READ_CALLBACKS* pReadCallbacks, const char* fileName, SDSIncludeExclude* pFolderName, int32_t core) { @@ -4059,10 +4059,10 @@ class SDSDecompressFile { goto EXIT_DECOMPRESS; } - //------------- META DATA ------------------------------- + //------------- META DATA ------------------------------- DecompressMetaData(pFileHeader, NULL, core); - //------------- SECTION DATA ------------------------------- + //------------- SECTION DATA ------------------------------- // only offset ==0 (first fileheader) is responsible for reading the sections if (startOffset == 0) { if (pFileHeader->SectionBlockCount && cSectionName.ReadListSections(SDSFile, pFileHeader) == NULL) { @@ -4110,9 +4110,9 @@ class SDSDecompressFile { //--------- ALLOCATE COMPRESS ARRAYS --- pstCompressArrays = AllocDecompressArrays( - pReadCallbacks, - tupleSize, - false, + pReadCallbacks, + tupleSize, + false, pFileHeader->FileType == SDS_FILE_TYPE_ONEFILE, fileTypeStackable ); @@ -4688,7 +4688,7 @@ void ConvertDType( typedef void(*SDS_COPY_FORTRAN)( void* pDest, // (upper left corner of array) void* pSrc, // the value to fill each element with - int64_t totalRowsize, // all the rows of all the files + int64_t totalRowsize, // all the rows of all the files int64_t arrayOffset, // start row for the current file int64_t arrayRowsize, // rowLength for the current file int64_t colSize); @@ -4710,7 +4710,7 @@ template void CopyFortran( void* pDestT, // (upper left corner of array) void* pSrcT, // the value to fill each element with - int64_t totalRowsize, // all the rows of all the files + int64_t totalRowsize, // all the rows of all the files int64_t arrayOffset, // start row for the current file int64_t arrayRowsize, // rowLength for the current file int64_t colSize) { @@ -4753,7 +4753,7 @@ SDS_COPY_FORTRAN GetCopyFortran(int32_t dtype) { //----------------------------------------- typedef void(*SDS_GAP_FILL_SPECIAL)( void* pDest, // (upper left corner of array) - int64_t totalRowsize, // all the rows of all the files + int64_t totalRowsize, // all the rows of all the files int64_t arrayOffset, // start row for the current file int64_t arrayRowsize, // rowLength for the current file int64_t colSize); @@ -4765,7 +4765,7 @@ typedef void(*SDS_GAP_FILL_SPECIAL)( template void GapFillSpecial( void* pDestT, // (upper left corner of array) - int64_t totalRowsize, // all the rows of all the files + int64_t totalRowsize, // all the rows of all the files int64_t arrayOffset, // start row for the current file int64_t arrayRowsize, // rowLength for the current file int64_t colSize) { @@ -4803,7 +4803,7 @@ void GapFillSpecial( template void GapFillSpecialRowMajor( void* pDestT, // (upper left corner of array) - int64_t totalRowsize, // all the rows of all the files + int64_t totalRowsize, // all the rows of all the files int64_t arrayOffset, // start row for the current file int64_t arrayRowsize, // rowLength for the current file int64_t colSize) { @@ -4814,7 +4814,7 @@ void GapFillSpecialRowMajor( fill = GET_INVALID(fill); LOG_THREAD("In gap fill special sizeof:%lld %lld %lld %lld %lld %lf\n", sizeof(T), totalRowsize, arrayOffset, arrayRowsize, colSize, (double)fill); - + pDest = pDest + (arrayOffset * colSize); // Read horizontally (contiguous) @@ -4849,13 +4849,13 @@ void GapFillSpecialRowMajor( template void MatlabStringFill( uint16_t* pDest, // 2 byte unicode (see note on location) - int64_t totalRowsize, // all the rows of all the files + int64_t totalRowsize, // all the rows of all the files int64_t arrayOffset, // start row for the current file int64_t arrayRowsize, // rowLength for the current file int64_t itemSizeMaster) { // NOTE: - // the pDest is not top left corner.. rather it is + // the pDest is not top left corner.. rather it is // the top left corner + itemsize * row // // NOTE what happens for a gap fill for UNICODE and matlab is the loader? @@ -4888,13 +4888,13 @@ void MatlabStringFill( template void MatlabStringFillFromUnicode( uint16_t* pDest, // 2 byte unicode (see note on location) - int64_t totalRowsize, // all the rows of all the files + int64_t totalRowsize, // all the rows of all the files int64_t arrayOffset, // start row for the current file int64_t arrayRowsize, // rowLength for the current file int64_t itemSizeMaster) { // NOTE: - // the pDest is not top left corner.. rather it is + // the pDest is not top left corner.. rather it is // the top left corner + itemsize * row // // NOTE what happens for a gap fill for UNICODE and matlab is the loader? @@ -5030,7 +5030,7 @@ void GapFillAny( // row major layout: // A1, B1, C1, D1, E1 <-- first file // A2, B2, C2, D2, E2 <-- second file (row length:2, col:5) (arrayOffset =1 ) -// A3, B3, C3, D3, E3 +// A3, B3, C3, D3, E3 // // Fortran layout (total row length 3, col length:5) // A1, *A2, *A3 @@ -5040,9 +5040,9 @@ void GapFillAny( // E1, *E2, *E3 void RotationalFixup( - char* pDest, // - char* pSrc, // - int64_t totalRowsize, // all the rows of all the files + char* pDest, // + char* pSrc, // + int64_t totalRowsize, // all the rows of all the files int64_t arrayOffset, // start row for the current file int64_t arrayRowsize, // rowLength for the current file int64_t arrayColsize, // colSize of the current file @@ -5086,7 +5086,7 @@ template void MatlabStringFixup( uint16_t* pDest, // 2 byte unicode (upper left corner of array) T* pSrc, // 1 or 4 byte unicode (file upper left corner) - int64_t totalRowsize, // all the rows of all the files + int64_t totalRowsize, // all the rows of all the files int64_t arrayOffset, // start row for the current file int64_t arrayRowsize, // rowLength for the current file int64_t itemSize, @@ -5156,7 +5156,7 @@ void StringFixup( // Code below to be deleted when ugs in filtering gone //int64_t rows = pSlaveBlock->Dimensions[0]; - + // new code for filtering.. int64_t rows = slaveRowLength; @@ -5318,7 +5318,7 @@ SDS_COMPATIBLE IsArrayCompatible( if (hightype == SDS_ULONGLONG && lowtype == SDS_LONGLONG) { // Due to bug with filtering - instead of convert will give warning here... precision loss - // This should be an option + // This should be an option if (handleInt64ToUInt64) { printf("Warning: ignoring sign loss going to from int/uint64 for col: %s\n", colName); // flip it back off @@ -5683,7 +5683,7 @@ bool DecompressMultiArray( } } } - + else { LOG_THREAD("!!bad destBuffer\n"); } @@ -5842,7 +5842,7 @@ class SDSDecompressManyFiles { // input - new or existing column // if new, a new ColumnVector will be created // - // + // // if the name is not found, the unique count goes up void AddColumnList( int64_t validPos, @@ -6076,9 +6076,9 @@ class SDSDecompressManyFiles { // fileSize is the size of the input file // set localOffset to non-zero to indicate a section copy static int64_t AppendToFile( - SDS_FILE_HANDLE outFileHandle, + SDS_FILE_HANDLE outFileHandle, SDSDecompressFile* pSDSDecompress, - int64_t fileOffset, + int64_t fileOffset, int64_t fileSize, char* pSectionData, int64_t ¤tSection) { @@ -6341,14 +6341,14 @@ class SDSDecompressManyFiles { // main routine for reading in multiple files // this routine does NOT stack // may return NULL if not all files exist - // + // void* ReadManyFiles(SDS_READ_CALLBACKS* pReadCallbacks, int32_t multiMode) { // Open what might be 100+ files GetFileInfo(multiMode); // If any of the files have sections, we have to grow the list of files SDSDecompressFile** pSDSDecompressFileExtra = NULL; - + // Check if concat mode if (multiMode != SDS_MULTI_MODE_CONCAT_MANY) { pSDSDecompressFileExtra = ScanForSections(); @@ -6416,7 +6416,7 @@ class SDSDecompressManyFiles { for (int64_t t = 0; t < FileCount; t++) { SDSDecompressFile* pSDSDecompress = pSDSDecompressFile[t]; - // + // pReadFinalCallback[t].strFileName = pSDSDecompress->FileName; if (pSDSDecompress->IsFileValid) { @@ -6481,7 +6481,7 @@ class SDSDecompressManyFiles { WORKSPACE_FREE(pReadFinalCallback); } - + // Tear it down ClearColumnList(); @@ -6503,7 +6503,7 @@ class SDSDecompressManyFiles { pSDSDecompress->FileName, pSDSDecompress->pInclude, instance, // instanceindex - pSDSDecompress->ShareName, + pSDSDecompress->ShareName, pSDSDecompress->pFolderName, pSDSDecompress->pSectionsName, pSDSDecompress->Mode); @@ -6555,7 +6555,7 @@ class SDSDecompressManyFiles { SDSDecompressFile* pSDSDecompressExtra = CopyDecompressFileFrom(pSDSDecompress, instance); pSDSDecompressFileExtra[instance] = pSDSDecompressExtra; - + // read in header for section (note if this is the first header (i.e. section ==0), we are reading it again) // TODO: optimization here int64_t fileoffset = pSDSDecompress->cSectionName.pSectionOffsets[section]; @@ -6565,7 +6565,7 @@ class SDSDecompressManyFiles { } else { pSDSDecompressFileExtra[FileWithSectionsCount] = CopyDecompressFileFrom(pSDSDecompress, FileWithSectionsCount); - + // TJD: Check to make sure no memory leaks, also, future optimization - we do not have to read it again pSDSDecompressFileExtra[FileWithSectionsCount]->DecompressFileInternal(pReadCallbacks, 0, 0); FileWithSectionsCount += 1; @@ -6774,7 +6774,7 @@ class SDSDecompressManyFiles { // Also for each column we create SDSArrayInfo SDSArrayInfo* pManyDestInfo = (SDSArrayInfo*)WORKSPACE_ALLOC(sizeof(SDSArrayInfo)*TotalUniqueColumns); SDSFilterInfo* pFilterInfo = NULL; - + // Check if we are filtering the data with a boolean mask if (pReadCallbacks->Filter.pBoolMask) { // Worst case allocation (only valid files calculated) @@ -6830,9 +6830,9 @@ class SDSDecompressManyFiles { //LOGGING("master %p ", pMasterBlock); int32_t mask = SDS_FLAGS_ORIGINAL_CONTAINER; - bool isFilterable = false; + bool isFilterable = false; - if (pFilterInfo && + if (pFilterInfo && (pKing->ArrayEnum & mask) == mask && (pKing->ArrayEnum & (SDS_FLAGS_SCALAR | SDS_FLAGS_META | SDS_FLAGS_NESTED)) == 0 && fileTypeStackable) { @@ -6905,7 +6905,7 @@ class SDSDecompressManyFiles { if (filterTrueCount == 0) { pSDSDecompress->IsFileValidAndNotFilteredOut = false; filteredOut++; - } + } // The length has been shortened by the filter currentUnfilteredOffset += calcLength; @@ -7228,7 +7228,7 @@ extern "C" { // totalItemSize -?? notused // // metaData -- block of bytes to store as metadata - // metaDataSize -- + // metaDataSize -- // DllExport int32_t SDSWriteFile( const char *fileName, @@ -7253,7 +7253,7 @@ extern "C" { // --- // sharedMemory if provided will check shared memory first // ReadFromSharedMemory must be provided - // + // // Returns what the user specified in ReadFinal DllExport void* SDSReadFile( const char *fileName, @@ -7297,7 +7297,7 @@ extern "C" { // --- // sharedMemory if provided will check shared memory first // ReadFromSharedMemory must be provided - // + // // Returns what the user specified in ReadFinal DllExport void* SDSReadManyFiles( SDS_MULTI_READ* pMultiRead, @@ -7345,6 +7345,7 @@ extern "C" { ReadFileHeader(fileHandle, &tempFileHeader, 0, pMultiRead[0].pFileName); if (result != 0) { + // ReadFileHeader() has already closed the fileHandle upon error. multiMode = SDS_MULTI_MODE_ERROR; } else { @@ -7354,9 +7355,9 @@ extern "C" { else { multiMode = SDS_MULTI_MODE_READ_MANY; } - } - DefaultFileIO.FileClose(fileHandle); + DefaultFileIO.FileClose(fileHandle); + } } } } @@ -7406,7 +7407,7 @@ extern "C" { return false; } - DllExport int32_t CloseDecompressFile(void* pInput) { + DllExport int32_t CloseDecompressFile(void* pInput) { SDSDecompressFile* pSDSDecompressFile = (SDSDecompressFile * )pInput; delete pSDSDecompressFile; return true;