diff --git a/src/Array.cpp b/src/Array.cpp index 3cbee39e..5811930f 100644 --- a/src/Array.cpp +++ b/src/Array.cpp @@ -76,7 +76,7 @@ static PyObject * ConvertInt32(PyObject * object) Py_DecRef((PyObject *)pArray); return NULL; } - pInt32[i] = (int32_t)val; + pInt32[i] = static_cast(val); } return (PyObject *)pArray; } diff --git a/src/BasicMath.cpp b/src/BasicMath.cpp index 05b799f6..a40530d8 100644 --- a/src/BasicMath.cpp +++ b/src/BasicMath.cpp @@ -236,7 +236,7 @@ static const inline T MaxOp(T x, T y) // x & y; } static const int64_t NAN_FOR_INT64 = (int64_t)0x8000000000000000LL; -static const int64_t NAN_FOR_INT32 = (int32_t)0x80000000; +static const int64_t NAN_FOR_INT32 = 0x80000000; template static const inline double DivOp(T x, T y) @@ -3428,9 +3428,9 @@ PyObject * TwoInputsInternal(CTwoInputs & twoInputs, int64_t funcNumber) if (twoInputs.len2 > 1) fl.Input2Strides = PyArray_STRIDE(twoInputs.inArr2, 0); - fl.InputItemSize = - twoInputs.len1 >= twoInputs.len2 ? PyArray_ITEMSIZE(twoInputs.inArr) : PyArray_ITEMSIZE(twoInputs.inArr2); - fl.OutputItemSize = PyArray_ITEMSIZE(outputArray); + fl.InputItemSize = twoInputs.len1 >= twoInputs.len2 ? static_cast(PyArray_ITEMSIZE(twoInputs.inArr)) : + static_cast(PyArray_ITEMSIZE(twoInputs.inArr2)); + fl.OutputItemSize = static_cast(PyArray_ITEMSIZE(outputArray)); fl.NumpyOutputType = wantedOutputType; fl.NumpyType = twoInputs.len1 >= twoInputs.len2 ? twoInputs.numpyInType1 : twoInputs.numpyInType2; diff --git a/src/Compress.cpp b/src/Compress.cpp index 1dffd99d..d4850934 100644 --- a/src/Compress.cpp +++ b/src/Compress.cpp @@ -23,11 +23,11 @@ void FillInNumpyHeader(NUMPY_HEADERSIZE * pstNumpyHeader, int32_t dtype, int32_t pstNumpyHeader->magic = COMPRESSION_MAGIC; pstNumpyHeader->compressiontype = COMPRESSION_TYPE_ZSTD; - pstNumpyHeader->dtype = (int8_t)(dtype); - pstNumpyHeader->ndim = (int8_t)ndim; + pstNumpyHeader->dtype = static_cast(dtype); + pstNumpyHeader->ndim = static_cast(ndim); pstNumpyHeader->flags = flags; - pstNumpyHeader->itemsize = (int32_t)itemsize; + pstNumpyHeader->itemsize = static_cast< int32_t >(itemsize); if (pstNumpyHeader->ndim > 3) { @@ -188,7 +188,7 @@ PyObject * CompressDecompressArrays(PyObject * self, PyObject * args) pstCompressArrays->pNumpyHeaders[t] = AllocCompressedMemory(aInfo[t].ArrayLength, aInfo[t].NumpyDType, aInfo[t].NDim, (int64_t *)(((PyArrayObject_fields *)aInfo[t].pObject)->dimensions), - PyArray_FLAGS(aInfo[t].pObject), aInfo[t].ItemSize); + PyArray_FLAGS(aInfo[t].pObject), static_cast(aInfo[t].ItemSize)); } // This will kick off the workerthread and call CompressMemoryArray diff --git a/src/Convert.cpp b/src/Convert.cpp index 7fd98f5e..c4c91d28 100644 --- a/src/Convert.cpp +++ b/src/Convert.cpp @@ -7,6 +7,8 @@ #include "Reduce.h" #include +#include +#include #define LOGGING(...) //#define LOGGING printf @@ -1049,8 +1051,8 @@ void * GetInvalid(int dtype) // NOTE: if they are the same type, special fast routine called PyObject * ConvertSafeInternal(PyArrayObject * const inArr1, const int64_t out_dtype) { - const int32_t numpyOutType = (int32_t)out_dtype; - const int32_t numpyInType = PyArray_TYPE(inArr1); + const int32_t numpyOutType = static_cast(out_dtype); + const int32_t numpyInType = static_cast(PyArray_TYPE(inArr1)); if (numpyOutType < 0 || numpyInType > NPY_LONGDOUBLE || numpyOutType > NPY_LONGDOUBLE) { @@ -1251,8 +1253,8 @@ PyObject * ConvertSafe(PyObject * self, PyObject * args) // GetConversionFunctionUnsafe. PyObject * ConvertUnsafeInternal(PyArrayObject * inArr1, int64_t out_dtype) { - const int32_t numpyOutType = (int32_t)out_dtype; - const int32_t numpyInType = ObjectToDtype(inArr1); + const int32_t numpyOutType = static_cast(out_dtype); + const int32_t numpyInType = static_cast(ObjectToDtype(inArr1)); if (numpyOutType < 0 || numpyInType < 0 || numpyInType > NPY_LONGDOUBLE || numpyOutType > NPY_LONGDOUBLE) { @@ -2085,6 +2087,12 @@ int64_t Combine1Filter(void * pInputIndex, { for (int64_t i = 0; i < arrayLength; i++) { + if (i > std::numeric_limits::max()) + { + std::string errmsg{"Combine1Filter has attempted to exceed the size of the index value "}; + errmsg += std::to_string(i); + throw std::overflow_error(errmsg); + } if (pFilter[i]) { INDEX index = pInput[i]; @@ -2096,7 +2104,7 @@ int64_t Combine1Filter(void * pInputIndex, if (pHash[index] == 0) { // First time, assign FirstKey - pNewFirst[uniquecount] = (int32_t)i; + pNewFirst[uniquecount] = static_cast(i); uniquecount++; // printf("reassign index:%lld to bin:%d\n", (int64_t)index, @@ -2110,7 +2118,7 @@ int64_t Combine1Filter(void * pInputIndex, { // Get reassigned key // printf("exiting index:%lld to bin:%d\n", (int64_t)index, - // (int32_t)pHash[index]); + // pHash[index]); pOutput[i] = (INDEX)pHash[index]; } } @@ -2132,6 +2140,12 @@ int64_t Combine1Filter(void * pInputIndex, // When no filter provided for (int64_t i = 0; i < arrayLength; i++) { + if (i > std::numeric_limits::max()) + { + std::string errmsg{"Combine1Filter has attempted to exceed the size of the index value, no filter provided "}; + errmsg += std::to_string(i); + throw std::overflow_error(errmsg); + } INDEX index = pInput[i]; // printf("[%lld] got index\n", (int64_t)index); @@ -2141,7 +2155,7 @@ int64_t Combine1Filter(void * pInputIndex, if (pHash[index] == 0) { // First time, assign FirstKey - pNewFirst[uniquecount] = (int32_t)i; + pNewFirst[uniquecount] = static_cast(i); uniquecount++; // ReassignKey @@ -2258,7 +2272,17 @@ PyObject * CombineAccum1Filter(PyObject * self, PyObject * args) { int32_t * pFirst = (int32_t *)PyArray_BYTES(firstArray); - int64_t uniqueCount = pFunction(pDataIn1, PyArray_BYTES(outArray), pFirst, pFilterIn, arraySize1, hashSize); + int64_t uniqueCount{}; + + try + { + uniqueCount = pFunction(pDataIn1, PyArray_BYTES(outArray), pFirst, pFilterIn, arraySize1, hashSize); + } + catch ( std::exception &e ) + { + PyErr_Format(PyExc_ValueError, e.what()); + return nullptr; + } if (uniqueCount < arraySize1) { @@ -3242,7 +3266,7 @@ PyObject * HomogenizeArrays(PyObject * self, PyObject * args) if (aInfo) { - int32_t dtype = (int32_t)PyLong_AsLong(dtypeObject); + int32_t dtype = PyLong_AsLong(dtypeObject); if (dtype != -1) { @@ -4017,7 +4041,7 @@ PyObject * ApplyRows(PyObject * self, PyObject * args, PyObject * kwargs) if (aInfo) { - int32_t dtype = (int32_t)PyLong_AsLong(dtypeObject); + int32_t dtype = PyLong_AsLong(dtypeObject); if (dtype != -1) { diff --git a/src/Ema.cpp b/src/Ema.cpp index 38605196..9bec05d1 100644 --- a/src/Ema.cpp +++ b/src/Ema.cpp @@ -725,7 +725,7 @@ static void CumSum(void * pKeyT, void * pAccumBin, void * pColumn, int64_t numUn U Invalid = GET_INVALID(pDest[0]); - int32_t windowSize = (int32_t)windowSize1; + int32_t windowSize = std::lround(windowSize1); LOGGING("cumsum %lld %lld %lld %p %p\n", numUnique, totalInputRows, (int64_t)Invalid, pIncludeMask, pResetMask); @@ -848,7 +848,7 @@ static void CumProd(void * pKeyT, void * pAccumBin, void * pColumn, int64_t numU U Invalid = GET_INVALID(pDest[0]); - int32_t windowSize = (int32_t)windowSize1; + int32_t windowSize = std::lround(windowSize1); LOGGING("cumprod %lld %lld %p %p\n", numUnique, totalInputRows, pIncludeMask, pResetMask); @@ -1196,7 +1196,7 @@ class EmaByBase // (double)pLastEma[location], (double)-decayRate, (double)pTime[i], // (double)pLastTime[location] ); pLastEma[location] = pSrc[i] + pLastEma[location] * exp(-decayRate * (pTime[i] - pLastTime[location])); - // printf("[%d][%d] %lf\n", i, (int32_t)location, + // printf("[%d][%d] %lf\n", i, location, // (double)pLastEma[location]); pLastTime[location] = pTime[i]; pDest[i] = pLastEma[location]; @@ -1827,7 +1827,7 @@ void EmaByCall(void * pEmaBy, int64_t i) //--------------------------------------------------------------- // Arg1 = LIST of numpy arrays which has the values to accumulate (often all the -// columns in a dataset) Arg2 = iKey = numpy array (int32_t) which has the index +// columns in a dataset) Arg2 = iKey = numpy array which has the index // to the unique keys (ikey from MultiKeyGroupBy32) Arg3 = integer unique rows // Arg4 = integer (function number to execute for cumsum, ema) // Arg5 = params for function must be (decay/window, time, includemask, @@ -1958,7 +1958,7 @@ PyObject * EmaAll32(PyObject * self, PyObject * args) stEma32 * pstEma32 = (stEma32 *)WORKSPACE_ALLOC((sizeof(stEma32) + 8 + sizeof(stEmaReturn)) * tupleSize); pstEma32->aInfo = aInfo; - pstEma32->funcNum = (int32_t)funcNum; + pstEma32->funcNum = static_cast(funcNum); pstEma32->pKey = (int32_t *)PyArray_BYTES(iKey); pstEma32->tupleSize = tupleSize; pstEma32->uniqueRows = unique_rows; diff --git a/src/FileReadWrite.cpp b/src/FileReadWrite.cpp index 1b1d6326..3b8ef3da 100644 --- a/src/FileReadWrite.cpp +++ b/src/FileReadWrite.cpp @@ -158,8 +158,8 @@ DWORD CFileReadWrite::ReadChunk(void * buffer, uint32_t count) OverlappedIO.hEvent = NULL; OverlappedIO.InternalHigh = 0; OverlappedIO.Internal = 0; - OverlappedIO.OffsetHigh = (uint32_t)(BufferPos >> 32); - OverlappedIO.Offset = (uint32_t)BufferPos; + OverlappedIO.OffsetHigh = (BufferPos >> 32); + OverlappedIO.Offset = BufferPos; bool bReadDone; @@ -215,8 +215,8 @@ DWORD CFileReadWrite::ReadChunkAsync(void * buffer, uint32_t count, DWORD * last { pOverlapped->InternalHigh = 0; pOverlapped->Internal = 0; - pOverlapped->OffsetHigh = (uint32_t)(BufferPos >> 32); - pOverlapped->Offset = (uint32_t)BufferPos; + pOverlapped->OffsetHigh = (BufferPos >> 32); + pOverlapped->Offset = BufferPos; bool bReadDone; @@ -330,8 +330,8 @@ DWORD CFileReadWrite::WriteChunk(void * buffer, uint32_t count) OverlappedIO.hEvent = 0; OverlappedIO.InternalHigh = 0; OverlappedIO.Internal = 0; - OverlappedIO.OffsetHigh = (uint32_t)(BufferPos >> 32); - OverlappedIO.Offset = (uint32_t)BufferPos; + OverlappedIO.OffsetHigh = (BufferPos >> 32); + OverlappedIO.Offset = BufferPos; bool bWriteDone; @@ -391,8 +391,8 @@ DWORD CFileReadWrite::WriteChunkAsync(void * buffer, uint32_t count, DWORD * las { pOverlapped->InternalHigh = 0; pOverlapped->Internal = 0; - pOverlapped->OffsetHigh = (uint32_t)(BufferPos >> 32); - pOverlapped->Offset = (uint32_t)BufferPos; + pOverlapped->OffsetHigh = (BufferPos >> 32); + pOverlapped->Offset = BufferPos; bool bWriteDone; diff --git a/src/GroupBy.cpp b/src/GroupBy.cpp index 7ef5827d..68005840 100644 --- a/src/GroupBy.cpp +++ b/src/GroupBy.cpp @@ -1031,7 +1031,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t nth = (int32_t)funcParam; + int64_t nth = funcParam; U invalid = GET_INVALID(pDest[0]); @@ -1040,7 +1040,7 @@ class GroupByBase { if (pCount[i] > 0 && nth < pCount[i]) { - int32_t grpIndex = pFirst[i] + nth; + int64_t grpIndex = pFirst[i] + nth; int32_t bin = pGroup[grpIndex]; pDest[i] = pSrc[bin]; } @@ -1060,14 +1060,14 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t nth = (int32_t)funcParam; + int64_t nth = funcParam; // For all the bins we have to fill for (int64_t i = binLow; i < binHigh; i++) { if (pCount[i] > 0 && nth < pCount[i]) { - int32_t grpIndex = pFirst[i] + nth; + int64_t grpIndex = pFirst[i] + nth; int32_t bin = pGroup[grpIndex]; memcpy(&pDest[i * itemSize], &pSrc[bin * itemSize], itemSize); } @@ -1203,17 +1203,17 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = (int32_t)funcParam; + int64_t windowSize = funcParam; U invalid = GET_INVALID(pDest[0]); if (binLow == 0) { // Mark all invalid if invalid bin int32_t start = pFirst[0]; - int32_t last = start + pCount[0]; - for (int j = start; j < last; j++) + int64_t last = start + pCount[0]; + for (int64_t j = start; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = invalid; } binLow++; @@ -1227,12 +1227,12 @@ class GroupByBase for (int64_t i = binLow; i < binHigh; i++) { int32_t start = pFirst[i]; - int32_t last = start + pCount[i]; + int64_t last = start + pCount[i]; U currentSum = 0; // Priming of the summation - for (int j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { int32_t index = pGroup[j]; @@ -1240,7 +1240,7 @@ class GroupByBase pDest[index] = currentSum; } - for (int j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { int32_t index = pGroup[j]; @@ -1267,17 +1267,17 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = (int32_t)funcParam; + int64_t windowSize = funcParam; U invalid = GET_INVALID(pDest[0]); if (binLow == 0) { // Mark all invalid if invalid bin int32_t start = pFirst[0]; - int32_t last = start + pCount[0]; - for (int j = start; j < last; j++) + int64_t last = start + pCount[0]; + for (int64_t j = start; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = invalid; } binLow++; @@ -1294,14 +1294,14 @@ class GroupByBase for (int64_t i = binLow; i < binHigh; i++) { int32_t start = pFirst[i]; - int32_t last = start + pCount[i]; + int64_t last = start + pCount[i]; U currentSum = 0; // Priming of the summation - for (int j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; U value = (U)pSrc[index]; if (value != invalid) { @@ -1310,9 +1310,9 @@ class GroupByBase pDest[index] = currentSum; } - for (int j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; U value = (U)pSrc[index]; if (value != invalid) @@ -1337,12 +1337,12 @@ class GroupByBase for (int64_t i = binLow; i < binHigh; i++) { int32_t start = pFirst[i]; - int32_t last = start + pCount[i]; + int64_t last = start + pCount[i]; U currentSum = 0; // Priming of the summation - for (int j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { int32_t index = pGroup[j]; U value = (U)pSrc[index]; @@ -1353,7 +1353,7 @@ class GroupByBase pDest[index] = currentSum; } - for (int j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { int32_t index = pGroup[j]; @@ -1387,7 +1387,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = (int32_t)funcParam; + int64_t windowSize = funcParam; U invalid = GET_INVALID((U)0); double invalid_out = GET_INVALID(pDest[0]); @@ -1395,10 +1395,10 @@ class GroupByBase { // Mark all invalid if invalid bin int32_t start = pFirst[0]; - int32_t last = start + pCount[0]; - for (int j = start; j < last; j++) + int64_t last = start + pCount[0]; + for (int64_t j = start; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = invalid_out; } binLow++; @@ -1412,22 +1412,22 @@ class GroupByBase for (int64_t i = binLow; i < binHigh; i++) { int32_t start = pFirst[i]; - int32_t last = start + pCount[i]; + int64_t last = start + pCount[i]; double currentSum = 0; // Priming of the summation - for (int j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; currentSum += pSrc[index]; pDest[index] = currentSum / (j - start + 1); } - for (int j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; currentSum += pSrc[index]; @@ -1452,7 +1452,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = (int32_t)funcParam; + int64_t windowSize = funcParam; U invalid = GET_INVALID((U)0); double invalid_out = GET_INVALID(pDest[0]); @@ -1486,9 +1486,9 @@ class GroupByBase double count = 0; // Priming of the summation - for (int j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; U value = (U)pSrc[index]; if (value != invalid) { @@ -1498,9 +1498,9 @@ class GroupByBase pDest[index] = count > 0 ? currentSum / count : invalid_out; } - for (int j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; U value = (U)pSrc[index]; if (value != invalid) @@ -1533,9 +1533,9 @@ class GroupByBase double count = 0; // Priming of the summation - for (int j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; U value = (U)pSrc[index]; if (value == value) { @@ -1545,9 +1545,9 @@ class GroupByBase pDest[index] = count > 0 ? currentSum / count : invalid_out; } - for (int j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; U value = (U)pSrc[index]; if (value == value) @@ -1581,7 +1581,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = (int32_t)funcParam; + int64_t windowSize = funcParam; U invalid = GET_INVALID(pDest[0]); LOGGING("in rolling count %lld %lld sizeofdest %lld\n", binLow, binHigh, sizeof(U)); @@ -1611,18 +1611,18 @@ class GroupByBase if (windowSize < 0) { - for (int j = last - 1; j >= start; j--) + for (int64_t j = last - 1; j >= start; j--) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = currentSum; currentSum += 1; } } else { - for (int j = start; j < last; j++) + for (int64_t j = start; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = currentSum; currentSum += 1; } @@ -1642,7 +1642,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = (int32_t)funcParam; + int64_t windowSize = funcParam; U invalid = GET_INVALID(pDest[0]); // printf("binlow %lld, binhigh %lld, windowSize: %d\n", binLow, binHigh, @@ -1665,20 +1665,20 @@ class GroupByBase for (int64_t i = binLow; i < binHigh; i++) { int32_t start = pFirst[i]; - int32_t last = start + pCount[i]; + int64_t last = start + pCount[i]; if (windowSize >= 0) { // invalid for window - for (int32_t j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { int32_t index = pGroup[j]; pDest[index] = invalid; } - for (int32_t j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = (U)pSrc[pGroup[j - windowSize]]; } } @@ -1691,15 +1691,15 @@ class GroupByBase // printf("bin[%lld] start:%d last:%d windowSize:%d\n", i, start, // last, windowSize); - for (int32_t j = last; j > start && j > (last - windowSize); j--) + for (int64_t j = last; j > start && j > (last - windowSize); j--) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = invalid; } - for (int32_t j = last - windowSize; j > start; j--) + for (int64_t j = last - windowSize; j > start; j--) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = (U)pSrc[pGroup[j + windowSize]]; } // put it back to what it was @@ -1719,7 +1719,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = (int32_t)funcParam; + int64_t windowSize = funcParam; U invalid = GET_INVALID(pDest[0]); if (binLow == 0) @@ -1773,13 +1773,13 @@ class GroupByBase // invalid for window U previous = 0; - for (int j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { int32_t index = pGroup[j]; pDest[index] = invalid; } - for (int j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { int32_t index = pGroup[j]; U temp = (U)pSrc[index]; @@ -1794,13 +1794,13 @@ class GroupByBase last--; start--; - for (int j = last; j > start && j > (last - windowSize); j--) + for (int64_t j = last; j > start && j > (last - windowSize); j--) { int32_t index = pGroup[j]; pDest[index] = invalid; } - for (int j = last - windowSize; j > start; j--) + for (int64_t j = last - windowSize; j > start; j--) { int32_t index = pGroup[j]; U temp = (U)pSrc[index]; @@ -1914,7 +1914,7 @@ class GroupByBase for (int64_t i = binLow; i < binHigh; i++) { int32_t index = pFirst[i]; - int32_t nCount = pCount[i]; + int64_t nCount = pCount[i]; if (nCount == 0) { @@ -1941,7 +1941,7 @@ class GroupByBase pEnd--; } - nCount = (int32_t)((pEnd + 1) - pSort); + nCount = ((pEnd + 1) - pSort); if (nCount == 0) { @@ -1999,7 +1999,7 @@ class GroupByBase for (int64_t i = binLow; i < binHigh; i++) { int32_t index = pFirst[i]; - int32_t nCount = pCount[i]; + int64_t nCount = pCount[i]; if (nCount == 0) { @@ -2010,7 +2010,7 @@ class GroupByBase // Copy over the items for this group for (int j = 0; j < nCount; j++) { - // printf("**[%lld][%d] %d\n", i, index + j, (int32_t)pGroup[index + + // printf("**[%lld][%d] %d\n", i, index + j, pGroup[index + // j]); pSort[j] = pSrc[pGroup[index + j]]; } @@ -2029,7 +2029,7 @@ class GroupByBase pEnd--; } - nCount = (int32_t)((pEnd + 1) - pSort); + nCount = ((pEnd + 1) - pSort); if (nCount == 0) { @@ -3498,7 +3498,7 @@ void GroupByCall(void * pGroupBy, int64_t i) //--------------------------------------------------------------- // Arg1 = LIST of numpy arrays which has the values to accumulate (often all the -// columns in a dataset) Arg2 = numpy array (int32_t) which has the index to the +// columns in a dataset) Arg2 = numpy array which has the index to the // unique keys (ikey from MultiKeyGroupBy32) Arg3 = integer unique rows Arg4 = // integer (function number to execute for sum,mean,min, max) Example: // GroupByOp2(array, ikey, 3, np.float32) Returns cells @@ -3687,7 +3687,7 @@ PyObject * GroupBySingleOpMultiBands(ArrayInfo * aInfo, PyArrayObject * iKey, Py // next low bin is the previous high bin low = high; - pstGroupBy32->returnObjects[i].funcNum = (int32_t)firstFuncNum; + pstGroupBy32->returnObjects[i].funcNum = firstFuncNum; pstGroupBy32->returnObjects[i].didWork = 0; // Assign working memory per call @@ -3816,7 +3816,7 @@ PyObject * GroupBySingleOpMultithreaded(ArrayInfo * aInfo, PyArrayObject * iKey, for (int i = 0; i < numCores; i++) { - pstGroupBy32->returnObjects[i].funcNum = (int32_t)firstFuncNum; + pstGroupBy32->returnObjects[i].funcNum = firstFuncNum; pstGroupBy32->returnObjects[i].binLow = binLow; pstGroupBy32->returnObjects[i].binHigh = binHigh; @@ -3872,7 +3872,7 @@ PyObject * GroupBySingleOpMultithreaded(ArrayInfo * aInfo, PyArrayObject * iKey, //--------------------------------------------------------------- // Arg1 = LIST of numpy arrays which has the values to accumulate (often all the -// columns in a dataset) Arg2 = iKey = numpy array (int32_t) which has the index +// columns in a dataset) Arg2 = iKey = numpy array which has the index // to the unique keys (ikey from MultiKeyGroupBy32) Arg3 = integer unique rows // Arg4 = LIST of integer (function number to execute for sum,mean,min, max) // Arg5 = LIST of integers (binLow -- invalid bin) @@ -4014,8 +4014,8 @@ PyObject * GroupByAll32(PyObject * self, PyObject * args) bool hasCounts = false; int overflow = 0; - int64_t funcNum = PyLong_AsLongLongAndOverflow(PyList_GET_ITEM(listFuncNum, i), &overflow); - pstGroupBy32->returnObjects[i].funcNum = (int32_t)funcNum; + int32_t funcNum = PyLong_AsLongAndOverflow(PyList_GET_ITEM(listFuncNum, i), &overflow); + pstGroupBy32->returnObjects[i].funcNum = funcNum; int64_t binLow = PyLong_AsLongLongAndOverflow(PyList_GET_ITEM(listBinLow, i), &overflow); pstGroupBy32->returnObjects[i].binLow = binLow; @@ -4108,11 +4108,11 @@ PyObject * GroupByAll32(PyObject * self, PyObject * args) //--------------------------------------------------------------- // Arg1 = LIST of numpy arrays which has the values to accumulate (often all the -// columns in a dataset) Arg2 =iKey = numpy array (int32_t) which has the index -// to the unique keys (ikey from MultiKeyGroupBy32) Arg3 =iGroup: (int32_t) +// columns in a dataset) Arg2 =iKey = numpy array which has the index +// to the unique keys (ikey from MultiKeyGroupBy32) Arg3 =iGroup: // array size is same as multikey, unique keys are grouped together Arg4 -// =iFirst: (int32_t) array size is number of unique keys, indexes into iGroup -// Arg5 =nCount: (int32_t) array size is number of unique keys for the group, is +// =iFirst: array size is number of unique keys, indexes into iGroup +// Arg5 =nCount: array size is number of unique keys for the group, is // how many member of the group (paired with iFirst) Arg6 = integer unique rows // Arg7 = LIST of integers (function number to execute for sum,mean,min, max) // Arg8 = LIST of integers (binLow -- invalid bin) @@ -4254,8 +4254,8 @@ PyObject * GroupByAllPack32(PyObject * self, PyObject * args) for (int i = 0; i < tupleSize; i++) { int overflow = 0; - int64_t funcNum = PyLong_AsLongLongAndOverflow(PyList_GET_ITEM(listFuncNum, i), &overflow); - pstGroupBy32->returnObjects[i].funcNum = (int32_t)funcNum; + int32_t funcNum = PyLong_AsLongAndOverflow(PyList_GET_ITEM(listFuncNum, i), &overflow); + pstGroupBy32->returnObjects[i].funcNum = funcNum; int64_t binLow = PyLong_AsLongLongAndOverflow(PyList_GET_ITEM(listBinLow, i), &overflow); pstGroupBy32->returnObjects[i].binLow = binLow; diff --git a/src/HashFunctions.cpp b/src/HashFunctions.cpp index 9453d617..3dd7833e 100644 --- a/src/HashFunctions.cpp +++ b/src/HashFunctions.cpp @@ -300,9 +300,9 @@ PyObject * IsMemberCategorical(PyObject * self, PyObject * args) // // final result 6 0 6 2 0 4 6 4 0 template -[[nodiscard]] char const * -FindFirstOccurence(T const * pArray2, int32_t const * pUnique1, int32_t * pReIndex, int32_t * pReverseMap, int64_t array2Length, - int32_t unique1Length, int32_t unique2Length, int32_t baseOffset2T) +[[nodiscard]] char const * FindFirstOccurence(T const * pArray2, int32_t const * pUnique1, int32_t * pReIndex, + int32_t * pReverseMap, int64_t array2Length, int32_t unique1Length, + int32_t unique2Length, int32_t baseOffset2T) { T baseOffset2 = (T)baseOffset2T; @@ -402,7 +402,8 @@ FindFirstOccurence(T const * pArray2, int32_t const * pUnique1, int32_t * pReInd // Returns: pOutput and pBoolOutput // TODO: this routine needs to output INT8/16/32/64 instead of just INT32 template -void FinalMatch(T const * pArray1, int32_t * pOutput, int8_t * pBoolOutput, int32_t const * pReIndex, int64_t array1Length, int32_t baseoffset) +void FinalMatch(T const * pArray1, int32_t * pOutput, int8_t * pBoolOutput, int32_t const * pReIndex, int64_t array1Length, + int32_t baseoffset) { const int32_t invalid = *(int32_t *)GetDefaultForType(NPY_INT32); @@ -501,24 +502,24 @@ PyObject * IsMemberCategoricalFixup(PyObject * self, PyObject * args) int32_t * pUnique = (int32_t *)PyArray_BYTES(isMemberUnique); void * pArray2 = PyArray_BYTES(inArr2); - char const * err{nullptr}; + char const * err{ nullptr }; switch (array2Type) { case NPY_INT8: - err = FindFirstOccurence((int8_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, arraySizeUnique, - unique2Length, baseoffset2); + err = FindFirstOccurence((int8_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, + arraySizeUnique, unique2Length, baseoffset2); break; case NPY_INT16: - err = FindFirstOccurence((int16_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, arraySizeUnique, - unique2Length, baseoffset2); + err = FindFirstOccurence((int16_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, + arraySizeUnique, unique2Length, baseoffset2); break; CASE_NPY_INT32: - err = FindFirstOccurence((int32_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, arraySizeUnique, - unique2Length, baseoffset2); + err = FindFirstOccurence((int32_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, + arraySizeUnique, unique2Length, baseoffset2); break; CASE_NPY_INT64: - err = FindFirstOccurence((int64_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, arraySizeUnique, - unique2Length, baseoffset2); + err = FindFirstOccurence((int64_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, + arraySizeUnique, unique2Length, baseoffset2); break; default: PyErr_Format(PyExc_ValueError, "IsMemberCategoricalFixup second argument is not INT8/16/32/64"); diff --git a/src/HashLinear.cpp b/src/HashLinear.cpp index 8b9fc3dd..48c49a9d 100644 --- a/src/HashLinear.cpp +++ b/src/HashLinear.cpp @@ -1224,7 +1224,7 @@ static void IsMemberMK(void * pHashLinearVoid, int64_t arraySize, void * pInputT //#define HASH_int32_t uint64_t h = (uint64_t)item; h ^= (h >> 16); h= h & //(HashSize-1); #define HASH_int32_t uint64_t h= fasthash64_8(item) & //(HashSize-1); -#define HASH_int32_t uint64_t h = _mm_crc32_u32(0, (uint32_t)item) & (HashSize - 1); +#define HASH_int32_t uint64_t h = _mm_crc32_u32(0, item) & (HashSize - 1); //#define HASH_int64_t uint64_t h= _mm_crc32_u64(0, item) & (HashSize-1); #define HASH_int64_t uint64_t h = fasthash64_8(item) & (HashSize - 1); @@ -1308,7 +1308,7 @@ void CHashLinear::MakeHashLocation(int64_t arraySize, T * pHashList, int64 // printf("%llu |", pBitFields[i]); //} - if (sizeof(T) <= 2) + if constexpr (sizeof(T) <= 2) { for (U i = 0; i < arraySize; i++) { @@ -1321,7 +1321,7 @@ void CHashLinear::MakeHashLocation(int64_t arraySize, T * pHashList, int64 } else { - if (sizeof(T) == 4) + if constexpr (sizeof(T) == 4) { if (HashMode == HASH_MODE_PRIME) { @@ -1346,7 +1346,7 @@ void CHashLinear::MakeHashLocation(int64_t arraySize, T * pHashList, int64 } } } - else if (sizeof(T) == 8) + else if constexpr (sizeof(T) == 8) { if (HashMode == HASH_MODE_PRIME) { @@ -1409,7 +1409,7 @@ int64_t CHashLinear::IsMemberCategorical(int64_t arraySize, T * pHashList, } else { - if (sizeof(T) == 4) + if constexpr (sizeof(T) == 4) { if (HashMode == HASH_MODE_PRIME) { @@ -1430,7 +1430,7 @@ int64_t CHashLinear::IsMemberCategorical(int64_t arraySize, T * pHashList, } } } - else if (sizeof(T) == 8) + else if constexpr (sizeof(T) == 8) { if (HashMode == HASH_MODE_PRIME) { @@ -1540,7 +1540,7 @@ void IsMember(void * pHashLinearVoid, int64_t arraySize, void * pHashListT, int8 // printf("\n"); - if (sizeof(T) <= 2) + if constexpr (sizeof(T) <= 2) { LOGGING("Perfect hash\n"); // perfect hash @@ -1555,7 +1555,7 @@ void IsMember(void * pHashLinearVoid, int64_t arraySize, void * pHashListT, int8 { HASH_MODE HashMode = pHashLinear->HashMode; - if (sizeof(T) == 4) + if constexpr (sizeof(T) == 4) { if (HashMode == HASH_MODE_PRIME) { @@ -1582,7 +1582,7 @@ void IsMember(void * pHashLinearVoid, int64_t arraySize, void * pHashListT, int8 } } } - else if (sizeof(T) == 8) + else if constexpr (sizeof(T) == 8) { if (HashMode == HASH_MODE_PRIME) { @@ -2221,9 +2221,8 @@ uint64_t CHashLinear::GroupByItemSize(int64_t totalRows, int64_t totalItem // make local reference on stack uint64_t * pBitFieldsX = pBitFields; - switch (sizeof(T)) + if constexpr (sizeof(T) == 1) { - case 1: // TODO: Specially handle bools here -- they're 1-byte but logically have // only two buckets (zero/nonzero). // if (coreType == NPY_BOOL) @@ -2254,8 +2253,9 @@ uint64_t CHashLinear::GroupByItemSize(int64_t totalRows, int64_t totalItem pIndexArray[i] = 0; } } - break; - case 2: + } + if constexpr (sizeof(T) == 2) + { if (pBoolFilter == NULL) { for (U i = 0; i < totalRows; i++) @@ -2279,8 +2279,9 @@ uint64_t CHashLinear::GroupByItemSize(int64_t totalRows, int64_t totalItem pIndexArray[i] = 0; } } - break; - case 4: + } + if constexpr (sizeof(T) == 4) + { if (pBoolFilter == NULL) { for (U i = 0; i < totalRows; i++) @@ -2306,8 +2307,9 @@ uint64_t CHashLinear::GroupByItemSize(int64_t totalRows, int64_t totalItem pIndexArray[i] = 0; } } - break; - case 8: + } + if constexpr (sizeof(T) == 8) + { if (pBoolFilter == NULL) { for (U i = 0; i < totalRows; i++) @@ -2333,8 +2335,9 @@ uint64_t CHashLinear::GroupByItemSize(int64_t totalRows, int64_t totalItem pIndexArray[i] = 0; } } - break; - case 16: + } + if constexpr (sizeof(T) == 16) + { // TO BE WORKED ON... if (pBoolFilter == NULL) { @@ -2361,7 +2364,6 @@ uint64_t CHashLinear::GroupByItemSize(int64_t totalRows, int64_t totalItem pIndexArray[i] = 0; } } - break; } // printf("GroupByItem end! %d\n", numUnique); diff --git a/src/MathThreads.cpp b/src/MathThreads.cpp index 117c17ed..b24c44e7 100644 --- a/src/MathThreads.cpp +++ b/src/MathThreads.cpp @@ -82,7 +82,7 @@ void * WorkerThreadFunction(void * lpParam) { stWorkerRing * pWorkerRing = (stWorkerRing *)lpParam; - uint32_t core = (uint32_t)(InterlockedIncrement64(&pWorkerRing->WorkThread)); + uint32_t core = static_cast((InterlockedIncrement64(&pWorkerRing->WorkThread))); core = core - 1; // if (core > 3) core += 16; diff --git a/src/MathWorker.cpp b/src/MathWorker.cpp index f994600b..37551c26 100644 --- a/src/MathWorker.cpp +++ b/src/MathWorker.cpp @@ -76,14 +76,14 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) if (n >= 1) { __cpuid((int *)reg, 1); - f1c = (uint32_t)reg[2]; - f1d = (uint32_t)reg[3]; + f1c = reg[2]; + f1d = reg[3]; } if (n >= 7) { __cpuidex((int *)reg, 7, 0); - f7b = (uint32_t)reg[1]; - f7c = (uint32_t)reg[2]; + f7b = reg[1]; + f7c = reg[2]; } } #elif defined(__i386__) && defined(__PIC__) && ! defined(__clang__) && defined(__GNUC__) diff --git a/src/Merge.cpp b/src/Merge.cpp index 338320b8..91668f9d 100644 --- a/src/Merge.cpp +++ b/src/Merge.cpp @@ -1637,13 +1637,18 @@ PyObject * BooleanToFancy(PyObject * self, PyObject * args, PyObject * kwargs) for (int64_t i = start; i < (start + length); i++) { + if (i > INT_MAX) + { + PyErr_Format(PyExc_ValueError, "Cannot represent lengths > 31 bits in type [%d]", callbackArg->dtype); + return false; + } if (pBooleanMask[i]) { - *pOut++ = (int32_t)i; + *pOut++ = static_cast(i); } else { - *pOutFalse++ = (int32_t)i; + *pOutFalse++ = static_cast(i); } } } @@ -1657,9 +1662,14 @@ PyObject * BooleanToFancy(PyObject * self, PyObject * args, PyObject * kwargs) for (int64_t i = start; i < (start + length); i++) { + if (i > INT_MAX) + { + PyErr_Format(PyExc_ValueError, "Cannot represent lengths > 31 bits in type [%d]", callbackArg->dtype); + return false; + } if (pBooleanMask[i]) { - *pOut++ = i; + *pOut++ = static_cast(i); } } } @@ -1670,9 +1680,14 @@ PyObject * BooleanToFancy(PyObject * self, PyObject * args, PyObject * kwargs) for (int64_t i = start; i < (start + length); i++) { + if (i > INT_MAX) + { + PyErr_Format(PyExc_ValueError, "Cannot represent lengths > 31 bits in type [%d]", callbackArg->dtype); + return false; + } if (pBooleanMask[i]) { - *pOut++ = (int32_t)i; + *pOut++ = static_cast(i); } } } diff --git a/src/MultiKey.cpp b/src/MultiKey.cpp index 6ede9e42..31c59e4f 100644 --- a/src/MultiKey.cpp +++ b/src/MultiKey.cpp @@ -13,7 +13,7 @@ char * RotateArrays(int64_t tupleSize, ArrayInfo * aInfo) { // TODO: Is it fast to calculate the hash here? - // uint32_t* pHashArray = (uint32_t*)malloc(sizeof(uint32_t) * totalRows); + // uint32_t* pHashArray = (uint32_t*)malloc(sizeof * totalRows); int64_t totalItemSize = 0; int64_t totalRows = 0; @@ -609,14 +609,14 @@ int32_t * GroupByPackFixup32(int64_t numUnique, int64_t totalRows, void * pIndex } // Go backwards - for (int32_t i = (int32_t)totalRows - 1; i >= 0; i--) + for (int64_t i = totalRows - 1; i >= 0; i--) { K group = pIndexArray[i]; pNextArray[i] = pGroupArray[group]; // printf("%d - group %d next %d\n", i, (int)group, pNextArray[i]); - pGroupArray[group] = i; + pGroupArray[group] = static_cast(i); } return pNextArray; @@ -657,7 +657,12 @@ bool GroupByPackFinal32(int64_t numUnique, int64_t totalRows, void * pIndexArray int32_t * pFirstArray = (int32_t *)PyArray_BYTES(firstArray); int32_t * pCountArray = (int32_t *)PyArray_BYTES(countArray); - int32_t i = (int32_t)totalRows; + if (totalRows > INT_MAX) + { + return false; + } + + int32_t i = static_cast(totalRows); // TODO -- how to handle empty? pSortArray[0] = GB_INVALID_INDEX; @@ -1513,7 +1518,7 @@ PyObject * MultiKeyHash(PyObject * self, PyObject * args) assert(mkp.totalRows < 2100000000); - int32_t i = (int32_t)mkp.totalRows; + int32_t i = static_cast(mkp.totalRows); while (i > 0) { diff --git a/src/Recycler.cpp b/src/Recycler.cpp index 8bb07f52..9b9b1de4 100644 --- a/src/Recycler.cpp +++ b/src/Recycler.cpp @@ -344,7 +344,7 @@ static inline void RemoveFromList(stRecycleList * pItems, int32_t slot) // On decrement it will decref array, set the slot to null and inc Tail // On increment it will incref array, time stamp, and inc Head // On Entry: the recycledArray must be valid -static void RefCountNumpyArray(stRecycleList * pItems, int32_t lzcount, int32_t type, int32_t slot, bool bIncrement) +static void RefCountNumpyArray(stRecycleList * pItems, int64_t lzcount, int32_t type, int32_t slot, bool bIncrement) { if (pItems->Item[slot].recycledArray == NULL) { @@ -359,7 +359,7 @@ static void RefCountNumpyArray(stRecycleList * pItems, int32_t lzcount, int32_t // timestamp pItems->Item[slot].tsc = __rdtsc(); - LOGRECYCLE("Adding item to slot %d,%d,%d -- refcnt now is %llu\n", lzcount, type, slot, + LOGRECYCLE("Adding item to slot %lld,%d,%d -- refcnt now is %llu\n", lzcount, type, slot, pItems->Item[slot].recycledArray->ob_base.ob_refcnt); // Only location head is incremented @@ -374,7 +374,8 @@ static void RefCountNumpyArray(stRecycleList * pItems, int32_t lzcount, int32_t RemoveFromList(pItems, slot); // If this is the last decref, python may free memory - LOGRECYCLE("Removing item in slot %d,%d,%d -- refcnt before is %llu\n", lzcount, type, slot, toDelete->ob_base.ob_refcnt); + LOGRECYCLE("Removing item in slot %lld,%d,%d -- refcnt before is %llu\n", lzcount, type, slot, + toDelete->ob_base.ob_refcnt); // NOTE: If this is the last decref, it might "free" the memory which might // take time @@ -664,7 +665,7 @@ static bool DeleteNumpyArray(PyArrayObject * inArr) ndim == 1 && strides != NULL && itemSize == strides[0]) { // Based on size and type, lookup - int32_t log2 = (int32_t)lzcnt_64(totalSize); + int64_t log2 = lzcnt_64(totalSize); stRecycleList * pItems = &g_stRecycleList[log2][type]; @@ -743,7 +744,7 @@ static bool DeleteNumpyArray(PyArrayObject * inArr) if (deltaHead < 0 || deltaHead > RECYCLE_MAXIMUM_SEARCH) { LogError( - "!!! inner critical error with recycler items %d,%d,%d with " + "!!! inner critical error with recycler items %lld,%d,%d with " "deltahead %d totalsize%lld\n", log2, type, slot, deltaHead, pItems->Item[slot].totalSize); } diff --git a/src/Reduce.cpp b/src/Reduce.cpp index e677f414..23dd107d 100644 --- a/src/Reduce.cpp +++ b/src/Reduce.cpp @@ -2942,7 +2942,7 @@ static PyObject * ReduceInternal(PyArrayObject * inArr1, REDUCE_FUNCTIONS func, Py_RETURN_NONE; } - stScatterGatherFunc sgFunc = { (int32_t)numpyInType, 0, 0, 0, 0, 0 }; + stScatterGatherFunc sgFunc = { numpyInType, 0, 0, 0, 0, 0 }; FUNCTION_LIST fl{}; fl.AnyScatterGatherCall = pFunction; diff --git a/src/RipTide.cpp b/src/RipTide.cpp index f195d77a..6979af01 100644 --- a/src/RipTide.cpp +++ b/src/RipTide.cpp @@ -920,6 +920,7 @@ int32_t GetArrDType(PyArrayObject * inArr) return dtype; } +// TODO: Refactor this to be variant based. //----------------------------------------------------------------------------------- // Convert python object to // pInput: pointer to array value that needs to be converted @@ -1015,7 +1016,7 @@ bool ConvertSingleItemArray(void * pInput, int16_t numpyInType, _m256all * pDest break; CASE_NPY_UINT32: CASE_NPY_INT32: - pDest->i = _mm256_set1_epi32((int32_t)value); + pDest->i = _mm256_set1_epi32(static_cast(value)); break; CASE_NPY_UINT64: @@ -1092,7 +1093,7 @@ bool ConvertScalarObject(PyObject * inObject1, _m256all * pDest, int16_t numpyOu break; CASE_NPY_UINT32: CASE_NPY_INT32: - pDest->i = _mm256_set1_epi32((int32_t)value); + pDest->i = _mm256_set1_epi32(static_cast(value)); break; CASE_NPY_UINT64: @@ -1177,10 +1178,10 @@ bool ConvertScalarObject(PyObject * inObject1, _m256all * pDest, int16_t numpyOu pDest->i = _mm256_set1_epi16((uint16_t)value2); break; CASE_NPY_INT32: - pDest->i = _mm256_set1_epi32((int32_t)value); + pDest->i = _mm256_set1_epi32(static_cast(value)); break; CASE_NPY_UINT32: - pDest->i = _mm256_set1_epi32((uint32_t)value2); + pDest->i = _mm256_set1_epi32(static_cast(value2)); break; CASE_NPY_INT64: @@ -1223,10 +1224,10 @@ bool ConvertScalarObject(PyObject * inObject1, _m256all * pDest, int16_t numpyOu pDest->i = _mm256_set1_epi16((uint16_t)value); break; CASE_NPY_UINT32: - pDest->i = _mm256_set1_epi32((uint32_t)value); + pDest->i = _mm256_set1_epi32(static_cast(value)); break; CASE_NPY_INT32: - pDest->i = _mm256_set1_epi32((int32_t)value); + pDest->i = _mm256_set1_epi32(static_cast(value)); break; CASE_NPY_UINT64: @@ -2256,7 +2257,7 @@ bool GetUpcastType(int numpyInType1, int numpyInType2, int & convertType1, int & // int GetStridesAndContig(PyArrayObject const * inArray, int & ndim, int64_t & stride) { - stride = PyArray_ITEMSIZE(inArray); + stride = static_cast(PyArray_ITEMSIZE(inArray)); int direction = 0; ndim = PyArray_NDIM(inArray); if (ndim > 0) @@ -2296,7 +2297,7 @@ int GetStridesAndContig(PyArrayObject const * inArray, int & ndim, int64_t & str int64_t curStrideLen = stride; while (currentdim != (ndims - 1)) { - curStrideLen *= PyArray_DIM(inArray, currentdim); + curStrideLen *= static_cast(PyArray_DIM(inArray, currentdim)); LOGGING("'F' %lld vs %lld dim: %lld stride: %lld \n", curStrideLen, PyArray_STRIDE(inArray, currentdim + 1), PyArray_DIM(inArray, currentdim + 1), stride); if (PyArray_STRIDE(inArray, currentdim + 1) != curStrideLen) diff --git a/src/SDSFile.cpp b/src/SDSFile.cpp index 2dcf51c4..2ed31c1e 100644 --- a/src/SDSFile.cpp +++ b/src/SDSFile.cpp @@ -505,11 +505,17 @@ int64_t SDSFileReadChunk(SDS_EVENT_HANDLE eventHandle, SDS_FILE_HANDLE Handle, v // LastError); OVERLAPPED OverlappedIO; + if (BufferPos > INT_MAX) + { + LogError("!! Overlapped only supports 32-bit file offsets !! [%lld]", BufferPos); + return 0; + } + OverlappedIO.hEvent = eventHandle; OverlappedIO.InternalHigh = 0; OverlappedIO.Internal = 0; - OverlappedIO.OffsetHigh = (uint32_t)(BufferPos >> 32); - OverlappedIO.Offset = (uint32_t)BufferPos; + OverlappedIO.OffsetHigh = (BufferPos >> 32); + OverlappedIO.Offset = static_cast(BufferPos & 0x0FFFFFFFFULL); bool bReadDone; @@ -591,11 +597,17 @@ int64_t SDSFileWriteChunk(SDS_EVENT_HANDLE eventHandle, SDS_FILE_HANDLE Handle, // LastError); OVERLAPPED OverlappedIO; + if (BufferPos > INT_MAX) + { + LogError("!! Overlapped only supports 32-bit file offsets !! [%lld]", BufferPos); + return 0; + } + OverlappedIO.hEvent = eventHandle; OverlappedIO.InternalHigh = 0; OverlappedIO.Internal = 0; - OverlappedIO.OffsetHigh = (uint32_t)(BufferPos >> 32); - OverlappedIO.Offset = (uint32_t)BufferPos; + OverlappedIO.OffsetHigh = (BufferPos >> 32); + OverlappedIO.Offset = static_cast(BufferPos & 0x0FFFFFFFFULL); OVERLAPPED * pos = &OverlappedIO; DWORD n; @@ -1575,7 +1587,7 @@ int64_t ReadAndDecompressBandWithFilter(SDS_ARRAY_BLOCK * pBlockInfo, // may con rowOffset, pBlockInfo->ArrayDataOffset, pBlockInfo->ArrayCompressedSize, pBlockInfo->ArrayUncompressedSize, stackIndex, bytesPerRow); - for (int32_t i = 0; i < pBlockInfo->ArrayBandCount; i++) + for (int64_t i = 0; i < pBlockInfo->ArrayBandCount; i++) { int64_t compressedSize = pBands[i] - previousSize; previousSize = pBands[i]; @@ -1719,7 +1731,7 @@ static size_t ReadAndDecompressArrayBlockWithFilter(SDS_ARRAY_BLOCK * pBlockInfo else { printf( - "out of mem no temp buffer bandcount:%d bandsize:%d uncomp size: " + "out of mem no temp buffer bandcount:%lld bandsize:%lld uncomp size: " "%lld\n", pBlockInfo->ArrayBandCount, pBlockInfo->ArrayBandSize, pBlockInfo->ArrayUncompressedSize); } @@ -1791,7 +1803,7 @@ static size_t ReadAndDecompressArrayBlock(SDS_ARRAY_BLOCK * pBlockInfo, // may c { int64_t previousSize = 0; - for (int32_t i = 0; i < pBlockInfo->ArrayBandCount; i++) + for (int64_t i = 0; i < pBlockInfo->ArrayBandCount; i++) { int64_t compressedSize = pBands[i] - previousSize; previousSize = pBands[i]; @@ -1812,7 +1824,7 @@ static size_t ReadAndDecompressArrayBlock(SDS_ARRAY_BLOCK * pBlockInfo, // may c if (result != uncompressedSize) { printf( - "[%d][%lld][%d] MTDecompression (uncompressed) band error " + "[%d][%lld][%lld] MTDecompression (uncompressed) band error " "size %lld vs %lld\n", core, arrayIndex, i, uncompressedSize, compressedSize); result = -1; @@ -1829,7 +1841,7 @@ static size_t ReadAndDecompressArrayBlock(SDS_ARRAY_BLOCK * pBlockInfo, // may c if (dcSize != uncompressedSize) { printf( - "[%d][%lld][%d] MTDecompression band error size %lld vs " + "[%d][%lld][%lld] MTDecompression band error size %lld vs " "%lld vs %lld\n", core, arrayIndex, i, dcSize, uncompressedSize, compressedSize); result = -1; @@ -2725,8 +2737,8 @@ bool CompressFileArray(void * pstCompressArraysV, int32_t core, int64_t t) pArrayBlock->CompressionType = COMPRESSION_TYPE_ZSTD; // New version 4.3 - pArrayBlock->ArrayBandCount = (int32_t)bandCount; - pArrayBlock->ArrayBandSize = (int32_t)bandSize; + pArrayBlock->ArrayBandCount = bandCount; + pArrayBlock->ArrayBandSize = bandSize; // record array dimensions int32_t ndim = pArrayInfo->NDim; @@ -2752,7 +2764,7 @@ bool CompressFileArray(void * pstCompressArraysV, int32_t core, int64_t t) pArrayBlock->Flags = pArrayInfo->Flags; pArrayBlock->DType = pArrayInfo->NumpyDType; - pArrayBlock->ItemSize = (int32_t)pArrayInfo->ItemSize; + pArrayBlock->ItemSize = pArrayInfo->ItemSize; pArrayBlock->HeaderLength = sizeof(SDS_ARRAY_BLOCK); pArrayBlock->Magic = COMPRESSION_MAGIC; @@ -2991,7 +3003,7 @@ bool SDSWriteFileInternal(const char * fileName, } pDestArrayBlock[arrayNumber].Flags = aInfo[arrayNumber].Flags; pDestArrayBlock[arrayNumber].DType = aInfo[arrayNumber].NumpyDType; - pDestArrayBlock[arrayNumber].ItemSize = (int32_t)aInfo[arrayNumber].ItemSize; + pDestArrayBlock[arrayNumber].ItemSize = aInfo[arrayNumber].ItemSize; pDestArrayBlock[arrayNumber].HeaderLength = sizeof(SDS_ARRAY_BLOCK); pDestArrayBlock[arrayNumber].Magic = COMPRESSION_MAGIC; @@ -6292,7 +6304,7 @@ class SDSDecompressManyFiles } // track last valid row? - LastRow = (int32_t)fileRow; + LastRow = fileRow; } //------------------------------------------------ diff --git a/src/SDSFile.h b/src/SDSFile.h index c524364f..2bf49858 100644 --- a/src/SDSFile.h +++ b/src/SDSFile.h @@ -45,8 +45,9 @@ // TJD added version low = 3 July 2019 for "bandsize" // TJD added version low = 4 August 2019 for "section" // TJD added version low = 5 March 2020 for timestamps and better section, +// EST added version low = 6 for 64-bit safe operations // boolean bitmasks -#define SDS_VERSION_LOW 5 +#define SDS_VERSION_LOW 6 #define SDS_VALUE_ERROR 1 @@ -265,7 +266,6 @@ enum SDS_TYPES // File format layout for one array block header struct SDS_ARRAY_BLOCK { - //----- offset 0 ----- int16_t HeaderTag; int16_t HeaderLength; @@ -274,12 +274,9 @@ struct SDS_ARRAY_BLOCK int8_t DType; int8_t NDim; - //----- offset 8 ----- // BUGBUG All these offsets are wrong, this struct has - // default member alignments. int32_t ItemSize; int32_t Flags; - //----- offset 16 ----- // no more than 5 dims int64_t Dimensions[SDS_MAX_DIMS]; int64_t Strides[SDS_MAX_DIMS]; @@ -289,10 +286,14 @@ struct SDS_ARRAY_BLOCK // / BandSize => # of bands int64_t ArrayCompressedSize; int64_t ArrayUncompressedSize; - int32_t ArrayBandCount; // 0=no banding - int32_t ArrayBandSize; // 0=no banding + int64_t ArrayBandCount; // 0=no banding + int64_t ArrayBandSize; // 0=no banding }; +static_assert(offsetof(SDS_ARRAY_BLOCK, HeaderTag) == 0); +static_assert(offsetof(SDS_ARRAY_BLOCK, ItemSize) == 8); +static_assert(offsetof(SDS_ARRAY_BLOCK, Dimensions) == 16); + //---------------------------------- // At offset 0 of the file is this header struct SDS_FILE_HEADER @@ -305,7 +306,6 @@ struct SDS_FILE_HEADER int16_t CompType; int16_t CompLevel; - //----- offset 16 ----- int64_t NameBlockSize; int64_t NameBlockOffset; int64_t NameBlockCount; @@ -314,28 +314,22 @@ struct SDS_FILE_HEADER int16_t StackType; // see SDS_STACK_TYPE int32_t AuthorId; // see SDS_AUTHOR_ID - //----- offset 48 ----- int64_t MetaBlockSize; int64_t MetaBlockOffset; - //----- offset 64 ----- int64_t TotalMetaCompressedSize; int64_t TotalMetaUncompressedSize; - //----- offset 80 ----- int64_t ArrayBlockSize; int64_t ArrayBlockOffset; - //----- offset 96 ----- int64_t ArraysWritten; int64_t ArrayFirstOffset; - //----- offset 112 ----- int64_t TotalArrayCompressedSize; // Includes the SDS_PADDING, next relative // offset to write to int64_t TotalArrayUncompressedSize; - //----- offset 128 ----- //------------- End of Version 4.1 --------- //------------- Version 4.3 starts here ---- int64_t BandBlockSize; // 00 if nothing @@ -343,7 +337,6 @@ struct SDS_FILE_HEADER int64_t BandBlockCount; // Number of names int64_t BandSize; // How many elements before creating another band - //----- offset 160 ----- //------------- End of Version 4.3 --------- //------------- Version 4.4 starts here ---- int64_t SectionBlockSize; // how many bytes valid data @@ -357,14 +350,11 @@ struct SDS_FILE_HEADER int64_t SectionBlockReservedSize; // total size of what we reserved for this // block (so we can append names) - //----- offset 192 ----- int64_t FileOffset; // this file offset within the file uint64_t TimeStampUTCNanos; // time stamp when file was last written (0s // indicate no time stamp) - //----- offset 208 ----- - char Reserved[512 - 208]; // BUGBUG This should be done using align statements, - // not magic number math with incorrect arguments. + char Reserved[512 - 208]; //----------------------------- // function to multithreaded safe calculate the next array offset to write to @@ -387,6 +377,17 @@ struct SDS_FILE_HEADER } }; +static_assert(offsetof(SDS_FILE_HEADER, MetaBlockSize) == 48); +static_assert(offsetof(SDS_FILE_HEADER, TotalMetaCompressedSize) == 64); +static_assert(offsetof(SDS_FILE_HEADER, ArrayBlockSize) == 80); +static_assert(offsetof(SDS_FILE_HEADER, ArraysWritten) == 96); +static_assert(offsetof(SDS_FILE_HEADER, TotalArrayCompressedSize) == 112); +static_assert(offsetof(SDS_FILE_HEADER, BandBlockSize) == 128); +static_assert(offsetof(SDS_FILE_HEADER, SectionBlockSize) == 160); +static_assert(offsetof(SDS_FILE_HEADER, FileOffset) == 192); +static_assert(offsetof(SDS_FILE_HEADER, Reserved) == 208); +static_assert(sizeof(SDS_FILE_HEADER) == 512); + //--------------------------------------- class SDSSectionName { diff --git a/src/SDSFilePython.cpp b/src/SDSFilePython.cpp index 9905eee1..921f33b7 100644 --- a/src/SDSFilePython.cpp +++ b/src/SDSFilePython.cpp @@ -1385,7 +1385,7 @@ PyObject * CompressFile(PyObject * self, PyObject * args, PyObject * kwargs) for (int64_t i = 0; i < arrayCount; i++) { pDest->ArrayLength = pSrc->ArrayLength; - pDest->ItemSize = (int32_t)pSrc->ItemSize; + pDest->ItemSize = static_cast(pSrc->ItemSize); pDest->pArrayObject = pSrc->pObject; // We do not need this.. pDest->NumBytes = pSrc->NumBytes; pDest->NumpyDType = pSrc->NumpyDType; diff --git a/src/Sort.cpp b/src/Sort.cpp index 316f1ff7..d2e8dfa6 100644 --- a/src/Sort.cpp +++ b/src/Sort.cpp @@ -2507,8 +2507,6 @@ static int par_amergesort(int64_t * pCutOffs, // May be NULL (if so no partition PAR_SORT_TYPE sortType; int64_t sizeofT; - int64_t sizeofUINDEX; - } psort; psort.funcSingleMerge = single_amergesort; @@ -2519,7 +2517,6 @@ static int par_amergesort(int64_t * pCutOffs, // May be NULL (if so no partition psort.strlen = strlen; psort.sortType = sortType; - psort.sizeofUINDEX = sizeof(UINDEX); if (strlen > 0) { psort.sizeofT = strlen; @@ -2553,7 +2550,7 @@ static int par_amergesort(int64_t * pCutOffs, // May be NULL (if so no partition // shift the data pointers to match the partition // call a single threaded merge callbackArg->funcSingleMerge(callbackArg->pValues + (partStart * callbackArg->sizeofT), - callbackArg->pToSort + (partStart * callbackArg->sizeofUINDEX), partLength, + callbackArg->pToSort + (partStart * sizeof(UINDEX)), partLength, callbackArg->strlen, callbackArg->sortType); return true; @@ -3014,7 +3011,7 @@ PyObject * SortInPlaceIndirect(PyObject * self, PyObject * args) for (int i = 0; i < arraySize1; i++) { - pDataIn[i] = (int32_t)inverseSort[pDataIn[i]]; + pDataIn[i] = static_cast(inverseSort[pDataIn[i]]); } WORKSPACE_FREE(inverseSort); } @@ -3978,7 +3975,6 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n bool * pFilter; int64_t base_index; int64_t strlen; - int64_t sizeofUINDEX; } pgroup; @@ -3996,9 +3992,6 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n pgroup.pFilter = pFilter; pgroup.base_index = base_index; pgroup.strlen = itemSizeValues; - const int64_t INDEX_SIZE = (int64_t)sizeof(UINDEX); - - pgroup.sizeofUINDEX = INDEX_SIZE; // Use threads per partition auto lambdaPSCallback = [](void * callbackArgT, int core, int64_t workIndex) -> bool @@ -4026,10 +4019,10 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n // call a single threaded merge callbackArg->pUniqueCounts[t] = callbackArg->funcSingleGroup(callbackArg->pValues + (partStart * callbackArg->strlen), partLength, - callbackArg->pIndex + (partStart * callbackArg->sizeofUINDEX), - callbackArg->pKeyOut + (partStart * callbackArg->sizeofUINDEX), - callbackArg->pFirstOut + (partStart * callbackArg->sizeofUINDEX), - callbackArg->pCountOut + (partStart * callbackArg->sizeofUINDEX), + callbackArg->pIndex + (partStart * sizeof(UINDEX)), + callbackArg->pKeyOut + (partStart * sizeof(UINDEX)), + callbackArg->pFirstOut + (partStart * sizeof(UINDEX)), + callbackArg->pCountOut + (partStart * sizeof(UINDEX)), callbackArg->pFilter + partStart, 0, // callbackArg->base_index, fix for countout callbackArg->strlen); @@ -4052,10 +4045,10 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n // TODO: fix up keys // parallel add? - PyArrayObject * firstReduced = AllocateNumpyArray(1, (npy_intp *)&totalUniques, INDEX_SIZE == 4 ? NPY_INT32 : NPY_INT64); + PyArrayObject * firstReduced = AllocateNumpyArray(1, (npy_intp *)&totalUniques, sizeof(UINDEX) == 4 ? NPY_INT32 : NPY_INT64); totalUniques++; - PyArrayObject * countReduced = AllocateNumpyArray(1, (npy_intp *)&totalUniques, INDEX_SIZE == 4 ? NPY_INT32 : NPY_INT64); + PyArrayObject * countReduced = AllocateNumpyArray(1, (npy_intp *)&totalUniques, sizeof(UINDEX) == 4 ? NPY_INT32 : NPY_INT64); // ANOTHER PARALEL ROUTINE to copy struct stPGROUPADD @@ -4075,8 +4068,6 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n bool * pFilter; int64_t base_index; - int64_t sizeofUINDEX; - } pgroupadd; pgroupadd.pUniqueCounts = pUniqueCounts; @@ -4094,13 +4085,12 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n pgroupadd.pCountReduced = (char *)PyArray_BYTES(countReduced); // skip first value since reserved for zero bin (and assign it 0) - for (int64_t c = 0; c < INDEX_SIZE; c++) + for (int64_t c = 0; c < sizeof(UINDEX); c++) { *pgroupadd.pCountReduced++ = 0; } pgroupadd.base_index = base_index; - pgroupadd.sizeofUINDEX = sizeof(UINDEX); // Use threads per partition auto lambdaPGADDCallback = [](void * callbackArgT, int core, int64_t workIndex) -> bool @@ -4126,7 +4116,7 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n // printf("[%lld] start: %lld length: %lld ubefore: %lld\n", t, // partStart, partLength, uniquesBefore); - if (callbackArg->sizeofUINDEX == 4) + if constexpr (sizeof(UINDEX) == 4) { int32_t * pKey = (int32_t *)callbackArg->pKeyOut; @@ -4142,7 +4132,7 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n int32_t * pCount = (int32_t *)callbackArg->pCountOut; int32_t * pCountReduced = (int32_t *)callbackArg->pCountReduced; - int32_t ubefore = (int32_t)uniquesBefore; + int32_t ubefore = static_cast(callbackArg->pUniqueCounts[t - 1]); if (t != 0) { @@ -4151,8 +4141,8 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n for (int64_t i = 0; i < partLength; i++) { - pKey[i] += ((int32_t)ubefore + 1); // start at 1 (to reserve zero bin), becomes ikey - pIndex[i] += (int32_t)partStart; + pKey[i] += static_cast(++ubefore); // start at 1 (to reserve zero bin), becomes ikey + pIndex[i] += static_cast(partStart); } } else @@ -4161,7 +4151,7 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n for (int64_t i = 0; i < partLength; i++) { - pKey[i] += ((int32_t)partStart + 1); // start at 1, becomes ikey + pKey[i] += static_cast(++partStart); // start at 1, becomes ikey } } @@ -4178,7 +4168,7 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n for (int64_t i = 0; i < uniqueLength; i++) { - pFirstReduced[i] = pFirst[i] + (int32_t)partStart; + pFirstReduced[i] = static_cast(pFirst[i] + partStart); // printf("setting %lld ", (int64_t)pCount[i]); pCountReduced[i] = pCount[i]; } diff --git a/src/TileRepeat.cpp b/src/TileRepeat.cpp index 6e90a637..f7aa38db 100644 --- a/src/TileRepeat.cpp +++ b/src/TileRepeat.cpp @@ -25,8 +25,7 @@ const union { int32_t i[8]; __m256i m; - //} __vindex8_strides = { 7, 6, 5, 4, 3, 2, 1, 0 }; -} __vindex8_strides = { 0, 1, 2, 3, 4, 5, 6, 7 }; +} vindex8_strides = { 0, 1, 2, 3, 4, 5, 6, 7 }; //----------------------------------- // @@ -41,7 +40,7 @@ void ConvertRecArray(char * pStartOffset, int64_t startRow, int64_t totalRows, s CHUNKROWS = 1; } - __m256i vindex = _mm256_mullo_epi32(_mm256_set1_epi32((int32_t)itemSize), _mm256_loadu_si256(&__vindex8_strides.m)); + __m256i vindex = _mm256_mullo_epi32(_mm256_set1_epi32(static_cast(itemSize)), _mm256_loadu_si256(&vindex8_strides.m)); __m128i vindex128 = _mm256_extracti128_si256(vindex, 0); while (startRow < totalRows) diff --git a/src/UnaryOps.cpp b/src/UnaryOps.cpp index 671e380f..865b6246 100644 --- a/src/UnaryOps.cpp +++ b/src/UnaryOps.cpp @@ -733,7 +733,7 @@ static inline void UnaryOpFastStrided(void * pDataIn, void * pDataOut, int64_t l U256 * pIn1_256 = (U256 *)pIn; U256 * pOut_256 = (U256 *)pOut; - int32_t babyStride = (int32_t)strideIn; + int32_t babyStride = static_cast< int32_t >(strideIn); // add 8 strides everytime we process 8 __m256i mindex = _mm256_mullo_epi32(_mm256_set1_epi32(babyStride), __vec8_strides.m); @@ -2512,7 +2512,7 @@ PyObject * ProcessOneInput(PyArrayObject * inArray, PyArrayObject * outObject1, { // Threading not allowed for this work item, call it directly from // main thread - pUnaryFunc(pDataIn, pDataOut, len, strideIn, PyArray_ITEMSIZE(outputArray)); + pUnaryFunc(pDataIn, pDataOut, len, strideIn, static_cast(PyArray_ITEMSIZE(outputArray))); } else { @@ -2526,7 +2526,7 @@ PyObject * ProcessOneInput(PyArrayObject * inArray, PyArrayObject * outObject1, stCallback.pDataOut = (char *)pDataOut; stCallback.pDataIn = (char *)pDataIn; stCallback.itemSizeIn = strideIn; - stCallback.itemSizeOut = PyArray_ITEMSIZE(outputArray); + stCallback.itemSizeOut = static_cast(PyArray_ITEMSIZE(outputArray)); // This will notify the worker threads of a new work item // most functions are so fast, we do not need more than 3 worker diff --git a/src/one_input.cpp b/src/one_input.cpp index a7528527..836e8e54 100644 --- a/src/one_input.cpp +++ b/src/one_input.cpp @@ -30,7 +30,7 @@ PyObject * process_one_input(PyArrayObject const * in_array, PyArrayObject * out if (direction == 0 && numpy_outtype == -1) { numpy_outtype = riptable_cpp::get_active_value_return( - *opt_op_trait, std::make_index_sequence>{}) ? + *opt_op_trait, std::make_index_sequence>{}) ? numpy_intype : NPY_BOOL; PyArrayObject * result_array{ (ndim <= 1) ? AllocateNumpyArray(1, &len, numpy_outtype) : @@ -53,10 +53,11 @@ PyObject * process_one_input(PyArrayObject const * in_array, PyArrayObject * out } else { - int wanted_outtype = riptable_cpp::get_active_value_return( - *opt_op_trait, std::make_index_sequence>{}) ? - numpy_intype : - NPY_BOOL; + int wanted_outtype = + riptable_cpp::get_active_value_return( + *opt_op_trait, std::make_index_sequence>{}) ? + numpy_intype : + NPY_BOOL; if (numpy_outtype != -1 && numpy_outtype != wanted_outtype) { @@ -164,6 +165,12 @@ namespace riptable_cpp case MATH_OPERATION::ABS: retval.first = abs_op{}; break; + case MATH_OPERATION::FABS: + retval.first = fabs_op{}; + break; + case MATH_OPERATION::SIGN: + retval.first = sign_op{}; + break; case MATH_OPERATION::ISNAN: retval.first = isnan_op{}; break; @@ -177,7 +184,7 @@ namespace riptable_cpp retval.first = isnotfinite_op{}; break; case MATH_OPERATION::NEG: - retval.first = bitwise_not_op{}; + retval.first = neg_op{}; break; case MATH_OPERATION::INVERT: retval.first = bitwise_not_op{}; @@ -197,6 +204,55 @@ namespace riptable_cpp case MATH_OPERATION::SQRT: retval.first = sqrt_op{}; break; + case MATH_OPERATION::LOG: + retval.first = log_op{}; + break; + case MATH_OPERATION::LOG2: + retval.first = log2_op{}; + break; + case MATH_OPERATION::LOG10: + retval.first = log10_op{}; + break; + case MATH_OPERATION::EXP: + retval.first = exp_op{}; + break; + case MATH_OPERATION::EXP2: + retval.first = exp2_op{}; + break; + case MATH_OPERATION::CBRT: + retval.first = cbrt_op{}; + break; + // BUG:: These are defined, but never called + // case MATH_OPERATION::TAN: + // retval.first = tan_op{}; + // break; + // case MATH_OPERATION::SIN: + // retval.first = sin_op{}; + // break; + // case MATH_OPERATION::COS: + // retval.first = cos_op{}; + // break; + case MATH_OPERATION::SIGNBIT: + retval.first = signbit_op{}; + break; + case MATH_OPERATION::LOGICAL_NOT: + retval.first = not_op{}; + break; + case MATH_OPERATION::ISINF: + retval.first = isinf_op{}; + break; + case MATH_OPERATION::ISNOTINF: + retval.first = isnotinf_op{}; + break; + case MATH_OPERATION::ISNORMAL: + retval.first = isnormal_op{}; + break; + case MATH_OPERATION::ISNOTNORMAL: + retval.first = isnotnormal_op{}; + break; + case MATH_OPERATION::ISNANORZERO: + retval.first = isnanorzero_op{}; + break; } return retval; diff --git a/src/one_input.h b/src/one_input.h index 66e44619..0d8f9176 100644 --- a/src/one_input.h +++ b/src/one_input.h @@ -19,7 +19,7 @@ extern "C" namespace riptable_cpp { - using chosen_traits_t = std::pair, std::optional>; + using chosen_traits_t = std::pair, std::optional>; chosen_traits_t set_traits(int32_t const function_num, int32_t const numpy_intype); } // namespace riptable_cpp diff --git a/src/one_input_impl.h b/src/one_input_impl.h index a38cf4cf..0702cc30 100644 --- a/src/one_input_impl.h +++ b/src/one_input_impl.h @@ -616,12 +616,13 @@ namespace riptable_cpp template void calculate_for_active_data_type(char const * in_p, char * out_p, ptrdiff_t & starting_element, - int64_t const in_array_stride, size_t contig_elems, operation_t const & requested_op, - type_variant const & in_type, std::index_sequence) + int64_t const in_array_stride, size_t contig_elems, + single_operation_t const & requested_op, type_variant const & in_type, + std::index_sequence) { (calculate_for_active_operation(in_p, out_p, starting_element, in_array_stride, contig_elems, requested_op, std::get_if(&in_type), - std::make_index_sequence>{}), + std::make_index_sequence>{}), ...); } diff --git a/src/operation_traits.h b/src/operation_traits.h index 2bd0421a..a13e232d 100644 --- a/src/operation_traits.h +++ b/src/operation_traits.h @@ -192,10 +192,10 @@ namespace riptable_cpp using simd_implementation = std::false_type; }; - using operation_t = std::variant; + using single_operation_t = std::variant; } // namespace riptable_cpp diff --git a/src/setup.py b/src/setup.py index 0a1f45ee..4a3233d4 100644 --- a/src/setup.py +++ b/src/setup.py @@ -75,6 +75,7 @@ #extra_compile_args = ['/MT /Ox /Ob2 /Oi /Ot'], # For MSVC windows compiler 2019 it has the new __CxxFrameHandler4 which is found in vcrntime140_1.dll which is not on all systems # We use /dsFH4- to disable this frame handler + #extra_compile_args = ['/Ox','/Ob2','/Oi','/Ot','/d2FH4-','/W3','/FC','/Zc:__cplusplus','/std:c++17','/permissive-','/Zc:strictStrings-'], extra_compile_args = ['/Ox','/Ob2','/Oi','/Ot','/d2FH4-','/W3','/WX','/FC','/Zc:__cplusplus','/std:c++17','/permissive-','/Zc:strictStrings-'], #extra_compile_args = ['/Od','/Z7','/d2FH4-','/W3','/WX','/FC','/Zc:__cplusplus','/std:c++17','/permissive-','/Zc:strictStrings-'], #extra_link_args = ['/debug'] diff --git a/test/riptide_test/test_one_input.cpp b/test/riptide_test/test_one_input.cpp index 67d57e5f..afe14d39 100644 --- a/test/riptide_test/test_one_input.cpp +++ b/test/riptide_test/test_one_input.cpp @@ -79,7 +79,7 @@ namespace "walk_fabs_float"_test = [&] { - operation_t op{ fabs_op{} }; + single_operation_t op{ fabs_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 28, 4, 4, reinterpret_cast(p_float + 5), reinterpret_cast(x.data()), op, @@ -96,7 +96,7 @@ namespace "walk_abs_float"_test = [&] { - operation_t op{ abs_op{} }; + single_operation_t op{ abs_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 28, 4, 4, reinterpret_cast(p_float + 5), reinterpret_cast(x.data()), op, @@ -306,7 +306,7 @@ namespace "walk_round_float"_test = [&] { - operation_t op{ round_op{} }; + single_operation_t op{ round_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_float + 5), reinterpret_cast(x.data()), op, @@ -362,7 +362,7 @@ namespace "walk_floor_float"_test = [&] { - operation_t op{ floor_op{} }; + single_operation_t op{ floor_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_float + 5), reinterpret_cast(x.data()), op, @@ -418,7 +418,7 @@ namespace "walk_trunc_float"_test = [&] { - operation_t op{ trunc_op{} }; + single_operation_t op{ trunc_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_float + 5), reinterpret_cast(x.data()), op, @@ -474,7 +474,7 @@ namespace "walk_ceil_float"_test = [&] { - operation_t op{ ceil_op{} }; + single_operation_t op{ ceil_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_float + 5), reinterpret_cast(x.data()), op, @@ -525,7 +525,7 @@ namespace "walk_sqrt_float"_test = [&] { - operation_t op{ sqrt_op{} }; + single_operation_t op{ sqrt_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_float + 5), reinterpret_cast(x.data()), op, @@ -779,7 +779,7 @@ namespace "walk_isnotnan_float"_test = [&] { - operation_t op{ isnotnan_op{} }; + single_operation_t op{ isnotnan_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_nans + 5), reinterpret_cast(x.data()), op, @@ -836,7 +836,7 @@ namespace "walk_isnan_float"_test = [&] { - operation_t op{ isnan_op{} }; + single_operation_t op{ isnan_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_nans + 5), reinterpret_cast(x.data()), op, @@ -887,7 +887,7 @@ namespace "walk_isfinite_float"_test = [&] { - operation_t op{ isfinite_op{} }; + single_operation_t op{ isfinite_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_inf + 5), reinterpret_cast(x.data()), op, @@ -938,7 +938,7 @@ namespace "walk_isnotfinite_float"_test = [&] { - operation_t op{ isnotfinite_op{} }; + single_operation_t op{ isnotfinite_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_inf + 5), reinterpret_cast(x.data()), op, @@ -989,7 +989,7 @@ namespace "walk_isinf_float"_test = [&] { - operation_t op{ isinf_op{} }; + single_operation_t op{ isinf_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_inf + 5), reinterpret_cast(x.data()), op, @@ -1040,7 +1040,7 @@ namespace "walk_isnotinf_float"_test = [&] { - operation_t op{ isnotinf_op{} }; + single_operation_t op{ isnotinf_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_inf + 5), reinterpret_cast(x.data()), op, @@ -1097,7 +1097,7 @@ namespace "walk_isnormal_float"_test = [&] { - operation_t op{ isnormal_op{} }; + single_operation_t op{ isnormal_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_norm + 5), reinterpret_cast(x.data()), op, @@ -1154,7 +1154,7 @@ namespace "walk_isnotnormal_float"_test = [&] { - operation_t op{ isnotnormal_op{} }; + single_operation_t op{ isnotnormal_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_norm + 5), reinterpret_cast(x.data()), op, @@ -1217,7 +1217,7 @@ namespace "walk_isnanorzero_float"_test = [&] { - operation_t op{ isnanorzero_op{} }; + single_operation_t op{ isnanorzero_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_norm + 5), reinterpret_cast(x.data()), op,