From 701230877326d1601723e46212f0de4233a845fa Mon Sep 17 00:00:00 2001 From: Staffan Tjernstrom <8728287+staffantj@users.noreply.github.com> Date: Wed, 10 Nov 2021 12:11:08 -0500 Subject: [PATCH 1/8] Extend the checking of MATH_OPERATIONS to handle the implemented functions. Also some slight re-naming to distinguish single, from multiple, types --- src/one_input.cpp | 68 +++++++++++++++++++++++++--- src/one_input.h | 2 +- src/one_input_impl.h | 4 +- src/operation_traits.h | 2 +- test/riptide_test/test_one_input.cpp | 32 ++++++------- 5 files changed, 82 insertions(+), 26 deletions(-) diff --git a/src/one_input.cpp b/src/one_input.cpp index a7528527..e31f7889 100644 --- a/src/one_input.cpp +++ b/src/one_input.cpp @@ -30,7 +30,7 @@ PyObject * process_one_input(PyArrayObject const * in_array, PyArrayObject * out if (direction == 0 && numpy_outtype == -1) { numpy_outtype = riptable_cpp::get_active_value_return( - *opt_op_trait, std::make_index_sequence>{}) ? + *opt_op_trait, std::make_index_sequence>{}) ? numpy_intype : NPY_BOOL; PyArrayObject * result_array{ (ndim <= 1) ? AllocateNumpyArray(1, &len, numpy_outtype) : @@ -53,10 +53,11 @@ PyObject * process_one_input(PyArrayObject const * in_array, PyArrayObject * out } else { - int wanted_outtype = riptable_cpp::get_active_value_return( - *opt_op_trait, std::make_index_sequence>{}) ? - numpy_intype : - NPY_BOOL; + int wanted_outtype = + riptable_cpp::get_active_value_return( + *opt_op_trait, std::make_index_sequence>{}) ? + numpy_intype : + NPY_BOOL; if (numpy_outtype != -1 && numpy_outtype != wanted_outtype) { @@ -164,6 +165,12 @@ namespace riptable_cpp case MATH_OPERATION::ABS: retval.first = abs_op{}; break; + case MATH_OPERATION::FABS: + retval.first = fabs_op{}; + break; + case MATH_OPERATION::SIGN: + retval.first = sign_op{}; + break; case MATH_OPERATION::ISNAN: retval.first = isnan_op{}; break; @@ -177,7 +184,7 @@ namespace riptable_cpp retval.first = isnotfinite_op{}; break; case MATH_OPERATION::NEG: - retval.first = bitwise_not_op{}; + retval.first = neg_op{}; break; case MATH_OPERATION::INVERT: retval.first = bitwise_not_op{}; @@ -197,6 +204,55 @@ namespace riptable_cpp case MATH_OPERATION::SQRT: retval.first = sqrt_op{}; break; + case MATH_OPERATION::LOG: + retval.first = log_op{}; + break; + case MATH_OPERATION::LOG2: + retval.first = log2_op{}; + break; + case MATH_OPERATION::LOG10: + retval.first = log10_op{}; + break; + case MATH_OPERATION::EXP: + retval.first = exp_op{}; + break; + case MATH_OPERATION::EXP2: + retval.first = exp2_op{}; + break; + case MATH_OPERATION::CBRT: + retval.first = cbrt_op{}; + break; +// BUG:: These are defined, but never called +// case MATH_OPERATION::TAN: +// retval.first = tan_op{}; +// break; +// case MATH_OPERATION::SIN: +// retval.first = sin_op{}; +// break; +// case MATH_OPERATION::COS: +// retval.first = cos_op{}; +// break; + case MATH_OPERATION::SIGNBIT: + retval.first = signbit_op{}; + break; + case MATH_OPERATION::LOGICAL_NOT: + retval.first = not_op{}; + break; + case MATH_OPERATION::ISINF: + retval.first = isinf_op{}; + break; + case MATH_OPERATION::ISNOTINF: + retval.first = isnotinf_op{}; + break; + case MATH_OPERATION::ISNORMAL: + retval.first = isnormal_op{}; + break; + case MATH_OPERATION::ISNOTNORMAL: + retval.first = isnotnormal_op{}; + break; + case MATH_OPERATION::ISNANORZERO: + retval.first = isnanorzero_op{}; + break; } return retval; diff --git a/src/one_input.h b/src/one_input.h index 66e44619..0d8f9176 100644 --- a/src/one_input.h +++ b/src/one_input.h @@ -19,7 +19,7 @@ extern "C" namespace riptable_cpp { - using chosen_traits_t = std::pair, std::optional>; + using chosen_traits_t = std::pair, std::optional>; chosen_traits_t set_traits(int32_t const function_num, int32_t const numpy_intype); } // namespace riptable_cpp diff --git a/src/one_input_impl.h b/src/one_input_impl.h index a38cf4cf..d78b4fbe 100644 --- a/src/one_input_impl.h +++ b/src/one_input_impl.h @@ -616,12 +616,12 @@ namespace riptable_cpp template void calculate_for_active_data_type(char const * in_p, char * out_p, ptrdiff_t & starting_element, - int64_t const in_array_stride, size_t contig_elems, operation_t const & requested_op, + int64_t const in_array_stride, size_t contig_elems, single_operation_t const & requested_op, type_variant const & in_type, std::index_sequence) { (calculate_for_active_operation(in_p, out_p, starting_element, in_array_stride, contig_elems, requested_op, std::get_if(&in_type), - std::make_index_sequence>{}), + std::make_index_sequence>{}), ...); } diff --git a/src/operation_traits.h b/src/operation_traits.h index 2bd0421a..30323aab 100644 --- a/src/operation_traits.h +++ b/src/operation_traits.h @@ -192,7 +192,7 @@ namespace riptable_cpp using simd_implementation = std::false_type; }; - using operation_t = std::variant; diff --git a/test/riptide_test/test_one_input.cpp b/test/riptide_test/test_one_input.cpp index 67d57e5f..afe14d39 100644 --- a/test/riptide_test/test_one_input.cpp +++ b/test/riptide_test/test_one_input.cpp @@ -79,7 +79,7 @@ namespace "walk_fabs_float"_test = [&] { - operation_t op{ fabs_op{} }; + single_operation_t op{ fabs_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 28, 4, 4, reinterpret_cast(p_float + 5), reinterpret_cast(x.data()), op, @@ -96,7 +96,7 @@ namespace "walk_abs_float"_test = [&] { - operation_t op{ abs_op{} }; + single_operation_t op{ abs_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 28, 4, 4, reinterpret_cast(p_float + 5), reinterpret_cast(x.data()), op, @@ -306,7 +306,7 @@ namespace "walk_round_float"_test = [&] { - operation_t op{ round_op{} }; + single_operation_t op{ round_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_float + 5), reinterpret_cast(x.data()), op, @@ -362,7 +362,7 @@ namespace "walk_floor_float"_test = [&] { - operation_t op{ floor_op{} }; + single_operation_t op{ floor_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_float + 5), reinterpret_cast(x.data()), op, @@ -418,7 +418,7 @@ namespace "walk_trunc_float"_test = [&] { - operation_t op{ trunc_op{} }; + single_operation_t op{ trunc_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_float + 5), reinterpret_cast(x.data()), op, @@ -474,7 +474,7 @@ namespace "walk_ceil_float"_test = [&] { - operation_t op{ ceil_op{} }; + single_operation_t op{ ceil_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_float + 5), reinterpret_cast(x.data()), op, @@ -525,7 +525,7 @@ namespace "walk_sqrt_float"_test = [&] { - operation_t op{ sqrt_op{} }; + single_operation_t op{ sqrt_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_float + 5), reinterpret_cast(x.data()), op, @@ -779,7 +779,7 @@ namespace "walk_isnotnan_float"_test = [&] { - operation_t op{ isnotnan_op{} }; + single_operation_t op{ isnotnan_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_nans + 5), reinterpret_cast(x.data()), op, @@ -836,7 +836,7 @@ namespace "walk_isnan_float"_test = [&] { - operation_t op{ isnan_op{} }; + single_operation_t op{ isnan_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_nans + 5), reinterpret_cast(x.data()), op, @@ -887,7 +887,7 @@ namespace "walk_isfinite_float"_test = [&] { - operation_t op{ isfinite_op{} }; + single_operation_t op{ isfinite_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_inf + 5), reinterpret_cast(x.data()), op, @@ -938,7 +938,7 @@ namespace "walk_isnotfinite_float"_test = [&] { - operation_t op{ isnotfinite_op{} }; + single_operation_t op{ isnotfinite_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_inf + 5), reinterpret_cast(x.data()), op, @@ -989,7 +989,7 @@ namespace "walk_isinf_float"_test = [&] { - operation_t op{ isinf_op{} }; + single_operation_t op{ isinf_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_inf + 5), reinterpret_cast(x.data()), op, @@ -1040,7 +1040,7 @@ namespace "walk_isnotinf_float"_test = [&] { - operation_t op{ isnotinf_op{} }; + single_operation_t op{ isnotinf_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_inf + 5), reinterpret_cast(x.data()), op, @@ -1097,7 +1097,7 @@ namespace "walk_isnormal_float"_test = [&] { - operation_t op{ isnormal_op{} }; + single_operation_t op{ isnormal_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_norm + 5), reinterpret_cast(x.data()), op, @@ -1154,7 +1154,7 @@ namespace "walk_isnotnormal_float"_test = [&] { - operation_t op{ isnotnormal_op{} }; + single_operation_t op{ isnotnormal_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_norm + 5), reinterpret_cast(x.data()), op, @@ -1217,7 +1217,7 @@ namespace "walk_isnanorzero_float"_test = [&] { - operation_t op{ isnanorzero_op{} }; + single_operation_t op{ isnanorzero_op{} }; data_type_t data_type{ float_traits{} }; std::array x{}; walk_data_array(1, 26, 4, 4, reinterpret_cast(p_norm + 5), reinterpret_cast(x.data()), op, From 78401fd41acb2f330b74ce51d41dc2222f9a86bf Mon Sep 17 00:00:00 2001 From: Staffan Tjernstrom <8728287+staffantj@users.noreply.github.com> Date: Tue, 16 Nov 2021 10:05:59 -0500 Subject: [PATCH 2/8] First we remove all the casts and fix up sizeof's --- src/Array.cpp | 2 +- src/BasicMath.cpp | 2 +- src/Compress.cpp | 2 +- src/Convert.cpp | 18 +++++++-------- src/Ema.cpp | 10 ++++---- src/FileReadWrite.cpp | 16 ++++++------- src/GroupBy.cpp | 54 +++++++++++++++++++++---------------------- src/HashLinear.cpp | 2 +- src/MathThreads.cpp | 2 +- src/MathWorker.cpp | 8 +++---- src/Merge.cpp | 6 ++--- src/MultiKey.cpp | 28 +++++++++++----------- src/Recycler.cpp | 2 +- src/Reduce.cpp | 2 +- src/RipTide.cpp | 12 +++++----- src/SDSFile.cpp | 20 ++++++++-------- src/SDSFilePython.cpp | 2 +- src/Sort.cpp | 12 +++++----- src/TileRepeat.cpp | 2 +- src/UnaryOps.cpp | 2 +- src/setup.py | 3 ++- 21 files changed, 104 insertions(+), 103 deletions(-) diff --git a/src/Array.cpp b/src/Array.cpp index 3cbee39e..478483e2 100644 --- a/src/Array.cpp +++ b/src/Array.cpp @@ -76,7 +76,7 @@ static PyObject * ConvertInt32(PyObject * object) Py_DecRef((PyObject *)pArray); return NULL; } - pInt32[i] = (int32_t)val; + pInt32[i] = val; } return (PyObject *)pArray; } diff --git a/src/BasicMath.cpp b/src/BasicMath.cpp index 05b799f6..aab90d4d 100644 --- a/src/BasicMath.cpp +++ b/src/BasicMath.cpp @@ -236,7 +236,7 @@ static const inline T MaxOp(T x, T y) // x & y; } static const int64_t NAN_FOR_INT64 = (int64_t)0x8000000000000000LL; -static const int64_t NAN_FOR_INT32 = (int32_t)0x80000000; +static const int64_t NAN_FOR_INT32 = 0x80000000; template static const inline double DivOp(T x, T y) diff --git a/src/Compress.cpp b/src/Compress.cpp index 1dffd99d..35818a23 100644 --- a/src/Compress.cpp +++ b/src/Compress.cpp @@ -27,7 +27,7 @@ void FillInNumpyHeader(NUMPY_HEADERSIZE * pstNumpyHeader, int32_t dtype, int32_t pstNumpyHeader->ndim = (int8_t)ndim; pstNumpyHeader->flags = flags; - pstNumpyHeader->itemsize = (int32_t)itemsize; + pstNumpyHeader->itemsize = itemsize; if (pstNumpyHeader->ndim > 3) { diff --git a/src/Convert.cpp b/src/Convert.cpp index 7fd98f5e..bfc5eabc 100644 --- a/src/Convert.cpp +++ b/src/Convert.cpp @@ -1049,7 +1049,7 @@ void * GetInvalid(int dtype) // NOTE: if they are the same type, special fast routine called PyObject * ConvertSafeInternal(PyArrayObject * const inArr1, const int64_t out_dtype) { - const int32_t numpyOutType = (int32_t)out_dtype; + const int32_t numpyOutType = out_dtype; const int32_t numpyInType = PyArray_TYPE(inArr1); if (numpyOutType < 0 || numpyInType > NPY_LONGDOUBLE || numpyOutType > NPY_LONGDOUBLE) @@ -1251,7 +1251,7 @@ PyObject * ConvertSafe(PyObject * self, PyObject * args) // GetConversionFunctionUnsafe. PyObject * ConvertUnsafeInternal(PyArrayObject * inArr1, int64_t out_dtype) { - const int32_t numpyOutType = (int32_t)out_dtype; + const int32_t numpyOutType = out_dtype; const int32_t numpyInType = ObjectToDtype(inArr1); if (numpyOutType < 0 || numpyInType < 0 || numpyInType > NPY_LONGDOUBLE || numpyOutType > NPY_LONGDOUBLE) @@ -2075,7 +2075,7 @@ int64_t Combine1Filter(void * pInputIndex, INDEX * pOutput = (INDEX *)pOutputIndex; // WORKSPACE_ALLOC - int64_t allocSize = hashLength * sizeof(int32_t); + int64_t allocSize = hashLength * sizeof; int32_t * pHash = (int32_t *)WorkSpaceAllocLarge(allocSize); memset(pHash, 0, allocSize); @@ -2096,7 +2096,7 @@ int64_t Combine1Filter(void * pInputIndex, if (pHash[index] == 0) { // First time, assign FirstKey - pNewFirst[uniquecount] = (int32_t)i; + pNewFirst[uniquecount] = i; uniquecount++; // printf("reassign index:%lld to bin:%d\n", (int64_t)index, @@ -2110,7 +2110,7 @@ int64_t Combine1Filter(void * pInputIndex, { // Get reassigned key // printf("exiting index:%lld to bin:%d\n", (int64_t)index, - // (int32_t)pHash[index]); + // pHash[index]); pOutput[i] = (INDEX)pHash[index]; } } @@ -2141,7 +2141,7 @@ int64_t Combine1Filter(void * pInputIndex, if (pHash[index] == 0) { // First time, assign FirstKey - pNewFirst[uniquecount] = (int32_t)i; + pNewFirst[uniquecount] = i; uniquecount++; // ReassignKey @@ -2271,7 +2271,7 @@ PyObject * CombineAccum1Filter(PyObject * self, PyObject * args) { int32_t * pFirstReduced = (int32_t *)PyArray_BYTES(firstArrayReduced); - memcpy(pFirstReduced, pFirst, uniqueCount * sizeof(int32_t)); + memcpy(pFirstReduced, pFirst, uniqueCount * sizeof); } Py_DecRef((PyObject *)firstArray); firstArray = firstArrayReduced; @@ -3242,7 +3242,7 @@ PyObject * HomogenizeArrays(PyObject * self, PyObject * args) if (aInfo) { - int32_t dtype = (int32_t)PyLong_AsLong(dtypeObject); + int32_t dtype = PyLong_AsLong(dtypeObject); if (dtype != -1) { @@ -4017,7 +4017,7 @@ PyObject * ApplyRows(PyObject * self, PyObject * args, PyObject * kwargs) if (aInfo) { - int32_t dtype = (int32_t)PyLong_AsLong(dtypeObject); + int32_t dtype = PyLong_AsLong(dtypeObject); if (dtype != -1) { diff --git a/src/Ema.cpp b/src/Ema.cpp index 38605196..4aea425d 100644 --- a/src/Ema.cpp +++ b/src/Ema.cpp @@ -725,7 +725,7 @@ static void CumSum(void * pKeyT, void * pAccumBin, void * pColumn, int64_t numUn U Invalid = GET_INVALID(pDest[0]); - int32_t windowSize = (int32_t)windowSize1; + int32_t windowSize = windowSize1; LOGGING("cumsum %lld %lld %lld %p %p\n", numUnique, totalInputRows, (int64_t)Invalid, pIncludeMask, pResetMask); @@ -848,7 +848,7 @@ static void CumProd(void * pKeyT, void * pAccumBin, void * pColumn, int64_t numU U Invalid = GET_INVALID(pDest[0]); - int32_t windowSize = (int32_t)windowSize1; + int32_t windowSize = windowSize1; LOGGING("cumprod %lld %lld %p %p\n", numUnique, totalInputRows, pIncludeMask, pResetMask); @@ -1196,7 +1196,7 @@ class EmaByBase // (double)pLastEma[location], (double)-decayRate, (double)pTime[i], // (double)pLastTime[location] ); pLastEma[location] = pSrc[i] + pLastEma[location] * exp(-decayRate * (pTime[i] - pLastTime[location])); - // printf("[%d][%d] %lf\n", i, (int32_t)location, + // printf("[%d][%d] %lf\n", i, location, // (double)pLastEma[location]); pLastTime[location] = pTime[i]; pDest[i] = pLastEma[location]; @@ -1827,7 +1827,7 @@ void EmaByCall(void * pEmaBy, int64_t i) //--------------------------------------------------------------- // Arg1 = LIST of numpy arrays which has the values to accumulate (often all the -// columns in a dataset) Arg2 = iKey = numpy array (int32_t) which has the index +// columns in a dataset) Arg2 = iKey = numpy array which has the index // to the unique keys (ikey from MultiKeyGroupBy32) Arg3 = integer unique rows // Arg4 = integer (function number to execute for cumsum, ema) // Arg5 = params for function must be (decay/window, time, includemask, @@ -1958,7 +1958,7 @@ PyObject * EmaAll32(PyObject * self, PyObject * args) stEma32 * pstEma32 = (stEma32 *)WORKSPACE_ALLOC((sizeof(stEma32) + 8 + sizeof(stEmaReturn)) * tupleSize); pstEma32->aInfo = aInfo; - pstEma32->funcNum = (int32_t)funcNum; + pstEma32->funcNum = funcNum; pstEma32->pKey = (int32_t *)PyArray_BYTES(iKey); pstEma32->tupleSize = tupleSize; pstEma32->uniqueRows = unique_rows; diff --git a/src/FileReadWrite.cpp b/src/FileReadWrite.cpp index 1b1d6326..3b8ef3da 100644 --- a/src/FileReadWrite.cpp +++ b/src/FileReadWrite.cpp @@ -158,8 +158,8 @@ DWORD CFileReadWrite::ReadChunk(void * buffer, uint32_t count) OverlappedIO.hEvent = NULL; OverlappedIO.InternalHigh = 0; OverlappedIO.Internal = 0; - OverlappedIO.OffsetHigh = (uint32_t)(BufferPos >> 32); - OverlappedIO.Offset = (uint32_t)BufferPos; + OverlappedIO.OffsetHigh = (BufferPos >> 32); + OverlappedIO.Offset = BufferPos; bool bReadDone; @@ -215,8 +215,8 @@ DWORD CFileReadWrite::ReadChunkAsync(void * buffer, uint32_t count, DWORD * last { pOverlapped->InternalHigh = 0; pOverlapped->Internal = 0; - pOverlapped->OffsetHigh = (uint32_t)(BufferPos >> 32); - pOverlapped->Offset = (uint32_t)BufferPos; + pOverlapped->OffsetHigh = (BufferPos >> 32); + pOverlapped->Offset = BufferPos; bool bReadDone; @@ -330,8 +330,8 @@ DWORD CFileReadWrite::WriteChunk(void * buffer, uint32_t count) OverlappedIO.hEvent = 0; OverlappedIO.InternalHigh = 0; OverlappedIO.Internal = 0; - OverlappedIO.OffsetHigh = (uint32_t)(BufferPos >> 32); - OverlappedIO.Offset = (uint32_t)BufferPos; + OverlappedIO.OffsetHigh = (BufferPos >> 32); + OverlappedIO.Offset = BufferPos; bool bWriteDone; @@ -391,8 +391,8 @@ DWORD CFileReadWrite::WriteChunkAsync(void * buffer, uint32_t count, DWORD * las { pOverlapped->InternalHigh = 0; pOverlapped->Internal = 0; - pOverlapped->OffsetHigh = (uint32_t)(BufferPos >> 32); - pOverlapped->Offset = (uint32_t)BufferPos; + pOverlapped->OffsetHigh = (BufferPos >> 32); + pOverlapped->Offset = BufferPos; bool bWriteDone; diff --git a/src/GroupBy.cpp b/src/GroupBy.cpp index 7ef5827d..76c01559 100644 --- a/src/GroupBy.cpp +++ b/src/GroupBy.cpp @@ -1031,7 +1031,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t nth = (int32_t)funcParam; + int32_t nth = funcParam; U invalid = GET_INVALID(pDest[0]); @@ -1060,7 +1060,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t nth = (int32_t)funcParam; + int32_t nth = funcParam; // For all the bins we have to fill for (int64_t i = binLow; i < binHigh; i++) @@ -1203,7 +1203,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = (int32_t)funcParam; + int32_t windowSize = funcParam; U invalid = GET_INVALID(pDest[0]); if (binLow == 0) @@ -1267,7 +1267,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = (int32_t)funcParam; + int32_t windowSize = funcParam; U invalid = GET_INVALID(pDest[0]); if (binLow == 0) @@ -1387,7 +1387,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = (int32_t)funcParam; + int32_t windowSize = funcParam; U invalid = GET_INVALID((U)0); double invalid_out = GET_INVALID(pDest[0]); @@ -1452,7 +1452,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = (int32_t)funcParam; + int32_t windowSize = funcParam; U invalid = GET_INVALID((U)0); double invalid_out = GET_INVALID(pDest[0]); @@ -1581,7 +1581,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = (int32_t)funcParam; + int32_t windowSize = funcParam; U invalid = GET_INVALID(pDest[0]); LOGGING("in rolling count %lld %lld sizeofdest %lld\n", binLow, binHigh, sizeof(U)); @@ -1642,7 +1642,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = (int32_t)funcParam; + int32_t windowSize = funcParam; U invalid = GET_INVALID(pDest[0]); // printf("binlow %lld, binhigh %lld, windowSize: %d\n", binLow, binHigh, @@ -1719,7 +1719,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = (int32_t)funcParam; + int32_t windowSize = funcParam; U invalid = GET_INVALID(pDest[0]); if (binLow == 0) @@ -1941,7 +1941,7 @@ class GroupByBase pEnd--; } - nCount = (int32_t)((pEnd + 1) - pSort); + nCount = ((pEnd + 1) - pSort); if (nCount == 0) { @@ -2010,7 +2010,7 @@ class GroupByBase // Copy over the items for this group for (int j = 0; j < nCount; j++) { - // printf("**[%lld][%d] %d\n", i, index + j, (int32_t)pGroup[index + + // printf("**[%lld][%d] %d\n", i, index + j, pGroup[index + // j]); pSort[j] = pSrc[pGroup[index + j]]; } @@ -2029,7 +2029,7 @@ class GroupByBase pEnd--; } - nCount = (int32_t)((pEnd + 1) - pSort); + nCount = ((pEnd + 1) - pSort); if (nCount == 0) { @@ -2189,7 +2189,7 @@ static void GatherMean(stGroupBy32 * pstGroupBy32, void * pDataInT, void * pData U * pDataInBase = (U *)pDataInT; U * pDataOut = (U *)pDataOutT; - int64_t allocSize = sizeof(int32_t) * numUnique; + int64_t allocSize = sizeof * numUnique; int32_t * pCountOut = (int32_t *)WORKSPACE_ALLOC(allocSize); memset(pCountOut, 0, allocSize); @@ -3498,7 +3498,7 @@ void GroupByCall(void * pGroupBy, int64_t i) //--------------------------------------------------------------- // Arg1 = LIST of numpy arrays which has the values to accumulate (often all the -// columns in a dataset) Arg2 = numpy array (int32_t) which has the index to the +// columns in a dataset) Arg2 = numpy array which has the index to the // unique keys (ikey from MultiKeyGroupBy32) Arg3 = integer unique rows Arg4 = // integer (function number to execute for sum,mean,min, max) Example: // GroupByOp2(array, ikey, 3, np.float32) Returns cells @@ -3636,7 +3636,7 @@ PyObject * GroupBySingleOpMultiBands(ArrayInfo * aInfo, PyArrayObject * iKey, Py if (hasCounts) { // Zero out for them - int64_t allocSize = sizeof(int32_t) * unique_rows; + int64_t allocSize = sizeof * unique_rows; pCountOut = (int32_t *)WORKSPACE_ALLOC(allocSize); if (pCountOut == NULL) { @@ -3687,7 +3687,7 @@ PyObject * GroupBySingleOpMultiBands(ArrayInfo * aInfo, PyArrayObject * iKey, Py // next low bin is the previous high bin low = high; - pstGroupBy32->returnObjects[i].funcNum = (int32_t)firstFuncNum; + pstGroupBy32->returnObjects[i].funcNum = firstFuncNum; pstGroupBy32->returnObjects[i].didWork = 0; // Assign working memory per call @@ -3783,7 +3783,7 @@ PyObject * GroupBySingleOpMultithreaded(ArrayInfo * aInfo, PyArrayObject * iKey, if (hasCounts) { // Zero out for them - int64_t allocSize = sizeof(int32_t) * unique_rows * numCores; + int64_t allocSize = sizeof * unique_rows * numCores; pCountOut = (int32_t *)WORKSPACE_ALLOC(allocSize); if (pCountOut == NULL) { @@ -3816,7 +3816,7 @@ PyObject * GroupBySingleOpMultithreaded(ArrayInfo * aInfo, PyArrayObject * iKey, for (int i = 0; i < numCores; i++) { - pstGroupBy32->returnObjects[i].funcNum = (int32_t)firstFuncNum; + pstGroupBy32->returnObjects[i].funcNum = firstFuncNum; pstGroupBy32->returnObjects[i].binLow = binLow; pstGroupBy32->returnObjects[i].binHigh = binHigh; @@ -3872,7 +3872,7 @@ PyObject * GroupBySingleOpMultithreaded(ArrayInfo * aInfo, PyArrayObject * iKey, //--------------------------------------------------------------- // Arg1 = LIST of numpy arrays which has the values to accumulate (often all the -// columns in a dataset) Arg2 = iKey = numpy array (int32_t) which has the index +// columns in a dataset) Arg2 = iKey = numpy array which has the index // to the unique keys (ikey from MultiKeyGroupBy32) Arg3 = integer unique rows // Arg4 = LIST of integer (function number to execute for sum,mean,min, max) // Arg5 = LIST of integers (binLow -- invalid bin) @@ -4015,7 +4015,7 @@ PyObject * GroupByAll32(PyObject * self, PyObject * args) int overflow = 0; int64_t funcNum = PyLong_AsLongLongAndOverflow(PyList_GET_ITEM(listFuncNum, i), &overflow); - pstGroupBy32->returnObjects[i].funcNum = (int32_t)funcNum; + pstGroupBy32->returnObjects[i].funcNum = funcNum; int64_t binLow = PyLong_AsLongLongAndOverflow(PyList_GET_ITEM(listBinLow, i), &overflow); pstGroupBy32->returnObjects[i].binLow = binLow; @@ -4047,12 +4047,12 @@ PyObject * GroupByAll32(PyObject * self, PyObject * args) if (hasCounts) { // Zero out for them - pCountOut = (int32_t *)WORKSPACE_ALLOC(sizeof(int32_t) * unique_rows); + pCountOut = (int32_t *)WORKSPACE_ALLOC(sizeof * unique_rows); if (pCountOut == NULL) { return NULL; } - memset(pCountOut, 0, sizeof(int32_t) * unique_rows); + memset(pCountOut, 0, sizeof * unique_rows); } } else @@ -4108,11 +4108,11 @@ PyObject * GroupByAll32(PyObject * self, PyObject * args) //--------------------------------------------------------------- // Arg1 = LIST of numpy arrays which has the values to accumulate (often all the -// columns in a dataset) Arg2 =iKey = numpy array (int32_t) which has the index -// to the unique keys (ikey from MultiKeyGroupBy32) Arg3 =iGroup: (int32_t) +// columns in a dataset) Arg2 =iKey = numpy array which has the index +// to the unique keys (ikey from MultiKeyGroupBy32) Arg3 =iGroup: // array size is same as multikey, unique keys are grouped together Arg4 -// =iFirst: (int32_t) array size is number of unique keys, indexes into iGroup -// Arg5 =nCount: (int32_t) array size is number of unique keys for the group, is +// =iFirst: array size is number of unique keys, indexes into iGroup +// Arg5 =nCount: array size is number of unique keys for the group, is // how many member of the group (paired with iFirst) Arg6 = integer unique rows // Arg7 = LIST of integers (function number to execute for sum,mean,min, max) // Arg8 = LIST of integers (binLow -- invalid bin) @@ -4255,7 +4255,7 @@ PyObject * GroupByAllPack32(PyObject * self, PyObject * args) { int overflow = 0; int64_t funcNum = PyLong_AsLongLongAndOverflow(PyList_GET_ITEM(listFuncNum, i), &overflow); - pstGroupBy32->returnObjects[i].funcNum = (int32_t)funcNum; + pstGroupBy32->returnObjects[i].funcNum = funcNum; int64_t binLow = PyLong_AsLongLongAndOverflow(PyList_GET_ITEM(listBinLow, i), &overflow); pstGroupBy32->returnObjects[i].binLow = binLow; diff --git a/src/HashLinear.cpp b/src/HashLinear.cpp index 8b9fc3dd..e55ded21 100644 --- a/src/HashLinear.cpp +++ b/src/HashLinear.cpp @@ -1224,7 +1224,7 @@ static void IsMemberMK(void * pHashLinearVoid, int64_t arraySize, void * pInputT //#define HASH_int32_t uint64_t h = (uint64_t)item; h ^= (h >> 16); h= h & //(HashSize-1); #define HASH_int32_t uint64_t h= fasthash64_8(item) & //(HashSize-1); -#define HASH_int32_t uint64_t h = _mm_crc32_u32(0, (uint32_t)item) & (HashSize - 1); +#define HASH_int32_t uint64_t h = _mm_crc32_u32(0, item) & (HashSize - 1); //#define HASH_int64_t uint64_t h= _mm_crc32_u64(0, item) & (HashSize-1); #define HASH_int64_t uint64_t h = fasthash64_8(item) & (HashSize - 1); diff --git a/src/MathThreads.cpp b/src/MathThreads.cpp index 117c17ed..acdb671c 100644 --- a/src/MathThreads.cpp +++ b/src/MathThreads.cpp @@ -82,7 +82,7 @@ void * WorkerThreadFunction(void * lpParam) { stWorkerRing * pWorkerRing = (stWorkerRing *)lpParam; - uint32_t core = (uint32_t)(InterlockedIncrement64(&pWorkerRing->WorkThread)); + uint32_t core = (InterlockedIncrement64(&pWorkerRing->WorkThread)); core = core - 1; // if (core > 3) core += 16; diff --git a/src/MathWorker.cpp b/src/MathWorker.cpp index f994600b..37551c26 100644 --- a/src/MathWorker.cpp +++ b/src/MathWorker.cpp @@ -76,14 +76,14 @@ MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) if (n >= 1) { __cpuid((int *)reg, 1); - f1c = (uint32_t)reg[2]; - f1d = (uint32_t)reg[3]; + f1c = reg[2]; + f1d = reg[3]; } if (n >= 7) { __cpuidex((int *)reg, 7, 0); - f7b = (uint32_t)reg[1]; - f7c = (uint32_t)reg[2]; + f7b = reg[1]; + f7c = reg[2]; } } #elif defined(__i386__) && defined(__PIC__) && ! defined(__clang__) && defined(__GNUC__) diff --git a/src/Merge.cpp b/src/Merge.cpp index 338320b8..c3165e9e 100644 --- a/src/Merge.cpp +++ b/src/Merge.cpp @@ -1639,11 +1639,11 @@ PyObject * BooleanToFancy(PyObject * self, PyObject * args, PyObject * kwargs) { if (pBooleanMask[i]) { - *pOut++ = (int32_t)i; + *pOut++ = i; } else { - *pOutFalse++ = (int32_t)i; + *pOutFalse++ = i; } } } @@ -1672,7 +1672,7 @@ PyObject * BooleanToFancy(PyObject * self, PyObject * args, PyObject * kwargs) { if (pBooleanMask[i]) { - *pOut++ = (int32_t)i; + *pOut++ = i; } } } diff --git a/src/MultiKey.cpp b/src/MultiKey.cpp index 6ede9e42..2f6ed7c5 100644 --- a/src/MultiKey.cpp +++ b/src/MultiKey.cpp @@ -13,7 +13,7 @@ char * RotateArrays(int64_t tupleSize, ArrayInfo * aInfo) { // TODO: Is it fast to calculate the hash here? - // uint32_t* pHashArray = (uint32_t*)malloc(sizeof(uint32_t) * totalRows); + // uint32_t* pHashArray = (uint32_t*)malloc(sizeof * totalRows); int64_t totalItemSize = 0; int64_t totalRows = 0; @@ -599,7 +599,7 @@ int32_t * GroupByPackFixup32(int64_t numUnique, int64_t totalRows, void * pIndex // reserve for invalid bin numUnique += GB_BASE_INDEX; - int64_t size = sizeof(int32_t) * totalRows; + int64_t size = sizeof * totalRows; int32_t * pNextArray = (int32_t *)WORKSPACE_ALLOC(size); // mark all invalid @@ -609,7 +609,7 @@ int32_t * GroupByPackFixup32(int64_t numUnique, int64_t totalRows, void * pIndex } // Go backwards - for (int32_t i = (int32_t)totalRows - 1; i >= 0; i--) + for (int32_t i = totalRows - 1; i >= 0; i--) { K group = pIndexArray[i]; @@ -657,7 +657,7 @@ bool GroupByPackFinal32(int64_t numUnique, int64_t totalRows, void * pIndexArray int32_t * pFirstArray = (int32_t *)PyArray_BYTES(firstArray); int32_t * pCountArray = (int32_t *)PyArray_BYTES(countArray); - int32_t i = (int32_t)totalRows; + int32_t i = totalRows; // TODO -- how to handle empty? pSortArray[0] = GB_INVALID_INDEX; @@ -844,7 +844,7 @@ PyObject * GroupByPack32(PyObject * self, PyObject * args) // Next was not supplied and must be calculated bMustFree = true; - int64_t allocsize = sizeof(int32_t) * (numUnique + GB_BASE_INDEX); + int64_t allocsize = sizeof * (numUnique + GB_BASE_INDEX); pGroupArray = (int32_t *)WORKSPACE_ALLOC(allocsize); switch (numpyIndexType) @@ -1151,8 +1151,8 @@ PyObject * MultiKeyGroupBy32Super(PyObject * self, PyObject * args) int32_t * pNextArray = (int32_t *)PyArray_BYTES(nextArray); // now one based, so zero reserved - int32_t * pUniqueArray = (int32_t *)WORKSPACE_ALLOC((mkp.aInfo[0].ArrayLength + 1) * sizeof(int32_t)); - int32_t * pUniqueCountArray = (int32_t *)WORKSPACE_ALLOC((mkp.aInfo[0].ArrayLength + 1) * sizeof(int32_t)); + int32_t * pUniqueArray = (int32_t *)WORKSPACE_ALLOC((mkp.aInfo[0].ArrayLength + 1) * sizeof); + int32_t * pUniqueCountArray = (int32_t *)WORKSPACE_ALLOC((mkp.aInfo[0].ArrayLength + 1) * sizeof); int64_t numUnique = 0; @@ -1190,9 +1190,9 @@ PyObject * MultiKeyGroupBy32Super(PyObject * self, PyObject * args) if (uniqueArray != NULL && uniqueCountArray != NULL) { int32_t * pUniqueArrayDest = (int32_t *)PyArray_BYTES(uniqueArray); - memcpy(pUniqueArrayDest, pUniqueArray, numUnique * sizeof(int32_t)); + memcpy(pUniqueArrayDest, pUniqueArray, numUnique * sizeof); int32_t * pUniqueCountArrayDest = (int32_t *)PyArray_BYTES(uniqueCountArray); - memcpy(pUniqueCountArrayDest, pUniqueCountArray, numUnique * sizeof(int32_t)); + memcpy(pUniqueCountArrayDest, pUniqueCountArray, numUnique * sizeof); } WORKSPACE_FREE(pUniqueArray); WORKSPACE_FREE(pUniqueCountArray); @@ -1260,10 +1260,10 @@ PyObject * MultiKeyUnique32(PyObject * self, PyObject * args) if (mkp.totalRows < 2100000000) { // worst case alloc - int32_t * pIndexArray = (int32_t *)WORKSPACE_ALLOC(mkp.totalRows * sizeof(int32_t)); + int32_t * pIndexArray = (int32_t *)WORKSPACE_ALLOC(mkp.totalRows * sizeof); // worst case alloc - int32_t * pCountArray = (int32_t *)WORKSPACE_ALLOC(mkp.totalRows * sizeof(int32_t)); + int32_t * pCountArray = (int32_t *)WORKSPACE_ALLOC(mkp.totalRows * sizeof); if (pIndexArray == NULL || pCountArray == NULL) { @@ -1288,7 +1288,7 @@ PyObject * MultiKeyUnique32(PyObject * self, PyObject * args) if (indexArray2 != NULL) { int32_t * pIndexArray2 = (int32_t *)PyArray_BYTES(indexArray2); - memcpy(pIndexArray2, pIndexArray, numUnique * sizeof(int32_t)); + memcpy(pIndexArray2, pIndexArray, numUnique * sizeof); } PyArrayObject * countArray2 = AllocateNumpyArray(1, (npy_intp *)&numUnique, NPY_INT32); @@ -1297,7 +1297,7 @@ PyObject * MultiKeyUnique32(PyObject * self, PyObject * args) if (countArray2 != NULL) { int32_t * pCountArray2 = (int32_t *)PyArray_BYTES(countArray2); - memcpy(pCountArray2, pCountArray, numUnique * sizeof(int32_t)); + memcpy(pCountArray2, pCountArray, numUnique * sizeof); } // free the side array @@ -1513,7 +1513,7 @@ PyObject * MultiKeyHash(PyObject * self, PyObject * args) assert(mkp.totalRows < 2100000000); - int32_t i = (int32_t)mkp.totalRows; + int32_t i = mkp.totalRows; while (i > 0) { diff --git a/src/Recycler.cpp b/src/Recycler.cpp index 8bb07f52..376dbfb9 100644 --- a/src/Recycler.cpp +++ b/src/Recycler.cpp @@ -664,7 +664,7 @@ static bool DeleteNumpyArray(PyArrayObject * inArr) ndim == 1 && strides != NULL && itemSize == strides[0]) { // Based on size and type, lookup - int32_t log2 = (int32_t)lzcnt_64(totalSize); + int32_t log2 = lzcnt_64(totalSize); stRecycleList * pItems = &g_stRecycleList[log2][type]; diff --git a/src/Reduce.cpp b/src/Reduce.cpp index e677f414..23dd107d 100644 --- a/src/Reduce.cpp +++ b/src/Reduce.cpp @@ -2942,7 +2942,7 @@ static PyObject * ReduceInternal(PyArrayObject * inArr1, REDUCE_FUNCTIONS func, Py_RETURN_NONE; } - stScatterGatherFunc sgFunc = { (int32_t)numpyInType, 0, 0, 0, 0, 0 }; + stScatterGatherFunc sgFunc = { numpyInType, 0, 0, 0, 0, 0 }; FUNCTION_LIST fl{}; fl.AnyScatterGatherCall = pFunction; diff --git a/src/RipTide.cpp b/src/RipTide.cpp index f195d77a..1547860b 100644 --- a/src/RipTide.cpp +++ b/src/RipTide.cpp @@ -1015,7 +1015,7 @@ bool ConvertSingleItemArray(void * pInput, int16_t numpyInType, _m256all * pDest break; CASE_NPY_UINT32: CASE_NPY_INT32: - pDest->i = _mm256_set1_epi32((int32_t)value); + pDest->i = _mm256_set1_epi32(value); break; CASE_NPY_UINT64: @@ -1092,7 +1092,7 @@ bool ConvertScalarObject(PyObject * inObject1, _m256all * pDest, int16_t numpyOu break; CASE_NPY_UINT32: CASE_NPY_INT32: - pDest->i = _mm256_set1_epi32((int32_t)value); + pDest->i = _mm256_set1_epi32(value); break; CASE_NPY_UINT64: @@ -1177,10 +1177,10 @@ bool ConvertScalarObject(PyObject * inObject1, _m256all * pDest, int16_t numpyOu pDest->i = _mm256_set1_epi16((uint16_t)value2); break; CASE_NPY_INT32: - pDest->i = _mm256_set1_epi32((int32_t)value); + pDest->i = _mm256_set1_epi32(value); break; CASE_NPY_UINT32: - pDest->i = _mm256_set1_epi32((uint32_t)value2); + pDest->i = _mm256_set1_epi32(value2); break; CASE_NPY_INT64: @@ -1223,10 +1223,10 @@ bool ConvertScalarObject(PyObject * inObject1, _m256all * pDest, int16_t numpyOu pDest->i = _mm256_set1_epi16((uint16_t)value); break; CASE_NPY_UINT32: - pDest->i = _mm256_set1_epi32((uint32_t)value); + pDest->i = _mm256_set1_epi32(value); break; CASE_NPY_INT32: - pDest->i = _mm256_set1_epi32((int32_t)value); + pDest->i = _mm256_set1_epi32(value); break; CASE_NPY_UINT64: diff --git a/src/SDSFile.cpp b/src/SDSFile.cpp index 2dcf51c4..f4d47d65 100644 --- a/src/SDSFile.cpp +++ b/src/SDSFile.cpp @@ -508,8 +508,8 @@ int64_t SDSFileReadChunk(SDS_EVENT_HANDLE eventHandle, SDS_FILE_HANDLE Handle, v OverlappedIO.hEvent = eventHandle; OverlappedIO.InternalHigh = 0; OverlappedIO.Internal = 0; - OverlappedIO.OffsetHigh = (uint32_t)(BufferPos >> 32); - OverlappedIO.Offset = (uint32_t)BufferPos; + OverlappedIO.OffsetHigh = (BufferPos >> 32); + OverlappedIO.Offset = BufferPos; bool bReadDone; @@ -594,8 +594,8 @@ int64_t SDSFileWriteChunk(SDS_EVENT_HANDLE eventHandle, SDS_FILE_HANDLE Handle, OverlappedIO.hEvent = eventHandle; OverlappedIO.InternalHigh = 0; OverlappedIO.Internal = 0; - OverlappedIO.OffsetHigh = (uint32_t)(BufferPos >> 32); - OverlappedIO.Offset = (uint32_t)BufferPos; + OverlappedIO.OffsetHigh = (BufferPos >> 32); + OverlappedIO.Offset = BufferPos; OVERLAPPED * pos = &OverlappedIO; DWORD n; @@ -2725,8 +2725,8 @@ bool CompressFileArray(void * pstCompressArraysV, int32_t core, int64_t t) pArrayBlock->CompressionType = COMPRESSION_TYPE_ZSTD; // New version 4.3 - pArrayBlock->ArrayBandCount = (int32_t)bandCount; - pArrayBlock->ArrayBandSize = (int32_t)bandSize; + pArrayBlock->ArrayBandCount = bandCount; + pArrayBlock->ArrayBandSize = bandSize; // record array dimensions int32_t ndim = pArrayInfo->NDim; @@ -2752,7 +2752,7 @@ bool CompressFileArray(void * pstCompressArraysV, int32_t core, int64_t t) pArrayBlock->Flags = pArrayInfo->Flags; pArrayBlock->DType = pArrayInfo->NumpyDType; - pArrayBlock->ItemSize = (int32_t)pArrayInfo->ItemSize; + pArrayBlock->ItemSize = pArrayInfo->ItemSize; pArrayBlock->HeaderLength = sizeof(SDS_ARRAY_BLOCK); pArrayBlock->Magic = COMPRESSION_MAGIC; @@ -2991,7 +2991,7 @@ bool SDSWriteFileInternal(const char * fileName, } pDestArrayBlock[arrayNumber].Flags = aInfo[arrayNumber].Flags; pDestArrayBlock[arrayNumber].DType = aInfo[arrayNumber].NumpyDType; - pDestArrayBlock[arrayNumber].ItemSize = (int32_t)aInfo[arrayNumber].ItemSize; + pDestArrayBlock[arrayNumber].ItemSize = aInfo[arrayNumber].ItemSize; pDestArrayBlock[arrayNumber].HeaderLength = sizeof(SDS_ARRAY_BLOCK); pDestArrayBlock[arrayNumber].Magic = COMPRESSION_MAGIC; @@ -3748,7 +3748,7 @@ class SDSDecompressFile { pArrayNames[i] = NULL; } - pArrayEnums = (int32_t *)WORKSPACE_ALLOC(NameCount * sizeof(int32_t)); + pArrayEnums = (int32_t *)WORKSPACE_ALLOC(NameCount * sizeof); for (int32_t i = 0; i < NameCount; i++) { pArrayEnums[i] = 0; @@ -6292,7 +6292,7 @@ class SDSDecompressManyFiles } // track last valid row? - LastRow = (int32_t)fileRow; + LastRow = fileRow; } //------------------------------------------------ diff --git a/src/SDSFilePython.cpp b/src/SDSFilePython.cpp index 9905eee1..a9f91926 100644 --- a/src/SDSFilePython.cpp +++ b/src/SDSFilePython.cpp @@ -1385,7 +1385,7 @@ PyObject * CompressFile(PyObject * self, PyObject * args, PyObject * kwargs) for (int64_t i = 0; i < arrayCount; i++) { pDest->ArrayLength = pSrc->ArrayLength; - pDest->ItemSize = (int32_t)pSrc->ItemSize; + pDest->ItemSize = pSrc->ItemSize; pDest->pArrayObject = pSrc->pObject; // We do not need this.. pDest->NumBytes = pSrc->NumBytes; pDest->NumpyDType = pSrc->NumpyDType; diff --git a/src/Sort.cpp b/src/Sort.cpp index 316f1ff7..37951a3d 100644 --- a/src/Sort.cpp +++ b/src/Sort.cpp @@ -3014,7 +3014,7 @@ PyObject * SortInPlaceIndirect(PyObject * self, PyObject * args) for (int i = 0; i < arraySize1; i++) { - pDataIn[i] = (int32_t)inverseSort[pDataIn[i]]; + pDataIn[i] = inverseSort[pDataIn[i]]; } WORKSPACE_FREE(inverseSort); } @@ -4142,7 +4142,7 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n int32_t * pCount = (int32_t *)callbackArg->pCountOut; int32_t * pCountReduced = (int32_t *)callbackArg->pCountReduced; - int32_t ubefore = (int32_t)uniquesBefore; + int32_t ubefore = uniquesBefore; if (t != 0) { @@ -4151,8 +4151,8 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n for (int64_t i = 0; i < partLength; i++) { - pKey[i] += ((int32_t)ubefore + 1); // start at 1 (to reserve zero bin), becomes ikey - pIndex[i] += (int32_t)partStart; + pKey[i] += (ubefore + 1); // start at 1 (to reserve zero bin), becomes ikey + pIndex[i] += partStart; } } else @@ -4161,7 +4161,7 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n for (int64_t i = 0; i < partLength; i++) { - pKey[i] += ((int32_t)partStart + 1); // start at 1, becomes ikey + pKey[i] += (partStart + 1); // start at 1, becomes ikey } } @@ -4178,7 +4178,7 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n for (int64_t i = 0; i < uniqueLength; i++) { - pFirstReduced[i] = pFirst[i] + (int32_t)partStart; + pFirstReduced[i] = pFirst[i] + partStart; // printf("setting %lld ", (int64_t)pCount[i]); pCountReduced[i] = pCount[i]; } diff --git a/src/TileRepeat.cpp b/src/TileRepeat.cpp index 6e90a637..7434f930 100644 --- a/src/TileRepeat.cpp +++ b/src/TileRepeat.cpp @@ -41,7 +41,7 @@ void ConvertRecArray(char * pStartOffset, int64_t startRow, int64_t totalRows, s CHUNKROWS = 1; } - __m256i vindex = _mm256_mullo_epi32(_mm256_set1_epi32((int32_t)itemSize), _mm256_loadu_si256(&__vindex8_strides.m)); + __m256i vindex = _mm256_mullo_epi32(_mm256_set1_epi32(itemSize), _mm256_loadu_si256(&__vindex8_strides.m)); __m128i vindex128 = _mm256_extracti128_si256(vindex, 0); while (startRow < totalRows) diff --git a/src/UnaryOps.cpp b/src/UnaryOps.cpp index 671e380f..1026a1aa 100644 --- a/src/UnaryOps.cpp +++ b/src/UnaryOps.cpp @@ -733,7 +733,7 @@ static inline void UnaryOpFastStrided(void * pDataIn, void * pDataOut, int64_t l U256 * pIn1_256 = (U256 *)pIn; U256 * pOut_256 = (U256 *)pOut; - int32_t babyStride = (int32_t)strideIn; + ssize_t babyStride = strideIn; // add 8 strides everytime we process 8 __m256i mindex = _mm256_mullo_epi32(_mm256_set1_epi32(babyStride), __vec8_strides.m); diff --git a/src/setup.py b/src/setup.py index 0a1f45ee..dedec5eb 100644 --- a/src/setup.py +++ b/src/setup.py @@ -75,7 +75,8 @@ #extra_compile_args = ['/MT /Ox /Ob2 /Oi /Ot'], # For MSVC windows compiler 2019 it has the new __CxxFrameHandler4 which is found in vcrntime140_1.dll which is not on all systems # We use /dsFH4- to disable this frame handler - extra_compile_args = ['/Ox','/Ob2','/Oi','/Ot','/d2FH4-','/W3','/WX','/FC','/Zc:__cplusplus','/std:c++17','/permissive-','/Zc:strictStrings-'], + extra_compile_args = ['/Ox','/Ob2','/Oi','/Ot','/d2FH4-','/W3','/FC','/Zc:__cplusplus','/std:c++17','/permissive-','/Zc:strictStrings-'], + #extra_compile_args = ['/Ox','/Ob2','/Oi','/Ot','/d2FH4-','/W3','/WX''/FC','/Zc:__cplusplus','/std:c++17','/permissive-','/Zc:strictStrings-'], #extra_compile_args = ['/Od','/Z7','/d2FH4-','/W3','/WX','/FC','/Zc:__cplusplus','/std:c++17','/permissive-','/Zc:strictStrings-'], #extra_link_args = ['/debug'] ) From 809bfaf6193a0ae2cfcde810f0a2f7ae8b34e700 Mon Sep 17 00:00:00 2001 From: Staffan Tjernstrom <8728287+staffantj@users.noreply.github.com> Date: Tue, 16 Nov 2021 14:17:30 -0500 Subject: [PATCH 3/8] At this point we have a lot of warnings --- src/Convert.cpp | 6 +++--- src/GroupBy.cpp | 12 ++++++------ src/HashFunctions.cpp | 2 +- src/MultiKey.cpp | 22 +++++++++++----------- src/SDSFile.cpp | 2 +- src/Sort.cpp | 2 +- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/Convert.cpp b/src/Convert.cpp index bfc5eabc..da5409a1 100644 --- a/src/Convert.cpp +++ b/src/Convert.cpp @@ -2075,7 +2075,7 @@ int64_t Combine1Filter(void * pInputIndex, INDEX * pOutput = (INDEX *)pOutputIndex; // WORKSPACE_ALLOC - int64_t allocSize = hashLength * sizeof; + int64_t allocSize = hashLength * sizeof(int32_t); int32_t * pHash = (int32_t *)WorkSpaceAllocLarge(allocSize); memset(pHash, 0, allocSize); @@ -2271,7 +2271,7 @@ PyObject * CombineAccum1Filter(PyObject * self, PyObject * args) { int32_t * pFirstReduced = (int32_t *)PyArray_BYTES(firstArrayReduced); - memcpy(pFirstReduced, pFirst, uniqueCount * sizeof); + memcpy(pFirstReduced, pFirst, uniqueCount * sizeof(int32_t)); } Py_DecRef((PyObject *)firstArray); firstArray = firstArrayReduced; @@ -2303,7 +2303,7 @@ int64_t iFirstFilter(void * pInputIndex, INDEX * pInput = (INDEX *)pInputIndex; int64_t * pNewFirst = (int64_t *)pNewFirstIndex; int64_t invalid = (int64_t)(1LL << (sizeof(int64_t) * 8 - 1)); - + // Fill with invalid for (int64_t i = 0; i < hashLength; i++) { diff --git a/src/GroupBy.cpp b/src/GroupBy.cpp index 76c01559..e55be8e3 100644 --- a/src/GroupBy.cpp +++ b/src/GroupBy.cpp @@ -2189,7 +2189,7 @@ static void GatherMean(stGroupBy32 * pstGroupBy32, void * pDataInT, void * pData U * pDataInBase = (U *)pDataInT; U * pDataOut = (U *)pDataOutT; - int64_t allocSize = sizeof * numUnique; + int64_t allocSize = sizeof(int32_t) * numUnique; int32_t * pCountOut = (int32_t *)WORKSPACE_ALLOC(allocSize); memset(pCountOut, 0, allocSize); @@ -3636,7 +3636,7 @@ PyObject * GroupBySingleOpMultiBands(ArrayInfo * aInfo, PyArrayObject * iKey, Py if (hasCounts) { // Zero out for them - int64_t allocSize = sizeof * unique_rows; + int64_t allocSize = sizeof(int32_t) * unique_rows; pCountOut = (int32_t *)WORKSPACE_ALLOC(allocSize); if (pCountOut == NULL) { @@ -3783,7 +3783,7 @@ PyObject * GroupBySingleOpMultithreaded(ArrayInfo * aInfo, PyArrayObject * iKey, if (hasCounts) { // Zero out for them - int64_t allocSize = sizeof * unique_rows * numCores; + int64_t allocSize = sizeof(int32_t) * unique_rows * numCores; pCountOut = (int32_t *)WORKSPACE_ALLOC(allocSize); if (pCountOut == NULL) { @@ -4047,12 +4047,12 @@ PyObject * GroupByAll32(PyObject * self, PyObject * args) if (hasCounts) { // Zero out for them - pCountOut = (int32_t *)WORKSPACE_ALLOC(sizeof * unique_rows); + pCountOut = (int32_t *)WORKSPACE_ALLOC(sizeof(int32_t) * unique_rows); if (pCountOut == NULL) { return NULL; } - memset(pCountOut, 0, sizeof * unique_rows); + memset(pCountOut, 0, sizeof(int32_t) * unique_rows); } } else @@ -4076,7 +4076,7 @@ PyObject * GroupByAll32(PyObject * self, PyObject * args) returnTuple = PyTuple_New(tupleSize); PyObject * returnCount = NULL; - // Fill in results + // Fill in results for (int i = 0; i < tupleSize; i++) { PyObject * item = pstGroupBy32->returnObjects[i].returnObject; diff --git a/src/HashFunctions.cpp b/src/HashFunctions.cpp index 9453d617..621176bc 100644 --- a/src/HashFunctions.cpp +++ b/src/HashFunctions.cpp @@ -482,7 +482,7 @@ PyObject * IsMemberCategoricalFixup(PyObject * self, PyObject * args) PyErr_Format(PyExc_ValueError, "IsMemberCategoricalFixup third argument must be type int32_t not %s", NpyToString(uniqueType)); return NULL; - } + } LOGGING( "IsMemberCategoricalFixup uniqlength:%d baseoffset1:%d " diff --git a/src/MultiKey.cpp b/src/MultiKey.cpp index 2f6ed7c5..a49aafce 100644 --- a/src/MultiKey.cpp +++ b/src/MultiKey.cpp @@ -599,7 +599,7 @@ int32_t * GroupByPackFixup32(int64_t numUnique, int64_t totalRows, void * pIndex // reserve for invalid bin numUnique += GB_BASE_INDEX; - int64_t size = sizeof * totalRows; + int64_t size = sizeof(int32_t) * totalRows; int32_t * pNextArray = (int32_t *)WORKSPACE_ALLOC(size); // mark all invalid @@ -844,7 +844,7 @@ PyObject * GroupByPack32(PyObject * self, PyObject * args) // Next was not supplied and must be calculated bMustFree = true; - int64_t allocsize = sizeof * (numUnique + GB_BASE_INDEX); + int64_t allocsize = sizeof(int32_t) * (numUnique + GB_BASE_INDEX); pGroupArray = (int32_t *)WORKSPACE_ALLOC(allocsize); switch (numpyIndexType) @@ -1151,8 +1151,8 @@ PyObject * MultiKeyGroupBy32Super(PyObject * self, PyObject * args) int32_t * pNextArray = (int32_t *)PyArray_BYTES(nextArray); // now one based, so zero reserved - int32_t * pUniqueArray = (int32_t *)WORKSPACE_ALLOC((mkp.aInfo[0].ArrayLength + 1) * sizeof); - int32_t * pUniqueCountArray = (int32_t *)WORKSPACE_ALLOC((mkp.aInfo[0].ArrayLength + 1) * sizeof); + int32_t * pUniqueArray = (int32_t *)WORKSPACE_ALLOC((mkp.aInfo[0].ArrayLength + 1) * sizeof(int32_t)); + int32_t * pUniqueCountArray = (int32_t *)WORKSPACE_ALLOC((mkp.aInfo[0].ArrayLength + 1) * sizeof(int32_t)); int64_t numUnique = 0; @@ -1190,9 +1190,9 @@ PyObject * MultiKeyGroupBy32Super(PyObject * self, PyObject * args) if (uniqueArray != NULL && uniqueCountArray != NULL) { int32_t * pUniqueArrayDest = (int32_t *)PyArray_BYTES(uniqueArray); - memcpy(pUniqueArrayDest, pUniqueArray, numUnique * sizeof); + memcpy(pUniqueArrayDest, pUniqueArray, numUnique * sizeof(int32_t)); int32_t * pUniqueCountArrayDest = (int32_t *)PyArray_BYTES(uniqueCountArray); - memcpy(pUniqueCountArrayDest, pUniqueCountArray, numUnique * sizeof); + memcpy(pUniqueCountArrayDest, pUniqueCountArray, numUnique * sizeof(int32_t)); } WORKSPACE_FREE(pUniqueArray); WORKSPACE_FREE(pUniqueCountArray); @@ -1260,10 +1260,10 @@ PyObject * MultiKeyUnique32(PyObject * self, PyObject * args) if (mkp.totalRows < 2100000000) { // worst case alloc - int32_t * pIndexArray = (int32_t *)WORKSPACE_ALLOC(mkp.totalRows * sizeof); + int32_t * pIndexArray = (int32_t *)WORKSPACE_ALLOC(mkp.totalRows * sizeof(int32_t)); // worst case alloc - int32_t * pCountArray = (int32_t *)WORKSPACE_ALLOC(mkp.totalRows * sizeof); + int32_t * pCountArray = (int32_t *)WORKSPACE_ALLOC(mkp.totalRows * sizeof(int32_t)); if (pIndexArray == NULL || pCountArray == NULL) { @@ -1288,7 +1288,7 @@ PyObject * MultiKeyUnique32(PyObject * self, PyObject * args) if (indexArray2 != NULL) { int32_t * pIndexArray2 = (int32_t *)PyArray_BYTES(indexArray2); - memcpy(pIndexArray2, pIndexArray, numUnique * sizeof); + memcpy(pIndexArray2, pIndexArray, numUnique * sizeof(int32_t)); } PyArrayObject * countArray2 = AllocateNumpyArray(1, (npy_intp *)&numUnique, NPY_INT32); @@ -1297,7 +1297,7 @@ PyObject * MultiKeyUnique32(PyObject * self, PyObject * args) if (countArray2 != NULL) { int32_t * pCountArray2 = (int32_t *)PyArray_BYTES(countArray2); - memcpy(pCountArray2, pCountArray, numUnique * sizeof); + memcpy(pCountArray2, pCountArray, numUnique * sizeof(int32_t)); } // free the side array @@ -2148,7 +2148,7 @@ PyObject * GroupFromBinCount(PyObject * self, PyObject * args) LOGGING("multithread makeigroup %lld cores %lld unique %lld rows\n", cores, totalUnique, totalRows); g_cMathWorker->DoMultiThreadedWork((int)cores, lambdaBinCountCallback, &stMaster); - WORKSPACE_FREE(pstBinCount); + WORKSPACE_FREE(pstBinCount); } else { diff --git a/src/SDSFile.cpp b/src/SDSFile.cpp index f4d47d65..2225be5c 100644 --- a/src/SDSFile.cpp +++ b/src/SDSFile.cpp @@ -3748,7 +3748,7 @@ class SDSDecompressFile { pArrayNames[i] = NULL; } - pArrayEnums = (int32_t *)WORKSPACE_ALLOC(NameCount * sizeof); + pArrayEnums = (int32_t *)WORKSPACE_ALLOC(NameCount * sizeof(int32_t)); for (int32_t i = 0; i < NameCount; i++) { pArrayEnums[i] = 0; diff --git a/src/Sort.cpp b/src/Sort.cpp index 37951a3d..ff8def8e 100644 --- a/src/Sort.cpp +++ b/src/Sort.cpp @@ -2988,7 +2988,7 @@ PyObject * SortInPlaceIndirect(PyObject * self, PyObject * args) int32_t * pDataIn = (int32_t *)PyArray_BYTES(inArr1); int32_t * pSort = (int32_t *)PyArray_BYTES(inSort); - int32_t * inverseSort = (int32_t *)WORKSPACE_ALLOC(sortSize * sizeof(int32_t)); + int32_t * inverseSort = (int32_t *)WORKSPACE_ALLOC(sortSize * sizeof(int32_t)); for (int i = 0; i < sortSize; i++) { inverseSort[pSort[i]] = i; From d9c2f506a49933d2b11027d34abb444730eba2a7 Mon Sep 17 00:00:00 2001 From: Staffan Tjernstrom <8728287+staffantj@users.noreply.github.com> Date: Thu, 18 Nov 2021 13:32:41 -0500 Subject: [PATCH 4/8] The completed pass of fixing up casts to int32_t (and sometimes re-instating them) --- src/Array.cpp | 2 +- src/BasicMath.cpp | 6 +- src/CommonInc.h | 8 +-- src/Compress.cpp | 6 +- src/Convert.cpp | 28 +++++--- src/Ema.cpp | 6 +- src/GroupBy.cpp | 132 +++++++++++++++++------------------ src/HashFunctions.cpp | 29 ++++---- src/HashLinear.cpp | 40 +++++------ src/MathThreads.cpp | 2 +- src/MathWorker.h | 4 +- src/Merge.cpp | 35 +++++++--- src/MultiKey.cpp | 15 ++-- src/Recycler.cpp | 11 +-- src/Reduce.cpp | 2 +- src/RipTide.cpp | 29 ++++---- src/RipTide.h | 2 +- src/SDSFile.cpp | 25 +++++-- src/SDSFilePython.cpp | 2 +- src/Sort.cpp | 16 ++--- src/TileRepeat.cpp | 6 +- src/UnaryOps.cpp | 152 ++++++++++++++++++++--------------------- src/UnaryOps.h | 4 +- src/one_input.cpp | 24 +++---- src/one_input_impl.h | 5 +- src/operation_traits.h | 8 +-- src/setup.py | 4 +- 27 files changed, 328 insertions(+), 275 deletions(-) diff --git a/src/Array.cpp b/src/Array.cpp index 478483e2..5811930f 100644 --- a/src/Array.cpp +++ b/src/Array.cpp @@ -76,7 +76,7 @@ static PyObject * ConvertInt32(PyObject * object) Py_DecRef((PyObject *)pArray); return NULL; } - pInt32[i] = val; + pInt32[i] = static_cast(val); } return (PyObject *)pArray; } diff --git a/src/BasicMath.cpp b/src/BasicMath.cpp index aab90d4d..a40530d8 100644 --- a/src/BasicMath.cpp +++ b/src/BasicMath.cpp @@ -3428,9 +3428,9 @@ PyObject * TwoInputsInternal(CTwoInputs & twoInputs, int64_t funcNumber) if (twoInputs.len2 > 1) fl.Input2Strides = PyArray_STRIDE(twoInputs.inArr2, 0); - fl.InputItemSize = - twoInputs.len1 >= twoInputs.len2 ? PyArray_ITEMSIZE(twoInputs.inArr) : PyArray_ITEMSIZE(twoInputs.inArr2); - fl.OutputItemSize = PyArray_ITEMSIZE(outputArray); + fl.InputItemSize = twoInputs.len1 >= twoInputs.len2 ? static_cast(PyArray_ITEMSIZE(twoInputs.inArr)) : + static_cast(PyArray_ITEMSIZE(twoInputs.inArr2)); + fl.OutputItemSize = static_cast(PyArray_ITEMSIZE(outputArray)); fl.NumpyOutputType = wantedOutputType; fl.NumpyType = twoInputs.len1 >= twoInputs.len2 ? twoInputs.numpyInType1 : twoInputs.numpyInType2; diff --git a/src/CommonInc.h b/src/CommonInc.h index e8e37365..a547d0e5 100644 --- a/src/CommonInc.h +++ b/src/CommonInc.h @@ -344,8 +344,8 @@ struct stScatterGatherFunc typedef double (*ANY_SCATTER_GATHER_FUNC)(void * pDataIn, int64_t len, stScatterGatherFunc * pstScatterGatherFunc); // typedef void(*UNARY_FUNC)(void* pDataIn, void* pDataOut, int64_t len); -typedef void (*UNARY_FUNC)(void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut); -typedef void (*UNARY_FUNC_STRIDED)(void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut); +typedef void (*UNARY_FUNC)(void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut); +typedef void (*UNARY_FUNC_STRIDED)(void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut); // Pass in two vectors and return one vector // Used for operations like C = A + B @@ -474,8 +474,8 @@ struct FUNCTION_LIST int16_t NumpyOutputType; // The item size for two input arrays assumed to be the same - int64_t InputItemSize; - int64_t OutputItemSize; + int32_t InputItemSize; + int32_t OutputItemSize; // Strides may be 0 if it is a scalar or length 1 int64_t Input1Strides; diff --git a/src/Compress.cpp b/src/Compress.cpp index 35818a23..61918d7f 100644 --- a/src/Compress.cpp +++ b/src/Compress.cpp @@ -16,7 +16,7 @@ // Used when compressing from one numpy array to another numpy array // void FillInNumpyHeader(NUMPY_HEADERSIZE * pstNumpyHeader, int32_t dtype, int32_t ndim, int64_t * dims, int32_t flags, - int64_t itemsize) + int32_t itemsize) { if (pstNumpyHeader) { @@ -46,7 +46,7 @@ void FillInNumpyHeader(NUMPY_HEADERSIZE * pstNumpyHeader, int32_t dtype, int32_t // Returns temporary array to compress into // Caller is responsible for freeing memory NUMPY_HEADERSIZE * AllocCompressedMemory(int64_t arraylength, int32_t dtype, int32_t ndim, int64_t * dims, int32_t flags, - int64_t itemsize) + int32_t itemsize) { int64_t source_size = arraylength * itemsize; int64_t dest_size = ZSTD_compressBound(source_size); @@ -188,7 +188,7 @@ PyObject * CompressDecompressArrays(PyObject * self, PyObject * args) pstCompressArrays->pNumpyHeaders[t] = AllocCompressedMemory(aInfo[t].ArrayLength, aInfo[t].NumpyDType, aInfo[t].NDim, (int64_t *)(((PyArrayObject_fields *)aInfo[t].pObject)->dimensions), - PyArray_FLAGS(aInfo[t].pObject), aInfo[t].ItemSize); + PyArray_FLAGS(aInfo[t].pObject), static_cast(aInfo[t].ItemSize)); } // This will kick off the workerthread and call CompressMemoryArray diff --git a/src/Convert.cpp b/src/Convert.cpp index da5409a1..1aa83261 100644 --- a/src/Convert.cpp +++ b/src/Convert.cpp @@ -1049,8 +1049,8 @@ void * GetInvalid(int dtype) // NOTE: if they are the same type, special fast routine called PyObject * ConvertSafeInternal(PyArrayObject * const inArr1, const int64_t out_dtype) { - const int32_t numpyOutType = out_dtype; - const int32_t numpyInType = PyArray_TYPE(inArr1); + const int32_t numpyOutType = static_cast(out_dtype); + const int32_t numpyInType = static_cast(PyArray_TYPE(inArr1)); if (numpyOutType < 0 || numpyInType > NPY_LONGDOUBLE || numpyOutType > NPY_LONGDOUBLE) { @@ -1095,11 +1095,11 @@ PyObject * ConvertSafeInternal(PyArrayObject * const inArr1, const int64_t out_d void * pBadOutput1 = GetDefaultForType(numpyOutType); // Check the strides of both the input and output to make sure we can handle - int64_t strideIn; + int32_t strideIn; int directionIn = GetStridesAndContig(inArr1, ndim, strideIn); int ndimOut; - int64_t strideOut; + int32_t strideOut; int directionOut = GetStridesAndContig(outArray, ndimOut, strideOut); // If the input is C and/or F-contiguous, the output should have @@ -1251,8 +1251,8 @@ PyObject * ConvertSafe(PyObject * self, PyObject * args) // GetConversionFunctionUnsafe. PyObject * ConvertUnsafeInternal(PyArrayObject * inArr1, int64_t out_dtype) { - const int32_t numpyOutType = out_dtype; - const int32_t numpyInType = ObjectToDtype(inArr1); + const int32_t numpyOutType = static_cast(out_dtype); + const int32_t numpyInType = static_cast(ObjectToDtype(inArr1)); if (numpyOutType < 0 || numpyInType < 0 || numpyInType > NPY_LONGDOUBLE || numpyOutType > NPY_LONGDOUBLE) { @@ -2085,6 +2085,11 @@ int64_t Combine1Filter(void * pInputIndex, { for (int64_t i = 0; i < arrayLength; i++) { + if (i > std::numeric_limits::max()) + { + PyErr_Format(PyExc_ValueError, "Combine1Filter has attempted to exceed the size of the index value [%lld]", i); + return 0; + } if (pFilter[i]) { INDEX index = pInput[i]; @@ -2096,7 +2101,7 @@ int64_t Combine1Filter(void * pInputIndex, if (pHash[index] == 0) { // First time, assign FirstKey - pNewFirst[uniquecount] = i; + pNewFirst[uniquecount] = static_cast(i); uniquecount++; // printf("reassign index:%lld to bin:%d\n", (int64_t)index, @@ -2132,6 +2137,11 @@ int64_t Combine1Filter(void * pInputIndex, // When no filter provided for (int64_t i = 0; i < arrayLength; i++) { + if (i > std::numeric_limits::max()) + { + PyErr_Format(PyExc_ValueError, "Combine1Filter has attempted to exceed the size of the index value [%lld]", i); + return 0; + } INDEX index = pInput[i]; // printf("[%lld] got index\n", (int64_t)index); @@ -2141,7 +2151,7 @@ int64_t Combine1Filter(void * pInputIndex, if (pHash[index] == 0) { // First time, assign FirstKey - pNewFirst[uniquecount] = i; + pNewFirst[uniquecount] = static_cast(i); uniquecount++; // ReassignKey @@ -2303,7 +2313,7 @@ int64_t iFirstFilter(void * pInputIndex, INDEX * pInput = (INDEX *)pInputIndex; int64_t * pNewFirst = (int64_t *)pNewFirstIndex; int64_t invalid = (int64_t)(1LL << (sizeof(int64_t) * 8 - 1)); - + // Fill with invalid for (int64_t i = 0; i < hashLength; i++) { diff --git a/src/Ema.cpp b/src/Ema.cpp index 4aea425d..9bec05d1 100644 --- a/src/Ema.cpp +++ b/src/Ema.cpp @@ -725,7 +725,7 @@ static void CumSum(void * pKeyT, void * pAccumBin, void * pColumn, int64_t numUn U Invalid = GET_INVALID(pDest[0]); - int32_t windowSize = windowSize1; + int32_t windowSize = std::lround(windowSize1); LOGGING("cumsum %lld %lld %lld %p %p\n", numUnique, totalInputRows, (int64_t)Invalid, pIncludeMask, pResetMask); @@ -848,7 +848,7 @@ static void CumProd(void * pKeyT, void * pAccumBin, void * pColumn, int64_t numU U Invalid = GET_INVALID(pDest[0]); - int32_t windowSize = windowSize1; + int32_t windowSize = std::lround(windowSize1); LOGGING("cumprod %lld %lld %p %p\n", numUnique, totalInputRows, pIncludeMask, pResetMask); @@ -1958,7 +1958,7 @@ PyObject * EmaAll32(PyObject * self, PyObject * args) stEma32 * pstEma32 = (stEma32 *)WORKSPACE_ALLOC((sizeof(stEma32) + 8 + sizeof(stEmaReturn)) * tupleSize); pstEma32->aInfo = aInfo; - pstEma32->funcNum = funcNum; + pstEma32->funcNum = static_cast(funcNum); pstEma32->pKey = (int32_t *)PyArray_BYTES(iKey); pstEma32->tupleSize = tupleSize; pstEma32->uniqueRows = unique_rows; diff --git a/src/GroupBy.cpp b/src/GroupBy.cpp index e55be8e3..68005840 100644 --- a/src/GroupBy.cpp +++ b/src/GroupBy.cpp @@ -1031,7 +1031,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t nth = funcParam; + int64_t nth = funcParam; U invalid = GET_INVALID(pDest[0]); @@ -1040,7 +1040,7 @@ class GroupByBase { if (pCount[i] > 0 && nth < pCount[i]) { - int32_t grpIndex = pFirst[i] + nth; + int64_t grpIndex = pFirst[i] + nth; int32_t bin = pGroup[grpIndex]; pDest[i] = pSrc[bin]; } @@ -1060,14 +1060,14 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t nth = funcParam; + int64_t nth = funcParam; // For all the bins we have to fill for (int64_t i = binLow; i < binHigh; i++) { if (pCount[i] > 0 && nth < pCount[i]) { - int32_t grpIndex = pFirst[i] + nth; + int64_t grpIndex = pFirst[i] + nth; int32_t bin = pGroup[grpIndex]; memcpy(&pDest[i * itemSize], &pSrc[bin * itemSize], itemSize); } @@ -1203,17 +1203,17 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = funcParam; + int64_t windowSize = funcParam; U invalid = GET_INVALID(pDest[0]); if (binLow == 0) { // Mark all invalid if invalid bin int32_t start = pFirst[0]; - int32_t last = start + pCount[0]; - for (int j = start; j < last; j++) + int64_t last = start + pCount[0]; + for (int64_t j = start; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = invalid; } binLow++; @@ -1227,12 +1227,12 @@ class GroupByBase for (int64_t i = binLow; i < binHigh; i++) { int32_t start = pFirst[i]; - int32_t last = start + pCount[i]; + int64_t last = start + pCount[i]; U currentSum = 0; // Priming of the summation - for (int j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { int32_t index = pGroup[j]; @@ -1240,7 +1240,7 @@ class GroupByBase pDest[index] = currentSum; } - for (int j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { int32_t index = pGroup[j]; @@ -1267,17 +1267,17 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = funcParam; + int64_t windowSize = funcParam; U invalid = GET_INVALID(pDest[0]); if (binLow == 0) { // Mark all invalid if invalid bin int32_t start = pFirst[0]; - int32_t last = start + pCount[0]; - for (int j = start; j < last; j++) + int64_t last = start + pCount[0]; + for (int64_t j = start; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = invalid; } binLow++; @@ -1294,14 +1294,14 @@ class GroupByBase for (int64_t i = binLow; i < binHigh; i++) { int32_t start = pFirst[i]; - int32_t last = start + pCount[i]; + int64_t last = start + pCount[i]; U currentSum = 0; // Priming of the summation - for (int j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; U value = (U)pSrc[index]; if (value != invalid) { @@ -1310,9 +1310,9 @@ class GroupByBase pDest[index] = currentSum; } - for (int j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; U value = (U)pSrc[index]; if (value != invalid) @@ -1337,12 +1337,12 @@ class GroupByBase for (int64_t i = binLow; i < binHigh; i++) { int32_t start = pFirst[i]; - int32_t last = start + pCount[i]; + int64_t last = start + pCount[i]; U currentSum = 0; // Priming of the summation - for (int j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { int32_t index = pGroup[j]; U value = (U)pSrc[index]; @@ -1353,7 +1353,7 @@ class GroupByBase pDest[index] = currentSum; } - for (int j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { int32_t index = pGroup[j]; @@ -1387,7 +1387,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = funcParam; + int64_t windowSize = funcParam; U invalid = GET_INVALID((U)0); double invalid_out = GET_INVALID(pDest[0]); @@ -1395,10 +1395,10 @@ class GroupByBase { // Mark all invalid if invalid bin int32_t start = pFirst[0]; - int32_t last = start + pCount[0]; - for (int j = start; j < last; j++) + int64_t last = start + pCount[0]; + for (int64_t j = start; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = invalid_out; } binLow++; @@ -1412,22 +1412,22 @@ class GroupByBase for (int64_t i = binLow; i < binHigh; i++) { int32_t start = pFirst[i]; - int32_t last = start + pCount[i]; + int64_t last = start + pCount[i]; double currentSum = 0; // Priming of the summation - for (int j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; currentSum += pSrc[index]; pDest[index] = currentSum / (j - start + 1); } - for (int j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; currentSum += pSrc[index]; @@ -1452,7 +1452,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = funcParam; + int64_t windowSize = funcParam; U invalid = GET_INVALID((U)0); double invalid_out = GET_INVALID(pDest[0]); @@ -1486,9 +1486,9 @@ class GroupByBase double count = 0; // Priming of the summation - for (int j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; U value = (U)pSrc[index]; if (value != invalid) { @@ -1498,9 +1498,9 @@ class GroupByBase pDest[index] = count > 0 ? currentSum / count : invalid_out; } - for (int j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; U value = (U)pSrc[index]; if (value != invalid) @@ -1533,9 +1533,9 @@ class GroupByBase double count = 0; // Priming of the summation - for (int j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; U value = (U)pSrc[index]; if (value == value) { @@ -1545,9 +1545,9 @@ class GroupByBase pDest[index] = count > 0 ? currentSum / count : invalid_out; } - for (int j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; U value = (U)pSrc[index]; if (value == value) @@ -1581,7 +1581,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = funcParam; + int64_t windowSize = funcParam; U invalid = GET_INVALID(pDest[0]); LOGGING("in rolling count %lld %lld sizeofdest %lld\n", binLow, binHigh, sizeof(U)); @@ -1611,18 +1611,18 @@ class GroupByBase if (windowSize < 0) { - for (int j = last - 1; j >= start; j--) + for (int64_t j = last - 1; j >= start; j--) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = currentSum; currentSum += 1; } } else { - for (int j = start; j < last; j++) + for (int64_t j = start; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = currentSum; currentSum += 1; } @@ -1642,7 +1642,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = funcParam; + int64_t windowSize = funcParam; U invalid = GET_INVALID(pDest[0]); // printf("binlow %lld, binhigh %lld, windowSize: %d\n", binLow, binHigh, @@ -1665,20 +1665,20 @@ class GroupByBase for (int64_t i = binLow; i < binHigh; i++) { int32_t start = pFirst[i]; - int32_t last = start + pCount[i]; + int64_t last = start + pCount[i]; if (windowSize >= 0) { // invalid for window - for (int32_t j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { int32_t index = pGroup[j]; pDest[index] = invalid; } - for (int32_t j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = (U)pSrc[pGroup[j - windowSize]]; } } @@ -1691,15 +1691,15 @@ class GroupByBase // printf("bin[%lld] start:%d last:%d windowSize:%d\n", i, start, // last, windowSize); - for (int32_t j = last; j > start && j > (last - windowSize); j--) + for (int64_t j = last; j > start && j > (last - windowSize); j--) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = invalid; } - for (int32_t j = last - windowSize; j > start; j--) + for (int64_t j = last - windowSize; j > start; j--) { - int32_t index = pGroup[j]; + int64_t index = pGroup[j]; pDest[index] = (U)pSrc[pGroup[j + windowSize]]; } // put it back to what it was @@ -1719,7 +1719,7 @@ class GroupByBase // only allow int32_t since comes from group and not ikey int32_t * pGroup = (int32_t *)pGroupT; - int32_t windowSize = funcParam; + int64_t windowSize = funcParam; U invalid = GET_INVALID(pDest[0]); if (binLow == 0) @@ -1773,13 +1773,13 @@ class GroupByBase // invalid for window U previous = 0; - for (int j = start; j < last && j < (start + windowSize); j++) + for (int64_t j = start; j < last && j < (start + windowSize); j++) { int32_t index = pGroup[j]; pDest[index] = invalid; } - for (int j = start + windowSize; j < last; j++) + for (int64_t j = start + windowSize; j < last; j++) { int32_t index = pGroup[j]; U temp = (U)pSrc[index]; @@ -1794,13 +1794,13 @@ class GroupByBase last--; start--; - for (int j = last; j > start && j > (last - windowSize); j--) + for (int64_t j = last; j > start && j > (last - windowSize); j--) { int32_t index = pGroup[j]; pDest[index] = invalid; } - for (int j = last - windowSize; j > start; j--) + for (int64_t j = last - windowSize; j > start; j--) { int32_t index = pGroup[j]; U temp = (U)pSrc[index]; @@ -1914,7 +1914,7 @@ class GroupByBase for (int64_t i = binLow; i < binHigh; i++) { int32_t index = pFirst[i]; - int32_t nCount = pCount[i]; + int64_t nCount = pCount[i]; if (nCount == 0) { @@ -1999,7 +1999,7 @@ class GroupByBase for (int64_t i = binLow; i < binHigh; i++) { int32_t index = pFirst[i]; - int32_t nCount = pCount[i]; + int64_t nCount = pCount[i]; if (nCount == 0) { @@ -4014,7 +4014,7 @@ PyObject * GroupByAll32(PyObject * self, PyObject * args) bool hasCounts = false; int overflow = 0; - int64_t funcNum = PyLong_AsLongLongAndOverflow(PyList_GET_ITEM(listFuncNum, i), &overflow); + int32_t funcNum = PyLong_AsLongAndOverflow(PyList_GET_ITEM(listFuncNum, i), &overflow); pstGroupBy32->returnObjects[i].funcNum = funcNum; int64_t binLow = PyLong_AsLongLongAndOverflow(PyList_GET_ITEM(listBinLow, i), &overflow); @@ -4076,7 +4076,7 @@ PyObject * GroupByAll32(PyObject * self, PyObject * args) returnTuple = PyTuple_New(tupleSize); PyObject * returnCount = NULL; - // Fill in results + // Fill in results for (int i = 0; i < tupleSize; i++) { PyObject * item = pstGroupBy32->returnObjects[i].returnObject; @@ -4109,7 +4109,7 @@ PyObject * GroupByAll32(PyObject * self, PyObject * args) //--------------------------------------------------------------- // Arg1 = LIST of numpy arrays which has the values to accumulate (often all the // columns in a dataset) Arg2 =iKey = numpy array which has the index -// to the unique keys (ikey from MultiKeyGroupBy32) Arg3 =iGroup: +// to the unique keys (ikey from MultiKeyGroupBy32) Arg3 =iGroup: // array size is same as multikey, unique keys are grouped together Arg4 // =iFirst: array size is number of unique keys, indexes into iGroup // Arg5 =nCount: array size is number of unique keys for the group, is @@ -4254,7 +4254,7 @@ PyObject * GroupByAllPack32(PyObject * self, PyObject * args) for (int i = 0; i < tupleSize; i++) { int overflow = 0; - int64_t funcNum = PyLong_AsLongLongAndOverflow(PyList_GET_ITEM(listFuncNum, i), &overflow); + int32_t funcNum = PyLong_AsLongAndOverflow(PyList_GET_ITEM(listFuncNum, i), &overflow); pstGroupBy32->returnObjects[i].funcNum = funcNum; int64_t binLow = PyLong_AsLongLongAndOverflow(PyList_GET_ITEM(listBinLow, i), &overflow); diff --git a/src/HashFunctions.cpp b/src/HashFunctions.cpp index 621176bc..3dd7833e 100644 --- a/src/HashFunctions.cpp +++ b/src/HashFunctions.cpp @@ -300,9 +300,9 @@ PyObject * IsMemberCategorical(PyObject * self, PyObject * args) // // final result 6 0 6 2 0 4 6 4 0 template -[[nodiscard]] char const * -FindFirstOccurence(T const * pArray2, int32_t const * pUnique1, int32_t * pReIndex, int32_t * pReverseMap, int64_t array2Length, - int32_t unique1Length, int32_t unique2Length, int32_t baseOffset2T) +[[nodiscard]] char const * FindFirstOccurence(T const * pArray2, int32_t const * pUnique1, int32_t * pReIndex, + int32_t * pReverseMap, int64_t array2Length, int32_t unique1Length, + int32_t unique2Length, int32_t baseOffset2T) { T baseOffset2 = (T)baseOffset2T; @@ -402,7 +402,8 @@ FindFirstOccurence(T const * pArray2, int32_t const * pUnique1, int32_t * pReInd // Returns: pOutput and pBoolOutput // TODO: this routine needs to output INT8/16/32/64 instead of just INT32 template -void FinalMatch(T const * pArray1, int32_t * pOutput, int8_t * pBoolOutput, int32_t const * pReIndex, int64_t array1Length, int32_t baseoffset) +void FinalMatch(T const * pArray1, int32_t * pOutput, int8_t * pBoolOutput, int32_t const * pReIndex, int64_t array1Length, + int32_t baseoffset) { const int32_t invalid = *(int32_t *)GetDefaultForType(NPY_INT32); @@ -482,7 +483,7 @@ PyObject * IsMemberCategoricalFixup(PyObject * self, PyObject * args) PyErr_Format(PyExc_ValueError, "IsMemberCategoricalFixup third argument must be type int32_t not %s", NpyToString(uniqueType)); return NULL; - } + } LOGGING( "IsMemberCategoricalFixup uniqlength:%d baseoffset1:%d " @@ -501,24 +502,24 @@ PyObject * IsMemberCategoricalFixup(PyObject * self, PyObject * args) int32_t * pUnique = (int32_t *)PyArray_BYTES(isMemberUnique); void * pArray2 = PyArray_BYTES(inArr2); - char const * err{nullptr}; + char const * err{ nullptr }; switch (array2Type) { case NPY_INT8: - err = FindFirstOccurence((int8_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, arraySizeUnique, - unique2Length, baseoffset2); + err = FindFirstOccurence((int8_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, + arraySizeUnique, unique2Length, baseoffset2); break; case NPY_INT16: - err = FindFirstOccurence((int16_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, arraySizeUnique, - unique2Length, baseoffset2); + err = FindFirstOccurence((int16_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, + arraySizeUnique, unique2Length, baseoffset2); break; CASE_NPY_INT32: - err = FindFirstOccurence((int32_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, arraySizeUnique, - unique2Length, baseoffset2); + err = FindFirstOccurence((int32_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, + arraySizeUnique, unique2Length, baseoffset2); break; CASE_NPY_INT64: - err = FindFirstOccurence((int64_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, arraySizeUnique, - unique2Length, baseoffset2); + err = FindFirstOccurence((int64_t *)pArray2, pUnique, reIndexArray, reverseMapArray, arraySize2, + arraySizeUnique, unique2Length, baseoffset2); break; default: PyErr_Format(PyExc_ValueError, "IsMemberCategoricalFixup second argument is not INT8/16/32/64"); diff --git a/src/HashLinear.cpp b/src/HashLinear.cpp index e55ded21..48c49a9d 100644 --- a/src/HashLinear.cpp +++ b/src/HashLinear.cpp @@ -1308,7 +1308,7 @@ void CHashLinear::MakeHashLocation(int64_t arraySize, T * pHashList, int64 // printf("%llu |", pBitFields[i]); //} - if (sizeof(T) <= 2) + if constexpr (sizeof(T) <= 2) { for (U i = 0; i < arraySize; i++) { @@ -1321,7 +1321,7 @@ void CHashLinear::MakeHashLocation(int64_t arraySize, T * pHashList, int64 } else { - if (sizeof(T) == 4) + if constexpr (sizeof(T) == 4) { if (HashMode == HASH_MODE_PRIME) { @@ -1346,7 +1346,7 @@ void CHashLinear::MakeHashLocation(int64_t arraySize, T * pHashList, int64 } } } - else if (sizeof(T) == 8) + else if constexpr (sizeof(T) == 8) { if (HashMode == HASH_MODE_PRIME) { @@ -1409,7 +1409,7 @@ int64_t CHashLinear::IsMemberCategorical(int64_t arraySize, T * pHashList, } else { - if (sizeof(T) == 4) + if constexpr (sizeof(T) == 4) { if (HashMode == HASH_MODE_PRIME) { @@ -1430,7 +1430,7 @@ int64_t CHashLinear::IsMemberCategorical(int64_t arraySize, T * pHashList, } } } - else if (sizeof(T) == 8) + else if constexpr (sizeof(T) == 8) { if (HashMode == HASH_MODE_PRIME) { @@ -1540,7 +1540,7 @@ void IsMember(void * pHashLinearVoid, int64_t arraySize, void * pHashListT, int8 // printf("\n"); - if (sizeof(T) <= 2) + if constexpr (sizeof(T) <= 2) { LOGGING("Perfect hash\n"); // perfect hash @@ -1555,7 +1555,7 @@ void IsMember(void * pHashLinearVoid, int64_t arraySize, void * pHashListT, int8 { HASH_MODE HashMode = pHashLinear->HashMode; - if (sizeof(T) == 4) + if constexpr (sizeof(T) == 4) { if (HashMode == HASH_MODE_PRIME) { @@ -1582,7 +1582,7 @@ void IsMember(void * pHashLinearVoid, int64_t arraySize, void * pHashListT, int8 } } } - else if (sizeof(T) == 8) + else if constexpr (sizeof(T) == 8) { if (HashMode == HASH_MODE_PRIME) { @@ -2221,9 +2221,8 @@ uint64_t CHashLinear::GroupByItemSize(int64_t totalRows, int64_t totalItem // make local reference on stack uint64_t * pBitFieldsX = pBitFields; - switch (sizeof(T)) + if constexpr (sizeof(T) == 1) { - case 1: // TODO: Specially handle bools here -- they're 1-byte but logically have // only two buckets (zero/nonzero). // if (coreType == NPY_BOOL) @@ -2254,8 +2253,9 @@ uint64_t CHashLinear::GroupByItemSize(int64_t totalRows, int64_t totalItem pIndexArray[i] = 0; } } - break; - case 2: + } + if constexpr (sizeof(T) == 2) + { if (pBoolFilter == NULL) { for (U i = 0; i < totalRows; i++) @@ -2279,8 +2279,9 @@ uint64_t CHashLinear::GroupByItemSize(int64_t totalRows, int64_t totalItem pIndexArray[i] = 0; } } - break; - case 4: + } + if constexpr (sizeof(T) == 4) + { if (pBoolFilter == NULL) { for (U i = 0; i < totalRows; i++) @@ -2306,8 +2307,9 @@ uint64_t CHashLinear::GroupByItemSize(int64_t totalRows, int64_t totalItem pIndexArray[i] = 0; } } - break; - case 8: + } + if constexpr (sizeof(T) == 8) + { if (pBoolFilter == NULL) { for (U i = 0; i < totalRows; i++) @@ -2333,8 +2335,9 @@ uint64_t CHashLinear::GroupByItemSize(int64_t totalRows, int64_t totalItem pIndexArray[i] = 0; } } - break; - case 16: + } + if constexpr (sizeof(T) == 16) + { // TO BE WORKED ON... if (pBoolFilter == NULL) { @@ -2361,7 +2364,6 @@ uint64_t CHashLinear::GroupByItemSize(int64_t totalRows, int64_t totalItem pIndexArray[i] = 0; } } - break; } // printf("GroupByItem end! %d\n", numUnique); diff --git a/src/MathThreads.cpp b/src/MathThreads.cpp index acdb671c..b24c44e7 100644 --- a/src/MathThreads.cpp +++ b/src/MathThreads.cpp @@ -82,7 +82,7 @@ void * WorkerThreadFunction(void * lpParam) { stWorkerRing * pWorkerRing = (stWorkerRing *)lpParam; - uint32_t core = (InterlockedIncrement64(&pWorkerRing->WorkThread)); + uint32_t core = static_cast((InterlockedIncrement64(&pWorkerRing->WorkThread))); core = core - 1; // if (core > 3) core += 16; diff --git a/src/MathWorker.h b/src/MathWorker.h index 97384039..d8a55f28 100644 --- a/src/MathWorker.h +++ b/src/MathWorker.h @@ -562,8 +562,8 @@ class CMathWorker bool didSomeWork = false; OLD_CALLBACK * OldCallback = &pstWorkerItem->OldCallback; - int64_t strideSizeIn = OldCallback->FunctionList->InputItemSize; - int64_t strideSizeOut = OldCallback->FunctionList->OutputItemSize; + int32_t strideSizeIn = OldCallback->FunctionList->InputItemSize; + int32_t strideSizeOut = OldCallback->FunctionList->OutputItemSize; char * pDataInX = (char *)OldCallback->pDataInBase1; char * pDataInX2 = (char *)OldCallback->pDataInBase2; diff --git a/src/Merge.cpp b/src/Merge.cpp index c3165e9e..70123ed1 100644 --- a/src/Merge.cpp +++ b/src/Merge.cpp @@ -255,8 +255,8 @@ PyObject * BooleanIndexInternal(PyArrayObject * aValues, PyArrayObject * aIndex) int ndimValue = 0; int ndimBoolean = 0; - int64_t strideValue = 0; - int64_t strideBoolean = 0; + int32_t strideValue = 0; + int32_t strideBoolean = 0; int result1 = GetStridesAndContig(aValues, ndimValue, strideValue); int result2 = GetStridesAndContig(aIndex, ndimBoolean, strideBoolean); @@ -741,7 +741,7 @@ PyObject * BooleanSum(PyObject * self, PyObject * args) } int ndimBoolean; - int64_t strideBoolean; + int32_t strideBoolean; int result1 = GetStridesAndContig(aIndex, ndimBoolean, strideBoolean); @@ -1329,8 +1329,8 @@ PyObject * MBGet(PyObject * self, PyObject * args) int ndimValue; int ndimIndex; - int64_t strideValue = 0; - int64_t strideIndex = 0; + int32_t strideValue = 0; + int32_t strideIndex = 0; int result1 = GetStridesAndContig(aValues, ndimValue, strideValue); int result2 = GetStridesAndContig(aIndex, ndimIndex, strideIndex); @@ -1505,7 +1505,7 @@ PyObject * BooleanToFancy(PyObject * self, PyObject * args, PyObject * kwargs) } int ndimBoolean; - int64_t strideBoolean; + int32_t strideBoolean; int result1 = GetStridesAndContig(aIndex, ndimBoolean, strideBoolean); @@ -1637,13 +1637,18 @@ PyObject * BooleanToFancy(PyObject * self, PyObject * args, PyObject * kwargs) for (int64_t i = start; i < (start + length); i++) { + if (i > INT_MAX) + { + PyErr_Format(PyExc_ValueError, "Cannot represent lengths > 31 bits in type [%d]", callbackArg->dtype); + return false; + } if (pBooleanMask[i]) { - *pOut++ = i; + *pOut++ = static_cast(i); } else { - *pOutFalse++ = i; + *pOutFalse++ = static_cast(i); } } } @@ -1657,9 +1662,14 @@ PyObject * BooleanToFancy(PyObject * self, PyObject * args, PyObject * kwargs) for (int64_t i = start; i < (start + length); i++) { + if (i > INT_MAX) + { + PyErr_Format(PyExc_ValueError, "Cannot represent lengths > 31 bits in type [%d]", callbackArg->dtype); + return false; + } if (pBooleanMask[i]) { - *pOut++ = i; + *pOut++ = static_cast(i); } } } @@ -1670,9 +1680,14 @@ PyObject * BooleanToFancy(PyObject * self, PyObject * args, PyObject * kwargs) for (int64_t i = start; i < (start + length); i++) { + if (i > INT_MAX) + { + PyErr_Format(PyExc_ValueError, "Cannot represent lengths > 31 bits in type [%d]", callbackArg->dtype); + return false; + } if (pBooleanMask[i]) { - *pOut++ = i; + *pOut++ = static_cast(i); } } } diff --git a/src/MultiKey.cpp b/src/MultiKey.cpp index a49aafce..31c59e4f 100644 --- a/src/MultiKey.cpp +++ b/src/MultiKey.cpp @@ -609,14 +609,14 @@ int32_t * GroupByPackFixup32(int64_t numUnique, int64_t totalRows, void * pIndex } // Go backwards - for (int32_t i = totalRows - 1; i >= 0; i--) + for (int64_t i = totalRows - 1; i >= 0; i--) { K group = pIndexArray[i]; pNextArray[i] = pGroupArray[group]; // printf("%d - group %d next %d\n", i, (int)group, pNextArray[i]); - pGroupArray[group] = i; + pGroupArray[group] = static_cast(i); } return pNextArray; @@ -657,7 +657,12 @@ bool GroupByPackFinal32(int64_t numUnique, int64_t totalRows, void * pIndexArray int32_t * pFirstArray = (int32_t *)PyArray_BYTES(firstArray); int32_t * pCountArray = (int32_t *)PyArray_BYTES(countArray); - int32_t i = totalRows; + if (totalRows > INT_MAX) + { + return false; + } + + int32_t i = static_cast(totalRows); // TODO -- how to handle empty? pSortArray[0] = GB_INVALID_INDEX; @@ -1513,7 +1518,7 @@ PyObject * MultiKeyHash(PyObject * self, PyObject * args) assert(mkp.totalRows < 2100000000); - int32_t i = mkp.totalRows; + int32_t i = static_cast(mkp.totalRows); while (i > 0) { @@ -2148,7 +2153,7 @@ PyObject * GroupFromBinCount(PyObject * self, PyObject * args) LOGGING("multithread makeigroup %lld cores %lld unique %lld rows\n", cores, totalUnique, totalRows); g_cMathWorker->DoMultiThreadedWork((int)cores, lambdaBinCountCallback, &stMaster); - WORKSPACE_FREE(pstBinCount); + WORKSPACE_FREE(pstBinCount); } else { diff --git a/src/Recycler.cpp b/src/Recycler.cpp index 376dbfb9..9b9b1de4 100644 --- a/src/Recycler.cpp +++ b/src/Recycler.cpp @@ -344,7 +344,7 @@ static inline void RemoveFromList(stRecycleList * pItems, int32_t slot) // On decrement it will decref array, set the slot to null and inc Tail // On increment it will incref array, time stamp, and inc Head // On Entry: the recycledArray must be valid -static void RefCountNumpyArray(stRecycleList * pItems, int32_t lzcount, int32_t type, int32_t slot, bool bIncrement) +static void RefCountNumpyArray(stRecycleList * pItems, int64_t lzcount, int32_t type, int32_t slot, bool bIncrement) { if (pItems->Item[slot].recycledArray == NULL) { @@ -359,7 +359,7 @@ static void RefCountNumpyArray(stRecycleList * pItems, int32_t lzcount, int32_t // timestamp pItems->Item[slot].tsc = __rdtsc(); - LOGRECYCLE("Adding item to slot %d,%d,%d -- refcnt now is %llu\n", lzcount, type, slot, + LOGRECYCLE("Adding item to slot %lld,%d,%d -- refcnt now is %llu\n", lzcount, type, slot, pItems->Item[slot].recycledArray->ob_base.ob_refcnt); // Only location head is incremented @@ -374,7 +374,8 @@ static void RefCountNumpyArray(stRecycleList * pItems, int32_t lzcount, int32_t RemoveFromList(pItems, slot); // If this is the last decref, python may free memory - LOGRECYCLE("Removing item in slot %d,%d,%d -- refcnt before is %llu\n", lzcount, type, slot, toDelete->ob_base.ob_refcnt); + LOGRECYCLE("Removing item in slot %lld,%d,%d -- refcnt before is %llu\n", lzcount, type, slot, + toDelete->ob_base.ob_refcnt); // NOTE: If this is the last decref, it might "free" the memory which might // take time @@ -664,7 +665,7 @@ static bool DeleteNumpyArray(PyArrayObject * inArr) ndim == 1 && strides != NULL && itemSize == strides[0]) { // Based on size and type, lookup - int32_t log2 = lzcnt_64(totalSize); + int64_t log2 = lzcnt_64(totalSize); stRecycleList * pItems = &g_stRecycleList[log2][type]; @@ -743,7 +744,7 @@ static bool DeleteNumpyArray(PyArrayObject * inArr) if (deltaHead < 0 || deltaHead > RECYCLE_MAXIMUM_SEARCH) { LogError( - "!!! inner critical error with recycler items %d,%d,%d with " + "!!! inner critical error with recycler items %lld,%d,%d with " "deltahead %d totalsize%lld\n", log2, type, slot, deltaHead, pItems->Item[slot].totalSize); } diff --git a/src/Reduce.cpp b/src/Reduce.cpp index 23dd107d..198e75b8 100644 --- a/src/Reduce.cpp +++ b/src/Reduce.cpp @@ -2949,7 +2949,7 @@ static PyObject * ReduceInternal(PyArrayObject * inArr1, REDUCE_FUNCTIONS func, fl.FunctionName = "Reduce"; fl.NumpyOutputType = 0; fl.NumpyType = numpyInType; - fl.InputItemSize = PyArray_ITEMSIZE(inArr1); + fl.InputItemSize = static_cast(PyArray_ITEMSIZE(inArr1)); fl.Input1Strides = len > 1 ? PyArray_STRIDE(inArr1, 0) : 0; fl.OutputItemSize = 0; fl.TypeOfFunctionCall = TYPE_OF_FUNCTION_CALL::ANY_SCATTER_GATHER; diff --git a/src/RipTide.cpp b/src/RipTide.cpp index 1547860b..29f8187c 100644 --- a/src/RipTide.cpp +++ b/src/RipTide.cpp @@ -920,6 +920,7 @@ int32_t GetArrDType(PyArrayObject * inArr) return dtype; } +// TODO: Refactor this to be variant based. //----------------------------------------------------------------------------------- // Convert python object to // pInput: pointer to array value that needs to be converted @@ -1015,7 +1016,7 @@ bool ConvertSingleItemArray(void * pInput, int16_t numpyInType, _m256all * pDest break; CASE_NPY_UINT32: CASE_NPY_INT32: - pDest->i = _mm256_set1_epi32(value); + pDest->i = _mm256_set1_epi32(static_cast(value)); break; CASE_NPY_UINT64: @@ -1092,7 +1093,7 @@ bool ConvertScalarObject(PyObject * inObject1, _m256all * pDest, int16_t numpyOu break; CASE_NPY_UINT32: CASE_NPY_INT32: - pDest->i = _mm256_set1_epi32(value); + pDest->i = _mm256_set1_epi32(static_cast(value)); break; CASE_NPY_UINT64: @@ -1177,10 +1178,10 @@ bool ConvertScalarObject(PyObject * inObject1, _m256all * pDest, int16_t numpyOu pDest->i = _mm256_set1_epi16((uint16_t)value2); break; CASE_NPY_INT32: - pDest->i = _mm256_set1_epi32(value); + pDest->i = _mm256_set1_epi32(static_cast(value)); break; CASE_NPY_UINT32: - pDest->i = _mm256_set1_epi32(value2); + pDest->i = _mm256_set1_epi32(static_cast(value2)); break; CASE_NPY_INT64: @@ -1223,10 +1224,10 @@ bool ConvertScalarObject(PyObject * inObject1, _m256all * pDest, int16_t numpyOu pDest->i = _mm256_set1_epi16((uint16_t)value); break; CASE_NPY_UINT32: - pDest->i = _mm256_set1_epi32(value); + pDest->i = _mm256_set1_epi32(static_cast(value)); break; CASE_NPY_INT32: - pDest->i = _mm256_set1_epi32(value); + pDest->i = _mm256_set1_epi32(static_cast(value)); break; CASE_NPY_UINT64: @@ -2254,19 +2255,19 @@ bool GetUpcastType(int numpyInType1, int numpyInType2, int & convertType1, int & // return value 0: one loop can process all data, false = multiple loops // NOTE: if return value is 0 and itemsze == stride, then vector math possible // -int GetStridesAndContig(PyArrayObject const * inArray, int & ndim, int64_t & stride) +int GetStridesAndContig(PyArrayObject const * inArray, int & ndim, int32_t & stride) { - stride = PyArray_ITEMSIZE(inArray); + stride = static_cast(PyArray_ITEMSIZE(inArray)); int direction = 0; ndim = PyArray_NDIM(inArray); if (ndim > 0) { - stride = PyArray_STRIDE(inArray, 0); + stride = static_cast(PyArray_STRIDE(inArray, 0)); if (ndim > 1) { // at least two strides int ndims = PyArray_NDIM(inArray); - int64_t lastStride = PyArray_STRIDE(inArray, ndims - 1); + int32_t lastStride = static_cast(PyArray_STRIDE(inArray, ndims - 1)); if (lastStride == stride) { // contiguous with one of the dimensions having length 1 @@ -2276,10 +2277,10 @@ int GetStridesAndContig(PyArrayObject const * inArray, int & ndim, int64_t & str // Row Major - 'C' Style // work backwards int currentdim = ndims - 1; - int64_t curStrideLen = lastStride; + int32_t curStrideLen = lastStride; while (currentdim != 0) { - curStrideLen *= PyArray_DIM(inArray, currentdim); + curStrideLen *= static_cast(PyArray_DIM(inArray, currentdim)); LOGGING("'C' %lld vs %lld dim: %lld stride: %lld \n", curStrideLen, PyArray_STRIDE(inArray, currentdim - 1), PyArray_DIM(inArray, currentdim - 1), lastStride); if (PyArray_STRIDE(inArray, currentdim - 1) != curStrideLen) @@ -2293,10 +2294,10 @@ int GetStridesAndContig(PyArrayObject const * inArray, int & ndim, int64_t & str { // Col Major - 'F' Style int currentdim = 0; - int64_t curStrideLen = stride; + int32_t curStrideLen = stride; while (currentdim != (ndims - 1)) { - curStrideLen *= PyArray_DIM(inArray, currentdim); + curStrideLen *= static_cast(PyArray_DIM(inArray, currentdim)); LOGGING("'F' %lld vs %lld dim: %lld stride: %lld \n", curStrideLen, PyArray_STRIDE(inArray, currentdim + 1), PyArray_DIM(inArray, currentdim + 1), stride); if (PyArray_STRIDE(inArray, currentdim + 1) != curStrideLen) diff --git a/src/RipTide.h b/src/RipTide.h index 7dbebb7d..c0c7c984 100644 --- a/src/RipTide.h +++ b/src/RipTide.h @@ -99,7 +99,7 @@ typedef struct } PyBoolScalarObject; extern bool GetUpcastType(int numpyInType1, int numpyInType2, int & convertType1, int & convertType2, int64_t funcNumber); -extern int GetStridesAndContig(PyArrayObject const * inArray, int & ndim, int64_t & stride); +extern int GetStridesAndContig(PyArrayObject const * inArray, int & ndim, int32_t & stride); /** * @brief Calculate the number of elements in an array with the given diff --git a/src/SDSFile.cpp b/src/SDSFile.cpp index 2225be5c..02e8f300 100644 --- a/src/SDSFile.cpp +++ b/src/SDSFile.cpp @@ -505,11 +505,17 @@ int64_t SDSFileReadChunk(SDS_EVENT_HANDLE eventHandle, SDS_FILE_HANDLE Handle, v // LastError); OVERLAPPED OverlappedIO; + if (BufferPos > INT_MAX) + { + LogError("!! Overlapped only supports 32-bit file offsets !! [%lld]", BufferPos); + return 0; + } + OverlappedIO.hEvent = eventHandle; OverlappedIO.InternalHigh = 0; OverlappedIO.Internal = 0; OverlappedIO.OffsetHigh = (BufferPos >> 32); - OverlappedIO.Offset = BufferPos; + OverlappedIO.Offset = static_cast(BufferPos); bool bReadDone; @@ -591,11 +597,17 @@ int64_t SDSFileWriteChunk(SDS_EVENT_HANDLE eventHandle, SDS_FILE_HANDLE Handle, // LastError); OVERLAPPED OverlappedIO; + if (BufferPos > INT_MAX) + { + LogError("!! Overlapped only supports 32-bit file offsets !! [%lld]", BufferPos); + return 0; + } + OverlappedIO.hEvent = eventHandle; OverlappedIO.InternalHigh = 0; OverlappedIO.Internal = 0; OverlappedIO.OffsetHigh = (BufferPos >> 32); - OverlappedIO.Offset = BufferPos; + OverlappedIO.Offset = static_cast(BufferPos); OVERLAPPED * pos = &OverlappedIO; DWORD n; @@ -2725,8 +2737,13 @@ bool CompressFileArray(void * pstCompressArraysV, int32_t core, int64_t t) pArrayBlock->CompressionType = COMPRESSION_TYPE_ZSTD; // New version 4.3 - pArrayBlock->ArrayBandCount = bandCount; - pArrayBlock->ArrayBandSize = bandSize; + if (bandCount > INT_MAX || bandSize > INT_MAX) + { + LOGGING("bandCount [%lld] and bandSize [%lld] are only 32-bit values in the file array block", bandCount, bandSize); + return false; + } + pArrayBlock->ArrayBandCount = static_cast(bandCount); + pArrayBlock->ArrayBandSize = static_cast(bandSize); // record array dimensions int32_t ndim = pArrayInfo->NDim; diff --git a/src/SDSFilePython.cpp b/src/SDSFilePython.cpp index a9f91926..921f33b7 100644 --- a/src/SDSFilePython.cpp +++ b/src/SDSFilePython.cpp @@ -1385,7 +1385,7 @@ PyObject * CompressFile(PyObject * self, PyObject * args, PyObject * kwargs) for (int64_t i = 0; i < arrayCount; i++) { pDest->ArrayLength = pSrc->ArrayLength; - pDest->ItemSize = pSrc->ItemSize; + pDest->ItemSize = static_cast(pSrc->ItemSize); pDest->pArrayObject = pSrc->pObject; // We do not need this.. pDest->NumBytes = pSrc->NumBytes; pDest->NumpyDType = pSrc->NumpyDType; diff --git a/src/Sort.cpp b/src/Sort.cpp index ff8def8e..50c1a7f7 100644 --- a/src/Sort.cpp +++ b/src/Sort.cpp @@ -2988,7 +2988,7 @@ PyObject * SortInPlaceIndirect(PyObject * self, PyObject * args) int32_t * pDataIn = (int32_t *)PyArray_BYTES(inArr1); int32_t * pSort = (int32_t *)PyArray_BYTES(inSort); - int32_t * inverseSort = (int32_t *)WORKSPACE_ALLOC(sortSize * sizeof(int32_t)); + int32_t * inverseSort = (int32_t *)WORKSPACE_ALLOC(sortSize * sizeof(int32_t)); for (int i = 0; i < sortSize; i++) { inverseSort[pSort[i]] = i; @@ -3014,7 +3014,7 @@ PyObject * SortInPlaceIndirect(PyObject * self, PyObject * args) for (int i = 0; i < arraySize1; i++) { - pDataIn[i] = inverseSort[pDataIn[i]]; + pDataIn[i] = static_cast(inverseSort[pDataIn[i]]); } WORKSPACE_FREE(inverseSort); } @@ -4126,7 +4126,7 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n // printf("[%lld] start: %lld length: %lld ubefore: %lld\n", t, // partStart, partLength, uniquesBefore); - if (callbackArg->sizeofUINDEX == 4) + if constexpr (sizeof(UINDEX) == 4) { int32_t * pKey = (int32_t *)callbackArg->pKeyOut; @@ -4142,7 +4142,7 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n int32_t * pCount = (int32_t *)callbackArg->pCountOut; int32_t * pCountReduced = (int32_t *)callbackArg->pCountReduced; - int32_t ubefore = uniquesBefore; + int32_t ubefore = static_cast(callbackArg->pUniqueCounts[t - 1]); if (t != 0) { @@ -4151,8 +4151,8 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n for (int64_t i = 0; i < partLength; i++) { - pKey[i] += (ubefore + 1); // start at 1 (to reserve zero bin), becomes ikey - pIndex[i] += partStart; + pKey[i] += static_cast(++ubefore); // start at 1 (to reserve zero bin), becomes ikey + pIndex[i] += static_cast(partStart); } } else @@ -4161,7 +4161,7 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n for (int64_t i = 0; i < partLength; i++) { - pKey[i] += (partStart + 1); // start at 1, becomes ikey + pKey[i] += static_cast(++partStart); // start at 1, becomes ikey } } @@ -4178,7 +4178,7 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n for (int64_t i = 0; i < uniqueLength; i++) { - pFirstReduced[i] = pFirst[i] + partStart; + pFirstReduced[i] = static_cast(pFirst[i] + partStart); // printf("setting %lld ", (int64_t)pCount[i]); pCountReduced[i] = pCount[i]; } diff --git a/src/TileRepeat.cpp b/src/TileRepeat.cpp index 7434f930..ef636e26 100644 --- a/src/TileRepeat.cpp +++ b/src/TileRepeat.cpp @@ -31,7 +31,7 @@ const union //----------------------------------- // void ConvertRecArray(char * pStartOffset, int64_t startRow, int64_t totalRows, stOffsets * pstOffset, int64_t * pOffsets, - int64_t numArrays, int64_t itemSize) + int64_t numArrays, int32_t itemSize) { // Try to keep everything in L1Cache const int64_t L1CACHE = 32768; @@ -172,7 +172,7 @@ PyObject * RecordArrayToColMajor(PyObject * self, PyObject * args) return NULL; } - int64_t itemSize = PyArray_ITEMSIZE(inArr); + int32_t itemSize = static_cast(PyArray_ITEMSIZE(inArr)); if (itemSize != PyArray_STRIDE(inArr, 0)) { @@ -219,7 +219,7 @@ PyObject * RecordArrayToColMajor(PyObject * self, PyObject * args) stOffsets * pstOffset; int64_t * pOffsets; int64_t numArrays; - int64_t itemSize; + int32_t itemSize; int64_t lastRow; } stConvert; diff --git a/src/UnaryOps.cpp b/src/UnaryOps.cpp index 1026a1aa..6239deaf 100644 --- a/src/UnaryOps.cpp +++ b/src/UnaryOps.cpp @@ -659,7 +659,7 @@ static const inline __m256d SQRT_OP_256(__m256d x) // T = data type as input // MathOp operation to perform template -static inline void UnaryOpFast(void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static inline void UnaryOpFast(void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { T * pIn = (T *)pDataIn; T * pOut = (T *)pDataOut; @@ -714,7 +714,7 @@ static inline void UnaryOpFast(void * pDataIn, void * pDataOut, int64_t len, int // T = data type as input // MathOp operation to perform template -static inline void UnaryOpFastStrided(void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static inline void UnaryOpFastStrided(void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { T * pIn = (T *)pDataIn; T * pOut = (T *)pDataOut; @@ -733,7 +733,7 @@ static inline void UnaryOpFastStrided(void * pDataIn, void * pDataOut, int64_t l U256 * pIn1_256 = (U256 *)pIn; U256 * pOut_256 = (U256 *)pOut; - ssize_t babyStride = strideIn; + int32_t babyStride = strideIn; // add 8 strides everytime we process 8 __m256i mindex = _mm256_mullo_epi32(_mm256_set1_epi32(babyStride), __vec8_strides.m); @@ -768,8 +768,8 @@ static inline void UnaryOpFastStrided(void * pDataIn, void * pDataOut, int64_t l // T = data type as input // MathOp operation to perform template -static inline void UnaryNanFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, - int64_t strideOut) +static inline void UnaryNanFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, + int32_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -816,8 +816,8 @@ static inline void UnaryNanFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, vo // T = data type as input // MathOp operation to perform template -static inline void UnaryNanFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, - int64_t strideOut) +static inline void UnaryNanFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, + int32_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -859,8 +859,8 @@ static inline void UnaryNanFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, v // T = data type as input // MathOp operation to perform template -static inline void UnaryNotNanFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, - int64_t strideOut) +static inline void UnaryNotNanFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, + int32_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -904,8 +904,8 @@ static inline void UnaryNotNanFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, // T = data type as input // MathOp operation to perform template -static inline void UnaryNotNanFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, - int64_t strideOut) +static inline void UnaryNotNanFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, + int32_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -947,8 +947,8 @@ static inline void UnaryNotNanFastDouble(MathFunctionPtr MATH_OP, void * pDataIn // T = data type as input // MathOp operation to perform template -static inline void UnaryFiniteFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, - int64_t strideOut) +static inline void UnaryFiniteFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, + int32_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -995,8 +995,8 @@ static inline void UnaryFiniteFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, // T = data type as input // MathOp operation to perform template -static inline void UnaryFiniteFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, - int64_t strideOut) +static inline void UnaryFiniteFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, + int32_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -1041,8 +1041,8 @@ static inline void UnaryFiniteFastDouble(MathFunctionPtr MATH_OP, void * pDataIn // T = data type as input // MathOp operation to perform template -static inline void UnaryNotFiniteFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, - int64_t strideOut) +static inline void UnaryNotFiniteFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, + int32_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -1090,7 +1090,7 @@ static inline void UnaryNotFiniteFastFloat(MathFunctionPtr MATH_OP, void * pData // MathOp operation to perform template static inline void UnaryNotFiniteFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, - int64_t strideIn, int64_t strideOut) + int32_t strideIn, int32_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -1138,7 +1138,7 @@ static inline void UnaryNotFiniteFastDouble(MathFunctionPtr MATH_OP, void * pDat // Output same as input type // MathOp operation to perform template -static void UnaryOpSlow(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { T * pIn = (T *)pDataIn; T * pOut = (T *)pDataOut; @@ -1158,8 +1158,8 @@ static void UnaryOpSlow(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut // Output always returns a double // MathOp operation to perform template -static void UnaryOpSlowDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, - int64_t strideOut) +static void UnaryOpSlowDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, + int32_t strideOut) { T * pIn = (T *)pDataIn; double * pOut = (double *)pDataOut; @@ -1179,8 +1179,8 @@ static void UnaryOpSlowDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pD // Output always returns a double // MathOp operation to perform template -static void UnaryOpSlowBool(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, - int64_t strideOut) +static void UnaryOpSlowBool(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, + int32_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -1195,7 +1195,7 @@ static void UnaryOpSlowBool(MathFunctionPtr MATH_OP, void * pDataIn, void * pDat } } -static void UnaryOpSlow_FillTrue(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_FillTrue(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { if (strideOut == sizeof(bool)) { @@ -1213,7 +1213,7 @@ static void UnaryOpSlow_FillTrue(void * pDataIn1, void * pDataOut, int64_t len, } } -static void UnaryOpSlow_FillFalse(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_FillFalse(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { if (strideOut == sizeof(bool)) { @@ -1233,59 +1233,59 @@ static void UnaryOpSlow_FillFalse(void * pDataIn1, void * pDataOut, int64_t len, //------------------------------------------------------------------- template -static void UnaryOpSlow_ABS(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_ABS(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(ABS_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_FABS(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_FABS(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(FABS_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_SIGN(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_SIGN(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(SIGN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_FLOATSIGN(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_FLOATSIGN(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(FLOATSIGN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_NEG(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_NEG(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(NEG_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_ISNOTNAN(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_ISNOTNAN(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowBool(ISNOTNAN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISNAN(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_ISNAN(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowBool(ISNAN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISNANORZERO(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_ISNANORZERO(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowBool(ISNANORZERO_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISINVALID(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_ISINVALID(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { T * pIn = (T *)pDataIn1; T invalid = *(T *)GetInvalid(); @@ -1309,7 +1309,7 @@ static void UnaryOpSlow_ISINVALID(void * pDataIn1, void * pDataOut, int64_t len, } template -static void UnaryOpSlow_ISINVALIDORZERO(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_ISINVALIDORZERO(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { T * pIn = (T *)pDataIn1; T invalid = *(T *)GetInvalid(); @@ -1333,7 +1333,7 @@ static void UnaryOpSlow_ISINVALIDORZERO(void * pDataIn1, void * pDataOut, int64_ } template -static void UnaryOpSlow_ISNOTINVALID(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_ISNOTINVALID(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { T * pIn = (T *)pDataIn1; T invalid = *(T *)GetInvalid(); @@ -1357,173 +1357,173 @@ static void UnaryOpSlow_ISNOTINVALID(void * pDataIn1, void * pDataOut, int64_t l } template -static void UnaryOpSlow_ISFINITE(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_ISFINITE(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowBool(ISFINITE_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISNOTFINITE(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_ISNOTFINITE(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowBool(ISNOTFINITE_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISINF(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_ISINF(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowBool(ISINF_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISNOTINF(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_ISNOTINF(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowBool(ISNOTINF_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISNORMAL(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_ISNORMAL(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowBool(ISNORMAL_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISNOTNORMAL(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_ISNOTNORMAL(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowBool(ISNOTNORMAL_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_SIGNBIT(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_SIGNBIT(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowBool(SIGNBIT_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_NOT(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_NOT(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowBool(NOT_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_INVERT(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_INVERT(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(INVERT_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_INVERT_BOOL(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_INVERT_BOOL(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(INVERT_OP_BOOL, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_FLOOR(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_FLOOR(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(FLOOR_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_CEIL(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_CEIL(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(CEIL_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_TRUNC(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_TRUNC(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(TRUNC_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_ROUND(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_ROUND(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(ROUND_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_SQRT(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_SQRT(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(SQRT_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_CBRT(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_CBRT(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(CBRT_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_LOG(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_LOG(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(LOG_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_LOG2(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_LOG2(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(LOG2_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_LOG10(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_LOG10(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(LOG10_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_EXP(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_EXP(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(EXP_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_EXP2(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlow_EXP2(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlow(EXP2_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlowDouble_SQRT(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlowDouble_SQRT(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowDouble(SQRT_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlowDouble_CBRT(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlowDouble_CBRT(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowDouble(CBRT_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlowDouble_LOG(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlowDouble_LOG(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowDouble(LOG_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlowDouble_LOG2(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlowDouble_LOG2(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowDouble(LOG2_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlowDouble_LOG10(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlowDouble_LOG10(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowDouble(LOG10_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlowDouble_EXP(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlowDouble_EXP(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowDouble(EXP_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlowDouble_EXP2(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static void UnaryOpSlowDouble_EXP2(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryOpSlowDouble(EXP2_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } @@ -1536,45 +1536,45 @@ static void UnaryOpSlowDouble_EXP2(void * pDataIn1, void * pDataOut, int64_t len // This routines will call the proper templated ABS function based on the type // It provides the standard signature template -static inline void UnaryOpFast_NANF32(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static inline void UnaryOpFast_NANF32(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryNanFastFloat(ISNAN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static inline void UnaryOpFast_NANF64(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static inline void UnaryOpFast_NANF64(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryNanFastDouble(ISNAN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static inline void UnaryOpFast_NOTNANF32(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static inline void UnaryOpFast_NOTNANF32(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryNotNanFastFloat(ISNOTNAN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static inline void UnaryOpFast_NOTNANF64(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static inline void UnaryOpFast_NOTNANF64(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryNotNanFastDouble(ISNOTNAN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static inline void UnaryOpFast_FINITEF32(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static inline void UnaryOpFast_FINITEF32(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryFiniteFastFloat(ISFINITE_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static inline void UnaryOpFast_FINITEF64(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static inline void UnaryOpFast_FINITEF64(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryFiniteFastDouble(ISFINITE_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static inline void UnaryOpFast_NOTFINITEF32(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static inline void UnaryOpFast_NOTFINITEF32(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryNotFiniteFastFloat(ISNOTFINITE_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static inline void UnaryOpFast_NOTFINITEF64(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) +static inline void UnaryOpFast_NOTFINITEF64(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) { return UnaryNotFiniteFastDouble(ISNOTFINITE_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } @@ -2481,7 +2481,7 @@ PyObject * ProcessOneInput(PyArrayObject * inArray, PyArrayObject * outObject1, { LOGGING("In process one input %d %d %d\n", funcNumber, numpyInType1, numpyOutType); int ndim; - int64_t strideIn; + int32_t strideIn; int directionIn = GetStridesAndContig(inArray, ndim, strideIn); int64_t len = CALC_ARRAY_LENGTH(ndim, PyArray_DIMS(inArray)); @@ -2512,7 +2512,7 @@ PyObject * ProcessOneInput(PyArrayObject * inArray, PyArrayObject * outObject1, { // Threading not allowed for this work item, call it directly from // main thread - pUnaryFunc(pDataIn, pDataOut, len, strideIn, PyArray_ITEMSIZE(outputArray)); + pUnaryFunc(pDataIn, pDataOut, len, strideIn, static_cast(PyArray_ITEMSIZE(outputArray))); } else { @@ -2526,7 +2526,7 @@ PyObject * ProcessOneInput(PyArrayObject * inArray, PyArrayObject * outObject1, stCallback.pDataOut = (char *)pDataOut; stCallback.pDataIn = (char *)pDataIn; stCallback.itemSizeIn = strideIn; - stCallback.itemSizeOut = PyArray_ITEMSIZE(outputArray); + stCallback.itemSizeOut = static_cast(PyArray_ITEMSIZE(outputArray)); // This will notify the worker threads of a new work item // most functions are so fast, we do not need more than 3 worker @@ -2600,7 +2600,7 @@ PyObject * ProcessOneInput(PyArrayObject * inArray, PyArrayObject * outObject1, void * pDataOut = PyArray_BYTES(outputArray); int ndimOut; - int64_t strideOut; + int32_t strideOut; int directionOut = GetStridesAndContig(outputArray, ndimOut, strideOut); // F contig on 2 dim => dim0 * stride0 == stride1 diff --git a/src/UnaryOps.h b/src/UnaryOps.h index 022c2e21..48d9a0c0 100644 --- a/src/UnaryOps.h +++ b/src/UnaryOps.h @@ -22,8 +22,8 @@ struct UNARY_CALLBACK char * pDataIn; char * pDataOut; - int64_t itemSizeIn; - int64_t itemSizeOut; + int32_t itemSizeIn; + int32_t itemSizeOut; }; #endif diff --git a/src/one_input.cpp b/src/one_input.cpp index e31f7889..a6c2840e 100644 --- a/src/one_input.cpp +++ b/src/one_input.cpp @@ -18,7 +18,7 @@ PyObject * process_one_input(PyArrayObject const * in_array, PyArrayObject * out int32_t numpy_intype, int32_t numpy_outtype) { int32_t ndim{}; - int64_t stride{}; + int32_t stride{}; int32_t direction{ GetStridesAndContig(in_array, ndim, stride) }; npy_intp len{ CALC_ARRAY_LENGTH(ndim, PyArray_DIMS(const_cast(in_array))) }; @@ -83,7 +83,7 @@ PyObject * process_one_input(PyArrayObject const * in_array, PyArrayObject * out char * out_p{ PyArray_BYTES(const_cast(result_array)) }; int num_dims_out{}; - int64_t stride_out{}; + int32_t stride_out{}; int direction_out = GetStridesAndContig(result_array, num_dims_out, stride_out); if (direction_out == 0) @@ -222,16 +222,16 @@ namespace riptable_cpp case MATH_OPERATION::CBRT: retval.first = cbrt_op{}; break; -// BUG:: These are defined, but never called -// case MATH_OPERATION::TAN: -// retval.first = tan_op{}; -// break; -// case MATH_OPERATION::SIN: -// retval.first = sin_op{}; -// break; -// case MATH_OPERATION::COS: -// retval.first = cos_op{}; -// break; + // BUG:: These are defined, but never called + // case MATH_OPERATION::TAN: + // retval.first = tan_op{}; + // break; + // case MATH_OPERATION::SIN: + // retval.first = sin_op{}; + // break; + // case MATH_OPERATION::COS: + // retval.first = cos_op{}; + // break; case MATH_OPERATION::SIGNBIT: retval.first = signbit_op{}; break; diff --git a/src/one_input_impl.h b/src/one_input_impl.h index d78b4fbe..0702cc30 100644 --- a/src/one_input_impl.h +++ b/src/one_input_impl.h @@ -616,8 +616,9 @@ namespace riptable_cpp template void calculate_for_active_data_type(char const * in_p, char * out_p, ptrdiff_t & starting_element, - int64_t const in_array_stride, size_t contig_elems, single_operation_t const & requested_op, - type_variant const & in_type, std::index_sequence) + int64_t const in_array_stride, size_t contig_elems, + single_operation_t const & requested_op, type_variant const & in_type, + std::index_sequence) { (calculate_for_active_operation(in_p, out_p, starting_element, in_array_stride, contig_elems, requested_op, std::get_if(&in_type), diff --git a/src/operation_traits.h b/src/operation_traits.h index 30323aab..a13e232d 100644 --- a/src/operation_traits.h +++ b/src/operation_traits.h @@ -192,10 +192,10 @@ namespace riptable_cpp using simd_implementation = std::false_type; }; - using single_operation_t = std::variant; + using single_operation_t = std::variant; } // namespace riptable_cpp diff --git a/src/setup.py b/src/setup.py index dedec5eb..4a3233d4 100644 --- a/src/setup.py +++ b/src/setup.py @@ -75,8 +75,8 @@ #extra_compile_args = ['/MT /Ox /Ob2 /Oi /Ot'], # For MSVC windows compiler 2019 it has the new __CxxFrameHandler4 which is found in vcrntime140_1.dll which is not on all systems # We use /dsFH4- to disable this frame handler - extra_compile_args = ['/Ox','/Ob2','/Oi','/Ot','/d2FH4-','/W3','/FC','/Zc:__cplusplus','/std:c++17','/permissive-','/Zc:strictStrings-'], - #extra_compile_args = ['/Ox','/Ob2','/Oi','/Ot','/d2FH4-','/W3','/WX''/FC','/Zc:__cplusplus','/std:c++17','/permissive-','/Zc:strictStrings-'], + #extra_compile_args = ['/Ox','/Ob2','/Oi','/Ot','/d2FH4-','/W3','/FC','/Zc:__cplusplus','/std:c++17','/permissive-','/Zc:strictStrings-'], + extra_compile_args = ['/Ox','/Ob2','/Oi','/Ot','/d2FH4-','/W3','/WX','/FC','/Zc:__cplusplus','/std:c++17','/permissive-','/Zc:strictStrings-'], #extra_compile_args = ['/Od','/Z7','/d2FH4-','/W3','/WX','/FC','/Zc:__cplusplus','/std:c++17','/permissive-','/Zc:strictStrings-'], #extra_link_args = ['/debug'] ) From d2c5368ebca6bfe6991e8d53e28abc557fcc16c2 Mon Sep 17 00:00:00 2001 From: Staffan Tjernstrom <8728287+staffantj@users.noreply.github.com> Date: Thu, 18 Nov 2021 16:06:15 -0500 Subject: [PATCH 5/8] Modified SDSFile to be more 64-bit accurate. Updates the SDS file minor version --- src/SDSFile.cpp | 23 +++++++++-------------- src/SDSFile.h | 38 +++++++++++++++++++------------------- 2 files changed, 28 insertions(+), 33 deletions(-) diff --git a/src/SDSFile.cpp b/src/SDSFile.cpp index 02e8f300..2ed31c1e 100644 --- a/src/SDSFile.cpp +++ b/src/SDSFile.cpp @@ -515,7 +515,7 @@ int64_t SDSFileReadChunk(SDS_EVENT_HANDLE eventHandle, SDS_FILE_HANDLE Handle, v OverlappedIO.InternalHigh = 0; OverlappedIO.Internal = 0; OverlappedIO.OffsetHigh = (BufferPos >> 32); - OverlappedIO.Offset = static_cast(BufferPos); + OverlappedIO.Offset = static_cast(BufferPos & 0x0FFFFFFFFULL); bool bReadDone; @@ -607,7 +607,7 @@ int64_t SDSFileWriteChunk(SDS_EVENT_HANDLE eventHandle, SDS_FILE_HANDLE Handle, OverlappedIO.InternalHigh = 0; OverlappedIO.Internal = 0; OverlappedIO.OffsetHigh = (BufferPos >> 32); - OverlappedIO.Offset = static_cast(BufferPos); + OverlappedIO.Offset = static_cast(BufferPos & 0x0FFFFFFFFULL); OVERLAPPED * pos = &OverlappedIO; DWORD n; @@ -1587,7 +1587,7 @@ int64_t ReadAndDecompressBandWithFilter(SDS_ARRAY_BLOCK * pBlockInfo, // may con rowOffset, pBlockInfo->ArrayDataOffset, pBlockInfo->ArrayCompressedSize, pBlockInfo->ArrayUncompressedSize, stackIndex, bytesPerRow); - for (int32_t i = 0; i < pBlockInfo->ArrayBandCount; i++) + for (int64_t i = 0; i < pBlockInfo->ArrayBandCount; i++) { int64_t compressedSize = pBands[i] - previousSize; previousSize = pBands[i]; @@ -1731,7 +1731,7 @@ static size_t ReadAndDecompressArrayBlockWithFilter(SDS_ARRAY_BLOCK * pBlockInfo else { printf( - "out of mem no temp buffer bandcount:%d bandsize:%d uncomp size: " + "out of mem no temp buffer bandcount:%lld bandsize:%lld uncomp size: " "%lld\n", pBlockInfo->ArrayBandCount, pBlockInfo->ArrayBandSize, pBlockInfo->ArrayUncompressedSize); } @@ -1803,7 +1803,7 @@ static size_t ReadAndDecompressArrayBlock(SDS_ARRAY_BLOCK * pBlockInfo, // may c { int64_t previousSize = 0; - for (int32_t i = 0; i < pBlockInfo->ArrayBandCount; i++) + for (int64_t i = 0; i < pBlockInfo->ArrayBandCount; i++) { int64_t compressedSize = pBands[i] - previousSize; previousSize = pBands[i]; @@ -1824,7 +1824,7 @@ static size_t ReadAndDecompressArrayBlock(SDS_ARRAY_BLOCK * pBlockInfo, // may c if (result != uncompressedSize) { printf( - "[%d][%lld][%d] MTDecompression (uncompressed) band error " + "[%d][%lld][%lld] MTDecompression (uncompressed) band error " "size %lld vs %lld\n", core, arrayIndex, i, uncompressedSize, compressedSize); result = -1; @@ -1841,7 +1841,7 @@ static size_t ReadAndDecompressArrayBlock(SDS_ARRAY_BLOCK * pBlockInfo, // may c if (dcSize != uncompressedSize) { printf( - "[%d][%lld][%d] MTDecompression band error size %lld vs " + "[%d][%lld][%lld] MTDecompression band error size %lld vs " "%lld vs %lld\n", core, arrayIndex, i, dcSize, uncompressedSize, compressedSize); result = -1; @@ -2737,13 +2737,8 @@ bool CompressFileArray(void * pstCompressArraysV, int32_t core, int64_t t) pArrayBlock->CompressionType = COMPRESSION_TYPE_ZSTD; // New version 4.3 - if (bandCount > INT_MAX || bandSize > INT_MAX) - { - LOGGING("bandCount [%lld] and bandSize [%lld] are only 32-bit values in the file array block", bandCount, bandSize); - return false; - } - pArrayBlock->ArrayBandCount = static_cast(bandCount); - pArrayBlock->ArrayBandSize = static_cast(bandSize); + pArrayBlock->ArrayBandCount = bandCount; + pArrayBlock->ArrayBandSize = bandSize; // record array dimensions int32_t ndim = pArrayInfo->NDim; diff --git a/src/SDSFile.h b/src/SDSFile.h index c524364f..cc16f79b 100644 --- a/src/SDSFile.h +++ b/src/SDSFile.h @@ -45,8 +45,9 @@ // TJD added version low = 3 July 2019 for "bandsize" // TJD added version low = 4 August 2019 for "section" // TJD added version low = 5 March 2020 for timestamps and better section, +// EST added version low = 6 for 64-bit safe operations // boolean bitmasks -#define SDS_VERSION_LOW 5 +#define SDS_VERSION_LOW 6 #define SDS_VALUE_ERROR 1 @@ -265,7 +266,6 @@ enum SDS_TYPES // File format layout for one array block header struct SDS_ARRAY_BLOCK { - //----- offset 0 ----- int16_t HeaderTag; int16_t HeaderLength; @@ -274,12 +274,9 @@ struct SDS_ARRAY_BLOCK int8_t DType; int8_t NDim; - //----- offset 8 ----- // BUGBUG All these offsets are wrong, this struct has - // default member alignments. int32_t ItemSize; int32_t Flags; - //----- offset 16 ----- // no more than 5 dims int64_t Dimensions[SDS_MAX_DIMS]; int64_t Strides[SDS_MAX_DIMS]; @@ -289,10 +286,14 @@ struct SDS_ARRAY_BLOCK // / BandSize => # of bands int64_t ArrayCompressedSize; int64_t ArrayUncompressedSize; - int32_t ArrayBandCount; // 0=no banding - int32_t ArrayBandSize; // 0=no banding + int64_t ArrayBandCount; // 0=no banding + int64_t ArrayBandSize; // 0=no banding }; +static_assert(offsetof(SDS_ARRAY_BLOCK, HeaderTag) == 0); +static_assert(offsetof(SDS_ARRAY_BLOCK, ItemSize) == 8); +static_assert(offsetof(SDS_ARRAY_BLOCK, Dimensions) == 16); + //---------------------------------- // At offset 0 of the file is this header struct SDS_FILE_HEADER @@ -305,7 +306,6 @@ struct SDS_FILE_HEADER int16_t CompType; int16_t CompLevel; - //----- offset 16 ----- int64_t NameBlockSize; int64_t NameBlockOffset; int64_t NameBlockCount; @@ -314,28 +314,22 @@ struct SDS_FILE_HEADER int16_t StackType; // see SDS_STACK_TYPE int32_t AuthorId; // see SDS_AUTHOR_ID - //----- offset 48 ----- int64_t MetaBlockSize; int64_t MetaBlockOffset; - //----- offset 64 ----- int64_t TotalMetaCompressedSize; int64_t TotalMetaUncompressedSize; - //----- offset 80 ----- int64_t ArrayBlockSize; int64_t ArrayBlockOffset; - //----- offset 96 ----- int64_t ArraysWritten; int64_t ArrayFirstOffset; - //----- offset 112 ----- int64_t TotalArrayCompressedSize; // Includes the SDS_PADDING, next relative // offset to write to int64_t TotalArrayUncompressedSize; - //----- offset 128 ----- //------------- End of Version 4.1 --------- //------------- Version 4.3 starts here ---- int64_t BandBlockSize; // 00 if nothing @@ -343,7 +337,6 @@ struct SDS_FILE_HEADER int64_t BandBlockCount; // Number of names int64_t BandSize; // How many elements before creating another band - //----- offset 160 ----- //------------- End of Version 4.3 --------- //------------- Version 4.4 starts here ---- int64_t SectionBlockSize; // how many bytes valid data @@ -357,14 +350,11 @@ struct SDS_FILE_HEADER int64_t SectionBlockReservedSize; // total size of what we reserved for this // block (so we can append names) - //----- offset 192 ----- int64_t FileOffset; // this file offset within the file uint64_t TimeStampUTCNanos; // time stamp when file was last written (0s // indicate no time stamp) - //----- offset 208 ----- - char Reserved[512 - 208]; // BUGBUG This should be done using align statements, - // not magic number math with incorrect arguments. + char Reserved[512 - 208]; //----------------------------- // function to multithreaded safe calculate the next array offset to write to @@ -387,6 +377,16 @@ struct SDS_FILE_HEADER } }; +static_assert(offsetof(SDS_FILE_HEADER, MetaBlockSize) == 48); +static_assert(offsetof(SDS_FILE_HEADER, TotalMetaCompressedSize) == 64); +static_assert(offsetof(SDS_FILE_HEADER, ArrayBlockSize) == 80); +static_assert(offsetof(SDS_FILE_HEADER, ArraysWritten) == 96); +static_assert(offsetof(SDS_FILE_HEADER, TotalArrayCompressedSize) == 112); +static_assert(offsetof(SDS_FILE_HEADER, BandBlockSize) == 128); +static_assert(offsetof(SDS_FILE_HEADER, SectionBlockSize) == 160); +static_assert(offsetof(SDS_FILE_HEADER, FileOffset) == 192); +static_assert(offsetof(SDS_FILE_HEADER, Reserved) == 208); + //--------------------------------------- class SDSSectionName { From f00a4ee5dc0334b7843efda8ea188ad8a14535be Mon Sep 17 00:00:00 2001 From: Staffan Tjernstrom <8728287+staffantj@users.noreply.github.com> Date: Fri, 19 Nov 2021 09:37:35 -0500 Subject: [PATCH 6/8] Check the size of SDS_FILE_HEADER --- src/SDSFile.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/SDSFile.h b/src/SDSFile.h index cc16f79b..2bf49858 100644 --- a/src/SDSFile.h +++ b/src/SDSFile.h @@ -386,6 +386,7 @@ static_assert(offsetof(SDS_FILE_HEADER, BandBlockSize) == 128); static_assert(offsetof(SDS_FILE_HEADER, SectionBlockSize) == 160); static_assert(offsetof(SDS_FILE_HEADER, FileOffset) == 192); static_assert(offsetof(SDS_FILE_HEADER, Reserved) == 208); +static_assert(sizeof(SDS_FILE_HEADER) == 512); //--------------------------------------- class SDSSectionName From 06da762c44644610036451da8ca97dc395d42e22 Mon Sep 17 00:00:00 2001 From: Staffan Tjernstrom <8728287+staffantj@users.noreply.github.com> Date: Thu, 2 Dec 2021 11:57:21 -0500 Subject: [PATCH 7/8] Rolling back the experiment to see how enforcing 32-bit strides would look like --- src/CommonInc.h | 8 +-- src/Compress.cpp | 10 +-- src/Convert.cpp | 4 +- src/MathWorker.h | 4 +- src/Merge.cpp | 12 ++-- src/Reduce.cpp | 2 +- src/RipTide.cpp | 14 ++--- src/RipTide.h | 2 +- src/Sort.cpp | 26 +++----- src/TileRepeat.cpp | 11 ++-- src/UnaryOps.cpp | 148 ++++++++++++++++++++++----------------------- src/UnaryOps.h | 4 +- src/one_input.cpp | 4 +- 13 files changed, 119 insertions(+), 130 deletions(-) diff --git a/src/CommonInc.h b/src/CommonInc.h index a547d0e5..e8e37365 100644 --- a/src/CommonInc.h +++ b/src/CommonInc.h @@ -344,8 +344,8 @@ struct stScatterGatherFunc typedef double (*ANY_SCATTER_GATHER_FUNC)(void * pDataIn, int64_t len, stScatterGatherFunc * pstScatterGatherFunc); // typedef void(*UNARY_FUNC)(void* pDataIn, void* pDataOut, int64_t len); -typedef void (*UNARY_FUNC)(void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut); -typedef void (*UNARY_FUNC_STRIDED)(void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut); +typedef void (*UNARY_FUNC)(void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut); +typedef void (*UNARY_FUNC_STRIDED)(void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut); // Pass in two vectors and return one vector // Used for operations like C = A + B @@ -474,8 +474,8 @@ struct FUNCTION_LIST int16_t NumpyOutputType; // The item size for two input arrays assumed to be the same - int32_t InputItemSize; - int32_t OutputItemSize; + int64_t InputItemSize; + int64_t OutputItemSize; // Strides may be 0 if it is a scalar or length 1 int64_t Input1Strides; diff --git a/src/Compress.cpp b/src/Compress.cpp index 61918d7f..d4850934 100644 --- a/src/Compress.cpp +++ b/src/Compress.cpp @@ -16,18 +16,18 @@ // Used when compressing from one numpy array to another numpy array // void FillInNumpyHeader(NUMPY_HEADERSIZE * pstNumpyHeader, int32_t dtype, int32_t ndim, int64_t * dims, int32_t flags, - int32_t itemsize) + int64_t itemsize) { if (pstNumpyHeader) { pstNumpyHeader->magic = COMPRESSION_MAGIC; pstNumpyHeader->compressiontype = COMPRESSION_TYPE_ZSTD; - pstNumpyHeader->dtype = (int8_t)(dtype); - pstNumpyHeader->ndim = (int8_t)ndim; + pstNumpyHeader->dtype = static_cast(dtype); + pstNumpyHeader->ndim = static_cast(ndim); pstNumpyHeader->flags = flags; - pstNumpyHeader->itemsize = itemsize; + pstNumpyHeader->itemsize = static_cast< int32_t >(itemsize); if (pstNumpyHeader->ndim > 3) { @@ -46,7 +46,7 @@ void FillInNumpyHeader(NUMPY_HEADERSIZE * pstNumpyHeader, int32_t dtype, int32_t // Returns temporary array to compress into // Caller is responsible for freeing memory NUMPY_HEADERSIZE * AllocCompressedMemory(int64_t arraylength, int32_t dtype, int32_t ndim, int64_t * dims, int32_t flags, - int32_t itemsize) + int64_t itemsize) { int64_t source_size = arraylength * itemsize; int64_t dest_size = ZSTD_compressBound(source_size); diff --git a/src/Convert.cpp b/src/Convert.cpp index 1aa83261..91e008bd 100644 --- a/src/Convert.cpp +++ b/src/Convert.cpp @@ -1095,11 +1095,11 @@ PyObject * ConvertSafeInternal(PyArrayObject * const inArr1, const int64_t out_d void * pBadOutput1 = GetDefaultForType(numpyOutType); // Check the strides of both the input and output to make sure we can handle - int32_t strideIn; + int64_t strideIn; int directionIn = GetStridesAndContig(inArr1, ndim, strideIn); int ndimOut; - int32_t strideOut; + int64_t strideOut; int directionOut = GetStridesAndContig(outArray, ndimOut, strideOut); // If the input is C and/or F-contiguous, the output should have diff --git a/src/MathWorker.h b/src/MathWorker.h index d8a55f28..97384039 100644 --- a/src/MathWorker.h +++ b/src/MathWorker.h @@ -562,8 +562,8 @@ class CMathWorker bool didSomeWork = false; OLD_CALLBACK * OldCallback = &pstWorkerItem->OldCallback; - int32_t strideSizeIn = OldCallback->FunctionList->InputItemSize; - int32_t strideSizeOut = OldCallback->FunctionList->OutputItemSize; + int64_t strideSizeIn = OldCallback->FunctionList->InputItemSize; + int64_t strideSizeOut = OldCallback->FunctionList->OutputItemSize; char * pDataInX = (char *)OldCallback->pDataInBase1; char * pDataInX2 = (char *)OldCallback->pDataInBase2; diff --git a/src/Merge.cpp b/src/Merge.cpp index 70123ed1..91668f9d 100644 --- a/src/Merge.cpp +++ b/src/Merge.cpp @@ -255,8 +255,8 @@ PyObject * BooleanIndexInternal(PyArrayObject * aValues, PyArrayObject * aIndex) int ndimValue = 0; int ndimBoolean = 0; - int32_t strideValue = 0; - int32_t strideBoolean = 0; + int64_t strideValue = 0; + int64_t strideBoolean = 0; int result1 = GetStridesAndContig(aValues, ndimValue, strideValue); int result2 = GetStridesAndContig(aIndex, ndimBoolean, strideBoolean); @@ -741,7 +741,7 @@ PyObject * BooleanSum(PyObject * self, PyObject * args) } int ndimBoolean; - int32_t strideBoolean; + int64_t strideBoolean; int result1 = GetStridesAndContig(aIndex, ndimBoolean, strideBoolean); @@ -1329,8 +1329,8 @@ PyObject * MBGet(PyObject * self, PyObject * args) int ndimValue; int ndimIndex; - int32_t strideValue = 0; - int32_t strideIndex = 0; + int64_t strideValue = 0; + int64_t strideIndex = 0; int result1 = GetStridesAndContig(aValues, ndimValue, strideValue); int result2 = GetStridesAndContig(aIndex, ndimIndex, strideIndex); @@ -1505,7 +1505,7 @@ PyObject * BooleanToFancy(PyObject * self, PyObject * args, PyObject * kwargs) } int ndimBoolean; - int32_t strideBoolean; + int64_t strideBoolean; int result1 = GetStridesAndContig(aIndex, ndimBoolean, strideBoolean); diff --git a/src/Reduce.cpp b/src/Reduce.cpp index 198e75b8..23dd107d 100644 --- a/src/Reduce.cpp +++ b/src/Reduce.cpp @@ -2949,7 +2949,7 @@ static PyObject * ReduceInternal(PyArrayObject * inArr1, REDUCE_FUNCTIONS func, fl.FunctionName = "Reduce"; fl.NumpyOutputType = 0; fl.NumpyType = numpyInType; - fl.InputItemSize = static_cast(PyArray_ITEMSIZE(inArr1)); + fl.InputItemSize = PyArray_ITEMSIZE(inArr1); fl.Input1Strides = len > 1 ? PyArray_STRIDE(inArr1, 0) : 0; fl.OutputItemSize = 0; fl.TypeOfFunctionCall = TYPE_OF_FUNCTION_CALL::ANY_SCATTER_GATHER; diff --git a/src/RipTide.cpp b/src/RipTide.cpp index 29f8187c..6979af01 100644 --- a/src/RipTide.cpp +++ b/src/RipTide.cpp @@ -2255,19 +2255,19 @@ bool GetUpcastType(int numpyInType1, int numpyInType2, int & convertType1, int & // return value 0: one loop can process all data, false = multiple loops // NOTE: if return value is 0 and itemsze == stride, then vector math possible // -int GetStridesAndContig(PyArrayObject const * inArray, int & ndim, int32_t & stride) +int GetStridesAndContig(PyArrayObject const * inArray, int & ndim, int64_t & stride) { stride = static_cast(PyArray_ITEMSIZE(inArray)); int direction = 0; ndim = PyArray_NDIM(inArray); if (ndim > 0) { - stride = static_cast(PyArray_STRIDE(inArray, 0)); + stride = PyArray_STRIDE(inArray, 0); if (ndim > 1) { // at least two strides int ndims = PyArray_NDIM(inArray); - int32_t lastStride = static_cast(PyArray_STRIDE(inArray, ndims - 1)); + int64_t lastStride = PyArray_STRIDE(inArray, ndims - 1); if (lastStride == stride) { // contiguous with one of the dimensions having length 1 @@ -2277,10 +2277,10 @@ int GetStridesAndContig(PyArrayObject const * inArray, int & ndim, int32_t & str // Row Major - 'C' Style // work backwards int currentdim = ndims - 1; - int32_t curStrideLen = lastStride; + int64_t curStrideLen = lastStride; while (currentdim != 0) { - curStrideLen *= static_cast(PyArray_DIM(inArray, currentdim)); + curStrideLen *= PyArray_DIM(inArray, currentdim); LOGGING("'C' %lld vs %lld dim: %lld stride: %lld \n", curStrideLen, PyArray_STRIDE(inArray, currentdim - 1), PyArray_DIM(inArray, currentdim - 1), lastStride); if (PyArray_STRIDE(inArray, currentdim - 1) != curStrideLen) @@ -2294,10 +2294,10 @@ int GetStridesAndContig(PyArrayObject const * inArray, int & ndim, int32_t & str { // Col Major - 'F' Style int currentdim = 0; - int32_t curStrideLen = stride; + int64_t curStrideLen = stride; while (currentdim != (ndims - 1)) { - curStrideLen *= static_cast(PyArray_DIM(inArray, currentdim)); + curStrideLen *= static_cast(PyArray_DIM(inArray, currentdim)); LOGGING("'F' %lld vs %lld dim: %lld stride: %lld \n", curStrideLen, PyArray_STRIDE(inArray, currentdim + 1), PyArray_DIM(inArray, currentdim + 1), stride); if (PyArray_STRIDE(inArray, currentdim + 1) != curStrideLen) diff --git a/src/RipTide.h b/src/RipTide.h index c0c7c984..7dbebb7d 100644 --- a/src/RipTide.h +++ b/src/RipTide.h @@ -99,7 +99,7 @@ typedef struct } PyBoolScalarObject; extern bool GetUpcastType(int numpyInType1, int numpyInType2, int & convertType1, int & convertType2, int64_t funcNumber); -extern int GetStridesAndContig(PyArrayObject const * inArray, int & ndim, int32_t & stride); +extern int GetStridesAndContig(PyArrayObject const * inArray, int & ndim, int64_t & stride); /** * @brief Calculate the number of elements in an array with the given diff --git a/src/Sort.cpp b/src/Sort.cpp index 50c1a7f7..d2e8dfa6 100644 --- a/src/Sort.cpp +++ b/src/Sort.cpp @@ -2507,8 +2507,6 @@ static int par_amergesort(int64_t * pCutOffs, // May be NULL (if so no partition PAR_SORT_TYPE sortType; int64_t sizeofT; - int64_t sizeofUINDEX; - } psort; psort.funcSingleMerge = single_amergesort; @@ -2519,7 +2517,6 @@ static int par_amergesort(int64_t * pCutOffs, // May be NULL (if so no partition psort.strlen = strlen; psort.sortType = sortType; - psort.sizeofUINDEX = sizeof(UINDEX); if (strlen > 0) { psort.sizeofT = strlen; @@ -2553,7 +2550,7 @@ static int par_amergesort(int64_t * pCutOffs, // May be NULL (if so no partition // shift the data pointers to match the partition // call a single threaded merge callbackArg->funcSingleMerge(callbackArg->pValues + (partStart * callbackArg->sizeofT), - callbackArg->pToSort + (partStart * callbackArg->sizeofUINDEX), partLength, + callbackArg->pToSort + (partStart * sizeof(UINDEX)), partLength, callbackArg->strlen, callbackArg->sortType); return true; @@ -3978,7 +3975,6 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n bool * pFilter; int64_t base_index; int64_t strlen; - int64_t sizeofUINDEX; } pgroup; @@ -3996,9 +3992,6 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n pgroup.pFilter = pFilter; pgroup.base_index = base_index; pgroup.strlen = itemSizeValues; - const int64_t INDEX_SIZE = (int64_t)sizeof(UINDEX); - - pgroup.sizeofUINDEX = INDEX_SIZE; // Use threads per partition auto lambdaPSCallback = [](void * callbackArgT, int core, int64_t workIndex) -> bool @@ -4026,10 +4019,10 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n // call a single threaded merge callbackArg->pUniqueCounts[t] = callbackArg->funcSingleGroup(callbackArg->pValues + (partStart * callbackArg->strlen), partLength, - callbackArg->pIndex + (partStart * callbackArg->sizeofUINDEX), - callbackArg->pKeyOut + (partStart * callbackArg->sizeofUINDEX), - callbackArg->pFirstOut + (partStart * callbackArg->sizeofUINDEX), - callbackArg->pCountOut + (partStart * callbackArg->sizeofUINDEX), + callbackArg->pIndex + (partStart * sizeof(UINDEX)), + callbackArg->pKeyOut + (partStart * sizeof(UINDEX)), + callbackArg->pFirstOut + (partStart * sizeof(UINDEX)), + callbackArg->pCountOut + (partStart * sizeof(UINDEX)), callbackArg->pFilter + partStart, 0, // callbackArg->base_index, fix for countout callbackArg->strlen); @@ -4052,10 +4045,10 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n // TODO: fix up keys // parallel add? - PyArrayObject * firstReduced = AllocateNumpyArray(1, (npy_intp *)&totalUniques, INDEX_SIZE == 4 ? NPY_INT32 : NPY_INT64); + PyArrayObject * firstReduced = AllocateNumpyArray(1, (npy_intp *)&totalUniques, sizeof(UINDEX) == 4 ? NPY_INT32 : NPY_INT64); totalUniques++; - PyArrayObject * countReduced = AllocateNumpyArray(1, (npy_intp *)&totalUniques, INDEX_SIZE == 4 ? NPY_INT32 : NPY_INT64); + PyArrayObject * countReduced = AllocateNumpyArray(1, (npy_intp *)&totalUniques, sizeof(UINDEX) == 4 ? NPY_INT32 : NPY_INT64); // ANOTHER PARALEL ROUTINE to copy struct stPGROUPADD @@ -4075,8 +4068,6 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n bool * pFilter; int64_t base_index; - int64_t sizeofUINDEX; - } pgroupadd; pgroupadd.pUniqueCounts = pUniqueCounts; @@ -4094,13 +4085,12 @@ static PyObject * GroupFromLexSortInternal(PyObject * kwargs, UINDEX * pIndex, n pgroupadd.pCountReduced = (char *)PyArray_BYTES(countReduced); // skip first value since reserved for zero bin (and assign it 0) - for (int64_t c = 0; c < INDEX_SIZE; c++) + for (int64_t c = 0; c < sizeof(UINDEX); c++) { *pgroupadd.pCountReduced++ = 0; } pgroupadd.base_index = base_index; - pgroupadd.sizeofUINDEX = sizeof(UINDEX); // Use threads per partition auto lambdaPGADDCallback = [](void * callbackArgT, int core, int64_t workIndex) -> bool diff --git a/src/TileRepeat.cpp b/src/TileRepeat.cpp index ef636e26..f7aa38db 100644 --- a/src/TileRepeat.cpp +++ b/src/TileRepeat.cpp @@ -25,13 +25,12 @@ const union { int32_t i[8]; __m256i m; - //} __vindex8_strides = { 7, 6, 5, 4, 3, 2, 1, 0 }; -} __vindex8_strides = { 0, 1, 2, 3, 4, 5, 6, 7 }; +} vindex8_strides = { 0, 1, 2, 3, 4, 5, 6, 7 }; //----------------------------------- // void ConvertRecArray(char * pStartOffset, int64_t startRow, int64_t totalRows, stOffsets * pstOffset, int64_t * pOffsets, - int64_t numArrays, int32_t itemSize) + int64_t numArrays, int64_t itemSize) { // Try to keep everything in L1Cache const int64_t L1CACHE = 32768; @@ -41,7 +40,7 @@ void ConvertRecArray(char * pStartOffset, int64_t startRow, int64_t totalRows, s CHUNKROWS = 1; } - __m256i vindex = _mm256_mullo_epi32(_mm256_set1_epi32(itemSize), _mm256_loadu_si256(&__vindex8_strides.m)); + __m256i vindex = _mm256_mullo_epi32(_mm256_set1_epi32(static_cast(itemSize)), _mm256_loadu_si256(&vindex8_strides.m)); __m128i vindex128 = _mm256_extracti128_si256(vindex, 0); while (startRow < totalRows) @@ -172,7 +171,7 @@ PyObject * RecordArrayToColMajor(PyObject * self, PyObject * args) return NULL; } - int32_t itemSize = static_cast(PyArray_ITEMSIZE(inArr)); + int64_t itemSize = PyArray_ITEMSIZE(inArr); if (itemSize != PyArray_STRIDE(inArr, 0)) { @@ -219,7 +218,7 @@ PyObject * RecordArrayToColMajor(PyObject * self, PyObject * args) stOffsets * pstOffset; int64_t * pOffsets; int64_t numArrays; - int32_t itemSize; + int64_t itemSize; int64_t lastRow; } stConvert; diff --git a/src/UnaryOps.cpp b/src/UnaryOps.cpp index 6239deaf..865b6246 100644 --- a/src/UnaryOps.cpp +++ b/src/UnaryOps.cpp @@ -659,7 +659,7 @@ static const inline __m256d SQRT_OP_256(__m256d x) // T = data type as input // MathOp operation to perform template -static inline void UnaryOpFast(void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static inline void UnaryOpFast(void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { T * pIn = (T *)pDataIn; T * pOut = (T *)pDataOut; @@ -714,7 +714,7 @@ static inline void UnaryOpFast(void * pDataIn, void * pDataOut, int64_t len, int // T = data type as input // MathOp operation to perform template -static inline void UnaryOpFastStrided(void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static inline void UnaryOpFastStrided(void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { T * pIn = (T *)pDataIn; T * pOut = (T *)pDataOut; @@ -733,7 +733,7 @@ static inline void UnaryOpFastStrided(void * pDataIn, void * pDataOut, int64_t l U256 * pIn1_256 = (U256 *)pIn; U256 * pOut_256 = (U256 *)pOut; - int32_t babyStride = strideIn; + int32_t babyStride = static_cast< int32_t >(strideIn); // add 8 strides everytime we process 8 __m256i mindex = _mm256_mullo_epi32(_mm256_set1_epi32(babyStride), __vec8_strides.m); @@ -768,8 +768,8 @@ static inline void UnaryOpFastStrided(void * pDataIn, void * pDataOut, int64_t l // T = data type as input // MathOp operation to perform template -static inline void UnaryNanFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, - int32_t strideOut) +static inline void UnaryNanFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, + int64_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -816,8 +816,8 @@ static inline void UnaryNanFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, vo // T = data type as input // MathOp operation to perform template -static inline void UnaryNanFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, - int32_t strideOut) +static inline void UnaryNanFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, + int64_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -859,8 +859,8 @@ static inline void UnaryNanFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, v // T = data type as input // MathOp operation to perform template -static inline void UnaryNotNanFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, - int32_t strideOut) +static inline void UnaryNotNanFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, + int64_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -904,8 +904,8 @@ static inline void UnaryNotNanFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, // T = data type as input // MathOp operation to perform template -static inline void UnaryNotNanFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, - int32_t strideOut) +static inline void UnaryNotNanFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, + int64_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -947,8 +947,8 @@ static inline void UnaryNotNanFastDouble(MathFunctionPtr MATH_OP, void * pDataIn // T = data type as input // MathOp operation to perform template -static inline void UnaryFiniteFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, - int32_t strideOut) +static inline void UnaryFiniteFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, + int64_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -995,8 +995,8 @@ static inline void UnaryFiniteFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, // T = data type as input // MathOp operation to perform template -static inline void UnaryFiniteFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, - int32_t strideOut) +static inline void UnaryFiniteFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, + int64_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -1041,8 +1041,8 @@ static inline void UnaryFiniteFastDouble(MathFunctionPtr MATH_OP, void * pDataIn // T = data type as input // MathOp operation to perform template -static inline void UnaryNotFiniteFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, - int32_t strideOut) +static inline void UnaryNotFiniteFastFloat(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, + int64_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -1090,7 +1090,7 @@ static inline void UnaryNotFiniteFastFloat(MathFunctionPtr MATH_OP, void * pData // MathOp operation to perform template static inline void UnaryNotFiniteFastDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, - int32_t strideIn, int32_t strideOut) + int64_t strideIn, int64_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -1138,7 +1138,7 @@ static inline void UnaryNotFiniteFastDouble(MathFunctionPtr MATH_OP, void * pDat // Output same as input type // MathOp operation to perform template -static void UnaryOpSlow(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { T * pIn = (T *)pDataIn; T * pOut = (T *)pDataOut; @@ -1158,8 +1158,8 @@ static void UnaryOpSlow(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut // Output always returns a double // MathOp operation to perform template -static void UnaryOpSlowDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, - int32_t strideOut) +static void UnaryOpSlowDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, + int64_t strideOut) { T * pIn = (T *)pDataIn; double * pOut = (double *)pDataOut; @@ -1179,8 +1179,8 @@ static void UnaryOpSlowDouble(MathFunctionPtr MATH_OP, void * pDataIn, void * pD // Output always returns a double // MathOp operation to perform template -static void UnaryOpSlowBool(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int32_t strideIn, - int32_t strideOut) +static void UnaryOpSlowBool(MathFunctionPtr MATH_OP, void * pDataIn, void * pDataOut, int64_t len, int64_t strideIn, + int64_t strideOut) { T * pIn = (T *)pDataIn; bool * pOut = (bool *)pDataOut; @@ -1195,7 +1195,7 @@ static void UnaryOpSlowBool(MathFunctionPtr MATH_OP, void * pDataIn, void * pDat } } -static void UnaryOpSlow_FillTrue(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_FillTrue(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { if (strideOut == sizeof(bool)) { @@ -1213,7 +1213,7 @@ static void UnaryOpSlow_FillTrue(void * pDataIn1, void * pDataOut, int64_t len, } } -static void UnaryOpSlow_FillFalse(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_FillFalse(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { if (strideOut == sizeof(bool)) { @@ -1233,59 +1233,59 @@ static void UnaryOpSlow_FillFalse(void * pDataIn1, void * pDataOut, int64_t len, //------------------------------------------------------------------- template -static void UnaryOpSlow_ABS(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_ABS(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(ABS_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_FABS(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_FABS(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(FABS_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_SIGN(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_SIGN(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(SIGN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_FLOATSIGN(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_FLOATSIGN(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(FLOATSIGN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_NEG(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_NEG(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(NEG_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_ISNOTNAN(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_ISNOTNAN(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowBool(ISNOTNAN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISNAN(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_ISNAN(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowBool(ISNAN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISNANORZERO(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_ISNANORZERO(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowBool(ISNANORZERO_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISINVALID(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_ISINVALID(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { T * pIn = (T *)pDataIn1; T invalid = *(T *)GetInvalid(); @@ -1309,7 +1309,7 @@ static void UnaryOpSlow_ISINVALID(void * pDataIn1, void * pDataOut, int64_t len, } template -static void UnaryOpSlow_ISINVALIDORZERO(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_ISINVALIDORZERO(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { T * pIn = (T *)pDataIn1; T invalid = *(T *)GetInvalid(); @@ -1333,7 +1333,7 @@ static void UnaryOpSlow_ISINVALIDORZERO(void * pDataIn1, void * pDataOut, int64_ } template -static void UnaryOpSlow_ISNOTINVALID(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_ISNOTINVALID(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { T * pIn = (T *)pDataIn1; T invalid = *(T *)GetInvalid(); @@ -1357,173 +1357,173 @@ static void UnaryOpSlow_ISNOTINVALID(void * pDataIn1, void * pDataOut, int64_t l } template -static void UnaryOpSlow_ISFINITE(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_ISFINITE(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowBool(ISFINITE_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISNOTFINITE(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_ISNOTFINITE(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowBool(ISNOTFINITE_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISINF(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_ISINF(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowBool(ISINF_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISNOTINF(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_ISNOTINF(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowBool(ISNOTINF_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISNORMAL(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_ISNORMAL(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowBool(ISNORMAL_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_ISNOTNORMAL(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_ISNOTNORMAL(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowBool(ISNOTNORMAL_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static void UnaryOpSlow_SIGNBIT(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_SIGNBIT(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowBool(SIGNBIT_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_NOT(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_NOT(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowBool(NOT_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_INVERT(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_INVERT(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(INVERT_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_INVERT_BOOL(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_INVERT_BOOL(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(INVERT_OP_BOOL, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_FLOOR(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_FLOOR(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(FLOOR_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_CEIL(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_CEIL(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(CEIL_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_TRUNC(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_TRUNC(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(TRUNC_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_ROUND(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_ROUND(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(ROUND_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_SQRT(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_SQRT(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(SQRT_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_CBRT(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_CBRT(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(CBRT_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_LOG(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_LOG(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(LOG_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_LOG2(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_LOG2(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(LOG2_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_LOG10(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_LOG10(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(LOG10_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_EXP(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_EXP(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(EXP_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlow_EXP2(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlow_EXP2(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlow(EXP2_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlowDouble_SQRT(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlowDouble_SQRT(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowDouble(SQRT_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlowDouble_CBRT(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlowDouble_CBRT(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowDouble(CBRT_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlowDouble_LOG(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlowDouble_LOG(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowDouble(LOG_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlowDouble_LOG2(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlowDouble_LOG2(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowDouble(LOG2_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlowDouble_LOG10(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlowDouble_LOG10(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowDouble(LOG10_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlowDouble_EXP(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlowDouble_EXP(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowDouble(EXP_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } //------------------------------------------------------------------- template -static void UnaryOpSlowDouble_EXP2(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static void UnaryOpSlowDouble_EXP2(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryOpSlowDouble(EXP2_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } @@ -1536,45 +1536,45 @@ static void UnaryOpSlowDouble_EXP2(void * pDataIn1, void * pDataOut, int64_t len // This routines will call the proper templated ABS function based on the type // It provides the standard signature template -static inline void UnaryOpFast_NANF32(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static inline void UnaryOpFast_NANF32(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryNanFastFloat(ISNAN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static inline void UnaryOpFast_NANF64(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static inline void UnaryOpFast_NANF64(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryNanFastDouble(ISNAN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static inline void UnaryOpFast_NOTNANF32(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static inline void UnaryOpFast_NOTNANF32(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryNotNanFastFloat(ISNOTNAN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static inline void UnaryOpFast_NOTNANF64(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static inline void UnaryOpFast_NOTNANF64(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryNotNanFastDouble(ISNOTNAN_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static inline void UnaryOpFast_FINITEF32(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static inline void UnaryOpFast_FINITEF32(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryFiniteFastFloat(ISFINITE_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static inline void UnaryOpFast_FINITEF64(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static inline void UnaryOpFast_FINITEF64(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryFiniteFastDouble(ISFINITE_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static inline void UnaryOpFast_NOTFINITEF32(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static inline void UnaryOpFast_NOTFINITEF32(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryNotFiniteFastFloat(ISNOTFINITE_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } template -static inline void UnaryOpFast_NOTFINITEF64(void * pDataIn1, void * pDataOut, int64_t len, int32_t strideIn, int32_t strideOut) +static inline void UnaryOpFast_NOTFINITEF64(void * pDataIn1, void * pDataOut, int64_t len, int64_t strideIn, int64_t strideOut) { return UnaryNotFiniteFastDouble(ISNOTFINITE_OP, pDataIn1, pDataOut, len, strideIn, strideOut); } @@ -2481,7 +2481,7 @@ PyObject * ProcessOneInput(PyArrayObject * inArray, PyArrayObject * outObject1, { LOGGING("In process one input %d %d %d\n", funcNumber, numpyInType1, numpyOutType); int ndim; - int32_t strideIn; + int64_t strideIn; int directionIn = GetStridesAndContig(inArray, ndim, strideIn); int64_t len = CALC_ARRAY_LENGTH(ndim, PyArray_DIMS(inArray)); @@ -2600,7 +2600,7 @@ PyObject * ProcessOneInput(PyArrayObject * inArray, PyArrayObject * outObject1, void * pDataOut = PyArray_BYTES(outputArray); int ndimOut; - int32_t strideOut; + int64_t strideOut; int directionOut = GetStridesAndContig(outputArray, ndimOut, strideOut); // F contig on 2 dim => dim0 * stride0 == stride1 diff --git a/src/UnaryOps.h b/src/UnaryOps.h index 48d9a0c0..022c2e21 100644 --- a/src/UnaryOps.h +++ b/src/UnaryOps.h @@ -22,8 +22,8 @@ struct UNARY_CALLBACK char * pDataIn; char * pDataOut; - int32_t itemSizeIn; - int32_t itemSizeOut; + int64_t itemSizeIn; + int64_t itemSizeOut; }; #endif diff --git a/src/one_input.cpp b/src/one_input.cpp index a6c2840e..836e8e54 100644 --- a/src/one_input.cpp +++ b/src/one_input.cpp @@ -18,7 +18,7 @@ PyObject * process_one_input(PyArrayObject const * in_array, PyArrayObject * out int32_t numpy_intype, int32_t numpy_outtype) { int32_t ndim{}; - int32_t stride{}; + int64_t stride{}; int32_t direction{ GetStridesAndContig(in_array, ndim, stride) }; npy_intp len{ CALC_ARRAY_LENGTH(ndim, PyArray_DIMS(const_cast(in_array))) }; @@ -83,7 +83,7 @@ PyObject * process_one_input(PyArrayObject const * in_array, PyArrayObject * out char * out_p{ PyArray_BYTES(const_cast(result_array)) }; int num_dims_out{}; - int32_t stride_out{}; + int64_t stride_out{}; int direction_out = GetStridesAndContig(result_array, num_dims_out, stride_out); if (direction_out == 0) From 9f5bc19e92e6e182f5966a7859eb9d20d39dfa16 Mon Sep 17 00:00:00 2001 From: Staffan Tjernstrom <8728287+staffantj@users.noreply.github.com> Date: Thu, 2 Dec 2021 13:54:45 -0500 Subject: [PATCH 8/8] Throw on range error until we reach somewhere to talk to Python --- src/Convert.cpp | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/Convert.cpp b/src/Convert.cpp index 91e008bd..c4c91d28 100644 --- a/src/Convert.cpp +++ b/src/Convert.cpp @@ -7,6 +7,8 @@ #include "Reduce.h" #include +#include +#include #define LOGGING(...) //#define LOGGING printf @@ -2087,8 +2089,9 @@ int64_t Combine1Filter(void * pInputIndex, { if (i > std::numeric_limits::max()) { - PyErr_Format(PyExc_ValueError, "Combine1Filter has attempted to exceed the size of the index value [%lld]", i); - return 0; + std::string errmsg{"Combine1Filter has attempted to exceed the size of the index value "}; + errmsg += std::to_string(i); + throw std::overflow_error(errmsg); } if (pFilter[i]) { @@ -2139,8 +2142,9 @@ int64_t Combine1Filter(void * pInputIndex, { if (i > std::numeric_limits::max()) { - PyErr_Format(PyExc_ValueError, "Combine1Filter has attempted to exceed the size of the index value [%lld]", i); - return 0; + std::string errmsg{"Combine1Filter has attempted to exceed the size of the index value, no filter provided "}; + errmsg += std::to_string(i); + throw std::overflow_error(errmsg); } INDEX index = pInput[i]; // printf("[%lld] got index\n", (int64_t)index); @@ -2268,7 +2272,17 @@ PyObject * CombineAccum1Filter(PyObject * self, PyObject * args) { int32_t * pFirst = (int32_t *)PyArray_BYTES(firstArray); - int64_t uniqueCount = pFunction(pDataIn1, PyArray_BYTES(outArray), pFirst, pFilterIn, arraySize1, hashSize); + int64_t uniqueCount{}; + + try + { + uniqueCount = pFunction(pDataIn1, PyArray_BYTES(outArray), pFirst, pFilterIn, arraySize1, hashSize); + } + catch ( std::exception &e ) + { + PyErr_Format(PyExc_ValueError, e.what()); + return nullptr; + } if (uniqueCount < arraySize1) {