diff --git a/src/Microsoft.ML.FastTree/Dataset/DenseIntArray.cs b/src/Microsoft.ML.FastTree/Dataset/DenseIntArray.cs
index 6e3faac9a0..b3cc3191f9 100644
--- a/src/Microsoft.ML.FastTree/Dataset/DenseIntArray.cs
+++ b/src/Microsoft.ML.FastTree/Dataset/DenseIntArray.cs
@@ -111,6 +111,123 @@ protected static unsafe void SumupCPlusPlusDense(SumupInputData input, FeatureHi
}
}
+ ///
+ /// Managed equivalent of , used on platforms where the
+ /// native FastTree library is not available (e.g. arm64). This mirrors the native C_Sumup
+ /// loop in src/Native/FastTreeNative (Sumup.h / SumupNibbles.h) exactly, including the
+ /// per-document iteration and accumulation order, so histogram results are bit-identical
+ /// to the native implementation. Reads go through fixed pointers to avoid the per-element
+ /// bounds checks and interface-indexer dispatch of the generic .
+ ///
+ protected static unsafe void SumupManagedDense(SumupInputData input, FeatureHistogram histogram,
+ byte* data, int numBits)
+ {
+ using (Timer.Time(TimerEvent.SumupCppDense))
+ {
+ fixed (FloatType* pSumTargetsByBin = histogram.SumTargetsByBin)
+ fixed (FloatType* pSampleOutputs = input.Outputs)
+ fixed (double* pSumWeightsByBin = histogram.SumWeightsByBin)
+ fixed (double* pSampleWeights = input.Weights)
+ fixed (int* pIndices = input.DocIndices)
+ fixed (int* pCountByBin = histogram.CountByBin)
+ {
+ int count = input.TotalCount;
+ ushort* data16 = (ushort*)data;
+ int* data32 = (int*)data;
+
+ // numBits is switched outside the loop (it never varies within a call) so the
+ // hot loop stays a tight scalar accumulation matching the native code. The
+ // "pIndices == null ? i : pIndices[i]" ternary is loop-invariant and free.
+ if (pSumWeightsByBin != null)
+ {
+ switch (numBits)
+ {
+ case 4:
+ for (int i = 0; i < count; i++)
+ {
+ int p = pIndices == null ? i : pIndices[i];
+ int featureBin = (data[p >> 1] >> ((~(p << 2)) & 4)) & 0xf;
+ pSumTargetsByBin[featureBin] += pSampleOutputs[i];
+ pSumWeightsByBin[featureBin] += pSampleWeights[i];
+ ++pCountByBin[featureBin];
+ }
+ break;
+ case 8:
+ for (int i = 0; i < count; i++)
+ {
+ int featureBin = data[pIndices == null ? i : pIndices[i]];
+ pSumTargetsByBin[featureBin] += pSampleOutputs[i];
+ pSumWeightsByBin[featureBin] += pSampleWeights[i];
+ ++pCountByBin[featureBin];
+ }
+ break;
+ case 16:
+ for (int i = 0; i < count; i++)
+ {
+ int featureBin = data16[pIndices == null ? i : pIndices[i]];
+ pSumTargetsByBin[featureBin] += pSampleOutputs[i];
+ pSumWeightsByBin[featureBin] += pSampleWeights[i];
+ ++pCountByBin[featureBin];
+ }
+ break;
+ case 32:
+ for (int i = 0; i < count; i++)
+ {
+ int featureBin = data32[pIndices == null ? i : pIndices[i]];
+ pSumTargetsByBin[featureBin] += pSampleOutputs[i];
+ pSumWeightsByBin[featureBin] += pSampleWeights[i];
+ ++pCountByBin[featureBin];
+ }
+ break;
+ default:
+ throw Contracts.Except("Unsupported bits per item {0}", numBits);
+ }
+ }
+ else
+ {
+ switch (numBits)
+ {
+ case 4:
+ for (int i = 0; i < count; i++)
+ {
+ int p = pIndices == null ? i : pIndices[i];
+ int featureBin = (data[p >> 1] >> ((~(p << 2)) & 4)) & 0xf;
+ pSumTargetsByBin[featureBin] += pSampleOutputs[i];
+ ++pCountByBin[featureBin];
+ }
+ break;
+ case 8:
+ for (int i = 0; i < count; i++)
+ {
+ int featureBin = data[pIndices == null ? i : pIndices[i]];
+ pSumTargetsByBin[featureBin] += pSampleOutputs[i];
+ ++pCountByBin[featureBin];
+ }
+ break;
+ case 16:
+ for (int i = 0; i < count; i++)
+ {
+ int featureBin = data16[pIndices == null ? i : pIndices[i]];
+ pSumTargetsByBin[featureBin] += pSampleOutputs[i];
+ ++pCountByBin[featureBin];
+ }
+ break;
+ case 32:
+ for (int i = 0; i < count; i++)
+ {
+ int featureBin = data32[pIndices == null ? i : pIndices[i]];
+ pSumTargetsByBin[featureBin] += pSampleOutputs[i];
+ ++pCountByBin[featureBin];
+ }
+ break;
+ default:
+ throw Contracts.Except("Unsupported bits per item {0}", numBits);
+ }
+ }
+ }
+ }
+ }
+
public override IIntArrayForwardIndexer GetIndexer()
{
return this;
@@ -389,21 +506,21 @@ public Dense8BitIntArray(int len)
: base(len)
{
_data = new byte[len];
- SetupSumupHandler(SumupNative, base.Sumup);
+ SetupSumupHandler(SumupNative, SumupManaged);
}
public Dense8BitIntArray(byte[] buffer, ref int position)
: base(buffer.ToInt(ref position))
{
_data = buffer.ToByteArray(ref position);
- SetupSumupHandler(SumupNative, base.Sumup);
+ SetupSumupHandler(SumupNative, SumupManaged);
}
public Dense8BitIntArray(int len, IEnumerable values)
: base(len)
{
_data = values.Select(i => (byte)i).ToArray(len);
- SetupSumupHandler(SumupNative, base.Sumup);
+ SetupSumupHandler(SumupNative, SumupManaged);
}
///
@@ -457,6 +574,17 @@ private void SumupNative(SumupInputData input, FeatureHistogram histogram)
}
}
+ private void SumupManaged(SumupInputData input, FeatureHistogram histogram)
+ {
+ unsafe
+ {
+ fixed (byte* pData = _data)
+ {
+ SumupManagedDense(input, histogram, pData, 8);
+ }
+ }
+ }
+
public override void Sumup(SumupInputData input, FeatureHistogram histogram) => SumupHandler(input, histogram);
}
@@ -476,14 +604,14 @@ public Dense4BitIntArray(int len)
: base(len)
{
_data = new byte[(len + 1) / 2]; // Even length = half the bytes. Odd length = half the bytes+0.5.
- SetupSumupHandler(SumupNative, base.Sumup);
+ SetupSumupHandler(SumupNative, SumupManaged);
}
public Dense4BitIntArray(int len, IEnumerable values)
: base(len)
{
_data = new byte[(len + 1) / 2];
- SetupSumupHandler(SumupNative, base.Sumup);
+ SetupSumupHandler(SumupNative, SumupManaged);
int currentIndex = 0;
bool upper = true;
@@ -508,7 +636,7 @@ public Dense4BitIntArray(byte[] buffer, ref int position)
: base(buffer.ToInt(ref position))
{
_data = buffer.ToByteArray(ref position);
- SetupSumupHandler(SumupNative, base.Sumup);
+ SetupSumupHandler(SumupNative, SumupManaged);
}
///
@@ -580,6 +708,17 @@ public void SumupNative(SumupInputData input, FeatureHistogram histogram)
}
}
+ private void SumupManaged(SumupInputData input, FeatureHistogram histogram)
+ {
+ unsafe
+ {
+ fixed (byte* pData = _data)
+ {
+ SumupManagedDense(input, histogram, pData, 4);
+ }
+ }
+ }
+
public override void Sumup(SumupInputData input, FeatureHistogram histogram) => SumupHandler(input, histogram);
}
@@ -596,21 +735,21 @@ public Dense16BitIntArray(int len)
: base(len)
{
_data = new ushort[len];
- SetupSumupHandler(SumupNative, base.Sumup);
+ SetupSumupHandler(SumupNative, SumupManaged);
}
public Dense16BitIntArray(int len, IEnumerable values)
: base(len)
{
_data = values.Select(i => (ushort)i).ToArray(len);
- SetupSumupHandler(SumupNative, base.Sumup);
+ SetupSumupHandler(SumupNative, SumupManaged);
}
public Dense16BitIntArray(byte[] buffer, ref int position)
: base(buffer.ToInt(ref position))
{
_data = buffer.ToUShortArray(ref position);
- SetupSumupHandler(SumupNative, base.Sumup);
+ SetupSumupHandler(SumupNative, SumupManaged);
}
public override unsafe void Callback(Action callback)
@@ -668,6 +807,18 @@ public void SumupNative(SumupInputData input, FeatureHistogram histogram)
}
}
+ private void SumupManaged(SumupInputData input, FeatureHistogram histogram)
+ {
+ unsafe
+ {
+ fixed (ushort* pData = _data)
+ {
+ byte* pDataBytes = (byte*)pData;
+ SumupManagedDense(input, histogram, pDataBytes, 16);
+ }
+ }
+ }
+
public override void Sumup(SumupInputData input, FeatureHistogram histogram) => SumupHandler(input, histogram);
}
@@ -685,21 +836,21 @@ public Dense32BitIntArray(int len)
: base(len)
{
_data = new int[len];
- SetupSumupHandler(SumupNative, base.Sumup);
+ SetupSumupHandler(SumupNative, SumupManaged);
}
public Dense32BitIntArray(int len, IEnumerable values)
: base(len)
{
_data = values.ToArray(len);
- SetupSumupHandler(SumupNative, base.Sumup);
+ SetupSumupHandler(SumupNative, SumupManaged);
}
public Dense32BitIntArray(byte[] buffer, ref int position)
: base(buffer.ToInt(ref position))
{
_data = buffer.ToIntArray(ref position);
- SetupSumupHandler(SumupNative, base.Sumup);
+ SetupSumupHandler(SumupNative, SumupManaged);
}
public override unsafe void Callback(Action callback)
@@ -757,6 +908,18 @@ public void SumupNative(SumupInputData input, FeatureHistogram histogram)
}
}
+ private void SumupManaged(SumupInputData input, FeatureHistogram histogram)
+ {
+ unsafe
+ {
+ fixed (int* pData = _data)
+ {
+ byte* pDataBytes = (byte*)pData;
+ SumupManagedDense(input, histogram, pDataBytes, 32);
+ }
+ }
+ }
+
public override void Sumup(SumupInputData input, FeatureHistogram histogram) => SumupHandler(input, histogram);
}
}
diff --git a/src/Microsoft.ML.FastTree/Dataset/SegmentIntArray.cs b/src/Microsoft.ML.FastTree/Dataset/SegmentIntArray.cs
index eb70897fed..2515834900 100644
--- a/src/Microsoft.ML.FastTree/Dataset/SegmentIntArray.cs
+++ b/src/Microsoft.ML.FastTree/Dataset/SegmentIntArray.cs
@@ -73,7 +73,7 @@ public SegmentIntArray(int length, IEnumerable values)
{
using (Timer.Time(TimerEvent.SparseConstruction))
{
- SetupSumupHandler(SumupCPlusPlus, base.Sumup);
+ SetupSumupHandler(SumupCPlusPlus, SumupManaged);
uint[] vals = new uint[length];
uint pos = 0;
@@ -576,6 +576,106 @@ public unsafe void SumupCPlusPlus(SumupInputData input, FeatureHistogram histogr
}
}
}
+
+ ///
+ /// Managed equivalent of , used on platforms where the native
+ /// FastTree library is not available (e.g. arm64). This mirrors the native SumupSegment /
+ /// SumupSegment_noindices templates in src/Native/FastTreeNative/SumupSegment.h exactly,
+ /// including the segment bit-unpacking and accumulation order, so histogram results are
+ /// bit-identical to the native implementation. Reads go through fixed pointers to avoid the
+ /// per-element bounds checks and interface-indexer dispatch of the generic
+ /// fallback.
+ ///
+ public unsafe void SumupManaged(SumupInputData input, FeatureHistogram histogram)
+ {
+ // Note: timing is handled by the public Sumup override which wraps SumupHandler in
+ // Timer.Time(TimerEvent.SumupSegment); do not add a nested timer here or it double-counts.
+ fixed (FloatType* pSumTargetsByBin = histogram.SumTargetsByBin)
+ fixed (FloatType* pSampleOutputs = input.Outputs)
+ fixed (double* pSumWeightsByBin = histogram.SumWeightsByBin)
+ fixed (double* pSampleOutputWeights = input.Weights)
+ fixed (uint* pDataFixed = _data)
+ fixed (byte* pSegTypeFixed = _segType)
+ fixed (int* pSegLengthFixed = _segLength)
+ fixed (int* pIndicesFixed = input.DocIndices)
+ fixed (int* pCountByBin = histogram.CountByBin)
+ {
+ int count = input.TotalCount;
+
+ if (pIndicesFixed == null)
+ {
+ // Sequential (root) case: SumupSegment_noindices.
+ uint* pData = pDataFixed;
+ byte* pSegType = pSegTypeFixed;
+ int* pSegLength = pSegLengthFixed;
+
+ ulong workingBits = pData[0] | ((ulong)pData[1] << 32);
+ int bitsOffset = 0;
+ pData += 2;
+
+ int i = 0;
+ while (i < count)
+ {
+ int segEnd = *(pSegLength++);
+ int segType = *(pSegType++);
+ uint mask = (uint)(~((-1) << segType));
+
+ while (segEnd-- > 0)
+ {
+ int featureBin = (int)((workingBits >> bitsOffset) & mask);
+ pSumTargetsByBin[featureBin] += pSampleOutputs[i];
+ if (pSumWeightsByBin != null)
+ pSumWeightsByBin[featureBin] += pSampleOutputWeights[i];
+ ++pCountByBin[featureBin];
+ ++i;
+ bitsOffset += segType;
+ if (bitsOffset >= 32)
+ {
+ workingBits = (workingBits >> 32) | ((ulong)*(pData++) << 32);
+ bitsOffset &= 31;
+ }
+ }
+ }
+ }
+ else
+ {
+ // Leaf case with document indices: SumupSegment.
+ uint* pData = pDataFixed;
+ byte* pSegType = pSegTypeFixed;
+ int* pSegLength = pSegLengthFixed;
+ int* pIndices = pIndicesFixed;
+
+ long globalBitOffset = 0;
+ int currIndex = 0;
+ int segEnd = *(pSegLength++);
+ int nextIndex = segEnd;
+ int segType = *(pSegType++);
+ uint mask = (uint)(~((-1) << segType));
+
+ for (int i = 0; i < count; i++)
+ {
+ int index = *(pIndices++);
+ while (index >= nextIndex)
+ {
+ globalBitOffset += (long)segEnd * segType;
+ currIndex = nextIndex;
+ segEnd = *(pSegLength++);
+ nextIndex += segEnd;
+ segType = *(pSegType++);
+ mask = (uint)(~((-1) << segType));
+ }
+ long bitOffset = globalBitOffset + (long)(index - currIndex) * segType;
+ int major = (int)(bitOffset >> 5);
+ int minor = (int)(bitOffset & 0x1f);
+ int featureBin = (int)(((((ulong)pData[major]) >> minor) | (((ulong)pData[major + 1]) << (32 - minor))) & mask);
+ pSumTargetsByBin[featureBin] += pSampleOutputs[i];
+ if (pSumWeightsByBin != null)
+ pSumWeightsByBin[featureBin] += pSampleOutputWeights[i];
+ ++pCountByBin[featureBin];
+ }
+ }
+ }
+ }
public static void ManagedSegmentFindOptimalPath(uint[] array, int len, int bitsNeeded, out long bits, out int transitions)
{
uint max;