diff --git a/X10D.Tests/src/Core/SpanTest.cs b/X10D.Tests/src/Core/SpanTest.cs index 1c86feb..8209ee6 100644 --- a/X10D.Tests/src/Core/SpanTest.cs +++ b/X10D.Tests/src/Core/SpanTest.cs @@ -1,4 +1,9 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; +#if NET5_0_OR_GREATER +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; +#endif +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq.AutoMock; using X10D.Collections; using X10D.Core; @@ -144,93 +149,253 @@ public class SpanTest } [TestMethod] - public void Pack8Bit_Should_Pack_Correctly() + public void PackByteInternal_Fallback_ShouldReturnCorrectByte_GivenReadOnlySpan_Using() { - Span span = stackalloc bool[8] {true, true, false, false, true, true, false, false}; - Assert.AreEqual(0b00110011, span.PackByte()); + const byte expected = 0b00110011; + ReadOnlySpan span = stackalloc bool[8] {true, true, false, false, true, true, false, false}; + + byte actual = span.PackByteInternal_Fallback(); + + Assert.AreEqual(expected, actual); + } + +#if NET5_0_OR_GREATER + [TestMethod] + public void PackByteInternal_Sse2_ShouldReturnCorrectByte_GivenReadOnlySpan_Using() + { + if (!Sse2.IsSupported) + { + return; + } + + const byte expected = 0b00110011; + ReadOnlySpan span = stackalloc bool[8] {true, true, false, false, true, true, false, false}; + + byte actual = span.PackByteInternal_Sse2(); + + Assert.AreEqual(expected, actual); } [TestMethod] - public void Pack8Bit_Should_Pack_Correctly_Randomize() + public void PackByteInternal_AdvSimd_ShouldReturnCorrectByte_GivenReadOnlySpan_Using() { - var value = new Random().NextByte(); + if (!AdvSimd.IsSupported) + { + return; + } - Span unpacks = stackalloc bool[8]; + const byte expected = 0b00110011; + ReadOnlySpan span = stackalloc bool[8] {true, true, false, false, true, true, false, false}; - value.Unpack(unpacks); + byte actual = span.PackByteInternal_AdvSimd(); - Assert.AreEqual(value, unpacks.PackByte()); + Assert.AreEqual(expected, actual); + } +#endif + + [TestMethod] + public void PackInt16_ShouldReturnSameAsPackByte_WhenSpanHasLength8() + { + ReadOnlySpan span = stackalloc bool[8] {true, true, false, false, true, true, false, false}; + + short expected = span.PackByte(); + short actual = span.PackInt16(); + + Assert.AreEqual(expected, actual); } [TestMethod] - public void Pack16Bit_Should_Pack_Correctly() + public void PackInt16Internal_Fallback_ShouldReturnCorrectInt16_GivenReadOnlySpan() + { + const short expected = 0b00101101_11010100; + ReadOnlySpan span = stackalloc bool[16] + { + false, false, true, false, true, false, true, true, true, false, true, true, false, true, false, false, + }; + + short actual = span.PackInt16Internal_Fallback(); + + Assert.AreEqual(expected, actual); + } + +#if NET5_0_OR_GREATER + [TestMethod] + public void PackInt16Internal_Sse2_ShouldReturnCorrectInt16_GivenReadOnlySpan_Using() + { + if (!Sse2.IsSupported) + { + return; + } + + const short expected = 0b00101101_11010100; + ReadOnlySpan span = stackalloc bool[16] + { + false, false, true, false, true, false, true, true, true, false, true, true, false, true, false, false, + }; + + short actual = span.PackInt16Internal_Sse2(); + + Assert.AreEqual(expected, actual); + } +#endif + + [TestMethod] + public void PackInt32Internal_Fallback_ShouldReturnCorrectInt32_GivenReadOnlySpan() + { + const int expected = 0b01010101_10101010_01010101_10101010; + ReadOnlySpan span = stackalloc bool[32] + { + false, true, false, true, false, true, false, true, true, false, true, false, true, false, true, false, false, + true, false, true, false, true, false, true, true, false, true, false, true, false, true, false, + }; + + int actual = span.PackInt32Internal_Fallback(); + + Assert.AreEqual(expected, actual); + } + +#if NET5_0_OR_GREATER + [TestMethod] + public void PackInt32Internal_Sse2_ShouldReturnCorrectInt32_GivenReadOnlySpan() + { + if (!Sse2.IsSupported) + { + return; + } + + const int expected = 0b01010101_10101010_01010101_10101010; + ReadOnlySpan span = stackalloc bool[32] + { + false, true, false, true, false, true, false, true, true, false, true, false, true, false, true, false, false, + true, false, true, false, true, false, true, true, false, true, false, true, false, true, false, + }; + + int actual = span.PackInt32Internal_Sse2(); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void PackInt32Internal_Avx2_ShouldReturnCorrectInt32_GivenReadOnlySpan() + { + if (!Avx2.IsSupported) + { + return; + } + + const int expected = 0b01010101_10101010_01010101_10101010; + ReadOnlySpan span = stackalloc bool[32] + { + false, true, false, true, false, true, false, true, true, false, true, false, true, false, true, false, false, + true, false, true, false, true, false, true, true, false, true, false, true, false, true, false, + }; + + int actual = span.PackInt32Internal_Avx2(); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void PackInt32Internal_AdvSimd_ShouldReturnCorrectInt32_GivenReadOnlySpan() + { + if (!AdvSimd.IsSupported) + { + return; + } + + const int expected = 0b01010101_10101010_01010101_10101010; + ReadOnlySpan span = stackalloc bool[32] + { + false, true, false, true, false, true, false, true, true, false, true, false, true, false, true, false, false, + true, false, true, false, true, false, true, true, false, true, false, true, false, true, false, + }; + + int actual = span.PackInt32Internal_AdvSimd(); + + Assert.AreEqual(expected, actual); + } +#endif + + [TestMethod] + public void PackInt32_ShouldReturnSameAsPackByte_WhenSpanHasLength8() + { + ReadOnlySpan span = stackalloc bool[8] {true, true, false, false, true, true, false, false}; + + int expected = span.PackByte(); + int actual = span.PackInt32(); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void PackInt32_ShouldReturnSameAsPackInt16_WhenSpanHasLength16() { ReadOnlySpan span = stackalloc bool[16] { false, false, true, false, true, false, true, true, true, false, true, true, false, true, false, false, }; - Assert.AreEqual(0b00101101_11010100, span.PackInt16()); + + int expected = span.PackInt16(); + int actual = span.PackInt32(); + + Assert.AreEqual(expected, actual); } [TestMethod] - public void Pack16Bit_Should_Pack_Correctly_Randomize() + public void PackInt64_ShouldReturnCorrectInt64_GivenReadOnlySpan() { - var value = new Random().NextInt16(); - - Span unpacks = stackalloc bool[16]; - - value.Unpack(unpacks); - - Assert.AreEqual(value, unpacks.PackInt16()); - } - - [TestMethod] - public void Pack32Bit_Should_Pack_Correctly() - { - ReadOnlySpan span = stackalloc bool[] - { - false, true, false, true, false, true, false, true, true, false, true, false, true, false, true, false, false, - true, false, true, false, true, false, true, true, false, true, false, true, false, true, false, - }; - Assert.AreEqual(0b01010101_10101010_01010101_10101010, span.PackInt32()); - } - - [TestMethod] - public void Pack32Bit_Should_Pack_Correctly_Randomize() - { - var value = new Random().Next(int.MinValue, int.MaxValue); - - Span unpacks = stackalloc bool[32]; - - value.Unpack(unpacks); - - Assert.AreEqual(value, unpacks.PackInt32()); - } - - [TestMethod] - public void Pack64Bit_Should_Pack_Correctly() - { - ReadOnlySpan span = stackalloc bool[] + const long expected = 0b01010101_11010110_01101001_11010110_00010010_10010111_00101100_10100101; + ReadOnlySpan span = stackalloc bool[64] { true, false, true, false, false, true, false, true, false, false, true, true, false, true, false, false, true, true, true, false, true, false, false, true, false, true, false, false, true, false, false, false, false, true, true, false, true, false, true, true, true, false, false, true, false, true, true, false, false, true, true, false, true, false, true, true, true, false, true, false, true, false, true, false, }; - Assert.AreEqual(0b01010101_11010110_01101001_11010110_00010010_10010111_00101100_10100101, span.PackInt64()); + + long actual = span.PackInt64(); + + Assert.AreEqual(expected, actual); } [TestMethod] - public void Pack64Bit_Should_Pack_Correctly_Randomize() + public void PackInt64_ShouldReturnSameAsPackByte_WhenSpanHasLength8() { - var rand = new Random(); - long value = ((long)rand.Next() << 32) | (long)rand.Next(); + ReadOnlySpan span = stackalloc bool[8] {true, true, false, false, true, true, false, false}; - Span unpacks = stackalloc bool[64]; + long expected = span.PackByte(); + long actual = span.PackInt64(); - value.Unpack(unpacks); + Assert.AreEqual(expected, actual); + } - Assert.AreEqual(value, unpacks.PackInt64()); + [TestMethod] + public void PackInt64_ShouldReturnSameAsPackInt16_WhenSpanHasLength16() + { + ReadOnlySpan span = stackalloc bool[16] + { + false, false, true, false, true, false, true, true, true, false, true, true, false, true, false, false, + }; + + long expected = span.PackInt16(); + long actual = span.PackInt64(); + + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void PackInt64_ShouldReturnSameAsPackInt32_WhenSpanHasLength16() + { + ReadOnlySpan span = stackalloc bool[32] + { + false, true, false, true, false, true, false, true, true, false, true, false, true, false, true, false, false, + true, false, true, false, true, false, true, true, false, true, false, true, false, true, false, + }; + + long expected = span.PackInt32(); + long actual = span.PackInt64(); + + Assert.AreEqual(expected, actual); } private enum EnumByte : byte diff --git a/X10D/src/Core/SpanExtensions.cs b/X10D/src/Core/SpanExtensions.cs index bc5cb1d..505f294 100644 --- a/X10D/src/Core/SpanExtensions.cs +++ b/X10D/src/Core/SpanExtensions.cs @@ -23,12 +23,6 @@ public static class SpanExtensions #if NETCOREAPP3_0_OR_GREATER private const ulong IntegerPackingMagic = 0x0102040810204080; - [ExcludeFromCodeCoverage] - private static Vector64 IntegerPackingMagicV64 - { - get => Vector64.Create(IntegerPackingMagic); - } - [ExcludeFromCodeCoverage] private static Vector128 IntegerPackingMagicV128 { @@ -53,6 +47,7 @@ public static class SpanExtensions /// . /// /// The size of is unsupported. + [Pure] #if NETSTANDARD2_1 [MethodImpl(MethodImplOptions.AggressiveInlining)] #else @@ -75,10 +70,11 @@ public static class SpanExtensions /// . /// /// The size of is unsupported. -#if NETSTANDARD2_1 - [MethodImpl(MethodImplOptions.AggressiveInlining)] -#else + [Pure] +#if NETCOREAPP3_0_OR_GREATER [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] +#else + [MethodImpl(MethodImplOptions.AggressiveInlining)] #endif public static bool Contains(this ReadOnlySpan span, T value) where T : struct, Enum { @@ -153,7 +149,11 @@ public static class SpanExtensions /// An 8-bit unsigned integer containing the packed booleans. /// contains more than 8 elements. [Pure] +#if NETCOREAPP3_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] +#else [MethodImpl(MethodImplOptions.AggressiveInlining)] +#endif public static byte PackByte(this Span source) { return PackByte((ReadOnlySpan)source); @@ -166,57 +166,42 @@ public static class SpanExtensions /// An 8-bit unsigned integer containing the packed booleans. /// contains more than 8 elements. [Pure] - public static unsafe byte PackByte(this ReadOnlySpan source) - { - switch (source.Length) - { - case > 8: throw new ArgumentException("Source cannot contain more than 8 elements.", nameof(source)); - case 8: -#if NETSTANDARD2_1 - // TODO: Think of a way to do fast boolean correctness without using SIMD API. - goto default; +#if NETCOREAPP3_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] #else - // TODO: Acceleration in Big Endian environment. - if (!BitConverter.IsLittleEndian) - { - goto default; - } - - fixed (bool* pSource = source) - { - // TODO: .NET 8.0 Wasm support. - - if (Sse2.IsSupported) - { - Vector128 load = Sse2.LoadScalarVector128((ulong*)pSource).AsByte(); - - return unchecked((byte)(IntegerPackingMagic * load.CorrectBoolean().AsUInt64().GetElement(0) >> 56)); - } - - // Probably should remove this piece of code because it is untested, but I see no reason why it should fail - // unless vld1_u8 reverse positions of 8 bytes for some reason. - - if (AdvSimd.IsSupported) - { - // Hasn't been tested since March 6th 2023 (Reason: Unavailable hardware). - Vector64 load = AdvSimd.LoadVector64((byte*)pSource); - - return unchecked((byte)(IntegerPackingMagic * load.CorrectBoolean().AsUInt64().GetElement(0) >> 56)); - } - - goto default; - } + [MethodImpl(MethodImplOptions.AggressiveInlining)] #endif - default: - byte result = 0; - - for (var index = 0; index < source.Length; index++) - { - result |= (byte)(source[index] ? 1 << index : 0); - } - - return result; + [ExcludeFromCodeCoverage] + public static byte PackByte(this ReadOnlySpan source) + { + if (source.Length > 8) + { + throw new ArgumentException(ExceptionMessages.SourceSpanIsTooLarge, nameof(source)); } + + if (source.Length < 8) + { + return PackByteInternal_Fallback(source); + } + +#if NETCOREAPP3_0_OR_GREATER + if (!BitConverter.IsLittleEndian) + { + return PackByteInternal_Fallback(source); + } + + if (Sse2.IsSupported) + { + return PackByteInternal_Sse2(source); + } + + if (AdvSimd.IsSupported) + { + return PackByteInternal_AdvSimd(source); + } +#endif + + return PackByteInternal_Fallback(source); } /// @@ -239,52 +224,39 @@ public static class SpanExtensions /// A 16-bit signed integer containing the packed booleans. /// contains more than 16 elements. [Pure] - public static unsafe short PackInt16(this ReadOnlySpan source) +#if NETCOREAPP3_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] +#else + [MethodImpl(MethodImplOptions.AggressiveInlining)] +#endif + [ExcludeFromCodeCoverage] + public static short PackInt16(this ReadOnlySpan source) { switch (source.Length) { - case > 16: throw new ArgumentException("Source cannot contain more than than 16 elements.", nameof(source)); - case 8: return PackByte(source); // Potential optimization - + case > 16: + throw new ArgumentException(ExceptionMessages.SourceSpanIsTooLarge, nameof(source)); + case 8: + return PackByte(source); case 16: -#if NETSTANDARD2_1 - // TODO: Think of a way to do fast boolean correctness without using SIMD API. - goto default; -#else - // TODO: Acceleration in Big Endian environment. if (!BitConverter.IsLittleEndian) { goto default; } - // TODO: AdvSimd implementation. - // TODO: WasmSimd implementation. - +#if NETCOREAPP3_0_OR_GREATER if (Sse2.IsSupported) { - fixed (bool* pSource = source) - { - Vector128 load = Sse2.LoadVector128((byte*)pSource); - Vector128 correct = load.CorrectBoolean().AsUInt64(); - Vector128 multiply = IntrinsicUtility.Multiply(IntegerPackingMagicV128, correct); - Vector128 shift = Sse2.ShiftRightLogical(multiply, 56); - - return (short)(shift.GetElement(0) | (shift.GetElement(1) << 8)); - } + return PackInt16Internal_Sse2(source); } - - goto default; #endif + goto default; + case < 16: + return PackInt16Internal_Fallback(source); + default: - short result = 0; - - for (var index = 0; index < source.Length; index++) - { - result |= (short)(source[index] ? 1 << index : 0); - } - - return result; + return PackInt16Internal_Fallback(source); } } @@ -295,7 +267,11 @@ public static class SpanExtensions /// A 32-bit signed integer containing the packed booleans. /// contains more than 32 elements. [Pure] +#if NETCOREAPP3_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] +#else [MethodImpl(MethodImplOptions.AggressiveInlining)] +#endif public static int PackInt32(this Span source) { return PackInt32((ReadOnlySpan)source); @@ -308,99 +284,51 @@ public static class SpanExtensions /// A 32-bit signed integer containing the packed booleans. /// contains more than 32 elements. [Pure] - public static unsafe int PackInt32(this ReadOnlySpan source) +#if NETCOREAPP3_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] +#else + [MethodImpl(MethodImplOptions.AggressiveInlining)] +#endif + [ExcludeFromCodeCoverage] + public static int PackInt32(this ReadOnlySpan source) { switch (source.Length) { - case > 32: throw new ArgumentException("Source cannot contain more than than 32 elements.", nameof(source)); - case 8: return PackByte(source); - case 16: return PackInt16(source); + case > 32: + throw new ArgumentException(ExceptionMessages.SourceSpanIsTooLarge, nameof(source)); + + case 8: + return PackByte(source); + + case 16: + return PackInt16(source); case 32: -#if NETSTANDARD2_1 - // TODO: Think of a way to do fast boolean correctness without using SIMD API. - goto default; -#else - // TODO: Acceleration in Big Endian environment. +#if NETCOREAPP3_0_OR_GREATER if (!BitConverter.IsLittleEndian) { goto default; } - fixed (bool* pSource = source) + if (Avx2.IsSupported) { - if (Avx2.IsSupported) - { - Vector256 load = Avx.LoadVector256((byte*)pSource); - Vector256 correct = load.CorrectBoolean().AsUInt64(); + return PackInt32Internal_Avx2(source); + } - Vector256 multiply = IntrinsicUtility.Multiply(IntegerPackingMagicV256, correct); - Vector256 shift = Avx2.ShiftRightLogical(multiply, 56); - shift = Avx2.ShiftLeftLogicalVariable(shift, Vector256.Create(0UL, 8, 16, 24)); + if (Sse2.IsSupported) + { + return PackInt32Internal_Sse2(source); + } - Vector256 p1 = Avx2.Permute4x64(shift, 0b10_11_00_01); - Vector256 or1 = Avx2.Or(shift, p1); - Vector256 p2 = Avx2.Permute4x64(or1, 0b00_00_10_10); - Vector256 or2 = Avx2.Or(or1, p2); - - return (int)or2.GetElement(0); - } - - if (Sse2.IsSupported) - { - Vector128 load = Sse2.LoadVector128((byte*)pSource); - Vector128 correct = load.CorrectBoolean().AsUInt64(); - - Vector128 multiply = IntrinsicUtility.Multiply(IntegerPackingMagicV128, correct); - Vector128 shift1 = Sse2.ShiftRightLogical(multiply, 56); - shift1 = Sse2.ShiftLeftLogical(shift1, Vector128.Create(0UL, 8UL)); - - load = Sse2.LoadVector128((byte*)(pSource + 16)); - correct = load.CorrectBoolean().AsUInt64(); - - multiply = IntrinsicUtility.Multiply(IntegerPackingMagicV128, correct); - Vector128 shift2 = Sse2.ShiftRightLogical(multiply, 56); - shift2 = Sse2.ShiftLeftLogical(shift2, Vector128.Create(16UL, 24UL)); - - Vector128 or1 = Sse2.Or(shift1, shift2); - Vector128 or2 = Sse2.Or(or1, or1.ReverseElements()); - - return (int)or2.GetElement(0); - } - - if (AdvSimd.IsSupported) - { - // Hasn't been tested since March 6th 2023 (Reason: Unavailable hardware). - Vector128 vector1 = AdvSimd.LoadVector128((byte*)pSource).CorrectBoolean().AsUInt64(); - Vector128 vector2 = AdvSimd.LoadVector128((byte*)(pSource + 16)).CorrectBoolean().AsUInt64(); - - Vector128 calc1 = IntrinsicUtility.Multiply(IntegerPackingMagicV128, vector1); - Vector128 calc2 = IntrinsicUtility.Multiply(IntegerPackingMagicV128, vector2); - - calc1 = AdvSimd.ShiftRightLogical(calc1, 56); - calc2 = AdvSimd.ShiftRightLogical(calc2, 56); - - Vector128 shift1 = AdvSimd.ShiftLogical(calc1, Vector128.Create(0, 8)); - Vector128 shift2 = AdvSimd.ShiftLogical(calc2, Vector128.Create(16, 24)); - - return (int)(shift1.GetElement(0) | shift1.GetElement(1) | shift2.GetElement(0) | shift2.GetElement(1)); - } - else - { - goto default; - } + if (AdvSimd.IsSupported) + { + return PackInt32Internal_AdvSimd(source); } #endif + goto default; default: - int result = 0; - - for (var i = 0; i < source.Length; i++) - { - result |= source[i] ? 1 << i : 0; - } - - return result; + return PackInt32Internal_Fallback(source); } } @@ -411,7 +339,11 @@ public static class SpanExtensions /// A 64-bit signed integer containing the packed booleans. /// contains more than 64 elements. [Pure] +#if NETCOREAPP3_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] +#else [MethodImpl(MethodImplOptions.AggressiveInlining)] +#endif public static long PackInt64(this Span source) { return PackInt64((ReadOnlySpan)source); @@ -424,17 +356,21 @@ public static class SpanExtensions /// A 64-bit signed integer containing the packed booleans. /// contains more than 64 elements. [Pure] - public static unsafe long PackInt64(this ReadOnlySpan source) +#if NETCOREAPP3_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] +#else + [MethodImpl(MethodImplOptions.AggressiveInlining)] +#endif + public static long PackInt64(this ReadOnlySpan source) { switch (source.Length) { - case > 64: throw new ArgumentException("Source cannot contain more than than 64 elements.", nameof(source)); + case > 64: throw new ArgumentException(ExceptionMessages.SourceSpanIsTooLarge, nameof(source)); case 8: return PackByte(source); case 16: return PackInt16(source); case 32: return PackInt32(source); - case 64: - // TODO: Reimplement when Vector512 is in standard API. - return (long)PackInt32(source[..32]) | ((long)PackInt32(source[32..]) << 32); + // ReSharper disable once RedundantCast + case 64: return (long)PackInt32(source[..32]) | ((long)PackInt32(source[32..]) << 32); default: long result = 0; @@ -447,4 +383,194 @@ public static class SpanExtensions return result; } } + + [Pure] +#if NETCOREAPP3_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] +#else + [MethodImpl(MethodImplOptions.AggressiveInlining)] +#endif + internal static byte PackByteInternal_Fallback(this ReadOnlySpan source) + { + byte result = 0; + + for (var index = 0; index < source.Length; index++) + { + result |= (byte)(source[index] ? 1 << index : 0); + } + + return result; + } + + [Pure] +#if NETCOREAPP3_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] +#else + [MethodImpl(MethodImplOptions.AggressiveInlining)] +#endif + internal static short PackInt16Internal_Fallback(this ReadOnlySpan source) + { + short result = 0; + + for (var index = 0; index < source.Length; index++) + { + result |= (short)(source[index] ? 1 << index : 0); + } + + return result; + } + + [Pure] +#if NETCOREAPP3_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] +#else + [MethodImpl(MethodImplOptions.AggressiveInlining)] +#endif + internal static int PackInt32Internal_Fallback(this ReadOnlySpan source) + { + var result = 0; + + for (var index = 0; index < source.Length; index++) + { + result |= source[index] ? 1 << index : 0; + } + + return result; + } + +#if NETCOREAPP3_0_OR_GREATER + [Pure] + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + internal static byte PackByteInternal_Sse2(this ReadOnlySpan source) + { + unsafe + { + fixed (bool* pSource = source) + { + Vector128 load = Sse2.LoadScalarVector128((ulong*)pSource).AsByte(); + return unchecked((byte)(IntegerPackingMagic * load.CorrectBoolean().AsUInt64().GetElement(0) >> 56)); + } + } + } + + [Pure] + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + internal static short PackInt16Internal_Sse2(this ReadOnlySpan source) + { + unsafe + { + fixed (bool* pSource = source) + { + Vector128 load = Sse2.LoadVector128((byte*)pSource); + Vector128 correct = load.CorrectBoolean().AsUInt64(); + Vector128 multiply = IntrinsicUtility.Multiply(IntegerPackingMagicV128, correct); + Vector128 shift = Sse2.ShiftRightLogical(multiply, 56); + + return (short)(shift.GetElement(0) | (shift.GetElement(1) << 8)); + } + } + } + + [Pure] + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + internal static int PackInt32Internal_AdvSimd(this ReadOnlySpan source) + { + unsafe + { + fixed (bool* pSource = source) + { + // TODO Hasn't been tested since March 6th 2023 (Reason: Unavailable hardware). + Vector128 vector1 = AdvSimd.LoadVector128((byte*)pSource).CorrectBoolean().AsUInt64(); + Vector128 vector2 = AdvSimd.LoadVector128((byte*)(pSource + 16)).CorrectBoolean().AsUInt64(); + + Vector128 calc1 = IntrinsicUtility.Multiply(IntegerPackingMagicV128, vector1); + Vector128 calc2 = IntrinsicUtility.Multiply(IntegerPackingMagicV128, vector2); + + calc1 = AdvSimd.ShiftRightLogical(calc1, 56); + calc2 = AdvSimd.ShiftRightLogical(calc2, 56); + + Vector128 shift1 = AdvSimd.ShiftLogical(calc1, Vector128.Create(0, 8)); + Vector128 shift2 = AdvSimd.ShiftLogical(calc2, Vector128.Create(16, 24)); + + return (int)(shift1.GetElement(0) | shift1.GetElement(1) | shift2.GetElement(0) | shift2.GetElement(1)); + } + } + } + + [Pure] + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + internal static int PackInt32Internal_Avx2(this ReadOnlySpan source) + { + unsafe + { + fixed (bool* pSource = source) + { + Vector256 load = Avx.LoadVector256((byte*)pSource); + Vector256 correct = load.CorrectBoolean().AsUInt64(); + + Vector256 multiply = IntrinsicUtility.Multiply(IntegerPackingMagicV256, correct); + Vector256 shift = Avx2.ShiftRightLogical(multiply, 56); + shift = Avx2.ShiftLeftLogicalVariable(shift, Vector256.Create(0UL, 8, 16, 24)); + + Vector256 p1 = Avx2.Permute4x64(shift, 0b10_11_00_01); + Vector256 or1 = Avx2.Or(shift, p1); + Vector256 p2 = Avx2.Permute4x64(or1, 0b00_00_10_10); + Vector256 or2 = Avx2.Or(or1, p2); + + return (int)or2.GetElement(0); + } + } + } + + [Pure] + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + internal static int PackInt32Internal_Sse2(this ReadOnlySpan source) + { + unsafe + { + fixed (bool* pSource = source) + { + Vector128 load = Sse2.LoadVector128((byte*)pSource); + Vector128 correct = load.CorrectBoolean().AsUInt64(); + + Vector128 multiply = IntrinsicUtility.Multiply(IntegerPackingMagicV128, correct); + Vector128 shift1 = Sse2.ShiftRightLogical(multiply, 56); + shift1 = Sse2.ShiftLeftLogical(shift1, Vector128.Create(0UL, 8UL)); + + load = Sse2.LoadVector128((byte*)(pSource + 16)); + correct = load.CorrectBoolean().AsUInt64(); + + multiply = IntrinsicUtility.Multiply(IntegerPackingMagicV128, correct); + Vector128 shift2 = Sse2.ShiftRightLogical(multiply, 56); + shift2 = Sse2.ShiftLeftLogical(shift2, Vector128.Create(16UL, 24UL)); + + Vector128 or1 = Sse2.Or(shift1, shift2); + Vector128 or2 = Sse2.Or(or1, or1.ReverseElements()); + + return (int)or2.GetElement(0); + } + } + } + +#if NET5_0_OR_GREATER + // dotcover disable + //NOSONAR + [Pure] + [MethodImpl(MethodImplOptions.AggressiveInlining | MethodImplOptions.AggressiveOptimization)] + internal static byte PackByteInternal_AdvSimd(this ReadOnlySpan source) + { + unsafe + { + fixed (bool* pSource = source) + { + // TODO Hasn't been tested since March 6th 2023 (Reason: Unavailable hardware). + Vector64 load = AdvSimd.LoadVector64((byte*)pSource); + return unchecked((byte)(IntegerPackingMagic * load.CorrectBoolean().AsUInt64().GetElement(0) >> 56)); + } + } + } + //NOSONAR + // dotcover enable +#endif +#endif } diff --git a/X10D/src/ExceptionMessages.Designer.cs b/X10D/src/ExceptionMessages.Designer.cs index 704181c..208b385 100644 --- a/X10D/src/ExceptionMessages.Designer.cs +++ b/X10D/src/ExceptionMessages.Designer.cs @@ -176,6 +176,15 @@ namespace X10D { } } + /// + /// Looks up a localized string similar to The source contains too many elements.. + /// + internal static string SourceSpanIsTooLarge { + get { + return ResourceManager.GetString("SourceSpanIsTooLarge", resourceCulture); + } + } + /// /// Looks up a localized string similar to The stream does not support reading.. /// diff --git a/X10D/src/ExceptionMessages.resx b/X10D/src/ExceptionMessages.resx index 480548d..1b91b9b 100644 --- a/X10D/src/ExceptionMessages.resx +++ b/X10D/src/ExceptionMessages.resx @@ -153,6 +153,9 @@ Length must be greater than or equal to 0. + + The source contains too many elements. + The stream does not support reading.