C# endianness fixes (#1964)

crates/bindings-csharp/BSATN.Runtime.Tests/BSATN.Runtime.Tests.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <TargetFrameworks>net8.0</TargetFrameworks>
+    <RootNamespace>SpacetimeDB</RootNamespace>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="CsCheck" Version="4.1.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />
+    <PackageReference Include="xunit" Version="2.9.0" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.1" PrivateAssets="all" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="../BSATN.Runtime/BSATN.Runtime.csproj" />
+  </ItemGroup>
+
+</Project>

crates/bindings-csharp/BSATN.Runtime.Tests/Tests.cs

@@ -0,0 +1,142 @@
+namespace SpacetimeDB;
+
+using CsCheck;
+using Xunit;
+
+public static class BSATNRuntimeTests
+{
+    [Fact]
+    public static void AddressRoundtrips()
+    {
+        var str = "00112233445566778899AABBCCDDEEFF";
+        var addr = Address.FromHexString(str);
+
+        Assert.NotNull(addr);
+        Assert.Equal(addr.ToString(), str);
+
+        var bytes = Convert.FromHexString(str);
+
+        var addr2 = Address.FromBigEndian(bytes);
+        Assert.Equal(addr2, addr);
+
+        Array.Reverse(bytes);
+        var addr3 = Address.From(bytes);
+        Assert.Equal(addr3, addr);
+
+        var memoryStream = new MemoryStream();
+        var bsatn = new Address.BSATN();
+        using (var writer = new BinaryWriter(memoryStream))
+        {
+            if (addr is { } addrNotNull)
+            {
+                bsatn.Write(writer, addrNotNull);
+            }
+            else
+            {
+                Assert.Fail("Impossible");
+            }
+        }
+
+        var littleEndianBytes = memoryStream.ToArray();
+        var reader = new BinaryReader(new MemoryStream(littleEndianBytes));
+        var addr4 = bsatn.Read(reader);
+        Assert.Equal(addr4, addr);
+
+        // Note: From = FromLittleEndian
+        var addr5 = Address.From(littleEndianBytes);
+        Assert.Equal(addr5, addr);
+    }
+
+    static readonly Gen<string> genHex = Gen.String[Gen.Char["0123456789abcdef"], 0, 128];
+
+    [Fact]
+    public static void AddressLengthCheck()
+    {
+        genHex.Sample(s =>
+        {
+            if (s.Length == 32)
+            {
+                return;
+            }
+            Assert.ThrowsAny<Exception>(() => Address.FromHexString(s));
+        });
+        Gen.Byte.Array[0, 64]
+            .Sample(arr =>
+            {
+                if (arr.Length == 16)
+                {
+                    return;
+                }
+                Assert.ThrowsAny<Exception>(() => Address.FromBigEndian(arr));
+                Assert.ThrowsAny<Exception>(() => Address.From(arr));
+            });
+    }
+
+    [Fact]
+    public static void IdentityRoundtrips()
+    {
+        var str = "00112233445566778899AABBCCDDEEFF00112233445566778899AABBCCDDEEFF";
+        var ident = Identity.FromHexString(str);
+
+        Assert.Equal(ident.ToString(), str);
+
+        // We can't use this in the implementation because it isn't available
+        // in Unity's .NET. But we can use it in tests.
+        var bytes = Convert.FromHexString(str);
+
+        var ident2 = Identity.FromBigEndian(bytes);
+        Assert.Equal(ident2, ident);
+
+        Array.Reverse(bytes);
+        var ident3 = Identity.From(bytes);
+        Assert.Equal(ident3, ident);
+
+        var memoryStream = new MemoryStream();
+        var bsatn = new Identity.BSATN();
+        using (var writer = new BinaryWriter(memoryStream))
+        {
+            bsatn.Write(writer, ident);
+        }
+
+        var littleEndianBytes = memoryStream.ToArray();
+        var reader = new BinaryReader(new MemoryStream(littleEndianBytes));
+        var ident4 = bsatn.Read(reader);
+        Assert.Equal(ident4, ident);
+
+        // Note: From = FromLittleEndian
+        var ident5 = Identity.From(littleEndianBytes);
+        Assert.Equal(ident5, ident);
+    }
+
+    [Fact]
+    public static void IdentityLengthCheck()
+    {
+        genHex.Sample(s =>
+        {
+            if (s.Length == 64)
+            {
+                return;
+            }
+            Assert.ThrowsAny<Exception>(() => Identity.FromHexString(s));
+        });
+        Gen.Byte.Array[0, 64]
+            .Sample(arr =>
+            {
+                if (arr.Length == 32)
+                {
+                    return;
+                }
+                Assert.ThrowsAny<Exception>(() => Identity.FromBigEndian(arr));
+                Assert.ThrowsAny<Exception>(() => Identity.From(arr));
+            });
+    }
+
+    [Fact]
+    public static void NonHexStrings()
+    {
+        // n.b. 32 chars long
+        Assert.ThrowsAny<Exception>(
+            () => Address.FromHexString("these are not hex characters....")
+        );
+    }
+}

crates/bindings-csharp/BSATN.Runtime/Builtins.cs

@@ -8,11 +8,144 @@ using SpacetimeDB.Internal;
 
 internal static class Util
 {
-    // Same as `Convert.ToHexString`, but that method is not available in .NET Standard
-    // which we need to target for Unity support.
-    public static string ToHex<T>(T val)
-        where T : struct =>
-        BitConverter.ToString(MemoryMarshal.AsBytes([val]).ToArray()).Replace("-", "");
+    /// <summary>
+    /// Convert this object to a BIG-ENDIAN hex string.
+    ///
+    /// Big endian is almost always the correct convention here. It puts the most significant bytes
+    /// of the number at the lowest indexes of the resulting string; assuming the string is printed
+    /// with low indexes to the left, this will result in the correct hex number being displayed.
+    ///
+    /// (This might be wrong if the string is printed after, say, a unicode right-to-left marker.
+    /// But, well, what can you do.)
+    ///
+    /// Similar to `Convert.ToHexString`, but that method is not available in .NET Standard
+    /// which we need to target for Unity support.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    /// <param name="val"></param>
+    /// <returns></returns>
+    public static string ToHexBigEndian<T>(T val)
+        where T : struct => BitConverter.ToString(AsBytesBigEndian(val).ToArray()).Replace("-", "");
+
+    /// <summary>
+    /// Read a value of type T from the passed span, which is assumed to be in little-endian format.
+    /// The behavior of this method is independent of the endianness of the host, unlike MemoryMarshal.Read.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    /// <param name="source"></param>
+    /// <returns></returns>
+    public static T ReadLittleEndian<T>(ReadOnlySpan<byte> source)
+        where T : struct => Read<T>(source, !BitConverter.IsLittleEndian);
+
+    /// <summary>
+    /// Read a value of type T from the passed span, which is assumed to be in big-endian format.
+    /// The behavior of this method is independent of the endianness of the host, unlike MemoryMarshal.Read.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    /// <param name="source"></param>
+    /// <returns></returns>
+    public static T ReadBigEndian<T>(ReadOnlySpan<byte> source)
+        where T : struct => Read<T>(source, BitConverter.IsLittleEndian);
+
+    /// <summary>
+    /// Convert the passed byte array to a value of type T, optionally reversing it before performing the conversion.
+    /// If the input is not reversed, it is treated as having the native endianness of the host system.
+    /// (The endianness of the host system can be checked via System.BitConverter.IsLittleEndian.)
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    /// <param name="source"></param>
+    /// <param name="reverse"></param>
+    /// <returns></returns>
+    static T Read<T>(ReadOnlySpan<byte> source, bool reverse)
+        where T : struct
+    {
+        Debug.Assert(
+            source.Length == Marshal.SizeOf<T>(),
+            $"Error while reading ${typeof(T).FullName}: expected source span to be {Marshal.SizeOf<T>()} bytes long, but was {source.Length} bytes."
+        );
+
+        var result = MemoryMarshal.Read<T>(source);
+
+        if (reverse)
+        {
+            var resultSpan = MemoryMarshal.CreateSpan(ref result, 1);
+            MemoryMarshal.AsBytes(resultSpan).Reverse();
+        }
+
+        return result;
+    }
+
+    /// <summary>
+    /// Convert the passed T to a little-endian byte array.
+    /// The behavior of this method is independent of the endianness of the host, unlike MemoryMarshal.Read.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    /// <param name="source"></param>
+    /// <returns></returns>
+    public static byte[] AsBytesLittleEndian<T>(T source)
+        where T : struct => AsBytes(source, !BitConverter.IsLittleEndian);
+
+    /// <summary>
+    /// Convert the passed T to a big-endian byte array.
+    /// The behavior of this method is independent of the endianness of the host, unlike MemoryMarshal.Read.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    /// <param name="source"></param>
+    /// <returns></returns>
+    public static byte[] AsBytesBigEndian<T>(T source)
+        where T : struct => AsBytes<T>(source, BitConverter.IsLittleEndian);
+
+    /// <summary>
+    /// Convert the passed T to a byte array, and optionally reverse the array before returning it.
+    /// If the output is not reversed, it will have the native endianness of the host system.
+    /// (The endianness of the host system can be checked via System.BitConverter.IsLittleEndian.)
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    /// <param name="source"></param>
+    /// <param name="reverse"></param>
+    /// <returns></returns>
+    static byte[] AsBytes<T>(T source, bool reverse)
+        where T : struct
+    {
+        var result = MemoryMarshal.AsBytes([source]).ToArray();
+        if (reverse)
+        {
+            Array.Reverse(result, 0, result.Length);
+        }
+        return result;
+    }
+
+    /// <summary>
+    /// Convert a hex string to a byte array.
+    /// </summary>
+    /// <param name="hex"></param>
+    /// <returns></returns>
+    public static byte[] StringToByteArray(string hex)
+    {
+        Debug.Assert(
+            hex.Length % 2 == 0,
+            $"Expected input string (\"{hex}\") to be of even length"
+        );
+
+        var NumberChars = hex.Length;
+        var bytes = new byte[NumberChars / 2];
+        for (var i = 0; i < NumberChars; i += 2)
+        {
+            bytes[i / 2] = Convert.ToByte(hex.Substring(i, 2), 16);
+        }
+        return bytes;
+    }
+
+    /// <summary>
+    /// Read a value from a "big-endian" hex string.
+    /// All hex strings we expect to encounter are big-endian (store most significant bytes
+    /// at low indexes) so this should always be used.
+    /// </summary>
+    /// <typeparam name="T"></typeparam>
+    /// <param name="hex"></param>
+    /// <returns></returns>
+    public static T ReadFromBigEndianHexString<T>(string hex)
+        where T : struct => ReadBigEndian<T>(StringToByteArray(hex));
 }
 
 public readonly partial struct Unit
@@ -35,10 +168,48 @@ public readonly record struct Address
 
     internal Address(U128 v) => value = v;
 
+    /// <summary>
+    /// Create an Address from a LITTLE-ENDIAN byte array.
+    ///
+    /// If you are parsing an Address from a string, you probably want FromHexString instead,
+    /// or, failing that, FromBigEndian.
+    ///
+    /// Returns null if the resulting address is the default.
+    /// </summary>
+    /// <param name="bytes"></param>
     public static Address? From(byte[] bytes)
     {
-        Debug.Assert(bytes.Length == 16);
-        var addr = new Address(MemoryMarshal.Read<U128>(bytes));
+        var addr = new Address(Util.ReadLittleEndian<U128>(bytes));
+        return addr == default ? null : addr;
+    }
+
+    /// <summary>
+    /// Create an Address from a BIG-ENDIAN byte array.
+    ///
+    /// This method is the correct choice if you have converted the bytes of a hexadecimal-formatted Address
+    /// to a byte array in the following way:
+    ///
+    /// "0xb0b1b2..."
+    /// ->
+    /// [0xb0, 0xb1, 0xb2, ...]
+    ///
+    /// Returns null if the resulting address is the default.
+    /// </summary>
+    /// <param name="bytes"></param>
+    public static Address? FromBigEndian(byte[] bytes)
+    {
+        var addr = new Address(Util.ReadBigEndian<U128>(bytes));
+        return addr == default ? null : addr;
+    }
+
+    /// <summary>
+    /// Create an Address from a hex string.
+    /// </summary>
+    /// <param name="hex"></param>
+    /// <returns></returns>
+    public static Address? FromHexString(string hex)
+    {
+        var addr = new Address(Util.ReadFromBigEndianHexString<U128>(hex));
         return addr == default ? null : addr;
     }
 
@@ -62,7 +233,7 @@ public readonly record struct Address
             new AlgebraicType.Product([new("__address__", new AlgebraicType.U128(default))]);
     }
 
-    public override string ToString() => Util.ToHex(value);
+    public override string ToString() => Util.ToHexBigEndian(value);
 }
 
 public readonly record struct Identity
@@ -71,14 +242,51 @@ public readonly record struct Identity
 
     internal Identity(U256 val) => value = val;
 
+    /// <summary>
+    /// Create an Identity from a LITTLE-ENDIAN byte array.
+    ///
+    /// If you are parsing an Identity from a string, you probably want FromHexString instead,
+    /// or, failing that, FromBigEndian.
+    /// </summary>
+    /// <param name="bytes"></param>
     public Identity(byte[] bytes)
     {
-        Debug.Assert(bytes.Length == 32);
-        value = MemoryMarshal.Read<U256>(bytes);
+        value = Util.ReadLittleEndian<U256>(bytes);
     }
 
+    /// <summary>
+    /// Create an Identity from a LITTLE-ENDIAN byte array.
+    ///
+    /// If you are parsing an Identity from a string, you probably want FromHexString instead,
+    /// or, failing that, FromBigEndian.
+    /// </summary>
+    /// <param name="bytes"></param>
     public static Identity From(byte[] bytes) => new(bytes);
 
+    /// <summary>
+    /// Create an Identity from a BIG-ENDIAN byte array.
+    ///
+    /// This method is the correct choice if you have converted the bytes of a hexadecimal-formatted `Identity`
+    /// to a byte array in the following way:
+    ///
+    /// "0xb0b1b2..."
+    /// ->
+    /// [0xb0, 0xb1, 0xb2, ...]
+    /// </summary>
+    /// <param name="bytes"></param>
+    public static Identity FromBigEndian(byte[] bytes)
+    {
+        return new Identity(Util.ReadBigEndian<U256>(bytes));
+    }
+
+    /// <summary>
+    /// Create an Identity from a hex string.
+    /// </summary>
+    /// <param name="hex"></param>
+    /// <returns></returns>
+    public static Identity FromHexString(string hex) =>
+        new Identity(Util.ReadFromBigEndianHexString<U256>(hex));
+
     public readonly struct BSATN : IReadWrite<Identity>
     {
         public Identity Read(BinaryReader reader) => new(new SpacetimeDB.BSATN.U256().Read(reader));
@@ -91,7 +299,7 @@ public readonly record struct Identity
     }
 
     // This must be explicitly forwarded to base, otherwise record will generate a new implementation.
-    public override string ToString() => Util.ToHex(value);
+    public override string ToString() => Util.ToHexBigEndian(value);
 }
 
 // [SpacetimeDB.Type] - we have custom representation of time in microseconds, so implementing BSATN manually

crates/bindings-csharp/SpacetimeSharpSATS.sln

@@ -28,6 +28,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "sdk-tests", "sdk-tests", "{
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "benchmarks-cs", "..\..\modules\benchmarks-cs\benchmarks-cs.csproj", "{50E1AAE1-C42C-4C2F-B708-5190B0362165}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BSATN.Runtime.Tests", "BSATN.Runtime.Tests\BSATN.Runtime.Tests.csproj", "{FCF18E21-FB59-4A4D-A9ED-B85D2874E536}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -62,6 +64,10 @@ Global
 		{FDACD960-168E-44F9-B036-2E29EA391BE7}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{50E1AAE1-C42C-4C2F-B708-5190B0362165}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{50E1AAE1-C42C-4C2F-B708-5190B0362165}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FCF18E21-FB59-4A4D-A9ED-B85D2874E536}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{FCF18E21-FB59-4A4D-A9ED-B85D2874E536}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{FCF18E21-FB59-4A4D-A9ED-B85D2874E536}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{FCF18E21-FB59-4A4D-A9ED-B85D2874E536}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE