Misc changes

src/core/IronPython.Modules/_codecs.cs

@@ -335,12 +335,12 @@ public static PythonTuple unicode_internal_decode(CodeContext context, [NotNone]
         public static PythonTuple unicode_internal_decode(CodeContext context, [NotNone] IBufferProtocol input, string? errors = null) {
             PythonOps.Warn(context, PythonExceptions.DeprecationWarning, "unicode_internal codec has been deprecated");
             using IPythonBuffer buffer = input.GetBuffer();
-            return DoDecode(context, "unicode-internal", Encoding.Unicode, buffer, errors).ToPythonTuple();
+            return DoDecode(context, "unicode-internal", UnicodeInternalEncoding, buffer, errors).ToPythonTuple();
         }
 
         public static PythonTuple unicode_internal_encode(CodeContext context, [NotNone] string input, string? errors = null) {
             PythonOps.Warn(context, PythonExceptions.DeprecationWarning, "unicode_internal codec has been deprecated");
-            return DoEncode(context, "unicode-internal", Encoding.Unicode, input, errors, false).ToPythonTuple();
+            return DoEncode(context, "unicode-internal", UnicodeInternalEncoding, input, errors, false).ToPythonTuple();
         }
 
         public static PythonTuple unicode_internal_encode(CodeContext context, [NotNone] IBufferProtocol input, string? errors = null) {
@@ -350,6 +350,9 @@ public static PythonTuple unicode_internal_encode(CodeContext context, [NotNone]
             return PythonTuple.MakeTuple(bytes, bytes.Count);
         }
 
+        // native byte order of the internal UTF-16 string representation
+        private static Encoding UnicodeInternalEncoding => BitConverter.IsLittleEndian ? Utf16LeEncoding : Utf16BeEncoding;
+
         #endregion
 
         #region Utf-16 Functions

src/core/IronPython.Modules/_ctypes/CFuncPtrType.cs

@@ -39,7 +39,7 @@ public class CFuncPtrType : PythonType, INativeType {
             private Type _reverseDelegateType;
             private static Dictionary<DelegateCacheKey, Type> _reverseDelegates = new Dictionary<DelegateCacheKey, Type>();
 
-            //from_buffer_copy,  from_param, from_address, from_buffer, __doc__ __mul__ __rmul__ in_dll __new__ 
+            //from_buffer_copy,  from_param, from_address, from_buffer, __doc__ __mul__ __rmul__ in_dll __new__
             public CFuncPtrType(CodeContext/*!*/ context, string name, PythonTuple bases, PythonDictionary members)
                 : base(context, name, bases, members) {
 
@@ -109,7 +109,9 @@ object INativeType.GetValue(MemoryHolder owner, object readingFrom, int offset,
                     return funcAddr.ToPython();
                 }
 
-                return CreateInstance(Context.SharedContext, funcAddr);
+                _CFuncPtr res = (_CFuncPtr)CreateInstance(Context.SharedContext, funcAddr);
+                res.MemHolder.AddObject(offset, readingFrom); // TODO: CPython keeps the owner in _b_base_ instead
+                return res;
             }
 
             object INativeType.SetValue(MemoryHolder address, int offset, object value) {
@@ -193,7 +195,7 @@ internal Delegate MakeReverseDelegate(CodeContext/*!*/ context, object target) {
                 return _reverseDelegate.CreateDelegate(_reverseDelegateType, constantPool);
             }
 
-           
+
             private void MakeReverseDelegateWorker(CodeContext context) {
                 Type[] sigTypes;
                 Type[] callSiteType;
@@ -229,7 +231,7 @@ private void MakeReverseDelegateWorker(CodeContext context) {
                 // load code context
                 int contextIndex = constantPool.Count;
                 Debug.Assert(pc.SharedContext != null);
-                constantPool.Add(pc.SharedContext);                
+                constantPool.Add(pc.SharedContext);
                 ilGen.Emit(OpCodes.Ldarg_0);
                 ilGen.Emit(OpCodes.Ldc_I4, contextIndex);
                 ilGen.Emit(OpCodes.Ldelem_Ref);
@@ -260,7 +262,7 @@ private void MakeReverseDelegateWorker(CodeContext context) {
                     ilGen.Emit(OpCodes.Pop);
                 }
 
-                // } catch(Exception e) { 
+                // } catch(Exception e) {
                 // emit the cleanup code
 
                 ilGen.BeginCatchBlock(typeof(Exception));

src/core/IronPython/Runtime/Operations/StringOps.cs

@@ -1865,7 +1865,7 @@ internal static string DoDecode(CodeContext context, IPythonBuffer buffer, strin
             Debug.Assert(buffer.IsCContiguous());
 
             var span = buffer.AsReadOnlySpan();
-            int start = GetStartingOffset(span, e);
+            int start = GetStartingOffset(span, ref e);
             int length = (numBytes >= 0 ? numBytes : span.Length) - start;
 
             // CLR's encoder exceptions have a 1-1 mapping w/ Python's encoder exceptions
@@ -1914,9 +1914,25 @@ Encoding setFallback(Encoding enc, DecoderFallback fb) {
         /// <summary>
         /// Gets the starting offset checking to see if the incoming bytes already include a preamble.
         /// </summary>
-        private static int GetStartingOffset(ReadOnlySpan<byte> bytes, Encoding e) {
+        private static int GetStartingOffset(ReadOnlySpan<byte> bytes, ref Encoding e) {
             byte[] preamble = e.GetPreamble();
-            return preamble.Length > 0 && bytes.StartsWith(preamble) ? preamble.Length : 0;
+            if (preamble.Length == 0) return 0;
+            if (bytes.StartsWith(preamble)) return preamble.Length;
+
+            // utf-16 and utf-32 also accept a BOM of the opposite endianness
+            bool isLE = preamble[0] == 0xFF;
+            Encoding? swapped = e switch {
+                UnicodeEncoding => CodecsInfo.Codecs[isLE ? "utf_16be" : "utf_16le"].Value,
+                UTF32Encoding => CodecsInfo.Codecs[isLE ? "utf_32be" : "utf_32le"].Value,
+                _ => null,
+            };
+            if (swapped is null) return 0;
+
+            // the swapped encodings have no preamble, the opposite BOM is the reversed preamble
+            Array.Reverse(preamble);
+            if (!bytes.StartsWith(preamble)) return 0;
+            e = swapped;
+            return preamble.Length;
         }
 
         internal static Bytes RawEncode(CodeContext/*!*/ context, string s, string encoding, string? errors) {

tests/IronPython.Tests/Cases/CPythonCasesManifest.ini

@@ -891,9 +891,6 @@ Reason=Blocking
 [CPython.test_threaded_import]
 Ignore=true
 
-[CPython.test_threadedtempfile]
-RetryCount=2 # https://github.com/IronLanguages/ironpython3/issues/1063
-
 [CPython.test_threading]
 IsolationLevel=PROCESS
 RunCondition=NOT $(IS_MONO) # TODO: figure out

tests/suite/interop/net/field/test_instance_fields.py

@@ -2,6 +2,7 @@
 # The .NET Foundation licenses this file to you under the Apache 2.0 License.
 # See the LICENSE file in the project root for more information.
 
+import contextlib
 import unittest
 import warnings
 
@@ -15,6 +16,12 @@ def setUp(self):
         super(InstanceFieldsTest, self).setUp()
         self.add_clr_assemblies("fieldtests", "typesamples")
 
+    def _assertWarnsIfValueType(self, t):
+        import clr
+        if clr.GetClrType(t).IsValueType:
+            return self.assertWarnsRegex(RuntimeWarning, 'may result in updating a copy')
+        return contextlib.ExitStack() # no-op context manager
+
     def _test_get_by_instance(self, o):
         import System
         from Merlin.Testing.TypeSample import EnumInt64
@@ -203,7 +210,9 @@ def f24(): v.InstanceSimpleInterfaceField = ClassImplementSimpleInterface(78)
         def f1(): v.InstanceInt32Field = "abc"
         def f2(): v.InstanceEnumField = 3
 
-        for f in [f1, f2]: self.assertRaises(TypeError, f)
+        for f in [f1, f2]:
+            with self._assertWarnsIfValueType(t):
+                self.assertRaises(TypeError, f)
 
     def _test_set_by_type(self, o, vf, t):
         import clr
@@ -242,9 +251,7 @@ def f24(): t.InstanceSimpleInterfaceField.__set__(v, ClassImplementSimpleInterfa
 
         for i in range(1, 25):
             f = eval("f%s" % i)
-            if clr.GetClrType(t).IsValueType:
-                self.assertWarnsRegex(RuntimeWarning, 'may result in updating a copy', f)
-            else:
+            with self._assertWarnsIfValueType(t):
                 f()
 
         if not clr.GetClrType(t).IsValueType:
@@ -259,9 +266,12 @@ def f1(): t.InstanceInt16Field.__set__(v, "abc")
         def f2(): t.InstanceCharField.__set__(v, "abc")
         def f3(): t.InstanceEnumField.__set__(v, EnumInt32.B)
 
-        self.assertRaisesRegex(TypeError, "expected Int16, got str",  f1)
-        self.assertRaisesRegex(TypeError, "expected string of length 1 when converting to char, got 'abc'", f2)
-        self.assertRaisesRegex(TypeError, "expected EnumInt64, got EnumInt32",  f3)
+        with self._assertWarnsIfValueType(t):
+            self.assertRaisesRegex(TypeError, "expected Int16, got str",  f1)
+        with self._assertWarnsIfValueType(t):
+            self.assertRaisesRegex(TypeError, "expected string of length 1 when converting to char, got 'abc'", f2)
+        with self._assertWarnsIfValueType(t):
+            self.assertRaisesRegex(TypeError, "expected EnumInt64, got EnumInt32",  f3)
 
     def _test_set_by_descriptor(self, o, vf, t):
         import clr
@@ -301,9 +311,7 @@ def f24(): t.__dict__['InstanceSimpleInterfaceField'].__set__(v, ClassImplementS
 
         for i in range(1, 25):
             f = eval("f%s" % i)
-            if clr.GetClrType(t).IsValueType:
-                self.assertWarns(RuntimeWarning, f)
-            else:
+            with self._assertWarnsIfValueType(t):
                 f()
 
         if not clr.GetClrType(t).IsValueType:
@@ -473,7 +481,7 @@ def test_generic_fields(self):
         self.assertEqual(o.InstanceStructTField.Flag, None)
 
         def tryUpdate():
-            o.InstanceTField="abc"
+            o.InstanceTField = "abc"
 
         self.assertWarnsRegex(RuntimeWarning, 'may result in updating a copy', tryUpdate)
 

tests/suite/modules/type_related/test_ctypes.py

@@ -105,6 +105,15 @@ def test_from_bytes(self):
         self.assertEqual(c[:], b[1:])
         self.assertRaisesMessage(TypeError, self.readonly_error_msg, (c_char * 15).from_buffer, b, sizeof(c_char))
 
+    def test_funcptr_field_keepalive(self):
+        PROC = CFUNCTYPE(c_int, c_int)
+        class S(Structure):
+            _fields_ = [("p", PROC)]
+
+        f = S(PROC(lambda a: a)).p
+        gc.collect() # f must keep the struct alive, try to collect it
+        self.assertEqual(f(10), 10)
+
     def test_bitfield_bool(self):
         """
         Tests pertaining of bitfields of type c_bool

tests/suite/test_bytes.py

@@ -408,6 +408,14 @@ def test_count(self):
     def test_decode(self):
         for testType in types:
             self.assertEqual(testType(b'\xff\xfea\x00b\x00c\x00').decode('utf-16'), 'abc')
+            # https://github.com/IronLanguages/ironpython3/issues/1445
+            self.assertEqual(testType(b'\xfe\xff\x00a\x00b\x00c').decode('utf-16'), 'abc')
+            self.assertEqual(testType(b'\xff\xfe\x00\x00a\x00\x00\x00').decode('utf-32'), 'a')
+            self.assertEqual(testType(b'\x00\x00\xfe\xff\x00\x00\x00a').decode('utf-32'), 'a')
+            self.assertEqual(testType(b'\xfe\xff\x00a').decode('utf-16-le'), '\ufffe\u6100')
+            with warnings.catch_warnings():
+                warnings.simplefilter('ignore', DeprecationWarning)
+                self.assertEqual(testType(b'\xff\xfea\x00').decode('unicode_internal'), '\ufeffa')
 
     def test_endswith(self):
         for testType in types:

tests/suite/test_cliclass.py

@@ -1703,7 +1703,8 @@ def warnMethod():
     def test_ctor_field_assign_conversions(self):
         from IronPythonTest.BinderTest import ValueTypeWithFields
         res = ValueTypeWithFields(Y=42)
-        res.Y = 42
+        with self.assertWarnsRegex(RuntimeWarning, 'may result in updating a copy'):
+            res.Y = 42
         self.assertEqual(ValueTypeWithFields(Y=42), res)
 
         class myint(int): pass

tests/suite/test_isinstance.py

@@ -701,21 +701,24 @@ def test_mutable_Valuetypes(self):
         # But writes to value type fields should fail with ValueError.
         success = 0
         try:
-            direct_vt.width = 5
+            with self.assertWarnsRegex(RuntimeWarning, 'may result in updating a copy'):
+                direct_vt.width = 5
         except ValueError:
             success = 1
         self.assertTrue(success == 0 and direct_vt.width == 1)
 
         success = 0
         try:
-            embedded_vt.size.width = 5
+            with self.assertWarnsRegex(RuntimeWarning, 'may result in updating a copy'):
+                embedded_vt.size.width = 5
         except ValueError:
             success = 1
         self.assertTrue(success == 0 and embedded_vt.size.width == 3)
 
         success = 0
         try:
-            embedded_vt.Size.width = 5
+            with self.assertWarnsRegex(RuntimeWarning, 'may result in updating a copy'):
+                embedded_vt.Size.width = 5
         except ValueError:
             success = 1
         self.assertTrue(success == 0 and embedded_vt.Size.width == 3)