From c05f5784f6d2c8bfbc16302b3fd2d8ae4ee39894 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 11 Oct 2023 09:32:09 +0200 Subject: [PATCH 01/21] Reset version to dev --- CHANGELOG.md | 11 ++++++++++- version.txt | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 411356775..fdab9bf64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ project adheres to [Semantic Versioning][]. This document follows the conventions laid out in [Keep a CHANGELOG][]. +## [Unreleased][] + +### Added + +### Changed + +### Fixed + + ## [3.0.3](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.3) - 2023-10-11 ### Added @@ -833,7 +842,7 @@ This version improves performance on benchmarks significantly compared to 2.3. [semantic versioning]: http://semver.org/ -[unreleased]: ../../compare/v2.3.0...HEAD +[unreleased]: ../../compare/v3.0.1...HEAD [2.3.0]: ../../compare/v2.2.2...v2.3.0 diff --git a/version.txt b/version.txt index 75a22a26a..0f9d6b15d 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.0.3 +3.1.0-dev From 8e8c3f3c921720076b6eb7ee217a14d115240d9c Mon Sep 17 00:00:00 2001 From: bmello4688 Date: Mon, 16 Oct 2023 06:47:26 -0400 Subject: [PATCH 02/21] Allow setting of the python module file (#2044) Allow the setting of the python module file in order to create a virtual package structure --------- Co-authored-by: Benedikt Reinartz --- src/embed_tests/Modules.cs | 70 ++++++++++++++++++++++++++--- src/runtime/PythonTypes/PyModule.cs | 13 +++++- 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/src/embed_tests/Modules.cs b/src/embed_tests/Modules.cs index a88ab8552..6cab4dd07 100644 --- a/src/embed_tests/Modules.cs +++ b/src/embed_tests/Modules.cs @@ -51,7 +51,7 @@ public void TestEval() ps.Set("a", 1); var result = ps.Eval("a + 2"); Assert.AreEqual(3, result); - } + } } /// @@ -169,6 +169,62 @@ public void TestScopeClass() } } + /// + /// Create a class in the scope, the class can read variables in the scope. + /// Its methods can write the variables with the help of 'global' keyword. + /// + [Test] + public void TestCreateVirtualPackageStructure() + { + using (Py.GIL()) + { + using var _p1 = PyModule.FromString("test", ""); + // Sub-module + using var _p2 = PyModule.FromString("test.scope", + "class Class1():\n" + + " def __init__(self, value):\n" + + " self.value = value\n" + + " def call(self, arg):\n" + + " return self.value + bb + arg\n" + // use scope variables + " def update(self, arg):\n" + + " global bb\n" + + " bb = self.value + arg\n", // update scope variable + "test" + ); + + dynamic ps2 = Py.Import("test.scope"); + ps2.bb = 100; + + dynamic obj1 = ps2.Class1(20); + var result = obj1.call(10).As(); + Assert.AreEqual(130, result); + + obj1.update(10); + result = ps2.Get("bb"); + Assert.AreEqual(30, result); + } + } + + /// + /// Test setting the file attribute via a FromString parameter + /// + [Test] + public void TestCreateModuleWithFilename() + { + using var _gil = Py.GIL(); + + using var mod = PyModule.FromString("mod", ""); + using var modWithoutName = PyModule.FromString("mod_without_name", "", " "); + using var modNullName = PyModule.FromString("mod_null_name", "", null); + + using var modWithName = PyModule.FromString("mod_with_name", "", "some_filename"); + + Assert.AreEqual("none", mod.Get("__file__")); + Assert.AreEqual("none", modWithoutName.Get("__file__")); + Assert.AreEqual("none", modNullName.Get("__file__")); + Assert.AreEqual("some_filename", modWithName.Get("__file__")); + } + /// /// Import a python module into the session. /// Equivalent to the Python "import" statement. @@ -194,7 +250,7 @@ public void TestImportModule() } /// - /// Create a scope and import variables from a scope, + /// Create a scope and import variables from a scope, /// exec Python statements in the scope then discard it. /// [Test] @@ -218,7 +274,7 @@ public void TestImportScope() } /// - /// Create a scope and import variables from a scope, + /// Create a scope and import variables from a scope, /// exec Python statements in the scope then discard it. /// [Test] @@ -241,7 +297,7 @@ public void TestImportAllFromScope() } /// - /// Create a scope and import variables from a scope, + /// Create a scope and import variables from a scope, /// call the function imported. /// [Test] @@ -286,7 +342,7 @@ public void TestImportScopeFunction() public void TestVariables() { using (Py.GIL()) - { + { (ps.Variables() as dynamic)["ee"] = new PyInt(200); var a0 = ps.Get("ee"); Assert.AreEqual(200, a0); @@ -326,8 +382,8 @@ public void TestThread() _ps.res = 0; _ps.bb = 100; _ps.th_cnt = 0; - //add function to the scope - //can be call many times, more efficient than ast + //add function to the scope + //can be call many times, more efficient than ast ps.Exec( "import threading\n"+ "lock = threading.Lock()\n"+ diff --git a/src/runtime/PythonTypes/PyModule.cs b/src/runtime/PythonTypes/PyModule.cs index 4549678ed..243f77ecc 100644 --- a/src/runtime/PythonTypes/PyModule.cs +++ b/src/runtime/PythonTypes/PyModule.cs @@ -82,7 +82,18 @@ public PyModule Reload() public static PyModule FromString(string name, string code) { - using NewReference c = Runtime.Py_CompileString(code, "none", (int)RunFlagType.File); + return FromString(name, code, ""); + } + + public static PyModule FromString(string name, string code, string file) + { + // Force valid value + if (string.IsNullOrWhiteSpace(file)) + { + file = "none"; + } + + using NewReference c = Runtime.Py_CompileString(code, file, (int)RunFlagType.File); NewReference m = Runtime.PyImport_ExecCodeModule(name, c.BorrowOrThrow()); return new PyModule(m.StealOrThrow()); } From eef67db7ff800c550b509fd4aa0eeedd86647801 Mon Sep 17 00:00:00 2001 From: Jake Date: Tue, 7 Nov 2023 22:27:54 +0900 Subject: [PATCH 03/21] To fix memory access exception when iteration breaks in the middle of the list before reaching end. --- .../CollectionWrappers/IterableWrapper.cs | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/runtime/CollectionWrappers/IterableWrapper.cs b/src/runtime/CollectionWrappers/IterableWrapper.cs index d30e584d3..849e3997b 100644 --- a/src/runtime/CollectionWrappers/IterableWrapper.cs +++ b/src/runtime/CollectionWrappers/IterableWrapper.cs @@ -24,18 +24,22 @@ public IEnumerator GetEnumerator() { iterObject = PyIter.GetIter(pyObject); } - - using var _ = iterObject; - while (true) + try { - using var GIL = Py.GIL(); - - if (!iterObject.MoveNext()) + while (true) { - iterObject.Dispose(); - break; + using var _ = Py.GIL(); + if (!iterObject.MoveNext()) + { + break; + } + yield return iterObject.Current.As()!; } - yield return iterObject.Current.As()!; + } + finally + { + using var _ = Py.GIL(); + iterObject.Dispose(); } } } From 87fc365a5fad5c2a6ab43ca12abd140fe0085443 Mon Sep 17 00:00:00 2001 From: Gert Dreyer Date: Fri, 23 Feb 2024 09:27:01 +0200 Subject: [PATCH 04/21] Tests for operators on type CS type with codec --- src/embed_tests/TestOperator.cs | 230 ++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) diff --git a/src/embed_tests/TestOperator.cs b/src/embed_tests/TestOperator.cs index a5713274a..1d66903ca 100644 --- a/src/embed_tests/TestOperator.cs +++ b/src/embed_tests/TestOperator.cs @@ -15,6 +15,7 @@ public class TestOperator public void SetUp() { PythonEngine.Initialize(); + OwnIntCodec.Setup(); } [OneTimeTearDown] @@ -23,6 +24,120 @@ public void Dispose() PythonEngine.Shutdown(); } + // Mock Integer class to test math ops on non-native dotnet types + public struct OwnInt + { + private int _value; + + public int Num => _value; + + public OwnInt() + { + _value = 0; + } + + public OwnInt(int value) + { + _value = value; + } + + public static OwnInt operator -(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value - p2._value); + } + + public static OwnInt operator +(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value + p2._value); + } + + public static OwnInt operator *(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value * p2._value); + } + + public static OwnInt operator /(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value / p2._value); + } + + public static OwnInt operator %(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value % p2._value); + } + + public static OwnInt operator ^(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value ^ p2._value); + } + + public static bool operator <(OwnInt p1, OwnInt p2) + { + return p1._value < p2._value; + } + + public static bool operator >(OwnInt p1, OwnInt p2) + { + return p1._value > p2._value; + } + + public static bool operator ==(OwnInt p1, OwnInt p2) + { + return p1._value == p2._value; + } + + public static bool operator !=(OwnInt p1, OwnInt p2) + { + return p1._value != p2._value; + } + + public static OwnInt operator |(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value | p2._value); + } + + public static OwnInt operator &(OwnInt p1, OwnInt p2) + { + return new OwnInt(p1._value & p2._value); + } + + public static bool operator <=(OwnInt p1, OwnInt p2) + { + return p1._value <= p2._value; + } + + public static bool operator >=(OwnInt p1, OwnInt p2) + { + return p1._value >= p2._value; + } + } + + // Codec for mock class above. + public class OwnIntCodec : IPyObjectDecoder + { + public static void Setup() + { + PyObjectConversions.RegisterDecoder(new OwnIntCodec()); + } + + public bool CanDecode(PyType objectType, Type targetType) + { + return objectType.Name == "int" && targetType == typeof(OwnInt); + } + + public bool TryDecode(PyObject pyObj, out T? value) + { + if (pyObj.PyType.Name != "int" || typeof(T) != typeof(OwnInt)) + { + value = default(T); + return false; + } + + value = (T)(object)new OwnInt(pyObj.As()); + return true; + } + } + public class OperableObject { public int Num { get; set; } @@ -524,6 +639,121 @@ public void ShiftOperatorOverloads() c = a >> b.Num assert c.Num == a.Num >> b.Num +"); + } + + [Test] + public void ReverseOperatorWithCodec() + { + string name = string.Format("{0}.{1}", + typeof(OwnInt).DeclaringType.Name, + typeof(OwnInt).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = 2 +b = cls(10) + +c = a + b +assert c.Num == a + b.Num + +c = a - b +assert c.Num == a - b.Num + +c = a * b +assert c.Num == a * b.Num + +c = a / b +assert c.Num == a // b.Num + +c = a % b +assert c.Num == a % b.Num + +c = a & b +assert c.Num == a & b.Num + +c = a | b +assert c.Num == a | b.Num + +c = a ^ b +assert c.Num == a ^ b.Num + +c = a == b +assert c == (a == b.Num) + +c = a != b +assert c == (a != b.Num) + +c = a <= b +assert c == (a <= b.Num) + +c = a >= b +assert c == (a >= b.Num) + +c = a < b +assert c == (a < b.Num) + +c = a > b +assert c == (a > b.Num) +"); + } + + [Test] + public void ForwardOperatorWithCodec() + { + string name = string.Format("{0}.{1}", + typeof(OwnInt).DeclaringType.Name, + typeof(OwnInt).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = cls(2) +b = 10 +c = a + b +assert c.Num == a.Num + b + +c = a - b +assert c.Num == a.Num - b + +c = a * b +assert c.Num == a.Num * b + +c = a / b +assert c.Num == a.Num // b + +c = a % b +assert c.Num == a.Num % b + +c = a & b +assert c.Num == a.Num & b + +c = a | b +assert c.Num == a.Num | b + +c = a ^ b +assert c.Num == a.Num ^ b + +c = a == b +assert c == (a.Num == b) + +c = a != b +assert c == (a.Num != b) + +c = a <= b +assert c == (a.Num <= b) + +c = a >= b +assert c == (a.Num >= b) + +c = a < b +assert c == (a.Num < b) + +c = a > b +assert c == (a.Num > b) "); } } From 01d6772b05e88cf7b1472aad751fcde5aa8008c5 Mon Sep 17 00:00:00 2001 From: Gert Dreyer Date: Wed, 21 Feb 2024 22:07:18 +0200 Subject: [PATCH 05/21] Bugfix: RecursionError when reverse/righthand operations invoked. e.g. __rsub__, __rmul__ --- src/runtime/ClassManager.cs | 2 +- src/runtime/MethodBinder.cs | 40 +++++++++++++++++------------ src/runtime/Types/MethodBinding.cs | 3 ++- src/runtime/Types/MethodObject.cs | 18 ++++++------- src/runtime/Types/OperatorMethod.cs | 18 +++++-------- 5 files changed, 42 insertions(+), 39 deletions(-) diff --git a/src/runtime/ClassManager.cs b/src/runtime/ClassManager.cs index 79ab20e82..25f8639ab 100644 --- a/src/runtime/ClassManager.cs +++ b/src/runtime/ClassManager.cs @@ -546,7 +546,7 @@ private static ClassInfo GetClassInfo(Type type, ClassBase impl) ci.members[pyName] = new MethodObject(type, name, forwardMethods).AllocObject(); // Only methods where only the right operand is the declaring type. if (reverseMethods.Length > 0) - ci.members[pyNameReverse] = new MethodObject(type, name, reverseMethods).AllocObject(); + ci.members[pyNameReverse] = new MethodObject(type, name, reverseMethods, reverse_args: true).AllocObject(); } } diff --git a/src/runtime/MethodBinder.cs b/src/runtime/MethodBinder.cs index 07ed4fe22..18ef573d0 100644 --- a/src/runtime/MethodBinder.cs +++ b/src/runtime/MethodBinder.cs @@ -28,17 +28,22 @@ internal class MethodBinder [NonSerialized] public bool init = false; + public const bool DefaultAllowThreads = true; public bool allow_threads = DefaultAllowThreads; - internal MethodBinder() + public bool args_reversed = false; + + internal MethodBinder(bool reverse_args = false) { list = new List(); + args_reversed = reverse_args; } - internal MethodBinder(MethodInfo mi) + internal MethodBinder(MethodInfo mi, bool reverse_args = false) { list = new List { new MaybeMethodBase(mi) }; + args_reversed = reverse_args; } public int Count @@ -271,10 +276,11 @@ internal static int ArgPrecedence(Type t) /// The Python target of the method invocation. /// The Python arguments. /// The Python keyword arguments. + /// Reverse arguments of methods. Used for methods such as __radd__, __rsub__, __rmod__ etc /// A Binding if successful. Otherwise null. - internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw) + internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, bool reverse_args = false) { - return Bind(inst, args, kw, null, null); + return Bind(inst, args, kw, null, null, reverse_args); } /// @@ -287,10 +293,11 @@ internal static int ArgPrecedence(Type t) /// The Python arguments. /// The Python keyword arguments. /// If not null, only bind to that method. + /// Reverse arguments of methods. Used for methods such as __radd__, __rsub__, __rmod__ etc /// A Binding if successful. Otherwise null. - internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info) + internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, bool reverse_args = false) { - return Bind(inst, args, kw, info, null); + return Bind(inst, args, kw, info, null, reverse_args); } private readonly struct MatchedMethod @@ -334,8 +341,9 @@ public MismatchedMethod(Exception exception, MethodBase mb) /// The Python keyword arguments. /// If not null, only bind to that method. /// If not null, additionally attempt to bind to the generic methods in this array by inferring generic type parameters. + /// Reverse arguments of methods. Used for methods such as __radd__, __rsub__, __rmod__ etc /// A Binding if successful. Otherwise null. - internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, MethodBase[]? methodinfo) + internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, MethodBase[]? methodinfo, bool reverse_args = false) { // loop to find match, return invoker w/ or w/o error var kwargDict = new Dictionary(); @@ -363,10 +371,10 @@ public MismatchedMethod(Exception exception, MethodBase mb) _methods = GetMethods(); } - return Bind(inst, args, kwargDict, _methods, matchGenerics: true); + return Bind(inst, args, kwargDict, _methods, matchGenerics: true, reverse_args); } - static Binding? Bind(BorrowedReference inst, BorrowedReference args, Dictionary kwargDict, MethodBase[] methods, bool matchGenerics) + private static Binding? Bind(BorrowedReference inst, BorrowedReference args, Dictionary kwargDict, MethodBase[] methods, bool matchGenerics, bool reversed = false) { var pynargs = (int)Runtime.PyTuple_Size(args); var isGeneric = false; @@ -386,7 +394,7 @@ public MismatchedMethod(Exception exception, MethodBase mb) // Binary operator methods will have 2 CLR args but only one Python arg // (unary operators will have 1 less each), since Python operator methods are bound. isOperator = isOperator && pynargs == pi.Length - 1; - bool isReverse = isOperator && OperatorMethod.IsReverse((MethodInfo)mi); // Only cast if isOperator. + bool isReverse = isOperator && reversed; // Only cast if isOperator. if (isReverse && OperatorMethod.IsComparisonOp((MethodInfo)mi)) continue; // Comparison operators in Python have no reverse mode. if (!MatchesArgumentCount(pynargs, pi, kwargDict, out bool paramsArray, out ArrayList? defaultArgList, out int kwargsMatched, out int defaultsNeeded) && !isOperator) @@ -809,14 +817,14 @@ static bool MatchesArgumentCount(int positionalArgumentCount, ParameterInfo[] pa return match; } - internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw) + internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, bool reverse_args = false) { - return Invoke(inst, args, kw, null, null); + return Invoke(inst, args, kw, null, null, reverse_args); } - internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info) + internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, bool reverse_args = false) { - return Invoke(inst, args, kw, info, null); + return Invoke(inst, args, kw, info, null, reverse_args = false); } protected static void AppendArgumentTypes(StringBuilder to, BorrowedReference args) @@ -852,7 +860,7 @@ protected static void AppendArgumentTypes(StringBuilder to, BorrowedReference ar to.Append(')'); } - internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, MethodBase[]? methodinfo) + internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, MethodBase[]? methodinfo, bool reverse_args = false) { // No valid methods, nothing to bind. if (GetMethods().Length == 0) @@ -865,7 +873,7 @@ internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference a return Exceptions.RaiseTypeError(msg.ToString()); } - Binding? binding = Bind(inst, args, kw, info, methodinfo); + Binding? binding = Bind(inst, args, kw, info, methodinfo, reverse_args); object result; IntPtr ts = IntPtr.Zero; diff --git a/src/runtime/Types/MethodBinding.cs b/src/runtime/Types/MethodBinding.cs index 334d705a6..2f943f3fb 100644 --- a/src/runtime/Types/MethodBinding.cs +++ b/src/runtime/Types/MethodBinding.cs @@ -237,7 +237,8 @@ public static NewReference tp_call(BorrowedReference ob, BorrowedReference args, } } } - return self.m.Invoke(target is null ? BorrowedReference.Null : target, args, kw, self.info.UnsafeValue); + + return self.m.Invoke(target is null ? BorrowedReference.Null : target, args, kw, self.info.UnsafeValue, self.m.binder.args_reversed); } finally { diff --git a/src/runtime/Types/MethodObject.cs b/src/runtime/Types/MethodObject.cs index 05198da76..1943ed884 100644 --- a/src/runtime/Types/MethodObject.cs +++ b/src/runtime/Types/MethodObject.cs @@ -19,20 +19,20 @@ internal class MethodObject : ExtensionType { [NonSerialized] private MethodBase[]? _info = null; + private readonly List infoList; internal string name; internal readonly MethodBinder binder; internal bool is_static = false; - internal PyString? doc; internal MaybeType type; - public MethodObject(MaybeType type, string name, MethodBase[] info, bool allow_threads) + public MethodObject(MaybeType type, string name, MethodBase[] info, bool allow_threads, bool reverse_args = false) { this.type = type; this.name = name; this.infoList = new List(); - binder = new MethodBinder(); + binder = new MethodBinder(reverse_args); foreach (MethodBase item in info) { this.infoList.Add(item); @@ -45,8 +45,8 @@ public MethodObject(MaybeType type, string name, MethodBase[] info, bool allow_t binder.allow_threads = allow_threads; } - public MethodObject(MaybeType type, string name, MethodBase[] info) - : this(type, name, info, allow_threads: AllowThreads(info)) + public MethodObject(MaybeType type, string name, MethodBase[] info, bool reverse_args = false) + : this(type, name, info, allow_threads: AllowThreads(info), reverse_args) { } @@ -67,14 +67,14 @@ internal MethodBase[] info } } - public virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw) + public virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, bool reverse_args = false) { - return Invoke(inst, args, kw, null); + return Invoke(inst, args, kw, null, reverse_args); } - public virtual NewReference Invoke(BorrowedReference target, BorrowedReference args, BorrowedReference kw, MethodBase? info) + public virtual NewReference Invoke(BorrowedReference target, BorrowedReference args, BorrowedReference kw, MethodBase? info, bool reverse_args = false) { - return binder.Invoke(target, args, kw, info, this.info); + return binder.Invoke(target, args, kw, info, this.info, reverse_args); } /// diff --git a/src/runtime/Types/OperatorMethod.cs b/src/runtime/Types/OperatorMethod.cs index e3cc23370..a2ca73982 100644 --- a/src/runtime/Types/OperatorMethod.cs +++ b/src/runtime/Types/OperatorMethod.cs @@ -177,17 +177,14 @@ public static string ReversePyMethodName(string pyName) } /// - /// Check if the method is performing a reverse operation. + /// Check if the method should have a reversed operation. /// /// The operator method. /// - public static bool IsReverse(MethodBase method) + public static bool HaveReverse(MethodBase method) { - Type primaryType = method.IsOpsHelper() - ? method.DeclaringType.GetGenericArguments()[0] - : method.DeclaringType; - Type leftOperandType = method.GetParameters()[0].ParameterType; - return leftOperandType != primaryType; + var pi = method.GetParameters(); + return OpMethodMap.ContainsKey(method.Name) && pi.Length == 2; } public static void FilterMethods(MethodBase[] methods, out MethodBase[] forwardMethods, out MethodBase[] reverseMethods) @@ -196,14 +193,11 @@ public static void FilterMethods(MethodBase[] methods, out MethodBase[] forwardM var reverseMethodsList = new List(); foreach (var method in methods) { - if (IsReverse(method)) + forwardMethodsList.Add(method); + if (HaveReverse(method)) { reverseMethodsList.Add(method); - } else - { - forwardMethodsList.Add(method); } - } forwardMethods = forwardMethodsList.ToArray(); reverseMethods = reverseMethodsList.ToArray(); From d3bde9d6d39e5bcf5990d7dba43cda42321287d2 Mon Sep 17 00:00:00 2001 From: Gert Dreyer Date: Fri, 23 Feb 2024 15:33:44 +0200 Subject: [PATCH 06/21] Update Authors and Changelog --- AUTHORS.md | 1 + CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS.md b/AUTHORS.md index 577e898aa..18435671c 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -86,3 +86,4 @@ - ([@gpetrou](https://github.com/gpetrou)) - Ehsan Iran-Nejad ([@eirannejad](https://github.com/eirannejad)) - ([@legomanww](https://github.com/legomanww)) +- ([@gertdreyer](https://github.com/gertdreyer)) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdab9bf64..5b545045f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Fixed +- Fixed RecursionError for reverse operators on C# operable types from python. See #2240 ## [3.0.3](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.3) - 2023-10-11 From 1542cc96ae1e69f672da85f14ff51ed6a430b6f8 Mon Sep 17 00:00:00 2001 From: Gert Dreyer Date: Mon, 26 Feb 2024 13:11:41 +0200 Subject: [PATCH 07/21] Normalized names. Added HashCode and Equals to testing objects --- src/embed_tests/TestOperator.cs | 19 +++++++----- src/runtime/ClassManager.cs | 2 +- src/runtime/MethodBinder.cs | 50 ++++++++++++++++-------------- src/runtime/Types/MethodBinding.cs | 2 +- src/runtime/Types/MethodObject.cs | 16 +++++----- 5 files changed, 48 insertions(+), 41 deletions(-) diff --git a/src/embed_tests/TestOperator.cs b/src/embed_tests/TestOperator.cs index 1d66903ca..1ec3268ac 100644 --- a/src/embed_tests/TestOperator.cs +++ b/src/embed_tests/TestOperator.cs @@ -41,6 +41,17 @@ public OwnInt(int value) _value = value; } + public override int GetHashCode() + { + return unchecked(65535 + _value.GetHashCode()); + } + + public override bool Equals(object obj) + { + return obj is OwnInt @object && + Num == @object.Num; + } + public static OwnInt operator -(OwnInt p1, OwnInt p2) { return new OwnInt(p1._value - p2._value); @@ -125,14 +136,8 @@ public bool CanDecode(PyType objectType, Type targetType) return objectType.Name == "int" && targetType == typeof(OwnInt); } - public bool TryDecode(PyObject pyObj, out T? value) + public bool TryDecode(PyObject pyObj, out T value) { - if (pyObj.PyType.Name != "int" || typeof(T) != typeof(OwnInt)) - { - value = default(T); - return false; - } - value = (T)(object)new OwnInt(pyObj.As()); return true; } diff --git a/src/runtime/ClassManager.cs b/src/runtime/ClassManager.cs index 25f8639ab..ecb6055a8 100644 --- a/src/runtime/ClassManager.cs +++ b/src/runtime/ClassManager.cs @@ -546,7 +546,7 @@ private static ClassInfo GetClassInfo(Type type, ClassBase impl) ci.members[pyName] = new MethodObject(type, name, forwardMethods).AllocObject(); // Only methods where only the right operand is the declaring type. if (reverseMethods.Length > 0) - ci.members[pyNameReverse] = new MethodObject(type, name, reverseMethods, reverse_args: true).AllocObject(); + ci.members[pyNameReverse] = new MethodObject(type, name, reverseMethods, argsReversed: true).AllocObject(); } } diff --git a/src/runtime/MethodBinder.cs b/src/runtime/MethodBinder.cs index 18ef573d0..836e1da3e 100644 --- a/src/runtime/MethodBinder.cs +++ b/src/runtime/MethodBinder.cs @@ -32,18 +32,18 @@ internal class MethodBinder public const bool DefaultAllowThreads = true; public bool allow_threads = DefaultAllowThreads; - public bool args_reversed = false; + public bool argsReversed = false; - internal MethodBinder(bool reverse_args = false) + internal MethodBinder(bool argsReversed = false) { list = new List(); - args_reversed = reverse_args; + this.argsReversed = argsReversed; } - internal MethodBinder(MethodInfo mi, bool reverse_args = false) + internal MethodBinder(MethodInfo mi, bool argsReversed = false) { list = new List { new MaybeMethodBase(mi) }; - args_reversed = reverse_args; + this.argsReversed = argsReversed; } public int Count @@ -276,11 +276,11 @@ internal static int ArgPrecedence(Type t) /// The Python target of the method invocation. /// The Python arguments. /// The Python keyword arguments. - /// Reverse arguments of methods. Used for methods such as __radd__, __rsub__, __rmod__ etc + /// Reverse arguments of methods. Used for methods such as __radd__, __rsub__, __rmod__ etc /// A Binding if successful. Otherwise null. - internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, bool reverse_args = false) + internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, bool argsReversed = false) { - return Bind(inst, args, kw, null, null, reverse_args); + return Bind(inst, args, kw, null, null, argsReversed); } /// @@ -293,11 +293,11 @@ internal static int ArgPrecedence(Type t) /// The Python arguments. /// The Python keyword arguments. /// If not null, only bind to that method. - /// Reverse arguments of methods. Used for methods such as __radd__, __rsub__, __rmod__ etc + /// Reverse arguments of methods. Used for methods such as __radd__, __rsub__, __rmod__ etc /// A Binding if successful. Otherwise null. - internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, bool reverse_args = false) + internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, bool argsReversed = false) { - return Bind(inst, args, kw, info, null, reverse_args); + return Bind(inst, args, kw, info, null, argsReversed); } private readonly struct MatchedMethod @@ -341,9 +341,9 @@ public MismatchedMethod(Exception exception, MethodBase mb) /// The Python keyword arguments. /// If not null, only bind to that method. /// If not null, additionally attempt to bind to the generic methods in this array by inferring generic type parameters. - /// Reverse arguments of methods. Used for methods such as __radd__, __rsub__, __rmod__ etc + /// Reverse arguments of methods. Used for methods such as __radd__, __rsub__, __rmod__ etc /// A Binding if successful. Otherwise null. - internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, MethodBase[]? methodinfo, bool reverse_args = false) + internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, MethodBase[]? methodinfo, bool argsReversed = false) { // loop to find match, return invoker w/ or w/o error var kwargDict = new Dictionary(); @@ -371,10 +371,10 @@ public MismatchedMethod(Exception exception, MethodBase mb) _methods = GetMethods(); } - return Bind(inst, args, kwargDict, _methods, matchGenerics: true, reverse_args); + return Bind(inst, args, kwargDict, _methods, matchGenerics: true, argsReversed); } - private static Binding? Bind(BorrowedReference inst, BorrowedReference args, Dictionary kwargDict, MethodBase[] methods, bool matchGenerics, bool reversed = false) + private static Binding? Bind(BorrowedReference inst, BorrowedReference args, Dictionary kwargDict, MethodBase[] methods, bool matchGenerics, bool argsReversed = false) { var pynargs = (int)Runtime.PyTuple_Size(args); var isGeneric = false; @@ -394,7 +394,7 @@ public MismatchedMethod(Exception exception, MethodBase mb) // Binary operator methods will have 2 CLR args but only one Python arg // (unary operators will have 1 less each), since Python operator methods are bound. isOperator = isOperator && pynargs == pi.Length - 1; - bool isReverse = isOperator && reversed; // Only cast if isOperator. + bool isReverse = isOperator && argsReversed; // Only cast if isOperator. if (isReverse && OperatorMethod.IsComparisonOp((MethodInfo)mi)) continue; // Comparison operators in Python have no reverse mode. if (!MatchesArgumentCount(pynargs, pi, kwargDict, out bool paramsArray, out ArrayList? defaultArgList, out int kwargsMatched, out int defaultsNeeded) && !isOperator) @@ -402,12 +402,14 @@ public MismatchedMethod(Exception exception, MethodBase mb) continue; } // Preprocessing pi to remove either the first or second argument. - if (isOperator && !isReverse) { + if (isOperator && !isReverse) + { // The first Python arg is the right operand, while the bound instance is the left. // We need to skip the first (left operand) CLR argument. pi = pi.Skip(1).ToArray(); } - else if (isOperator && isReverse) { + else if (isOperator && isReverse) + { // The first Python arg is the left operand. // We need to take the first CLR argument. pi = pi.Take(1).ToArray(); @@ -817,14 +819,14 @@ static bool MatchesArgumentCount(int positionalArgumentCount, ParameterInfo[] pa return match; } - internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, bool reverse_args = false) + internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, bool argsReversed = false) { - return Invoke(inst, args, kw, null, null, reverse_args); + return Invoke(inst, args, kw, null, null, argsReversed); } - internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, bool reverse_args = false) + internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, bool argsReversed = false) { - return Invoke(inst, args, kw, info, null, reverse_args = false); + return Invoke(inst, args, kw, info, null, argsReversed = false); } protected static void AppendArgumentTypes(StringBuilder to, BorrowedReference args) @@ -860,7 +862,7 @@ protected static void AppendArgumentTypes(StringBuilder to, BorrowedReference ar to.Append(')'); } - internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, MethodBase[]? methodinfo, bool reverse_args = false) + internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, MethodBase[]? methodinfo, bool argsReversed = false) { // No valid methods, nothing to bind. if (GetMethods().Length == 0) @@ -873,7 +875,7 @@ internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference a return Exceptions.RaiseTypeError(msg.ToString()); } - Binding? binding = Bind(inst, args, kw, info, methodinfo, reverse_args); + Binding? binding = Bind(inst, args, kw, info, methodinfo, argsReversed); object result; IntPtr ts = IntPtr.Zero; diff --git a/src/runtime/Types/MethodBinding.cs b/src/runtime/Types/MethodBinding.cs index 2f943f3fb..79607d1ae 100644 --- a/src/runtime/Types/MethodBinding.cs +++ b/src/runtime/Types/MethodBinding.cs @@ -238,7 +238,7 @@ public static NewReference tp_call(BorrowedReference ob, BorrowedReference args, } } - return self.m.Invoke(target is null ? BorrowedReference.Null : target, args, kw, self.info.UnsafeValue, self.m.binder.args_reversed); + return self.m.Invoke(target is null ? BorrowedReference.Null : target, args, kw, self.info.UnsafeValue, self.m.binder.argsReversed); } finally { diff --git a/src/runtime/Types/MethodObject.cs b/src/runtime/Types/MethodObject.cs index 1943ed884..4bc21458b 100644 --- a/src/runtime/Types/MethodObject.cs +++ b/src/runtime/Types/MethodObject.cs @@ -27,12 +27,12 @@ internal class MethodObject : ExtensionType internal PyString? doc; internal MaybeType type; - public MethodObject(MaybeType type, string name, MethodBase[] info, bool allow_threads, bool reverse_args = false) + public MethodObject(MaybeType type, string name, MethodBase[] info, bool allow_threads, bool argsReversed = false) { this.type = type; this.name = name; this.infoList = new List(); - binder = new MethodBinder(reverse_args); + binder = new MethodBinder(argsReversed); foreach (MethodBase item in info) { this.infoList.Add(item); @@ -45,8 +45,8 @@ public MethodObject(MaybeType type, string name, MethodBase[] info, bool allow_t binder.allow_threads = allow_threads; } - public MethodObject(MaybeType type, string name, MethodBase[] info, bool reverse_args = false) - : this(type, name, info, allow_threads: AllowThreads(info), reverse_args) + public MethodObject(MaybeType type, string name, MethodBase[] info, bool argsReversed = false) + : this(type, name, info, allow_threads: AllowThreads(info), argsReversed) { } @@ -67,14 +67,14 @@ internal MethodBase[] info } } - public virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, bool reverse_args = false) + public virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, bool argsReversed = false) { - return Invoke(inst, args, kw, null, reverse_args); + return Invoke(inst, args, kw, null, argsReversed); } - public virtual NewReference Invoke(BorrowedReference target, BorrowedReference args, BorrowedReference kw, MethodBase? info, bool reverse_args = false) + public virtual NewReference Invoke(BorrowedReference target, BorrowedReference args, BorrowedReference kw, MethodBase? info, bool argsReversed = false) { - return binder.Invoke(target, args, kw, info, this.info, reverse_args); + return binder.Invoke(target, args, kw, info, this.info, argsReversed); } /// From 71ca0634696be47ac52066097802d5a53a716ede Mon Sep 17 00:00:00 2001 From: Gert Dreyer Date: Mon, 26 Feb 2024 20:49:55 +0200 Subject: [PATCH 08/21] Cleanup Codec/Argument Reversing Passing. --- src/embed_tests/TestOperator.cs | 4 ++-- src/runtime/MethodBinder.cs | 31 +++++++++++++----------------- src/runtime/Types/MethodBinding.cs | 2 +- src/runtime/Types/MethodObject.cs | 10 +++++----- 4 files changed, 21 insertions(+), 26 deletions(-) diff --git a/src/embed_tests/TestOperator.cs b/src/embed_tests/TestOperator.cs index 1ec3268ac..6bfb81bdb 100644 --- a/src/embed_tests/TestOperator.cs +++ b/src/embed_tests/TestOperator.cs @@ -25,9 +25,9 @@ public void Dispose() } // Mock Integer class to test math ops on non-native dotnet types - public struct OwnInt + public readonly struct OwnInt { - private int _value; + private readonly int _value; public int Num => _value; diff --git a/src/runtime/MethodBinder.cs b/src/runtime/MethodBinder.cs index 836e1da3e..9a5515c8e 100644 --- a/src/runtime/MethodBinder.cs +++ b/src/runtime/MethodBinder.cs @@ -34,16 +34,14 @@ internal class MethodBinder public bool argsReversed = false; - internal MethodBinder(bool argsReversed = false) + internal MethodBinder() { list = new List(); - this.argsReversed = argsReversed; } - internal MethodBinder(MethodInfo mi, bool argsReversed = false) + internal MethodBinder(MethodInfo mi) { list = new List { new MaybeMethodBase(mi) }; - this.argsReversed = argsReversed; } public int Count @@ -276,11 +274,10 @@ internal static int ArgPrecedence(Type t) /// The Python target of the method invocation. /// The Python arguments. /// The Python keyword arguments. - /// Reverse arguments of methods. Used for methods such as __radd__, __rsub__, __rmod__ etc /// A Binding if successful. Otherwise null. - internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, bool argsReversed = false) + internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw) { - return Bind(inst, args, kw, null, null, argsReversed); + return Bind(inst, args, kw, null, null); } /// @@ -293,11 +290,10 @@ internal static int ArgPrecedence(Type t) /// The Python arguments. /// The Python keyword arguments. /// If not null, only bind to that method. - /// Reverse arguments of methods. Used for methods such as __radd__, __rsub__, __rmod__ etc /// A Binding if successful. Otherwise null. - internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, bool argsReversed = false) + internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info) { - return Bind(inst, args, kw, info, null, argsReversed); + return Bind(inst, args, kw, info, null); } private readonly struct MatchedMethod @@ -341,9 +337,8 @@ public MismatchedMethod(Exception exception, MethodBase mb) /// The Python keyword arguments. /// If not null, only bind to that method. /// If not null, additionally attempt to bind to the generic methods in this array by inferring generic type parameters. - /// Reverse arguments of methods. Used for methods such as __radd__, __rsub__, __rmod__ etc /// A Binding if successful. Otherwise null. - internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, MethodBase[]? methodinfo, bool argsReversed = false) + internal Binding? Bind(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, MethodBase[]? methodinfo) { // loop to find match, return invoker w/ or w/o error var kwargDict = new Dictionary(); @@ -819,14 +814,14 @@ static bool MatchesArgumentCount(int positionalArgumentCount, ParameterInfo[] pa return match; } - internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, bool argsReversed = false) + internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw) { - return Invoke(inst, args, kw, null, null, argsReversed); + return Invoke(inst, args, kw, null, null); } - internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, bool argsReversed = false) + internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info) { - return Invoke(inst, args, kw, info, null, argsReversed = false); + return Invoke(inst, args, kw, info, null); } protected static void AppendArgumentTypes(StringBuilder to, BorrowedReference args) @@ -862,7 +857,7 @@ protected static void AppendArgumentTypes(StringBuilder to, BorrowedReference ar to.Append(')'); } - internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, MethodBase[]? methodinfo, bool argsReversed = false) + internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, MethodBase? info, MethodBase[]? methodinfo) { // No valid methods, nothing to bind. if (GetMethods().Length == 0) @@ -875,7 +870,7 @@ internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference a return Exceptions.RaiseTypeError(msg.ToString()); } - Binding? binding = Bind(inst, args, kw, info, methodinfo, argsReversed); + Binding? binding = Bind(inst, args, kw, info, methodinfo); object result; IntPtr ts = IntPtr.Zero; diff --git a/src/runtime/Types/MethodBinding.cs b/src/runtime/Types/MethodBinding.cs index 79607d1ae..bfe22b0f3 100644 --- a/src/runtime/Types/MethodBinding.cs +++ b/src/runtime/Types/MethodBinding.cs @@ -238,7 +238,7 @@ public static NewReference tp_call(BorrowedReference ob, BorrowedReference args, } } - return self.m.Invoke(target is null ? BorrowedReference.Null : target, args, kw, self.info.UnsafeValue, self.m.binder.argsReversed); + return self.m.Invoke(target is null ? BorrowedReference.Null : target, args, kw, self.info.UnsafeValue); } finally { diff --git a/src/runtime/Types/MethodObject.cs b/src/runtime/Types/MethodObject.cs index 4bc21458b..12484d301 100644 --- a/src/runtime/Types/MethodObject.cs +++ b/src/runtime/Types/MethodObject.cs @@ -32,7 +32,7 @@ public MethodObject(MaybeType type, string name, MethodBase[] info, bool allow_t this.type = type; this.name = name; this.infoList = new List(); - binder = new MethodBinder(argsReversed); + binder = new MethodBinder() { argsReversed = argsReversed }; foreach (MethodBase item in info) { this.infoList.Add(item); @@ -67,14 +67,14 @@ internal MethodBase[] info } } - public virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw, bool argsReversed = false) + public virtual NewReference Invoke(BorrowedReference inst, BorrowedReference args, BorrowedReference kw) { - return Invoke(inst, args, kw, null, argsReversed); + return Invoke(inst, args, kw, null); } - public virtual NewReference Invoke(BorrowedReference target, BorrowedReference args, BorrowedReference kw, MethodBase? info, bool argsReversed = false) + public virtual NewReference Invoke(BorrowedReference target, BorrowedReference args, BorrowedReference kw, MethodBase? info) { - return binder.Invoke(target, args, kw, info, this.info, argsReversed); + return binder.Invoke(target, args, kw, info, this.info); } /// From 9d18a243f507e18143ec97c743a0dfe76fdce67d Mon Sep 17 00:00:00 2001 From: Victor Date: Tue, 27 Feb 2024 23:36:05 -0800 Subject: [PATCH 09/21] Added `ToPythonAs()` extension method to allow for explicit conversion using a specific type (#2330) fixes https://github.com/pythonnet/pythonnet/issues/2311 --- CHANGELOG.md | 3 +++ src/embed_tests/TestConverter.cs | 9 +++++++++ src/runtime/Converter.cs | 11 +++++++++-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b545045f..83f9d4bd1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added +- Added `ToPythonAs()` extension method to allow for explicit conversion using a specific type. ([#2311][i2311]) + ### Changed ### Fixed @@ -960,3 +962,4 @@ This version improves performance on benchmarks significantly compared to 2.3. [i238]: https://github.com/pythonnet/pythonnet/issues/238 [i1481]: https://github.com/pythonnet/pythonnet/issues/1481 [i1672]: https://github.com/pythonnet/pythonnet/pull/1672 +[i2311]: https://github.com/pythonnet/pythonnet/issues/2311 diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index 0686d528b..a59b9c97b 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -185,6 +185,15 @@ public void RawPyObjectProxy() Assert.AreEqual(pyObject.DangerousGetAddressOrNull(), proxiedHandle); } + [Test] + public void GenericToPython() + { + int i = 42; + var pyObject = i.ToPythonAs(); + var type = pyObject.GetPythonType(); + Assert.AreEqual(nameof(IConvertible), type.Name); + } + // regression for https://github.com/pythonnet/pythonnet/issues/451 [Test] public void CanGetListFromDerivedClass() diff --git a/src/runtime/Converter.cs b/src/runtime/Converter.cs index 412f3b711..50b33e60e 100644 --- a/src/runtime/Converter.cs +++ b/src/runtime/Converter.cs @@ -133,7 +133,8 @@ internal static NewReference ToPython(object? value, Type type) if (EncodableByUser(type, value)) { var encoded = PyObjectConversions.TryEncode(value, type); - if (encoded != null) { + if (encoded != null) + { return new NewReference(encoded); } } @@ -334,7 +335,7 @@ internal static bool ToManagedValue(BorrowedReference value, Type obType, if (obType.IsGenericType && obType.GetGenericTypeDefinition() == typeof(Nullable<>)) { - if( value == Runtime.PyNone ) + if (value == Runtime.PyNone) { result = null; return true; @@ -980,5 +981,11 @@ public static PyObject ToPython(this object? o) if (o is null) return Runtime.None; return Converter.ToPython(o, o.GetType()).MoveToPyObject(); } + + public static PyObject ToPythonAs(this T? o) + { + if (o is null) return Runtime.None; + return Converter.ToPython(o, typeof(T)).MoveToPyObject(); + } } } From 563e3695f284b0269c73048875b7d2031832993a Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Thu, 15 Feb 2024 13:07:51 -0800 Subject: [PATCH 10/21] IComparable and IEquatable implementations for PyInt, PyFloat, and PyString for primitive .NET types --- CHANGELOG.md | 3 + src/embed_tests/TestPyFloat.cs | 27 ++++ src/embed_tests/TestPyInt.cs | 70 +++++++++ src/embed_tests/TestPyString.cs | 19 +++ .../PythonTypes/PyFloat.IComparable.cs | 34 +++++ src/runtime/PythonTypes/PyFloat.cs | 4 +- src/runtime/PythonTypes/PyInt.IComparable.cs | 136 ++++++++++++++++++ src/runtime/PythonTypes/PyInt.cs | 2 +- src/runtime/PythonTypes/PyObject.cs | 19 ++- src/runtime/PythonTypes/PyString.cs | 21 ++- 10 files changed, 331 insertions(+), 4 deletions(-) create mode 100644 src/runtime/PythonTypes/PyFloat.IComparable.cs create mode 100644 src/runtime/PythonTypes/PyInt.IComparable.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 83f9d4bd1..e6cc52d72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Added `ToPythonAs()` extension method to allow for explicit conversion using a specific type. ([#2311][i2311]) +- Added `IComparable` and `IEquatable` implementations to `PyInt`, `PyFloat`, and `PyString` + to compare with primitive .NET types like `long`. + ### Changed ### Fixed diff --git a/src/embed_tests/TestPyFloat.cs b/src/embed_tests/TestPyFloat.cs index 36531cb6a..89e29e5fd 100644 --- a/src/embed_tests/TestPyFloat.cs +++ b/src/embed_tests/TestPyFloat.cs @@ -126,5 +126,32 @@ public void AsFloatBad() StringAssert.StartsWith("could not convert string to float", ex.Message); Assert.IsNull(a); } + + [Test] + public void CompareTo() + { + var v = new PyFloat(42); + + Assert.AreEqual(0, v.CompareTo(42f)); + Assert.AreEqual(0, v.CompareTo(42d)); + + Assert.AreEqual(1, v.CompareTo(41f)); + Assert.AreEqual(1, v.CompareTo(41d)); + + Assert.AreEqual(-1, v.CompareTo(43f)); + Assert.AreEqual(-1, v.CompareTo(43d)); + } + + [Test] + public void Equals() + { + var v = new PyFloat(42); + + Assert.IsTrue(v.Equals(42f)); + Assert.IsTrue(v.Equals(42d)); + + Assert.IsFalse(v.Equals(41f)); + Assert.IsFalse(v.Equals(41d)); + } } } diff --git a/src/embed_tests/TestPyInt.cs b/src/embed_tests/TestPyInt.cs index c147e074b..d2767e664 100644 --- a/src/embed_tests/TestPyInt.cs +++ b/src/embed_tests/TestPyInt.cs @@ -210,6 +210,76 @@ public void ToBigInteger() CollectionAssert.AreEqual(expected, actual); } + [Test] + public void CompareTo() + { + var v = new PyInt(42); + + #region Signed + Assert.AreEqual(0, v.CompareTo(42L)); + Assert.AreEqual(0, v.CompareTo(42)); + Assert.AreEqual(0, v.CompareTo((short)42)); + Assert.AreEqual(0, v.CompareTo((sbyte)42)); + + Assert.AreEqual(1, v.CompareTo(41L)); + Assert.AreEqual(1, v.CompareTo(41)); + Assert.AreEqual(1, v.CompareTo((short)41)); + Assert.AreEqual(1, v.CompareTo((sbyte)41)); + + Assert.AreEqual(-1, v.CompareTo(43L)); + Assert.AreEqual(-1, v.CompareTo(43)); + Assert.AreEqual(-1, v.CompareTo((short)43)); + Assert.AreEqual(-1, v.CompareTo((sbyte)43)); + #endregion Signed + + #region Unsigned + Assert.AreEqual(0, v.CompareTo(42UL)); + Assert.AreEqual(0, v.CompareTo(42U)); + Assert.AreEqual(0, v.CompareTo((ushort)42)); + Assert.AreEqual(0, v.CompareTo((byte)42)); + + Assert.AreEqual(1, v.CompareTo(41UL)); + Assert.AreEqual(1, v.CompareTo(41U)); + Assert.AreEqual(1, v.CompareTo((ushort)41)); + Assert.AreEqual(1, v.CompareTo((byte)41)); + + Assert.AreEqual(-1, v.CompareTo(43UL)); + Assert.AreEqual(-1, v.CompareTo(43U)); + Assert.AreEqual(-1, v.CompareTo((ushort)43)); + Assert.AreEqual(-1, v.CompareTo((byte)43)); + #endregion Unsigned + } + + [Test] + public void Equals() + { + var v = new PyInt(42); + + #region Signed + Assert.True(v.Equals(42L)); + Assert.True(v.Equals(42)); + Assert.True(v.Equals((short)42)); + Assert.True(v.Equals((sbyte)42)); + + Assert.False(v.Equals(41L)); + Assert.False(v.Equals(41)); + Assert.False(v.Equals((short)41)); + Assert.False(v.Equals((sbyte)41)); + #endregion Signed + + #region Unsigned + Assert.True(v.Equals(42UL)); + Assert.True(v.Equals(42U)); + Assert.True(v.Equals((ushort)42)); + Assert.True(v.Equals((byte)42)); + + Assert.False(v.Equals(41UL)); + Assert.False(v.Equals(41U)); + Assert.False(v.Equals((ushort)41)); + Assert.False(v.Equals((byte)41)); + #endregion Unsigned + } + [Test] public void ToBigIntegerLarge() { diff --git a/src/embed_tests/TestPyString.cs b/src/embed_tests/TestPyString.cs index b12e08c23..35c6339ee 100644 --- a/src/embed_tests/TestPyString.cs +++ b/src/embed_tests/TestPyString.cs @@ -112,5 +112,24 @@ public void TestUnicodeSurrogate() Assert.AreEqual(4, actual.Length()); Assert.AreEqual(expected, actual.ToString()); } + + [Test] + public void CompareTo() + { + var a = new PyString("foo"); + + Assert.AreEqual(0, a.CompareTo("foo")); + Assert.AreEqual("foo".CompareTo("bar"), a.CompareTo("bar")); + Assert.AreEqual("foo".CompareTo("foz"), a.CompareTo("foz")); + } + + [Test] + public void Equals() + { + var a = new PyString("foo"); + + Assert.True(a.Equals("foo")); + Assert.False(a.Equals("bar")); + } } } diff --git a/src/runtime/PythonTypes/PyFloat.IComparable.cs b/src/runtime/PythonTypes/PyFloat.IComparable.cs new file mode 100644 index 000000000..c12fc283a --- /dev/null +++ b/src/runtime/PythonTypes/PyFloat.IComparable.cs @@ -0,0 +1,34 @@ +using System; + +namespace Python.Runtime; + +partial class PyFloat : IComparable, IComparable + , IEquatable, IEquatable + , IComparable, IEquatable +{ + public override bool Equals(object o) + { + using var _ = Py.GIL(); + return o switch + { + double f64 => this.Equals(f64), + float f32 => this.Equals(f32), + _ => base.Equals(o), + }; + } + + public int CompareTo(double other) => this.ToDouble().CompareTo(other); + + public int CompareTo(float other) => this.ToDouble().CompareTo(other); + + public bool Equals(double other) => this.ToDouble().Equals(other); + + public bool Equals(float other) => this.ToDouble().Equals(other); + + public int CompareTo(PyFloat? other) + { + return other is null ? 1 : this.CompareTo(other.BorrowNullable()); + } + + public bool Equals(PyFloat? other) => base.Equals(other); +} diff --git a/src/runtime/PythonTypes/PyFloat.cs b/src/runtime/PythonTypes/PyFloat.cs index c09ec93ba..50621d5c2 100644 --- a/src/runtime/PythonTypes/PyFloat.cs +++ b/src/runtime/PythonTypes/PyFloat.cs @@ -8,7 +8,7 @@ namespace Python.Runtime /// PY3: https://docs.python.org/3/c-api/float.html /// for details. /// - public class PyFloat : PyNumber + public partial class PyFloat : PyNumber { internal PyFloat(in StolenReference ptr) : base(ptr) { @@ -100,6 +100,8 @@ public static PyFloat AsFloat(PyObject value) return new PyFloat(op.Steal()); } + public double ToDouble() => Runtime.PyFloat_AsDouble(obj); + public override TypeCode GetTypeCode() => TypeCode.Double; } } diff --git a/src/runtime/PythonTypes/PyInt.IComparable.cs b/src/runtime/PythonTypes/PyInt.IComparable.cs new file mode 100644 index 000000000..a96f02e10 --- /dev/null +++ b/src/runtime/PythonTypes/PyInt.IComparable.cs @@ -0,0 +1,136 @@ +using System; + +namespace Python.Runtime; + +partial class PyInt : IComparable, IComparable, IComparable, IComparable + , IComparable, IComparable, IComparable, IComparable + , IEquatable, IEquatable, IEquatable, IEquatable + , IEquatable, IEquatable, IEquatable, IEquatable + , IComparable, IEquatable +{ + public override bool Equals(object o) + { + using var _ = Py.GIL(); + return o switch + { + long i64 => this.Equals(i64), + int i32 => this.Equals(i32), + short i16 => this.Equals(i16), + sbyte i8 => this.Equals(i8), + + ulong u64 => this.Equals(u64), + uint u32 => this.Equals(u32), + ushort u16 => this.Equals(u16), + byte u8 => this.Equals(u8), + + _ => base.Equals(o), + }; + } + + #region Signed + public int CompareTo(long other) + { + using var pyOther = Runtime.PyInt_FromInt64(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public int CompareTo(int other) + { + using var pyOther = Runtime.PyInt_FromInt32(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public int CompareTo(short other) + { + using var pyOther = Runtime.PyInt_FromInt32(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public int CompareTo(sbyte other) + { + using var pyOther = Runtime.PyInt_FromInt32(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public bool Equals(long other) + { + using var pyOther = Runtime.PyInt_FromInt64(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + + public bool Equals(int other) + { + using var pyOther = Runtime.PyInt_FromInt32(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + + public bool Equals(short other) + { + using var pyOther = Runtime.PyInt_FromInt32(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + + public bool Equals(sbyte other) + { + using var pyOther = Runtime.PyInt_FromInt32(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + #endregion Signed + + #region Unsigned + public int CompareTo(ulong other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public int CompareTo(uint other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public int CompareTo(ushort other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public int CompareTo(byte other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.CompareTo(pyOther.BorrowOrThrow()); + } + + public bool Equals(ulong other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + + public bool Equals(uint other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + + public bool Equals(ushort other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + + public bool Equals(byte other) + { + using var pyOther = Runtime.PyLong_FromUnsignedLongLong(other); + return this.Equals(pyOther.BorrowOrThrow()); + } + #endregion Unsigned + + public int CompareTo(PyInt? other) + { + return other is null ? 1 : this.CompareTo(other.BorrowNullable()); + } + + public bool Equals(PyInt? other) => base.Equals(other); +} diff --git a/src/runtime/PythonTypes/PyInt.cs b/src/runtime/PythonTypes/PyInt.cs index e71462b74..0d00f5a13 100644 --- a/src/runtime/PythonTypes/PyInt.cs +++ b/src/runtime/PythonTypes/PyInt.cs @@ -9,7 +9,7 @@ namespace Python.Runtime /// Represents a Python integer object. /// See the documentation at https://docs.python.org/3/c-api/long.html /// - public class PyInt : PyNumber, IFormattable + public partial class PyInt : PyNumber, IFormattable { internal PyInt(in StolenReference ptr) : base(ptr) { diff --git a/src/runtime/PythonTypes/PyObject.cs b/src/runtime/PythonTypes/PyObject.cs index bda2d9c02..cf0c2a03f 100644 --- a/src/runtime/PythonTypes/PyObject.cs +++ b/src/runtime/PythonTypes/PyObject.cs @@ -1136,6 +1136,23 @@ public long Refcount } } + internal int CompareTo(BorrowedReference other) + { + int greater = Runtime.PyObject_RichCompareBool(this.Reference, other, Runtime.Py_GT); + Debug.Assert(greater != -1); + if (greater > 0) + return 1; + int less = Runtime.PyObject_RichCompareBool(this.Reference, other, Runtime.Py_LT); + Debug.Assert(less != -1); + return less > 0 ? -1 : 0; + } + + internal bool Equals(BorrowedReference other) + { + int equal = Runtime.PyObject_RichCompareBool(this.Reference, other, Runtime.Py_EQ); + Debug.Assert(equal != -1); + return equal > 0; + } public override bool TryGetMember(GetMemberBinder binder, out object? result) { @@ -1325,7 +1342,7 @@ private bool TryCompare(PyObject arg, int op, out object @out) } return true; } - + public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg, out object? result) { using var _ = Py.GIL(); diff --git a/src/runtime/PythonTypes/PyString.cs b/src/runtime/PythonTypes/PyString.cs index d54397fcf..6fed25c3e 100644 --- a/src/runtime/PythonTypes/PyString.cs +++ b/src/runtime/PythonTypes/PyString.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.Runtime.Serialization; namespace Python.Runtime @@ -13,7 +14,7 @@ namespace Python.Runtime /// 2011-01-29: ...Then why does the string constructor call PyUnicode_FromUnicode()??? /// [Serializable] - public class PyString : PySequence + public class PyString : PySequence, IComparable, IEquatable { internal PyString(in StolenReference reference) : base(reference) { } internal PyString(BorrowedReference reference) : base(reference) { } @@ -61,5 +62,23 @@ public static bool IsStringType(PyObject value) } public override TypeCode GetTypeCode() => TypeCode.String; + + internal string ToStringUnderGIL() + { + string? result = Runtime.GetManagedString(this.Reference); + Debug.Assert(result is not null); + return result!; + } + + public bool Equals(string? other) + => this.ToStringUnderGIL().Equals(other, StringComparison.CurrentCulture); + public int CompareTo(string? other) + => string.Compare(this.ToStringUnderGIL(), other, StringComparison.CurrentCulture); + + public override string ToString() + { + using var _ = Py.GIL(); + return this.ToStringUnderGIL(); + } } } From 6a8a97d0fc78ec11c754f8d8746c672cacefc586 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 5 May 2024 20:42:02 +0200 Subject: [PATCH 11/21] Fix CI by using macos-13 explicitly, adjust OS config (#2373) --- .github/workflows/main.yml | 48 +++++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c4af10c68..3396b83cc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,25 +9,35 @@ on: jobs: build-test: name: Build and Test - runs-on: ${{ matrix.os }}-latest + runs-on: ${{ matrix.os.instance }} timeout-minutes: 15 strategy: fail-fast: false matrix: - os: [windows, ubuntu, macos] - python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] - platform: [x64, x86] - exclude: - - os: ubuntu - platform: x86 - - os: macos + os: + - category: windows platform: x86 + instance: windows-latest + + - category: windows + platform: x64 + instance: windows-latest + + - category: ubuntu + platform: x64 + instance: ubuntu-latest + + - category: macos + platform: x64 + instance: macos-13 + + python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] steps: - name: Set Environment on macOS uses: maxim-lobanov/setup-xamarin@v1 - if: ${{ matrix.os == 'macos' }} + if: ${{ matrix.os.category == 'macos' }} with: mono-version: latest @@ -43,7 +53,7 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} - architecture: ${{ matrix.platform }} + architecture: ${{ matrix.os.platform }} - name: Install dependencies run: | @@ -55,42 +65,42 @@ jobs: pip install -v . - name: Set Python DLL path and PYTHONHOME (non Windows) - if: ${{ matrix.os != 'windows' }} + if: ${{ matrix.os.category != 'windows' }} run: | echo PYTHONNET_PYDLL=$(python -m find_libpython) >> $GITHUB_ENV echo PYTHONHOME=$(python -c 'import sys; print(sys.prefix)') >> $GITHUB_ENV - name: Set Python DLL path and PYTHONHOME (Windows) - if: ${{ matrix.os == 'windows' }} + if: ${{ matrix.os.category == 'windows' }} run: | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -InputObject "PYTHONNET_PYDLL=$(python -m find_libpython)" Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -InputObject "PYTHONHOME=$(python -c 'import sys; print(sys.prefix)')" - name: Embedding tests - run: dotnet test --runtime any-${{ matrix.platform }} --logger "console;verbosity=detailed" src/embed_tests/ + run: dotnet test --runtime any-${{ matrix.os.platform }} --logger "console;verbosity=detailed" src/embed_tests/ env: MONO_THREADS_SUSPEND: preemptive # https://github.com/mono/mono/issues/21466 - name: Python Tests (Mono) - if: ${{ matrix.os != 'windows' }} + if: ${{ matrix.os.category != 'windows' }} run: pytest --runtime mono # TODO: Run these tests on Windows x86 - name: Python Tests (.NET Core) - if: ${{ matrix.platform == 'x64' }} + if: ${{ matrix.os.platform == 'x64' }} run: pytest --runtime coreclr - name: Python Tests (.NET Framework) - if: ${{ matrix.os == 'windows' }} + if: ${{ matrix.os.category == 'windows' }} run: pytest --runtime netfx - name: Python tests run from .NET - run: dotnet test --runtime any-${{ matrix.platform }} src/python_tests_runner/ + run: dotnet test --runtime any-${{ matrix.os.platform }} src/python_tests_runner/ - name: Perf tests - if: ${{ (matrix.python == '3.8') && (matrix.platform == 'x64') }} + if: ${{ (matrix.python == '3.8') && (matrix.os.platform == 'x64') }} run: | pip install --force --no-deps --target src/perf_tests/baseline/ pythonnet==2.5.2 - dotnet test --configuration Release --runtime any-${{ matrix.platform }} --logger "console;verbosity=detailed" src/perf_tests/ + dotnet test --configuration Release --runtime any-${{ matrix.os.platform }} --logger "console;verbosity=detailed" src/perf_tests/ # TODO: Run mono tests on Windows? From 195cde67fffd06521f3bcb2294e60cad4ec506d6 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Fri, 10 May 2024 19:28:38 +0200 Subject: [PATCH 12/21] Use non-BOM encodings (#2370) * Use non-BOM encodings The documentation of the used `PyUnicode_DecodeUTF16` states that not passing `*byteorder` or passing a 0 results in the first two bytes, if they are the BOM (U+FEFF, zero-width no-break space), to be interpreted and skipped, which is incorrect when we convert a known "non BOM" string, which all strings from C# are. --- src/embed_tests/TestPyType.cs | 2 +- src/runtime/Loader.cs | 6 ++-- src/runtime/Native/CustomMarshaler.cs | 2 +- src/runtime/Native/NativeTypeSpec.cs | 2 +- src/runtime/PythonTypes/PyType.cs | 2 +- src/runtime/Runtime.cs | 46 ++++++++++++++------------- src/runtime/Util/Encodings.cs | 10 ++++++ tests/test_conversion.py | 3 ++ 8 files changed, 44 insertions(+), 29 deletions(-) create mode 100644 src/runtime/Util/Encodings.cs diff --git a/src/embed_tests/TestPyType.cs b/src/embed_tests/TestPyType.cs index 34645747d..0470070c3 100644 --- a/src/embed_tests/TestPyType.cs +++ b/src/embed_tests/TestPyType.cs @@ -28,7 +28,7 @@ public void CanCreateHeapType() const string name = "nÁmæ"; const string docStr = "dÁcæ"; - using var doc = new StrPtr(docStr, Encoding.UTF8); + using var doc = new StrPtr(docStr, Encodings.UTF8); var spec = new TypeSpec( name: name, basicSize: Util.ReadInt32(Runtime.Runtime.PyBaseObjectType, TypeOffset.tp_basicsize), diff --git a/src/runtime/Loader.cs b/src/runtime/Loader.cs index 516b9ab9c..c0e964abc 100644 --- a/src/runtime/Loader.cs +++ b/src/runtime/Loader.cs @@ -12,7 +12,7 @@ public unsafe static int Initialize(IntPtr data, int size) { try { - var dllPath = Encoding.UTF8.GetString((byte*)data.ToPointer(), size); + var dllPath = Encodings.UTF8.GetString((byte*)data.ToPointer(), size); if (!string.IsNullOrEmpty(dllPath)) { @@ -33,7 +33,7 @@ public unsafe static int Initialize(IntPtr data, int size) ); return 1; } - + return 0; } @@ -41,7 +41,7 @@ public unsafe static int Shutdown(IntPtr data, int size) { try { - var command = Encoding.UTF8.GetString((byte*)data.ToPointer(), size); + var command = Encodings.UTF8.GetString((byte*)data.ToPointer(), size); if (command == "full_shutdown") { diff --git a/src/runtime/Native/CustomMarshaler.cs b/src/runtime/Native/CustomMarshaler.cs index 62c027150..299af3a33 100644 --- a/src/runtime/Native/CustomMarshaler.cs +++ b/src/runtime/Native/CustomMarshaler.cs @@ -42,7 +42,7 @@ public int GetNativeDataSize() internal class UcsMarshaler : MarshalerBase { internal static readonly int _UCS = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? 2 : 4; - internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + internal static readonly Encoding PyEncoding = _UCS == 2 ? Encodings.UTF16 : Encodings.UTF32; private static readonly MarshalerBase Instance = new UcsMarshaler(); public override IntPtr MarshalManagedToNative(object managedObj) diff --git a/src/runtime/Native/NativeTypeSpec.cs b/src/runtime/Native/NativeTypeSpec.cs index 8b84df536..50019a148 100644 --- a/src/runtime/Native/NativeTypeSpec.cs +++ b/src/runtime/Native/NativeTypeSpec.cs @@ -17,7 +17,7 @@ public NativeTypeSpec(TypeSpec spec) { if (spec is null) throw new ArgumentNullException(nameof(spec)); - this.Name = new StrPtr(spec.Name, Encoding.UTF8); + this.Name = new StrPtr(spec.Name, Encodings.UTF8); this.BasicSize = spec.BasicSize; this.ItemSize = spec.ItemSize; this.Flags = (int)spec.Flags; diff --git a/src/runtime/PythonTypes/PyType.cs b/src/runtime/PythonTypes/PyType.cs index af796a5c5..28bda5d3e 100644 --- a/src/runtime/PythonTypes/PyType.cs +++ b/src/runtime/PythonTypes/PyType.cs @@ -53,7 +53,7 @@ public string Name { RawPointer = Util.ReadIntPtr(this, TypeOffset.tp_name), }; - return namePtr.ToString(System.Text.Encoding.UTF8)!; + return namePtr.ToString(Encodings.UTF8)!; } } diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 4e1c6156a..2f9e18f65 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -795,13 +795,13 @@ public static int Py_Main(int argc, string[] argv) internal static int PyRun_SimpleString(string code) { - using var codePtr = new StrPtr(code, Encoding.UTF8); + using var codePtr = new StrPtr(code, Encodings.UTF8); return Delegates.PyRun_SimpleStringFlags(codePtr, Utf8String); } internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedReference globals, BorrowedReference locals) { - using var codePtr = new StrPtr(code, Encoding.UTF8); + using var codePtr = new StrPtr(code, Encodings.UTF8); return Delegates.PyRun_StringFlags(codePtr, st, globals, locals, Utf8String); } @@ -813,14 +813,14 @@ internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedR /// internal static NewReference Py_CompileString(string str, string file, int start) { - using var strPtr = new StrPtr(str, Encoding.UTF8); + using var strPtr = new StrPtr(str, Encodings.UTF8); using var fileObj = new PyString(file); return Delegates.Py_CompileStringObject(strPtr, fileObj, start, Utf8String, -1); } internal static NewReference PyImport_ExecCodeModule(string name, BorrowedReference code) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyImport_ExecCodeModule(namePtr, code); } @@ -867,13 +867,13 @@ internal static bool PyObject_IsIterable(BorrowedReference ob) internal static int PyObject_HasAttrString(BorrowedReference pointer, string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyObject_HasAttrString(pointer, namePtr); } internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyObject_GetAttrString(pointer, namePtr); } @@ -884,12 +884,12 @@ internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, S internal static int PyObject_DelAttr(BorrowedReference @object, BorrowedReference name) => Delegates.PyObject_SetAttr(@object, name, null); internal static int PyObject_DelAttrString(BorrowedReference @object, string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyObject_SetAttrString(@object, namePtr, null); } internal static int PyObject_SetAttrString(BorrowedReference @object, string name, BorrowedReference value) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyObject_SetAttrString(@object, namePtr, value); } @@ -1071,7 +1071,7 @@ internal static bool PyBool_CheckExact(BorrowedReference ob) internal static NewReference PyLong_FromString(string value, int radix) { - using var valPtr = new StrPtr(value, Encoding.UTF8); + using var valPtr = new StrPtr(value, Encodings.UTF8); return Delegates.PyLong_FromString(valPtr, IntPtr.Zero, radix); } @@ -1252,12 +1252,14 @@ internal static bool PyString_CheckExact(BorrowedReference ob) internal static NewReference PyString_FromString(string value) { + int byteorder = BitConverter.IsLittleEndian ? -1 : 1; + int* byteorderPtr = &byteorder; fixed(char* ptr = value) return Delegates.PyUnicode_DecodeUTF16( (IntPtr)ptr, value.Length * sizeof(Char), IntPtr.Zero, - IntPtr.Zero + (IntPtr)byteorderPtr ); } @@ -1272,7 +1274,7 @@ internal static NewReference EmptyPyBytes() internal static NewReference PyByteArray_FromStringAndSize(IntPtr strPtr, nint len) => Delegates.PyByteArray_FromStringAndSize(strPtr, len); internal static NewReference PyByteArray_FromStringAndSize(string s) { - using var ptr = new StrPtr(s, Encoding.UTF8); + using var ptr = new StrPtr(s, Encodings.UTF8); return PyByteArray_FromStringAndSize(ptr.RawPointer, checked((nint)ptr.ByteCount)); } @@ -1300,7 +1302,7 @@ internal static IntPtr PyBytes_AsString(BorrowedReference ob) internal static NewReference PyUnicode_InternFromString(string s) { - using var ptr = new StrPtr(s, Encoding.UTF8); + using var ptr = new StrPtr(s, Encodings.UTF8); return Delegates.PyUnicode_InternFromString(ptr); } @@ -1375,7 +1377,7 @@ internal static bool PyDict_Check(BorrowedReference ob) internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer, string key) { - using var keyStr = new StrPtr(key, Encoding.UTF8); + using var keyStr = new StrPtr(key, Encodings.UTF8); return Delegates.PyDict_GetItemString(pointer, keyStr); } @@ -1391,7 +1393,7 @@ internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer /// internal static int PyDict_SetItemString(BorrowedReference dict, string key, BorrowedReference value) { - using var keyPtr = new StrPtr(key, Encoding.UTF8); + using var keyPtr = new StrPtr(key, Encodings.UTF8); return Delegates.PyDict_SetItemString(dict, keyPtr, value); } @@ -1400,7 +1402,7 @@ internal static int PyDict_SetItemString(BorrowedReference dict, string key, Bor internal static int PyDict_DelItemString(BorrowedReference pointer, string key) { - using var keyPtr = new StrPtr(key, Encoding.UTF8); + using var keyPtr = new StrPtr(key, Encodings.UTF8); return Delegates.PyDict_DelItemString(pointer, keyPtr); } @@ -1515,7 +1517,7 @@ internal static bool PyIter_Check(BorrowedReference ob) internal static NewReference PyModule_New(string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyModule_New(namePtr); } @@ -1529,7 +1531,7 @@ internal static NewReference PyModule_New(string name) /// Return -1 on error, 0 on success. internal static int PyModule_AddObject(BorrowedReference module, string name, StolenReference value) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); IntPtr valueAddr = value.DangerousGetAddressOrNull(); int res = Delegates.PyModule_AddObject(module, namePtr, valueAddr); // We can't just exit here because the reference is stolen only on success. @@ -1547,7 +1549,7 @@ internal static int PyModule_AddObject(BorrowedReference module, string name, St internal static NewReference PyImport_ImportModule(string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyImport_ImportModule(namePtr); } @@ -1556,7 +1558,7 @@ internal static NewReference PyImport_ImportModule(string name) internal static BorrowedReference PyImport_AddModule(string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PyImport_AddModule(namePtr); } @@ -1584,13 +1586,13 @@ internal static void PySys_SetArgvEx(int argc, string[] argv, int updatepath) internal static BorrowedReference PySys_GetObject(string name) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PySys_GetObject(namePtr); } internal static int PySys_SetObject(string name, BorrowedReference ob) { - using var namePtr = new StrPtr(name, Encoding.UTF8); + using var namePtr = new StrPtr(name, Encodings.UTF8); return Delegates.PySys_SetObject(namePtr, ob); } @@ -1689,7 +1691,7 @@ internal static IntPtr PyMem_Malloc(long size) internal static void PyErr_SetString(BorrowedReference ob, string message) { - using var msgPtr = new StrPtr(message, Encoding.UTF8); + using var msgPtr = new StrPtr(message, Encodings.UTF8); Delegates.PyErr_SetString(ob, msgPtr); } diff --git a/src/runtime/Util/Encodings.cs b/src/runtime/Util/Encodings.cs new file mode 100644 index 000000000..d5a0c6ff8 --- /dev/null +++ b/src/runtime/Util/Encodings.cs @@ -0,0 +1,10 @@ +using System; +using System.Text; + +namespace Python.Runtime; + +static class Encodings { + public static System.Text.Encoding UTF8 = new UTF8Encoding(false, true); + public static System.Text.Encoding UTF16 = new UnicodeEncoding(!BitConverter.IsLittleEndian, false, true); + public static System.Text.Encoding UTF32 = new UTF32Encoding(!BitConverter.IsLittleEndian, false, true); +} diff --git a/tests/test_conversion.py b/tests/test_conversion.py index bb686dd52..dd70f900a 100644 --- a/tests/test_conversion.py +++ b/tests/test_conversion.py @@ -510,6 +510,9 @@ def test_string_conversion(): ob.StringField = System.String(u'\uffff\uffff') assert ob.StringField == u'\uffff\uffff' + ob.StringField = System.String("\ufeffbom") + assert ob.StringField == "\ufeffbom" + ob.StringField = None assert ob.StringField is None From 32051cb3c7c2edffa031569043eac5aecaa573a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Bourbonnais?= <6788684+BadSingleton@users.noreply.github.com> Date: Fri, 10 May 2024 13:35:55 -0400 Subject: [PATCH 13/21] Expose serialization api (#2336) * Expose an API for users to specify their own formatter Adds post-serialization and pre-deserialization hooks for additional customization. * Add API for capsuling data when serializing * Add NoopFormatter and fall back to it if BinaryFormatter is not available --------- Co-authored-by: Benedikt Reinartz --- CHANGELOG.md | 3 + .../StateSerialization/NoopFormatter.cs | 14 ++ src/runtime/StateSerialization/RuntimeData.cs | 138 +++++++++++++++++- tests/domain_tests/TestRunner.cs | 117 +++++++++++++++ tests/domain_tests/test_domain_reload.py | 3 + 5 files changed, 269 insertions(+), 6 deletions(-) create mode 100644 src/runtime/StateSerialization/NoopFormatter.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index e6cc52d72..adef224e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. to compare with primitive .NET types like `long`. ### Changed +- Added a `FormatterFactory` member in RuntimeData to create formatters with parameters. For compatibility, the `FormatterType` member is still present and has precedence when defining both `FormatterFactory` and `FormatterType` +- Added a post-serialization and a pre-deserialization step callbacks to extend (de)serialization process +- Added an API to stash serialized data on Python capsules ### Fixed diff --git a/src/runtime/StateSerialization/NoopFormatter.cs b/src/runtime/StateSerialization/NoopFormatter.cs new file mode 100644 index 000000000..f05b7ebb2 --- /dev/null +++ b/src/runtime/StateSerialization/NoopFormatter.cs @@ -0,0 +1,14 @@ +using System; +using System.IO; +using System.Runtime.Serialization; + +namespace Python.Runtime; + +public class NoopFormatter : IFormatter { + public object Deserialize(Stream s) => throw new NotImplementedException(); + public void Serialize(Stream s, object o) {} + + public SerializationBinder? Binder { get; set; } + public StreamingContext Context { get; set; } + public ISurrogateSelector? SurrogateSelector { get; set; } +} diff --git a/src/runtime/StateSerialization/RuntimeData.cs b/src/runtime/StateSerialization/RuntimeData.cs index 204e15b5b..8eda9ce0b 100644 --- a/src/runtime/StateSerialization/RuntimeData.cs +++ b/src/runtime/StateSerialization/RuntimeData.cs @@ -1,7 +1,5 @@ using System; -using System.Collections; using System.Collections.Generic; -using System.Collections.ObjectModel; using System.Diagnostics; using System.IO; using System.Linq; @@ -17,7 +15,34 @@ namespace Python.Runtime { public static class RuntimeData { - private static Type? _formatterType; + + public readonly static Func DefaultFormatterFactory = () => + { + try + { + return new BinaryFormatter(); + } + catch + { + return new NoopFormatter(); + } + }; + + private static Func _formatterFactory { get; set; } = DefaultFormatterFactory; + + public static Func FormatterFactory + { + get => _formatterFactory; + set + { + if (value == null) + throw new ArgumentNullException(nameof(value)); + + _formatterFactory = value; + } + } + + private static Type? _formatterType = null; public static Type? FormatterType { get => _formatterType; @@ -31,6 +56,14 @@ public static Type? FormatterType } } + /// + /// Callback called as a last step in the serialization process + /// + public static Action? PostStashHook { get; set; } = null; + /// + /// Callback called as the first step in the deserialization process + /// + public static Action? PreRestoreHook { get; set; } = null; public static ICLRObjectStorer? WrappersStorer { get; set; } /// @@ -74,6 +107,7 @@ internal static void Stash() using NewReference capsule = PyCapsule_New(mem, IntPtr.Zero, IntPtr.Zero); int res = PySys_SetObject("clr_data", capsule.BorrowOrThrow()); PythonException.ThrowIfIsNotZero(res); + PostStashHook?.Invoke(); } internal static void RestoreRuntimeData() @@ -90,6 +124,7 @@ internal static void RestoreRuntimeData() private static void RestoreRuntimeDataImpl() { + PreRestoreHook?.Invoke(); BorrowedReference capsule = PySys_GetObject("clr_data"); if (capsule.IsNull) { @@ -250,11 +285,102 @@ private static void RestoreRuntimeDataObjects(SharedObjectsState storage) } } + static readonly string serialization_key_namepsace = "pythonnet_serialization_"; + /// + /// Removes the serialization capsule from the `sys` module object. + /// + /// + /// The serialization data must have been set with StashSerializationData + /// + /// The name given to the capsule on the `sys` module object + public static void FreeSerializationData(string key) + { + key = serialization_key_namepsace + key; + BorrowedReference oldCapsule = PySys_GetObject(key); + if (!oldCapsule.IsNull) + { + IntPtr oldData = PyCapsule_GetPointer(oldCapsule, IntPtr.Zero); + Marshal.FreeHGlobal(oldData); + PyCapsule_SetPointer(oldCapsule, IntPtr.Zero); + PySys_SetObject(key, null); + } + } + + /// + /// Stores the data in the argument in a Python capsule and stores + /// the capsule on the `sys` module object with the name . + /// + /// + /// No checks on pre-existing names on the `sys` module object are made. + /// + /// The name given to the capsule on the `sys` module object + /// A MemoryStream that contains the data to be placed in the capsule + public static void StashSerializationData(string key, MemoryStream stream) + { + if (stream.TryGetBuffer(out var data)) + { + IntPtr mem = Marshal.AllocHGlobal(IntPtr.Size + data.Count); + + // store the length of the buffer first + Marshal.WriteIntPtr(mem, (IntPtr)data.Count); + Marshal.Copy(data.Array, data.Offset, mem + IntPtr.Size, data.Count); + + try + { + using NewReference capsule = PyCapsule_New(mem, IntPtr.Zero, IntPtr.Zero); + int res = PySys_SetObject(key, capsule.BorrowOrThrow()); + PythonException.ThrowIfIsNotZero(res); + } + catch + { + Marshal.FreeHGlobal(mem); + } + } + else + { + throw new NotImplementedException($"{nameof(stream)} must be exposable"); + } + + } + + static byte[] emptyBuffer = new byte[0]; + /// + /// Retreives the previously stored data on a Python capsule. + /// Throws if the object corresponding to the parameter + /// on the `sys` module object is not a capsule. + /// + /// The name given to the capsule on the `sys` module object + /// A MemoryStream containing the previously saved serialization data. + /// The stream is empty if no name matches the key. + public static MemoryStream GetSerializationData(string key) + { + BorrowedReference capsule = PySys_GetObject(key); + if (capsule.IsNull) + { + // nothing to do. + return new MemoryStream(emptyBuffer, writable:false); + } + var ptr = PyCapsule_GetPointer(capsule, IntPtr.Zero); + if (ptr == IntPtr.Zero) + { + // The PyCapsule API returns NULL on error; NULL cannot be stored + // as a capsule's value + PythonException.ThrowIfIsNull(null); + } + var len = (int)Marshal.ReadIntPtr(ptr); + byte[] buffer = new byte[len]; + Marshal.Copy(ptr+IntPtr.Size, buffer, 0, len); + return new MemoryStream(buffer, writable:false); + } + internal static IFormatter CreateFormatter() { - return FormatterType != null ? - (IFormatter)Activator.CreateInstance(FormatterType) - : new BinaryFormatter(); + + if (FormatterType != null) + { + return (IFormatter)Activator.CreateInstance(FormatterType); + } + return FormatterFactory(); } } } diff --git a/tests/domain_tests/TestRunner.cs b/tests/domain_tests/TestRunner.cs index 4f6a3ea28..bbee81b3d 100644 --- a/tests/domain_tests/TestRunner.cs +++ b/tests/domain_tests/TestRunner.cs @@ -1132,6 +1132,66 @@ import System ", }, + new TestCase + { + Name = "test_serialize_unserializable_object", + DotNetBefore = @" + namespace TestNamespace + { + public class NotSerializableTextWriter : System.IO.TextWriter + { + override public System.Text.Encoding Encoding { get { return System.Text.Encoding.ASCII;} } + } + [System.Serializable] + public static class SerializableWriter + { + private static System.IO.TextWriter _writer = null; + public static System.IO.TextWriter Writer {get { return _writer; }} + public static void CreateInternalWriter() + { + _writer = System.IO.TextWriter.Synchronized(new NotSerializableTextWriter()); + } + } + } +", + DotNetAfter = @" + namespace TestNamespace + { + public class NotSerializableTextWriter : System.IO.TextWriter + { + override public System.Text.Encoding Encoding { get { return System.Text.Encoding.ASCII;} } + } + [System.Serializable] + public static class SerializableWriter + { + private static System.IO.TextWriter _writer = null; + public static System.IO.TextWriter Writer {get { return _writer; }} + public static void CreateInternalWriter() + { + _writer = System.IO.TextWriter.Synchronized(new NotSerializableTextWriter()); + } + } + } + ", + PythonCode = @" +import sys + +def before_reload(): + import clr + import System + clr.AddReference('DomainTests') + import TestNamespace + TestNamespace.SerializableWriter.CreateInternalWriter(); + sys.__obj = TestNamespace.SerializableWriter.Writer + sys.__obj.WriteLine('test') + +def after_reload(): + import clr + import System + sys.__obj.WriteLine('test') + + ", + } }; /// @@ -1142,7 +1202,59 @@ import System const string CaseRunnerTemplate = @" using System; using System.IO; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Formatters.Binary; using Python.Runtime; + +namespace Serialization +{{ + // Classes in this namespace is mostly useful for test_serialize_unserializable_object + class NotSerializableSerializer : ISerializationSurrogate + {{ + public NotSerializableSerializer() + {{ + }} + public void GetObjectData(object obj, SerializationInfo info, StreamingContext context) + {{ + info.AddValue(""notSerialized_tp"", obj.GetType()); + }} + public object SetObjectData(object obj, SerializationInfo info, StreamingContext context, ISurrogateSelector selector) + {{ + if (info == null) + {{ + return null; + }} + Type typeObj = info.GetValue(""notSerialized_tp"", typeof(Type)) as Type; + if (typeObj == null) + {{ + return null; + }} + + obj = Activator.CreateInstance(typeObj); + return obj; + }} + }} + class NonSerializableSelector : SurrogateSelector + {{ + public override ISerializationSurrogate GetSurrogate(Type type, StreamingContext context, out ISurrogateSelector selector) + {{ + if (type == null) + {{ + throw new ArgumentNullException(); + }} + selector = (ISurrogateSelector)this; + if (type.IsSerializable) + {{ + return null; // use whichever default + }} + else + {{ + return (ISerializationSurrogate)(new NotSerializableSerializer()); + }} + }} + }} +}} + namespace CaseRunner {{ class CaseRunner @@ -1151,6 +1263,11 @@ public static int Main() {{ try {{ + RuntimeData.FormatterFactory = () => + {{ + return new BinaryFormatter(){{SurrogateSelector = new Serialization.NonSerializableSelector()}}; + }}; + PythonEngine.Initialize(); using (Py.GIL()) {{ diff --git a/tests/domain_tests/test_domain_reload.py b/tests/domain_tests/test_domain_reload.py index 8999e481b..1e5e8e81b 100644 --- a/tests/domain_tests/test_domain_reload.py +++ b/tests/domain_tests/test_domain_reload.py @@ -88,3 +88,6 @@ def test_nested_type(): def test_import_after_reload(): _run_test("import_after_reload") + +def test_import_after_reload(): + _run_test("test_serialize_unserializable_object") \ No newline at end of file From b112885d19091f0e5fe1e7609236d6093fbd5a0b Mon Sep 17 00:00:00 2001 From: Victor Date: Sat, 11 May 2024 00:34:26 -0700 Subject: [PATCH 14/21] handle bad paths in sys.path (#2383) fixes #2376 --- CHANGELOG.md | 1 + src/runtime/AssemblyManager.cs | 7 +++++++ src/runtime/Exceptions.cs | 4 ++-- tests/test_module.py | 14 ++++++++++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adef224e0..23184258d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Fixed - Fixed RecursionError for reverse operators on C# operable types from python. See #2240 +- Fixed probing for assemblies in `sys.path` failing when a path in `sys.path` has invalid characters. See #2376 ## [3.0.3](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.3) - 2023-10-11 diff --git a/src/runtime/AssemblyManager.cs b/src/runtime/AssemblyManager.cs index a8bbd1f6c..82658bf50 100644 --- a/src/runtime/AssemblyManager.cs +++ b/src/runtime/AssemblyManager.cs @@ -200,6 +200,13 @@ static IEnumerable FindAssemblyCandidates(string name) } else { + int invalidCharIndex = head.IndexOfAny(Path.GetInvalidPathChars()); + if (invalidCharIndex >= 0) + { + using var importWarning = Runtime.PyObject_GetAttrString(Exceptions.exceptions_module, "ImportWarning"); + Exceptions.warn($"Path entry '{head}' has invalid char at position {invalidCharIndex}", importWarning.BorrowOrThrow()); + continue; + } path = Path.Combine(head, name); } diff --git a/src/runtime/Exceptions.cs b/src/runtime/Exceptions.cs index da095e030..85e56eace 100644 --- a/src/runtime/Exceptions.cs +++ b/src/runtime/Exceptions.cs @@ -270,7 +270,7 @@ public static void warn(string message, BorrowedReference exception, int stackle } using var warn = Runtime.PyObject_GetAttrString(warnings_module.obj, "warn"); - Exceptions.ErrorCheck(warn.Borrow()); + warn.BorrowOrThrow(); using var argsTemp = Runtime.PyTuple_New(3); BorrowedReference args = argsTemp.BorrowOrThrow(); @@ -283,7 +283,7 @@ public static void warn(string message, BorrowedReference exception, int stackle Runtime.PyTuple_SetItem(args, 2, level.StealOrThrow()); using var result = Runtime.PyObject_CallObject(warn.Borrow(), args); - Exceptions.ErrorCheck(result.Borrow()); + result.BorrowOrThrow(); } public static void warn(string message, BorrowedReference exception) diff --git a/tests/test_module.py b/tests/test_module.py index ddfa7bb36..0c20dcfc0 100644 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -344,6 +344,20 @@ def test_clr_add_reference(): with pytest.raises(FileNotFoundException): AddReference("somethingtotallysilly") + +def test_clr_add_reference_bad_path(): + import sys + from clr import AddReference + from System.IO import FileNotFoundException + bad_path = "hello\0world" + sys.path.append(bad_path) + try: + with pytest.raises(FileNotFoundException): + AddReference("test_clr_add_reference_bad_path") + finally: + sys.path.remove(bad_path) + + def test_clr_get_clr_type(): """Test clr.GetClrType().""" from clr import GetClrType From 4e5afdf973e29f1ae50413aa0cc092f2a03df68f Mon Sep 17 00:00:00 2001 From: Frank Witscher Date: Mon, 13 May 2024 16:25:11 +0200 Subject: [PATCH 15/21] Fix access violation exception on shutdown (#1977) When nulling the GC handles on shutdown the reference count of all objects pointed to by the IntPtr in the `CLRObject.reflectedObjects` are zero. This caused an exception in some scenarios because `Runtime.PyObject_TYPE(reflectedClrObject)` is called while the reference counter is at zero. After `TypeManager.RemoveTypes();` is called in the `Runtime.Shutdown()` method, reference count decrements to zero do not invoke `ClassBase.tp_clear` for managed objects anymore which normally is responsible for removing references from `CLRObject.reflectedObjects`. Collecting objects referenced in `CLRObject.reflectedObjects` only after leads to an unstable state in which the reference count for these object addresses is zero while still maintaining them to be used for further pseudo-cleanup. In that time, the memory could have been reclaimed already which leads to the exception. --- AUTHORS.md | 1 + CHANGELOG.md | 2 ++ src/runtime/Runtime.cs | 5 +++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index 18435671c..6aa4a6010 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -38,6 +38,7 @@ - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) - Félix Bourbonnais ([@BadSingleton](https://github.com/BadSingleton)) - Florian Treurniet ([@ftreurni](https://github.com/ftreurni)) +- Frank Witscher ([@Frawak](https://github.com/Frawak)) - He-chien Tsai ([@t3476](https://github.com/t3476)) - Inna Wiesel ([@inna-w](https://github.com/inna-w)) - Ivan Cronyn ([@cronan](https://github.com/cronan)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23184258d..7d2faa1b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Fixed RecursionError for reverse operators on C# operable types from python. See #2240 - Fixed probing for assemblies in `sys.path` failing when a path in `sys.path` has invalid characters. See #2376 +- Fixed possible access violation exception on shutdown. See ([#1977][i1977]) ## [3.0.3](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.3) - 2023-10-11 @@ -970,3 +971,4 @@ This version improves performance on benchmarks significantly compared to 2.3. [i1481]: https://github.com/pythonnet/pythonnet/issues/1481 [i1672]: https://github.com/pythonnet/pythonnet/pull/1672 [i2311]: https://github.com/pythonnet/pythonnet/issues/2311 +[i1977]: https://github.com/pythonnet/pythonnet/issues/1977 diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 2f9e18f65..a65fea66f 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -278,6 +278,8 @@ internal static void Shutdown() ClearClrModules(); RemoveClrRootModule(); + TryCollectingGarbage(MaxCollectRetriesOnShutdown, forceBreakLoops: true); + NullGCHandles(ExtensionType.loadedExtensions); ClassManager.RemoveClasses(); TypeManager.RemoveTypes(); @@ -295,8 +297,7 @@ internal static void Shutdown() PyObjectConversions.Reset(); PyGC_Collect(); - bool everythingSeemsCollected = TryCollectingGarbage(MaxCollectRetriesOnShutdown, - forceBreakLoops: true); + bool everythingSeemsCollected = TryCollectingGarbage(MaxCollectRetriesOnShutdown); Debug.Assert(everythingSeemsCollected); Finalizer.Shutdown(); From 6f0f6713e8f55a24ea7803584c5490eca0518739 Mon Sep 17 00:00:00 2001 From: Frank Witscher Date: Mon, 13 May 2024 22:38:59 +0200 Subject: [PATCH 16/21] Restrict first garbage collection Otherwise, collecting all at this earlier point results in corrupt memory for derived types. --- src/runtime/Finalizer.cs | 8 ++++---- src/runtime/Runtime.cs | 10 +++++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/runtime/Finalizer.cs b/src/runtime/Finalizer.cs index 713564f08..5b5ecfcfc 100644 --- a/src/runtime/Finalizer.cs +++ b/src/runtime/Finalizer.cs @@ -191,7 +191,7 @@ internal static void Shutdown() Instance.started = false; } - internal nint DisposeAll() + internal nint DisposeAll(bool disposeObj = true, bool disposeDerived = true, bool disposeBuffer = true) { if (_objQueue.IsEmpty && _derivedQueue.IsEmpty && _bufferQueue.IsEmpty) return 0; @@ -216,7 +216,7 @@ internal nint DisposeAll() try { - while (!_objQueue.IsEmpty) + if (disposeObj) while (!_objQueue.IsEmpty) { if (!_objQueue.TryDequeue(out var obj)) continue; @@ -240,7 +240,7 @@ internal nint DisposeAll() } } - while (!_derivedQueue.IsEmpty) + if (disposeDerived) while (!_derivedQueue.IsEmpty) { if (!_derivedQueue.TryDequeue(out var derived)) continue; @@ -258,7 +258,7 @@ internal nint DisposeAll() collected++; } - while (!_bufferQueue.IsEmpty) + if (disposeBuffer) while (!_bufferQueue.IsEmpty) { if (!_bufferQueue.TryDequeue(out var buffer)) continue; diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index a65fea66f..b3820270c 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -278,7 +278,8 @@ internal static void Shutdown() ClearClrModules(); RemoveClrRootModule(); - TryCollectingGarbage(MaxCollectRetriesOnShutdown, forceBreakLoops: true); + TryCollectingGarbage(MaxCollectRetriesOnShutdown, forceBreakLoops: true, + obj: true, derived: false, buffer: false); NullGCHandles(ExtensionType.loadedExtensions); ClassManager.RemoveClasses(); @@ -329,7 +330,8 @@ internal static void Shutdown() const int MaxCollectRetriesOnShutdown = 20; internal static int _collected; - static bool TryCollectingGarbage(int runs, bool forceBreakLoops) + static bool TryCollectingGarbage(int runs, bool forceBreakLoops, + bool obj = true, bool derived = true, bool buffer = true) { if (runs <= 0) throw new ArgumentOutOfRangeException(nameof(runs)); @@ -342,7 +344,9 @@ static bool TryCollectingGarbage(int runs, bool forceBreakLoops) GC.Collect(); GC.WaitForPendingFinalizers(); pyCollected += PyGC_Collect(); - pyCollected += Finalizer.Instance.DisposeAll(); + pyCollected += Finalizer.Instance.DisposeAll(disposeObj: obj, + disposeDerived: derived, + disposeBuffer: buffer); } if (Volatile.Read(ref _collected) == 0 && pyCollected == 0) { From f82aeea80cd463996d3ab376c94a57a8d2d7e774 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 9 Jun 2024 01:32:00 +0200 Subject: [PATCH 17/21] Simplify UTF8 StrPtr usage (#2374) * Use non-BOM encodings * Copy potential BOM to the output of PyString_FromString The documentation of the used `PyUnicode_DecodeUTF16` states that not passing `*byteorder` or passing a 0 results in the first two bytes, if they are the BOM (U+FEFF, zero-width no-break space), to be interpreted and skipped, which is incorrect when we convert a known "non BOM" string, which all strings from C# are. * Default to UTF8 for StrPtr --- src/embed_tests/TestPyType.cs | 3 +- src/runtime/Native/NativeTypeSpec.cs | 2 +- src/runtime/Native/StrPtr.cs | 2 ++ src/runtime/Runtime.cs | 43 ++++++++++++++-------------- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/embed_tests/TestPyType.cs b/src/embed_tests/TestPyType.cs index 0470070c3..d98dfda2e 100644 --- a/src/embed_tests/TestPyType.cs +++ b/src/embed_tests/TestPyType.cs @@ -28,7 +28,8 @@ public void CanCreateHeapType() const string name = "nÁmæ"; const string docStr = "dÁcæ"; - using var doc = new StrPtr(docStr, Encodings.UTF8); + using var doc = new StrPtr(docStr); + var spec = new TypeSpec( name: name, basicSize: Util.ReadInt32(Runtime.Runtime.PyBaseObjectType, TypeOffset.tp_basicsize), diff --git a/src/runtime/Native/NativeTypeSpec.cs b/src/runtime/Native/NativeTypeSpec.cs index 50019a148..90e07afd7 100644 --- a/src/runtime/Native/NativeTypeSpec.cs +++ b/src/runtime/Native/NativeTypeSpec.cs @@ -17,7 +17,7 @@ public NativeTypeSpec(TypeSpec spec) { if (spec is null) throw new ArgumentNullException(nameof(spec)); - this.Name = new StrPtr(spec.Name, Encodings.UTF8); + this.Name = new StrPtr(spec.Name); this.BasicSize = spec.BasicSize; this.ItemSize = spec.ItemSize; this.Flags = (int)spec.Flags; diff --git a/src/runtime/Native/StrPtr.cs b/src/runtime/Native/StrPtr.cs index 4f73be9b5..c9f4db660 100644 --- a/src/runtime/Native/StrPtr.cs +++ b/src/runtime/Native/StrPtr.cs @@ -10,6 +10,8 @@ struct StrPtr : IDisposable public IntPtr RawPointer { get; set; } unsafe byte* Bytes => (byte*)this.RawPointer; + public unsafe StrPtr(string value) : this(value, Encodings.UTF8) {} + public unsafe StrPtr(string value, Encoding encoding) { if (value is null) throw new ArgumentNullException(nameof(value)); diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 2f9e18f65..a26ad67a9 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -795,13 +795,13 @@ public static int Py_Main(int argc, string[] argv) internal static int PyRun_SimpleString(string code) { - using var codePtr = new StrPtr(code, Encodings.UTF8); + using var codePtr = new StrPtr(code); return Delegates.PyRun_SimpleStringFlags(codePtr, Utf8String); } internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedReference globals, BorrowedReference locals) { - using var codePtr = new StrPtr(code, Encodings.UTF8); + using var codePtr = new StrPtr(code); return Delegates.PyRun_StringFlags(codePtr, st, globals, locals, Utf8String); } @@ -813,14 +813,15 @@ internal static NewReference PyRun_String(string code, RunFlagType st, BorrowedR /// internal static NewReference Py_CompileString(string str, string file, int start) { - using var strPtr = new StrPtr(str, Encodings.UTF8); + using var strPtr = new StrPtr(str); + using var fileObj = new PyString(file); return Delegates.Py_CompileStringObject(strPtr, fileObj, start, Utf8String, -1); } internal static NewReference PyImport_ExecCodeModule(string name, BorrowedReference code) { - using var namePtr = new StrPtr(name, Encodings.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyImport_ExecCodeModule(namePtr, code); } @@ -867,13 +868,13 @@ internal static bool PyObject_IsIterable(BorrowedReference ob) internal static int PyObject_HasAttrString(BorrowedReference pointer, string name) { - using var namePtr = new StrPtr(name, Encodings.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyObject_HasAttrString(pointer, namePtr); } internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, string name) { - using var namePtr = new StrPtr(name, Encodings.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyObject_GetAttrString(pointer, namePtr); } @@ -884,12 +885,12 @@ internal static NewReference PyObject_GetAttrString(BorrowedReference pointer, S internal static int PyObject_DelAttr(BorrowedReference @object, BorrowedReference name) => Delegates.PyObject_SetAttr(@object, name, null); internal static int PyObject_DelAttrString(BorrowedReference @object, string name) { - using var namePtr = new StrPtr(name, Encodings.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyObject_SetAttrString(@object, namePtr, null); } internal static int PyObject_SetAttrString(BorrowedReference @object, string name, BorrowedReference value) { - using var namePtr = new StrPtr(name, Encodings.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyObject_SetAttrString(@object, namePtr, value); } @@ -1071,7 +1072,7 @@ internal static bool PyBool_CheckExact(BorrowedReference ob) internal static NewReference PyLong_FromString(string value, int radix) { - using var valPtr = new StrPtr(value, Encodings.UTF8); + using var valPtr = new StrPtr(value); return Delegates.PyLong_FromString(valPtr, IntPtr.Zero, radix); } @@ -1274,7 +1275,7 @@ internal static NewReference EmptyPyBytes() internal static NewReference PyByteArray_FromStringAndSize(IntPtr strPtr, nint len) => Delegates.PyByteArray_FromStringAndSize(strPtr, len); internal static NewReference PyByteArray_FromStringAndSize(string s) { - using var ptr = new StrPtr(s, Encodings.UTF8); + using var ptr = new StrPtr(s); return PyByteArray_FromStringAndSize(ptr.RawPointer, checked((nint)ptr.ByteCount)); } @@ -1302,7 +1303,7 @@ internal static IntPtr PyBytes_AsString(BorrowedReference ob) internal static NewReference PyUnicode_InternFromString(string s) { - using var ptr = new StrPtr(s, Encodings.UTF8); + using var ptr = new StrPtr(s); return Delegates.PyUnicode_InternFromString(ptr); } @@ -1377,7 +1378,7 @@ internal static bool PyDict_Check(BorrowedReference ob) internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer, string key) { - using var keyStr = new StrPtr(key, Encodings.UTF8); + using var keyStr = new StrPtr(key); return Delegates.PyDict_GetItemString(pointer, keyStr); } @@ -1393,7 +1394,7 @@ internal static BorrowedReference PyDict_GetItemString(BorrowedReference pointer /// internal static int PyDict_SetItemString(BorrowedReference dict, string key, BorrowedReference value) { - using var keyPtr = new StrPtr(key, Encodings.UTF8); + using var keyPtr = new StrPtr(key); return Delegates.PyDict_SetItemString(dict, keyPtr, value); } @@ -1402,7 +1403,7 @@ internal static int PyDict_SetItemString(BorrowedReference dict, string key, Bor internal static int PyDict_DelItemString(BorrowedReference pointer, string key) { - using var keyPtr = new StrPtr(key, Encodings.UTF8); + using var keyPtr = new StrPtr(key); return Delegates.PyDict_DelItemString(pointer, keyPtr); } @@ -1517,7 +1518,7 @@ internal static bool PyIter_Check(BorrowedReference ob) internal static NewReference PyModule_New(string name) { - using var namePtr = new StrPtr(name, Encodings.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyModule_New(namePtr); } @@ -1531,7 +1532,7 @@ internal static NewReference PyModule_New(string name) /// Return -1 on error, 0 on success. internal static int PyModule_AddObject(BorrowedReference module, string name, StolenReference value) { - using var namePtr = new StrPtr(name, Encodings.UTF8); + using var namePtr = new StrPtr(name); IntPtr valueAddr = value.DangerousGetAddressOrNull(); int res = Delegates.PyModule_AddObject(module, namePtr, valueAddr); // We can't just exit here because the reference is stolen only on success. @@ -1549,7 +1550,7 @@ internal static int PyModule_AddObject(BorrowedReference module, string name, St internal static NewReference PyImport_ImportModule(string name) { - using var namePtr = new StrPtr(name, Encodings.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyImport_ImportModule(namePtr); } @@ -1558,7 +1559,7 @@ internal static NewReference PyImport_ImportModule(string name) internal static BorrowedReference PyImport_AddModule(string name) { - using var namePtr = new StrPtr(name, Encodings.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PyImport_AddModule(namePtr); } @@ -1586,13 +1587,13 @@ internal static void PySys_SetArgvEx(int argc, string[] argv, int updatepath) internal static BorrowedReference PySys_GetObject(string name) { - using var namePtr = new StrPtr(name, Encodings.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PySys_GetObject(namePtr); } internal static int PySys_SetObject(string name, BorrowedReference ob) { - using var namePtr = new StrPtr(name, Encodings.UTF8); + using var namePtr = new StrPtr(name); return Delegates.PySys_SetObject(namePtr, ob); } @@ -1691,7 +1692,7 @@ internal static IntPtr PyMem_Malloc(long size) internal static void PyErr_SetString(BorrowedReference ob, string message) { - using var msgPtr = new StrPtr(message, Encodings.UTF8); + using var msgPtr = new StrPtr(message); Delegates.PyErr_SetString(ob, msgPtr); } From 9ebfbde35eef742e39b80d4fe2464bdece4e34be Mon Sep 17 00:00:00 2001 From: Victor Date: Mon, 1 Jul 2024 23:13:03 -0700 Subject: [PATCH 18/21] Fix crash when event does not have `Add` method and improve message for some other internal errors (#2409) * not all events have Add method fixes https://github.com/pythonnet/pythonnet/discussions/2405 * give users some idea of why we might be unable to reflect .NET types to Python for them * mentioned event Add method crash fix in changelog --- CHANGELOG.md | 1 + src/runtime/ClassManager.cs | 4 ++- src/runtime/InternalPythonnetException.cs | 9 +++++++ src/runtime/Types/ReflectedClrType.cs | 31 ++++++++++++++--------- 4 files changed, 32 insertions(+), 13 deletions(-) create mode 100644 src/runtime/InternalPythonnetException.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 23184258d..829180f40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Fixed - Fixed RecursionError for reverse operators on C# operable types from python. See #2240 +- Fixed crash when .NET event has no `AddMethod` - Fixed probing for assemblies in `sys.path` failing when a path in `sys.path` has invalid characters. See #2376 ## [3.0.3](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.3) - 2023-10-11 diff --git a/src/runtime/ClassManager.cs b/src/runtime/ClassManager.cs index ecb6055a8..d743bc006 100644 --- a/src/runtime/ClassManager.cs +++ b/src/runtime/ClassManager.cs @@ -290,11 +290,13 @@ internal static void InitClassBase(Type type, ClassBase impl, ReflectedClrType p internal static bool ShouldBindMethod(MethodBase mb) { + if (mb is null) throw new ArgumentNullException(nameof(mb)); return (mb.IsPublic || mb.IsFamily || mb.IsFamilyOrAssembly); } internal static bool ShouldBindField(FieldInfo fi) { + if (fi is null) throw new ArgumentNullException(nameof(fi)); return (fi.IsPublic || fi.IsFamily || fi.IsFamilyOrAssembly); } @@ -326,7 +328,7 @@ internal static bool ShouldBindProperty(PropertyInfo pi) internal static bool ShouldBindEvent(EventInfo ei) { - return ShouldBindMethod(ei.GetAddMethod(true)); + return ei.GetAddMethod(true) is { } add && ShouldBindMethod(add); } private static ClassInfo GetClassInfo(Type type, ClassBase impl) diff --git a/src/runtime/InternalPythonnetException.cs b/src/runtime/InternalPythonnetException.cs new file mode 100644 index 000000000..d0ea1bece --- /dev/null +++ b/src/runtime/InternalPythonnetException.cs @@ -0,0 +1,9 @@ +using System; + +namespace Python.Runtime; + +public class InternalPythonnetException : Exception +{ + public InternalPythonnetException(string message, Exception innerException) + : base(message, innerException) { } +} diff --git a/src/runtime/Types/ReflectedClrType.cs b/src/runtime/Types/ReflectedClrType.cs index 3d0aa7e99..df9b26c29 100644 --- a/src/runtime/Types/ReflectedClrType.cs +++ b/src/runtime/Types/ReflectedClrType.cs @@ -30,22 +30,29 @@ public static ReflectedClrType GetOrCreate(Type type) return pyType; } - // Ensure, that matching Python type exists first. - // It is required for self-referential classes - // (e.g. with members, that refer to the same class) - pyType = AllocateClass(type); - ClassManager.cache.Add(type, pyType); + try + { + // Ensure, that matching Python type exists first. + // It is required for self-referential classes + // (e.g. with members, that refer to the same class) + pyType = AllocateClass(type); + ClassManager.cache.Add(type, pyType); - var impl = ClassManager.CreateClass(type); + var impl = ClassManager.CreateClass(type); - TypeManager.InitializeClassCore(type, pyType, impl); + TypeManager.InitializeClassCore(type, pyType, impl); - ClassManager.InitClassBase(type, impl, pyType); + ClassManager.InitClassBase(type, impl, pyType); - // Now we force initialize the Python type object to reflect the given - // managed type, filling the Python type slots with thunks that - // point to the managed methods providing the implementation. - TypeManager.InitializeClass(pyType, impl, type); + // Now we force initialize the Python type object to reflect the given + // managed type, filling the Python type slots with thunks that + // point to the managed methods providing the implementation. + TypeManager.InitializeClass(pyType, impl, type); + } + catch (Exception e) + { + throw new InternalPythonnetException($"Failed to create Python type for {type.FullName}", e); + } return pyType; } From c99cdf3efef20451b96417d9422e21b8bcbf2cf4 Mon Sep 17 00:00:00 2001 From: Frank Witscher Date: Mon, 8 Jul 2024 09:49:16 +0200 Subject: [PATCH 19/21] Throw exception trying to add a reflected object after the hashset is cleared --- src/runtime/Runtime.cs | 2 ++ src/runtime/Types/ClrObject.cs | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index b3820270c..3b9b0ce48 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -158,6 +158,7 @@ internal static void Initialize(bool initSigs = false) ClassManager.Reset(); ClassDerivedObject.Reset(); TypeManager.Initialize(); + CLRObject.creationBlocked = false; _typesInitialized = true; // Initialize modules that depend on the runtime class. @@ -356,6 +357,7 @@ static bool TryCollectingGarbage(int runs, bool forceBreakLoops, { NullGCHandles(CLRObject.reflectedObjects); CLRObject.reflectedObjects.Clear(); + CLRObject.creationBlocked = true; } } return false; diff --git a/src/runtime/Types/ClrObject.cs b/src/runtime/Types/ClrObject.cs index 4cf9062cb..afa136414 100644 --- a/src/runtime/Types/ClrObject.cs +++ b/src/runtime/Types/ClrObject.cs @@ -11,10 +11,15 @@ internal sealed class CLRObject : ManagedType { internal readonly object inst; + internal static bool creationBlocked = false; + // "borrowed" references internal static readonly HashSet reflectedObjects = new(); static NewReference Create(object ob, BorrowedReference tp) { + if (creationBlocked) + throw new InvalidOperationException("Reflected objects should not be created anymore."); + Debug.Assert(tp != null); var py = Runtime.PyType_GenericAlloc(tp, 0); @@ -61,6 +66,9 @@ internal static void Restore(object ob, BorrowedReference pyHandle, Dictionary? context) { + if (creationBlocked) + throw new InvalidOperationException("Reflected objects should not be loaded anymore."); + base.OnLoad(ob, context); GCHandle gc = GCHandle.Alloc(this); SetGCHandle(ob, gc); From 6cdd6d7d7b7c50781390c5e59978cdc90967ef97 Mon Sep 17 00:00:00 2001 From: Frank Witscher Date: Mon, 5 Aug 2024 09:46:07 +0200 Subject: [PATCH 20/21] Move reflected object creation block Otherwise if you have a Python object that needs to temporarily create a .NET object in its destructor (for instance write log summary to a file), its destructor will fail if it happens to be freed on the second iteration of loop breaking. --- src/runtime/Runtime.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 3b9b0ce48..4fa4f5957 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -281,6 +281,7 @@ internal static void Shutdown() TryCollectingGarbage(MaxCollectRetriesOnShutdown, forceBreakLoops: true, obj: true, derived: false, buffer: false); + CLRObject.creationBlocked = true; NullGCHandles(ExtensionType.loadedExtensions); ClassManager.RemoveClasses(); @@ -357,7 +358,6 @@ static bool TryCollectingGarbage(int runs, bool forceBreakLoops, { NullGCHandles(CLRObject.reflectedObjects); CLRObject.reflectedObjects.Clear(); - CLRObject.creationBlocked = true; } } return false; From 6690310376b0b499be39bab59f15a54f31be196a Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 19 Sep 2024 19:21:33 +0200 Subject: [PATCH 21/21] Bump to 3.0.4 --- CHANGELOG.md | 20 ++++++++++++-------- version.txt | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1355bd692..66f66670e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,25 +5,29 @@ project adheres to [Semantic Versioning][]. This document follows the conventions laid out in [Keep a CHANGELOG][]. -## [Unreleased][] +## [3.0.4](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.4) - 2024-09-19 ### Added -- Added `ToPythonAs()` extension method to allow for explicit conversion using a specific type. ([#2311][i2311]) - -- Added `IComparable` and `IEquatable` implementations to `PyInt`, `PyFloat`, and `PyString` - to compare with primitive .NET types like `long`. +- Added `ToPythonAs()` extension method to allow for explicit conversion + using a specific type. ([#2311][i2311]) +- Added `IComparable` and `IEquatable` implementations to `PyInt`, `PyFloat`, + and `PyString` to compare with primitive .NET types like `long`. ### Changed -- Added a `FormatterFactory` member in RuntimeData to create formatters with parameters. For compatibility, the `FormatterType` member is still present and has precedence when defining both `FormatterFactory` and `FormatterType` -- Added a post-serialization and a pre-deserialization step callbacks to extend (de)serialization process +- Added a `FormatterFactory` member in RuntimeData to create formatters with + parameters. For compatibility, the `FormatterType` member is still present + and has precedence when defining both `FormatterFactory` and `FormatterType` +- Added a post-serialization and a pre-deserialization step callbacks to + extend (de)serialization process - Added an API to stash serialized data on Python capsules ### Fixed - Fixed RecursionError for reverse operators on C# operable types from python. See #2240 - Fixed crash when .NET event has no `AddMethod` -- Fixed probing for assemblies in `sys.path` failing when a path in `sys.path` has invalid characters. See #2376 +- Fixed probing for assemblies in `sys.path` failing when a path in `sys.path` + has invalid characters. See #2376 - Fixed possible access violation exception on shutdown. See ([#1977][i1977]) ## [3.0.3](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.3) - 2023-10-11 diff --git a/version.txt b/version.txt index 0f9d6b15d..b0f2dcb32 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.1.0-dev +3.0.4