From 57bd9e2138063a332e7f3a1f79fd9fcfcd36ed96 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 28 Aug 2026 16:48:14 +0300 Subject: [PATCH] gh-156518: Return the simple case mappings defined by the UCD Py_UNICODE_TOUPPER(), Py_UNICODE_TOLOWER() and Py_UNICODE_TOTITLE() returned the first character of the full mapping when it is longer than one character, which the UCD does not define as a simple mapping: the simple uppercase of LATIN SMALL LETTER SHARP S is itself, not "S". Return the character itself in that case, or the simple mapping which Tools/unicode/makeunicodedata.py does not store: the Greek letters with ypogegrammeni are uppercased to the letter with prosgegrammeni, and LATIN CAPITAL LETTER I WITH DOT ABOVE is lowercased to LATIN SMALL LETTER I. _sre.unicode_iscased() no longer infers casedness from the simple mappings, which a cased character can lack, but compares the full mappings. Co-Authored-By: Claude Opus 5 (1M context) --- ...-08-28-14-11-45.gh-issue-156518.0qDYQr.rst | 6 ++++ Modules/_sre/sre.c | 8 +++-- Objects/unicodectype.c | 33 +++++++++++++++++-- 3 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-08-28-14-11-45.gh-issue-156518.0qDYQr.rst diff --git a/Misc/NEWS.d/next/C_API/2026-08-28-14-11-45.gh-issue-156518.0qDYQr.rst b/Misc/NEWS.d/next/C_API/2026-08-28-14-11-45.gh-issue-156518.0qDYQr.rst new file mode 100644 index 000000000000000..9ddfb773b024176 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-08-28-14-11-45.gh-issue-156518.0qDYQr.rst @@ -0,0 +1,6 @@ +:c:func:`Py_UNICODE_TOUPPER`, :c:func:`Py_UNICODE_TOLOWER` and +:c:func:`Py_UNICODE_TOTITLE` now return the simple case mapping defined by the +Unicode database instead of the first character of the full case mapping. The +simple uppercase of ``'ß'`` (U+00DF) is now ``'ß'`` instead of ``'S'``, and the +simple uppercase of ``'ᾀ'`` (U+1F80) is ``'ᾈ'`` (U+1F88) instead of ``'Ἀ'`` +(U+1F08). diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 031e972aeba0312..af68bdb76a358da 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -595,8 +595,12 @@ static int _sre_unicode_iscased_impl(PyObject *module, int character) /*[clinic end generated code: output=9c5ddee0dc2bc258 input=51e42c3b8dddb78e]*/ { - unsigned int ch = (unsigned int)character; - return ch != sre_lower_unicode(ch) || ch != sre_upper_unicode(ch); + Py_UCS4 ch = (Py_UCS4)character; + Py_UCS4 mapped[3]; + /* A cased character can have no simple case mapping, as LATIN SMALL + LETTER SHARP S whose uppercase is "SS". */ + return _PyUnicode_ToUpperFull(ch, mapped) != 1 || mapped[0] != ch || + _PyUnicode_ToLowerFull(ch, mapped) != 1 || mapped[0] != ch; } /*[clinic input] diff --git a/Objects/unicodectype.c b/Objects/unicodectype.c index fdd380190ac1ecc..91c10b3aaceafe9 100644 --- a/Objects/unicodectype.c +++ b/Objects/unicodectype.c @@ -64,8 +64,12 @@ Py_UCS4 _PyUnicode_ToTitlecase(Py_UCS4 ch) { const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); - if (ctype->flags & EXTENDED_CASE_MASK) + if (ctype->flags & EXTENDED_CASE_MASK) { + /* No character with a longer full titlecase has a simple one. */ + if ((ctype->title >> 24) != 1) + return ch; return _PyUnicode_ExtendedCase[ctype->title & 0xFFFF]; + } return ch + ctype->title; } @@ -182,8 +186,25 @@ Py_UCS4 _PyUnicode_ToUppercase(Py_UCS4 ch) { const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); - if (ctype->flags & EXTENDED_CASE_MASK) + if (ctype->flags & EXTENDED_CASE_MASK) { + if ((ctype->upper >> 24) != 1) { + /* Only the full mapping is stored when it is longer than one + character (see Tools/unicode/makeunicodedata.py). Most such + characters have no simple uppercase, but the Greek letters + with ypogegrammeni have the one with prosgegrammeni. All of + them are Unicode 1.1. */ + if (ch >= 0x1F80 && ch <= 0x1FA7 && !(ch & 8)) { + return ch + 8; + } + switch (ch) { + case 0x1FB3: return 0x1FBC; /* ALPHA WITH YPOGEGRAMMENI */ + case 0x1FC3: return 0x1FCC; /* ETA WITH YPOGEGRAMMENI */ + case 0x1FF3: return 0x1FFC; /* OMEGA WITH YPOGEGRAMMENI */ + } + return ch; + } return _PyUnicode_ExtendedCase[ctype->upper & 0xFFFF]; + } return ch + ctype->upper; } @@ -194,8 +215,14 @@ Py_UCS4 _PyUnicode_ToLowercase(Py_UCS4 ch) { const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); - if (ctype->flags & EXTENDED_CASE_MASK) + if (ctype->flags & EXTENDED_CASE_MASK) { + if ((ctype->lower >> 24) != 1) { + /* LATIN CAPITAL LETTER I WITH DOT ABOVE is the only character + with a longer full lowercase and a simple one. */ + return ch == 0x0130 ? 0x0069 : ch; + } return _PyUnicode_ExtendedCase[ctype->lower & 0xFFFF]; + } return ch + ctype->lower; }