From 4f16561ce415fd01bf1ecca1a090e2e27054c8cf Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 25 Mar 2022 07:50:08 +0000 Subject: [PATCH 1/5] tweak deepfreeze output --- Tools/scripts/deepfreeze.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 1831c15784af771..cef51f7c66fcabc 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -12,6 +12,7 @@ import re import time import types +from string import ascii_letters from typing import Dict, FrozenSet, TextIO, Tuple import umarshal @@ -116,6 +117,7 @@ def __init__(self, file: TextIO) -> None: self.write('#include "internal/pycore_gc.h"') self.write('#include "internal/pycore_code.h"') self.write('#include "internal/pycore_long.h"') + self.write('#define AS_PYOBJECT(x) ((PyObject *)(&(x)))') self.write("") @contextlib.contextmanager @@ -152,9 +154,9 @@ def field(self, obj: object, name: str) -> None: def generate_bytes(self, name: str, b: bytes) -> str: if b == b"": - return "(PyObject *)&_Py_SINGLETON(bytes_empty)" + return "AS_PYOBJECT(_Py_SINGLETON(bytes_empty))" if len(b) == 1: - return f"(PyObject *)&_Py_SINGLETON(bytes_characters[{b[0]}])" + return f"AS_PYOBJECT(_Py_SINGLETON(bytes_characters[{b[0]}]))" self.write("static") with self.indent(): with self.block("struct"): @@ -165,11 +167,13 @@ def generate_bytes(self, name: str, b: bytes) -> str: self.object_var_head("PyBytes_Type", len(b)) self.write(".ob_shash = -1,") self.write(f".ob_sval = {make_string_literal(b)},") - return f"& {name}.ob_base.ob_base" + return f"AS_PYOBJECT({name})" def generate_unicode(self, name: str, s: str) -> str: if s in identifiers: return f"&_Py_ID({s})" + if all(i in ascii_letters or i == "_" for i in s): + name = f"const_str_{s}" kind, ascii = analyze_character_width(s) if kind == PyUnicode_1BYTE_KIND: datatype = "uint8_t" @@ -185,7 +189,7 @@ def generate_unicode(self, name: str, s: str) -> str: else: self.write("PyCompactUnicodeObject _compact;") self.write(f"{datatype} _data[{len(s)+1}];") - self.deallocs.append(f"_PyStaticUnicode_Dealloc((PyObject *)&{name});") + self.deallocs.append(f"_PyStaticUnicode_Dealloc(AS_PYOBJECT({name}));") with self.block(f"{name} =", ";"): if ascii: with self.block("._ascii =", ","): @@ -198,7 +202,7 @@ def generate_unicode(self, name: str, s: str) -> str: self.write(".ascii = 1,") self.write(".ready = 1,") self.write(f"._data = {make_string_literal(s.encode('ascii'))},") - return f"& {name}._ascii.ob_base" + return f"AS_PYOBJECT({name})" else: with self.block("._compact =", ","): with self.block("._base =", ","): @@ -224,7 +228,7 @@ def generate_unicode(self, name: str, s: str) -> str: self.patchups.append(f" {name}._compact._base.wstr = (wchar_t *) {name}._data;") self.patchups.append(f" {name}._compact.wstr_length = {len(s)};") self.patchups.append("}") - return f"& {name}._compact._base.ob_base" + return f"AS_PYOBJECT({name})" def generate_code(self, name: str, code: types.CodeType) -> str: @@ -282,11 +286,11 @@ def generate_code(self, name: str, code: types.CodeType) -> str: name_as_code = f"(PyCodeObject *)&{name}" self.deallocs.append(f"_PyStaticCode_Dealloc({name_as_code});") self.interns.append(f"_PyStaticCode_InternStrings({name_as_code})") - return f"& {name}.ob_base.ob_base" + return f"AS_PYOBJECT({name})" def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: if len(t) == 0: - return f"(PyObject *)& _Py_SINGLETON(tuple_empty)" + return f"AS_PYOBJECT(_Py_SINGLETON(tuple_empty))" items = [self.generate(f"{name}_{i}", it) for i, it in enumerate(t)] self.write("static") with self.indent(): @@ -303,7 +307,7 @@ def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: with self.block(f".ob_item =", ","): for item in items: self.write(item + ",") - return f"& {name}._object.ob_base.ob_base" + return f"AS_PYOBJECT({name}._object)" def _generate_int_for_bits(self, name: str, i: int, digit: int) -> None: sign = -1 if i < 0 else 0 if i == 0 else +1 @@ -325,7 +329,7 @@ def _generate_int_for_bits(self, name: str, i: int, digit: int) -> None: def generate_int(self, name: str, i: int) -> str: if -5 <= i <= 256: - return f"(PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + {i}]" + return f"AS_PYOBJECT(_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + {i}])" if abs(i) < 2**15: self._generate_int_for_bits(name, i, 2**15) else: @@ -338,19 +342,19 @@ def generate_int(self, name: str, i: int) -> str: self.write('#error "PYLONG_BITS_IN_DIGIT should be 15 or 30"') self.write("#endif") # If neither clause applies, it won't compile - return f"& {name}.ob_base.ob_base" + return f"AS_PYOBJECT({name})" def generate_float(self, name: str, x: float) -> str: with self.block(f"static PyFloatObject {name} =", ";"): self.object_head("PyFloat_Type") self.write(f".ob_fval = {x},") - return f"&{name}.ob_base" + return f"AS_PYOBJECT({name})" def generate_complex(self, name: str, z: complex) -> str: with self.block(f"static PyComplexObject {name} =", ";"): self.object_head("PyComplex_Type") self.write(f".cval = {{ {z.real}, {z.imag} }},") - return f"&{name}.ob_base" + return f"AS_PYOBJECT({name})" def generate_frozenset(self, name: str, fs: FrozenSet[object]) -> str: ret = self.generate_tuple(name, tuple(sorted(fs))) @@ -374,7 +378,7 @@ def generate(self, name: str, obj: object) -> str: # print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}") return self.cache[key] self.misses += 1 - if isinstance(obj, (types.CodeType, umarshal.Code)) : + if isinstance(obj, (types.CodeType, umarshal.Code)): val = self.generate_code(name, obj) elif isinstance(obj, tuple): val = self.generate_tuple(name, obj) @@ -411,7 +415,7 @@ def generate(self, name: str, obj: object) -> str: _Py_get_%%NAME%%_toplevel(void) { %%NAME%%_do_patchups(); - return Py_NewRef((PyObject *) &%%NAME%%_toplevel); + return Py_NewRef(AS_PYOBJECT(%%NAME%%_toplevel)); } """ From ab60757e4831ad5af583a33bf2e50a8f129fce45 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sat, 26 Mar 2022 12:41:21 +0000 Subject: [PATCH 2/5] code review --- Tools/scripts/deepfreeze.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index cef51f7c66fcabc..f0fd29e1ff9b13d 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -117,7 +117,6 @@ def __init__(self, file: TextIO) -> None: self.write('#include "internal/pycore_gc.h"') self.write('#include "internal/pycore_code.h"') self.write('#include "internal/pycore_long.h"') - self.write('#define AS_PYOBJECT(x) ((PyObject *)(&(x)))') self.write("") @contextlib.contextmanager @@ -154,9 +153,9 @@ def field(self, obj: object, name: str) -> None: def generate_bytes(self, name: str, b: bytes) -> str: if b == b"": - return "AS_PYOBJECT(_Py_SINGLETON(bytes_empty))" + return "(PyObject *)&_Py_SINGLETON(bytes_empty)" if len(b) == 1: - return f"AS_PYOBJECT(_Py_SINGLETON(bytes_characters[{b[0]}]))" + return f"(PyObject *)&_Py_SINGLETON(bytes_characters[{b[0]}])" self.write("static") with self.indent(): with self.block("struct"): @@ -167,7 +166,7 @@ def generate_bytes(self, name: str, b: bytes) -> str: self.object_var_head("PyBytes_Type", len(b)) self.write(".ob_shash = -1,") self.write(f".ob_sval = {make_string_literal(b)},") - return f"AS_PYOBJECT({name})" + return f"& {name}.ob_base.ob_base" def generate_unicode(self, name: str, s: str) -> str: if s in identifiers: @@ -189,7 +188,7 @@ def generate_unicode(self, name: str, s: str) -> str: else: self.write("PyCompactUnicodeObject _compact;") self.write(f"{datatype} _data[{len(s)+1}];") - self.deallocs.append(f"_PyStaticUnicode_Dealloc(AS_PYOBJECT({name}));") + self.deallocs.append(f"_PyStaticUnicode_Dealloc((PyObject *)&{name});") with self.block(f"{name} =", ";"): if ascii: with self.block("._ascii =", ","): @@ -202,7 +201,7 @@ def generate_unicode(self, name: str, s: str) -> str: self.write(".ascii = 1,") self.write(".ready = 1,") self.write(f"._data = {make_string_literal(s.encode('ascii'))},") - return f"AS_PYOBJECT({name})" + return f"& {name}._ascii.ob_base" else: with self.block("._compact =", ","): with self.block("._base =", ","): @@ -228,7 +227,7 @@ def generate_unicode(self, name: str, s: str) -> str: self.patchups.append(f" {name}._compact._base.wstr = (wchar_t *) {name}._data;") self.patchups.append(f" {name}._compact.wstr_length = {len(s)};") self.patchups.append("}") - return f"AS_PYOBJECT({name})" + return f"& {name}._compact._base.ob_base" def generate_code(self, name: str, code: types.CodeType) -> str: @@ -286,11 +285,11 @@ def generate_code(self, name: str, code: types.CodeType) -> str: name_as_code = f"(PyCodeObject *)&{name}" self.deallocs.append(f"_PyStaticCode_Dealloc({name_as_code});") self.interns.append(f"_PyStaticCode_InternStrings({name_as_code})") - return f"AS_PYOBJECT({name})" + return f"& {name}.ob_base.ob_base" def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: if len(t) == 0: - return f"AS_PYOBJECT(_Py_SINGLETON(tuple_empty))" + return f"(PyObject *)& _Py_SINGLETON(tuple_empty)" items = [self.generate(f"{name}_{i}", it) for i, it in enumerate(t)] self.write("static") with self.indent(): @@ -307,7 +306,7 @@ def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: with self.block(f".ob_item =", ","): for item in items: self.write(item + ",") - return f"AS_PYOBJECT({name}._object)" + return f"& {name}._object.ob_base.ob_base" def _generate_int_for_bits(self, name: str, i: int, digit: int) -> None: sign = -1 if i < 0 else 0 if i == 0 else +1 @@ -329,7 +328,11 @@ def _generate_int_for_bits(self, name: str, i: int, digit: int) -> None: def generate_int(self, name: str, i: int) -> str: if -5 <= i <= 256: - return f"AS_PYOBJECT(_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + {i}])" + return f"(PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + {i}]" + if i > 0: + name = f"const_int_{i}" + else: + name = f"const_int_negative_{abs(i)}" if abs(i) < 2**15: self._generate_int_for_bits(name, i, 2**15) else: @@ -342,19 +345,19 @@ def generate_int(self, name: str, i: int) -> str: self.write('#error "PYLONG_BITS_IN_DIGIT should be 15 or 30"') self.write("#endif") # If neither clause applies, it won't compile - return f"AS_PYOBJECT({name})" + return f"& {name}.ob_base.ob_base" def generate_float(self, name: str, x: float) -> str: with self.block(f"static PyFloatObject {name} =", ";"): self.object_head("PyFloat_Type") self.write(f".ob_fval = {x},") - return f"AS_PYOBJECT({name})" + return f"&{name}.ob_base" def generate_complex(self, name: str, z: complex) -> str: with self.block(f"static PyComplexObject {name} =", ";"): self.object_head("PyComplex_Type") self.write(f".cval = {{ {z.real}, {z.imag} }},") - return f"AS_PYOBJECT({name})" + return f"&{name}.ob_base" def generate_frozenset(self, name: str, fs: FrozenSet[object]) -> str: ret = self.generate_tuple(name, tuple(sorted(fs))) @@ -378,7 +381,7 @@ def generate(self, name: str, obj: object) -> str: # print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}") return self.cache[key] self.misses += 1 - if isinstance(obj, (types.CodeType, umarshal.Code)): + if isinstance(obj, (types.CodeType, umarshal.Code)) : val = self.generate_code(name, obj) elif isinstance(obj, tuple): val = self.generate_tuple(name, obj) @@ -415,7 +418,7 @@ def generate(self, name: str, obj: object) -> str: _Py_get_%%NAME%%_toplevel(void) { %%NAME%%_do_patchups(); - return Py_NewRef(AS_PYOBJECT(%%NAME%%_toplevel)); + return Py_NewRef((PyObject *) &%%NAME%%_toplevel); } """ From 69c6697375345dca0f5c6ae4bb9ffe781f5ed719 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sat, 26 Mar 2022 12:52:11 +0000 Subject: [PATCH 3/5] allow digit in string --- Tools/scripts/deepfreeze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index f0fd29e1ff9b13d..cfad45c41171b4b 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -171,7 +171,7 @@ def generate_bytes(self, name: str, b: bytes) -> str: def generate_unicode(self, name: str, s: str) -> str: if s in identifiers: return f"&_Py_ID({s})" - if all(i in ascii_letters or i == "_" for i in s): + if all(i in ascii_letters or i == "_" or i.isdigit() for i in s): name = f"const_str_{s}" kind, ascii = analyze_character_width(s) if kind == PyUnicode_1BYTE_KIND: From 22acafc1ba9a98e907facee86b5a96489df8d91c Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sun, 27 Mar 2022 16:00:21 +0000 Subject: [PATCH 4/5] code review --- Tools/scripts/deepfreeze.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index cfad45c41171b4b..f4915fd6f361e80 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -12,7 +12,6 @@ import re import time import types -from string import ascii_letters from typing import Dict, FrozenSet, TextIO, Tuple import umarshal @@ -171,7 +170,7 @@ def generate_bytes(self, name: str, b: bytes) -> str: def generate_unicode(self, name: str, s: str) -> str: if s in identifiers: return f"&_Py_ID({s})" - if all(i in ascii_letters or i == "_" or i.isdigit() for i in s): + if re.match(r'^[A-Za-z0-9_]+$', s): name = f"const_str_{s}" kind, ascii = analyze_character_width(s) if kind == PyUnicode_1BYTE_KIND: @@ -329,7 +328,7 @@ def _generate_int_for_bits(self, name: str, i: int, digit: int) -> None: def generate_int(self, name: str, i: int) -> str: if -5 <= i <= 256: return f"(PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + {i}]" - if i > 0: + if i >= 0: name = f"const_int_{i}" else: name = f"const_int_negative_{abs(i)}" From 049166cb424f24863d57af6fe0c308df3d0e7d8b Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sun, 27 Mar 2022 22:05:29 +0530 Subject: [PATCH 5/5] Update Tools/scripts/deepfreeze.py --- Tools/scripts/deepfreeze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index f4915fd6f361e80..dfa4b3a8eeb0112 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -170,7 +170,7 @@ def generate_bytes(self, name: str, b: bytes) -> str: def generate_unicode(self, name: str, s: str) -> str: if s in identifiers: return f"&_Py_ID({s})" - if re.match(r'^[A-Za-z0-9_]+$', s): + if re.match(r'\A[A-Za-z0-9_]+\Z', s): name = f"const_str_{s}" kind, ascii = analyze_character_width(s) if kind == PyUnicode_1BYTE_KIND: