diff --git a/src/specify_cli/agents.py b/src/specify_cli/agents.py index c127407d49..2c8fff004b 100644 --- a/src/specify_cli/agents.py +++ b/src/specify_cli/agents.py @@ -258,11 +258,17 @@ def render_toml_command(self, frontmatter: dict, body: str, source_id: str) -> s # ``C:\\Users\\...`` whose ``\\U`` reads as an invalid unicode escape) would # produce unparseable TOML — route those to the *literal* form ('''...'''), # which does not process escapes, or to the escaped basic string. - if '"""' not in body and "\\" not in body: + # Control characters TOML forbids in raw form (and bare CR) cannot + # appear in either multiline form, so those bodies also route to the + # escaped basic string. + from specify_cli.integrations.base import toml_contains_forbidden_ctrl + + has_forbidden_ctrl = toml_contains_forbidden_ctrl(body) + if '"""' not in body and "\\" not in body and not has_forbidden_ctrl: toml_lines.append('prompt = """') toml_lines.append(body) toml_lines.append('"""') - elif "'''" not in body: + elif "'''" not in body and not has_forbidden_ctrl: toml_lines.append("prompt = '''") toml_lines.append(body) toml_lines.append("'''") @@ -274,14 +280,9 @@ def render_toml_command(self, frontmatter: dict, body: str, source_id: str) -> s @staticmethod def _render_basic_toml_string(value: str) -> str: """Render *value* as a TOML basic string literal.""" - escaped = ( - value.replace("\\", "\\\\") - .replace('"', '\\"') - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\t", "\\t") - ) - return f'"{escaped}"' + from specify_cli.integrations.base import toml_escape_basic + + return f'"{toml_escape_basic(value)}"' def render_yaml_command( self, diff --git a/src/specify_cli/integrations/base.py b/src/specify_cli/integrations/base.py index 706a3cb5d2..6d09ad9793 100644 --- a/src/specify_cli/integrations/base.py +++ b/src/specify_cli/integrations/base.py @@ -50,6 +50,35 @@ command: index for index, command in enumerate(_CORE_COMMAND_TEMPLATE_ORDER) } +# TOML forbids these raw in every string form: C0 controls other than tab +# and newline, DEL, and a bare CR that is not part of a CRLF pair. +_TOML_FORBIDDEN_CTRL = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]|\r(?!\n)") + + +def toml_contains_forbidden_ctrl(value: str) -> bool: + """Return True if *value* contains characters TOML forbids raw.""" + return bool(_TOML_FORBIDDEN_CTRL.search(value)) + + +def toml_escape_basic(value: str) -> str: + """Escape *value* for a TOML basic string (quotes not included). + + Escapes backslashes, quotes, common whitespace escapes, and the + control characters TOML forbids in raw form (as ``\\uXXXX``). + """ + escaped = ( + value.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t") + ) + return re.sub( + r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]", + lambda match: f"\\u{ord(match.group()):04X}", + escaped, + ) + # --------------------------------------------------------------------------- # IntegrationOption @@ -960,8 +989,13 @@ def _render_toml_string(value: str) -> str: Uses a basic string for single-line values, multiline basic strings for values containing newlines, and falls back to a literal string or escaped basic string when delimiters appear in - the content. + the content. Values containing control characters TOML forbids + in raw form (or a bare CR) always use the escaped basic string, + since no other form can represent them. """ + if toml_contains_forbidden_ctrl(value): + return f'"{toml_escape_basic(value)}"' + if "\n" not in value and "\r" not in value: escaped = value.replace("\\", "\\\\").replace('"', '\\"') return f'"{escaped}"' @@ -974,17 +1008,7 @@ def _render_toml_string(value: str) -> str: if "'''" not in value and not value.endswith("'"): return "'''\n" + value + "'''" - return ( - '"' - + ( - value.replace("\\", "\\\\") - .replace('"', '\\"') - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\t", "\\t") - ) - + '"' - ) + return f'"{toml_escape_basic(value)}"' @staticmethod def _render_toml(description: str, body: str) -> str: diff --git a/tests/integrations/test_toml_control_chars.py b/tests/integrations/test_toml_control_chars.py new file mode 100644 index 0000000000..2bd1e19e62 --- /dev/null +++ b/tests/integrations/test_toml_control_chars.py @@ -0,0 +1,93 @@ +""" +Regression tests for control characters in the TOML renderers (issue #3340). + +TOML forbids raw control characters (U+0000-U+001F except tab and newline, +plus U+007F) in every string form, and a bare CR outside a CRLF pair. Both +renderers previously emitted them verbatim, producing files that parsers +reject. +""" + +import tomllib + +import pytest + +from specify_cli.agents import CommandRegistrar +from specify_cli.integrations.base import TomlIntegration, toml_escape_basic + +CTRL_SAMPLES = [ + pytest.param("ab\x07cd", id="bell"), + pytest.param("ab\x00cd", id="nul"), + pytest.param("ab\x1bcd", id="escape"), + pytest.param("ab\x7fcd", id="del"), + pytest.param("ab\rcd", id="bare-cr"), + pytest.param("line1\nbad\x07line\n", id="multiline-with-bell"), + pytest.param('has """ and \x07', id="delimiters-and-ctrl"), + pytest.param("C:\\Users\\x \x1b", id="backslash-and-ctrl"), +] + + +class TestTomlEscapeBasic: + @pytest.mark.parametrize("value", CTRL_SAMPLES) + def test_round_trips_as_basic_string(self, value: str): + parsed = tomllib.loads(f'v = "{toml_escape_basic(value)}"') + assert parsed["v"] == value + + def test_plain_strings_unchanged(self): + assert toml_escape_basic("plain text") == "plain text" + + +class TestRenderTomlString: + @pytest.mark.parametrize("value", CTRL_SAMPLES) + def test_round_trips(self, value: str): + rendered = TomlIntegration._render_toml_string(value) + parsed = tomllib.loads(f"prompt = {rendered}") + assert parsed["prompt"] == value + + def test_issue_repro(self): + # Verbatim repro from #3340. + rendered = TomlIntegration._render_toml_string("a\x07b") + assert tomllib.loads(f"prompt = {rendered}")["prompt"] == "a\x07b" + + def test_clean_single_line_output_unchanged(self): + assert TomlIntegration._render_toml_string("simple value") == '"simple value"' + + def test_clean_multiline_still_uses_multiline_form(self): + rendered = TomlIntegration._render_toml_string("line1\nline2") + assert rendered.startswith('"""') + assert tomllib.loads(f"prompt = {rendered}")["prompt"] == "line1\nline2" + + def test_crlf_pairs_stay_in_multiline_form(self): + # CRLF pairs are legal raw in multiline strings; the parser + # normalizes them to LF per the TOML newline rules. + value = "line1\r\nline2\r\n" + rendered = TomlIntegration._render_toml_string(value) + assert rendered.startswith('"""') + parsed = tomllib.loads(f"prompt = {rendered}")["prompt"] + assert parsed == "line1\nline2\n" + + def test_tab_stays_raw(self): + assert TomlIntegration._render_toml_string("a\tb") == '"a\tb"' + + +class TestRenderTomlCommand: + @pytest.fixture() + def registrar(self): + return CommandRegistrar() + + @pytest.mark.parametrize("value", CTRL_SAMPLES) + def test_body_round_trips(self, registrar, value: str): + content = registrar.render_toml_command({}, value, "ext") + assert tomllib.loads(content)["prompt"] == value + + @pytest.mark.parametrize("value", CTRL_SAMPLES) + def test_description_round_trips(self, registrar, value: str): + content = registrar.render_toml_command({"description": value}, "body", "ext") + parsed = tomllib.loads(content) + assert parsed["description"] == value + # The multiline prompt form appends a trailing newline by design. + assert parsed["prompt"] == "body\n" + + def test_clean_body_still_uses_multiline_form(self, registrar): + content = registrar.render_toml_command({}, "line1\nline2", "ext") + assert 'prompt = """' in content + assert tomllib.loads(content)["prompt"] == "line1\nline2\n"