Apply dictionary word break only to runs of dictionary script

Author: kipcole9

.github/workflows/ci.yml

@@ -58,7 +58,7 @@ jobs:
     # never share BEAMs compiled by a different toolchain.
     - name: Cache deps
       id: cache-deps
-      uses: actions/cache@v4
+      uses: actions/cache@v5
       env:
         cache-name: cache-elixir-deps
       with:
@@ -72,7 +72,7 @@ jobs:
     # project's downloaded deps every run.
     - name: Cache compiled build
       id: cache-build
-      uses: actions/cache@v4
+      uses: actions/cache@v5
       env:
         cache-name: cache-compiled-build
       with:
@@ -114,9 +114,24 @@ jobs:
       if: ${{ matrix.lint }}
       run: mix test --cover
 
+    # Step: Cache the dialyzer PLT. Keyed on the toolchain as well as the
+    # lockfile since a PLT is tagged with the BEAM that built it. Without
+    # this the PLT is rebuilt from scratch on every run, which takes several
+    # minutes.
+    - name: Restore PLT cache
+      if: ${{ matrix.lint }}
+      uses: actions/cache@v5
+      with:
+        path: _build/dev/dialyxir_*
+        key: plt-${{ matrix.elixir }}-${{ matrix.otp }}-${{ hashFiles('mix.lock') }}
+        restore-keys: |
+          plt-${{ matrix.elixir }}-${{ matrix.otp }}-
+
     # Step: Execute dialyzer. Runs on the lint entry only; a single
     # analysis per change is sufficient and avoids rebuilding a PLT for
     # every toolchain in the matrix.
     - name: Run dialyzer
       if: ${{ matrix.lint }}
+      env:
+        MIX_ENV: dev
       run: mix dialyzer

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## Unicode String v2.3.1
+
+This is the changelog for Unicode String v2.3.1 released on _unreleased_.  For older changelogs please consult the release tag on [GitHub](https://github.com/elixir-unicode/unicode_string/tags)
+
+### Bug Fixes
+
+* Word breaking in a dictionary locale now applies the dictionary only to runs of text written in the script(s) that dictionary covers, with the standard Unicode rules governing everything else. Previously `Unicode.String.split("Japanese", break: :word, locale: :ja)` returned each letter separately.
+
+* Word and line breaking no longer raise a `File.Error` when the ICU dictionaries have not been downloaded with `mix unicode.string.download.dictionaries`. Segmentation now falls back to the standard Unicode rules, and `Unicode.String.break/2` and `Unicode.String.splitter/2` return `{:error, reason}`.
+
 ## Unicode String v2.3.0
 
 This is the changelog for Unicode String v2.3.0 released on July 23rd, 2026.  For older changelogs please consult the release tag on [GitHub](https://github.com/elixir-unicode/unicode_string/tags)

lib/unicode/break.ex

@@ -213,34 +213,95 @@ defmodule Unicode.String.Break do
   @doc false
   def next("", _locale, _break, _options), do: nil
 
-  def next(string, locale, :word = break, options) when locale in @dictionary_locales do
-    <<char::utf8, rest::binary>> = string
+  def next(string, locale, :word = break, options)
+      when locale in @dictionary_locales do
+    # The guard restricts `locale` to the locales that have a dictionary, so
+    # this always resolves.
+    {:ok, dictionary} = Dictionary.dictionary_locale(locale)
+
+    # Without a downloaded dictionary the trie lookups find nothing and every
+    # character becomes its own segment. The standard rules are a much better
+    # answer than that, so fall back to them.
+    if Dictionary.loaded?(dictionary) do
+      next_word_with_dictionary(string, locale, dictionary, break, options)
+    else
+      next_by_rules(string, locale, break, options)
+    end
+  end
+
+  def next(string, locale, break, options) when break in @break_keys and is_binary(string) do
+    next_by_rules(string, locale, break, options)
+  end
 
-    {<<char::utf8>>, rest}
-    |> next_dict(locale, options)
+  # A dictionary only knows how to segment the script(s) it covers. Applying it
+  # to anything else - Latin words, digits, punctuation - shatters that text
+  # into single characters, so only runs of dictionary script go to the
+  # dictionary and everything else goes to the standard rules.
+
+  defp next_word_with_dictionary(string, locale, dictionary, break, options) do
+    string
+    |> word_pair_with_dictionary(dictionary, locale, break, options)
     |> repeat_if_trimming_required(locale, break, options, options[:trim])
   end
 
-  def next(string, locale, break, options) when break in @break_keys and is_binary(string) do
-    pair =
-      case Map.fetch!(@break_map, break) do
-        :grapheme_cluster_break ->
-          G.next(string)
+  defp word_pair_with_dictionary(
+         <<char::utf8, rest::binary>> = string,
+         dictionary,
+         locale,
+         break,
+         options
+       ) do
+    if Dictionary.dictionary_script?(char, dictionary) do
+      next_dict({<<char::utf8>>, rest}, dictionary, options)
+    else
+      string
+      |> break_pair(locale, break, options)
+      |> truncate_at_dictionary_run(dictionary)
+    end
+  end
 
-        :word_break ->
-          W.next(string)
+  # Not valid UTF-8. Leave it to the standard rules to deal with.
+  defp word_pair_with_dictionary(string, _dictionary, locale, break, options) do
+    break_pair(string, locale, break, options)
+  end
 
-        :sentence_break ->
-          S.next(string, locale, sentence_suppressions(locale, options))
+  # A rule such as WB13b (`$ExtendNumLet × $Katakana`) can carry a segment past
+  # the start of a dictionary script run. Cut the segment back so the
+  # dictionary, not the rules, decides how that run is broken.
 
-        :line_break ->
-          L.next(string)
-      end
+  defp truncate_at_dictionary_run(nil, _dictionary) do
+    nil
+  end
 
-    pair
+  defp truncate_at_dictionary_run({segment, rest}, dictionary) do
+    case Dictionary.split_at_dictionary_run(segment, dictionary) do
+      {^segment, ""} -> {segment, rest}
+      {truncated, remaining} -> {truncated, remaining <> rest}
+    end
+  end
+
+  defp next_by_rules(string, locale, break, options) do
+    string
+    |> break_pair(locale, break, options)
     |> repeat_if_trimming_required(locale, break, options, options[:trim])
   end
 
+  defp break_pair(string, locale, break, options) do
+    case Map.fetch!(@break_map, break) do
+      :grapheme_cluster_break ->
+        G.next(string)
+
+      :word_break ->
+        W.next(string)
+
+      :sentence_break ->
+        S.next(string, locale, sentence_suppressions(locale, options))
+
+      :line_break ->
+        L.next(string)
+    end
+  end
+
   defp repeat_if_trimming_required(nil, _locale, _break, _options, _) do
     nil
   end
@@ -263,6 +324,17 @@ defmodule Unicode.String.Break do
 
   defp next_dict({string_before, string_after}, locale, options) do
     <<next::utf8, rest::binary>> = string_after
+
+    if Dictionary.dictionary_script?(next, locale) do
+      next_dict_word({string_before, string_after}, {next, rest}, locale, options)
+    else
+      # The end of the dictionary script run. The standard rules take over from
+      # here so the dictionary must not consume any further.
+      {string_before, string_after}
+    end
+  end
+
+  defp next_dict_word({string_before, string_after}, {next, rest}, locale, options) do
     word = string_before <> <<next::utf8>>
 
     case Dictionary.find_prefix(word, locale) do

lib/unicode/dictionary.ex

@@ -16,6 +16,12 @@ defmodule Unicode.String.Dictionary do
   they are under an open source license and also for consistency with
   [ICU](https://icu.unicode.org).
 
+  A dictionary is applied only to runs of text written in the script(s) it covers -
+  Han, Hiragana and Katakana for Chinese and Japanese, and the corresponding script
+  for Thai, Lao, Khmer and Burmese. Text in any other script, such as Latin words or
+  digits embedded in Japanese text, is broken by the standard
+  [Unicode Segmentation](https://unicode.org/reports/tr29/) rules.
+
   Note that these dictionaries need to be downloaded with
   `mix unicode.string.download.dictionaries` prior to use. Each dictionary
   will be parsed and loaded into [persistent_term](https://www.erlang.org/doc/man/persistent_term)
@@ -34,6 +40,8 @@ defmodule Unicode.String.Dictionary do
 
   alias Unicode.String.Trie
 
+  require Unicode.Set
+
   @app_name :unicode_string
   @dictionary_dir "dictionaries/"
 
@@ -108,19 +116,97 @@ defmodule Unicode.String.Dictionary do
     :persistent_term.get({@app_name, locale}, nil)
   end
 
+  # The characters each dictionary is able to segment. These sets mirror the
+  # ones used by the ICU dictionary break engines so that a dictionary is
+  # applied only to text written in the script(s) it actually covers. Any other
+  # text - Latin words, digits, punctuation - is segmented by the standard
+  # Unicode word break rules.
+  #
+  # See https://github.com/unicode-org/icu/blob/main/icu4c/source/common/dictbe.cpp
+
+  @doc false
+  def dictionary_script?(codepoint, dictionary)
+
+  # ー and ｰ are the prolonged sound marks and ﾞ and ﾟ the halfwidth voiced
+  # sound marks. All four are Script=Common but only occur in Japanese text.
+  def dictionary_script?(codepoint, :zh)
+      when Unicode.Set.match?(
+             codepoint,
+             "[[:sc=Han:][:sc=Hiragana:][:sc=Katakana:]\\u30FC\\uFF70\\uFF9E\\uFF9F]"
+           ) do
+    true
+  end
+
+  def dictionary_script?(codepoint, :th)
+      when Unicode.Set.match?(codepoint, "[[:sc=Thai:]&[:lb=SA:]]") do
+    true
+  end
+
+  def dictionary_script?(codepoint, :lo)
+      when Unicode.Set.match?(codepoint, "[[:sc=Lao:]&[:lb=SA:]]") do
+    true
+  end
+
+  def dictionary_script?(codepoint, :km)
+      when Unicode.Set.match?(codepoint, "[[:sc=Khmer:]&[:lb=SA:]]") do
+    true
+  end
+
+  def dictionary_script?(codepoint, :my)
+      when Unicode.Set.match?(codepoint, "[[:sc=Myanmar:]&[:lb=SA:]]") do
+    true
+  end
+
+  def dictionary_script?(codepoint, _dictionary) when is_integer(codepoint) do
+    false
+  end
+
+  # Splits `string` at the start of the first run of dictionary script,
+  # returning `{text_before_the_run, run_and_everything_after_it}`. When there
+  # is no dictionary script in `string` the second element is `""`.
+
+  @doc false
+  def split_at_dictionary_run(string, dictionary) do
+    bytes = bytes_before_dictionary_run(string, dictionary, 0)
+    <<before_run::binary-size(^bytes), from_run::binary>> = string
+    {before_run, from_run}
+  end
+
+  defp bytes_before_dictionary_run(<<codepoint::utf8, rest::binary>> = string, dictionary, bytes) do
+    if dictionary_script?(codepoint, dictionary) do
+      bytes
+    else
+      bytes_before_dictionary_run(rest, dictionary, bytes + byte_size(string) - byte_size(rest))
+    end
+  end
+
+  # Either the end of the string or not valid UTF-8. Either way there is no
+  # dictionary script run left to split at.
+  defp bytes_before_dictionary_run(string, _dictionary, bytes) do
+    bytes + byte_size(string)
+  end
+
+  # These are called for every character of a dictionary script run so they
+  # take the dictionary directly from :persistent_term. A dictionary that was
+  # never downloaded is absent rather than empty, hence the explicit default.
+
   @doc false
   def has_key(string, locale) do
     with {:ok, locale} <- dictionary_locale(locale) do
-      dictionary = :persistent_term.get({@app_name, locale})
-      Trie.has_key(string, dictionary)
+      case :persistent_term.get({@app_name, locale}, nil) do
+        nil -> false
+        dictionary -> Trie.has_key(string, dictionary)
+      end
     end
   end
 
   @doc false
   def find_prefix(string, locale) do
     with {:ok, locale} <- dictionary_locale(locale) do
-      dictionary = :persistent_term.get({@app_name, locale})
-      Trie.find_prefix(string, dictionary)
+      case :persistent_term.get({@app_name, locale}, nil) do
+        nil -> :error
+        dictionary -> Trie.find_prefix(string, dictionary)
+      end
     end
   end
 
@@ -138,30 +224,45 @@ defmodule Unicode.String.Dictionary do
   defp load_dictionary(locale, file_name) do
     require Logger
 
-    trie =
-      file_name
-      |> read_dictionary()
-      |> String.split("\n")
-      |> Enum.reject(&(String.starts_with?(&1, @comment_marker) or String.length(&1) == 0))
-      |> Enum.map(fn line ->
-        case String.split(line, "\t") do
-          [word] -> word
-          [word, value] -> {word, String.to_integer(value)}
-        end
-      end)
-      |> Trie.new()
+    with {:ok, contents} <- read_dictionary(file_name) do
+      trie = contents |> dictionary_entries() |> Trie.new()
+      :ok = :persistent_term.put({@app_name, locale}, trie)
+      trie = :persistent_term.get({@app_name, locale})
 
-    :ok = :persistent_term.put({@app_name, locale}, trie)
-    trie = :persistent_term.get({@app_name, locale})
+      # Logger.debug("[unicode_string] Loaded word break dictionary for locale #{inspect locale}")
+      {:ok, trie}
+    end
+  end
 
-    # Logger.debug("[unicode_string] Loaded word break dictionary for locale #{inspect locale}")
-    {:ok, trie}
+  defp dictionary_entries(contents) do
+    contents
+    |> String.split("\n")
+    |> Enum.reject(&(String.starts_with?(&1, @comment_marker) or String.length(&1) == 0))
+    |> Enum.map(&dictionary_entry/1)
+  end
+
+  defp dictionary_entry(line) do
+    case String.split(line, "\t") do
+      [word] -> word
+      [word, value] -> {word, String.to_integer(value)}
+    end
   end
 
+  # A dictionary that has not been downloaded is not an error the caller
+  # should have to rescue - word breaking falls back to the standard
+  # Unicode rules - so the read returns an error rather than raising.
+
   defp read_dictionary(file_name) do
     priv_dir = :code.priv_dir(@app_name) |> to_string
     path = Path.join(priv_dir, [@dictionary_dir, file_name])
-    File.read!(path)
+
+    case File.read(path) do
+      {:ok, contents} ->
+        {:ok, contents}
+
+      {:error, reason} ->
+        {:error, "Could not read #{inspect(path)}: #{:file.format_error(reason)}"}
+    end
   end
 
   @doc false