Merge branch 'main' into reusable_state

Author: mr-martian

.gitignore

@@ -26,6 +26,7 @@ install_manifest.txt
 *.so
 *.dylib
 *.dll
+lttoolbox/liblttoolbox.so.*
 
 # Fortran module files
 *.mod
@@ -46,6 +47,7 @@ install_manifest.txt
 /lttoolbox/lt-comp
 /lttoolbox/lt-compose
 /lttoolbox/lt-proc
+/lttoolbox/lt-merge
 /lttoolbox/lt-trim
 /lttoolbox/Makefile
 /lttoolbox/Makefile.in

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
 cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
 project(lttoolbox
-	VERSION 3.7.14
+	VERSION 3.8.2
 	LANGUAGES CXX C
 	)
 set(VERSION ${PROJECT_VERSION})

cmake.sh

@@ -2,7 +2,7 @@
 set -e
 args=()
 
-while [[ $# > 0 ]];
+while [[ $# -gt 0 ]];
 do
 	case "$1" in
 	--prefix)

lttoolbox/CMakeLists.txt

@@ -26,6 +26,7 @@ set(LIBLTTOOLBOX_HEADERS
 	sorted_vector.h
 	sorted_vector.hpp
 	state.h
+	stream_reader.h
 	string_utils.h
 	symbol_iter.h
 	tmx_compiler.h
@@ -57,6 +58,7 @@ set(LIBLTTOOLBOX_SOURCES
 	reusable_state.cc
 	sorted_vector.cc
 	state.cc
+	stream_reader.cc
 	string_utils.cc
 	symbol_iter.cc
 	tmx_compiler.cc
@@ -103,6 +105,9 @@ target_link_libraries(lt-comp lttoolbox ${GETOPT_LIB})
 add_executable(lt-proc lt_proc.cc)
 target_link_libraries(lt-proc lttoolbox ${GETOPT_LIB})
 
+add_executable(lt-merge lt_merge.cc)
+target_link_libraries(lt-merge lttoolbox ${GETOPT_LIB})
+
 add_executable(lt-expand lt_expand.cc)
 target_link_libraries(lt-expand lttoolbox ${GETOPT_LIB})
 
@@ -144,11 +149,11 @@ install(TARGETS lttoolbox
 	ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
 install(FILES ${LIBLTTOOLBOX_HEADERS}
 	DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/lttoolbox)
-install(TARGETS lt-append lt-print lt-trim lt-compose lt-comp lt-proc lt-expand lt-paradigm lt-tmxcomp lt-tmxproc lt-invert lt-restrict lt-apply-acx
+install(TARGETS lt-append lt-print lt-trim lt-compose lt-comp lt-proc lt-merge lt-expand lt-paradigm lt-tmxcomp lt-tmxproc lt-invert lt-restrict lt-apply-acx
 	RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
 
 install(FILES dix.dtd dix.rng dix.rnc acx.rng xsd/dix.xsd xsd/acx.xsd
 	DESTINATION ${CMAKE_INSTALL_DATADIR}/lttoolbox)
 
-install(FILES lt-append.1 lt-comp.1 lt-expand.1 lt-paradigm.1 lt-proc.1 lt-tmxcomp.1 lt-tmxproc.1 lt-print.1 lt-trim.1 lt-compose.1
+install(FILES lt-append.1 lt-comp.1 lt-expand.1 lt-paradigm.1 lt-proc.1 lt-merge.1 lt-tmxcomp.1 lt-tmxproc.1 lt-print.1 lt-trim.1 lt-compose.1
 	DESTINATION ${CMAKE_INSTALL_MANDIR}/man1)

lttoolbox/att_compiler.cc

@@ -19,9 +19,9 @@
 #include <lttoolbox/alphabet.h>
 #include <lttoolbox/transducer.h>
 #include <lttoolbox/compression.h>
-#include <lttoolbox/string_utils.h>
 #include <lttoolbox/file_utils.h>
 #include <algorithm>
+#include <limits>
 #include <stack>
 #include <unicode/uchar.h>
 #include <unicode/ustring.h>
@@ -141,6 +141,53 @@ AttCompiler::add_transition(int from, int to,
   }
 }
 
+/*
+ * ICU number parsing does a lot of locale handling and this tends to
+ * dominate the runtime of lt-comp. Since we always force the locale
+ * to be C.UTF-8, we can make stronger assumptions leading to a
+ * ~2000x speedup for stoi and ~500x for stod relative to StringUtils.
+ * - DGS 2025-08-29
+ */
+
+int fast_stoi(UString s) {
+  int ret = 0;
+  for (size_t i = 0; i < s.size(); i++) {
+    if (s[i] < '0' || s[i] > '9') throw std::invalid_argument("bad int");
+    ret *= 10;
+    ret += (s[i] - '0');
+  }
+  return ret;
+}
+
+double fast_stod(UString s) {
+  if (s.size() == 0) throw std::invalid_argument("empty string");
+  double sign = 1;
+  size_t i = 0;
+  if (s[i] == '-') {
+    i++;
+    sign = -1;
+  }
+  if (i == s.size()) throw std::invalid_argument("no number");
+  if (i + 3 == s.size() && s[i] == 'i' && s[i+1] == 'n' && s[i+2] == 'f') {
+    return sign * std::numeric_limits<double>::infinity();
+  }
+  double ret = 0;
+  for (; i < s.size(); i++) {
+    if (s[i] == '.') break;
+    if (s[i] < '0' || s[i] > '9') throw std::invalid_argument("bad digit");
+    ret *= 10;
+    ret += (s[i] - '0');
+  }
+  i++;
+  double mul = 0.1;
+  for (; i < s.size(); i++) {
+    if (s[i] < '0' || s[i] > '9') throw std::invalid_argument("bad digit");
+    ret += (s[i] - '0') * mul;
+    mul *= 0.1;
+  }
+  return sign * ret;
+}
+
 void
 AttCompiler::parse(std::string const &file_name, bool read_rl)
 {
@@ -208,7 +255,7 @@ AttCompiler::parse(std::string const &file_name, bool read_rl)
     }
 
     try {
-      from = StringUtils::stoi(tokens[0]) + state_id_offset;
+      from = fast_stoi(tokens[0]) + state_id_offset;
     } catch (const std::invalid_argument& e) {
       std::cerr << "Error: invalid source state in file '" << file_name << "' on line " << line_number << "." << std::endl;
       exit(EXIT_FAILURE);
@@ -233,7 +280,7 @@ AttCompiler::parse(std::string const &file_name, bool read_rl)
       if (tokens.size() > 1)
       {
 	try {
-	  weight = StringUtils::stod(tokens[1]);
+    weight = fast_stod(tokens[1]);
 	} catch (const std::invalid_argument& e) {
 	  std::cerr << "Error: invalid weight in file '" << file_name << "' on line " << line_number << "." << std::endl;
 	  exit(EXIT_FAILURE);
@@ -248,7 +295,7 @@ AttCompiler::parse(std::string const &file_name, bool read_rl)
     else
     {
       try {
-	to = StringUtils::stoi(tokens[1]) + state_id_offset;
+        to = fast_stoi(tokens[1]) + state_id_offset;
       } catch (const std::invalid_argument& e) {
 	std::cerr << "Error: invalid target state in file '" << file_name << "' on line " << line_number << "." << std::endl;
 	exit(EXIT_FAILURE);
@@ -269,7 +316,7 @@ AttCompiler::parse(std::string const &file_name, bool read_rl)
       if(tokens.size() > 4)
       {
 	try {
-	  weight = StringUtils::stod(tokens[4]);
+    weight = fast_stod(tokens[4]);
 	} catch (const std::invalid_argument& e) {
 	  std::cerr << "Error: invalid weight in file '" << file_name << "' on line " << line_number << "." << std::endl;
 	  exit(EXIT_FAILURE);