cut: refactor -f to byte search and character processing

Not sure about this at all.
Only worthwhile if can also remove cut_fields_line_delim.

Refactored src/cut.c so the old UTF-8 byte-search path is now the
general byte-search field engine for all safe byte-search cases:
ordinary single-byte delimiters and valid UTF-8 delimiters. The old
cut_fields path did not go away completely; it is now
cut_fields_line_delim and is used only when the field delimiter equals
the record delimiter, because -d $'\n' and -z -d '' have different
semantics from normal line-based field splitting.

As part of that, I also folded the duplicated “start selected field”
logic into a shared helper, and renamed the byte-search helpers to match
their broader use. The current dispatcher in src/cut.c is now:
whitespace parser, then line- delimiter field mode, then byte-search
field mode, then the decoded multibyte parser.

diff --git a/src/cut.c b/src/cut.c
index 909eecb3f76ecd9972bde27fa9b280c846f7f225..455a978b683e40adb2e159f6f368066f52d93d4b 100644 (file)
--- a/src/cut.c
+++ b/src/cut.c
@@ -287,10 +287,9 @@ is_range_start_index (uintmax_t k)
 }
 
 static inline bool
-single_byte_field_delim_ok (void)
+field_delim_is_line_delim (void)
 {
-  return delim_length == 1
-         && (MB_CUR_MAX <= 1 || to_uchar (delim_bytes[0]) < 0x30);
+  return delim_length == 1 && delim_bytes[0] == line_delim;
 }
 
 /* Return true if the current charset is UTF-8.  */
@@ -313,6 +312,15 @@ utf8_field_delim_ok (void)
   return ! delim_mcel.err && is_utf8_charset ();
 }
 
+static inline bool
+bytesearch_field_delim_ok (void)
+{
+  return ! field_delim_is_line_delim ()
+         && (delim_length == 1
+             ? MB_CUR_MAX <= 1 || to_uchar (delim_bytes[0]) < 0x30
+             : utf8_field_delim_ok ());
+}
+
 static inline bool
 field_delim_eq (mcel_t g)
 {
@@ -497,13 +505,13 @@ scan_mb_field (mbbuf_t *mbbuf, struct mbfield_parser *parser,
     }
 }
 
-/* Return a pointer to the next field delimiter in the UTF-8 record BUF,
-   searching LEN bytes.  Return NULL if none is found.  DELIM_BYTES must
+/* Return a pointer to the next field delimiter in BUF, searching LEN bytes.
+   Return NULL if none is found.  DELIM_BYTES must be a single byte or
    represent a valid UTF-8 character.  BUF can contain invalid/NUL bytes,
-   but must be NUL terminated.  */
+   and must have room for a trailing NUL byte at BUF[LEN].  */
 ATTRIBUTE_PURE
 static char *
-find_utf8_field_delim (char *buf, size_t len)
+find_bytesearch_field_delim (char *buf, size_t len)
 {
 #if ! __GLIBC__  /* Only S390 has optimized memmem on glibc-2.42  */
   return memmem (buf, len, delim_bytes, delim_length);
@@ -547,11 +555,11 @@ find_utf8_field_delim (char *buf, size_t len)
 }
 
 static inline char *
-find_utf8_field_terminator (char *buf, idx_t len, bool *is_delim)
+find_bytesearch_field_terminator (char *buf, idx_t len, bool *is_delim)
 {
   char *line_end = memchr ((void *) buf, line_delim, len);
   idx_t line_len = line_end ? line_end - buf : len;
-  char *field_end = find_utf8_field_delim (buf, line_len);
+  char *field_end = find_bytesearch_field_delim (buf, line_len);
 
   if (field_end)
     {
@@ -564,8 +572,8 @@ find_utf8_field_terminator (char *buf, idx_t len, bool *is_delim)
 }
 
 static inline bool
-begin_utf8_field (uintmax_t field_idx, bool buffer_first_field,
-                  bool *found_any_selected_field)
+begin_field_output (uintmax_t field_idx, bool buffer_first_field,
+                    bool *found_any_selected_field)
 {
   bool write_field = print_kth (field_idx);
 
@@ -579,6 +587,19 @@ begin_utf8_field (uintmax_t field_idx, bool buffer_first_field,
   return write_field;
 }
 
+static inline idx_t
+bytesearch_safe_prefix (mbbuf_t *mbbuf, idx_t overlap)
+{
+  idx_t available = mbbuf_fill (mbbuf, overlap + 1);
+  if (available == 0)
+    return 0;
+
+  if (feof (mbbuf->fp))
+    return available;
+
+  return overlap < available ? available - overlap : 0;
+}
+
 static inline void
 reset_field_line (uintmax_t *field_idx, bool *found_any_selected_field,
                   bool *have_pending_line, struct mbfield_parser *parser)
@@ -780,14 +801,8 @@ cut_fields_mb_any (FILE *stream, bool whitespace_mode)
         }
 
       enum field_terminator terminator;
-      bool write_field = print_kth (field_idx);
-
-      if (write_field)
-        {
-          if (found_any_selected_field)
-            write_bytes (output_delimiter_string, output_delimiter_length);
-          found_any_selected_field = true;
-        }
+      bool write_field = begin_field_output (field_idx, buffer_first_field,
+                                             &found_any_selected_field);
 
       terminator = scan_mb_field (&mbbuf, &parser, &have_pending_line,
                                   write_field, NULL);
@@ -820,10 +835,10 @@ cut_fields_mb (FILE *stream)
 }
 
 /* Read from STREAM, printing to standard output any selected fields,
-   using UTF-8-aware byte searches for the field delimiter.  */
+   using byte searches for a single-byte or valid UTF-8 field delimiter.  */
 
 static void
-cut_fields_mb_utf8 (FILE *stream)
+cut_fields_bytesearch (FILE *stream)
 {
   /* Leave 1 byte unused so space to NUL terminate (for strstr).  */
   static char line_in[IO_BUFSIZE+1];
@@ -839,12 +854,12 @@ cut_fields_mb_utf8 (FILE *stream)
   current_rp = frp;
   buffer_first_field = suppress_non_delimited ^ !print_kth (1);
   mbbuf_init (&mbbuf, line_in, sizeof line_in - 1, stream);
-  write_field = begin_utf8_field (field_idx, buffer_first_field,
-                                  &found_any_selected_field);
+  write_field = begin_field_output (field_idx, buffer_first_field,
+                                    &found_any_selected_field);
 
   while (true)
     {
-      idx_t safe = mbbuf_utf8_safe_prefix (&mbbuf, overlap);
+      idx_t safe = bytesearch_safe_prefix (&mbbuf, overlap);
       idx_t processed = 0;
 
       if (safe == 0)
@@ -860,8 +875,8 @@ cut_fields_mb_utf8 (FILE *stream)
         {
           bool is_delim = false;
           char *terminator
-            = find_utf8_field_terminator (chunk + processed, safe - processed,
-                                          &is_delim);
+            = find_bytesearch_field_terminator (chunk + processed,
+                                                safe - processed, &is_delim);
           idx_t field_len = terminator ? terminator - (chunk + processed)
                                        : safe - processed;
 
@@ -892,8 +907,8 @@ cut_fields_mb_utf8 (FILE *stream)
 
               processed += delim_length;
               next_item (&field_idx);
-              write_field = begin_utf8_field (field_idx, buffer_first_field,
-                                              &found_any_selected_field);
+              write_field = begin_field_output (field_idx, buffer_first_field,
+                                                &found_any_selected_field);
             }
           else
             {
@@ -916,8 +931,8 @@ cut_fields_mb_utf8 (FILE *stream)
               current_rp = frp;
               found_any_selected_field = false;
               have_pending_line = false;
-              write_field = begin_utf8_field (field_idx, buffer_first_field,
-                                              &found_any_selected_field);
+              write_field = begin_field_output (field_idx, buffer_first_field,
+                                                &found_any_selected_field);
             }
         }
 
@@ -952,10 +967,11 @@ cut_fields_ws (FILE *stream)
   cut_fields_mb_any (stream, true);
 }
 
-/* Read from stream STREAM, printing to standard output any selected fields.  */
+/* Read from stream STREAM, printing to standard output any selected fields,
+   using the line delimiter as the field delimiter.  */
 
 static void
-cut_fields (FILE *stream)
+cut_fields_line_delim (FILE *stream)
 {
   int c;       /* Each character from the file.  */
   uintmax_t field_idx = 1;
@@ -1329,8 +1345,8 @@ main (int argc, char **argv)
 
     case CUT_MODE_FIELDS:
       cut_stream = whitespace_delimited ? cut_fields_ws
-                   : single_byte_field_delim_ok () ? cut_fields
-                   : utf8_field_delim_ok () ? cut_fields_mb_utf8
+                   : field_delim_is_line_delim () ? cut_fields_line_delim
+                   : bytesearch_field_delim_ok () ? cut_fields_bytesearch
                    : cut_fields_mb;
       break;
     }