all: use more consistent blank character determination

* src/system.h (c32issep): A new function that is essentially
iswblank() on GLIBC platforms, and iswspace() with exceptions elsewhere.
* src/expand.c: Use it instead of c32isblank().
* src/fold.c: Likewise.
* src/join.c: Likewise.
* src/numfmt.c: Likewise.
* src/unexpand.c: Likewise.
* src/uniq.c: Likewise.
* NEWS: Mention the improvement.

diff --git a/NEWS b/NEWS
index cec03a581b8635dbc2faab10b724b0c8657726d7..666968c86e067cf972f8e57c62914bfb604ef5a5 100644 (file)
--- a/NEWS
+++ b/NEWS
@@ -42,6 +42,10 @@ GNU coreutils NEWS                                    -*- outline -*-
   'install' now allows the combination of the --compare and
   --preserve-timestamps options.
 
+  'fold', 'join', 'numfmt', 'uniq' now use more consistent blank character
+  determination on non GLIBC platforms.  For example \u3000 (ideographic space)
+  will be considered a blank character on all platforms.
+
   'nl' now supports multi-byte --section-delimiter characters.
 
   'shuf -i' now operates up to two times faster on systems with unlocked stdio

diff --git a/src/expand.c b/src/expand.c
index 6d4223c9b2c9818e2b319816d1d9a36c491a7ba8..1d07590790a02d197b31f0da441f44ceaa12a5ce 100644 (file)
--- a/src/expand.c
+++ b/src/expand.c
@@ -140,8 +140,7 @@ expand (void)
 
           if (convert)
             {
-              convert &= convert_entire_line
-                         || !! (c32isblank (g.ch) && ! c32isnbspace (g.ch));
+              convert &= convert_entire_line || c32issep (g.ch);
 
               if (g.ch == '\t')
                 {

diff --git a/src/fold.c b/src/fold.c
index 666490f95d83ee8a399b1ee08fe89a545c31a37b..f49078f01a79e25a5124b5ac44e87257fb821f03 100644 (file)
--- a/src/fold.c
+++ b/src/fold.c
@@ -210,7 +210,7 @@ fold_file (char const *filename, size_t width)
               for (mcel_t g2; logical_p < logical_lim; logical_p += g2.len)
                 {
                   g2 = mcel_scan (logical_p, logical_lim);
-                  if (c32isblank (g2.ch) && ! c32isnbspace (g2.ch))
+                  if (c32issep (g2.ch))
                     {
                       space_length = g2.len;
                       logical_end = logical_p - line_out;

diff --git a/src/join.c b/src/join.c
index 883a4200576f7c47c1b38cd4e02d35324c15ce3e..4346758a6011a6a6617398dd82c868b0d357ba72 100644 (file)
--- a/src/join.c
+++ b/src/join.c
@@ -308,7 +308,7 @@ eq_tab (mcel_t g)
 static bool
 newline_or_blank (mcel_t g)
 {
-  return g.ch == '\n' || c32isblank (g.ch);
+  return g.ch == '\n' || c32issep (g.ch);
 }
 
 /* Fill in the 'fields' structure in LINE.  */
@@ -918,7 +918,7 @@ decode_field_spec (char const *s, int *file_index, idx_t *field_index)
 static bool
 comma_or_blank (mcel_t g)
 {
-  return g.ch == ',' || c32isblank (g.ch);
+  return g.ch == ',' || c32issep (g.ch);
 }
 
 /* Add the comma or blank separated field spec(s) in STR to 'outlist'.  */

diff --git a/src/numfmt.c b/src/numfmt.c
index fb6cb33969ce95f03c68d8136a5d9e5d0af15897..2436c54870c2ecf909e728f0b5c9259edbeb3459 100644 (file)
--- a/src/numfmt.c
+++ b/src/numfmt.c
@@ -215,8 +215,7 @@ static bool dev_debug = false;
 static bool
 newline_or_blank (mcel_t g)
 {
-  return g.ch == '\n'
-         || (c32isblank (g.ch) && ! c32isnbspace (g.ch));
+  return g.ch == '\n' || c32issep (g.ch);
 }
 
 static inline int
@@ -673,7 +672,7 @@ simple_strtod_human (char const *input_str,
       if (!matched_unit_sep)
         {
           mcel_t g = mcel_scanz (*endptr);
-          if (c32isblank (g.ch) || c32isnbspace (g.ch))
+          if (c32issep (g.ch) || c32isnbspace (g.ch))
             (*endptr) += g.len;
         }
 

diff --git a/src/system.h b/src/system.h
index 988c7cd9ec80cb93ef253c756a4f7f9d2ca3783d..79b3e606957518fe4ca5771e9d7720d9902deb39 100644 (file)
--- a/src/system.h
+++ b/src/system.h
@@ -160,6 +160,29 @@ c32isnbspace (char32_t wc)
   return wc == 0x00A0 || wc == 0x2007 || wc == 0x202F || wc == 0x2060;
 }
 
+ATTRIBUTE_PURE
+static inline int
+c32isvertspace (char32_t wc)
+{
+  return    wc == 0x000A || wc == 0x000B || wc == 0x000C || wc == 0x000D
+         || wc == 0x2028 || wc == 0x2029;
+}
+
+
+/* c32isblank() is too variable on non GLIBC platforms.
+   E.g., does not include \u3000 ideographic space on musl.
+   E.g., does include non-breaking space on Solaris and NetBSD.
+   This equivalent is more consistent across systems.  */
+ATTRIBUTE_PURE
+static inline bool
+c32issep (char32_t wc)
+{
+#if defined __GLIBC__
+  return !! c32isblank (wc);
+#endif
+  return !! (c32isspace (wc) && ! c32isvertspace (wc) && ! c32isnbspace (wc));
+}
+
 #include <locale.h>
 
 /* Take care of NLS matters.  */

diff --git a/src/unexpand.c b/src/unexpand.c
index 16d0f00315dcab7fb576d4920b35f581bebcc91d..4fbf9d3f8fd3ad077ca976836daa8b9c57c2cf1a 100644 (file)
--- a/src/unexpand.c
+++ b/src/unexpand.c
@@ -176,7 +176,7 @@ unexpand (void)
 
           if (convert)
             {
-              bool blank = !! (c32isblank (g.ch) && ! c32isnbspace (g.ch));
+              bool blank = c32issep (g.ch);
 
               if (blank)
                 {

diff --git a/src/uniq.c b/src/uniq.c
index eebff4b7b2bab594c2f462b21bc66356e91e86a0..30463598a58e3005e4a3b798d4c66371d2eab404 100644 (file)
--- a/src/uniq.c
+++ b/src/uniq.c
@@ -254,7 +254,7 @@ size_opt (char const *opt, char const *msgid)
 static bool
 newline_or_blank (mcel_t g)
 {
-  return g.ch == '\n' || c32isblank (g.ch);
+  return g.ch == '\n' || c32issep (g.ch);
 }
 
 /* Given a linebuffer LINE,