basic/utf8: add function to convert to ASCII

The conversion must be lossy because ASCII doesn't have enough chars.

diff --git a/src/basic/utf8.c b/src/basic/utf8.c
index 2ad215181667da19ae07c20895a603ee12387465..2532fcf81a7b8f842e9d8a02e8ed0c2389bf256b 100644 (file)
--- a/src/basic/utf8.c
+++ b/src/basic/utf8.c
@@ -312,6 +312,37 @@ char *ascii_is_valid_n(const char *str, size_t len) {
         return (char*) str;
 }
 
+int utf8_to_ascii(const char *str, char replacement_char, char **ret) {
+        /* Convert to a string that has only ASCII chars, replacing anything that is not ASCII
+         * by replacement_char. */
+
+        _cleanup_free_ char *ans = new(char, strlen(str) + 1);
+        if (!ans)
+                return -ENOMEM;
+
+        char *q = ans;
+
+        for (const char *p = str; *p; q++) {
+                int l;
+
+                l = utf8_encoded_valid_unichar(p, SIZE_MAX);
+                if (l < 0)  /* Non-UTF-8, let's not even try to propagate the garbage */
+                        return l;
+
+                if (l == 1)
+                        *q = *p;
+                else
+                        /* non-ASCII, we need to replace it */
+                        *q = replacement_char;
+
+                p += l;
+        }
+        *q = '\0';
+
+        *ret = TAKE_PTR(ans);
+        return 0;
+}
+
 /**
  * utf8_encode_unichar() - Encode single UCS-4 character as UTF-8
  * @out_utf8: output buffer of at least 4 bytes or NULL

diff --git a/src/basic/utf8.h b/src/basic/utf8.h
index b0e969f655dbfec865e007c81fa231a0e45bb0c2..4a06dd62c5be1b4113250bac52f2d99246711bb2 100644 (file)
--- a/src/basic/utf8.h
+++ b/src/basic/utf8.h
@@ -21,6 +21,8 @@ static inline char *utf8_is_valid(const char *s) {
 char *ascii_is_valid(const char *s) _pure_;
 char *ascii_is_valid_n(const char *str, size_t len);
 
+int utf8_to_ascii(const char *str, char replacement_char, char **ret);
+
 bool utf8_is_printable_newline(const char* str, size_t length, bool allow_newline) _pure_;
 #define utf8_is_printable(str, length) utf8_is_printable_newline(str, length, true)
 

diff --git a/src/test/test-utf8.c b/src/test/test-utf8.c
index 4ba9ca843943e794d902552f3e40cb5b4d651a13..763d17ce9ee53df830f4da5287feb8f98c497647 100644 (file)
--- a/src/test/test-utf8.c
+++ b/src/test/test-utf8.c
@@ -66,6 +66,33 @@ static void test_ascii_is_valid_n(void) {
         assert_se( ascii_is_valid_n("\342\204\242", 0));
 }
 
+static void test_utf8_to_ascii_one(const char *s, int r_expected, const char *expected) {
+        _cleanup_free_ char *ans = NULL;
+        int r;
+
+        r = utf8_to_ascii(s, '*', &ans);
+        log_debug("\"%s\" → %d/\"%s\" (expected %d/\"%s\")", s, r, strnull(ans), r_expected, strnull(expected));
+        assert_se(r == r_expected);
+        assert_se(streq_ptr(ans, expected));
+}
+
+static void test_utf8_to_ascii(void) {
+        log_info("/* %s */", __func__);
+
+        test_utf8_to_ascii_one("asdf", 0, "asdf");
+        test_utf8_to_ascii_one("dąb", 0, "d*b");
+        test_utf8_to_ascii_one("żęśłą óźń", 0, "***** ***");
+        test_utf8_to_ascii_one("\342\204\242", 0, "*");
+        test_utf8_to_ascii_one("\342\204", -EINVAL, NULL); /* truncated */
+        test_utf8_to_ascii_one("\342", -EINVAL, NULL); /* truncated */
+        test_utf8_to_ascii_one("\302\256", 0, "*");
+        test_utf8_to_ascii_one("", 0, "");
+        test_utf8_to_ascii_one(" ", 0, " ");
+        test_utf8_to_ascii_one("\t", 0, "\t");
+        test_utf8_to_ascii_one("串", 0, "*");
+        test_utf8_to_ascii_one("…👊🔪💐…", 0, "*****");
+}
+
 static void test_utf8_encoded_valid_unichar(void) {
         log_info("/* %s */", __func__);
 
@@ -241,6 +268,7 @@ int main(int argc, char *argv[]) {
         test_utf8_is_printable();
         test_ascii_is_valid();
         test_ascii_is_valid_n();
+        test_utf8_to_ascii();
         test_utf8_encoded_valid_unichar();
         test_utf8_escape_invalid();
         test_utf8_escape_non_printable();