Extend utf-8 support to 4 byte characters

[linenoise.git] / utf8.c

diff --git a/utf8.c b/utf8.c
index 26924b46c19154755c904038c949a98b033e3c50..db8f7c11f6dfafdbf0dccb72583606fbc9ca96b6 100644 (file)
--- a/utf8.c
+++ b/utf8.c
@@ -1,7 +1,7 @@
 /**
  * UTF-8 utility functions
  *
- * (c) 2010 Steve Bennett <steveb@workware.net.au>
+ * (c) 2010-2016 Steve Bennett <steveb@workware.net.au>
  *
  * See LICENCE for licence details.
  */
@@ -13,7 +13,7 @@
 #include "utf8.h"
 
 #ifdef USE_UTF8
-int utf8_fromunicode(char *p, unsigned short uc)
+int utf8_fromunicode(char *p, unsigned uc)
 {
     if (uc <= 0x7f) {
         *p = uc;
@@ -24,12 +24,20 @@ int utf8_fromunicode(char *p, unsigned short uc)
         *p = 0x80 | (uc & 0x3f);
         return 2;
     }
-    else {
+    else if (uc <= 0xffff) {
         *p++ = 0xe0 | ((uc & 0xf000) >> 12);
         *p++ = 0x80 | ((uc & 0xfc0) >> 6);
         *p = 0x80 | (uc & 0x3f);
         return 3;
     }
+    /* Note: We silently truncate to 21 bits here: 0x1fffff */
+    else {
+        *p++ = 0xf0 | ((uc & 0x1c0000) >> 18);
+        *p++ = 0x80 | ((uc & 0x3f000) >> 12);
+        *p++ = 0x80 | ((uc & 0xfc0) >> 6);
+        *p = 0x80 | (uc & 0x3f);
+        return 4;
+    }
 }
 
 int utf8_charlen(int c)
@@ -76,16 +84,6 @@ int utf8_index(const char *str, int index)
     return s - str;
 }
 
-int utf8_charequal(const char *s1, const char *s2)
-{
-    int c1, c2;
-
-    utf8_tounicode(s1, &c1);
-    utf8_tounicode(s2, &c2);
-
-    return c1 == c2;
-}
-
 int utf8_tounicode(const char *str, int *uc)
 {
     unsigned const char *s = (unsigned const char *)str;
@@ -106,6 +104,12 @@ int utf8_tounicode(const char *str, int *uc)
             return 3;
         }
     }
+    else if (s[0] < 0xf8) {
+        if (((str[1] & 0xc0) == 0x80) && ((str[2] & 0xc0) == 0x80) && ((str[3] & 0xc0) == 0x80)) {
+            *uc = ((s[0] & ~0xf0) << 18) | ((s[1] & ~0x80) << 12) | ((s[2] & ~0x80) << 6) | (s[3] & ~0x80);
+            return 4;
+        }
+    }
 
     /* Invalid sequence, so just return the byte */
     *uc = *s;