From 1076663c8ed00a7e4288ae6fd16aadf4a5b2a618 Mon Sep 17 00:00:00 2001 From: Paul Brinkmeier Date: Wed, 5 Aug 2026 02:37:24 +0200 Subject: [PATCH] UTF-63: First draft --- blog/utf63/index.html | 74 ++++++++++++++++++++++++++++++++++++++ blog/utf63/utf63.css | 83 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 blog/utf63/index.html create mode 100644 blog/utf63/utf63.css diff --git a/blog/utf63/index.html b/blog/utf63/index.html new file mode 100644 index 0000000..2a2effb --- /dev/null +++ b/blog/utf63/index.html @@ -0,0 +1,74 @@ + + + + UTF-63 + + + + + +
+

UTF-63

+
+ + · + By +
+

+ UTF-8 is almost everywhere these days. + The web runs on it, most common file formats use it, macOS, Linux and increasingly even Windows systems use it as the default option. + It makes sense: It's quite compact for mostly-ASCII text and therefore most keywords and symbols of text-based file formats and protocols. +

+ +

+ What if I told you that in this age of memory bandwidth-constrained workloads you are using a wasteful encoding for text storage? + Most text these days is 99% ASCII with a few emoji sprinkled in (source: Made it up). + + Even if we ignore the costly multibyte emoji encodings, using UTF-8, 1 out of every 8 bits of memory is wasted on the encoding, because 7-bit ASCII characters are encoded using a whole byte, i.e. we have an encoding overhead of roughly 15%. +

+

+ For example, consider the UTF-8 encoding of the string “🙈🙉🙊 encoding” (see here for codepoints): +

+ 
11110000100111111001100110001000  🙈
+11110000100111111001100110001001  🙉
+11110000100111111001100110001010  🙊
+00100000011001010110111001100011   enc
+01101111011001000110100101101110  odin
+01100111                          g
+

+ Colored spans show which code points are encoded by which bits. + Neutral (white on black) bits are encoding overhead required by UTF-8. + For emoji, the most important medium of modern communication, this encoding is obviously horrible. + Every monkey requires a full 32 bits to encode its 21 bits. + And even for good old reliable ASCII, useless bits are sprinkled here and there. + There's gotta be a better way! +

+

UTF-63

+

+ Enter: An entirely new way to encode Unicode! +

+ 
10000111110110010010000000111110  🙈🙉
+11001001001000011111011001001010  🙊
+00100000110010111011101100011110   enco
+11111100100110100111011101100111  ding
+

+ For our entirely random example, this UTF-63 encoding uses only 16 bytes, a clear win over the whopping 21 bytes required by UTF-8 (that's 30% more!). + So how does it work? +

+

+ Decoding is dead simple: + We consider each set of 8 bytes a Big Endian 64-bit code unit. + If the most significant bit is 0, the other 63 bits contain nine Basic Latin characters (i.e. ASCII characters) at 7 bits each. + If it's 1, they contain any three Unicode code points at 21 bits each (the maximum codepoint is U+10FFFF). + When any of the encoded code points is U+0000 NULL, the string ends there (yes, we're bringing null-terminated strings back. So. many. applications. don't allow NULL anyways). +

+

Usage

+

+ Admittedly, this page actually still uses UTF-8, simply because most modern browsers don't support UTF-63 yet. + To my knowledge, most OSes, desktop applications and programming languages don't either. + It wouldn't surprise me if VLC supported UTF-63 subtitles or something though. + You can do your part by spreading the word among friends, family, flatmates, coworkers and anybody in between. +

+
+ + diff --git a/blog/utf63/utf63.css b/blog/utf63/utf63.css new file mode 100644 index 0000000..3b46e68 --- /dev/null +++ b/blog/utf63/utf63.css @@ -0,0 +1,83 @@ +:root { + /* https://github.com/morhetz/gruvbox */ + --red: #cc241d; + --red2: #fb4934; + --orange: #d65d0e; + --orange2: #fe8019; + --yellow: #d79921; + --yellow2: #fabd2f; + --aqua: #689d6a; + --aqua2: #8ec07c; + --blue: #458588; + --blue2: #83a598; + --purple: #b16286; + --purple2: #d3869b; + --bg: #928374; + --fg0_h: #fbf1c7; + --bg0_h: #1d2021; +} + +html { + font-size: 20px; +} + +body { + color: var(--bg0_h); + background-color: var(--fg0_h); + font-family: 'Source Sans Pro'; +} + +address { + font-size: .8em; + font-style: normal; +} + +h1, h2, h3, h4, h5, h6 { + font-weight: normal; +} + +p { + line-height: 1.5; +} + +article { + margin: 0 auto; + width: 80ch; +} + +pre { + padding: 1em; + background-color: var(--bg0_h); + color: var(--fg0_h); + line-height: 1.3; +} + +.red2 { + color: var(--bg0_h); + background-color: var(--red2); +} + +.orange2 { + color: var(--bg0_h); + background-color: var(--orange2); +} + +.yellow2 { + color: var(--bg0_h); + background-color: var(--yellow2); +} + +.aqua2 { + color: var(--bg0_h); + background-color: var(--aqua2); +} + +.blue2 { + color: var(--bg0_h); + background-color: var(--blue2); +} + +.purple2 { + color: var(--bg0_h); + background-color: var(--purple2); +}