feat(unicode): accept unicode characters through the app's input handler to be handled correctly; key introduce isUnicode method

With the `isUnicode` method the read unicode characters from the user
can be checked against displayable text and rendered on the screen accordingly.

examples/continuous.zig

@@ -87,7 +87,7 @@ const InputField = struct {
         const this: *@This() = @ptrCast(@alignCast(ctx));
         switch (event) {
             .key => |key| {
-                if (key.isAscii()) try this.input.append(this.allocator, key.cp);
+                if (key.isUnicode()) try this.input.append(this.allocator, key.cp);
 
                 if (key.eql(.{ .cp = zterm.input.Enter }) or key.eql(.{ .cp = zterm.input.KpEnter }))
                     this.queue.push(.{ .accept = try this.input.toOwnedSlice(this.allocator) });

src/app.zig

@@ -123,7 +123,7 @@ pub fn App(comptime M: type, comptime E: type) type {
 
         fn run(this: *@This()) !void {
             // thread to read user inputs
-            var buf: [256]u8 = undefined;
+            var buf: [512]u8 = undefined;
             // NOTE set the `NONBLOCK` option for the stdin file, such that reading is not blocking!
             {
                 // TODO is there a better way to do this through the `std.Io` interface?
@@ -147,9 +147,10 @@ pub fn App(comptime M: type, comptime E: type) type {
             }
             while (true) {
                 this.io.checkCancel() catch break;
+                var remaining_bytes: usize = 0;
 
                 // non-blocking read
-                const read_bytes = terminal.read(&buf) catch |err| switch (err) {
+                const read_bytes = terminal.read(buf[remaining_bytes..]) catch |err| switch (err) {
                     error.WouldBlock => {
                         // wait a bit
                         this.io.sleep(.fromMilliseconds(20), .awake) catch |e| switch (e) {
@@ -159,7 +160,7 @@ pub fn App(comptime M: type, comptime E: type) type {
                         continue;
                     },
                     else => return err,
-                };
+                } + remaining_bytes;
 
                 // escape key presses
                 if (buf[0] == 0x1b and read_bytes > 1) {
@@ -414,7 +415,10 @@ pub fn App(comptime M: type, comptime E: type) type {
                         },
                         0x7f => .{ .cp = input.Backspace },
                         else => {
-                            var iter: std.unicode.Utf8Iterator = .{ .bytes = buf[0..read_bytes], .i = 0 };
+                            var len = read_bytes;
+                            while (!std.unicode.utf8ValidateSlice(buf[0..len])) len -= 1;
+                            remaining_bytes = read_bytes - len;
+                            var iter: std.unicode.Utf8Iterator = .{ .bytes = buf[0..len], .i = 0 };
                             while (iter.nextCodepoint()) |cp| this.postEvent(.{ .key = .{ .cp = cp } });
                             continue;
                         },
@@ -456,7 +460,7 @@ pub fn App(comptime M: type, comptime E: type) type {
         pub const RadioButton = element.RadioButton(Model, Event);
         pub const Scrollable = element.Scrollable(Model, Event);
         pub const Selection = element.Selection(Model, Event);
-        pub const Queue = queue.Queue(Event, 256);
+        pub const Queue = queue.Queue(Event, 512);
     };
 }
 

src/element.zig

@@ -631,7 +631,7 @@ pub fn TextField(Model: type, Event: type) type {
 
                     // TODO should this also accept `input.Enter` and insert `\n` into the value of the field?
                     // usual input keys
-                    if (key.isAscii()) try this.input.insert(this.allocator, this.input.items.len - this.cursor_offset, key.cp);
+                    if (key.isUnicode()) try this.input.insert(this.allocator, this.input.items.len - this.cursor_offset, key.cp);
 
                     if (key.eql(.{ .cp = input.Backspace })) {
                         if (this.cursor_offset < this.input.items.len) {
@@ -666,8 +666,20 @@ pub fn TextField(Model: type, Event: type) type {
             const value = switch (t) {
                 .bytes => blk: {
                     // NOTE convert unicode characters to ascii characters; if non ascii characters are found this is will fail!
-                    var slice = try this.allocator.alloc(u8, this.input.items.len);
-                    for (0.., this.input.items) |i, c| slice[i] = @intCast(c);
+                    const len: usize = len: {
+                        var res: usize = 0;
+                        for (this.input.items) |cp| res += try std.unicode.utf8CodepointSequenceLength(cp);
+                        break :len res;
+                    };
+                    var slice = try this.allocator.alloc(u8, len);
+                    errdefer this.allocator.free(slice);
+
+                    var i: usize = 0;
+                    for (this.input.items) |cp| {
+                        const size = try std.unicode.utf8CodepointSequenceLength(cp);
+                        _ = try std.unicode.utf8Encode(cp, slice[i .. i + size]);
+                        i += size;
+                    }
                     this.input.clearAndFree(this.allocator);
                     break :blk slice;
                 },

src/input.zig

@@ -65,6 +65,31 @@ pub const Key = packed struct {
         return meta.eql(this, other);
     }
 
+    /// Determine if the `Key` is an unicode character that can be printed to
+    /// the screen. This means that the code point of the `Key` is an ascii
+    /// character between 32 - 255 (with the exception of 127 = Delete) and no
+    /// modifiers (alt and/or ctrl) are used.
+    ///
+    /// # Example
+    ///
+    /// Get user input's from the .key event from the application event loop:
+    ///
+    /// ```zig
+    /// switch (event) {
+    ///   .key => |key| if (key.isUnicode()) try this.input.append(key.cp),
+    ///   else => {},
+    /// }
+    /// ```
+    pub fn isUnicode(this: @This()) bool {
+        return this.mod.alt == false and this.mod.ctrl == false and // no modifier keys
+            (this.cp >= 32 and this.cp <= 126 or // ascii printable characters (except for input.Delete)
+                this.cp >= 128 and this.cp <= 255) or // extended ascii codes
+            ((this.cp >= 0x0080 and this.cp <= 0x07FF) or
+                (this.cp >= 0x0800 and this.cp <= 0xFFFF) or
+                (this.cp >= 0x100000 and this.cp <= 0x10FFFF)) and // allowed unicode character ranges (2 - 4 byte characters)
+                (this.cp < 57348 or this.cp > 57454); // no other predifined meanings (i.e. arrow keys, etc.)
+    }
+
     /// Determine if the `Key` is an ascii character that can be printed to
     /// the screen. This means that the code point of the `Key` is an ascii
     /// character between 32 - 255 (with the exception of 127 = Delete) and no