encoding.utf8

def utf8_encode(text: str) -> List[int]

from fune.encoding.utf8 import utf8_encode  # encoding.utf8@^1

from typing import List


def utf8_encode(text: str) -> List[int]:
    """Text as UTF-8 bytes.

    A Python str can hold a lone surrogate (json.loads("\\"\\\\ud800\\"") makes
    one), which str.encode refuses; it becomes U+FFFD here, as TextEncoder
    does in the browser, so the three languages agree.
    """
    if not isinstance(text, str):
        raise TypeError("text must be a string")
    out = []
    for ch in text:
        cp = ord(ch)
        if 0xD800 <= cp <= 0xDFFF:
            cp = 0xFFFD
        if cp < 0x80:
            out.append(cp)
        elif cp < 0x800:
            out.extend((0xC0 | (cp >> 6), 0x80 | (cp & 63)))
        elif cp < 0x10000:
            out.extend((0xE0 | (cp >> 12), 0x80 | ((cp >> 6) & 63), 0x80 | (cp & 63)))
        else:
            out.extend((0xF0 | (cp >> 18), 0x80 | ((cp >> 12) & 63), 0x80 | ((cp >> 6) & 63), 0x80 | (cp & 63)))
    return out

def utf8_decode(bytes: Sequence[int]) -> str

from fune.encoding.utf8 import utf8_decode  # encoding.utf8@^1

import builtins
from typing import Sequence


def utf8_decode(bytes: Sequence[int]) -> str:
    """UTF-8 bytes back to text, strictly (RFC 3629).

    Python's own decoder is already strict about overlong forms, surrogates
    and code points past U+10FFFF, and the plain "utf-8" codec keeps a leading
    byte order mark, so once the byte values are checked it does the work.
    (The parameter is named by the manifest, hence builtins.bytes.)
    """
    if isinstance(bytes, (str, dict)) or not hasattr(bytes, "__len__"):
        raise TypeError("bytes must be a list of integers from 0 to 255")
    for b in bytes:
        if type(b) is not int or b < 0 or b > 255:
            raise ValueError("bytes must be a list of integers from 0 to 255")
    try:
        return builtins.bytes(bytes).decode("utf-8")
    except UnicodeDecodeError:
        raise ValueError("bytes are not valid UTF-8") from None

Install

fune build

With that line in your source, in a Python project (language python in fune.project), fune build resolves it and nothing else, pins them in fune.lock, downloads only the Python package of each, and builds the code above into your project’s .fune/build, one readable file per capability with a header linking back here. Or pin a range in fune.project and build in one step:

fune add encoding.utf8

That builds the whole group. To build only what you call, and whatever it uses inside the group:

fune add encoding.utf8 --only utf8Encode

The manifest, vectors and README with only the Python implementation. Install it without the registry with fune add ./encoding.utf8-1.0.0-python.fune, or fetch it from a terminal with fune pull encoding.utf8@1.0.0:python.

The whole function, every language, is one file too: encoding.utf8-1.0.0.fune, 14,837 bytes, sha256 a476924f3a2599c51305953b1cb504a9ed4b90e0a92bd71556a9bb45bc0c9f86. It installs into a project of any language.

# fune: before encoding.utf8.utf8Encode
# fune: before encoding.utf8.utf8Decode

# fune: after encoding.utf8.utf8Encode
# fune: after encoding.utf8.utf8Decode

# fune: step encoding.utf8.<fn> after <n|label>

More from the author

**Encoding never fails.** A JavaScript or Python string can hold a lone UTF-16 surrogate (half of an emoji, usually from cutting a string in the wrong place), which has no UTF-8 form. It is encoded as U+FFFD, the replacement character, which is what `TextEncoder` does in every browser. A Rust `&str` cannot hold one at all.

**Decoding is strict**, per RFC 3629 section 3 and the table in section 4: an overlong form (`C0 80` for NUL), an encoded surrogate (`ED A0 80`), a code point above U+10FFFF, a truncated sequence or a stray continuation byte is an error, never replaced with U+FFFD. Bytes that are not text should be carried as bytes (or `encoding.base64`), not decoded and hoped for.

A byte order mark (`EF BB BF`) at the start is decoded as U+FEFF and kept: this is a codec, not a file reader, and stripping it would make `utf8Encode(utf8Decode(b))` differ from `b`. No Unicode normalisation is applied in either direction (`"é"` precomposed and `"é"` as `e` plus a combining accent are different bytes).

Source: RFC 3629, UTF-8, a transformation format of ISO 10646 (https://www.rfc-editor.org/rfc/rfc3629); the encoded examples in section 7 are among the vectors.

Files