diff --git a/README.md b/README.md index 938d072..d53017b 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,8 @@ Pick the guide that fits you: | Arrays | `nums := [1, 2, 3]`, `nums.push(4)`, `nums[0]`, `nums.len`, `for i, n in nums { }` | | Maps | `ages := ["ann": 31]`, `ages["bob"] = 25`, `ages.has("cy")`, `counts[w] += 1`, `for k, v in ages { }` | | Errors | `fn parse(str s) -> int! { … return error("bad") }` → `parse(s) or 0`, `parse(s) or { … }`, `try parse(s)` | -| Vectors (SIMD) | `v := f32x8(list, i)`, `v * 2.0 + w`, `v.sum()`, `v.dot(w)`, `min(v, 1.0)`, `v.store(list, i)`, `v[3]` | +| Vectors (SIMD) | `v := f32x8(list, i)`, `v * 2.0 + w`, `v.sum()`, `select(v > 0.0, v, 0.0)`, `t.lookup(idx)`, `acc.dot_add(a, b)`, `v.store(list, i)` | +| Processors | `--cpu native`; `@cpu("avx2") fn …` with `if cpu_has("avx2")`; floats computed as written (`fma`/`muladd` to fuse) | | Text | `"hp \(hp)"`, `"a" + "b"`, `str(42)`, `int("42")`, `s.split(",")`, `s.upper()`, `s[0]` | | Control | `if` / `else if` / `else` (also as a value), `while`, `for i in 0..10`, `break`, `continue`, `defer` | | Memory | automatic for arrays, text and structs; `new(...)` / `free(...)` for your own heap structures | @@ -161,6 +162,7 @@ Pick the guide that fits you: - [x] Threads (`import thread`) and atomics; C++ files; `--target android` - [x] The compiler on every system it builds for: Linux, macOS (Apple Silicon and Intel), Windows; releases made by CI - [x] Vector types (`f32x8`, `i8x32`, …) for SIMD, and `--cpu native` +- [x] Masks, shuffles, byte lookup and `dot_add` for kernels; `@cpu` and `cpu_has`; `fma`/`muladd`, float bits, f16 and bf16 ## Layout diff --git a/README.pl.md b/README.pl.md index 098f317..68dcc85 100644 --- a/README.pl.md +++ b/README.pl.md @@ -105,7 +105,8 @@ Wybierz przewodnik dla siebie: | Tablice | `liczby := [1, 2, 3]`, `liczby.push(4)`, `liczby[0]`, `liczby.len`, `for i, n in liczby { }` | | Mapy | `wiek := ["ania": 31]`, `wiek["bob"] = 25`, `wiek.has("cyryl")`, `ile[slowo] += 1`, `for k, v in wiek { }` | | Błędy | `fn parsuj(str s) -> int! { … return error("zła liczba") }` → `parsuj(s) or 0`, `parsuj(s) or { … }`, `try parsuj(s)` | -| Wektory (SIMD) | `v := f32x8(lista, i)`, `v * 2.0 + w`, `v.sum()`, `v.dot(w)`, `min(v, 1.0)`, `v.store(lista, i)`, `v[3]` | +| Wektory (SIMD) | `v := f32x8(lista, i)`, `v * 2.0 + w`, `v.sum()`, `select(v > 0.0, v, 0.0)`, `t.lookup(idx)`, `acc.dot_add(a, b)`, `v.store(lista, i)` | +| Procesory | `--cpu native`; `@cpu("avx2") fn …` razem z `if cpu_has("avx2")`; liczby zmiennoprzecinkowe liczone tak, jak zapisane (`fma`/`muladd` do łączenia) | | Tekst | `"hp: \(hp)"`, `"a" + "b"`, `str(42)`, `int("42")`, `s.split(",")`, `s.upper()`, `s[0]` | | Sterowanie | `if` / `else if` / `else` (również jako wartość), `while`, `for i in 0..10`, `break`, `continue`, `defer` | | Pamięć | tablice, teksty i struktury zwalniają się same; `new(...)` / `free(...)` dla własnych struktur na stercie | @@ -163,6 +164,7 @@ Wybierz przewodnik dla siebie: - [x] Wątki (`import thread`) i operacje atomowe; pliki C++; `--target android` - [x] Kompilator na każdym systemie, na który buduje: Linux, macOS (Apple Silicon i Intel), Windows; wydania robi CI - [x] Typy wektorowe (`f32x8`, `i8x32`, …) dla SIMD i `--cpu native` +- [x] Maski, przestawianie, tabele bajtów i `dot_add` dla jąder obliczeniowych; `@cpu` i `cpu_has`; `fma`/`muladd`, bity liczb, f16 i bf16 ## Co gdzie leży diff --git a/docs/en/cheatsheet.md b/docs/en/cheatsheet.md index 615856d..6e21422 100644 --- a/docs/en/cheatsheet.md +++ b/docs/en/cheatsheet.md @@ -255,12 +255,19 @@ Types: `f32x4/8/16`, `f64x2/4/8`, `i8x16/32/64`, `i16x8/16/32`, `i32x4/8/16`, `i | `f32x8(v)` | an 8-lane vector converted lane by lane (`f32x8(ints)`, `i32x8(floats)`) | | `v + w`, `v * 2.0`, `-v` | lane by lane; a plain number counts for every lane (`/` only on floats) | | `& \| ^ << >> ~` | lane by lane, whole-number lanes | -| `v == w`, `v != w` | bool: every lane equal / not | +| `v < w`, `v == w`, … | a mask `boolx8`: `m.all()`, `m.any()`, `m.count()`, `m & n`, `!m`, `m[i]` | +| `select(m, a, b)` | lane by lane: `a` where `m` is true, else `b` | | `v[i]`, `v[i] = x`, `v.len` | a lane; the number of lanes | | `min(v, w)`, `max`, `abs(v)`, `clamp(v, lo, hi)` | lane by lane | | `v.sum()`, `v.min()`, `v.max()`, `v.dot(w)` | one number from all the lanes | -| `v.sqrt()`, `v.floor()`, `v.ceil()`, `v.round()`, `v.fma(b, c)` | every lane (float lanes; `fma` = `v * b + c`) | +| `v.sqrt()`, `v.floor()`, `v.ceil()`, `v.round()` | every lane (float lanes) | +| `v.fma(b, c)`, `v.muladd(b, c)` | `v * b + c`: one rounding always / the fastest way here | | `v.store(list, i)` | lanes into `list[i]` … | +| `f32x8(list, i, fill)`, `v.store_part(list, i)` | the end of a list: only the lanes inside it (the others are `fill`) | +| `v.shuffle(3, 2, 1, 0)`, `v.lo()`, `v.hi()`, `concat(a, b)` | lanes rearranged (shuffle takes numbers written in the code) | +| `table.lookup(idx)` | `table[idx % table.len]` per lane (`pshufb`, `tbl`) | +| `acc.dot_add(a, b)` | `i32x8` sums gain 4 byte products each (`u8x32` × `i8x32`): `vpdpbusd`, `usdot` where there | +| `@cpu("avx2,fma") fn …`, `cpu_has("avx2")` | a function that may use these features; check at run time before calling it | --- @@ -324,6 +331,9 @@ Automatic only when nothing is lost: smaller → bigger number, any int → floa | `shell(command)` | run a command, its exit code | | `exit(code)` | stop the program now (also ends an `or { }` block) | | `min(a, b, …)`, `max(a, b, …)`, `abs(x)`, `clamp(x, lo, hi)` | for any numbers (your own function of that name wins) | +| `fma(a, b, c)`, `muladd(a, b, c)` | `a * b + c` with one rounding / fused where the processor can (`a * b + c` written out is never fused) | +| `f32_bits(x)`, `f32_from_bits(u)`, `f64_bits`, `f64_from_bits` | a float's bits as `u32`/`u64` and back, unchanged | +| `f16_to_f32(h)`, `f32_to_f16(x)`, `bf16_to_f32(b)`, `f32_to_bf16(x)` | half and bfloat16 numbers (in a `u16`) ↔ `f32`; also on vectors | | `error(message)` | `return error("…")` in a `-> T!` function | | `addr(x)` | pointer to x; `addr(fn)` = C callback | | `assert(cond)`, `assert(cond, "why")` | stop with a runtime error when cond is false (in a `test` block: that test fails) | diff --git a/docs/en/for-engineers.md b/docs/en/for-engineers.md index 68bc9e8..7b055a2 100644 --- a/docs/en/for-engineers.md +++ b/docs/en/for-engineers.md @@ -304,11 +304,18 @@ type goes; a call `f32x8(…)` goes to `vecMake()`. | `f32x8(i32x8 v)` | `sitofp`, `fptosi`, `sext`, `trunc`… on the whole vector | | `v + w`, `v * 2.0` | `fadd`/`add`… on vectors; a scalar operand is coerced to the lane type, then splatted | | `v[i]` | a place: `getelementptr , ptr %v, i64 i` after a bounds check; or `extractelement` on a temporary | -| `v == w` | `fcmp oeq`/`icmp eq` → `` → `llvm.vector.reduce.and` | +| `v < w`, `v == w` … | `fcmp`/`icmp` → ``, a mask (`boolxN`: `vec` with `bool` lanes) | +| `select(m, a, b)` | `select ` | +| `m.any()`, `.all()`, `.count()` | `llvm.vector.reduce.or`/`.and`; `zext` + `llvm.vector.reduce.add` | +| `f32x8(list, i, fill)`, `v.store_part(list, i)` | `llvm.masked.load`/`llvm.masked.store` with the mask `<0, 1, …> < len - i` | +| `v.shuffle(…)`, `.lo()`, `.hi()`, `concat(a, b)` | `shufflevector` with a constant mask | +| `table.lookup(idx)` | `and` with `len - 1`, then `extractelement`/`insertelement` per lane: the pattern the x86 and ARM backends turn into `pshufb`/`tbl` | +| `acc.dot_add(a, b)` | `llvm.x86.avx512.vpdpbusd.{128,256,512}` or `llvm.aarch64.neon.{usdot,sdot,udot}` when the function's features have them; else `zext`/`sext`, `mul`, four strided `shufflevector`s and `add`s | | `min`, `max`, `abs` | `llvm.minnum`/`maxnum`, `llvm.smin`/`umin`…, `llvm.fabs`/`llvm.abs` | | `v.sum()`, `.min()`, `.max()` | `llvm.vector.reduce.fadd` (with `reassoc`), `.add`, `.fmin`, `.smin`… | | `v.sqrt()`, `.floor()`, `.ceil()`, `.round()` | the `llvm.*` intrinsic of that name | -| `v.fma(b, c)` | `llvm.fmuladd` (fused where the target has FMA; `llvm.fma` would be a libm call per lane without it) | +| `v.fma(b, c)`, `fma(a, b, c)` | `llvm.fma` (one rounding; a libm call per lane without FMA hardware) | +| `v.muladd(b, c)`, `muladd(a, b, c)` | `llvm.fmuladd` (fused where the target has FMA, `fmul` + `fadd` where not) | | `v.store(list, i)` | bounds check, `store <8 x float> …, align 4` | Integer `/` and `%` are refused: neither x86 nor ARM has a lane-wise integer divide, and LLVM would scalarise it. @@ -322,6 +329,20 @@ aligned address and crash. **Which instructions** is clang's choice: `--cpu ` adds `-march=` (x86-64) or `-mcpu=` (ARM64) to the IR's compile, so the same IR becomes SSE2, AVX2 + FMA, AVX-512 or NEON code. Lanes wider than the registers are split. +Some operations need to know the features at code generation time (`dot_add` picks `vpdpbusd` only where it exists). +`linker.cpuFeatures()` asks clang for its predefined macros with the same target and `--cpu` +(`clang -march=X -dM -E empty.c`: `__AVXVNNI__`, `__AVX512VNNI__`, `__ARM_FEATURE_MATMUL_INT8` …), maps them to Finch's +feature names and caches the answer per clang and flags; they reach `gen` as `Options.features`. `@cpu("…")` on a function +(`Node.cpu`) adds its own: `define … "target-features"="+avx2,+fma"` (with `+evex512` for any AVX-512 feature, which +LLVM 18+ needs for 512-bit registers), and `g.curFeatures` is the union while that function is generated. Features of the +other architecture are dropped (`cpuOf`). `checkCpuCall` refuses a vector wider than 128 bits as a parameter or result +between functions whose `@cpu` differ, because caller and callee would lower it to different registers. +`cpu_has(name)` calls `finch_cpu_has` in the runtime: CPUID leaves 1 and 7 with XGETBV for the AVX/AVX-512 state on x86, +`getauxval(AT_HWCAP/AT_HWCAP2)` on Linux ARM64, `sysctlbyname("hw.optional.arm.FEAT_…")` on macOS. + +Clang is always run as `NIX_ENFORCE_NO_NATIVE=0 clang …` on Unix: Nix's cc-wrapper otherwise drops `-march=native` +inside `nix-shell`. If `--cpu native` still yields no SSE4.2 macro on x86, the driver warns. + --- ## 8. Ownership and memory @@ -507,6 +528,19 @@ test. | Missing return | UB if used | compile error | | Order of evaluation | mostly unspecified | left to right; in an assignment the right side first | | Shift ≥ width, out-of-range float→int | UB | poison, as in LLVM (not checked) | +| `a * b + c` on floats | may be fused into an FMA (`-ffp-contract`) | never fused: two roundings | + +### Floating point + +Float operations are emitted as plain `fadd`/`fsub`/`fmul`/`fdiv`/`fneg` with **no fast-math flags**, so LLVM may not +reassociate, contract or assume no NaNs; `linker.fch` also compiles the IR with `-ffp-contract=on`, which fuses only the +`llvm.fmuladd` calls Finch asks for. The only flagged instruction is `v.sum()`'s `llvm.vector.reduce.fadd` (`reassoc`). +`fma` → `llvm.fma`, `muladd` → `llvm.fmuladd`, on scalars and vectors (`fmaCall`, `fusedOp`). + +`f32_bits`/`f32_from_bits`/`f64_…` are `bitcast` (also between vectors). `f16_to_f32` is `bitcast i16 → half` + `fpext`, +`f32_to_f16` is `fptrunc` + `bitcast`: one instruction with F16C or on ARM64, `__extendhfsf2`/`__truncsfhf2` from the C +compiler's runtime library elsewhere. `bf16_to_f32` is `zext` + `shl 16` + `bitcast`; `f32_to_bf16` rounds to nearest even +in integer ops (`u + 0x7fff + ((u >> 16) & 1)`, then `>> 16`) and keeps NaNs quiet (`| 0x40`), so it needs no library. | Dangling pointer from `addr()` / `new` + `free` | UB | still UB | ### The chain of calls on runtime errors @@ -558,5 +592,5 @@ evaluated twice and temporaries are never pointed to. - Generics are monomorphised (code size grows with instantiations); there are no trait bounds, a wrong use is an error inside the instantiated body. - A failed test's owned memory is not freed (the test runner `longjmp`s out of it). -- Vectors have no lane masks (`v < w`), shuffles or integer division. +- Vectors have no integer division, and `lookup` on baseline x86-64 (no SSSE3) is done lane by lane. diff --git a/docs/en/for-technicians.md b/docs/en/for-technicians.md index 34aa27c..c3ab1b7 100644 --- a/docs/en/for-technicians.md +++ b/docs/en/for-technicians.md @@ -345,6 +345,40 @@ str(p) // a C char pointer as text **Fixed-size numbers wrap around**, like real hardware: a `u8` holding 255 plus 1 becomes 0. Calculations happen in the type of the values involved, so `u8(250) + 10` is `4`, not `260`. +### Decimal numbers: what is guaranteed + +`f32` and `f64` are IEEE 754 numbers, and Finch computes them **exactly as written**, the same on every processor and +with every `--cpu`: + +- Each operation rounds once, in the order written. `a * b - c * d` is two multiplies and a subtract, never folded into a + fused multiply-add (C compilers often do that by default, which changes the last digits). +- Nothing is reordered or simplified: `(a + b) + c` is not turned into `a + (b + c)`, and `x * 0.0` is not assumed to be 0 + (it is NaN when `x` is). The one exception is `v.sum()` on a vector, which adds the lanes in the fastest order. + +When speed matters more than the last bit, ask for it by name: + +| | | +|---|---| +| `fma(a, b, c)` | `a * b + c` with a single rounding, always. One instruction where the processor has FMA (with `--cpu`, on ARM64); a slower library call elsewhere. | +| `muladd(a, b, c)` | `a * b + c` the fastest way on this processor: fused where it can be, a multiply and an add where not. | + +Both work on numbers and on vectors (`v.fma(b, c)`, `v.muladd(b, c)`). + +**Numbers as bits.** File formats and model weights store numbers as raw bits. These look at the bits without +converting the value (all of them also work lane by lane on vectors): + +| | | +|---|---| +| `f32_bits(x)` / `f32_from_bits(u)` | `f32` ↔ the `u32` with the same bits (`f32_bits(1.0)` is `0x3f800000`) | +| `f64_bits(x)` / `f64_from_bits(u)` | `f64` ↔ `u64` | +| `f16_to_f32(h)` / `f32_to_f16(x)` | half precision, kept in a `u16`, ↔ `f32` (rounded to the nearest) | +| `bf16_to_f32(b)` / `f32_to_bf16(x)` | bfloat16, kept in a `u16`, ↔ `f32` (rounded to the nearest, ties to even) | + +```c +[]u16 weights = read_weights() // f16 numbers from a file +v := f16_to_f32(u16x8(weights, 0)) // 8 of them as f32, at once +``` + --- ## 7. Operators @@ -697,6 +731,8 @@ Using `.value`, `.field` or `[i]` on a `null` pointer **stops the program** with | `exit(code)` | Ends the program right away. | | `addr(x)` | A pointer to `x`; `addr(function)` gives C a function pointer (callback). | | `new(value)`, `free(p)` | Manual heap memory (see [Memory](#13-memory-who-frees-what)). | +| `fma(a, b, c)`, `muladd(a, b, c)` | `a * b + c` with one rounding / the fastest way (see [Decimal numbers](#decimal-numbers-what-is-guaranteed)). | +| `f32_bits(x)`, `f32_from_bits(u)`, `f16_to_f32(h)`, … | A number's bits, and half-precision numbers (see [Decimal numbers](#decimal-numbers-what-is-guaranteed)). | | `int(x)`, `str(x)`, `u8(x)`, … | Conversions (see [Conversions](#conversions)). `str(x)` works on every type and gives the same text `print` shows. `int(text) or 0` handles text that isn't a number. | --- @@ -1364,17 +1400,15 @@ arrays, maps and function parameters, and `print` and `str` show it. | `+ - * /` | lane by lane (`/` on floats only: x86 and ARM can't divide whole numbers lane by lane) | | `& \| ^ << >> ~` | lane by lane, on whole-number lanes | | `-v` | every lane negated | -| `v == w`, `v != w` | `true` if every lane is equal / if any differs | | `v[i]`, `v[i] = x`, `v.len` | one lane; the number of lanes | | `min(v, w)`, `max(v, w)`, `abs(v)`, `clamp(v, lo, hi)` | lane by lane; `lo`, `hi` and `w` can be plain numbers | | `v.sum()`, `v.min()`, `v.max()` | all the lanes made into one number | | `v.dot(w)` | `(v * w).sum()` | | `v.sqrt()`, `v.floor()`, `v.ceil()`, `v.round()` | every lane (float lanes) | -| `v.fma(b, c)` | `v * b + c`, as one instruction where the processor has one | +| `v.fma(b, c)` | `v * b + c` with one rounding, always (the same on every processor) | +| `v.muladd(b, c)` | `v * b + c` the fastest way here: one instruction where the processor can (the last bit may differ between processors) | | `v.store(list, i)` | writes the lanes to `list[i]` … (or through a `ptr`) | -`<` and the other comparisons would give an answer per lane, so they aren't there; use `min`, `max` or `v[i] < w[i]`. - **A real loop.** The usual shape: as many whole vectors as fit, then the rest one by one. ```c @@ -1382,7 +1416,7 @@ fn dot([]f32 a, []f32 b) -> f32 { acc := f32x8(0.0) i := 0 while i + 8 <= a.len { - acc = f32x8(a, i).fma(f32x8(b, i), acc) + acc = f32x8(a, i).muladd(f32x8(b, i), acc) i += 8 } total := acc.sum() @@ -1396,10 +1430,125 @@ fn dot([]f32 a, []f32 b) -> f32 { `examples/simd.fch` times this against the plain loop. -**Which instructions.** By default a program runs on every processor of its system (on x86-64 that means SSE2, so an -`f32x8` is done as two halves). `--cpu native` (or `cpu = "native"` in `finch.toml`) lets LLVM use everything the processor -that builds it has: AVX2 and FMA, AVX-512, or NEON's newer parts on ARM. Such a program needs a processor like that one to -run. To pick a level instead: `--cpu x86-64-v3` (AVX2, most PCs since 2015), `--cpu x86-64-v4` (AVX-512), `--cpu apple-m1`. +### Masks: comparing, choosing, the end of a list + +Comparing vectors compares every lane, so the answer is a **mask**: one `true`/`false` a lane, of type `boolx8` (for 8 lanes). + +```c +v := f32x8(-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, -4.0, 5.0) +m := v > 0.0 // boolx8(false, false, false, true, true, true, false, true) +relu := select(m, v, 0.0) // v where the mask is true, 0 elsewhere +print(m.count(), m.any(), m.all()) // 4 true false +if (v == w).all() { ... } // a condition needs one bool: all() or any() +``` + +| | | +|---|---| +| `v < w`, `<=`, `>`, `>=`, `==`, `!=` | a mask (a plain number on one side counts for every lane) | +| `select(m, a, b)` | lane by lane: `a` where `m` is true, `b` where false (either may be a plain number) | +| `m.any()`, `m.all()`, `m.count()` | is any lane true, are all, how many | +| `m & n`, `m \| n`, `m ^ n`, `!m` | masks combined | +| `m[i]` | one lane, as a `bool` | +| `i32x8(m)`, `boolx8(v)` | true is 1; a lane that isn't 0 is true | + +**The end of a list.** A list is rarely a whole number of vectors long. Two forms touch only the lanes that are inside it: + +| | | +|---|---| +| `f32x8(list, i, fill)` | `list[i]` … as far as the list goes; the lanes past its end are `fill` | +| `v.store_part(list, i)` | writes only the lanes that fit in the list | + +```c +fn add_one([]f32 xs) { + i := 0 + while i < xs.len { // no separate loop for the rest + (f32x8(xs, i, 0.0) + 1.0).store_part(xs, i) + i += 8 + } +} +``` + +### Moving lanes + +| | | +|---|---| +| `v.shuffle(3, 2, 1, 0)` | the lanes with these numbers (written in the code), in this order; as many lanes as numbers | +| `v.lo()`, `v.hi()` | the lower and the upper half | +| `concat(a, b)` | `a`'s lanes, then `b`'s | +| `table.lookup(idx)` | for every lane of `idx`: `table[idx % table.len]`. A byte table (`u8x16`, `i8x16`) is one `pshufb` on x86, `tbl` on ARM | + +Unpacking 4-bit numbers (two to a byte, as quantized models store them) is shuffle, mask and lookup: + +```c +fn unpack4(u8x8 packed, i8x16 meaning) -> i8x16 { + pairs := packed.shuffle(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7) // each byte twice + odd := u8x16(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1) == 1 + nibbles := select(odd, pairs >> 4, pairs & 15) // low nibble, then high + return meaning.lookup(nibbles) // 0..15 -> what it stands for +} +``` + +### Int8 dot products: dot_add + +`acc.dot_add(a, b)` is the step of a quantized model's matrix multiply. `acc` is an `i32x8` of sums; `a` and `b` have four +times as many bytes (`u8x32` and `i8x32`; `i8` × `i8` and `u8` × `u8` work too). Each lane of `acc` gains the sum of four +products, `a[4k]*b[4k] + … + a[4k+3]*b[4k+3]`, exactly (no saturation), and the sums stay in the vector: + +```c +acc := i32x8(0) +i := 0 +while i + 32 <= weights.len { + acc = acc.dot_add(u8x32(activations, i), i8x32(weights, i)) + i += 32 +} +total := acc.sum() // once, at the end +``` + +It is one instruction where the processor has one: `vpdpbusd` with AVX-VNNI (Intel since Alder Lake, AMD Zen 5) or +AVX-512 VNNI, `usdot`/`sdot`/`udot` on ARM with the dot-product features. Elsewhere it is a few instructions +(`pmaddwd` and adds), with the same result. Which one is used depends on `--cpu` or the function's `@cpu` (below). + +### Choosing the instructions: --cpu, @cpu and cpu_has + +By default a program runs on every processor of its system (on x86-64 that means SSE2, so an `f32x8` is done as two +halves). Three ways to use more: + +**`--cpu`, for the whole program.** `--cpu native` (or `cpu = "native"` in `finch.toml`) lets LLVM use everything the +processor that builds it has: AVX2 and FMA, AVX-512, VNNI, or NEON's newer parts on ARM. The program then needs a processor +like that one. To pick a level instead: `--cpu x86-64-v3` (AVX2, most PCs since 2015), `--cpu x86-64-v4` (AVX-512), +`--cpu alderlake`, `--cpu apple-m1`, `--cpu neoverse-v1`. + +**`@cpu`, for one function, and `cpu_has` to choose at run time.** One program can carry an AVX-512 version, an AVX2 +version and a plain one, and pick the best the computer it runs on has: + +```c +@cpu("avx512f,avx512bw,avx512vl,avx512vnni") +fn matmul_avx512([]u8 a, []i8 b, []i32 out) { ... } // may use AVX-512 inside + +@cpu("x86-64-v3") +fn matmul_avx2([]u8 a, []i8 b, []i32 out) { ... } // a level works too + +fn matmul([]u8 a, []i8 b, []i32 out) { + if cpu_has("avx512vnni") && cpu_has("avx512vl") { + matmul_avx512(a, b, out) + } else if cpu_has("avx2") { + matmul_avx2(a, b, out) + } else { + matmul_plain(a, b, out) + } +} +``` + +- The features are x86's `sse3 ssse3 sse4.1 sse4.2 popcnt avx avx2 fma f16c bmi1 bmi2 avx512f avx512bw avx512dq avx512vl + avx512vnni avx512bf16 avx512fp16 avxvnni`, the levels `x86-64-v2`/`v3`/`v4`, and ARM's `neon fp16 dotprod i8mm bf16 sve sve2`. +- Names of the other architecture are left out when building, so the x86 and ARM versions sit side by side in one file + (and `cpu_has` says `false` for them). +- Call a `@cpu` function only after `cpu_has` said yes: on a processor without the feature it stops with *Illegal instruction*. +- A vector wider than 16 bytes (`f32x8`, `i32x16` …) can't be passed between functions with different `@cpu`: the two + sides would expect it in different registers. Pass the list (or a `ptr`) instead; Finch says so if you try. Lanes wider than the processor's registers are split by LLVM, and the same code builds for x86-64 and ARM64 alike. `v.sum()` on float lanes adds them in whatever order is fastest, so the last digits can differ from adding them one by one. + +On NixOS, Nix's clang wrapper removes `-march=native` inside `nix-shell`. Finch tells the wrapper to keep it, so +`--cpu native` works there too; if it ever has no effect, Finch warns. diff --git a/docs/pl/dla-inzynierow.md b/docs/pl/dla-inzynierow.md index 425c6d0..eefbeb0 100644 --- a/docs/pl/dla-inzynierow.md +++ b/docs/pl/dla-inzynierow.md @@ -310,11 +310,18 @@ wszędzie tam, gdzie może stać typ; wywołanie `f32x8(…)` trafia do `vecMake | `f32x8(i32x8 v)` | `sitofp`, `fptosi`, `sext`, `trunc`… na całym wektorze | | `v + w`, `v * 2.0` | `fadd`/`add`… na wektorach; argument skalarny jest dopasowywany do typu elementu i rozpowielany | | `v[i]` | miejsce w pamięci: `getelementptr , ptr %v, i64 i` po sprawdzeniu zakresu; albo `extractelement` na wartości tymczasowej | -| `v == w` | `fcmp oeq`/`icmp eq` → `` → `llvm.vector.reduce.and` | +| `v < w`, `v == w` … | `fcmp`/`icmp` → ``, maska (`boolxN`: `vec` z elementami `bool`) | +| `select(m, a, b)` | `select ` | +| `m.any()`, `.all()`, `.count()` | `llvm.vector.reduce.or`/`.and`; `zext` + `llvm.vector.reduce.add` | +| `f32x8(lista, i, wypelnienie)`, `v.store_part(lista, i)` | `llvm.masked.load`/`llvm.masked.store` z maską `<0, 1, …> < len - i` | +| `v.shuffle(…)`, `.lo()`, `.hi()`, `concat(a, b)` | `shufflevector` ze stałą maską | +| `tabela.lookup(idx)` | `and` z `len - 1`, potem `extractelement`/`insertelement` dla każdego elementu: wzorzec, który zaplecza x86 i ARM zamieniają na `pshufb`/`tbl` | +| `acc.dot_add(a, b)` | `llvm.x86.avx512.vpdpbusd.{128,256,512}` albo `llvm.aarch64.neon.{usdot,sdot,udot}`, gdy funkcja ma te rozszerzenia; w przeciwnym razie `zext`/`sext`, `mul`, cztery `shufflevector` co czwarty element i `add` | | `min`, `max`, `abs` | `llvm.minnum`/`maxnum`, `llvm.smin`/`umin`…, `llvm.fabs`/`llvm.abs` | | `v.sum()`, `.min()`, `.max()` | `llvm.vector.reduce.fadd` (z `reassoc`), `.add`, `.fmin`, `.smin`… | | `v.sqrt()`, `.floor()`, `.ceil()`, `.round()` | funkcja wewnętrzna `llvm.*` o tej nazwie | -| `v.fma(b, c)` | `llvm.fmuladd` (jedna instrukcja tam, gdzie procesor ma FMA; `llvm.fma` bez FMA wołałoby funkcję z libm dla każdego elementu) | +| `v.fma(b, c)`, `fma(a, b, c)` | `llvm.fma` (jedno zaokrąglenie; bez sprzętowego FMA wywołanie libm dla każdego elementu) | +| `v.muladd(b, c)`, `muladd(a, b, c)` | `llvm.fmuladd` (jedna instrukcja tam, gdzie procesor ma FMA, a gdzie nie, `fmul` + `fadd`) | | `v.store(lista, i)` | sprawdzenie zakresu, `store <8 x float> …, align 4` | Dzielenie całkowite `/` i `%` jest odrzucane: ani x86, ani ARM nie mają instrukcji dzielącej liczby całkowite element @@ -330,6 +337,20 @@ użyć `vmovaps` na adresie wyrównanym do 16 i program by się wysypał. do kompilacji IR, więc to samo IR staje się kodem SSE2, AVX2 + FMA, AVX-512 albo NEON. Wektory szersze niż rejestry procesora są dzielone na części. +Niektóre działania muszą znać rozszerzenia procesora już przy generowaniu kodu (`dot_add` wybiera `vpdpbusd` tylko tam, +gdzie ono istnieje). `linker.cpuFeatures()` pyta clanga o makra, które sam definiuje dla tego samego systemu i `--cpu` +(`clang -march=X -dM -E pusty.c`: `__AVXVNNI__`, `__AVX512VNNI__`, `__ARM_FEATURE_MATMUL_INT8` …), zamienia je na nazwy +rozszerzeń Fincha i zapamiętuje odpowiedź dla danego clanga i flag; do `gen` trafiają jako `Options.features`. `@cpu("…")` +przy funkcji (`Node.cpu`) dodaje własne: `define … "target-features"="+avx2,+fma"` (z `+evex512` przy każdym rozszerzeniu +AVX-512, którego LLVM 18+ wymaga dla rejestrów 512-bitowych), a `g.curFeatures` to suma obu na czas generowania tej +funkcji. Rozszerzenia drugiej architektury są pomijane (`cpuOf`). `checkCpuCall` nie pozwala przekazać wektora szerszego +niż 128 bitów jako parametru albo wyniku między funkcjami o różnym `@cpu`, bo wywołujący i wywoływany umieściliby go +w innych rejestrach. `cpu_has(nazwa)` woła `finch_cpu_has` z runtime'u: na x86 CPUID (liście 1 i 7) z XGETBV dla stanu +AVX/AVX-512, na Linuksie ARM64 `getauxval(AT_HWCAP/AT_HWCAP2)`, na macOS `sysctlbyname("hw.optional.arm.FEAT_…")`. + +Na Uniksach clang jest zawsze uruchamiany jako `NIX_ENFORCE_NO_NATIVE=0 clang …`: inaczej nakładka cc-wrapper z Niksa +usuwa `-march=native` w `nix-shell`. Jeśli `--cpu native` mimo to nie daje na x86 makra SSE4.2, sterownik ostrzega. + --- ## 8. Własność i pamięć @@ -515,6 +536,21 @@ Testem akceptacyjnym jest `tests/lsp_test.py`. | Brak `return` | UB, jeśli wynik jest użyty | błąd kompilacji | | Kolejność obliczania | w dużej mierze nieokreślona | od lewej do prawej; w przypisaniu najpierw prawa strona | | Przesunięcie o ≥ szerokość, liczba zmiennoprzecinkowa poza zakresem przy zamianie na całkowitą | UB | poison, jak w LLVM (niesprawdzane) | +| `a * b + c` na liczbach zmiennoprzecinkowych | może zostać złożone w FMA (`-ffp-contract`) | nigdy nie jest łączone: dwa zaokrąglenia | + +### Liczby zmiennoprzecinkowe + +Działania na liczbach zmiennoprzecinkowych to zwykłe `fadd`/`fsub`/`fmul`/`fdiv`/`fneg` **bez flag fast-math**, więc LLVM +nie może zmieniać kolejności działań, łączyć ich ani zakładać braku NaN; `linker.fch` kompiluje też IR z +`-ffp-contract=on`, które łączy tylko wywołania `llvm.fmuladd`, o które prosi Finch. Jedyna instrukcja z flagą to +`llvm.vector.reduce.fadd` w `v.sum()` (`reassoc`). `fma` → `llvm.fma`, `muladd` → `llvm.fmuladd`, na liczbach i wektorach +(`fmaCall`, `fusedOp`). + +`f32_bits`/`f32_from_bits`/`f64_…` to `bitcast` (także między wektorami). `f16_to_f32` to `bitcast i16 → half` + `fpext`, +`f32_to_f16` to `fptrunc` + `bitcast`: jedna instrukcja z F16C albo na ARM64, a gdzie indziej `__extendhfsf2`/`__truncsfhf2` +z biblioteki uruchomieniowej kompilatora C. `bf16_to_f32` to `zext` + `shl 16` + `bitcast`; `f32_to_bf16` zaokrągla do +najbliższej parzystej na liczbach całkowitych (`u + 0x7fff + ((u >> 16) & 1)`, potem `>> 16`) i zostawia NaN cichym +(`| 0x40`), więc nie potrzebuje żadnej biblioteki. | Wiszący wskaźnik z `addr()` / `new` + `free` | UB | nadal UB | ### Ścieżka wywołań przy błędach w trakcie działania @@ -569,4 +605,4 @@ nie jest obliczane dwa razy, a wskaźnik nigdy nie wskazuje na wartość tymczas - Typy generyczne są monomorfizowane (kod rośnie z każdą instancją); nie ma ograniczeń na parametry typu, więc złe użycie daje błąd wewnątrz ciała instancji. - Pamięć, którą miał nieudany test, nie jest zwalniana (uruchamianie testów wychodzi z niego przez `longjmp`). -- Wektory nie mają masek (`v < w`), przetasowań ani dzielenia całkowitego. +- Wektory nie mają dzielenia całkowitego, a `lookup` na podstawowym x86-64 (bez SSSE3) działa element po elemencie. diff --git a/docs/pl/dla-technikow.md b/docs/pl/dla-technikow.md index a925f03..1285f4a 100644 --- a/docs/pl/dla-technikow.md +++ b/docs/pl/dla-technikow.md @@ -348,6 +348,41 @@ str(p) // wskaźnik na znaki z C jako tekst **Liczby o stałym rozmiarze się zawijają**, jak w prawdziwym sprzęcie: `u8` równe 255 plus 1 daje 0. Obliczenia odbywają się w typie użytych wartości, więc `u8(250) + 10` to `4`, a nie `260`. +### Liczby zmiennoprzecinkowe: co jest gwarantowane + +`f32` i `f64` to liczby w standardzie IEEE 754, a Finch liczy je **dokładnie tak, jak są zapisane**, tak samo na każdym +procesorze i z każdym `--cpu`: + +- Każde działanie zaokrągla raz, w zapisanej kolejności. `a * b - c * d` to dwa mnożenia i odejmowanie, nigdy nie + złożone w jedno „mnożenie z dodawaniem” (FMA); kompilatory C często robią to domyślnie, co zmienia ostatnie cyfry. +- Nic nie jest przestawiane ani upraszczane: `(a + b) + c` nie zamienia się w `a + (b + c)`, a `x * 0.0` nie jest + z góry uznawane za 0 (gdy `x` to NaN, wynik też jest NaN). Jedyny wyjątek to `v.sum()` na wektorze, które dodaje + elementy w najszybszej kolejności. + +Gdy szybkość jest ważniejsza niż ostatni bit, poproś o nią wprost: + +| | | +|---|---| +| `fma(a, b, c)` | `a * b + c` z jednym zaokrągleniem, zawsze. Jedna instrukcja tam, gdzie procesor ma FMA (z `--cpu`, na ARM64); gdzie indziej wolniejsza funkcja biblioteczna. | +| `muladd(a, b, c)` | `a * b + c` najszybciej, jak się da na tym procesorze: jedną instrukcją, gdzie można, a gdzie nie, mnożeniem i dodawaniem. | + +Obie działają na liczbach i na wektorach (`v.fma(b, c)`, `v.muladd(b, c)`). + +**Liczby jako bity.** Formaty plików i wagi modeli zapisują liczby jako surowe bity. Te funkcje patrzą na bity bez +przeliczania wartości (wszystkie działają też na wektorach, element po elemencie): + +| | | +|---|---| +| `f32_bits(x)` / `f32_from_bits(u)` | `f32` ↔ `u32` z tymi samymi bitami (`f32_bits(1.0)` to `0x3f800000`) | +| `f64_bits(x)` / `f64_from_bits(u)` | `f64` ↔ `u64` | +| `f16_to_f32(h)` / `f32_to_f16(x)` | liczba połówkowej precyzji, trzymana w `u16`, ↔ `f32` (zaokrąglenie do najbliższej) | +| `bf16_to_f32(b)` / `f32_to_bf16(x)` | bfloat16, trzymany w `u16`, ↔ `f32` (zaokrąglenie do najbliższej, remisy do parzystej) | + +```c +[]u16 wagi = wczytaj_wagi() // liczby f16 z pliku +v := f16_to_f32(u16x8(wagi, 0)) // osiem z nich jako f32, naraz +``` + --- ## 7. Operatory @@ -700,6 +735,8 @@ Użycie `.value`, `.pole` albo `[i]` na wskaźniku `null` **zatrzymuje program** | `exit(kod)` | Natychmiast kończy program. | | `addr(x)` | Wskaźnik na `x`; `addr(funkcja)` daje C wskaźnik na funkcję (funkcję zwrotną). | | `new(wartosc)`, `free(p)` | Ręczna pamięć na stercie (zob. [Pamięć](#13-pamięć-kto-co-zwalnia)). | +| `fma(a, b, c)`, `muladd(a, b, c)` | `a * b + c` z jednym zaokrągleniem / najszybciej (zob. [Liczby zmiennoprzecinkowe](#liczby-zmiennoprzecinkowe-co-jest-gwarantowane)). | +| `f32_bits(x)`, `f32_from_bits(u)`, `f16_to_f32(h)`, … | Bity liczby i liczby połówkowej precyzji (zob. [Liczby zmiennoprzecinkowe](#liczby-zmiennoprzecinkowe-co-jest-gwarantowane)). | | `int(x)`, `str(x)`, `u8(x)`, … | Zamiany typów (zob. [Zamiana typów](#zamiana-typów)). `str(x)` działa na każdym typie i daje ten sam tekst, który pokazuje `print`. `int(tekst) or 0` obsługuje tekst, który nie jest liczbą. | --- @@ -1371,17 +1408,15 @@ struktury, elementem tablicy, wartością w mapie i parametrem funkcji, a `print | `+ - * /` | element po elemencie (`/` tylko na liczbach zmiennoprzecinkowych: x86 i ARM nie dzielą liczb całkowitych element po elemencie) | | `& \| ^ << >> ~` | element po elemencie, na liczbach całkowitych | | `-v` | każdy element ze zmienionym znakiem | -| `v == w`, `v != w` | `true`, jeśli wszystkie elementy są równe / jeśli któryś się różni | | `v[i]`, `v[i] = x`, `v.len` | jeden element; liczba elementów | | `min(v, w)`, `max(v, w)`, `abs(v)`, `clamp(v, lo, hi)` | element po elemencie; `lo`, `hi` i `w` mogą być zwykłymi liczbami | | `v.sum()`, `v.min()`, `v.max()` | wszystkie elementy złożone w jedną liczbę | | `v.dot(w)` | `(v * w).sum()` | | `v.sqrt()`, `v.floor()`, `v.ceil()`, `v.round()` | każdy element (liczby zmiennoprzecinkowe) | -| `v.fma(b, c)` | `v * b + c`, jedną instrukcją tam, gdzie procesor ją ma | +| `v.fma(b, c)` | `v * b + c` z jednym zaokrągleniem, zawsze (tak samo na każdym procesorze) | +| `v.muladd(b, c)` | `v * b + c` najszybciej, jak się da: jedną instrukcją, gdzie procesor to potrafi (ostatni bit może się różnić między procesorami) | | `v.store(lista, i)` | zapisuje elementy do `lista[i]` … (albo przez `ptr`) | -`<` i inne porównania dałyby osobną odpowiedź dla każdego elementu, więc ich nie ma; użyj `min`, `max` albo `v[i] < w[i]`. - **Prawdziwa pętla.** Zwykły schemat: tyle całych wektorów, ile się zmieści, a resztę po jednej liczbie. ```c @@ -1389,7 +1424,7 @@ fn iloczyn([]f32 a, []f32 b) -> f32 { suma := f32x8(0.0) i := 0 while i + 8 <= a.len { - suma = f32x8(a, i).fma(f32x8(b, i), suma) + suma = f32x8(a, i).muladd(f32x8(b, i), suma) i += 8 } wynik := suma.sum() @@ -1403,12 +1438,129 @@ fn iloczyn([]f32 a, []f32 b) -> f32 { `examples/simd.fch` mierzy czas tej pętli i porównuje go ze zwykłą. -**Jakie instrukcje.** Domyślnie program działa na każdym procesorze swojego systemu (na x86-64 oznacza to SSE2, więc -`f32x8` liczy się w dwóch połówkach). `--cpu native` (albo `cpu = "native"` w `finch.toml`) pozwala LLVM użyć wszystkiego, -co ma procesor, na którym budujesz: AVX2 i FMA, AVX-512 albo nowszych części NEON na ARM. Taki program potrzebuje potem +### Maski: porównywanie, wybieranie, koniec listy + +Porównanie wektorów porównuje każdy element, więc odpowiedzią jest **maska**: jedno `true`/`false` na element, +typu `boolx8` (dla 8 elementów). + +```c +v := f32x8(-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, -4.0, 5.0) +m := v > 0.0 // boolx8(false, false, false, true, true, true, false, true) +relu := select(m, v, 0.0) // v tam, gdzie maska jest prawdą, a gdzie indziej 0 +print(m.count(), m.any(), m.all()) // 4 true false +if (v == w).all() { ... } // warunek potrzebuje jednego bool: all() albo any() +``` + +| | | +|---|---| +| `v < w`, `<=`, `>`, `>=`, `==`, `!=` | maska (zwykła liczba po jednej stronie działa na każdy element) | +| `select(m, a, b)` | element po elemencie: `a` tam, gdzie `m` jest prawdą, `b` tam, gdzie fałszem (każde z nich może być zwykłą liczbą) | +| `m.any()`, `m.all()`, `m.count()` | czy któryś element jest prawdą, czy wszystkie, ile ich jest | +| `m & n`, `m \| n`, `m ^ n`, `!m` | łączenie masek | +| `m[i]` | jeden element jako `bool` | +| `i32x8(m)`, `boolx8(v)` | prawda to 1; element różny od 0 to prawda | + +**Koniec listy.** Lista rzadko ma długość będącą wielokrotnością wektora. Dwie formy dotykają tylko elementów, +które są w liście: + +| | | +|---|---| +| `f32x8(lista, i, wypelnienie)` | `lista[i]` … tak daleko, jak sięga lista; elementy za jej końcem dostają `wypelnienie` | +| `v.store_part(lista, i)` | zapisuje tylko elementy, które mieszczą się w liście | + +```c +fn dodaj_jeden([]f32 xs) { + i := 0 + while i < xs.len { // bez osobnej pętli na resztę + (f32x8(xs, i, 0.0) + 1.0).store_part(xs, i) + i += 8 + } +} +``` + +### Przestawianie elementów + +| | | +|---|---| +| `v.shuffle(3, 2, 1, 0)` | elementy o tych numerach (zapisanych w kodzie), w tej kolejności; tyle elementów, ile numerów | +| `v.lo()`, `v.hi()` | dolna i górna połowa | +| `concat(a, b)` | elementy `a`, a po nich elementy `b` | +| `tabela.lookup(idx)` | dla każdego elementu `idx`: `tabela[idx % tabela.len]`. Tabela bajtów (`u8x16`, `i8x16`) to jedno `pshufb` na x86 i `tbl` na ARM | + +Rozpakowanie liczb 4-bitowych (po dwie w bajcie, tak jak trzymają je skwantowane modele) to przestawienie, maska i tabela: + +```c +fn rozpakuj4(u8x8 spakowane, i8x16 znaczenie) -> i8x16 { + pary := spakowane.shuffle(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7) // każdy bajt dwa razy + nieparzyste := u8x16(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1) == 1 + polowki := select(nieparzyste, pary >> 4, pary & 15) // najpierw dolne 4 bity, potem górne + return znaczenie.lookup(polowki) // 0..15 -> to, co oznacza +} +``` + +### Iloczyn skalarny na bajtach: dot_add + +`acc.dot_add(a, b)` to krok mnożenia macierzy w skwantowanym modelu. `acc` to `i32x8` z sumami; `a` i `b` mają cztery +razy więcej bajtów (`u8x32` i `i8x32`; działają też `i8` × `i8` i `u8` × `u8`). Każdy element `acc` dostaje sumę czterech +iloczynów, `a[4k]*b[4k] + … + a[4k+3]*b[4k+3]`, dokładnie (bez nasycania), a sumy zostają w wektorze: + +```c +acc := i32x8(0) +i := 0 +while i + 32 <= wagi.len { + acc = acc.dot_add(u8x32(aktywacje, i), i8x32(wagi, i)) + i += 32 +} +suma := acc.sum() // raz, na końcu +``` + +To jedna instrukcja tam, gdzie procesor ją ma: `vpdpbusd` z AVX-VNNI (Intel od Alder Lake, AMD Zen 5) albo +AVX-512 VNNI, `usdot`/`sdot`/`udot` na ARM z rozszerzeniami do iloczynu skalarnego. Gdzie indziej to kilka instrukcji +(`pmaddwd` i dodawania), z tym samym wynikiem. Wybór zależy od `--cpu` albo `@cpu` funkcji (niżej). + +### Wybór instrukcji: --cpu, @cpu i cpu_has + +Domyślnie program działa na każdym procesorze swojego systemu (na x86-64 oznacza to SSE2, więc `f32x8` liczy się +w dwóch połówkach). Są trzy sposoby, żeby użyć więcej: + +**`--cpu`, dla całego programu.** `--cpu native` (albo `cpu = "native"` w `finch.toml`) pozwala LLVM użyć wszystkiego, +co ma procesor, na którym budujesz: AVX2 i FMA, AVX-512, VNNI albo nowszych części NEON na ARM. Program potrzebuje wtedy podobnego procesora. Można też wybrać poziom: `--cpu x86-64-v3` (AVX2, większość komputerów od 2015 roku), -`--cpu x86-64-v4` (AVX-512), `--cpu apple-m1`. +`--cpu x86-64-v4` (AVX-512), `--cpu alderlake`, `--cpu apple-m1`, `--cpu neoverse-v1`. + +**`@cpu` dla jednej funkcji i `cpu_has` do wyboru w trakcie działania.** Jeden program może mieć wersję z AVX-512, +wersję z AVX2 i zwykłą, i wybrać najlepszą, jaką ma komputer, na którym działa: + +```c +@cpu("avx512f,avx512bw,avx512vl,avx512vnni") +fn mnoz_avx512([]u8 a, []i8 b, []i32 wynik) { ... } // może w środku używać AVX-512 + +@cpu("x86-64-v3") +fn mnoz_avx2([]u8 a, []i8 b, []i32 wynik) { ... } // poziom też działa + +fn mnoz([]u8 a, []i8 b, []i32 wynik) { + if cpu_has("avx512vnni") && cpu_has("avx512vl") { + mnoz_avx512(a, b, wynik) + } else if cpu_has("avx2") { + mnoz_avx2(a, b, wynik) + } else { + mnoz_zwykle(a, b, wynik) + } +} +``` + +- Rozszerzenia to dla x86 `sse3 ssse3 sse4.1 sse4.2 popcnt avx avx2 fma f16c bmi1 bmi2 avx512f avx512bw avx512dq avx512vl + avx512vnni avx512bf16 avx512fp16 avxvnni`, poziomy `x86-64-v2`/`v3`/`v4`, a dla ARM `neon fp16 dotprod i8mm bf16 sve sve2`. +- Nazwy z drugiej architektury są przy budowaniu pomijane, więc wersje dla x86 i ARM mogą stać obok siebie w jednym pliku + (a `cpu_has` zwraca dla nich `false`). +- Funkcję z `@cpu` wołaj dopiero wtedy, gdy `cpu_has` odpowie „tak”: na procesorze bez tego rozszerzenia program + zatrzyma się z *Illegal instruction*. +- Wektora szerszego niż 16 bajtów (`f32x8`, `i32x16` …) nie da się przekazać między funkcjami o różnym `@cpu`: obie strony + oczekiwałyby go w innych rejestrach. Przekaż zamiast tego listę (albo `ptr`); Finch o tym powie, jeśli spróbujesz. Wektory szersze niż rejestry procesora LLVM dzieli na części, a ten sam kod buduje się tak samo na x86-64 i na ARM64. `v.sum()` na liczbach zmiennoprzecinkowych dodaje elementy w kolejności, która jest najszybsza, więc ostatnie cyfry mogą się różnić od dodawania ich po kolei. + +Na NixOS nakładka Niksa na clanga usuwa `-march=native` w `nix-shell`. Finch mówi jej, żeby to zostawiła, więc +`--cpu native` działa i tam; gdyby kiedyś nie miało żadnego skutku, Finch ostrzeże. diff --git a/docs/pl/sciaga.md b/docs/pl/sciaga.md index d0d6c49..1d88ed0 100644 --- a/docs/pl/sciaga.md +++ b/docs/pl/sciaga.md @@ -255,12 +255,19 @@ Typy: `f32x4/8/16`, `f64x2/4/8`, `i8x16/32/64`, `i16x8/16/32`, `i32x4/8/16`, `i6 | `f32x8(v)` | 8-elementowy wektor przeliczony element po elemencie (`f32x8(calkowite)`, `i32x8(ulamki)`) | | `v + w`, `v * 2.0`, `-v` | element po elemencie; zwykła liczba działa na każdy element (`/` tylko dla liczb zmiennoprzecinkowych) | | `& \| ^ << >> ~` | element po elemencie, dla liczb całkowitych | -| `v == w`, `v != w` | bool: czy wszystkie elementy są równe / czy któryś się różni | +| `v < w`, `v == w`, … | maska `boolx8`: `m.all()`, `m.any()`, `m.count()`, `m & n`, `!m`, `m[i]` | +| `select(m, a, b)` | element po elemencie: `a` tam, gdzie `m` jest prawdą, a gdzie nie, `b` | | `v[i]`, `v[i] = x`, `v.len` | jeden element; liczba elementów | | `min(v, w)`, `max`, `abs(v)`, `clamp(v, lo, hi)` | element po elemencie | | `v.sum()`, `v.min()`, `v.max()`, `v.dot(w)` | jedna liczba ze wszystkich elementów | -| `v.sqrt()`, `v.floor()`, `v.ceil()`, `v.round()`, `v.fma(b, c)` | każdy element (liczby zmiennoprzecinkowe; `fma` = `v * b + c`) | +| `v.sqrt()`, `v.floor()`, `v.ceil()`, `v.round()` | każdy element (liczby zmiennoprzecinkowe) | +| `v.fma(b, c)`, `v.muladd(b, c)` | `v * b + c`: zawsze z jednym zaokrągleniem / najszybciej na tym procesorze | | `v.store(lista, i)` | zapisuje elementy do `lista[i]` … | +| `f32x8(lista, i, wypelnienie)`, `v.store_part(lista, i)` | koniec listy: tylko elementy, które są w niej (pozostałe to `wypelnienie`) | +| `v.shuffle(3, 2, 1, 0)`, `v.lo()`, `v.hi()`, `concat(a, b)` | przestawione elementy (shuffle przyjmuje numery zapisane w kodzie) | +| `tabela.lookup(idx)` | `tabela[idx % tabela.len]` dla każdego elementu (`pshufb`, `tbl`) | +| `acc.dot_add(a, b)` | sumy `i32x8` dostają po 4 iloczyny bajtów (`u8x32` × `i8x32`): `vpdpbusd`, `usdot` tam, gdzie są | +| `@cpu("avx2,fma") fn …`, `cpu_has("avx2")` | funkcja, która może używać tych rozszerzeń; przed jej wywołaniem sprawdź procesor | --- @@ -324,6 +331,9 @@ Automatycznie tylko wtedy, gdy nic nie ginie: mniejsza liczba → większa, licz | `shell(polecenie)` | uruchamia polecenie, zwraca jego kod wyjścia | | `exit(kod)` | od razu kończy program (kończy też blok `or { }`) | | `min(a, b, …)`, `max(a, b, …)`, `abs(x)`, `clamp(x, lo, hi)` | dla dowolnych liczb (twoja funkcja o tej samej nazwie ma pierwszeństwo) | +| `fma(a, b, c)`, `muladd(a, b, c)` | `a * b + c` z jednym zaokrągleniem / jedną instrukcją, gdzie procesor potrafi (zapisane `a * b + c` nigdy nie jest łączone) | +| `f32_bits(x)`, `f32_from_bits(u)`, `f64_bits`, `f64_from_bits` | bity liczby zmiennoprzecinkowej jako `u32`/`u64` i z powrotem, bez zmian | +| `f16_to_f32(h)`, `f32_to_f16(x)`, `bf16_to_f32(b)`, `f32_to_bf16(x)` | liczby połówkowej precyzji i bfloat16 (w `u16`) ↔ `f32`; także na wektorach | | `error(komunikat)` | `return error("…")` w funkcji `-> T!` | | `addr(x)` | wskaźnik na x; `addr(fn)` = funkcja zwrotna dla C | | `assert(warunek)`, `assert(warunek, "dlaczego")` | zatrzymuje program, gdy warunek jest fałszywy (w bloku `test`: ten test nie przechodzi) | diff --git a/editors/kate/finch.xml b/editors/kate/finch.xml index f9f2642..b8a9229 100644 --- a/editors/kate/finch.xml +++ b/editors/kate/finch.xml @@ -82,6 +82,19 @@ atomic_store atomic_swap atomic_cas + select + concat + fma + muladd + cpu_has + f32_bits + f32_from_bits + f64_bits + f64_from_bits + f16_to_f32 + f32_to_f16 + bf16_to_f32 + f32_to_bf16 @@ -104,7 +117,8 @@ - + + diff --git a/editors/vscode/CHANGELOG.md b/editors/vscode/CHANGELOG.md index a0a750c..fc8f613 100644 --- a/editors/vscode/CHANGELOG.md +++ b/editors/vscode/CHANGELOG.md @@ -2,7 +2,8 @@ ## 2.6.0 -- Vector types (`f32x8`, `i8x32`, `f64x4`, …) highlighted as types; completion of their methods (`sum`, `dot`, `fma`, `store`, …). +- Vector types (`f32x8`, `i8x32`, `f64x4`, …) and masks (`boolx8`) highlighted as types; completion of their methods (`sum`, `dot_add`, `shuffle`, `lookup`, `store`, …). +- `@cpu(...)` and the new built-ins (`select`, `concat`, `fma`, `muladd`, `cpu_has`, `f32_bits`, `f16_to_f32`, …) highlighted. ## 2.5.0 diff --git a/editors/vscode/syntaxes/finch.tmLanguage.json b/editors/vscode/syntaxes/finch.tmLanguage.json index e3c2dea..8ce8dd8 100644 --- a/editors/vscode/syntaxes/finch.tmLanguage.json +++ b/editors/vscode/syntaxes/finch.tmLanguage.json @@ -147,11 +147,15 @@ "match": "\\bself\\b" }, { - "name": "storage.type.primitive.finch", - "match": "\\b(int|float|bool|char|str|ptr|map|i8|i16|i32|i64|u8|u16|u32|u64|f32|f64|[iuf](?:8|16|32|64)x(?:2|4|8|16|32|64))\\b" + "name": "storage.modifier.attribute.finch", + "match": "@cpu\\b" }, { - "match": "\\b(print|input|read_file|write_file|file_exists|delete_file|shell|exit|addr|new|free|error|min|max|abs|clamp|assert|atomic_add|atomic_sub|atomic_load|atomic_store|atomic_swap|atomic_cas)\\s*(?=\\()", + "name": "storage.type.primitive.finch", + "match": "\\b(int|float|bool|char|str|ptr|map|i8|i16|i32|i64|u8|u16|u32|u64|f32|f64|(?:[iuf](?:8|16|32|64)|bool)x(?:2|4|8|16|32|64))\\b" + }, + { + "match": "\\b(print|input|read_file|write_file|file_exists|delete_file|shell|exit|addr|new|free|error|min|max|abs|clamp|assert|atomic_add|atomic_sub|atomic_load|atomic_store|atomic_swap|atomic_cas|select|concat|fma|muladd|cpu_has|f32_bits|f32_from_bits|f64_bits|f64_from_bits|f16_to_f32|f32_to_f16|bf16_to_f32|f32_to_bf16)\\s*(?=\\()", "captures": { "1": { "name": "support.function.builtin.finch" diff --git a/editors/vscode/test/grammar.js b/editors/vscode/test/grammar.js index 0903ec5..ce951a9 100644 --- a/editors/vscode/test/grammar.js +++ b/editors/vscode/test/grammar.js @@ -30,6 +30,7 @@ const code = [ ' assert(clamp(x, 0, 9) == 3)', '}', 'fn dot(f32x8 a, i8x32 b) {', + '@cpu("avx2") fn k(boolx8 m) { select(m, a, b) }', ]; const expect = [ // [line, token text, scope that must be present] [0, 'import', 'keyword.control.import'], [0, '"stdio.h"', 'string.quoted.double'], @@ -51,6 +52,7 @@ const expect = [ // [line, token text, scope that must be present] [17, 'f', 'meta.embedded.line'], [17, 'left', 'string.quoted.double'], [18, 'assert', 'support.function.builtin'], [18, 'clamp', 'support.function.builtin'], [20, 'f32x8', 'storage.type.primitive'], [20, 'i8x32', 'storage.type.primitive'], + [21, '@cpu', 'storage.modifier.attribute'], [21, 'boolx8', 'storage.type.primitive'], [21, 'select', 'support.function.builtin'], ]; registry.loadGrammar('source.finch').then((g) => { let state = vsctm.INITIAL, toks = []; diff --git a/examples/simd.fch b/examples/simd.fch index 691227f..a2b2af1 100644 --- a/examples/simd.fch +++ b/examples/simd.fch @@ -13,12 +13,12 @@ fn dot_plain([]f32 a, []f32 b) -> f32 { return total } -// eight at a time: f32x8(list, i) takes list[i] ... list[i+7], fma multiplies and adds in one step +// eight at a time: f32x8(list, i) takes list[i] ... list[i+7]; muladd multiplies and adds, in one step where it can fn dot_simd([]f32 a, []f32 b) -> f32 { acc := f32x8(0.0) i := 0 while i + 8 <= a.len { - acc = f32x8(a, i).fma(f32x8(b, i), acc) + acc = f32x8(a, i).muladd(f32x8(b, i), acc) i += 8 } total := acc.sum() diff --git a/runtime/finch_rt.c b/runtime/finch_rt.c index e9a63fb..dd9afee 100644 --- a/runtime/finch_rt.c +++ b/runtime/finch_rt.c @@ -1147,3 +1147,96 @@ __attribute__((noreturn, cold)) void finch_map_missing(const char *file, int64_t (int)(key->len > 100 ? 100 : key->len), key->ptr ? key->ptr : ""); finch_panic(file, line, msg); } + +// ---------- cpu_has("avx2"): what the processor running the program can do ---------- +// +// x86: CPUID, and for the AVX and AVX-512 families also XGETBV (the system must save those registers, or the +// instructions fault). ARM64: the kernel's hwcaps on Linux and Android, sysctl on macOS. A name of the other +// architecture is false. + +#if defined(__x86_64__) || defined(__i386__) +#include +static int x86_leaf(unsigned leaf, unsigned sub, unsigned r[4]) { + unsigned max = __get_cpuid_max(0, 0); + if (leaf > max) return 0; + __cpuid_count(leaf, sub, r[0], r[1], r[2], r[3]); + return 1; +} +static unsigned long long x86_xcr0(void) { + unsigned lo, hi; + __asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0)); + return ((unsigned long long)hi << 32) | lo; +} +static int finch_cpu_has_c(const char *n) { + unsigned l1[4] = {0}, l7[4] = {0}, l71[4] = {0}; + x86_leaf(1, 0, l1); + x86_leaf(7, 0, l7); + x86_leaf(7, 1, l71); + int osx = (l1[2] >> 27) & 1; // OSXSAVE + unsigned long long xcr0 = osx ? x86_xcr0() : 0; + int ymm = osx && (xcr0 & 6) == 6; // the system saves xmm and ymm + int zmm = ymm && (xcr0 & 0xe0) == 0xe0; // and the AVX-512 registers + if (!strcmp(n, "sse3")) return l1[2] & 1; + if (!strcmp(n, "ssse3")) return (l1[2] >> 9) & 1; + if (!strcmp(n, "sse4.1")) return (l1[2] >> 19) & 1; + if (!strcmp(n, "sse4.2")) return (l1[2] >> 20) & 1; + if (!strcmp(n, "popcnt")) return (l1[2] >> 23) & 1; + if (!strcmp(n, "avx")) return ymm && ((l1[2] >> 28) & 1); + if (!strcmp(n, "fma")) return ymm && ((l1[2] >> 12) & 1); + if (!strcmp(n, "f16c")) return ymm && ((l1[2] >> 29) & 1); + if (!strcmp(n, "avx2")) return ymm && ((l7[1] >> 5) & 1); + if (!strcmp(n, "bmi1")) return (l7[1] >> 3) & 1; + if (!strcmp(n, "bmi2")) return (l7[1] >> 8) & 1; + if (!strcmp(n, "avxvnni")) return ymm && ((l71[0] >> 4) & 1); + if (!strcmp(n, "avx512f")) return zmm && ((l7[1] >> 16) & 1); + if (!strcmp(n, "avx512dq")) return zmm && ((l7[1] >> 17) & 1); + if (!strcmp(n, "avx512bw")) return zmm && ((l7[1] >> 30) & 1); + if (!strcmp(n, "avx512vl")) return zmm && ((l7[1] >> 31) & 1); + if (!strcmp(n, "avx512vnni")) return zmm && ((l7[2] >> 11) & 1); + if (!strcmp(n, "avx512fp16")) return zmm && ((l7[3] >> 23) & 1); + if (!strcmp(n, "avx512bf16")) return zmm && ((l71[0] >> 5) & 1); + return 0; +} +#elif defined(__aarch64__) && defined(__APPLE__) +#include +static int sysctl_flag(const char *name) { + int v = 0; + size_t len = sizeof v; + if (sysctlbyname(name, &v, &len, NULL, 0) != 0) return 0; + return v != 0; +} +static int finch_cpu_has_c(const char *n) { + if (!strcmp(n, "neon")) return 1; + if (!strcmp(n, "fp16")) return sysctl_flag("hw.optional.arm.FEAT_FP16"); + if (!strcmp(n, "dotprod")) return sysctl_flag("hw.optional.arm.FEAT_DotProd"); + if (!strcmp(n, "i8mm")) return sysctl_flag("hw.optional.arm.FEAT_I8MM"); + if (!strcmp(n, "bf16")) return sysctl_flag("hw.optional.arm.FEAT_BF16"); + return 0; // no SVE on Apple chips +} +#elif defined(__aarch64__) && defined(__linux__) +#include +static int finch_cpu_has_c(const char *n) { + unsigned long h = getauxval(AT_HWCAP), h2 = getauxval(AT_HWCAP2); + if (!strcmp(n, "neon")) return 1; + if (!strcmp(n, "fp16")) return (h >> 10) & 1; // HWCAP_ASIMDHP + if (!strcmp(n, "dotprod")) return (h >> 20) & 1; // HWCAP_ASIMDDP + if (!strcmp(n, "sve")) return (h >> 22) & 1; // HWCAP_SVE + if (!strcmp(n, "sve2")) return (h2 >> 1) & 1; // HWCAP2_SVE2 + if (!strcmp(n, "i8mm")) return (h2 >> 13) & 1; // HWCAP2_I8MM + if (!strcmp(n, "bf16")) return (h2 >> 14) & 1; // HWCAP2_BF16 + return 0; +} +#else +static int finch_cpu_has_c(const char *n) { + (void)n; + return 0; +} +#endif + +int finch_cpu_has(const FStr *name) { + char n[32]; + int64_t len = name->len < 31 ? name->len : 31; + memcpy(n, name->ptr ? name->ptr : "", (size_t)len); + n[len] = 0; + return finch_cpu_has_c(n) ? 1 : 0; +} diff --git a/shell.nix b/shell.nix index 1e3655a..90201e2 100644 --- a/shell.nix +++ b/shell.nix @@ -15,4 +15,7 @@ pkgs.mkShell { vulkan-headers vulkan-loader ]; + + # Nix's clang wrapper drops -march=native unless this is 0 (finch --cpu native also tells it so itself) + NIX_ENFORCE_NO_NATIVE = 0; } diff --git a/src/ast.fch b/src/ast.fch index 4c60451..abbc87b 100644 --- a/src/ast.fch +++ b/src/ast.fch @@ -44,7 +44,7 @@ fn vecType(str name) -> Type { return Type() } lane := name.sub(0, x) - if !["i8", "i16", "i32", "i64", "u8", "u16", "u32", "u64", "f32", "f64"].contains(lane) { + if !["i8", "i16", "i32", "i64", "u8", "u16", "u32", "u64", "f32", "f64", "bool"].contains(lane) { return Type() } n := 0 @@ -57,7 +57,7 @@ fn vecType(str name) -> Type { if n != 2 && n != 4 && n != 8 && n != 16 && n != 32 && n != 64 { return Type() } - bits := int(lane.sub(1, lane.len)) or 0 + bits := int(lane.sub(1, lane.len)) or 1 // bool: a mask, one bit a lane if n * bits > 512 { return Type() // the widest registers (AVX-512) hold 512 bits } @@ -147,6 +147,7 @@ struct Node { bool flag // decl: has a type; or: the right side is a { block } bool fallible // fn: -> T! or -> ! bool exported // export fn: a C function of a library (finch build --shared) + str cpu // @cpu("avx2,fma") fn: the processor features the function may use []Node kids []str names // named arguments: Point(x: 1) int line diff --git a/src/docs.fch b/src/docs.fch index 271660a..44f4615 100644 --- a/src/docs.fch +++ b/src/docs.fch @@ -26,6 +26,19 @@ fn builtins() -> []Doc { doc("max", "max(a, b, ...) -> number", "The biggest of the numbers, in the type they all fit."), doc("abs", "abs(x) -> number", "x without its minus sign (any number type)."), doc("clamp", "clamp(x, lo, hi) -> number", "x, but at least lo and at most hi."), + doc("cpu_has", "cpu_has(str feature) -> bool", "Does the processor running the program have this feature (\"avx2\", \"avx512f\", \"avxvnni\", \"dotprod\", \"i8mm\" ...)? Check before calling a @cpu(...) function."), + doc("concat", "concat(V a, V b) -> V2", "One vector with a's lanes, then b's (twice as many)."), + doc("select", "select(mask, a, b) -> V", "Lane by lane: a where the mask is true, b where it is false. The mask comes from comparing vectors (v > 0.0)."), + doc("fma", "fma(a, b, c) -> float", "a * b + c with one rounding, always (numbers or vectors). a * b + c written out is never fused: two roundings."), + doc("muladd", "muladd(a, b, c) -> float", "a * b + c the fastest way here: one fused instruction where the processor has one, else a multiply and an add."), + doc("f32_bits", "f32_bits(f32 x) -> u32", "The bits of x, unchanged (also f32xN -> u32xN). f64_bits gives a u64."), + doc("f32_from_bits", "f32_from_bits(u32 bits) -> f32", "The f32 with these bits (also u32xN -> f32xN). f64_from_bits takes a u64."), + doc("f64_bits", "f64_bits(float x) -> u64", "The bits of x, unchanged (also f64xN -> u64xN)."), + doc("f64_from_bits", "f64_from_bits(u64 bits) -> float", "The float with these bits (also u64xN -> f64xN)."), + doc("f16_to_f32", "f16_to_f32(u16 half) -> f32", "A half-precision number, kept in a u16, as an f32 (also u16xN -> f32xN)."), + doc("f32_to_f16", "f32_to_f16(f32 x) -> u16", "x as a half-precision number in a u16, rounded to the nearest (also f32xN -> u16xN)."), + doc("bf16_to_f32", "bf16_to_f32(u16 b) -> f32", "A bfloat16, kept in a u16, as an f32 (also u16xN -> f32xN)."), + doc("f32_to_bf16", "f32_to_bf16(f32 x) -> u16", "x as a bfloat16 in a u16, rounded to the nearest, ties to even (also f32xN -> u16xN)."), doc("atomic_add", "atomic_add(ptr[int] p, n) -> int", "Adds n to the number at p in one step other threads can't split; gives back what was there before."), doc("atomic_sub", "atomic_sub(ptr[int] p, n) -> int", "Takes n from the number at p in one step; gives back what was there before."), doc("atomic_load", "atomic_load(ptr[int] p) -> int", "Reads the number at p, as another thread last wrote it."), @@ -107,8 +120,18 @@ fn vecMethods() -> []Doc { doc("ceil", "ceil() -> V", "Every lane rounded up (f32/f64 lanes)."), doc("round", "round() -> V", "Every lane rounded to the nearest whole number, halves away from 0 (f32/f64 lanes)."), doc("abs", "abs() -> V", "Every lane without its minus sign."), - doc("fma", "fma(V b, V c) -> V", "v * b + c, as one instruction where the processor has one (f32/f64 lanes)."), - doc("store", "store([]T list, int i)", "Writes the lanes into list[i], list[i+1], ... (or through a ptr[T]).") + doc("fma", "fma(V b, V c) -> V", "v * b + c with one rounding, always (f32/f64 lanes). One instruction with FMA hardware, slower without."), + doc("muladd", "muladd(V b, V c) -> V", "v * b + c the fastest way here: fused into one instruction where the processor can (f32/f64 lanes)."), + doc("store", "store([]T list, int i)", "Writes the lanes into list[i], list[i+1], ... (or through a ptr[T])."), + doc("store_part", "store_part([]T list, int i)", "Writes the lanes that fit in the list (for its last, shorter piece)."), + doc("shuffle", "shuffle(int a, int b, ...) -> V", "A vector of the lanes with these numbers (written in the code): v.shuffle(3, 2, 1, 0) reverses an f32x4."), + doc("lo", "lo() -> V", "The lower half of the lanes."), + doc("hi", "hi() -> V", "The upper half of the lanes."), + doc("lookup", "lookup(I indices) -> V", "For every lane of indices: this vector's lane (index % len). A byte-table lookup (pshufb, tbl)."), + doc("dot_add", "dot_add(a, b) -> i32xN", "On an i32xN of sums: each lane gains the sum of 4 byte products of a and b (u8/i8, 4N lanes). One instruction with AVX-VNNI, AVX-512 VNNI or ARM's dot product."), + doc("any", "any() -> bool", "A mask: is any lane true?"), + doc("all", "all() -> bool", "A mask: are all lanes true?"), + doc("count", "count() -> int", "A mask: how many lanes are true.") ] } diff --git a/src/embedded.fch b/src/embedded.fch index 68230e4..465820d 100644 --- a/src/embedded.fch +++ b/src/embedded.fch @@ -20,7 +20,7 @@ fn names() -> []str { fn texts() -> []str { []str list - list.push("// The Finch runtime: string and array helpers, input, files and runtime errors.\n// It is embedded in the compiler, compiled once into a cache folder, and linked\n// statically into every Finch program. Everything here is plain C with no state.\n//\n// Layout shared with the compiler (src/gen.fch):\n// str = { char *ptr; int64 len; int64 cap } ptr is always NUL-terminated\n// cap > 0 : heap memory owned by the value (cap bytes allocated)\n// cap == 0 : a string literal, valid forever, never written to\n// cap == -1: borrowed from C, must be copied before it is kept\n// []T = { T *ptr; int64 len; int64 cap } owns ptr when cap > 0\n// map[K]V: see \"maps\" below\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#ifdef _WIN32\n#define WIN32_LEAN_AND_MEAN\n#define NOMINMAX\n#include \n#include \n#include \n#else\n#include \n#include \n#include \n#include \n#include \n#endif\n\ntypedef struct {\n char *ptr;\n int64_t len;\n int64_t cap;\n} FStr;\n\ntypedef struct {\n void *ptr;\n int64_t len;\n int64_t cap;\n} FArr;\n\nstatic char empty[1] = \"\";\n\n// ---------- runtime errors ----------\n\n// A program that loads Finch libraries (finch build --shared) can take runtime errors over\n// instead of the process exiting: finch_module_on_fatal(handler) in each library. The handler\n// gets the message and must not return (longjmp out, or exit itself).\nstatic void (*fatal_hook)(const char *msg);\nvoid finch_on_fatal(void (*hook)(const char *msg)) { fatal_hook = hook; }\n\n// The chain of Finch calls, kept by the program itself (see @finch_trace in the compiler's output):\n// each frame is { the caller's frame, the function's name, its file, the line it is at }.\ntypedef struct FFrame {\n struct FFrame *prev;\n const char *name;\n const char *file;\n int64_t line;\n} FFrame;\nstatic void *(*trace_top)(void);\nstatic void (*trace_set)(void *);\nvoid finch_trace_init(void *(*top)(void), void (*set)(void *)) {\n trace_top = top;\n trace_set = set;\n}\n\nstatic void print_trace(void) {\n if (!trace_top) return;\n FFrame *f = trace_top();\n for (int n = 0; f; n++, f = f->prev) {\n if (n == 30) {\n fprintf(stderr, \" ...\\n\");\n break;\n }\n fprintf(stderr, \" in %s (%s:%lld)\\n\", f->name, f->file, (long long)f->line);\n }\n}\n\nstatic __attribute__((noreturn)) void fatal(const char *msg) {\n if (fatal_hook) fatal_hook(msg);\n fflush(stdout);\n fprintf(stderr, \"%s\\n\", msg);\n print_trace();\n exit(1);\n}\n\n__attribute__((noreturn, cold)) void finch_panic(const char *file, int64_t line, const char *msg) {\n char full[1024];\n snprintf(full, sizeof full, \"%s:%lld: runtime error: %s\", file, (long long)line, msg);\n fflush(stdout);\n fatal(full);\n}\n\n__attribute__((noreturn, cold)) void finch_panic_index(const char *file, int64_t line, int64_t i, int64_t len) {\n char msg[128];\n snprintf(msg, sizeof msg, \"index %lld is out of range (the length is %lld)\", (long long)i, (long long)len);\n finch_panic(file, line, msg);\n}\n\nstatic void *must(void *p) {\n if (!p) {\n fflush(stdout);\n fatal(\"runtime error: out of memory\");\n }\n return p;\n}\n\nvoid *finch_alloc(int64_t n) { return must(malloc(n ? (size_t)n : 1)); }\nvoid *finch_zalloc(int64_t n) { return must(calloc(1, n ? (size_t)n : 1)); }\n\n// ---------- assert and finch test ----------\n\n__attribute__((noreturn, cold)) void finch_assert_failed(const char *file, int64_t line, const FStr *why) {\n char msg[600];\n if (why->len) snprintf(msg, sizeof msg, \"assert failed: %.*s\", (int)(why->len > 500 ? 500 : why->len), why->ptr);\n else snprintf(msg, sizeof msg, \"assert failed\");\n finch_panic(file, line, msg);\n}\n\n// Each test runs on its own: a runtime error (a failed assert too) ends that test, and the next one runs.\nstatic jmp_buf test_jump;\nstatic char test_message[1100];\nstatic int64_t tests_passed, tests_failed;\n\nstatic void test_fatal(const char *msg) {\n snprintf(test_message, sizeof test_message, \"%s\", msg);\n longjmp(test_jump, 1);\n}\n\nvoid finch_test_run(const char *name, void (*fn)(void)) {\n void (*saved)(const char *) = fatal_hook;\n void *chain = trace_top ? trace_top() : NULL;\n test_message[0] = 0;\n fatal_hook = test_fatal;\n int failed = setjmp(test_jump);\n if (!failed) fn();\n fatal_hook = saved;\n if (trace_set) trace_set(chain); // a test that stopped early left its calls on the chain\n fflush(stdout);\n if (failed) {\n tests_failed++;\n printf(\"FAIL %s\\n %s\\n\", name, test_message);\n } else {\n tests_passed++;\n printf(\"ok %s\\n\", name);\n }\n fflush(stdout);\n}\n\nint finch_test_summary(void) {\n printf(\"\\n%lld passed, %lld failed\\n\", (long long)tests_passed, (long long)tests_failed);\n return tests_failed ? 1 : 0;\n}\n\n// ---------- strings ----------\n\nstatic void str_heap(FStr *out, const char *p, int64_t len) {\n char *m = finch_alloc(len + 1);\n if (len) memcpy(m, p, (size_t)len);\n m[len] = 0;\n out->ptr = m;\n out->len = len;\n out->cap = len + 1;\n}\n\nstatic void str_static(FStr *out, const char *lit) {\n out->ptr = (char *)lit;\n out->len = (int64_t)strlen(lit);\n out->cap = 0;\n}\n\n// A copy that can be kept: literals are shared, everything else is duplicated.\nvoid finch_str_copy(FStr *out, const FStr *s) {\n if (s->cap == 0) *out = *s;\n else str_heap(out, s->ptr, s->len);\n}\n\n// Make `s` writable (before s[i] = c).\nvoid finch_str_own(FStr *s) {\n if (s->cap > 0) return;\n FStr t;\n str_heap(&t, s->ptr, s->len);\n *s = t;\n}\n\nvoid finch_str_drop(FStr *s) {\n if (s->cap > 0) free(s->ptr);\n}\n\nvoid finch_str_from_c(FStr *out, const char *p) {\n if (!p) {\n str_static(out, empty);\n return;\n }\n out->ptr = (char *)p;\n out->len = (int64_t)strlen(p);\n out->cap = -1;\n}\n\nvoid finch_str_concat(FStr *out, const FStr *a, const FStr *b) {\n int64_t len = a->len + b->len;\n char *m = finch_alloc(len + 1);\n memcpy(m, a->ptr, (size_t)a->len);\n memcpy(m + a->len, b->ptr, (size_t)b->len);\n m[len] = 0;\n out->ptr = m;\n out->len = len;\n out->cap = len + 1;\n}\n\nint finch_str_eq(const FStr *a, const FStr *b) {\n return a->len == b->len && memcmp(a->ptr, b->ptr, (size_t)a->len) == 0;\n}\n\nint64_t finch_str_cmp(const FStr *a, const FStr *b) {\n int64_t n = a->len < b->len ? a->len : b->len;\n int c = memcmp(a->ptr, b->ptr, (size_t)n);\n if (c) return c < 0 ? -1 : 1;\n return a->len < b->len ? -1 : a->len > b->len;\n}\n\nvoid finch_str_from_int(FStr *out, int64_t v) {\n char buf[32];\n int n = snprintf(buf, sizeof buf, \"%lld\", (long long)v);\n str_heap(out, buf, n);\n}\n\nvoid finch_str_from_uint(FStr *out, uint64_t v) {\n char buf[32];\n int n = snprintf(buf, sizeof buf, \"%llu\", (unsigned long long)v);\n str_heap(out, buf, n);\n}\n\n// The shortest text that reads back as the same number (as an f32 when `single`):\n// 0.1 -> \"0.1\", 1.0/3 -> \"0.3333333333333333\", 1e8 -> \"100000000\", 1e20 -> \"1e+20\".\nstatic int fmt_float(char *buf, size_t size, double v, int single) {\n if (isnan(v)) return snprintf(buf, size, \"nan\");\n if (isinf(v)) return snprintf(buf, size, v < 0 ? \"-inf\" : \"inf\");\n int p = 1;\n for (; p < 17; p++) {\n snprintf(buf, size, \"%.*e\", p - 1, v);\n double back = strtod(buf, NULL);\n if (single ? (float)back == (float)v : back == v) break;\n }\n int exp = atoi(strchr(buf, 'e') + 1);\n if (exp < -5 || exp >= 17) return snprintf(buf, size, \"%.*g\", p, v);\n int decimals = p - 1 - exp;\n return snprintf(buf, size, \"%.*f\", decimals > 0 ? decimals : 0, v);\n}\n\nvoid finch_str_from_float(FStr *out, double v) {\n char buf[64];\n int n = fmt_float(buf, sizeof buf, v, 0);\n str_heap(out, buf, n);\n}\n\nvoid finch_str_from_f32(FStr *out, double v) {\n char buf[64];\n int n = fmt_float(buf, sizeof buf, v, 1);\n str_heap(out, buf, n);\n}\n\n// str(x, 2): a fixed number of digits after the point\nvoid finch_str_fixed(FStr *out, double v, int64_t digits) {\n char buf[400];\n if (digits < 0) digits = 0;\n if (digits > 60) digits = 60;\n int n = snprintf(buf, sizeof buf, \"%.*f\", (int)digits, v);\n str_heap(out, buf, n);\n}\n\nvoid finch_print_float(double v, int32_t single) {\n char buf[64];\n fmt_float(buf, sizeof buf, v, single);\n fputs(buf, stdout);\n}\n\nvoid finch_str_from_char(FStr *out, char c) { str_heap(out, &c, 1); }\n\nvoid finch_str_from_bool(FStr *out, int b) { str_static(out, b ? \"true\" : \"false\"); }\n\nstatic void number_error(char *msg, size_t size, const FStr *s, const char *kind) {\n snprintf(msg, size, \"can't turn \\\"%.*s\\\" into %s\", (int)(s->len > 100 ? 100 : s->len), s->ptr ? s->ptr : \"\", kind);\n}\n\nstatic int parse_int(const FStr *s, int64_t *out) {\n const char *p = s->ptr ? s->ptr : \"\";\n while (isspace((unsigned char)*p)) p++;\n char *end;\n errno = 0;\n long long v = strtoll(p, &end, 10);\n while (isspace((unsigned char)*end)) end++;\n *out = v;\n return !(end == p || *end || errno == ERANGE);\n}\n\nstatic int parse_float(const FStr *s, double *out) {\n const char *p = s->ptr ? s->ptr : \"\";\n while (isspace((unsigned char)*p)) p++;\n char *end;\n double v = strtod(p, &end);\n while (isspace((unsigned char)*end)) end++;\n *out = v;\n return !(end == p || *end);\n}\n\nint64_t finch_str_to_int(const FStr *s, const char *file, int64_t line) {\n int64_t v;\n if (!parse_int(s, &v)) {\n char msg[200];\n number_error(msg, sizeof msg, s, \"int\");\n finch_panic(file, line, msg);\n }\n return v;\n}\n\ndouble finch_str_to_float(const FStr *s, const char *file, int64_t line) {\n double v;\n if (!parse_float(s, &v)) {\n char msg[200];\n number_error(msg, sizeof msg, s, \"float\");\n finch_panic(file, line, msg);\n }\n return v;\n}\n\n// ---------- errors as values: int(s) or 0, try read_file(p) ... ----------\n// Each gives back 1 on success; on failure 0, with the message in *err.\n\nstatic void set_error(FStr *err, const char *msg) { str_heap(err, msg, (int64_t)strlen(msg)); }\n\nint finch_str_try_int(const FStr *s, int64_t *out, FStr *err) {\n if (parse_int(s, out)) return 1;\n char msg[200];\n number_error(msg, sizeof msg, s, \"int\");\n set_error(err, msg);\n *out = 0;\n return 0;\n}\n\nint finch_str_try_float(const FStr *s, double *out, FStr *err) {\n if (parse_float(s, out)) return 1;\n char msg[200];\n number_error(msg, sizeof msg, s, \"float\");\n set_error(err, msg);\n *out = 0;\n return 0;\n}\n\nstatic void file_error(FStr *err, const char *what, const FStr *path) {\n char msg[300];\n snprintf(msg, sizeof msg, \"can't %s the file \\\"%.*s\\\": %s\", what, (int)(path->len > 200 ? 200 : path->len),\n path->ptr ? path->ptr : \"\", strerror(errno));\n set_error(err, msg);\n}\n\n// when `fn main() -> !` fails\nvoid finch_report_error(const FStr *msg) {\n fflush(stdout);\n fprintf(stderr, \"error: %.*s\\n\", (int)msg->len, msg->ptr ? msg->ptr : \"\");\n finch_str_drop((FStr *)msg);\n}\n\nvoid finch_str_sub(FStr *out, const FStr *s, int64_t start, int64_t end, const char *file, int64_t line) {\n if (start < 0 || end > s->len || start > end) {\n char msg[160];\n snprintf(msg, sizeof msg, \"sub(%lld, %lld) is out of range (the length is %lld)\", (long long)start,\n (long long)end, (long long)s->len);\n finch_panic(file, line, msg);\n }\n str_heap(out, s->ptr + start, end - start);\n}\n\nint64_t finch_str_find(const FStr *s, const FStr *t) {\n if (t->len == 0) return 0;\n for (int64_t i = 0; i + t->len <= s->len; i++)\n if (s->ptr[i] == t->ptr[0] && memcmp(s->ptr + i, t->ptr, (size_t)t->len) == 0) return i;\n return -1;\n}\n\nint finch_str_starts(const FStr *s, const FStr *t) {\n return t->len <= s->len && memcmp(s->ptr, t->ptr, (size_t)t->len) == 0;\n}\n\nint finch_str_ends(const FStr *s, const FStr *t) {\n return t->len <= s->len && memcmp(s->ptr + s->len - t->len, t->ptr, (size_t)t->len) == 0;\n}\n\nvoid finch_str_trim(FStr *out, const FStr *s) {\n int64_t a = 0, b = s->len;\n while (a < b && isspace((unsigned char)s->ptr[a])) a++;\n while (b > a && isspace((unsigned char)s->ptr[b - 1])) b--;\n str_heap(out, s->ptr + a, b - a);\n}\n\nvoid finch_str_upper(FStr *out, const FStr *s) {\n str_heap(out, s->ptr, s->len);\n for (int64_t i = 0; i < out->len; i++) out->ptr[i] = (char)toupper((unsigned char)out->ptr[i]);\n}\n\nvoid finch_str_lower(FStr *out, const FStr *s) {\n str_heap(out, s->ptr, s->len);\n for (int64_t i = 0; i < out->len; i++) out->ptr[i] = (char)tolower((unsigned char)out->ptr[i]);\n}\n\nvoid finch_str_replace(FStr *out, const FStr *s, const FStr *from, const FStr *to) {\n if (from->len == 0) {\n str_heap(out, s->ptr, s->len);\n return;\n }\n int64_t count = 0;\n for (int64_t i = 0; i + from->len <= s->len;) {\n if (memcmp(s->ptr + i, from->ptr, (size_t)from->len) == 0) {\n count++;\n i += from->len;\n } else {\n i++;\n }\n }\n int64_t len = s->len + count * (to->len - from->len);\n char *m = finch_alloc(len + 1), *w = m;\n for (int64_t i = 0; i < s->len;) {\n if (i + from->len <= s->len && memcmp(s->ptr + i, from->ptr, (size_t)from->len) == 0) {\n memcpy(w, to->ptr, (size_t)to->len);\n w += to->len;\n i += from->len;\n } else {\n *w++ = s->ptr[i++];\n }\n }\n *w = 0;\n out->ptr = m;\n out->len = len;\n out->cap = len + 1;\n}\n\nstatic void arr_push_str(FArr *a, const char *p, int64_t len);\n\n// \"a,b,,c\".split(\",\") -> [\"a\", \"b\", \"\", \"c\"]; split(\"\") gives single characters.\nvoid finch_str_split(FArr *out, const FStr *s, const FStr *sep) {\n out->ptr = NULL;\n out->len = out->cap = 0;\n if (sep->len == 0) {\n for (int64_t i = 0; i < s->len; i++) arr_push_str(out, s->ptr + i, 1);\n return;\n }\n int64_t start = 0;\n for (int64_t i = 0; i + sep->len <= s->len;) {\n if (memcmp(s->ptr + i, sep->ptr, (size_t)sep->len) == 0) {\n arr_push_str(out, s->ptr + start, i - start);\n i += sep->len;\n start = i;\n } else {\n i++;\n }\n }\n arr_push_str(out, s->ptr + start, s->len - start);\n}\n\nvoid finch_str_join(FStr *out, const FArr *parts, const FStr *sep) {\n const FStr *p = parts->ptr;\n int64_t len = 0;\n for (int64_t i = 0; i < parts->len; i++) len += p[i].len + (i ? sep->len : 0);\n char *m = finch_alloc(len + 1), *w = m;\n for (int64_t i = 0; i < parts->len; i++) {\n if (i) {\n memcpy(w, sep->ptr, (size_t)sep->len);\n w += sep->len;\n }\n memcpy(w, p[i].ptr, (size_t)p[i].len);\n w += p[i].len;\n }\n *w = 0;\n out->ptr = m;\n out->len = len;\n out->cap = len + 1;\n}\n\nvoid finch_str_repeat(FStr *out, const FStr *s, int64_t n) {\n if (n < 0) n = 0;\n int64_t len = s->len * n;\n char *m = finch_alloc(len + 1);\n for (int64_t i = 0; i < n; i++) memcpy(m + i * s->len, s->ptr, (size_t)s->len);\n m[len] = 0;\n out->ptr = m;\n out->len = len;\n out->cap = len + 1;\n}\n\nvoid finch_str_from_bytes(FStr *out, const FArr *bytes) { str_heap(out, bytes->ptr, bytes->len); }\n\n// ---------- building text: print(x) and str(x) of arrays, maps and structs ----------\n// A buffer starts as an empty str and grows on the heap.\n\nvoid finch_buf_add(FStr *b, const char *p, int64_t n) {\n if (b->len + n + 1 > b->cap) {\n int64_t cap = b->cap > 0 ? b->cap * 2 : 32;\n while (cap < b->len + n + 1) cap *= 2;\n b->ptr = must(realloc(b->cap > 0 ? b->ptr : NULL, (size_t)cap));\n b->cap = cap;\n }\n if (n) memcpy(b->ptr + b->len, p, (size_t)n);\n b->len += n;\n b->ptr[b->len] = 0;\n}\n\nstatic void buf_fmt(FStr *b, const char *fmt, ...) __attribute__((format(printf, 2, 3)));\nstatic void buf_fmt(FStr *b, const char *fmt, ...) {\n char t[64];\n va_list ap;\n va_start(ap, fmt);\n int n = vsnprintf(t, sizeof t, fmt, ap);\n va_end(ap);\n finch_buf_add(b, t, n);\n}\n\nvoid finch_buf_int(FStr *b, int64_t v) { buf_fmt(b, \"%lld\", (long long)v); }\nvoid finch_buf_uint(FStr *b, uint64_t v) { buf_fmt(b, \"%llu\", (unsigned long long)v); }\nvoid finch_buf_float(FStr *b, double v) {\n char t[64];\n finch_buf_add(b, t, fmt_float(t, sizeof t, v, 0));\n}\nvoid finch_buf_f32(FStr *b, double v) {\n char t[64];\n finch_buf_add(b, t, fmt_float(t, sizeof t, v, 1));\n}\nvoid finch_buf_bool(FStr *b, int v) { finch_buf_add(b, v ? \"true\" : \"false\", v ? 4 : 5); }\nvoid finch_buf_ptr(FStr *b, const void *p) {\n if (p) buf_fmt(b, \"0x%llx\", (unsigned long long)(uintptr_t)p);\n else finch_buf_add(b, \"null\", 4);\n}\n\nvoid finch_buf_char(FStr *b, char c, int quoted) {\n if (quoted) finch_buf_add(b, \"'\", 1);\n finch_buf_add(b, &c, 1);\n if (quoted) finch_buf_add(b, \"'\", 1);\n}\n\nvoid finch_buf_str(FStr *b, const FStr *s, int quoted) {\n if (quoted) finch_buf_add(b, \"\\\"\", 1);\n finch_buf_add(b, s->ptr, s->len);\n if (quoted) finch_buf_add(b, \"\\\"\", 1);\n}\n\n// print the buffer and free it\nvoid finch_buf_print(FStr *b) {\n if (b->len) fwrite(b->ptr, 1, (size_t)b->len, stdout);\n if (b->cap > 0) free(b->ptr);\n}\n\n// ---------- arrays ----------\n\nvoid finch_arr_reserve(FArr *a, int64_t need, int64_t es) {\n if (need <= a->cap) return;\n int64_t cap = a->cap * 2;\n if (cap < 4) cap = 4;\n if (cap < need) cap = need;\n a->ptr = must(realloc(a->cap > 0 ? a->ptr : NULL, (size_t)(cap * es)));\n a->cap = cap;\n}\n\nvoid finch_arr_make(FArr *out, int64_t len, int64_t es) {\n if (len < 0) len = 0;\n out->ptr = len ? finch_zalloc(len * es) : NULL;\n out->len = len;\n out->cap = len;\n}\n\n// grow or shrink to `len`; new elements are zero\nvoid finch_arr_resize(FArr *a, int64_t len, int64_t es) {\n if (len < 0) len = 0;\n finch_arr_reserve(a, len, es);\n if (len > a->len) memset((char *)a->ptr + a->len * es, 0, (size_t)((len - a->len) * es));\n a->len = len;\n}\n\n// a byte-for-byte copy of the elements (the compiler deep-copies owning elements afterwards)\nvoid finch_arr_clone_raw(FArr *out, const FArr *a, int64_t es) {\n out->len = a->len;\n out->cap = a->len;\n out->ptr = a->len ? finch_alloc(a->len * es) : NULL;\n if (a->len) memcpy(out->ptr, a->ptr, (size_t)(a->len * es));\n}\n\nvoid finch_arr_free(FArr *a) {\n if (a->cap > 0) free(a->ptr);\n}\n\n// open a hole at i (0 <= i <= len), len grows by one\nvoid finch_arr_insert_gap(FArr *a, int64_t i, int64_t es, const char *file, int64_t line) {\n if (i < 0 || i > a->len) finch_panic_index(file, line, i, a->len);\n finch_arr_reserve(a, a->len + 1, es);\n char *p = a->ptr;\n memmove(p + (i + 1) * es, p + i * es, (size_t)((a->len - i) * es));\n a->len++;\n}\n\n// close the hole at i (the element was already moved out), len shrinks by one\nvoid finch_arr_remove_gap(FArr *a, int64_t i, int64_t es) {\n char *p = a->ptr;\n memmove(p + i * es, p + (i + 1) * es, (size_t)((a->len - i - 1) * es));\n a->len--;\n}\n\nstatic void arr_push_str(FArr *a, const char *p, int64_t len) {\n finch_arr_reserve(a, a->len + 1, (int64_t)sizeof(FStr));\n str_heap((FStr *)a->ptr + a->len, p, len);\n a->len++;\n}\n\nvoid finch_bytes_from_str(FArr *out, const FStr *s) {\n out->len = out->cap = s->len;\n out->ptr = s->len ? finch_alloc(s->len) : NULL;\n if (s->len) memcpy(out->ptr, s->ptr, (size_t)s->len);\n}\n\n// ---------- program arguments, input and files ----------\n\nvoid finch_args(FArr *out, int argc, char **argv) {\n out->len = out->cap = argc;\n out->ptr = finch_alloc((int64_t)sizeof(FStr) * (argc ? argc : 1));\n for (int i = 0; i < argc; i++) str_heap((FStr *)out->ptr + i, argv[i], (int64_t)strlen(argv[i]));\n}\n\n// after main: the arguments go\nvoid finch_args_free(FArr *a) {\n for (int64_t i = 0; i < a->len; i++) finch_str_drop((FStr *)a->ptr + i);\n finch_arr_free(a);\n}\n\n// input(\"Name? \") prints the prompt and reads one line, without the newline.\nvoid finch_input(FStr *out, const FStr *prompt) {\n if (prompt->len) fwrite(prompt->ptr, 1, (size_t)prompt->len, stdout);\n fflush(stdout);\n // read one line of any length (getline isn't available everywhere)\n size_t size = 128, n = 0;\n char *line = finch_alloc((int64_t)size);\n int c, got = 0;\n while ((c = fgetc(stdin)) != EOF) {\n got = 1;\n if (c == '\\n') break;\n if (n + 1 >= size) line = must(realloc(line, size *= 2));\n line[n++] = (char)c;\n }\n if (!got) {\n free(line);\n str_static(out, empty);\n return;\n }\n while (n > 0 && line[n - 1] == '\\r') n--;\n line[n] = 0;\n out->ptr = line;\n out->len = (int64_t)n;\n out->cap = (int64_t)size;\n}\n\nint finch_file_exists(const FStr *path) {\n FILE *f = fopen(path->ptr, \"rb\");\n if (f) fclose(f);\n return f != NULL;\n}\n\nint finch_try_read_file(FStr *out, const FStr *path, FStr *err) {\n FILE *f = fopen(path->ptr ? path->ptr : \"\", \"rb\");\n if (!f) {\n file_error(err, \"read\", path);\n return 0;\n }\n fseek(f, 0, SEEK_END);\n long size = ftell(f);\n fseek(f, 0, SEEK_SET);\n char *m = finch_alloc(size + 1);\n size_t got = fread(m, 1, (size_t)size, f);\n fclose(f);\n m[got] = 0;\n out->ptr = m;\n out->len = (int64_t)got;\n out->cap = size + 1;\n return 1;\n}\n\nvoid finch_read_file(FStr *out, const FStr *path, const char *file, int64_t line) {\n FILE *f = fopen(path->ptr ? path->ptr : \"\", \"rb\");\n if (!f) {\n char msg[300];\n snprintf(msg, sizeof msg, \"can't read the file \\\"%.*s\\\": %s\", (int)(path->len > 200 ? 200 : path->len),\n path->ptr, strerror(errno));\n finch_panic(file, line, msg);\n }\n fseek(f, 0, SEEK_END);\n long size = ftell(f);\n fseek(f, 0, SEEK_SET);\n char *m = finch_alloc(size + 1);\n size_t got = fread(m, 1, (size_t)size, f);\n fclose(f);\n m[got] = 0;\n out->ptr = m;\n out->len = (int64_t)got;\n out->cap = size + 1;\n}\n\n// shell(\"ls -l\"): run a command, give back its exit code\nint64_t finch_shell(const FStr *cmd) {\n fflush(stdout);\n int status = system(cmd->ptr ? cmd->ptr : \"\");\n#ifdef _WIN32\n return status; // Windows gives the exit code directly\n#else\n if (status == -1) return -1;\n if (WIFEXITED(status)) return WEXITSTATUS(status);\n return 128 + (WIFSIGNALED(status) ? WTERMSIG(status) : 0);\n#endif\n}\n\nint finch_delete_file(const FStr *path) { return remove(path->ptr ? path->ptr : \"\") == 0; }\n\nint finch_try_delete_file(const FStr *path, FStr *err) {\n if (remove(path->ptr ? path->ptr : \"\") == 0) return 1;\n file_error(err, \"delete\", path);\n return 0;\n}\n\n// stdin/stdout/stderr are macros on some systems (Windows), so Finch asks for them here\nvoid *finch_std_stream(int i) { return i == 0 ? (void *)stdin : i == 1 ? (void *)stdout : (void *)stderr; }\n\nint finch_write_file(const FStr *path, const FStr *text) {\n FILE *f = fopen(path->ptr ? path->ptr : \"\", \"wb\");\n if (!f) return 0;\n size_t put = text->len ? fwrite(text->ptr, 1, (size_t)text->len, f) : 0;\n return fclose(f) == 0 && put == (size_t)text->len;\n}\n\nint finch_try_write_file(const FStr *path, const FStr *text, FStr *err) {\n errno = 0;\n if (finch_write_file(path, text)) return 1;\n if (!errno) errno = EIO;\n file_error(err, \"write\", path);\n return 0;\n}\n\n// ---------- folders (std/fs.fch, through std/finch_fs.h) ----------\n\nint finch_fs_is_dir(const char *path) {\n#ifdef _WIN32\n struct _finddata_t d;\n intptr_t h = _findfirst(path, &d);\n if (h == -1) return 0;\n _findclose(h);\n return (d.attrib & _A_SUBDIR) != 0;\n#else\n struct stat st;\n return stat(path, &st) == 0 && S_ISDIR(st.st_mode);\n#endif\n}\n\nint finch_fs_exists(const char *path) {\n#ifdef _WIN32\n return _access(path, 0) == 0;\n#else\n struct stat st;\n return stat(path, &st) == 0;\n#endif\n}\n\nstatic int make_one_dir(const char *p) {\n#ifdef _WIN32\n return _mkdir(p) == 0 || errno == EEXIST;\n#else\n return mkdir(p, 0777) == 0 || errno == EEXIST;\n#endif\n}\n\n// mkdir -p: the parents too; 1 if the folder is there afterwards\nint finch_fs_mkdir(const char *path) {\n size_t n = strlen(path);\n char *p = finch_alloc((int64_t)n + 1);\n memcpy(p, path, n + 1);\n for (size_t i = 1; i < n; i++) {\n if (p[i] == '/' || p[i] == '\\\\') {\n if (p[i - 1] == ':') continue; /* C: is a drive */\n char c = p[i];\n p[i] = 0;\n make_one_dir(p);\n p[i] = c;\n }\n }\n int ok = make_one_dir(p) && finch_fs_is_dir(p);\n free(p);\n return ok;\n}\n\nint finch_fs_remove_dir(const char *path) {\n#ifdef _WIN32\n return _rmdir(path) == 0;\n#else\n return rmdir(path) == 0;\n#endif\n}\n\n// the names in a folder, one per call (\"\" at the end), without . and ..\ntypedef struct {\n#ifdef _WIN32\n intptr_t h;\n struct _finddata_t d;\n int first;\n#else\n DIR *d;\n#endif\n} FsList;\n\nvoid *finch_fs_list_open(const char *path) {\n FsList *l = finch_zalloc((int64_t)sizeof(FsList));\n#ifdef _WIN32\n size_t n = strlen(path);\n char *pat = finch_alloc((int64_t)n + 3);\n memcpy(pat, path, n);\n memcpy(pat + n, \"\\\\*\", 3);\n l->h = _findfirst(pat, &l->d);\n free(pat);\n if (l->h == -1) {\n free(l);\n return NULL;\n }\n l->first = 1;\n#else\n l->d = opendir(path);\n if (!l->d) {\n free(l);\n return NULL;\n }\n#endif\n return l;\n}\n\nconst char *finch_fs_list_next(void *handle) {\n FsList *l = handle;\n for (;;) {\n#ifdef _WIN32\n if (!l->first && _findnext(l->h, &l->d) != 0) return \"\";\n l->first = 0;\n const char *name = l->d.name;\n#else\n struct dirent *e = readdir(l->d);\n if (!e) return \"\";\n const char *name = e->d_name;\n#endif\n if (strcmp(name, \".\") != 0 && strcmp(name, \"..\") != 0) return name;\n }\n}\n\nvoid finch_fs_list_close(void *handle) {\n FsList *l = handle;\n#ifdef _WIN32\n _findclose(l->h);\n#else\n closedir(l->d);\n#endif\n free(l);\n}\n\n// where a program keeps its own files for this user (saves, settings); \"\" if unknown\n// Linux: $XDG_DATA_HOME or ~/.local/share macOS: ~/Library/Application Support Windows: %APPDATA%\nconst char *finch_fs_data_home(void) {\n static char buf[4096];\n const char *v;\n#if defined(_WIN32)\n v = getenv(\"APPDATA\");\n snprintf(buf, sizeof buf, \"%s\", v ? v : \"\");\n#elif defined(__APPLE__)\n v = getenv(\"HOME\");\n if (v) snprintf(buf, sizeof buf, \"%s/Library/Application Support\", v);\n else buf[0] = 0;\n#else\n v = getenv(\"XDG_DATA_HOME\");\n if (v && *v) snprintf(buf, sizeof buf, \"%s\", v);\n else if ((v = getenv(\"HOME\"))) snprintf(buf, sizeof buf, \"%s/.local/share\", v);\n else buf[0] = 0;\n#endif\n return buf;\n}\n\n// ---------- threads (std/thread.fch, through std/finch_thread.h) ----------\n\ntypedef struct {\n void (*fn)(void *);\n void *arg;\n} ThreadStart;\n\n#ifdef _WIN32\nstatic DWORD WINAPI thread_main(LPVOID p) {\n ThreadStart st = *(ThreadStart *)p;\n free(p);\n st.fn(st.arg);\n return 0;\n}\n#else\nstatic void *thread_main(void *p) {\n ThreadStart st = *(ThreadStart *)p;\n free(p);\n st.fn(st.arg);\n return NULL;\n}\n#endif\n\n// run fn(arg) on a new thread; NULL if it couldn't start\nvoid *finch_thread_spawn(void (*fn)(void *), void *arg) {\n ThreadStart *st = finch_alloc((int64_t)sizeof(ThreadStart));\n st->fn = fn;\n st->arg = arg;\n#ifdef _WIN32\n HANDLE h = CreateThread(NULL, 0, thread_main, st, 0, NULL);\n if (!h) {\n free(st);\n return NULL;\n }\n return h;\n#else\n pthread_t *t = finch_alloc((int64_t)sizeof(pthread_t));\n if (pthread_create(t, NULL, thread_main, st) != 0) {\n free(st);\n free(t);\n return NULL;\n }\n return t;\n#endif\n}\n\n// wait for the thread to end; 1 if that worked\nint finch_thread_join(void *t) {\n if (!t) return 0;\n#ifdef _WIN32\n int ok = WaitForSingleObject((HANDLE)t, INFINITE) == WAIT_OBJECT_0;\n CloseHandle((HANDLE)t);\n return ok;\n#else\n int ok = pthread_join(*(pthread_t *)t, NULL) == 0;\n free(t);\n return ok;\n#endif\n}\n\nvoid *finch_mutex_new(void) {\n#ifdef _WIN32\n CRITICAL_SECTION *m = finch_alloc((int64_t)sizeof(CRITICAL_SECTION));\n InitializeCriticalSection(m);\n#else\n pthread_mutex_t *m = finch_alloc((int64_t)sizeof(pthread_mutex_t));\n pthread_mutex_init(m, NULL);\n#endif\n return m;\n}\n\nvoid finch_mutex_lock(void *m) {\n#ifdef _WIN32\n EnterCriticalSection((CRITICAL_SECTION *)m);\n#else\n pthread_mutex_lock((pthread_mutex_t *)m);\n#endif\n}\n\nvoid finch_mutex_unlock(void *m) {\n#ifdef _WIN32\n LeaveCriticalSection((CRITICAL_SECTION *)m);\n#else\n pthread_mutex_unlock((pthread_mutex_t *)m);\n#endif\n}\n\nvoid finch_mutex_free(void *m) {\n if (!m) return;\n#ifdef _WIN32\n DeleteCriticalSection((CRITICAL_SECTION *)m);\n#else\n pthread_mutex_destroy((pthread_mutex_t *)m);\n#endif\n free(m);\n}\n\nint finch_cpu_count(void) {\n#ifdef _WIN32\n SYSTEM_INFO si;\n GetSystemInfo(&si);\n return (int)si.dwNumberOfProcessors;\n#else\n long n = sysconf(_SC_NPROCESSORS_ONLN);\n return n > 0 ? (int)n : 1;\n#endif\n}\n\n// ---------- maps ----------\n// map[K]V = { entries; len; used; cap; index; icap; adds }\n// entries: `used` slots of es bytes, each { uint64 hash; K key; V value }, in insertion order.\n// A removed entry keeps its slot (hash 0) until the next resize packs the slots.\n// index: icap slots (a power of two, at least 2 * cap): -1 empty, -2 removed, else an entry number.\n// adds: how many keys were ever added; a `for` over the map stops with an error if it changes.\n// The compiler stores, copies and drops keys and values; the runtime only finds and places entries.\n// Keys are compared byte for byte (numbers, chars, bools) or as text (kstr).\n\ntypedef struct {\n char *entries;\n int64_t len, used, cap;\n int64_t *index;\n int64_t icap;\n int64_t adds;\n} FMap;\n\nstatic uint64_t map_hash(const void *key, int64_t ks, int kstr) {\n const unsigned char *p;\n size_t n;\n if (kstr) {\n const FStr *s = key;\n p = (const unsigned char *)s->ptr;\n n = (size_t)s->len;\n } else {\n p = key;\n n = (size_t)ks;\n }\n uint64_t h = 1469598103934665603ULL; // FNV-1a, then mixed so the low bits spread well\n for (size_t i = 0; i < n; i++) {\n h ^= p[i];\n h *= 1099511628211ULL;\n }\n h ^= h >> 31;\n h *= 0xbf58476d1ce4e5b9ULL;\n h ^= h >> 29;\n return h | 1; // 0 marks a removed entry\n}\n\nstatic int map_key_eq(const char *a, const void *b, int64_t ks, int kstr) {\n if (!kstr) return memcmp(a, b, (size_t)ks) == 0;\n const FStr *x = (const FStr *)a, *y = b;\n return x->len == y->len && (x->len == 0 || memcmp(x->ptr, y->ptr, (size_t)x->len) == 0);\n}\n\nstatic void map_reindex(FMap *m, int64_t es) {\n if (m->len != m->used) { // pack the slots of removed entries away\n int64_t j = 0;\n for (int64_t i = 0; i < m->used; i++) {\n char *e = m->entries + i * es;\n if (*(uint64_t *)e == 0) continue;\n if (i != j) memmove(m->entries + j * es, e, (size_t)es);\n j++;\n }\n m->used = j;\n }\n int64_t icap = 16;\n while (icap < m->cap * 2) icap *= 2;\n free(m->index);\n m->index = finch_alloc(icap * (int64_t)sizeof(int64_t));\n memset(m->index, 0xff, (size_t)icap * sizeof(int64_t));\n m->icap = icap;\n for (int64_t i = 0; i < m->used; i++) {\n uint64_t h = *(uint64_t *)(m->entries + i * es);\n int64_t s = (int64_t)(h & (uint64_t)(icap - 1));\n while (m->index[s] != -1) s = (s + 1) & (icap - 1);\n m->index[s] = i;\n }\n}\n\n// the entry number of `key`, or -1\nint64_t finch_map_find(const FMap *m, const void *key, int64_t es, int64_t ks, int32_t kstr) {\n if (m->len == 0) return -1;\n uint64_t h = map_hash(key, ks, kstr);\n for (int64_t s = (int64_t)(h & (uint64_t)(m->icap - 1));; s = (s + 1) & (m->icap - 1)) {\n int64_t i = m->index[s];\n if (i == -1) return -1;\n if (i < 0) continue;\n char *e = m->entries + i * es;\n if (*(uint64_t *)e == h && map_key_eq(e + 8, key, ks, kstr)) return i;\n }\n}\n\n// The entry for `key`. A new one is zeroed except its hash (*added = 1); the compiler then stores the key.\nint64_t finch_map_slot(FMap *m, const void *key, int64_t es, int64_t ks, int32_t kstr, int32_t *added) {\n int64_t i = finch_map_find(m, key, es, ks, kstr);\n *added = i < 0;\n if (i >= 0) return i;\n if (m->used == m->cap) {\n if (m->len * 4 < m->cap * 3) { // many removed slots: pack them instead of growing\n map_reindex(m, es);\n } else {\n m->cap = m->cap ? m->cap * 2 : 8;\n m->entries = must(realloc(m->entries, (size_t)(m->cap * es)));\n map_reindex(m, es);\n }\n }\n uint64_t h = map_hash(key, ks, kstr);\n i = m->used++;\n char *e = m->entries + i * es;\n memset(e, 0, (size_t)es);\n *(uint64_t *)e = h;\n int64_t s = (int64_t)(h & (uint64_t)(m->icap - 1));\n while (m->index[s] >= 0) s = (s + 1) & (m->icap - 1);\n m->index[s] = i;\n m->len++;\n m->adds++;\n return i;\n}\n\n// Forget entry i (its key and value were already dropped by the compiler).\nvoid finch_map_remove_at(FMap *m, int64_t i, int64_t es) {\n char *e = m->entries + i * es;\n uint64_t h = *(uint64_t *)e;\n int64_t s = (int64_t)(h & (uint64_t)(m->icap - 1));\n while (m->index[s] != i) s = (s + 1) & (m->icap - 1);\n m->index[s] = -2;\n *(uint64_t *)e = 0;\n if (--m->len == 0) {\n m->used = 0;\n memset(m->index, 0xff, (size_t)m->icap * sizeof(int64_t));\n }\n}\n\nvoid finch_map_clear(FMap *m) {\n m->len = m->used = 0;\n if (m->index) memset(m->index, 0xff, (size_t)m->icap * sizeof(int64_t));\n}\n\nvoid finch_map_free(FMap *m) {\n free(m->entries);\n free(m->index);\n}\n\n// A copy of the slots and the index; the compiler then copies the keys and values that own memory.\nvoid finch_map_clone_raw(FMap *out, const FMap *m, int64_t es) {\n *out = *m;\n if (!m->cap) return;\n out->entries = finch_alloc(m->cap * es);\n memcpy(out->entries, m->entries, (size_t)(m->used * es));\n out->index = finch_alloc(m->icap * (int64_t)sizeof(int64_t));\n memcpy(out->index, m->index, (size_t)m->icap * sizeof(int64_t));\n}\n\n__attribute__((noreturn, cold)) void finch_map_changed(const char *file, int64_t line) {\n finch_panic(file, line, \"a key was added to the map while looping over it (collect the new keys, and add them after the loop)\");\n}\n\n__attribute__((noreturn, cold)) void finch_map_missing(const char *file, int64_t line, const FStr *key) {\n char msg[200];\n snprintf(msg, sizeof msg, \"the key %.*s is not in the map (check with .has(key), or use .get(key, default))\",\n (int)(key->len > 100 ? 100 : key->len), key->ptr ? key->ptr : \"\");\n finch_panic(file, line, msg);\n}\n") + list.push("// The Finch runtime: string and array helpers, input, files and runtime errors.\n// It is embedded in the compiler, compiled once into a cache folder, and linked\n// statically into every Finch program. Everything here is plain C with no state.\n//\n// Layout shared with the compiler (src/gen.fch):\n// str = { char *ptr; int64 len; int64 cap } ptr is always NUL-terminated\n// cap > 0 : heap memory owned by the value (cap bytes allocated)\n// cap == 0 : a string literal, valid forever, never written to\n// cap == -1: borrowed from C, must be copied before it is kept\n// []T = { T *ptr; int64 len; int64 cap } owns ptr when cap > 0\n// map[K]V: see \"maps\" below\n\n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#include \n#ifdef _WIN32\n#define WIN32_LEAN_AND_MEAN\n#define NOMINMAX\n#include \n#include \n#include \n#else\n#include \n#include \n#include \n#include \n#include \n#endif\n\ntypedef struct {\n char *ptr;\n int64_t len;\n int64_t cap;\n} FStr;\n\ntypedef struct {\n void *ptr;\n int64_t len;\n int64_t cap;\n} FArr;\n\nstatic char empty[1] = \"\";\n\n// ---------- runtime errors ----------\n\n// A program that loads Finch libraries (finch build --shared) can take runtime errors over\n// instead of the process exiting: finch_module_on_fatal(handler) in each library. The handler\n// gets the message and must not return (longjmp out, or exit itself).\nstatic void (*fatal_hook)(const char *msg);\nvoid finch_on_fatal(void (*hook)(const char *msg)) { fatal_hook = hook; }\n\n// The chain of Finch calls, kept by the program itself (see @finch_trace in the compiler's output):\n// each frame is { the caller's frame, the function's name, its file, the line it is at }.\ntypedef struct FFrame {\n struct FFrame *prev;\n const char *name;\n const char *file;\n int64_t line;\n} FFrame;\nstatic void *(*trace_top)(void);\nstatic void (*trace_set)(void *);\nvoid finch_trace_init(void *(*top)(void), void (*set)(void *)) {\n trace_top = top;\n trace_set = set;\n}\n\nstatic void print_trace(void) {\n if (!trace_top) return;\n FFrame *f = trace_top();\n for (int n = 0; f; n++, f = f->prev) {\n if (n == 30) {\n fprintf(stderr, \" ...\\n\");\n break;\n }\n fprintf(stderr, \" in %s (%s:%lld)\\n\", f->name, f->file, (long long)f->line);\n }\n}\n\nstatic __attribute__((noreturn)) void fatal(const char *msg) {\n if (fatal_hook) fatal_hook(msg);\n fflush(stdout);\n fprintf(stderr, \"%s\\n\", msg);\n print_trace();\n exit(1);\n}\n\n__attribute__((noreturn, cold)) void finch_panic(const char *file, int64_t line, const char *msg) {\n char full[1024];\n snprintf(full, sizeof full, \"%s:%lld: runtime error: %s\", file, (long long)line, msg);\n fflush(stdout);\n fatal(full);\n}\n\n__attribute__((noreturn, cold)) void finch_panic_index(const char *file, int64_t line, int64_t i, int64_t len) {\n char msg[128];\n snprintf(msg, sizeof msg, \"index %lld is out of range (the length is %lld)\", (long long)i, (long long)len);\n finch_panic(file, line, msg);\n}\n\nstatic void *must(void *p) {\n if (!p) {\n fflush(stdout);\n fatal(\"runtime error: out of memory\");\n }\n return p;\n}\n\nvoid *finch_alloc(int64_t n) { return must(malloc(n ? (size_t)n : 1)); }\nvoid *finch_zalloc(int64_t n) { return must(calloc(1, n ? (size_t)n : 1)); }\n\n// ---------- assert and finch test ----------\n\n__attribute__((noreturn, cold)) void finch_assert_failed(const char *file, int64_t line, const FStr *why) {\n char msg[600];\n if (why->len) snprintf(msg, sizeof msg, \"assert failed: %.*s\", (int)(why->len > 500 ? 500 : why->len), why->ptr);\n else snprintf(msg, sizeof msg, \"assert failed\");\n finch_panic(file, line, msg);\n}\n\n// Each test runs on its own: a runtime error (a failed assert too) ends that test, and the next one runs.\nstatic jmp_buf test_jump;\nstatic char test_message[1100];\nstatic int64_t tests_passed, tests_failed;\n\nstatic void test_fatal(const char *msg) {\n snprintf(test_message, sizeof test_message, \"%s\", msg);\n longjmp(test_jump, 1);\n}\n\nvoid finch_test_run(const char *name, void (*fn)(void)) {\n void (*saved)(const char *) = fatal_hook;\n void *chain = trace_top ? trace_top() : NULL;\n test_message[0] = 0;\n fatal_hook = test_fatal;\n int failed = setjmp(test_jump);\n if (!failed) fn();\n fatal_hook = saved;\n if (trace_set) trace_set(chain); // a test that stopped early left its calls on the chain\n fflush(stdout);\n if (failed) {\n tests_failed++;\n printf(\"FAIL %s\\n %s\\n\", name, test_message);\n } else {\n tests_passed++;\n printf(\"ok %s\\n\", name);\n }\n fflush(stdout);\n}\n\nint finch_test_summary(void) {\n printf(\"\\n%lld passed, %lld failed\\n\", (long long)tests_passed, (long long)tests_failed);\n return tests_failed ? 1 : 0;\n}\n\n// ---------- strings ----------\n\nstatic void str_heap(FStr *out, const char *p, int64_t len) {\n char *m = finch_alloc(len + 1);\n if (len) memcpy(m, p, (size_t)len);\n m[len] = 0;\n out->ptr = m;\n out->len = len;\n out->cap = len + 1;\n}\n\nstatic void str_static(FStr *out, const char *lit) {\n out->ptr = (char *)lit;\n out->len = (int64_t)strlen(lit);\n out->cap = 0;\n}\n\n// A copy that can be kept: literals are shared, everything else is duplicated.\nvoid finch_str_copy(FStr *out, const FStr *s) {\n if (s->cap == 0) *out = *s;\n else str_heap(out, s->ptr, s->len);\n}\n\n// Make `s` writable (before s[i] = c).\nvoid finch_str_own(FStr *s) {\n if (s->cap > 0) return;\n FStr t;\n str_heap(&t, s->ptr, s->len);\n *s = t;\n}\n\nvoid finch_str_drop(FStr *s) {\n if (s->cap > 0) free(s->ptr);\n}\n\nvoid finch_str_from_c(FStr *out, const char *p) {\n if (!p) {\n str_static(out, empty);\n return;\n }\n out->ptr = (char *)p;\n out->len = (int64_t)strlen(p);\n out->cap = -1;\n}\n\nvoid finch_str_concat(FStr *out, const FStr *a, const FStr *b) {\n int64_t len = a->len + b->len;\n char *m = finch_alloc(len + 1);\n memcpy(m, a->ptr, (size_t)a->len);\n memcpy(m + a->len, b->ptr, (size_t)b->len);\n m[len] = 0;\n out->ptr = m;\n out->len = len;\n out->cap = len + 1;\n}\n\nint finch_str_eq(const FStr *a, const FStr *b) {\n return a->len == b->len && memcmp(a->ptr, b->ptr, (size_t)a->len) == 0;\n}\n\nint64_t finch_str_cmp(const FStr *a, const FStr *b) {\n int64_t n = a->len < b->len ? a->len : b->len;\n int c = memcmp(a->ptr, b->ptr, (size_t)n);\n if (c) return c < 0 ? -1 : 1;\n return a->len < b->len ? -1 : a->len > b->len;\n}\n\nvoid finch_str_from_int(FStr *out, int64_t v) {\n char buf[32];\n int n = snprintf(buf, sizeof buf, \"%lld\", (long long)v);\n str_heap(out, buf, n);\n}\n\nvoid finch_str_from_uint(FStr *out, uint64_t v) {\n char buf[32];\n int n = snprintf(buf, sizeof buf, \"%llu\", (unsigned long long)v);\n str_heap(out, buf, n);\n}\n\n// The shortest text that reads back as the same number (as an f32 when `single`):\n// 0.1 -> \"0.1\", 1.0/3 -> \"0.3333333333333333\", 1e8 -> \"100000000\", 1e20 -> \"1e+20\".\nstatic int fmt_float(char *buf, size_t size, double v, int single) {\n if (isnan(v)) return snprintf(buf, size, \"nan\");\n if (isinf(v)) return snprintf(buf, size, v < 0 ? \"-inf\" : \"inf\");\n int p = 1;\n for (; p < 17; p++) {\n snprintf(buf, size, \"%.*e\", p - 1, v);\n double back = strtod(buf, NULL);\n if (single ? (float)back == (float)v : back == v) break;\n }\n int exp = atoi(strchr(buf, 'e') + 1);\n if (exp < -5 || exp >= 17) return snprintf(buf, size, \"%.*g\", p, v);\n int decimals = p - 1 - exp;\n return snprintf(buf, size, \"%.*f\", decimals > 0 ? decimals : 0, v);\n}\n\nvoid finch_str_from_float(FStr *out, double v) {\n char buf[64];\n int n = fmt_float(buf, sizeof buf, v, 0);\n str_heap(out, buf, n);\n}\n\nvoid finch_str_from_f32(FStr *out, double v) {\n char buf[64];\n int n = fmt_float(buf, sizeof buf, v, 1);\n str_heap(out, buf, n);\n}\n\n// str(x, 2): a fixed number of digits after the point\nvoid finch_str_fixed(FStr *out, double v, int64_t digits) {\n char buf[400];\n if (digits < 0) digits = 0;\n if (digits > 60) digits = 60;\n int n = snprintf(buf, sizeof buf, \"%.*f\", (int)digits, v);\n str_heap(out, buf, n);\n}\n\nvoid finch_print_float(double v, int32_t single) {\n char buf[64];\n fmt_float(buf, sizeof buf, v, single);\n fputs(buf, stdout);\n}\n\nvoid finch_str_from_char(FStr *out, char c) { str_heap(out, &c, 1); }\n\nvoid finch_str_from_bool(FStr *out, int b) { str_static(out, b ? \"true\" : \"false\"); }\n\nstatic void number_error(char *msg, size_t size, const FStr *s, const char *kind) {\n snprintf(msg, size, \"can't turn \\\"%.*s\\\" into %s\", (int)(s->len > 100 ? 100 : s->len), s->ptr ? s->ptr : \"\", kind);\n}\n\nstatic int parse_int(const FStr *s, int64_t *out) {\n const char *p = s->ptr ? s->ptr : \"\";\n while (isspace((unsigned char)*p)) p++;\n char *end;\n errno = 0;\n long long v = strtoll(p, &end, 10);\n while (isspace((unsigned char)*end)) end++;\n *out = v;\n return !(end == p || *end || errno == ERANGE);\n}\n\nstatic int parse_float(const FStr *s, double *out) {\n const char *p = s->ptr ? s->ptr : \"\";\n while (isspace((unsigned char)*p)) p++;\n char *end;\n double v = strtod(p, &end);\n while (isspace((unsigned char)*end)) end++;\n *out = v;\n return !(end == p || *end);\n}\n\nint64_t finch_str_to_int(const FStr *s, const char *file, int64_t line) {\n int64_t v;\n if (!parse_int(s, &v)) {\n char msg[200];\n number_error(msg, sizeof msg, s, \"int\");\n finch_panic(file, line, msg);\n }\n return v;\n}\n\ndouble finch_str_to_float(const FStr *s, const char *file, int64_t line) {\n double v;\n if (!parse_float(s, &v)) {\n char msg[200];\n number_error(msg, sizeof msg, s, \"float\");\n finch_panic(file, line, msg);\n }\n return v;\n}\n\n// ---------- errors as values: int(s) or 0, try read_file(p) ... ----------\n// Each gives back 1 on success; on failure 0, with the message in *err.\n\nstatic void set_error(FStr *err, const char *msg) { str_heap(err, msg, (int64_t)strlen(msg)); }\n\nint finch_str_try_int(const FStr *s, int64_t *out, FStr *err) {\n if (parse_int(s, out)) return 1;\n char msg[200];\n number_error(msg, sizeof msg, s, \"int\");\n set_error(err, msg);\n *out = 0;\n return 0;\n}\n\nint finch_str_try_float(const FStr *s, double *out, FStr *err) {\n if (parse_float(s, out)) return 1;\n char msg[200];\n number_error(msg, sizeof msg, s, \"float\");\n set_error(err, msg);\n *out = 0;\n return 0;\n}\n\nstatic void file_error(FStr *err, const char *what, const FStr *path) {\n char msg[300];\n snprintf(msg, sizeof msg, \"can't %s the file \\\"%.*s\\\": %s\", what, (int)(path->len > 200 ? 200 : path->len),\n path->ptr ? path->ptr : \"\", strerror(errno));\n set_error(err, msg);\n}\n\n// when `fn main() -> !` fails\nvoid finch_report_error(const FStr *msg) {\n fflush(stdout);\n fprintf(stderr, \"error: %.*s\\n\", (int)msg->len, msg->ptr ? msg->ptr : \"\");\n finch_str_drop((FStr *)msg);\n}\n\nvoid finch_str_sub(FStr *out, const FStr *s, int64_t start, int64_t end, const char *file, int64_t line) {\n if (start < 0 || end > s->len || start > end) {\n char msg[160];\n snprintf(msg, sizeof msg, \"sub(%lld, %lld) is out of range (the length is %lld)\", (long long)start,\n (long long)end, (long long)s->len);\n finch_panic(file, line, msg);\n }\n str_heap(out, s->ptr + start, end - start);\n}\n\nint64_t finch_str_find(const FStr *s, const FStr *t) {\n if (t->len == 0) return 0;\n for (int64_t i = 0; i + t->len <= s->len; i++)\n if (s->ptr[i] == t->ptr[0] && memcmp(s->ptr + i, t->ptr, (size_t)t->len) == 0) return i;\n return -1;\n}\n\nint finch_str_starts(const FStr *s, const FStr *t) {\n return t->len <= s->len && memcmp(s->ptr, t->ptr, (size_t)t->len) == 0;\n}\n\nint finch_str_ends(const FStr *s, const FStr *t) {\n return t->len <= s->len && memcmp(s->ptr + s->len - t->len, t->ptr, (size_t)t->len) == 0;\n}\n\nvoid finch_str_trim(FStr *out, const FStr *s) {\n int64_t a = 0, b = s->len;\n while (a < b && isspace((unsigned char)s->ptr[a])) a++;\n while (b > a && isspace((unsigned char)s->ptr[b - 1])) b--;\n str_heap(out, s->ptr + a, b - a);\n}\n\nvoid finch_str_upper(FStr *out, const FStr *s) {\n str_heap(out, s->ptr, s->len);\n for (int64_t i = 0; i < out->len; i++) out->ptr[i] = (char)toupper((unsigned char)out->ptr[i]);\n}\n\nvoid finch_str_lower(FStr *out, const FStr *s) {\n str_heap(out, s->ptr, s->len);\n for (int64_t i = 0; i < out->len; i++) out->ptr[i] = (char)tolower((unsigned char)out->ptr[i]);\n}\n\nvoid finch_str_replace(FStr *out, const FStr *s, const FStr *from, const FStr *to) {\n if (from->len == 0) {\n str_heap(out, s->ptr, s->len);\n return;\n }\n int64_t count = 0;\n for (int64_t i = 0; i + from->len <= s->len;) {\n if (memcmp(s->ptr + i, from->ptr, (size_t)from->len) == 0) {\n count++;\n i += from->len;\n } else {\n i++;\n }\n }\n int64_t len = s->len + count * (to->len - from->len);\n char *m = finch_alloc(len + 1), *w = m;\n for (int64_t i = 0; i < s->len;) {\n if (i + from->len <= s->len && memcmp(s->ptr + i, from->ptr, (size_t)from->len) == 0) {\n memcpy(w, to->ptr, (size_t)to->len);\n w += to->len;\n i += from->len;\n } else {\n *w++ = s->ptr[i++];\n }\n }\n *w = 0;\n out->ptr = m;\n out->len = len;\n out->cap = len + 1;\n}\n\nstatic void arr_push_str(FArr *a, const char *p, int64_t len);\n\n// \"a,b,,c\".split(\",\") -> [\"a\", \"b\", \"\", \"c\"]; split(\"\") gives single characters.\nvoid finch_str_split(FArr *out, const FStr *s, const FStr *sep) {\n out->ptr = NULL;\n out->len = out->cap = 0;\n if (sep->len == 0) {\n for (int64_t i = 0; i < s->len; i++) arr_push_str(out, s->ptr + i, 1);\n return;\n }\n int64_t start = 0;\n for (int64_t i = 0; i + sep->len <= s->len;) {\n if (memcmp(s->ptr + i, sep->ptr, (size_t)sep->len) == 0) {\n arr_push_str(out, s->ptr + start, i - start);\n i += sep->len;\n start = i;\n } else {\n i++;\n }\n }\n arr_push_str(out, s->ptr + start, s->len - start);\n}\n\nvoid finch_str_join(FStr *out, const FArr *parts, const FStr *sep) {\n const FStr *p = parts->ptr;\n int64_t len = 0;\n for (int64_t i = 0; i < parts->len; i++) len += p[i].len + (i ? sep->len : 0);\n char *m = finch_alloc(len + 1), *w = m;\n for (int64_t i = 0; i < parts->len; i++) {\n if (i) {\n memcpy(w, sep->ptr, (size_t)sep->len);\n w += sep->len;\n }\n memcpy(w, p[i].ptr, (size_t)p[i].len);\n w += p[i].len;\n }\n *w = 0;\n out->ptr = m;\n out->len = len;\n out->cap = len + 1;\n}\n\nvoid finch_str_repeat(FStr *out, const FStr *s, int64_t n) {\n if (n < 0) n = 0;\n int64_t len = s->len * n;\n char *m = finch_alloc(len + 1);\n for (int64_t i = 0; i < n; i++) memcpy(m + i * s->len, s->ptr, (size_t)s->len);\n m[len] = 0;\n out->ptr = m;\n out->len = len;\n out->cap = len + 1;\n}\n\nvoid finch_str_from_bytes(FStr *out, const FArr *bytes) { str_heap(out, bytes->ptr, bytes->len); }\n\n// ---------- building text: print(x) and str(x) of arrays, maps and structs ----------\n// A buffer starts as an empty str and grows on the heap.\n\nvoid finch_buf_add(FStr *b, const char *p, int64_t n) {\n if (b->len + n + 1 > b->cap) {\n int64_t cap = b->cap > 0 ? b->cap * 2 : 32;\n while (cap < b->len + n + 1) cap *= 2;\n b->ptr = must(realloc(b->cap > 0 ? b->ptr : NULL, (size_t)cap));\n b->cap = cap;\n }\n if (n) memcpy(b->ptr + b->len, p, (size_t)n);\n b->len += n;\n b->ptr[b->len] = 0;\n}\n\nstatic void buf_fmt(FStr *b, const char *fmt, ...) __attribute__((format(printf, 2, 3)));\nstatic void buf_fmt(FStr *b, const char *fmt, ...) {\n char t[64];\n va_list ap;\n va_start(ap, fmt);\n int n = vsnprintf(t, sizeof t, fmt, ap);\n va_end(ap);\n finch_buf_add(b, t, n);\n}\n\nvoid finch_buf_int(FStr *b, int64_t v) { buf_fmt(b, \"%lld\", (long long)v); }\nvoid finch_buf_uint(FStr *b, uint64_t v) { buf_fmt(b, \"%llu\", (unsigned long long)v); }\nvoid finch_buf_float(FStr *b, double v) {\n char t[64];\n finch_buf_add(b, t, fmt_float(t, sizeof t, v, 0));\n}\nvoid finch_buf_f32(FStr *b, double v) {\n char t[64];\n finch_buf_add(b, t, fmt_float(t, sizeof t, v, 1));\n}\nvoid finch_buf_bool(FStr *b, int v) { finch_buf_add(b, v ? \"true\" : \"false\", v ? 4 : 5); }\nvoid finch_buf_ptr(FStr *b, const void *p) {\n if (p) buf_fmt(b, \"0x%llx\", (unsigned long long)(uintptr_t)p);\n else finch_buf_add(b, \"null\", 4);\n}\n\nvoid finch_buf_char(FStr *b, char c, int quoted) {\n if (quoted) finch_buf_add(b, \"'\", 1);\n finch_buf_add(b, &c, 1);\n if (quoted) finch_buf_add(b, \"'\", 1);\n}\n\nvoid finch_buf_str(FStr *b, const FStr *s, int quoted) {\n if (quoted) finch_buf_add(b, \"\\\"\", 1);\n finch_buf_add(b, s->ptr, s->len);\n if (quoted) finch_buf_add(b, \"\\\"\", 1);\n}\n\n// print the buffer and free it\nvoid finch_buf_print(FStr *b) {\n if (b->len) fwrite(b->ptr, 1, (size_t)b->len, stdout);\n if (b->cap > 0) free(b->ptr);\n}\n\n// ---------- arrays ----------\n\nvoid finch_arr_reserve(FArr *a, int64_t need, int64_t es) {\n if (need <= a->cap) return;\n int64_t cap = a->cap * 2;\n if (cap < 4) cap = 4;\n if (cap < need) cap = need;\n a->ptr = must(realloc(a->cap > 0 ? a->ptr : NULL, (size_t)(cap * es)));\n a->cap = cap;\n}\n\nvoid finch_arr_make(FArr *out, int64_t len, int64_t es) {\n if (len < 0) len = 0;\n out->ptr = len ? finch_zalloc(len * es) : NULL;\n out->len = len;\n out->cap = len;\n}\n\n// grow or shrink to `len`; new elements are zero\nvoid finch_arr_resize(FArr *a, int64_t len, int64_t es) {\n if (len < 0) len = 0;\n finch_arr_reserve(a, len, es);\n if (len > a->len) memset((char *)a->ptr + a->len * es, 0, (size_t)((len - a->len) * es));\n a->len = len;\n}\n\n// a byte-for-byte copy of the elements (the compiler deep-copies owning elements afterwards)\nvoid finch_arr_clone_raw(FArr *out, const FArr *a, int64_t es) {\n out->len = a->len;\n out->cap = a->len;\n out->ptr = a->len ? finch_alloc(a->len * es) : NULL;\n if (a->len) memcpy(out->ptr, a->ptr, (size_t)(a->len * es));\n}\n\nvoid finch_arr_free(FArr *a) {\n if (a->cap > 0) free(a->ptr);\n}\n\n// open a hole at i (0 <= i <= len), len grows by one\nvoid finch_arr_insert_gap(FArr *a, int64_t i, int64_t es, const char *file, int64_t line) {\n if (i < 0 || i > a->len) finch_panic_index(file, line, i, a->len);\n finch_arr_reserve(a, a->len + 1, es);\n char *p = a->ptr;\n memmove(p + (i + 1) * es, p + i * es, (size_t)((a->len - i) * es));\n a->len++;\n}\n\n// close the hole at i (the element was already moved out), len shrinks by one\nvoid finch_arr_remove_gap(FArr *a, int64_t i, int64_t es) {\n char *p = a->ptr;\n memmove(p + i * es, p + (i + 1) * es, (size_t)((a->len - i - 1) * es));\n a->len--;\n}\n\nstatic void arr_push_str(FArr *a, const char *p, int64_t len) {\n finch_arr_reserve(a, a->len + 1, (int64_t)sizeof(FStr));\n str_heap((FStr *)a->ptr + a->len, p, len);\n a->len++;\n}\n\nvoid finch_bytes_from_str(FArr *out, const FStr *s) {\n out->len = out->cap = s->len;\n out->ptr = s->len ? finch_alloc(s->len) : NULL;\n if (s->len) memcpy(out->ptr, s->ptr, (size_t)s->len);\n}\n\n// ---------- program arguments, input and files ----------\n\nvoid finch_args(FArr *out, int argc, char **argv) {\n out->len = out->cap = argc;\n out->ptr = finch_alloc((int64_t)sizeof(FStr) * (argc ? argc : 1));\n for (int i = 0; i < argc; i++) str_heap((FStr *)out->ptr + i, argv[i], (int64_t)strlen(argv[i]));\n}\n\n// after main: the arguments go\nvoid finch_args_free(FArr *a) {\n for (int64_t i = 0; i < a->len; i++) finch_str_drop((FStr *)a->ptr + i);\n finch_arr_free(a);\n}\n\n// input(\"Name? \") prints the prompt and reads one line, without the newline.\nvoid finch_input(FStr *out, const FStr *prompt) {\n if (prompt->len) fwrite(prompt->ptr, 1, (size_t)prompt->len, stdout);\n fflush(stdout);\n // read one line of any length (getline isn't available everywhere)\n size_t size = 128, n = 0;\n char *line = finch_alloc((int64_t)size);\n int c, got = 0;\n while ((c = fgetc(stdin)) != EOF) {\n got = 1;\n if (c == '\\n') break;\n if (n + 1 >= size) line = must(realloc(line, size *= 2));\n line[n++] = (char)c;\n }\n if (!got) {\n free(line);\n str_static(out, empty);\n return;\n }\n while (n > 0 && line[n - 1] == '\\r') n--;\n line[n] = 0;\n out->ptr = line;\n out->len = (int64_t)n;\n out->cap = (int64_t)size;\n}\n\nint finch_file_exists(const FStr *path) {\n FILE *f = fopen(path->ptr, \"rb\");\n if (f) fclose(f);\n return f != NULL;\n}\n\nint finch_try_read_file(FStr *out, const FStr *path, FStr *err) {\n FILE *f = fopen(path->ptr ? path->ptr : \"\", \"rb\");\n if (!f) {\n file_error(err, \"read\", path);\n return 0;\n }\n fseek(f, 0, SEEK_END);\n long size = ftell(f);\n fseek(f, 0, SEEK_SET);\n char *m = finch_alloc(size + 1);\n size_t got = fread(m, 1, (size_t)size, f);\n fclose(f);\n m[got] = 0;\n out->ptr = m;\n out->len = (int64_t)got;\n out->cap = size + 1;\n return 1;\n}\n\nvoid finch_read_file(FStr *out, const FStr *path, const char *file, int64_t line) {\n FILE *f = fopen(path->ptr ? path->ptr : \"\", \"rb\");\n if (!f) {\n char msg[300];\n snprintf(msg, sizeof msg, \"can't read the file \\\"%.*s\\\": %s\", (int)(path->len > 200 ? 200 : path->len),\n path->ptr, strerror(errno));\n finch_panic(file, line, msg);\n }\n fseek(f, 0, SEEK_END);\n long size = ftell(f);\n fseek(f, 0, SEEK_SET);\n char *m = finch_alloc(size + 1);\n size_t got = fread(m, 1, (size_t)size, f);\n fclose(f);\n m[got] = 0;\n out->ptr = m;\n out->len = (int64_t)got;\n out->cap = size + 1;\n}\n\n// shell(\"ls -l\"): run a command, give back its exit code\nint64_t finch_shell(const FStr *cmd) {\n fflush(stdout);\n int status = system(cmd->ptr ? cmd->ptr : \"\");\n#ifdef _WIN32\n return status; // Windows gives the exit code directly\n#else\n if (status == -1) return -1;\n if (WIFEXITED(status)) return WEXITSTATUS(status);\n return 128 + (WIFSIGNALED(status) ? WTERMSIG(status) : 0);\n#endif\n}\n\nint finch_delete_file(const FStr *path) { return remove(path->ptr ? path->ptr : \"\") == 0; }\n\nint finch_try_delete_file(const FStr *path, FStr *err) {\n if (remove(path->ptr ? path->ptr : \"\") == 0) return 1;\n file_error(err, \"delete\", path);\n return 0;\n}\n\n// stdin/stdout/stderr are macros on some systems (Windows), so Finch asks for them here\nvoid *finch_std_stream(int i) { return i == 0 ? (void *)stdin : i == 1 ? (void *)stdout : (void *)stderr; }\n\nint finch_write_file(const FStr *path, const FStr *text) {\n FILE *f = fopen(path->ptr ? path->ptr : \"\", \"wb\");\n if (!f) return 0;\n size_t put = text->len ? fwrite(text->ptr, 1, (size_t)text->len, f) : 0;\n return fclose(f) == 0 && put == (size_t)text->len;\n}\n\nint finch_try_write_file(const FStr *path, const FStr *text, FStr *err) {\n errno = 0;\n if (finch_write_file(path, text)) return 1;\n if (!errno) errno = EIO;\n file_error(err, \"write\", path);\n return 0;\n}\n\n// ---------- folders (std/fs.fch, through std/finch_fs.h) ----------\n\nint finch_fs_is_dir(const char *path) {\n#ifdef _WIN32\n struct _finddata_t d;\n intptr_t h = _findfirst(path, &d);\n if (h == -1) return 0;\n _findclose(h);\n return (d.attrib & _A_SUBDIR) != 0;\n#else\n struct stat st;\n return stat(path, &st) == 0 && S_ISDIR(st.st_mode);\n#endif\n}\n\nint finch_fs_exists(const char *path) {\n#ifdef _WIN32\n return _access(path, 0) == 0;\n#else\n struct stat st;\n return stat(path, &st) == 0;\n#endif\n}\n\nstatic int make_one_dir(const char *p) {\n#ifdef _WIN32\n return _mkdir(p) == 0 || errno == EEXIST;\n#else\n return mkdir(p, 0777) == 0 || errno == EEXIST;\n#endif\n}\n\n// mkdir -p: the parents too; 1 if the folder is there afterwards\nint finch_fs_mkdir(const char *path) {\n size_t n = strlen(path);\n char *p = finch_alloc((int64_t)n + 1);\n memcpy(p, path, n + 1);\n for (size_t i = 1; i < n; i++) {\n if (p[i] == '/' || p[i] == '\\\\') {\n if (p[i - 1] == ':') continue; /* C: is a drive */\n char c = p[i];\n p[i] = 0;\n make_one_dir(p);\n p[i] = c;\n }\n }\n int ok = make_one_dir(p) && finch_fs_is_dir(p);\n free(p);\n return ok;\n}\n\nint finch_fs_remove_dir(const char *path) {\n#ifdef _WIN32\n return _rmdir(path) == 0;\n#else\n return rmdir(path) == 0;\n#endif\n}\n\n// the names in a folder, one per call (\"\" at the end), without . and ..\ntypedef struct {\n#ifdef _WIN32\n intptr_t h;\n struct _finddata_t d;\n int first;\n#else\n DIR *d;\n#endif\n} FsList;\n\nvoid *finch_fs_list_open(const char *path) {\n FsList *l = finch_zalloc((int64_t)sizeof(FsList));\n#ifdef _WIN32\n size_t n = strlen(path);\n char *pat = finch_alloc((int64_t)n + 3);\n memcpy(pat, path, n);\n memcpy(pat + n, \"\\\\*\", 3);\n l->h = _findfirst(pat, &l->d);\n free(pat);\n if (l->h == -1) {\n free(l);\n return NULL;\n }\n l->first = 1;\n#else\n l->d = opendir(path);\n if (!l->d) {\n free(l);\n return NULL;\n }\n#endif\n return l;\n}\n\nconst char *finch_fs_list_next(void *handle) {\n FsList *l = handle;\n for (;;) {\n#ifdef _WIN32\n if (!l->first && _findnext(l->h, &l->d) != 0) return \"\";\n l->first = 0;\n const char *name = l->d.name;\n#else\n struct dirent *e = readdir(l->d);\n if (!e) return \"\";\n const char *name = e->d_name;\n#endif\n if (strcmp(name, \".\") != 0 && strcmp(name, \"..\") != 0) return name;\n }\n}\n\nvoid finch_fs_list_close(void *handle) {\n FsList *l = handle;\n#ifdef _WIN32\n _findclose(l->h);\n#else\n closedir(l->d);\n#endif\n free(l);\n}\n\n// where a program keeps its own files for this user (saves, settings); \"\" if unknown\n// Linux: $XDG_DATA_HOME or ~/.local/share macOS: ~/Library/Application Support Windows: %APPDATA%\nconst char *finch_fs_data_home(void) {\n static char buf[4096];\n const char *v;\n#if defined(_WIN32)\n v = getenv(\"APPDATA\");\n snprintf(buf, sizeof buf, \"%s\", v ? v : \"\");\n#elif defined(__APPLE__)\n v = getenv(\"HOME\");\n if (v) snprintf(buf, sizeof buf, \"%s/Library/Application Support\", v);\n else buf[0] = 0;\n#else\n v = getenv(\"XDG_DATA_HOME\");\n if (v && *v) snprintf(buf, sizeof buf, \"%s\", v);\n else if ((v = getenv(\"HOME\"))) snprintf(buf, sizeof buf, \"%s/.local/share\", v);\n else buf[0] = 0;\n#endif\n return buf;\n}\n\n// ---------- threads (std/thread.fch, through std/finch_thread.h) ----------\n\ntypedef struct {\n void (*fn)(void *);\n void *arg;\n} ThreadStart;\n\n#ifdef _WIN32\nstatic DWORD WINAPI thread_main(LPVOID p) {\n ThreadStart st = *(ThreadStart *)p;\n free(p);\n st.fn(st.arg);\n return 0;\n}\n#else\nstatic void *thread_main(void *p) {\n ThreadStart st = *(ThreadStart *)p;\n free(p);\n st.fn(st.arg);\n return NULL;\n}\n#endif\n\n// run fn(arg) on a new thread; NULL if it couldn't start\nvoid *finch_thread_spawn(void (*fn)(void *), void *arg) {\n ThreadStart *st = finch_alloc((int64_t)sizeof(ThreadStart));\n st->fn = fn;\n st->arg = arg;\n#ifdef _WIN32\n HANDLE h = CreateThread(NULL, 0, thread_main, st, 0, NULL);\n if (!h) {\n free(st);\n return NULL;\n }\n return h;\n#else\n pthread_t *t = finch_alloc((int64_t)sizeof(pthread_t));\n if (pthread_create(t, NULL, thread_main, st) != 0) {\n free(st);\n free(t);\n return NULL;\n }\n return t;\n#endif\n}\n\n// wait for the thread to end; 1 if that worked\nint finch_thread_join(void *t) {\n if (!t) return 0;\n#ifdef _WIN32\n int ok = WaitForSingleObject((HANDLE)t, INFINITE) == WAIT_OBJECT_0;\n CloseHandle((HANDLE)t);\n return ok;\n#else\n int ok = pthread_join(*(pthread_t *)t, NULL) == 0;\n free(t);\n return ok;\n#endif\n}\n\nvoid *finch_mutex_new(void) {\n#ifdef _WIN32\n CRITICAL_SECTION *m = finch_alloc((int64_t)sizeof(CRITICAL_SECTION));\n InitializeCriticalSection(m);\n#else\n pthread_mutex_t *m = finch_alloc((int64_t)sizeof(pthread_mutex_t));\n pthread_mutex_init(m, NULL);\n#endif\n return m;\n}\n\nvoid finch_mutex_lock(void *m) {\n#ifdef _WIN32\n EnterCriticalSection((CRITICAL_SECTION *)m);\n#else\n pthread_mutex_lock((pthread_mutex_t *)m);\n#endif\n}\n\nvoid finch_mutex_unlock(void *m) {\n#ifdef _WIN32\n LeaveCriticalSection((CRITICAL_SECTION *)m);\n#else\n pthread_mutex_unlock((pthread_mutex_t *)m);\n#endif\n}\n\nvoid finch_mutex_free(void *m) {\n if (!m) return;\n#ifdef _WIN32\n DeleteCriticalSection((CRITICAL_SECTION *)m);\n#else\n pthread_mutex_destroy((pthread_mutex_t *)m);\n#endif\n free(m);\n}\n\nint finch_cpu_count(void) {\n#ifdef _WIN32\n SYSTEM_INFO si;\n GetSystemInfo(&si);\n return (int)si.dwNumberOfProcessors;\n#else\n long n = sysconf(_SC_NPROCESSORS_ONLN);\n return n > 0 ? (int)n : 1;\n#endif\n}\n\n// ---------- maps ----------\n// map[K]V = { entries; len; used; cap; index; icap; adds }\n// entries: `used` slots of es bytes, each { uint64 hash; K key; V value }, in insertion order.\n// A removed entry keeps its slot (hash 0) until the next resize packs the slots.\n// index: icap slots (a power of two, at least 2 * cap): -1 empty, -2 removed, else an entry number.\n// adds: how many keys were ever added; a `for` over the map stops with an error if it changes.\n// The compiler stores, copies and drops keys and values; the runtime only finds and places entries.\n// Keys are compared byte for byte (numbers, chars, bools) or as text (kstr).\n\ntypedef struct {\n char *entries;\n int64_t len, used, cap;\n int64_t *index;\n int64_t icap;\n int64_t adds;\n} FMap;\n\nstatic uint64_t map_hash(const void *key, int64_t ks, int kstr) {\n const unsigned char *p;\n size_t n;\n if (kstr) {\n const FStr *s = key;\n p = (const unsigned char *)s->ptr;\n n = (size_t)s->len;\n } else {\n p = key;\n n = (size_t)ks;\n }\n uint64_t h = 1469598103934665603ULL; // FNV-1a, then mixed so the low bits spread well\n for (size_t i = 0; i < n; i++) {\n h ^= p[i];\n h *= 1099511628211ULL;\n }\n h ^= h >> 31;\n h *= 0xbf58476d1ce4e5b9ULL;\n h ^= h >> 29;\n return h | 1; // 0 marks a removed entry\n}\n\nstatic int map_key_eq(const char *a, const void *b, int64_t ks, int kstr) {\n if (!kstr) return memcmp(a, b, (size_t)ks) == 0;\n const FStr *x = (const FStr *)a, *y = b;\n return x->len == y->len && (x->len == 0 || memcmp(x->ptr, y->ptr, (size_t)x->len) == 0);\n}\n\nstatic void map_reindex(FMap *m, int64_t es) {\n if (m->len != m->used) { // pack the slots of removed entries away\n int64_t j = 0;\n for (int64_t i = 0; i < m->used; i++) {\n char *e = m->entries + i * es;\n if (*(uint64_t *)e == 0) continue;\n if (i != j) memmove(m->entries + j * es, e, (size_t)es);\n j++;\n }\n m->used = j;\n }\n int64_t icap = 16;\n while (icap < m->cap * 2) icap *= 2;\n free(m->index);\n m->index = finch_alloc(icap * (int64_t)sizeof(int64_t));\n memset(m->index, 0xff, (size_t)icap * sizeof(int64_t));\n m->icap = icap;\n for (int64_t i = 0; i < m->used; i++) {\n uint64_t h = *(uint64_t *)(m->entries + i * es);\n int64_t s = (int64_t)(h & (uint64_t)(icap - 1));\n while (m->index[s] != -1) s = (s + 1) & (icap - 1);\n m->index[s] = i;\n }\n}\n\n// the entry number of `key`, or -1\nint64_t finch_map_find(const FMap *m, const void *key, int64_t es, int64_t ks, int32_t kstr) {\n if (m->len == 0) return -1;\n uint64_t h = map_hash(key, ks, kstr);\n for (int64_t s = (int64_t)(h & (uint64_t)(m->icap - 1));; s = (s + 1) & (m->icap - 1)) {\n int64_t i = m->index[s];\n if (i == -1) return -1;\n if (i < 0) continue;\n char *e = m->entries + i * es;\n if (*(uint64_t *)e == h && map_key_eq(e + 8, key, ks, kstr)) return i;\n }\n}\n\n// The entry for `key`. A new one is zeroed except its hash (*added = 1); the compiler then stores the key.\nint64_t finch_map_slot(FMap *m, const void *key, int64_t es, int64_t ks, int32_t kstr, int32_t *added) {\n int64_t i = finch_map_find(m, key, es, ks, kstr);\n *added = i < 0;\n if (i >= 0) return i;\n if (m->used == m->cap) {\n if (m->len * 4 < m->cap * 3) { // many removed slots: pack them instead of growing\n map_reindex(m, es);\n } else {\n m->cap = m->cap ? m->cap * 2 : 8;\n m->entries = must(realloc(m->entries, (size_t)(m->cap * es)));\n map_reindex(m, es);\n }\n }\n uint64_t h = map_hash(key, ks, kstr);\n i = m->used++;\n char *e = m->entries + i * es;\n memset(e, 0, (size_t)es);\n *(uint64_t *)e = h;\n int64_t s = (int64_t)(h & (uint64_t)(m->icap - 1));\n while (m->index[s] >= 0) s = (s + 1) & (m->icap - 1);\n m->index[s] = i;\n m->len++;\n m->adds++;\n return i;\n}\n\n// Forget entry i (its key and value were already dropped by the compiler).\nvoid finch_map_remove_at(FMap *m, int64_t i, int64_t es) {\n char *e = m->entries + i * es;\n uint64_t h = *(uint64_t *)e;\n int64_t s = (int64_t)(h & (uint64_t)(m->icap - 1));\n while (m->index[s] != i) s = (s + 1) & (m->icap - 1);\n m->index[s] = -2;\n *(uint64_t *)e = 0;\n if (--m->len == 0) {\n m->used = 0;\n memset(m->index, 0xff, (size_t)m->icap * sizeof(int64_t));\n }\n}\n\nvoid finch_map_clear(FMap *m) {\n m->len = m->used = 0;\n if (m->index) memset(m->index, 0xff, (size_t)m->icap * sizeof(int64_t));\n}\n\nvoid finch_map_free(FMap *m) {\n free(m->entries);\n free(m->index);\n}\n\n// A copy of the slots and the index; the compiler then copies the keys and values that own memory.\nvoid finch_map_clone_raw(FMap *out, const FMap *m, int64_t es) {\n *out = *m;\n if (!m->cap) return;\n out->entries = finch_alloc(m->cap * es);\n memcpy(out->entries, m->entries, (size_t)(m->used * es));\n out->index = finch_alloc(m->icap * (int64_t)sizeof(int64_t));\n memcpy(out->index, m->index, (size_t)m->icap * sizeof(int64_t));\n}\n\n__attribute__((noreturn, cold)) void finch_map_changed(const char *file, int64_t line) {\n finch_panic(file, line, \"a key was added to the map while looping over it (collect the new keys, and add them after the loop)\");\n}\n\n__attribute__((noreturn, cold)) void finch_map_missing(const char *file, int64_t line, const FStr *key) {\n char msg[200];\n snprintf(msg, sizeof msg, \"the key %.*s is not in the map (check with .has(key), or use .get(key, default))\",\n (int)(key->len > 100 ? 100 : key->len), key->ptr ? key->ptr : \"\");\n finch_panic(file, line, msg);\n}\n\n// ---------- cpu_has(\"avx2\"): what the processor running the program can do ----------\n//\n// x86: CPUID, and for the AVX and AVX-512 families also XGETBV (the system must save those registers, or the\n// instructions fault). ARM64: the kernel's hwcaps on Linux and Android, sysctl on macOS. A name of the other\n// architecture is false.\n\n#if defined(__x86_64__) || defined(__i386__)\n#include \nstatic int x86_leaf(unsigned leaf, unsigned sub, unsigned r[4]) {\n unsigned max = __get_cpuid_max(0, 0);\n if (leaf > max) return 0;\n __cpuid_count(leaf, sub, r[0], r[1], r[2], r[3]);\n return 1;\n}\nstatic unsigned long long x86_xcr0(void) {\n unsigned lo, hi;\n __asm__ volatile(\"xgetbv\" : \"=a\"(lo), \"=d\"(hi) : \"c\"(0));\n return ((unsigned long long)hi << 32) | lo;\n}\nstatic int finch_cpu_has_c(const char *n) {\n unsigned l1[4] = {0}, l7[4] = {0}, l71[4] = {0};\n x86_leaf(1, 0, l1);\n x86_leaf(7, 0, l7);\n x86_leaf(7, 1, l71);\n int osx = (l1[2] >> 27) & 1; // OSXSAVE\n unsigned long long xcr0 = osx ? x86_xcr0() : 0;\n int ymm = osx && (xcr0 & 6) == 6; // the system saves xmm and ymm\n int zmm = ymm && (xcr0 & 0xe0) == 0xe0; // and the AVX-512 registers\n if (!strcmp(n, \"sse3\")) return l1[2] & 1;\n if (!strcmp(n, \"ssse3\")) return (l1[2] >> 9) & 1;\n if (!strcmp(n, \"sse4.1\")) return (l1[2] >> 19) & 1;\n if (!strcmp(n, \"sse4.2\")) return (l1[2] >> 20) & 1;\n if (!strcmp(n, \"popcnt\")) return (l1[2] >> 23) & 1;\n if (!strcmp(n, \"avx\")) return ymm && ((l1[2] >> 28) & 1);\n if (!strcmp(n, \"fma\")) return ymm && ((l1[2] >> 12) & 1);\n if (!strcmp(n, \"f16c\")) return ymm && ((l1[2] >> 29) & 1);\n if (!strcmp(n, \"avx2\")) return ymm && ((l7[1] >> 5) & 1);\n if (!strcmp(n, \"bmi1\")) return (l7[1] >> 3) & 1;\n if (!strcmp(n, \"bmi2\")) return (l7[1] >> 8) & 1;\n if (!strcmp(n, \"avxvnni\")) return ymm && ((l71[0] >> 4) & 1);\n if (!strcmp(n, \"avx512f\")) return zmm && ((l7[1] >> 16) & 1);\n if (!strcmp(n, \"avx512dq\")) return zmm && ((l7[1] >> 17) & 1);\n if (!strcmp(n, \"avx512bw\")) return zmm && ((l7[1] >> 30) & 1);\n if (!strcmp(n, \"avx512vl\")) return zmm && ((l7[1] >> 31) & 1);\n if (!strcmp(n, \"avx512vnni\")) return zmm && ((l7[2] >> 11) & 1);\n if (!strcmp(n, \"avx512fp16\")) return zmm && ((l7[3] >> 23) & 1);\n if (!strcmp(n, \"avx512bf16\")) return zmm && ((l71[0] >> 5) & 1);\n return 0;\n}\n#elif defined(__aarch64__) && defined(__APPLE__)\n#include \nstatic int sysctl_flag(const char *name) {\n int v = 0;\n size_t len = sizeof v;\n if (sysctlbyname(name, &v, &len, NULL, 0) != 0) return 0;\n return v != 0;\n}\nstatic int finch_cpu_has_c(const char *n) {\n if (!strcmp(n, \"neon\")) return 1;\n if (!strcmp(n, \"fp16\")) return sysctl_flag(\"hw.optional.arm.FEAT_FP16\");\n if (!strcmp(n, \"dotprod\")) return sysctl_flag(\"hw.optional.arm.FEAT_DotProd\");\n if (!strcmp(n, \"i8mm\")) return sysctl_flag(\"hw.optional.arm.FEAT_I8MM\");\n if (!strcmp(n, \"bf16\")) return sysctl_flag(\"hw.optional.arm.FEAT_BF16\");\n return 0; // no SVE on Apple chips\n}\n#elif defined(__aarch64__) && defined(__linux__)\n#include \nstatic int finch_cpu_has_c(const char *n) {\n unsigned long h = getauxval(AT_HWCAP), h2 = getauxval(AT_HWCAP2);\n if (!strcmp(n, \"neon\")) return 1;\n if (!strcmp(n, \"fp16\")) return (h >> 10) & 1; // HWCAP_ASIMDHP\n if (!strcmp(n, \"dotprod\")) return (h >> 20) & 1; // HWCAP_ASIMDDP\n if (!strcmp(n, \"sve\")) return (h >> 22) & 1; // HWCAP_SVE\n if (!strcmp(n, \"sve2\")) return (h2 >> 1) & 1; // HWCAP2_SVE2\n if (!strcmp(n, \"i8mm\")) return (h2 >> 13) & 1; // HWCAP2_I8MM\n if (!strcmp(n, \"bf16\")) return (h2 >> 14) & 1; // HWCAP2_BF16\n return 0;\n}\n#else\nstatic int finch_cpu_has_c(const char *n) {\n (void)n;\n return 0;\n}\n#endif\n\nint finch_cpu_has(const FStr *name) {\n char n[32];\n int64_t len = name->len < 31 ? name->len : 31;\n memcpy(n, name->ptr ? name->ptr : \"\", (size_t)len);\n n[len] = 0;\n return finch_cpu_has_c(n) ? 1 : 0;\n}\n") list.push("// Files and folders: bytes, folders, listing, and a place for a program's own files.\n//\n// import fs\n// data := try fs.read_bytes(\"atlas.png\") the whole file as []u8 (any bytes, NUL included)\n// try fs.write_bytes(\"out.bin\", data)\n// try fs.mkdir(\"saves/slot1\") with the folders above it\n// names := try fs.list(\"saves\") the names in a folder, sorted\n// fs.exists(p) fs.is_dir(p) try fs.remove_dir(p)\n// dir := try fs.data_dir(\"MyGame\") a folder for this user's saves, made if needed\nimport \"finch_fs.h\"\n\nfn read_bytes(str path) -> []u8! {\n text := try read_file(path)\n return text.bytes()\n}\n\nfn write_bytes(str path, []u8 data) -> ! {\n try write_file(path, str(data))\n}\n\nfn exists(str path) -> bool {\n return finch_fs_exists(path) != 0\n}\n\nfn is_dir(str path) -> bool {\n return finch_fs_is_dir(path) != 0\n}\n\nfn mkdir(str path) -> ! {\n if finch_fs_mkdir(path) == 0 {\n return error(\"can't make the folder \\\"\" + path + \"\\\"\")\n }\n}\n\n// only an empty folder\nfn remove_dir(str path) -> ! {\n if finch_fs_remove_dir(path) == 0 {\n return error(\"can't remove the folder \\\"\" + path + \"\\\" (is it empty?)\")\n }\n}\n\nfn list(str path) -> []str! {\n l := finch_fs_list_open(path)\n if l == null {\n return error(\"can't list the folder \\\"\" + path + \"\\\"\")\n }\n []str names\n while true {\n name := finch_fs_list_next(l)\n if name.len == 0 {\n break\n }\n names.push(name)\n }\n finch_fs_list_close(l)\n names.sort()\n return names\n}\n\n// The folder where a program keeps this user's files (saves, settings), made if it isn't there:\n// ~/.local/share/ on Linux, ~/Library/Application Support/ on macOS, %APPDATA%\\ on Windows.\nfn data_dir(str app) -> str! {\n home := finch_fs_data_home()\n if home.len == 0 {\n return error(\"this system doesn't say where programs keep their files\")\n }\n dir := home + \"/\" + app\n try mkdir(dir)\n return dir\n}\n") list.push("// JSON: read it, build it, write it.\n//\n// import json\n//\n// doc := try json.parse(\"{\\\"name\\\": \\\"Finch\\\", \\\"tags\\\": [\\\"fast\\\", \\\"small\\\"], \\\"stars\\\": 5}\")\n// print(doc.get(\"name\").str_or(\"\")) Finch\n// print(doc.get(\"tags\").at(1).str_or(\"\")) small\n// print(doc.get(\"stars\").int_or(0)) 5\n// print(doc.get(\"missing\").is_null()) true (a missing key is null, so chains never fail)\n//\n// out := json.new_object()\n// out.set(\"ok\", json.new_bool(true))\n// list := json.new_list()\n// list.push(json.new_number(1.5))\n// out.set(\"values\", list)\n// print(json.stringify(out)) {\"ok\":true,\"values\":[1.5]}\n// print(json.pretty(out)) the same, over several lines\n\nenum Kind { Null, Bool, Number, Text, List, Object }\n\n// One JSON value. `kind` says which of the other fields it uses: flag for Bool, num for Number,\n// text for Text, items for List, and items with keys (the same length, in the same order) for Object.\nstruct Json {\n Kind kind\n bool flag\n float num\n str text\n []Json items\n []str keys\n}\n\nfn new_null() -> Json {\n return Json(kind: Kind.Null)\n}\n\nfn new_bool(bool b) -> Json {\n return Json(kind: Kind.Bool, flag: b)\n}\n\nfn new_number(float n) -> Json {\n return Json(kind: Kind.Number, num: n)\n}\n\nfn new_text(str s) -> Json {\n return Json(kind: Kind.Text, text: s)\n}\n\nfn new_list() -> Json {\n return Json(kind: Kind.List)\n}\n\nfn new_object() -> Json {\n return Json(kind: Kind.Object)\n}\n\n// ---------- looking inside ----------\n\nfn Json.is_null() -> bool {\n return self.kind == Kind.Null\n}\n\n// the number of elements of a list, of keys of an object, of bytes of a text; 0 for the rest\nfn Json.len() -> int {\n return match self.kind {\n List, Object -> self.items.len\n Text -> self.text.len\n else -> 0\n }\n}\n\nfn Json.has(str key) -> bool {\n return self.kind == Kind.Object && self.keys.contains(key)\n}\n\n// the value of a key of an object; null if there is none (or this is not an object)\nfn Json.get(str key) -> Json {\n if self.kind != Kind.Object {\n return new_null()\n }\n i := self.keys.find(key)\n if i < 0 {\n return new_null()\n }\n return self.items[i]\n}\n\n// the element of a list; null if there is none\nfn Json.at(int i) -> Json {\n if self.kind != Kind.List || i < 0 || i >= self.items.len {\n return new_null()\n }\n return self.items[i]\n}\n\n// the value if it is of that kind, else what was given\nfn Json.str_or(str fallback) -> str {\n if self.kind == Kind.Text {\n return self.text\n }\n return fallback\n}\n\nfn Json.float_or(float fallback) -> float {\n if self.kind == Kind.Number {\n return self.num\n }\n return fallback\n}\n\nfn Json.int_or(int fallback) -> int {\n if self.kind == Kind.Number {\n return int(self.num)\n }\n return fallback\n}\n\nfn Json.bool_or(bool fallback) -> bool {\n if self.kind == Kind.Bool {\n return self.flag\n }\n return fallback\n}\n\n// the keys of an object, in the order they were written\nfn Json.names() -> []str {\n return self.keys\n}\n\n// ---------- building ----------\n\n// add to the end of a list\nfn Json.push(Json v) {\n self.items.push(v)\n}\n\n// give a key a value in an object; a key that is already there is replaced\nfn Json.set(str key, Json v) {\n i := self.keys.find(key)\n if i >= 0 {\n self.items[i] = v\n return\n }\n self.keys.push(key)\n self.items.push(v)\n}\n\n// ---------- writing ----------\n\nfn quote(str s) -> str {\n []str parts = [\"\\\"\"]\n for c in s {\n n := int(c)\n if c == '\"' {\n parts.push(\"\\\\\\\"\")\n } else if c == '\\\\' {\n parts.push(\"\\\\\\\\\")\n } else if c == '\\n' {\n parts.push(\"\\\\n\")\n } else if c == '\\r' {\n parts.push(\"\\\\r\")\n } else if c == '\\t' {\n parts.push(\"\\\\t\")\n } else if n < 32 {\n parts.push(\"\\\\u00\" + hexdigit(n / 16) + hexdigit(n % 16))\n } else {\n parts.push(str(c))\n }\n }\n parts.push(\"\\\"\")\n return parts.join(\"\")\n}\n\nfn hexdigit(int n) -> str {\n digits := \"0123456789abcdef\"\n return str(digits[n])\n}\n\nfn number_text(float n) -> str {\n if n == float(int(n)) && n > -1000000000000000.0 && n < 1000000000000000.0 {\n return str(int(n))\n }\n return str(n)\n}\n\nstruct Writer {\n []str out\n}\n\nfn write(Json j, str indent, str pad, ptr[Writer] w) {\n nl := \"\"\n inner := \"\"\n sep := \",\"\n colon := \":\"\n if indent.len > 0 {\n nl = \"\\n\"\n inner = pad + indent\n colon = \": \"\n }\n match j.kind {\n Null -> w.out.push(\"null\")\n Bool -> {\n if j.flag {\n w.out.push(\"true\")\n } else {\n w.out.push(\"false\")\n }\n }\n Number -> w.out.push(number_text(j.num))\n Text -> w.out.push(quote(j.text))\n List -> {\n if j.items.len == 0 {\n w.out.push(\"[]\")\n return\n }\n w.out.push(\"[\" + nl)\n for i, item in j.items {\n if i > 0 {\n w.out.push(sep + nl)\n }\n w.out.push(inner)\n write(item, indent, inner, w)\n }\n w.out.push(nl + pad + \"]\")\n }\n Object -> {\n if j.items.len == 0 {\n w.out.push(\"{}\")\n return\n }\n w.out.push(\"{\" + nl)\n for i, item in j.items {\n if i > 0 {\n w.out.push(sep + nl)\n }\n w.out.push(inner + quote(j.keys[i]) + colon)\n write(item, indent, inner, w)\n }\n w.out.push(nl + pad + \"}\")\n }\n }\n}\n\n// the value as text on one line\nfn stringify(Json j) -> str {\n w := Writer()\n write(j, \"\", \"\", addr(w))\n return w.out.join(\"\")\n}\n\n// the value as text over several lines, indented by two spaces\nfn pretty(Json j) -> str {\n w := Writer()\n write(j, \" \", \"\", addr(w))\n return w.out.join(\"\")\n}\n\n// ---------- reading ----------\n\nstruct Reader {\n str src\n int i\n}\n\n// the line and column of a position, for a message\nfn where(ptr[Reader] r) -> str {\n line := 1\n col := 1\n for k in 0..r.i {\n if k < r.src.len && r.src[k] == '\\n' {\n line += 1\n col = 1\n } else {\n col += 1\n }\n }\n return \"line \" + str(line) + \", column \" + str(col)\n}\n\nfn peek(ptr[Reader] r) -> char {\n if r.i < r.src.len {\n return r.src[r.i]\n }\n return char(0)\n}\n\nfn skip_space(ptr[Reader] r) {\n while r.i < r.src.len && (r.src[r.i] == ' ' || r.src[r.i] == '\\n' || r.src[r.i] == '\\t' || r.src[r.i] == '\\r') {\n r.i += 1\n }\n}\n\nfn unexpected(ptr[Reader] r, str wanted) -> str {\n if r.i >= r.src.len {\n return \"json: the text ends where \" + wanted + \" should be (\" + where(r) + \")\"\n }\n return \"json: found '\" + str(r.src[r.i]) + \"' where \" + wanted + \" should be (\" + where(r) + \")\"\n}\n\n// does `word` come next?\nfn comes(ptr[Reader] r, str word) -> bool {\n return r.src.len - r.i >= word.len && r.src.sub(r.i, r.i + word.len) == word\n}\n\nfn expect_word(ptr[Reader] r, str word) -> ! {\n if !comes(r, word) {\n return error(unexpected(r, \"'\" + word + \"'\"))\n }\n r.i += word.len\n}\n\nfn hex_value(char c) -> int {\n if c >= '0' && c <= '9' {\n return int(c) - int('0')\n }\n if c >= 'a' && c <= 'f' {\n return int(c) - int('a') + 10\n }\n if c >= 'A' && c <= 'F' {\n return int(c) - int('A') + 10\n }\n return -1\n}\n\n// a code point as UTF-8\nfn utf8(int cp) -> str {\n []u8 b\n if cp < 128 {\n b.push(u8(cp))\n } else if cp < 2048 {\n b.push(u8(192 + cp / 64))\n b.push(u8(128 + cp % 64))\n } else if cp < 65536 {\n b.push(u8(224 + cp / 4096))\n b.push(u8(128 + (cp / 64) % 64))\n b.push(u8(128 + cp % 64))\n } else {\n b.push(u8(240 + cp / 262144))\n b.push(u8(128 + (cp / 4096) % 64))\n b.push(u8(128 + (cp / 64) % 64))\n b.push(u8(128 + cp % 64))\n }\n return str(b)\n}\n\nfn read_hex4(ptr[Reader] r) -> int! {\n n := 0\n for k in 0..4 {\n if r.i >= r.src.len || hex_value(r.src[r.i]) < 0 {\n return error(unexpected(r, \"a hexadecimal digit\"))\n }\n n = n * 16 + hex_value(r.src[r.i])\n r.i += 1\n }\n return n\n}\n\nfn read_string(ptr[Reader] r) -> str! {\n r.i += 1 // the opening quote\n []str parts\n start := r.i\n while true {\n if r.i >= r.src.len {\n return error(\"json: a text is never closed (\" + where(r) + \")\")\n }\n c := r.src[r.i]\n if c == '\"' {\n parts.push(r.src.sub(start, r.i))\n r.i += 1\n return parts.join(\"\")\n }\n if int(c) < 32 {\n return error(\"json: a line break or control character inside a text (\" + where(r) + \")\")\n }\n if c != '\\\\' {\n r.i += 1\n continue\n }\n parts.push(r.src.sub(start, r.i))\n r.i += 1\n e := peek(r)\n r.i += 1\n if e == 'n' {\n parts.push(\"\\n\")\n } else if e == 't' {\n parts.push(\"\\t\")\n } else if e == 'r' {\n parts.push(\"\\r\")\n } else if e == 'b' {\n parts.push(utf8(8))\n } else if e == 'f' {\n parts.push(utf8(12))\n } else if e == '\"' || e == '\\\\' || e == '/' {\n parts.push(str(e))\n } else if e == 'u' {\n cp := try read_hex4(r)\n // a pair of 😀 stands for one character above U+FFFF\n if cp >= 55296 && cp < 56320 && comes(r, \"\\\\u\") {\n r.i += 2\n low := try read_hex4(r)\n cp = 65536 + (cp - 55296) * 1024 + (low - 56320)\n }\n parts.push(utf8(cp))\n } else {\n return error(\"json: unknown escape \\\\\" + str(e) + \" in a text (\" + where(r) + \")\")\n }\n start = r.i\n }\n}\n\nfn read_number(ptr[Reader] r) -> Json! {\n start := r.i\n if peek(r) == '-' {\n r.i += 1\n }\n while r.i < r.src.len && ((r.src[r.i] >= '0' && r.src[r.i] <= '9') || r.src[r.i] == '.' || r.src[r.i] == 'e' || r.src[r.i] == 'E' || r.src[r.i] == '+' || r.src[r.i] == '-') {\n r.i += 1\n }\n n := float(r.src.sub(start, r.i)) or {\n r.i = start\n return error(unexpected(r, \"a number\"))\n }\n return new_number(n)\n}\n\nfn read_value(ptr[Reader] r) -> Json! {\n skip_space(r)\n c := peek(r)\n if r.i >= r.src.len {\n return error(unexpected(r, \"a value\"))\n }\n if c == '{' {\n obj := new_object()\n r.i += 1\n skip_space(r)\n if peek(r) == '}' {\n r.i += 1\n return obj\n }\n while true {\n skip_space(r)\n if peek(r) != '\"' {\n return error(unexpected(r, \"a key in quotes\"))\n }\n key := try read_string(r)\n skip_space(r)\n if peek(r) != ':' {\n return error(unexpected(r, \"':'\"))\n }\n r.i += 1\n v := try read_value(r)\n obj.set(key, v)\n skip_space(r)\n if peek(r) == ',' {\n r.i += 1\n continue\n }\n if peek(r) == '}' {\n r.i += 1\n return obj\n }\n return error(unexpected(r, \"',' or '}'\"))\n }\n }\n if c == '[' {\n list := new_list()\n r.i += 1\n skip_space(r)\n if peek(r) == ']' {\n r.i += 1\n return list\n }\n while true {\n v := try read_value(r)\n list.push(v)\n skip_space(r)\n if peek(r) == ',' {\n r.i += 1\n continue\n }\n if peek(r) == ']' {\n r.i += 1\n return list\n }\n return error(unexpected(r, \"',' or ']'\"))\n }\n }\n if c == '\"' {\n s := try read_string(r)\n return new_text(s)\n }\n if c == 't' {\n try expect_word(r, \"true\")\n return new_bool(true)\n }\n if c == 'f' {\n try expect_word(r, \"false\")\n return new_bool(false)\n }\n if c == 'n' {\n try expect_word(r, \"null\")\n return new_null()\n }\n if c == '-' || (c >= '0' && c <= '9') {\n return try read_number(r)\n }\n return error(unexpected(r, \"a value\"))\n}\n\n// the JSON in `text`; an error with the line and column where it went wrong\nfn parse(str text) -> Json! {\n r := Reader(src: text)\n v := try read_value(addr(r))\n skip_space(addr(r))\n if r.i < r.src.len {\n return error(unexpected(addr(r), \"the end of the text\"))\n }\n return v\n}\n") list.push("// Paths, as text. Elements are separated by \"/\".\n//\n// import path\n// path.join(\"a\", \"b.txt\") a/b.txt\n// path.base(\"/tmp/x/a.tar.gz\") a.tar.gz path.stem(...) a.tar path.ext(...) .gz\n// path.dir(\"/tmp/x/a.txt\") /tmp/x\n// path.clean(\"a//b/../c/./d\") a/c/d\n\n// a and b with one separator between; an absolute b replaces a\nfn join(str a, str b) -> str {\n if a.len == 0 {\n return b\n }\n if b.len == 0 {\n return a\n }\n if b[0] == '/' {\n return b\n }\n if a[a.len - 1] == '/' {\n return a + b\n }\n return a + \"/\" + b\n}\n\nfn is_abs(str p) -> bool {\n return p.len > 0 && p[0] == '/'\n}\n\n// the last element: \"a/b.txt\" -> \"b.txt\", \"a/b/\" -> \"b\", \"\" -> \".\", \"/\" -> \"/\"\nfn base(str p) -> str {\n if p.len == 0 {\n return \".\"\n }\n end := p.len\n while end > 1 && p[end - 1] == '/' {\n end -= 1\n }\n if end == 1 && p[0] == '/' {\n return \"/\"\n }\n start := end\n while start > 0 && p[start - 1] != '/' {\n start -= 1\n }\n return p.sub(start, end)\n}\n\n// everything before the last element: \"a/b.txt\" -> \"a\", \"b.txt\" -> \".\", \"/b\" -> \"/\"\nfn dir(str p) -> str {\n end := p.len\n while end > 1 && p[end - 1] == '/' {\n end -= 1\n }\n i := end\n while i > 0 && p[i - 1] != '/' {\n i -= 1\n }\n if i == 0 {\n return \".\"\n }\n while i > 1 && p[i - 1] == '/' {\n i -= 1\n }\n return p.sub(0, i)\n}\n\n// the part of the last element from its last dot: \"a.tar.gz\" -> \".gz\"; \"\" when there is none\nfn ext(str p) -> str {\n b := base(p)\n i := b.len\n while i > 0 && b[i - 1] != '.' {\n i -= 1\n }\n if i <= 1 { // no dot, or a name that only starts with one (.bashrc)\n return \"\"\n }\n return b.sub(i - 1, b.len)\n}\n\n// the last element without its extension\nfn stem(str p) -> str {\n b := base(p)\n e := ext(p)\n return b.sub(0, b.len - e.len)\n}\n\n// the elements of a path: \"/a/b/c\" -> [\"a\", \"b\", \"c\"]\nfn parts(str p) -> []str {\n []str out\n for piece in p.split(\"/\") {\n if piece.len > 0 {\n out.push(piece)\n }\n }\n return out\n}\n\n// the shortest path with the same meaning: no \"//\", no \".\", and \"..\" resolved where it can be\nfn clean(str p) -> str {\n if p.len == 0 {\n return \".\"\n }\n abs := is_abs(p)\n []str out\n for piece in parts(p) {\n if piece == \".\" {\n continue\n }\n if piece == \"..\" {\n if out.len > 0 && out[out.len - 1] != \"..\" {\n out.pop()\n } else if !abs {\n out.push(\"..\")\n }\n continue\n }\n out.push(piece)\n }\n joined := out.join(\"/\")\n if abs {\n return \"/\" + joined\n }\n if joined.len == 0 {\n return \".\"\n }\n return joined\n}\n") diff --git a/src/gen.fch b/src/gen.fch index d3b3eb3..ad6ebce 100644 --- a/src/gen.fch +++ b/src/gen.fch @@ -33,6 +33,7 @@ struct Options { bool trace // runtime errors show the chain of calls (off for --shared and --no-trace) bool test // finch test: main runs the test "..." blocks instead []str testModules // the modules whose tests run ("" is the main file); not the standard library's + []str features // what the processor it is built for has (from --cpu, asked of clang): avx2, i8mm ... } // A generic struct or function, kept as written until something asks for a version of it. @@ -218,6 +219,8 @@ struct Gen { bool deferOn // running a defer: variables declared after it are out of sight int deferLimit int inDefer + []str features // the processor features of the whole program (--cpu) + []str curFeatures // and of the function being built: those and its @cpu(...) } // ---------- basics ---------- @@ -1460,6 +1463,9 @@ fn coerce(ptr[Gen] g, Val v, ast.Type want, ast.Node at, str what) -> Val { if want.kind == "str" && (isNumber(v.ty) || v.ty.kind == "bool" || v.ty.kind == "char") { msg = msg + " (turn it into text with str(...))" } + if want.kind == "bool" && v.ty.kind == "vec" && v.ty.elem[0].kind == "bool" { + msg = msg + " (a comparison of vectors has an answer for every lane: use .all() or .any())" + } if want.kind == "vec" && isNumber(v.ty) { msg = msg + " (" + ast.show(want) + "(x) puts x in every lane)" } @@ -2357,6 +2363,13 @@ fn define(ptr[Gen] g, int si) { g.instNote = " (while building " + g.cur.shown + ")" } f := g.progs[g.cur.prog].fns[g.cur.index] + ownCpu := cpuOf(g, f) + g.curFeatures = g.features + for x in ownCpu { + if !g.curFeatures.contains(x) { + g.curFeatures.push(x) + } + } []str empty g.body = empty g.allocas = empty @@ -2441,8 +2454,20 @@ fn define(ptr[Gen] g, int si) { } } attach := "" + if ownCpu.len > 0 { + []str plus + wide := false + for x in ownCpu { + plus.push("+" + llvmFeature(x)) + wide = wide || x.starts_with("avx512") + } + if wide { + plus.push("+evex512") // LLVM 18 and later: the 512-bit registers themselves (clang adds it for C too) + } + attach = " \"target-features\"=\"" + plus.join(",") + "\"" + } if dbgsp >= 0 { - attach = " !dbg !" + str(dbgsp) + attach = attach + " !dbg !" + str(dbgsp) } g.dbgScope = -1 g.out.push("define internal " + retTy(g, g.cur) + " " + g.cur.ll + "(" + params.join(", ") + ")" + attach + " {\nentry:\n" + @@ -3680,7 +3705,7 @@ fn expr(ptr[Gen] g, ast.Node e) -> Val { fn unary(ptr[Gen] g, ast.Node e) -> Val { v := expr(g, e.kids[0]) - if v.ty.kind == "vec" && e.name != "!" { + if v.ty.kind == "vec" && (e.name != "!" || v.ty.elem[0].kind == "bool") { return vecUnary(g, e, v) } if e.name == "!" { @@ -4453,6 +4478,18 @@ fn indexOf(ptr[Gen] g, ast.Node e, Ref obj, bool write) -> Ref { return mapIndex(g, e, obj, t, write) } i := coerce(g, expr(g, e.kids[1]), T("int"), e.kids[1], "an index").v + if t.kind == "vec" && t.elem[0].kind == "bool" { + // a mask keeps a lane in a bit, so a lane is read from the whole value + if write { + fail(g, e, "a lane of a mask can't be set by itself; make a new mask (with a comparison or select)") + } + boundsCheck(g, i, str(t.count), e) + mv := obj.val.v + if obj.isPlace { + mv = load(g, llty(g, t), obj.pl.addr) + } + return Ref(isPlace: false, val: Val(v: op(g, "extractelement " + llty(g, t) + " " + mv + ", i64 " + i), ty: T("bool"))) + } if t.kind == "vec" { // a lane: v[2], v[i] = x boundsCheck(g, i, str(t.count), e) @@ -5572,6 +5609,21 @@ fn call(ptr[Gen] g, ast.Node c) -> Val { if ei >= 0 { return enumFrom(g, c, ei) } + if n == "select" && lookup(g, n) < 0 && findFn(g, g.module, n) < 0 && !g.ci.fns.has(n) { + return selectCall(g, c) + } + if n == "cpu_has" && lookup(g, n) < 0 && findFn(g, g.module, n) < 0 { + return cpuHasCall(g, c) + } + if n == "concat" && lookup(g, n) < 0 && findFn(g, g.module, n) < 0 && !g.ci.fns.has(n) { + return concatCall(g, c) + } + if ["fma", "muladd"].contains(n) { + return fmaCall(g, c) // Finch's, also for f32 and vectors (C's fma is the same for one double) + } + if ["f32_bits", "f32_from_bits", "f64_bits", "f64_from_bits", "f16_to_f32", "f32_to_f16", "bf16_to_f32", "f32_to_bf16"].contains(n) && !g.ci.fns.has(n) { + return bitsCall(g, c) + } if g.ci.fns.has(n) && !["min", "max", "abs", "clamp"].contains(n) { return callC(g, c, g.ci.fns[n]) // (C's abs only takes an int; Finch's takes any number) } @@ -5810,6 +5862,7 @@ fn callFn(ptr[Gen] g, int fi, ast.Node c, int first, str shown, str selfAddr, st // The call itself, once the arguments are values ("i64 5"); the receiver's address, if any, is already first. fn emitCall(ptr[Gen] g, FnSig sig, ast.Node c, []str args, str shown) -> Val { + checkCpuCall(g, sig, c, shown) if sig.fallible && g.ftarget != nodeKey(c) { fail(g, c, "'" + shown + "' can fail, so say what happens then: " + shown + "(...) or , " + shown + "(...) or { ... }, or try " + shown + "(...)") } @@ -6498,6 +6551,7 @@ fn runtimeDecls() -> str { "declare i32 @finch_test_summary()", "declare void @finch_input(ptr, ptr)", "declare i32 @finch_file_exists(ptr)", + "declare i32 @finch_cpu_has(ptr)", "declare i32 @finch_str_try_int(ptr, ptr, ptr)", "declare i32 @finch_str_try_float(ptr, ptr, ptr)", "declare i32 @finch_try_read_file(ptr, ptr, ptr)", @@ -6546,7 +6600,7 @@ fn assemble(ptr[Gen] g) -> str { fn generate([]ast.Program progs, []str files, cimport.CImports ci, Options opt) -> str { shared := opt.shared - gen := Gen(progs: progs, files: files, ci: ci, shared: opt.shared, debug: opt.debug, index: opt.index, cwd: opt.cwd, tg: opt.target, dbgScope: -1) + gen := Gen(progs: progs, files: files, ci: ci, shared: opt.shared, debug: opt.debug, index: opt.index, cwd: opt.cwd, tg: opt.target, dbgScope: -1, features: opt.features) g := addr(gen) g.trace = opt.trace && !opt.shared if g.debug { @@ -6647,8 +6701,14 @@ fn laneBytes(ast.Type vt) -> str { return str(bitsOf(vt.elem[0]) / 8) } -// v8f32, v32i8: how LLVM's vector intrinsics are named for this type +// v8f32, v32i8 (or f32, i64 for a plain number): how LLVM's intrinsics are named for this type fn vecSuffix(ast.Type vt) -> str { + if vt.kind != "vec" { + if isFloat(vt) { + return "f" + str(bitsOf(vt)) + } + return "i" + str(bitsOf(vt)) + } if isFloat(vt.elem[0]) { return "v" + str(vt.count) + "f" + str(bitsOf(vt.elem[0])) } @@ -6731,9 +6791,15 @@ fn vecMake(ptr[Gen] g, ast.Node c, ast.Type vt) -> Val { ft = first.pl.ty } if ft.kind == "array" || ft.kind == "ptr" { - if c.kids.len != 2 { + if lane.kind == "bool" { + fail(g, c, "a mask isn't loaded from a list; compare instead, like f32x8(list, i) > 0.0") + } + if c.kids.len != 2 && c.kids.len != 3 { fail(g, c, shown + "(list, i): which index to start at? (" + shown + "(list, 0) for the first " + str(n) + ")") } + if c.kids.len == 3 && ft.kind == "ptr" { + fail(g, c, shown + "(list, i, fill) needs a list: a ptr doesn't know where its numbers end") + } if ft.elem.len == 0 || !ast.same(ft.elem[0], lane) { fail(g, c.kids[0], shown + "(list, i) loads " + ast.show(lane) + " numbers, but this is " + ast.show(ft)) } @@ -6742,6 +6808,10 @@ fn vecMake(ptr[Gen] g, ast.Node c, ast.Type vt) -> Val { hold(g, first.val) } i := coerce(g, expr(g, c.kids[1]), T("int"), c.kids[1], "an index").v + fill := "" + if c.kids.len == 3 { + fill = coerce(g, expr(g, c.kids[2]), lane, c.kids[2], "the number for the lanes past the end").v + } heldEnd(g, mark) data := "" if ft.kind == "ptr" { @@ -6754,8 +6824,22 @@ fn vecMake(ptr[Gen] g, ast.Node c, ast.Type vt) -> Val { if first.isPlace { av = load(g, "{ ptr, i64, i64 }", first.pl.addr) } - vecBounds(g, i, n, op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1"), c) + len := op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1") data = op(g, "extractvalue { ptr, i64, i64 } " + av + ", 0") + if fill.len > 0 { + // the end of a list: the lanes that are in it are loaded, the others get `fill` + m := tailMask(g, i, n, len, c) + ep := op(g, "getelementptr " + llty(g, lane) + ", ptr " + data + ", i64 " + i) + ml := llty(g, shaped(vt, "bool")) + fname := "@llvm.masked.load." + vecSuffix(vt) + ".p0" + declareOnce(g, "declare " + lt + " " + fname + "(ptr, i32, " + ml + ", " + lt + ")") + out := op(g, "call " + lt + " " + fname + "(ptr " + ep + ", i32 " + laneBytes(vt) + ", " + ml + " " + m + ", " + lt + " " + splat(g, fill, vt) + ")") + if !first.isPlace { + release(g, first.val) + } + return Val(v: out, ty: vt) + } + vecBounds(g, i, n, len, c) } ep := op(g, "getelementptr inbounds " + llty(g, lane) + ", ptr " + data + ", i64 " + i) out := op(g, "load " + lt + ", ptr " + ep + ", align " + laneBytes(vt)) @@ -6804,6 +6888,20 @@ fn vecConvert(ptr[Gen] g, Val v, ast.Type to, ast.Node at) -> Val { } fl := from.elem[0] tl := to.elem[0] + if tl.kind == "bool" { + // a lane that isn't 0 is true + if isFloat(fl) { + return Val(v: op(g, "fcmp une " + llty(g, from) + " " + v.v + ", zeroinitializer"), ty: to) + } + return Val(v: op(g, "icmp ne " + llty(g, from) + " " + v.v + ", zeroinitializer"), ty: to) + } + if fl.kind == "bool" { + // true is 1 + if isFloat(tl) { + return Val(v: op(g, "uitofp " + llty(g, from) + " " + v.v + " to " + llty(g, to)), ty: to) + } + return Val(v: op(g, "zext " + llty(g, from) + " " + v.v + " to " + llty(g, to)), ty: to) + } inst := "" if isInt(fl) && isInt(tl) { if bitsOf(fl) == bitsOf(tl) { @@ -6837,6 +6935,12 @@ fn vecConvert(ptr[Gen] g, Val v, ast.Type to, ast.Node at) -> Val { fn vecUnary(ptr[Gen] g, ast.Node e, Val v) -> Val { lane := v.ty.elem[0] lt := llty(g, v.ty) + if lane.kind == "bool" { + if e.name != "!" && e.name != "~" { + fail(g, e, "can't put '" + e.name + "' in front of a mask (" + ast.show(v.ty) + "); '!' flips it") + } + return Val(v: op(g, "xor " + lt + " " + v.v + ", " + splat(g, "true", v.ty)), ty: v.ty) + } if e.name == "~" { if !isInt(lane) { fail(g, e, "'~' flips the bits of whole numbers, not of " + ast.show(v.ty)) @@ -6857,6 +6961,9 @@ fn vecArith(ptr[Gen] g, str o, Val l, Val r, ast.Node at) -> Val { a := asVec(g, l, vt, o, at) b := asVec(g, r, vt, o, at) lane := vt.elem[0] + if lane.kind == "bool" && o != "&" && o != "|" && o != "^" { + fail(g, at, "can't use '" + o + "' on a mask (" + ast.show(vt) + "); masks take & | ^ and !") + } names := ["+", "-", "*", "/", "&", "|", "^", "<<", ">>"] insts := ["add", "sub", "mul", "", "and", "or", "xor", "shl", "ashr"] if isUnsigned(lane) { @@ -6879,27 +6986,22 @@ fn vecArith(ptr[Gen] g, str o, Val l, Val r, ast.Node at) -> Val { return Val(v: op(g, insts[k] + " " + llty(g, vt) + " " + a.v + ", " + b.v), ty: vt) } -// v == w: every lane is equal. (Lane by lane: min, max, or v[i] < w[i].) +// v < w, v == w ...: a mask (boolxN), the answer for every lane. m.all(), m.any(), select(m, a, b) use it. fn vecCompare(ptr[Gen] g, str o, Val l, Val r, ast.Node at) -> Val { vt := vecOf(l, r) - if o != "==" && o != "!=" { - fail(g, at, "'" + o + "' on " + ast.show(vt) + " would have an answer for every lane; use min(v, w), max(v, w), or compare lanes: v[i] " + o + " w[i]") - } a := asVec(g, l, vt, o, at) b := asVec(g, r, vt, o, at) - cmp := "" - if isFloat(vt.elem[0]) { - cmp = op(g, "fcmp oeq " + llty(g, vt) + " " + a.v + ", " + b.v) - } else { - cmp = op(g, "icmp eq " + llty(g, vt) + " " + a.v + ", " + b.v) + lane := vt.elem[0] + mask := shaped(vt, "bool") + if lane.kind == "bool" && o != "==" && o != "!=" { + fail(g, at, "masks can only be compared with == and !=") } - bits := ast.Type(kind: "vec", count: vt.count) - bits.elem.push(T("bool")) - all := vecIntrinsic(g, "i1", "vector.reduce.and", bits, ["<" + str(vt.count) + " x i1>"], [cmp], "") - if o == "!=" { - return Val(v: op(g, "xor i1 " + all + ", true"), ty: T("bool")) + if isFloat(lane) { + ops := ["==", "!=", "<", "<=", ">", ">="] + preds := ["oeq", "une", "olt", "ole", "ogt", "oge"] + return Val(v: op(g, "fcmp " + preds[ops.find(o)] + " " + llty(g, vt) + " " + a.v + ", " + b.v), ty: mask) } - return Val(v: all, ty: T("bool")) + return Val(v: op(g, "icmp " + intPred(o, isSigned(lane)) + " " + llty(g, vt) + " " + a.v + ", " + b.v), ty: mask) } // min(v, w) and max(v, w), lane by lane (clamp is the two) @@ -6967,6 +7069,9 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val { lane := vt.elem[0] lt := llty(g, vt) n := m.name + if lane.kind == "bool" && n != "shuffle" && n != "lo" && n != "hi" { + return maskMethod(g, m, v) + } if n == "sum" || n == "min" || n == "max" { need(g, m, 0) how := n @@ -6984,22 +7089,20 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val { need(g, m, 0) return vecAbs(g, v) } - if n == "sqrt" || n == "floor" || n == "ceil" || n == "round" || n == "fma" { + if n == "sqrt" || n == "floor" || n == "ceil" || n == "round" || n == "fma" || n == "muladd" { if !isFloat(lane) { fail(g, m, n + "() is for f32 and f64 lanes, but this is " + ast.show(vt)) } - if n == "fma" { + if n == "fma" || n == "muladd" { need(g, m, 2) - b := asVec(g, expr(g, m.kids[1]), vt, "fma", m.kids[1]) - c := asVec(g, expr(g, m.kids[2]), vt, "fma", m.kids[2]) - // fmuladd: one fused instruction where the processor has one (FMA, NEON), a multiply and an add where - // it doesn't (llvm.fma would call a slow library function there, to round only once) - return Val(v: vecIntrinsic(g, lt, "fmuladd", vt, [lt, lt, lt], [v.v, b.v, c.v], ""), ty: vt) + b := asVec(g, expr(g, m.kids[1]), vt, n, m.kids[1]) + c := asVec(g, expr(g, m.kids[2]), vt, n, m.kids[2]) + return Val(v: fusedOp(g, n, vt, v.v, b.v, c.v), ty: vt) } need(g, m, 0) return Val(v: vecIntrinsic(g, lt, n, vt, [lt], [v.v], ""), ty: vt) } - if n == "store" { + if n == "store" || n == "store_part" { need(g, m, 2) dst := ref(g, m.kids[1], true) dt := dst.val.ty @@ -7010,7 +7113,10 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val { fail(g, m.kids[1], "store(list, i) writes " + ast.show(lane) + " numbers into a []" + ast.show(lane) + " or a ptr[" + ast.show(lane) + "], but this is " + ast.show(dt)) } if dt.kind == "array" && !dst.isPlace { - fail(g, m.kids[1], "store(list, i) needs a list in a variable (this one would be thrown away)") + fail(g, m.kids[1], n + "(list, i) needs a list in a variable (this one would be thrown away)") + } + if n == "store_part" && dt.kind != "array" { + fail(g, m.kids[1], "store_part(list, i) needs a list: a ptr doesn't know where its numbers end") } i := coerce(g, expr(g, m.kids[2]), T("int"), m.kids[2], "an index").v data := "" @@ -7021,13 +7127,615 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val { } } else { av := load(g, "{ ptr, i64, i64 }", dst.pl.addr) - vecBounds(g, i, vt.count, op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1"), m) + len := op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1") data = op(g, "extractvalue { ptr, i64, i64 } " + av + ", 0") + if n == "store_part" { + // only the lanes that are in the list are written + mk := tailMask(g, i, vt.count, len, m) + ep := op(g, "getelementptr " + llty(g, lane) + ", ptr " + data + ", i64 " + i) + ml := llty(g, shaped(vt, "bool")) + fname := "@llvm.masked.store." + vecSuffix(vt) + ".p0" + declareOnce(g, "declare void " + fname + "(" + lt + ", ptr, i32, " + ml + ")") + emit(g, "call void " + fname + "(" + lt + " " + v.v + ", ptr " + ep + ", i32 " + laneBytes(vt) + ", " + ml + " " + mk + ")") + return Val(ty: T("void")) + } + vecBounds(g, i, vt.count, len, m) } ep := op(g, "getelementptr inbounds " + llty(g, lane) + ", ptr " + data + ", i64 " + i) emit(g, "store " + lt + " " + v.v + ", ptr " + ep + ", align " + laneBytes(vt)) return Val(ty: T("void")) } - fail(g, m, "a " + ast.show(vt) + " has no method '" + n + "' (it has: sum, min, max, dot, abs, sqrt, floor, ceil, round, fma, store)") + if n == "shuffle" || n == "lo" || n == "hi" || n == "lookup" { + return shuffleMethod(g, m, v) + } + if n == "dot_add" { + return dotAdd(g, m, v) + } + fail(g, m, "a " + ast.show(vt) + " has no method '" + n + "' (it has: sum, min, max, dot, abs, sqrt, floor, ceil, round, fma, muladd, store, store_part, shuffle, lo, hi, lookup, dot_add)") return v } + +// ---------- fused multiply-add, and numbers as bits ---------- +// +// Finch never fuses on its own: a * b + c is a multiply and an add, two roundings, on every processor and with every +// --cpu (the IR has no fast-math flags, and clang compiles it with -ffp-contract=on, which fuses only what asks to be). +// Fusing is asked for by name: +// fma(a, b, c) one rounding, always (one instruction with FMA hardware, a library call without it) +// muladd(a, b, c) whichever is faster here: fused where the processor can, a multiply and an add where it can't + +// the llvm.fma / llvm.fmuladd call for a number or a vector type +fn fusedOp(ptr[Gen] g, str name, ast.Type t, str a, str b, str c) -> str { + lt := llty(g, t) + which := "fma" + if name == "muladd" { + which = "fmuladd" + } + return vecIntrinsic(g, lt, which, t, [lt, lt, lt], [a, b, c], "") +} + +fn fmaCall(ptr[Gen] g, ast.Node c) -> Val { + n := c.name + if g.index { + noteRef(g, c, c.col, n.len, docs.hover(docs.find(docs.builtins(), n)), "", 0, 0) + } + checkArgs(g, c, 0, 3, n) + []Val vals + for a in c.kids { + v := expr(g, a) + if !isNumber(v.ty) && v.ty.kind != "vec" { + fail(g, a, n + "(...) is for numbers, but this is " + ast.show(v.ty)) + } + vals.push(v) + } + // the type all three take: a vector if one is, else the float type they fit + t := T("void") + for v in vals { + if v.ty.kind == "vec" { + t = v.ty + } + } + if t.kind == "vec" { + if !isFloat(t.elem[0]) { + fail(g, c, n + "(...) is for f32 and f64 lanes, but this is " + ast.show(t)) + } + []str vs + for i in 0..3 { + vs.push(asVec(g, vals[i], t, n, c.kids[i]).v) + } + return Val(v: fusedOp(g, n, t, vs[0], vs[1], vs[2]), ty: t) + } + // f32 when an f32 takes part and nothing wider does (numbers written in the code adapt), else float + narrow := false + wide := false + for v in vals { + if v.ty.kind == "f32" { + narrow = true + } else if !v.lit { + wide = true + } + } + t = T("float") + if narrow && !wide { + t = T("f32") + } + []str vs + for i in 0..3 { + vs.push(coerce(g, vals[i], t, c.kids[i], "each number of " + n + "(...)").v) + } + return Val(v: fusedOp(g, n, t, vs[0], vs[1], vs[2]), ty: t) +} + +// the same shape as t (a number, or a vector of as many lanes) with lanes of another kind +fn shaped(ast.Type t, str lane) -> ast.Type { + if t.kind != "vec" { + return T(lane) + } + out := ast.Type(kind: "vec", count: t.count) + out.elem.push(T(lane)) + return out +} + +// a constant in every lane of t (or just the number) +fn konst(ptr[Gen] g, ast.Type t, str n) -> str { + if t.kind != "vec" { + return n + } + return splat(g, n, t) +} + +// f32_bits(x) -> u32 and back, f64 the same; f16 and bf16 (as the u16 that holds them) to f32 and back. +// Each works on a number or on a vector, lane by lane: f16_to_f32(u16x8) is an f32x8. +fn bitsCall(ptr[Gen] g, ast.Node c) -> Val { + n := c.name + if g.index { + noteRef(g, c, c.col, n.len, docs.hover(docs.find(docs.builtins(), n)), "", 0, 0) + } + checkArgs(g, c, 0, 1, n) + v := expr(g, c.kids[0]) + from := "" // the lane type it takes + to := "" + if n == "f32_bits" || n == "f32_to_f16" || n == "f32_to_bf16" { + from = "f32" + to = "u32" + if n != "f32_bits" { + to = "u16" + } + } else if n == "f64_bits" { + from = "float" + to = "u64" + } else if n == "f32_from_bits" { + from = "u32" + to = "f32" + } else if n == "f64_from_bits" { + from = "u64" + to = "float" + } else { + from = "u16" + to = "f32" + } + lane := v.ty + if v.ty.kind == "vec" { + lane = v.ty.elem[0] + } + // the bits may come as a signed number of the same size (i32 for u32), and a number written in the code fits + if isInt(T(from)) && isInt(lane) && bitsOf(lane) == bitsOf(T(from)) { + v = Val(v: v.v, ty: shaped(v.ty, from)) + } else if v.ty.kind != "vec" && v.lit { + v = coerce(g, v, T(from), c.kids[0], "the value of " + n + "(...)") + } else if lane.kind != from { + fail(g, c.kids[0], n + "(...) takes " + ast.show(T(from)) + " (or a vector of them), but this is " + ast.show(v.ty)) + } + src := shaped(v.ty, from) + out := shaped(v.ty, to) + il := llty(g, src) + ol := llty(g, out) + if n == "f32_bits" || n == "f64_bits" || n == "f32_from_bits" || n == "f64_from_bits" { + return Val(v: op(g, "bitcast " + il + " " + v.v + " to " + ol), ty: out) + } + if n == "bf16_to_f32" { + // bf16 is the top half of an f32 + wide := shaped(v.ty, "u32") + wl := llty(g, wide) + z := op(g, "zext " + il + " " + v.v + " to " + wl) + sh := op(g, "shl " + wl + " " + z + ", " + konst(g, wide, "16")) + return Val(v: op(g, "bitcast " + wl + " " + sh + " to " + ol), ty: out) + } + if n == "f32_to_bf16" { + // the top half, rounded to the nearest (ties to even); a NaN stays a NaN + wide := shaped(v.ty, "u32") + wl := llty(g, wide) + u := op(g, "bitcast " + il + " " + v.v + " to " + wl) + hi := op(g, "lshr " + wl + " " + u + ", " + konst(g, wide, "16")) + odd := op(g, "and " + wl + " " + hi + ", " + konst(g, wide, "1")) + bias := op(g, "add " + wl + " " + odd + ", " + konst(g, wide, "32767")) + rounded := op(g, "lshr " + wl + " " + op(g, "add " + wl + " " + u + ", " + bias) + ", " + konst(g, wide, "16")) + quiet := op(g, "or " + wl + " " + hi + ", " + konst(g, wide, "64")) + isNan := op(g, "fcmp uno " + il + " " + v.v + ", " + v.v) + pickd := op(g, "select " + llty(g, shaped(v.ty, "bool")) + " " + isNan + ", " + wl + " " + quiet + ", " + wl + " " + rounded) + return Val(v: op(g, "trunc " + wl + " " + pickd + " to " + ol), ty: out) + } + // f16: LLVM's half type (one instruction with F16C or on ARM64, a small library function elsewhere) + hl := "half" + if v.ty.kind == "vec" { + hl = "<" + str(v.ty.count) + " x half>" + } + if n == "f16_to_f32" { + h := op(g, "bitcast " + il + " " + v.v + " to " + hl) + return Val(v: op(g, "fpext " + hl + " " + h + " to " + ol), ty: out) + } + h := op(g, "fptrunc " + il + " " + v.v + " to " + hl) + return Val(v: op(g, "bitcast " + hl + " " + h + " to " + ol), ty: out) +} + +// ---------- masks: boolxN ---------- +// +// A comparison of vectors gives a mask, one true/false a lane (). It picks lanes with select, says +// whether any or all lanes are true, and loads and stores the end of a list (f32x8(list, i, fill), store_part). + +// lanes 0, 1, ... n-1 that are inside a list of `len` from index i (i may be past the end: then none) +fn tailMask(ptr[Gen] g, str i, int n, str len, ast.Node at) -> str { + panicIf(g, op(g, "icmp slt i64 " + i + ", 0"), "a vector's index is below 0", at) + rest := op(g, "sub i64 " + len + ", " + i) + []str lanes + for k in 0..n { + lanes.push("i64 " + str(k)) + } + wide := ast.Type(kind: "vec", count: n) + wide.elem.push(T("int")) + return op(g, "icmp slt <" + str(n) + " x i64> <" + lanes.join(", ") + ">, " + splat(g, rest, wide)) +} + +// select(mask, a, b): a's lane where the mask is true, b's where it is false +fn selectCall(ptr[Gen] g, ast.Node c) -> Val { + if g.index { + noteRef(g, c, c.col, c.name.len, docs.hover(docs.find(docs.builtins(), "select")), "", 0, 0) + } + checkArgs(g, c, 0, 3, "select") + m := expr(g, c.kids[0]) + if m.ty.kind != "vec" || m.ty.elem[0].kind != "bool" { + fail(g, c.kids[0], "select(mask, a, b) takes a mask first, from a comparison of vectors like v > 0.0; this is " + ast.show(m.ty)) + } + a := expr(g, c.kids[1]) + b := expr(g, c.kids[2]) + vt := a.ty + if vt.kind != "vec" { + vt = b.ty + } + if vt.kind != "vec" { + fail(g, c, "select(mask, a, b): a or b has to be a vector (the other may be a number for every lane)") + } + if vt.count != m.ty.count { + fail(g, c, "the mask has " + str(m.ty.count) + " lanes, but " + ast.show(vt) + " has " + str(vt.count)) + } + av := asVec(g, a, vt, "select", c.kids[1]) + bv := asVec(g, b, vt, "select", c.kids[2]) + lt := llty(g, vt) + return Val(v: op(g, "select " + llty(g, m.ty) + " " + m.v + ", " + lt + " " + av.v + ", " + lt + " " + bv.v), ty: vt) +} + +fn maskMethod(ptr[Gen] g, ast.Node m, Val v) -> Val { + n := m.name + ml := llty(g, v.ty) + if n == "any" || n == "all" { + need(g, m, 0) + how := "or" + if n == "all" { + how = "and" + } + return Val(v: vecIntrinsic(g, "i1", "vector.reduce." + how, v.ty, [ml], [v.v], ""), ty: T("bool")) + } + if n == "count" { + need(g, m, 0) + wide := shaped(v.ty, "i32") + wl := llty(g, wide) + z := op(g, "zext " + ml + " " + v.v + " to " + wl) + sum := vecIntrinsic(g, "i32", "vector.reduce.add", wide, [wl], [z], "") + return Val(v: op(g, "zext i32 " + sum + " to i64"), ty: T("int")) + } + fail(g, m, "a mask (" + ast.show(v.ty) + ") has .len, [i] and the methods any, all, count; pick lanes with select(mask, a, b)") + return v +} + +// ---------- moving lanes around ---------- + +fn indexList([]int idx) -> str { + []str parts + for k in idx { + parts.push("i32 " + str(k)) + } + return "<" + parts.join(", ") + ">" +} + +fn shuffleOf(ptr[Gen] g, Val v, []int idx, str v2) -> Val { + vt := v.ty + out := ast.Type(kind: "vec", count: idx.len) + out.elem.push(vt.elem[0]) + second := v2 + if second.len == 0 { + second = "poison" + } + return Val(v: op(g, "shufflevector " + llty(g, vt) + " " + v.v + ", " + llty(g, vt) + " " + second + ", <" + str(idx.len) + " x i32> " + indexList(idx)), ty: out) +} + +fn fitsVector(ptr[Gen] g, ast.Node at, ast.Type lane, int n) { + if n != 2 && n != 4 && n != 8 && n != 16 && n != 32 && n != 64 { + fail(g, at, "a vector has 2, 4, 8, 16, 32 or 64 lanes, not " + str(n)) + } + bits := bitsOf(lane) + if n * bits > 512 { + fail(g, at, "that would be " + str(n) + " lanes of " + ast.show(lane) + ", more than 512 bits") + } +} + +// v.shuffle(3, 2, 1, 0): lanes picked by number (numbers written in the code); v.lo(), v.hi(): the halves; +// table.lookup(idx): table[idx % table.len] for every lane of idx (pshufb on x86, tbl on ARM) +fn shuffleMethod(ptr[Gen] g, ast.Node m, Val v) -> Val { + vt := v.ty + n := m.name + if n == "lo" || n == "hi" { + need(g, m, 0) + if vt.count < 4 { + fail(g, m, "a " + ast.show(vt) + " has only " + str(vt.count) + " lanes; take them with v[0] and v[1]") + } + []int idx + start := 0 + if n == "hi" { + start = vt.count / 2 + } + for k in 0..vt.count / 2 { + idx.push(start + k) + } + return shuffleOf(g, v, idx, "") + } + if n == "shuffle" { + []int idx + for q in 1..m.kids.len { + x := expr(g, m.kids[q]) + if !isInt(x.ty) || !x.lit { + fail(g, m.kids[q], "shuffle(...) takes lane numbers written in the code (or constants), like v.shuffle(3, 2, 1, 0)") + } + k := int(x.v) or -1 + if k < 0 || k >= vt.count { + fail(g, m.kids[q], "lane " + x.v + " isn't in a " + ast.show(vt) + " (its lanes are 0 to " + str(vt.count - 1) + ")") + } + idx.push(k) + } + fitsVector(g, m, vt.elem[0], idx.len) + return shuffleOf(g, v, idx, "") + } + // lookup + need(g, m, 1) + ix := expr(g, m.kids[1]) + if ix.ty.kind != "vec" || !isInt(ix.ty.elem[0]) { + fail(g, m.kids[1], "lookup(indices) takes a vector of whole numbers (like u8x16), but this is " + ast.show(ix.ty)) + } + if vt.elem[0].kind == "bool" { + fail(g, m, "a mask can't be a lookup table") + } + fitsVector(g, m.kids[1], vt.elem[0], ix.ty.count) + il := llty(g, ix.ty) + masked := op(g, "and " + il + " " + ix.v + ", " + splat(g, str(vt.count - 1), ix.ty)) + out := shaped(ix.ty, vt.elem[0].kind) + ol := llty(g, out) + el := llty(g, vt.elem[0]) + xl := llty(g, ix.ty.elem[0]) + cur := "poison" + for k in 0..ix.ty.count { + at := op(g, "extractelement " + il + " " + masked + ", i64 " + str(k)) + lane := op(g, "extractelement " + llty(g, vt) + " " + v.v + ", " + xl + " " + at) + cur = op(g, "insertelement " + ol + " " + cur + ", " + el + " " + lane + ", i64 " + str(k)) + } + return Val(v: cur, ty: out) +} + +// concat(a, b): a's lanes, then b's +fn concatCall(ptr[Gen] g, ast.Node c) -> Val { + if g.index { + noteRef(g, c, c.col, c.name.len, docs.hover(docs.find(docs.builtins(), "concat")), "", 0, 0) + } + checkArgs(g, c, 0, 2, "concat") + a := expr(g, c.kids[0]) + b := expr(g, c.kids[1]) + if a.ty.kind != "vec" || !ast.same(a.ty, b.ty) { + fail(g, c, "concat(a, b) joins two vectors of the same type, but these are " + ast.show(a.ty) + " and " + ast.show(b.ty)) + } + fitsVector(g, c, a.ty.elem[0], a.ty.count * 2) + []int idx + for k in 0..a.ty.count * 2 { + idx.push(k) + } + return shuffleOf(g, a, idx, b.v) +} + +// ---------- processor features: @cpu(...), cpu_has(...) ---------- +// +// --cpu gives the whole program a processor (main.fch asks clang which features that means); @cpu("avx2,fma") gives one +// function more, as LLVM's "target-features". Such a function is only called after cpu_has("avx2") said yes. Features of +// the other architecture are dropped, so one source holds the x86 and the ARM versions side by side. + +fn x86Features() -> []str { + return ["sse3", "ssse3", "sse4.1", "sse4.2", "popcnt", "avx", "avx2", "fma", "f16c", "bmi1", "bmi2", "avx512f", "avx512bw", + "avx512dq", "avx512vl", "avx512vnni", "avx512bf16", "avx512fp16", "avxvnni"] +} + +fn armFeatures() -> []str { + return ["neon", "fp16", "dotprod", "i8mm", "bf16", "sve", "sve2"] +} + +// the name LLVM uses for a feature +fn llvmFeature(str x) -> str { + if x == "bmi1" { + return "bmi" + } + if x == "fp16" { + return "fullfp16" + } + return x +} + +// the features of a level: x86-64-v3 is the AVX2 generation, v4 the AVX-512 one +fn levelFeatures(str name) -> []str { + []str out + if name == "x86-64-v2" || name == "x86-64-v3" || name == "x86-64-v4" { + for x in ["sse3", "ssse3", "sse4.1", "sse4.2", "popcnt"] { + out.push(x) + } + } + if name == "x86-64-v3" || name == "x86-64-v4" { + for x in ["avx", "avx2", "bmi1", "bmi2", "f16c", "fma"] { + out.push(x) + } + } + if name == "x86-64-v4" { + for x in ["avx512f", "avx512bw", "avx512dq", "avx512vl"] { + out.push(x) + } + } + return out +} + +// a function's own @cpu features, for the architecture it is built for +fn cpuOf(ptr[Gen] g, ast.Node f) -> []str { + []str out + if f.cpu.len == 0 { + return out + } + for part in f.cpu.split(",") { + x := part.trim() + names := levelFeatures(x) + if names.len == 0 { + if !x86Features().contains(x) && !armFeatures().contains(x) { + fail(g, f, "@cpu: '" + x + "' isn't a feature Finch knows (x86: " + x86Features().join(", ") + ", x86-64-v2/v3/v4; ARM: " + armFeatures().join(", ") + ")") + } + names.push(x) + } + for y in names { + if armFeatures().contains(y) == g.tg.arm64 && !out.contains(y) { + out.push(y) + } + } + } + return out +} + +// a vector wider than 16 bytes travels in different registers depending on the features (two xmm, or one ymm), so it +// can't cross from a function with one set of features to a function with another +fn checkCpuCall(ptr[Gen] g, FnSig sig, ast.Node c, str shown) { + if sig.prog < 0 || sig.prog >= g.progs.len || sig.index >= g.progs[sig.prog].fns.len { + return + } + callee := g.progs[sig.prog].fns[sig.index].cpu + mine := g.progs[g.cur.prog].fns[g.cur.index].cpu + if callee == mine { + return + } + []ast.Type ts = sig.params + ts.push(sig.ret) + for t in ts { + if t.kind == "vec" && t.count * bitsOf(t.elem[0]) > 128 { + fail(g, c, "'" + shown + "' has other @cpu features than this function, so a " + ast.show(t) + " can't be passed between them (its registers differ); pass a list or a ptr, or give both the same @cpu") + } + } +} + +// cpu_has("avx2"): does the processor running the program have it? (false for the other architecture's names) +fn cpuHasCall(ptr[Gen] g, ast.Node c) -> Val { + if g.index { + noteRef(g, c, c.col, c.name.len, docs.hover(docs.find(docs.builtins(), "cpu_has")), "", 0, 0) + } + checkArgs(g, c, 0, 1, "cpu_has") + if c.kids[0].kind == "str" { + x := c.kids[0].text + if !x86Features().contains(x) && !armFeatures().contains(x) { + fail(g, c.kids[0], "cpu_has: '" + x + "' isn't a feature Finch knows (x86: " + x86Features().join(", ") + "; ARM: " + armFeatures().join(", ") + ")") + } + } + s := strArg(g, c.kids[0], "the feature's name") + r := op(g, "call i32 @finch_cpu_has(ptr " + spill(g, s) + ")") + release(g, s) + return Val(v: op(g, "icmp ne i32 " + r + ", 0"), ty: T("bool")) +} + +// ---------- acc.dot_add(a, b): the step of an int8 dot product ---------- +// +// acc is i32xN; a and b have 4N bytes (a u8 or i8, b i8 or u8). Lane k of acc gains a[4k]*b[4k] + ... + a[4k+3]*b[4k+3], +// exactly (no saturation). That is vpdpbusd (AVX-VNNI, AVX-512 VNNI) and usdot / sdot / udot (ARM), used when the +// function's features have them; elsewhere the same sum is written out, which LLVM makes into pmaddwd and adds. +fn dotAdd(ptr[Gen] g, ast.Node m, Val acc) -> Val { + need(g, m, 2) + at := acc.ty + if at.elem[0].kind != "i32" && at.elem[0].kind != "u32" { + fail(g, m, "dot_add is called on an i32 vector (the sums), like acc.dot_add(a, b) with acc an i32x8; this is " + ast.show(at)) + } + a := expr(g, m.kids[1]) + b := expr(g, m.kids[2]) + for x in [a, b] { + if x.ty.kind != "vec" || (x.ty.elem[0].kind != "u8" && x.ty.elem[0].kind != "i8") || x.ty.count != at.count * 4 { + fail(g, m, "dot_add(a, b) on an " + ast.show(at) + " takes two vectors of " + str(at.count * 4) + " bytes (u8 or i8), but these are " + ast.show(a.ty) + " and " + ast.show(b.ty)) + } + } + au := a.ty.elem[0].kind == "u8" + bu := b.ty.elem[0].kind == "u8" + if !au && bu { + // u8 x i8 is the same sum as i8 x u8: put the unsigned one first + t := a + a = b + b = t + au = true + bu = false + } + al := llty(g, at) + bits := at.count * 32 + f := g.curFeatures + if !g.tg.arm64 && au && !bu { + vnni := (bits == 512 && f.contains("avx512vnni")) || (bits <= 256 && (f.contains("avxvnni") || (f.contains("avx512vnni") && f.contains("avx512vl")))) + if vnni { + a32 := op(g, "bitcast " + llty(g, a.ty) + " " + a.v + " to " + al) + b32 := op(g, "bitcast " + llty(g, b.ty) + " " + b.v + " to " + al) + name := "@llvm.x86.avx512.vpdpbusd." + str(bits) + declareOnce(g, "declare " + al + " " + name + "(" + al + ", " + al + ", " + al + ")") + return Val(v: op(g, "call " + al + " " + name + "(" + al + " " + acc.v + ", " + al + " " + a32 + ", " + al + " " + b32 + ")"), ty: at) + } + } + if g.tg.arm64 && bits % 128 == 0 { + which := "" + if au && !bu && f.contains("i8mm") { + which = "usdot" + } else if au && bu && f.contains("dotprod") { + which = "udot" + } else if !au && !bu && f.contains("dotprod") { + which = "sdot" + } + if which.len > 0 { + // one instruction per 16 bytes + name := "@llvm.aarch64.neon." + which + ".v4i32.v16i8" + declareOnce(g, "declare <4 x i32> " + name + "(<4 x i32>, <16 x i8>, <16 x i8>)") + parts := at.count / 4 + []str outs + for q in 0..parts { + []int ai + for k in 0..4 { + ai.push(q * 4 + k) + } + []int bi + for k in 0..16 { + bi.push(q * 16 + k) + } + ac := shuffleOf(g, acc, ai, "").v + ap := shuffleOf(g, a, bi, "").v + bp := shuffleOf(g, b, bi, "").v + outs.push(op(g, "call <4 x i32> " + name + "(<4 x i32> " + ac + ", <16 x i8> " + ap + ", <16 x i8> " + bp + ")")) + } + return Val(v: joinParts(g, outs, 4), ty: at) + } + } + // written out: widen, multiply, add each group of four + w := shaped(a.ty, "i32") + wl := llty(g, w) + ext := "sext" + if au { + ext = "zext" + } + aw := op(g, ext + " " + llty(g, a.ty) + " " + a.v + " to " + wl) + ext = "sext" + if bu { + ext = "zext" + } + bw := op(g, ext + " " + llty(g, b.ty) + " " + b.v + " to " + wl) + prod := Val(v: op(g, "mul " + wl + " " + aw + ", " + bw), ty: w) + sum := acc.v + for k in 0..4 { + []int idx + for j in 0..at.count { + idx.push(4 * j + k) + } + sum = op(g, "add " + al + " " + sum + ", " + shuffleOf(g, prod, idx, "").v) + } + return Val(v: sum, ty: at) +} + +// <4 x i32> pieces back into one vector of n * 4 lanes +fn joinParts(ptr[Gen] g, []str parts, int each) -> str { + if parts.len == 1 { + return parts[0] + } + []str cur = parts + size := each + while cur.len > 1 { + []str next + t := ast.Type(kind: "vec", count: size) + t.elem.push(T("i32")) + k := 0 + while k < cur.len { + []int idx + for j in 0..size * 2 { + idx.push(j) + } + next.push(shuffleOf(g, Val(v: cur[k], ty: t), idx, cur[k + 1]).v) + k += 2 + } + cur = next + size = size * 2 + } + return cur[0] +} diff --git a/src/lexer.fch b/src/lexer.fch index ed4337b..19accef 100644 --- a/src/lexer.fch +++ b/src/lexer.fch @@ -70,7 +70,7 @@ fn operator(ptr[Lexer] lx) -> str { return two } } - ones := "(){}[],.:+-*/%&|^~<>=!" + ones := "(){}[],.:+-*/%&|^~<>=!@" c := lx.src.sub(lx.i, lx.i + 1) if ones.contains(c) { return c diff --git a/src/linker.fch b/src/linker.fch index e55addc..9cda7a2 100644 --- a/src/linker.fch +++ b/src/linker.fch @@ -371,10 +371,7 @@ struct Build { // IR -> program. 0 if it worked. fn build(Build b, platform.Target tg, cimport.CImports ci) -> int { - clang := getenv("FINCH_CLANG") - if clang.len == 0 { - clang = "clang" - } + clang := clangCmd() // the C compiler of the runtime and the linker: the target's own when building for another system cc := clang if tg.cross { @@ -391,7 +388,8 @@ fn build(Build b, platform.Target tg, cimport.CImports ci) -> int { target = " --target=" + tg.triple } log := b.out + ".log" - obj := irObject(clang, target + " " + b.opt + b.extra, b.ir, b.name + "\n" + tg.triple, log) or { + // -ffp-contract=on: only what asks to be fused is (muladd); a * b + c stays two roundings, whatever clang's default + obj := irObject(clang, target + " " + b.opt + " -ffp-contract=on" + b.extra, b.ir, b.name + "\n" + tg.triple, log) or { print(err) host.remove(log) return 1 @@ -477,3 +475,74 @@ fn build(Build b, platform.Target tg, cimport.CImports ci) -> int { } return 0 } + +// clang, as the program is compiled with it. Nix's clang wrapper removes -march=native when NIX_ENFORCE_NO_NATIVE +// is 1, as it is in nix-shell; --cpu native asks for it on purpose, so the wrapper is told to keep it. +fn clangCmd() -> str { + clang := getenv("FINCH_CLANG") + if clang.len == 0 { + clang = "clang" + } + if host.pathSeparator() != ";" { + return "NIX_ENFORCE_NO_NATIVE=0 " + clang + } + return clang +} + +// -march=x86-64-v3 or -mcpu=apple-m1: how clang is told the processor +fn cpuFlag(platform.Target tg, str cpu) -> str { + if cpu.len == 0 { + return "" + } + if tg.arm64 { + return " -mcpu=" + quote(cpu) + } + return " -march=" + quote(cpu) +} + +// What the processor the program is for can do, as Finch's feature names (avx2, avxvnni, i8mm ...): clang says it in +// the macros it predefines for that target and --cpu. Kept in the cache, one file per clang, target and cpu. +fn cpuFeatures(platform.Target tg, str cpu) -> []str { + clang := clangCmd() + flags := cpuFlag(tg, cpu) + if tg.cross || tg.windows { + flags = " --target=" + tg.triple + flags + } + dir := host.cacheDir() + "/cpu" + host.makeDirs(dir) + slot := dir + "/" + fingerprint2(clang + flags) + text := read_file(slot + ".txt") or "" + if text.len == 0 { + empty := slot + ".part" + str(host.pid()) + ".c" + out := slot + ".part" + str(host.pid()) + ".txt" + write_file(empty, "") + host.run(clang + flags + " -dM -E " + quote(empty) + " > " + quote(out) + " 2> " + host.nullDevice()) + host.remove(empty) + text = read_file(out) or "" + if text.len == 0 { + host.remove(out) + []str none + return none + } + host.move(out, slot + ".txt") + } + macros := ["__SSE3__", "__SSSE3__", "__SSE4_1__", "__SSE4_2__", "__POPCNT__", "__AVX__", "__AVX2__", "__FMA__", "__F16C__", + "__BMI__", "__BMI2__", "__AVX512F__", "__AVX512BW__", "__AVX512DQ__", "__AVX512VL__", "__AVX512VNNI__", + "__AVX512BF16__", "__AVX512FP16__", "__AVXVNNI__", "__ARM_NEON", "__ARM_FEATURE_FP16_VECTOR_ARITHMETIC", + "__ARM_FEATURE_DOTPROD", "__ARM_FEATURE_MATMUL_INT8", "__ARM_FEATURE_BF16_VECTOR_ARITHMETIC", "__ARM_FEATURE_SVE", + "__ARM_FEATURE_SVE2"] + names := ["sse3", "ssse3", "sse4.1", "sse4.2", "popcnt", "avx", "avx2", "fma", "f16c", "bmi1", "bmi2", "avx512f", "avx512bw", + "avx512dq", "avx512vl", "avx512vnni", "avx512bf16", "avx512fp16", "avxvnni", "neon", "fp16", "dotprod", "i8mm", "bf16", + "sve", "sve2"] + []str have + for line in text.split("\n") { + parts := line.split(" ") + if parts.len >= 2 && parts[0] == "#define" { + k := macros.find(parts[1]) + if k >= 0 && !have.contains(names[k]) { + have.push(names[k]) + } + } + } + return have +} diff --git a/src/main.fch b/src/main.fch index 8bf35bd..6cf7f61 100644 --- a/src/main.fch +++ b/src/main.fch @@ -471,7 +471,15 @@ fn main([]str args) -> int { testModules.push(prog.module) // your own files' tests, not the standard library's } } - ir := gen.generate(loader.progs, loader.files, ci, gen.Options(shared: shared, debug: debug, index: cmd == "index", cwd: host.cwd(), target: tg, test: cmd == "test", testModules: testModules, trace: trace)) + // what the processor can do (dot_add and the like pick their instructions by it); not needed for the editor's index + []str features + if cmd != "index" { + features = linker.cpuFeatures(tg, cpu) + if cpu == "native" && !tg.arm64 && !features.contains("sse4.2") { + print("warning: --cpu native made no difference to clang (a clang wrapper may remove -march=native; try --cpu with a name, like x86-64-v3)") + } + } + ir := gen.generate(loader.progs, loader.files, ci, gen.Options(shared: shared, debug: debug, index: cmd == "index", cwd: host.cwd(), target: tg, test: cmd == "test", testModules: testModules, trace: trace, features: features)) if cmd == "ir" || cmd == "index" { print(ir) return 0 @@ -501,14 +509,7 @@ fn main([]str args) -> int { if debug { extra = extra + " -g" // or clang drops the debug info that is in the IR } - if cpu.len > 0 { - // x86-64 names a processor with -march, ARM with -mcpu - if tg.arm64 { - extra = extra + " -mcpu=" + linker.quote(cpu) - } else { - extra = extra + " -march=" + linker.quote(cpu) - } - } + extra = extra + linker.cpuFlag(tg, cpu) // x86-64 names a processor with -march, ARM with -mcpu []str cflagList for d in dirs { cflagList.push(" -I" + linker.quote(d)) // also the program's own folders (its headers, for inline C functions) diff --git a/src/parser.fch b/src/parser.fch index db97175..7fef3c4 100644 --- a/src/parser.fch +++ b/src/parser.fch @@ -289,6 +289,22 @@ fn parseProgram(ptr[Parser] p) -> ast.Program { if e.flag { prog.structs.push(sumStruct(e)) // an enum with values inside is stored as a struct (see sumStruct) } + } else if at(p, "@") { + // @cpu("avx2,fma") fn ...: a function that may use these processor features (check with cpu_has first) + a := next(p) + name := expect(p, "ident", "an attribute name, like @cpu(\"avx2\")").text + if name != "cpu" { + fail(p, a, "the attribute here is @cpu(\"...\"): the processor features a function may use") + } + expect(p, "(", "'('") + feats := expect(p, "string", "the features in quotes, like @cpu(\"avx2,fma\")").text + expect(p, ")", "')'") + if !at(p, "fn") && !at(p, "export") { + fail(p, cur(p), "@cpu(...) goes right before a function") + } + f := parseFn(p) + f.cpu = feats + prog.fns.push(f) } else { prog.fns.push(parseFn(p)) } diff --git a/tests/fail/cpu_attr_name.fch b/tests/fail/cpu_attr_name.fch new file mode 100644 index 0000000..f7702a9 --- /dev/null +++ b/tests/fail/cpu_attr_name.fch @@ -0,0 +1,7 @@ +// expect: @cpu: 'avx3' isn't a feature Finch knows +@cpu("avx2,avx3") +fn f() { +} +fn main() { + f() +} diff --git a/tests/fail/cpu_has_name.fch b/tests/fail/cpu_has_name.fch new file mode 100644 index 0000000..c1ba4be --- /dev/null +++ b/tests/fail/cpu_has_name.fch @@ -0,0 +1,4 @@ +// expect: cpu_has: 'avx-512' isn't a feature Finch knows +fn main() { + print(cpu_has("avx-512")) +} diff --git a/tests/fail/cpu_wide_vector.fch b/tests/fail/cpu_wide_vector.fch new file mode 100644 index 0000000..a384b7d --- /dev/null +++ b/tests/fail/cpu_wide_vector.fch @@ -0,0 +1,8 @@ +// expect: can't be passed between them (its registers differ) +@cpu("avx2") +fn double(f32x8 v) -> f32x8 { + return v * 2.0 +} +fn main() { + print(double(f32x8(1.0))) +} diff --git a/tests/fail/vec_less.fch b/tests/fail/vec_less.fch index e9dadb0..36c975f 100644 --- a/tests/fail/vec_less.fch +++ b/tests/fail/vec_less.fch @@ -1,5 +1,7 @@ -// expect: use min(v, w), max(v, w), or compare lanes: v[i] < w[i] +// expect: use .all() or .any() fn main() { a := f32x4(1.0) - print(a < a) + if a < a { + print("no") + } } diff --git a/tests/fail/vec_shuffle_range.fch b/tests/fail/vec_shuffle_range.fch new file mode 100644 index 0000000..516bac5 --- /dev/null +++ b/tests/fail/vec_shuffle_range.fch @@ -0,0 +1,5 @@ +// expect: lane 4 isn't in a f32x4 (its lanes are 0 to 3) +fn main() { + v := f32x4(1.0) + print(v.shuffle(0, 1, 2, 4)) +} diff --git a/tests/fail/vec_shuffle_var.fch b/tests/fail/vec_shuffle_var.fch new file mode 100644 index 0000000..94c98a2 --- /dev/null +++ b/tests/fail/vec_shuffle_var.fch @@ -0,0 +1,6 @@ +// expect: shuffle(...) takes lane numbers written in the code +fn main() { + v := f32x4(1.0) + k := 2 + print(v.shuffle(k, 0, 1, 3)) +} diff --git a/tests/run/dot_int8.fch b/tests/run/dot_int8.fch new file mode 100644 index 0000000..3615628 --- /dev/null +++ b/tests/run/dot_int8.fch @@ -0,0 +1,69 @@ +// The int8 dot product of quantized models: plain, with dot_add, and in @cpu functions picked by cpu_has + +fn plain([]u8 a, []i8 b) -> int { + total := 0 + for i in 0..a.len { + total += int(a[i]) * int(b[i]) + } + return total +} + +// 32 bytes a step, 8 sums kept in a vector until the end +fn with_vectors([]u8 a, []i8 b) -> int { + acc := i32x8(0) + i := 0 + while i + 32 <= a.len { + acc = acc.dot_add(u8x32(a, i), i8x32(b, i)) + i += 32 + } + return int(acc.sum()) +} + +@cpu("avxvnni") +fn with_vnni([]u8 a, []i8 b) -> int { + acc := i32x8(0) + i := 0 + while i + 32 <= a.len { + acc = acc.dot_add(u8x32(a, i), i8x32(b, i)) + i += 32 + } + return int(acc.sum()) +} + +@cpu("avx512f,avx512bw,avx512vl,avx512vnni,i8mm") +fn with_avx512([]u8 a, []i8 b) -> int { + acc := i32x16(0) + i := 0 + while i + 64 <= a.len { + acc = acc.dot_add(u8x64(a, i), i8x64(b, i)) + i += 64 + } + return int(acc.sum()) +} + +fn best([]u8 a, []i8 b) -> int { + if cpu_has("avx512vnni") && cpu_has("avx512vl") && cpu_has("avx512bw") { + return with_avx512(a, b) + } + if cpu_has("avxvnni") { + return with_vnni(a, b) + } + return with_vectors(a, b) +} + +fn main() { + []u8 a + []i8 b + for k in 0..1024 { + a.push(u8((k * 37 + 11) % 256)) + b.push(i8((k * 53 + 7) % 256 - 128)) + } + want := plain(a, b) + print(want, with_vectors(a, b) == want, best(a, b) == want) + + // the extremes: 255 * -128 four times in a lane, no saturation + acc := i32x4(1, 2, 3, 4).dot_add(u8x16(255), i8x16(-128)) + print(acc, i32x4(0).dot_add(i8x16(-128), i8x16(-128)), i32x4(0).dot_add(u8x16(255), u8x16(255))) + print(i32x4(0).dot_add(i8x16(-1), u8x16(200))) + print(cpu_has("avx2") || cpu_has("neon") || true) +} diff --git a/tests/run/dot_int8.out b/tests/run/dot_int8.out new file mode 100644 index 0000000..28479e1 --- /dev/null +++ b/tests/run/dot_int8.out @@ -0,0 +1,4 @@ +-68096 true true +i32x4(-130559, -130558, -130557, -130556) i32x4(65536, 65536, 65536, 65536) i32x4(260100, 260100, 260100, 260100) +i32x4(-800, -800, -800, -800) +true diff --git a/tests/run/floatbits.fch b/tests/run/floatbits.fch new file mode 100644 index 0000000..fe63f3a --- /dev/null +++ b/tests/run/floatbits.fch @@ -0,0 +1,31 @@ +// a * b + c is never fused; fma rounds once; numbers as their bits; f16 and bf16 + +fn plain(float a, float b, float c) -> float { + return a * b + c +} + +fn main() { + // 0.1 * 10 is 1.0000000000000000555..., which rounds to 1 before the - 1 (two roundings), but not inside fma + print(plain(0.1, 10.0, -1.0), fma(0.1, 10.0, -1.0)) + print(fma(f32(0.1), 10.0, -1.0), fma(2, 3, 4)) + m := muladd(1.5, 2.0, 0.25) // fused or not, exact here + print(m, f32x4(1.5).muladd(2.0, 0.25), fma(f32x4(0.1), 10.0, f32x4(-1.0))[0]) + det := f32x4(0.1) * f32x4(10.0) - f32x4(1.0) + print(det[0]) + + // bits + print(f32_bits(1.0), f64_bits(1.0), f32_bits(-0.0)) + print(f32_from_bits(0x40490fdb), f64_from_bits(f64_bits(2.5)), f32_from_bits(i32(-1082130432))) + u32 inf = 0x7f800000 + print(f32_from_bits(inf), f32_bits(f32x4(1.0, 2.0, -2.0, 0.5))) + + // f16 + print(f16_to_f32(0x3c00), f16_to_f32(0xc000), f16_to_f32(0x7bff), f16_to_f32(0x0001)) + print(f32_to_f16(0.1), f32_to_f16(65504.0), f32_to_f16(1.0e6), f16_to_f32(f32_to_f16(0.333))) + print(f16_to_f32(u16x8(0x3c00, 0x4000, 0x4200, 0x4400, 0, 0x8000, 0x3800, 0x3400))) + + // bf16 + print(bf16_to_f32(0x3f80), bf16_to_f32(0x4049), f32_to_bf16(3.14159)) + print(f32_to_bf16(f32_from_bits(0x3f808000)), f32_to_bf16(f32_from_bits(0x3f818000)), f32_to_bf16(f32_from_bits(0x7fc00001))) + print(f32_to_bf16(f32x4(1.0, -2.0, 0.5, 3.14159)), bf16_to_f32(u16x4(0x3f80, 0xc000, 0x3f00, 0x4049))) +} diff --git a/tests/run/floatbits.out b/tests/run/floatbits.out new file mode 100644 index 0000000..10ab32f --- /dev/null +++ b/tests/run/floatbits.out @@ -0,0 +1,13 @@ +0 5.551115123125783e-17 +1.4901161e-08 10 +3.25 f32x4(3.25, 3.25, 3.25, 3.25) 1.4901161e-08 +0 +1065353216 4607182418800017408 2147483648 +3.1415927 2.5 -1 +inf u32x4(1065353216, 1073741824, 3221225472, 1056964608) +1 -2 65504 5.9604645e-08 +11878 31743 31744 0.3330078 +f32x8(1, 2, 3, 4, 0, -0, 0.5, 0.25) +1 3.140625 16457 +16256 16258 32704 +u16x4(16256, 49152, 16128, 16457) f32x4(1, -2, 0.5, 3.140625) diff --git a/tests/run/masks.fch b/tests/run/masks.fch new file mode 100644 index 0000000..38cefed --- /dev/null +++ b/tests/run/masks.fch @@ -0,0 +1,50 @@ +// Masks: comparisons of vectors, select, any/all/count, and the end of a list + +fn relu(f32x8 v) -> f32x8 { + return select(v > 0.0, v, 0.0) +} + +// the sum of a list of any length: whole vectors, then the last piece with its missing lanes as 0 +fn total([]f32 xs) -> f32 { + acc := f32x8(0.0) + i := 0 + while i < xs.len { + acc = acc + f32x8(xs, i, 0.0) + i += 8 + } + return acc.sum() +} + +// add 1 to every number of a list, in place, the last piece too +fn bump([]f32 xs) { + i := 0 + while i < xs.len { + (f32x8(xs, i, 0.0) + 1.0).store_part(xs, i) + i += 8 + } +} + +fn main() { + v := f32x8(-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, -4.0, 5.0) + m := v > 0.0 + print(m, m.count(), m.any(), m.all(), m[3]) + print(relu(v), select(!m, v, -1.0)) + print((v >= -1.0) & (v < 3.0), (v == 0.0) | (v == 5.0), m ^ (v > 1.0)) + + a := i32x4(1, 5, 3, 7) + b := i32x4(4, 4, 4, 4) + print(select(a > b, a, b), (a == a).all(), (a != b).all()) + u := u8x16(200) + print((u > u8x16(100)).all(), i32x4(a > b), f32x4(a < b)) + k := i32x4(0, 2, 0, -1) + print(boolx4(k), boolx4(true, false, true, false)) + + []f32 xs + for n in 0..13 { + xs.push(f32(n)) + } + print(total(xs)) + bump(xs) + print(xs) + print(f32x8(xs, 8, -1.0), f32x8(xs, 13, 9.0), f32x8(xs, 100, 7.0)) +} diff --git a/tests/run/masks.out b/tests/run/masks.out new file mode 100644 index 0000000..a034c35 --- /dev/null +++ b/tests/run/masks.out @@ -0,0 +1,9 @@ +boolx8(false, false, false, true, true, true, false, true) 4 true false true +f32x8(0, 0, 0, 1, 2, 3, 0, 5) f32x8(-2, -1, 0, -1, -1, -1, -4, -1) +boolx8(false, true, true, true, true, false, false, false) boolx8(false, false, true, false, false, false, false, true) boolx8(false, false, false, true, false, false, false, false) +i32x4(4, 5, 4, 7) true true +true i32x4(0, 1, 0, 1) f32x4(1, 0, 1, 0) +boolx4(false, true, false, true) boolx4(true, false, true, false) +78 +[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13] +f32x8(9, 10, 11, 12, 13, -1, -1, -1) f32x8(9, 9, 9, 9, 9, 9, 9, 9) f32x8(7, 7, 7, 7, 7, 7, 7, 7) diff --git a/tests/run/shuffles.fch b/tests/run/shuffles.fch new file mode 100644 index 0000000..c7b06bf --- /dev/null +++ b/tests/run/shuffles.fch @@ -0,0 +1,27 @@ +// Moving lanes: shuffle, lo/hi, concat, and a byte table lookup (how 4-bit weights are unpacked) + +// sixteen 4-bit numbers, two to a byte, made into sixteen bytes through a table of what each nibble means +fn unpack4(u8x8 packed, i8x16 table) -> i8x16 { + pairs := packed.shuffle(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7) // u8x16: each byte twice + odd := u8x16(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1) == 1 + nibbles := select(odd, pairs >> 4, pairs & 15) // low nibble first, then the high one + return table.lookup(nibbles) +} + +fn main() { + v := f32x4(1.0, 2.0, 3.0, 4.0) + print(v.shuffle(3, 2, 1, 0), v.shuffle(0, 0, 1, 1, 2, 2, 3, 3), v.shuffle(1, 2)) + w := i32x8(0, 1, 2, 3, 4, 5, 6, 7) + print(w.lo(), w.hi(), concat(w.hi(), w.lo())) + print(concat(w, w).len, concat(v, v * 10.0).shuffle(0, 4, 1, 5, 2, 6, 3, 7)) + + table := u8x16(10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25) + print(table.lookup(u8x16(15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14))) + print(table.lookup(u8x16(16, 17, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)).lo()) + print(table.lookup(u8x32(3)).hi().lo().lo()) + + q4 := i8x16(-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7) // what each 4-bit number means + print(unpack4(u8x8(0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe), q4)) + m := f32x4(1.0, -1.0, 2.0, -2.0) > 0.0 + print(m.shuffle(1, 0, 3, 2), m.lo()) +} diff --git a/tests/run/shuffles.out b/tests/run/shuffles.out new file mode 100644 index 0000000..5d724e2 --- /dev/null +++ b/tests/run/shuffles.out @@ -0,0 +1,8 @@ +f32x4(4, 3, 2, 1) f32x8(1, 1, 2, 2, 3, 3, 4, 4) f32x2(2, 3) +i32x4(0, 1, 2, 3) i32x4(4, 5, 6, 7) i32x8(4, 5, 6, 7, 0, 1, 2, 3) +16 f32x8(1, 10, 2, 20, 3, 30, 4, 40) +u8x16(25, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24) +u8x8(10, 11, 25, 10, 10, 10, 10, 10) +u8x4(13, 13, 13, 13) +i8x16(-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7) +boolx4(false, true, false, true) boolx2(true, false) diff --git a/tests/run/vectors.fch b/tests/run/vectors.fch index 2d7d95f..5255421 100644 --- a/tests/run/vectors.fch +++ b/tests/run/vectors.fch @@ -14,7 +14,7 @@ fn dot([]f32 a, []f32 b) -> f32 { acc := f32x8(0.0) i := 0 while i + 8 <= a.len { - acc = f32x8(a, i).fma(f32x8(b, i), acc) + acc = f32x8(a, i).muladd(f32x8(b, i), acc) i += 8 } total := acc.sum() @@ -36,7 +36,7 @@ fn main() { // lanes a[0] = 9.0 print(a[0], a[3], a) - print(a == f32x4(9.0, 2.0, 3.0, 4.0), a != a) + print((a == f32x4(9.0, 2.0, 3.0, 4.0)).all(), (a != a).any()) // whole numbers, bits, conversions n := i32x8(1, 2, 3, 4, 5, 6, 7, 8)