Merge pull request #2 from Nexoniarz/claude/charming-cerf-c5s1a1

Claude/charming cerf c5s1a1

README.md

@@ -103,7 +103,8 @@ Pick the guide that fits you:
 | Arrays | `nums := [1, 2, 3]`, `nums.push(4)`, `nums[0]`, `nums.len`, `for i, n in nums { }` |
 | Maps | `ages := ["ann": 31]`, `ages["bob"] = 25`, `ages.has("cy")`, `counts[w] += 1`, `for k, v in ages { }` |
 | Errors | `fn parse(str s) -> int! { … return error("bad") }` → `parse(s) or 0`, `parse(s) or { … }`, `try parse(s)` |
-| Vectors (SIMD) | `v := f32x8(list, i)`, `v * 2.0 + w`, `v.sum()`, `v.dot(w)`, `min(v, 1.0)`, `v.store(list, i)`, `v[3]` |
+| Vectors (SIMD) | `v := f32x8(list, i)`, `v * 2.0 + w`, `v.sum()`, `select(v > 0.0, v, 0.0)`, `t.lookup(idx)`, `acc.dot_add(a, b)`, `v.store(list, i)` |
+| Processors | `--cpu native`; `@cpu("avx2") fn …` with `if cpu_has("avx2")`; floats computed as written (`fma`/`muladd` to fuse) |
 | Text | `"hp \(hp)"`, `"a" + "b"`, `str(42)`, `int("42")`, `s.split(",")`, `s.upper()`, `s[0]` |
 | Control | `if` / `else if` / `else` (also as a value), `while`, `for i in 0..10`, `break`, `continue`, `defer` |
 | Memory | automatic for arrays, text and structs; `new(...)` / `free(...)` for your own heap structures |
@@ -161,6 +162,7 @@ Pick the guide that fits you:
 - [x] Threads (`import thread`) and atomics; C++ files; `--target android`
 - [x] The compiler on every system it builds for: Linux, macOS (Apple Silicon and Intel), Windows; releases made by CI
 - [x] Vector types (`f32x8`, `i8x32`, …) for SIMD, and `--cpu native`
+- [x] Masks, shuffles, byte lookup and `dot_add` for kernels; `@cpu` and `cpu_has`; `fma`/`muladd`, float bits, f16 and bf16
 
 ## Layout
 

README.pl.md

@@ -105,7 +105,8 @@ Wybierz przewodnik dla siebie:
 | Tablice | `liczby := [1, 2, 3]`, `liczby.push(4)`, `liczby[0]`, `liczby.len`, `for i, n in liczby { }` |
 | Mapy | `wiek := ["ania": 31]`, `wiek["bob"] = 25`, `wiek.has("cyryl")`, `ile[slowo] += 1`, `for k, v in wiek { }` |
 | Błędy | `fn parsuj(str s) -> int! { … return error("zła liczba") }` → `parsuj(s) or 0`, `parsuj(s) or { … }`, `try parsuj(s)` |
-| Wektory (SIMD) | `v := f32x8(lista, i)`, `v * 2.0 + w`, `v.sum()`, `v.dot(w)`, `min(v, 1.0)`, `v.store(lista, i)`, `v[3]` |
+| Wektory (SIMD) | `v := f32x8(lista, i)`, `v * 2.0 + w`, `v.sum()`, `select(v > 0.0, v, 0.0)`, `t.lookup(idx)`, `acc.dot_add(a, b)`, `v.store(lista, i)` |
+| Procesory | `--cpu native`; `@cpu("avx2") fn …` razem z `if cpu_has("avx2")`; liczby zmiennoprzecinkowe liczone tak, jak zapisane (`fma`/`muladd` do łączenia) |
 | Tekst | `"hp: \(hp)"`, `"a" + "b"`, `str(42)`, `int("42")`, `s.split(",")`, `s.upper()`, `s[0]` |
 | Sterowanie | `if` / `else if` / `else` (również jako wartość), `while`, `for i in 0..10`, `break`, `continue`, `defer` |
 | Pamięć | tablice, teksty i struktury zwalniają się same; `new(...)` / `free(...)` dla własnych struktur na stercie |
@@ -163,6 +164,7 @@ Wybierz przewodnik dla siebie:
 - [x] Wątki (`import thread`) i operacje atomowe; pliki C++; `--target android`
 - [x] Kompilator na każdym systemie, na który buduje: Linux, macOS (Apple Silicon i Intel), Windows; wydania robi CI
 - [x] Typy wektorowe (`f32x8`, `i8x32`, …) dla SIMD i `--cpu native`
+- [x] Maski, przestawianie, tabele bajtów i `dot_add` dla jąder obliczeniowych; `@cpu` i `cpu_has`; `fma`/`muladd`, bity liczb, f16 i bf16
 
 ## Co gdzie leży
 

docs/en/cheatsheet.md

@@ -255,12 +255,19 @@ Types: `f32x4/8/16`, `f64x2/4/8`, `i8x16/32/64`, `i16x8/16/32`, `i32x4/8/16`, `i
 | `f32x8(v)` | an 8-lane vector converted lane by lane (`f32x8(ints)`, `i32x8(floats)`) |
 | `v + w`, `v * 2.0`, `-v` | lane by lane; a plain number counts for every lane (`/` only on floats) |
 | `& \| ^ << >> ~` | lane by lane, whole-number lanes |
-| `v == w`, `v != w` | bool: every lane equal / not |
+| `v < w`, `v == w`, … | a mask `boolx8`: `m.all()`, `m.any()`, `m.count()`, `m & n`, `!m`, `m[i]` |
+| `select(m, a, b)` | lane by lane: `a` where `m` is true, else `b` |
 | `v[i]`, `v[i] = x`, `v.len` | a lane; the number of lanes |
 | `min(v, w)`, `max`, `abs(v)`, `clamp(v, lo, hi)` | lane by lane |
 | `v.sum()`, `v.min()`, `v.max()`, `v.dot(w)` | one number from all the lanes |
-| `v.sqrt()`, `v.floor()`, `v.ceil()`, `v.round()`, `v.fma(b, c)` | every lane (float lanes; `fma` = `v * b + c`) |
+| `v.sqrt()`, `v.floor()`, `v.ceil()`, `v.round()` | every lane (float lanes) |
+| `v.fma(b, c)`, `v.muladd(b, c)` | `v * b + c`: one rounding always / the fastest way here |
 | `v.store(list, i)` | lanes into `list[i]` … |
+| `f32x8(list, i, fill)`, `v.store_part(list, i)` | the end of a list: only the lanes inside it (the others are `fill`) |
+| `v.shuffle(3, 2, 1, 0)`, `v.lo()`, `v.hi()`, `concat(a, b)` | lanes rearranged (shuffle takes numbers written in the code) |
+| `table.lookup(idx)` | `table[idx % table.len]` per lane (`pshufb`, `tbl`) |
+| `acc.dot_add(a, b)` | `i32x8` sums gain 4 byte products each (`u8x32` × `i8x32`): `vpdpbusd`, `usdot` where there |
+| `@cpu("avx2,fma") fn …`, `cpu_has("avx2")` | a function that may use these features; check at run time before calling it |
 
 ---
 
@@ -324,6 +331,9 @@ Automatic only when nothing is lost: smaller → bigger number, any int → floa
 | `shell(command)` | run a command, its exit code |
 | `exit(code)` | stop the program now (also ends an `or { }` block) |
 | `min(a, b, …)`, `max(a, b, …)`, `abs(x)`, `clamp(x, lo, hi)` | for any numbers (your own function of that name wins) |
+| `fma(a, b, c)`, `muladd(a, b, c)` | `a * b + c` with one rounding / fused where the processor can (`a * b + c` written out is never fused) |
+| `f32_bits(x)`, `f32_from_bits(u)`, `f64_bits`, `f64_from_bits` | a float's bits as `u32`/`u64` and back, unchanged |
+| `f16_to_f32(h)`, `f32_to_f16(x)`, `bf16_to_f32(b)`, `f32_to_bf16(x)` | half and bfloat16 numbers (in a `u16`) ↔ `f32`; also on vectors |
 | `error(message)` | `return error("…")` in a `-> T!` function |
 | `addr(x)` | pointer to x; `addr(fn)` = C callback |
 | `assert(cond)`, `assert(cond, "why")` | stop with a runtime error when cond is false (in a `test` block: that test fails) |

docs/en/for-engineers.md

@@ -304,11 +304,18 @@ type goes; a call `f32x8(…)` goes to `vecMake()`.
 | `f32x8(i32x8 v)` | `sitofp`, `fptosi`, `sext`, `trunc`… on the whole vector |
 | `v + w`, `v * 2.0` | `fadd`/`add`… on vectors; a scalar operand is coerced to the lane type, then splatted |
 | `v[i]` | a place: `getelementptr <lane>, ptr %v, i64 i` after a bounds check; or `extractelement` on a temporary |
-| `v == w` | `fcmp oeq`/`icmp eq` → `<N x i1>` → `llvm.vector.reduce.and` |
+| `v < w`, `v == w` … | `fcmp`/`icmp` → `<N x i1>`, a mask (`boolxN`: `vec` with `bool` lanes) |
+| `select(m, a, b)` | `select <N x i1>` |
+| `m.any()`, `.all()`, `.count()` | `llvm.vector.reduce.or`/`.and`; `zext` + `llvm.vector.reduce.add` |
+| `f32x8(list, i, fill)`, `v.store_part(list, i)` | `llvm.masked.load`/`llvm.masked.store` with the mask `<0, 1, …> < len - i` |
+| `v.shuffle(…)`, `.lo()`, `.hi()`, `concat(a, b)` | `shufflevector` with a constant mask |
+| `table.lookup(idx)` | `and` with `len - 1`, then `extractelement`/`insertelement` per lane: the pattern the x86 and ARM backends turn into `pshufb`/`tbl` |
+| `acc.dot_add(a, b)` | `llvm.x86.avx512.vpdpbusd.{128,256,512}` or `llvm.aarch64.neon.{usdot,sdot,udot}` when the function's features have them; else `zext`/`sext`, `mul`, four strided `shufflevector`s and `add`s |
 | `min`, `max`, `abs` | `llvm.minnum`/`maxnum`, `llvm.smin`/`umin`…, `llvm.fabs`/`llvm.abs` |
 | `v.sum()`, `.min()`, `.max()` | `llvm.vector.reduce.fadd` (with `reassoc`), `.add`, `.fmin`, `.smin`… |
 | `v.sqrt()`, `.floor()`, `.ceil()`, `.round()` | the `llvm.*` intrinsic of that name |
-| `v.fma(b, c)` | `llvm.fmuladd` (fused where the target has FMA; `llvm.fma` would be a libm call per lane without it) |
+| `v.fma(b, c)`, `fma(a, b, c)` | `llvm.fma` (one rounding; a libm call per lane without FMA hardware) |
+| `v.muladd(b, c)`, `muladd(a, b, c)` | `llvm.fmuladd` (fused where the target has FMA, `fmul` + `fadd` where not) |
 | `v.store(list, i)` | bounds check, `store <8 x float> …, align 4` |
 
 Integer `/` and `%` are refused: neither x86 nor ARM has a lane-wise integer divide, and LLVM would scalarise it.
@@ -322,6 +329,20 @@ aligned address and crash.
 **Which instructions** is clang's choice: `--cpu <name>` adds `-march=<name>` (x86-64) or `-mcpu=<name>` (ARM64) to the
 IR's compile, so the same IR becomes SSE2, AVX2 + FMA, AVX-512 or NEON code. Lanes wider than the registers are split.
 
+Some operations need to know the features at code generation time (`dot_add` picks `vpdpbusd` only where it exists).
+`linker.cpuFeatures()` asks clang for its predefined macros with the same target and `--cpu`
+(`clang -march=X -dM -E empty.c`: `__AVXVNNI__`, `__AVX512VNNI__`, `__ARM_FEATURE_MATMUL_INT8` …), maps them to Finch's
+feature names and caches the answer per clang and flags; they reach `gen` as `Options.features`. `@cpu("…")` on a function
+(`Node.cpu`) adds its own: `define … "target-features"="+avx2,+fma"` (with `+evex512` for any AVX-512 feature, which
+LLVM 18+ needs for 512-bit registers), and `g.curFeatures` is the union while that function is generated. Features of the
+other architecture are dropped (`cpuOf`). `checkCpuCall` refuses a vector wider than 128 bits as a parameter or result
+between functions whose `@cpu` differ, because caller and callee would lower it to different registers.
+`cpu_has(name)` calls `finch_cpu_has` in the runtime: CPUID leaves 1 and 7 with XGETBV for the AVX/AVX-512 state on x86,
+`getauxval(AT_HWCAP/AT_HWCAP2)` on Linux ARM64, `sysctlbyname("hw.optional.arm.FEAT_…")` on macOS.
+
+Clang is always run as `NIX_ENFORCE_NO_NATIVE=0 clang …` on Unix: Nix's cc-wrapper otherwise drops `-march=native`
+inside `nix-shell`. If `--cpu native` still yields no SSE4.2 macro on x86, the driver warns.
+
 ---
 
 ## 8. Ownership and memory
@@ -507,6 +528,19 @@ test.
 | Missing return | UB if used | compile error |
 | Order of evaluation | mostly unspecified | left to right; in an assignment the right side first |
 | Shift ≥ width, out-of-range float→int | UB | poison, as in LLVM (not checked) |
+| `a * b + c` on floats | may be fused into an FMA (`-ffp-contract`) | never fused: two roundings |
+
+### Floating point
+
+Float operations are emitted as plain `fadd`/`fsub`/`fmul`/`fdiv`/`fneg` with **no fast-math flags**, so LLVM may not
+reassociate, contract or assume no NaNs; `linker.fch` also compiles the IR with `-ffp-contract=on`, which fuses only the
+`llvm.fmuladd` calls Finch asks for. The only flagged instruction is `v.sum()`'s `llvm.vector.reduce.fadd` (`reassoc`).
+`fma` → `llvm.fma`, `muladd` → `llvm.fmuladd`, on scalars and vectors (`fmaCall`, `fusedOp`).
+
+`f32_bits`/`f32_from_bits`/`f64_…` are `bitcast` (also between vectors). `f16_to_f32` is `bitcast i16 → half` + `fpext`,
+`f32_to_f16` is `fptrunc` + `bitcast`: one instruction with F16C or on ARM64, `__extendhfsf2`/`__truncsfhf2` from the C
+compiler's runtime library elsewhere. `bf16_to_f32` is `zext` + `shl 16` + `bitcast`; `f32_to_bf16` rounds to nearest even
+in integer ops (`u + 0x7fff + ((u >> 16) & 1)`, then `>> 16`) and keeps NaNs quiet (`| 0x40`), so it needs no library.
 | Dangling pointer from `addr()` / `new` + `free` | UB | still UB |
 
 ### The chain of calls on runtime errors
@@ -558,5 +592,5 @@ evaluated twice and temporaries are never pointed to.
 - Generics are monomorphised (code size grows with instantiations); there are no trait bounds, a wrong use is an error
   inside the instantiated body.
 - A failed test's owned memory is not freed (the test runner `longjmp`s out of it).
-- Vectors have no lane masks (`v < w`), shuffles or integer division.
+- Vectors have no integer division, and `lookup` on baseline x86-64 (no SSSE3) is done lane by lane.
 

docs/en/for-technicians.md

@@ -345,6 +345,40 @@ str(p)           // a C char pointer as text
 **Fixed-size numbers wrap around**, like real hardware: a `u8` holding 255 plus 1 becomes 0.
 Calculations happen in the type of the values involved, so `u8(250) + 10` is `4`, not `260`.
 
+### Decimal numbers: what is guaranteed
+
+`f32` and `f64` are IEEE 754 numbers, and Finch computes them **exactly as written**, the same on every processor and
+with every `--cpu`:
+
+- Each operation rounds once, in the order written. `a * b - c * d` is two multiplies and a subtract, never folded into a
+  fused multiply-add (C compilers often do that by default, which changes the last digits).
+- Nothing is reordered or simplified: `(a + b) + c` is not turned into `a + (b + c)`, and `x * 0.0` is not assumed to be 0
+  (it is NaN when `x` is). The one exception is `v.sum()` on a vector, which adds the lanes in the fastest order.
+
+When speed matters more than the last bit, ask for it by name:
+
+| | |
+|---|---|
+| `fma(a, b, c)` | `a * b + c` with a single rounding, always. One instruction where the processor has FMA (with `--cpu`, on ARM64); a slower library call elsewhere. |
+| `muladd(a, b, c)` | `a * b + c` the fastest way on this processor: fused where it can be, a multiply and an add where not. |
+
+Both work on numbers and on vectors (`v.fma(b, c)`, `v.muladd(b, c)`).
+
+**Numbers as bits.** File formats and model weights store numbers as raw bits. These look at the bits without
+converting the value (all of them also work lane by lane on vectors):
+
+| | |
+|---|---|
+| `f32_bits(x)` / `f32_from_bits(u)` | `f32` ↔ the `u32` with the same bits (`f32_bits(1.0)` is `0x3f800000`) |
+| `f64_bits(x)` / `f64_from_bits(u)` | `f64` ↔ `u64` |
+| `f16_to_f32(h)` / `f32_to_f16(x)` | half precision, kept in a `u16`, ↔ `f32` (rounded to the nearest) |
+| `bf16_to_f32(b)` / `f32_to_bf16(x)` | bfloat16, kept in a `u16`, ↔ `f32` (rounded to the nearest, ties to even) |
+
+```c
+[]u16 weights = read_weights()               // f16 numbers from a file
+v := f16_to_f32(u16x8(weights, 0))           // 8 of them as f32, at once
+```
+
 ---
 
 ## 7. Operators
@@ -697,6 +731,8 @@ Using `.value`, `.field` or `[i]` on a `null` pointer **stops the program** with
 | `exit(code)` | Ends the program right away. |
 | `addr(x)` | A pointer to `x`; `addr(function)` gives C a function pointer (callback). |
 | `new(value)`, `free(p)` | Manual heap memory (see [Memory](#13-memory-who-frees-what)). |
+| `fma(a, b, c)`, `muladd(a, b, c)` | `a * b + c` with one rounding / the fastest way (see [Decimal numbers](#decimal-numbers-what-is-guaranteed)). |
+| `f32_bits(x)`, `f32_from_bits(u)`, `f16_to_f32(h)`, … | A number's bits, and half-precision numbers (see [Decimal numbers](#decimal-numbers-what-is-guaranteed)). |
 | `int(x)`, `str(x)`, `u8(x)`, … | Conversions (see [Conversions](#conversions)). `str(x)` works on every type and gives the same text `print` shows. `int(text) or 0` handles text that isn't a number. |
 
 ---
@@ -1364,17 +1400,15 @@ arrays, maps and function parameters, and `print` and `str` show it.
 | `+ - * /` | lane by lane (`/` on floats only: x86 and ARM can't divide whole numbers lane by lane) |
 | `& \| ^ << >> ~` | lane by lane, on whole-number lanes |
 | `-v` | every lane negated |
-| `v == w`, `v != w` | `true` if every lane is equal / if any differs |
 | `v[i]`, `v[i] = x`, `v.len` | one lane; the number of lanes |
 | `min(v, w)`, `max(v, w)`, `abs(v)`, `clamp(v, lo, hi)` | lane by lane; `lo`, `hi` and `w` can be plain numbers |
 | `v.sum()`, `v.min()`, `v.max()` | all the lanes made into one number |
 | `v.dot(w)` | `(v * w).sum()` |
 | `v.sqrt()`, `v.floor()`, `v.ceil()`, `v.round()` | every lane (float lanes) |
-| `v.fma(b, c)` | `v * b + c`, as one instruction where the processor has one |
+| `v.fma(b, c)` | `v * b + c` with one rounding, always (the same on every processor) |
+| `v.muladd(b, c)` | `v * b + c` the fastest way here: one instruction where the processor can (the last bit may differ between processors) |
 | `v.store(list, i)` | writes the lanes to `list[i]` … (or through a `ptr`) |
 
-`<` and the other comparisons would give an answer per lane, so they aren't there; use `min`, `max` or `v[i] < w[i]`.
-
 **A real loop.** The usual shape: as many whole vectors as fit, then the rest one by one.
 
 ```c
@@ -1382,7 +1416,7 @@ fn dot([]f32 a, []f32 b) -> f32 {
     acc := f32x8(0.0)
     i := 0
     while i + 8 <= a.len {
-        acc = f32x8(a, i).fma(f32x8(b, i), acc)
+        acc = f32x8(a, i).muladd(f32x8(b, i), acc)
         i += 8
     }
     total := acc.sum()
@@ -1396,10 +1430,125 @@ fn dot([]f32 a, []f32 b) -> f32 {
 
 `examples/simd.fch` times this against the plain loop.
 
-**Which instructions.** By default a program runs on every processor of its system (on x86-64 that means SSE2, so an
+### Masks: comparing, choosing, the end of a list
-`f32x8` is done as two halves). `--cpu native` (or `cpu = "native"` in `finch.toml`) lets LLVM use everything the processor
+
-that builds it has: AVX2 and FMA, AVX-512, or NEON's newer parts on ARM. Such a program needs a processor like that one to
+Comparing vectors compares every lane, so the answer is a **mask**: one `true`/`false` a lane, of type `boolx8` (for 8 lanes).
-run. To pick a level instead: `--cpu x86-64-v3` (AVX2, most PCs since 2015), `--cpu x86-64-v4` (AVX-512), `--cpu apple-m1`.
+
+```c
+v := f32x8(-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, -4.0, 5.0)
+m := v > 0.0                      // boolx8(false, false, false, true, true, true, false, true)
+relu := select(m, v, 0.0)         // v where the mask is true, 0 elsewhere
+print(m.count(), m.any(), m.all())    // 4 true false
+if (v == w).all() { ... }         // a condition needs one bool: all() or any()
+```
+
+| | |
+|---|---|
+| `v < w`, `<=`, `>`, `>=`, `==`, `!=` | a mask (a plain number on one side counts for every lane) |
+| `select(m, a, b)` | lane by lane: `a` where `m` is true, `b` where false (either may be a plain number) |
+| `m.any()`, `m.all()`, `m.count()` | is any lane true, are all, how many |
+| `m & n`, `m \| n`, `m ^ n`, `!m` | masks combined |
+| `m[i]` | one lane, as a `bool` |
+| `i32x8(m)`, `boolx8(v)` | true is 1; a lane that isn't 0 is true |
+
+**The end of a list.** A list is rarely a whole number of vectors long. Two forms touch only the lanes that are inside it:
+
+| | |
+|---|---|
+| `f32x8(list, i, fill)` | `list[i]` … as far as the list goes; the lanes past its end are `fill` |
+| `v.store_part(list, i)` | writes only the lanes that fit in the list |
+
+```c
+fn add_one([]f32 xs) {
+    i := 0
+    while i < xs.len {                          // no separate loop for the rest
+        (f32x8(xs, i, 0.0) + 1.0).store_part(xs, i)
+        i += 8
+    }
+}
+```
+
+### Moving lanes
+
+| | |
+|---|---|
+| `v.shuffle(3, 2, 1, 0)` | the lanes with these numbers (written in the code), in this order; as many lanes as numbers |
+| `v.lo()`, `v.hi()` | the lower and the upper half |
+| `concat(a, b)` | `a`'s lanes, then `b`'s |
+| `table.lookup(idx)` | for every lane of `idx`: `table[idx % table.len]`. A byte table (`u8x16`, `i8x16`) is one `pshufb` on x86, `tbl` on ARM |
+
+Unpacking 4-bit numbers (two to a byte, as quantized models store them) is shuffle, mask and lookup:
+
+```c
+fn unpack4(u8x8 packed, i8x16 meaning) -> i8x16 {
+    pairs := packed.shuffle(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7)   // each byte twice
+    odd := u8x16(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1) == 1
+    nibbles := select(odd, pairs >> 4, pairs & 15)                         // low nibble, then high
+    return meaning.lookup(nibbles)                                         // 0..15 -> what it stands for
+}
+```
+
+### Int8 dot products: dot_add
+
+`acc.dot_add(a, b)` is the step of a quantized model's matrix multiply. `acc` is an `i32x8` of sums; `a` and `b` have four
+times as many bytes (`u8x32` and `i8x32`; `i8` × `i8` and `u8` × `u8` work too). Each lane of `acc` gains the sum of four
+products, `a[4k]*b[4k] + … + a[4k+3]*b[4k+3]`, exactly (no saturation), and the sums stay in the vector:
+
+```c
+acc := i32x8(0)
+i := 0
+while i + 32 <= weights.len {
+    acc = acc.dot_add(u8x32(activations, i), i8x32(weights, i))
+    i += 32
+}
+total := acc.sum()                // once, at the end
+```
+
+It is one instruction where the processor has one: `vpdpbusd` with AVX-VNNI (Intel since Alder Lake, AMD Zen 5) or
+AVX-512 VNNI, `usdot`/`sdot`/`udot` on ARM with the dot-product features. Elsewhere it is a few instructions
+(`pmaddwd` and adds), with the same result. Which one is used depends on `--cpu` or the function's `@cpu` (below).
+
+### Choosing the instructions: --cpu, @cpu and cpu_has
+
+By default a program runs on every processor of its system (on x86-64 that means SSE2, so an `f32x8` is done as two
+halves). Three ways to use more:
+
+**`--cpu`, for the whole program.** `--cpu native` (or `cpu = "native"` in `finch.toml`) lets LLVM use everything the
+processor that builds it has: AVX2 and FMA, AVX-512, VNNI, or NEON's newer parts on ARM. The program then needs a processor
+like that one. To pick a level instead: `--cpu x86-64-v3` (AVX2, most PCs since 2015), `--cpu x86-64-v4` (AVX-512),
+`--cpu alderlake`, `--cpu apple-m1`, `--cpu neoverse-v1`.
+
+**`@cpu`, for one function, and `cpu_has` to choose at run time.** One program can carry an AVX-512 version, an AVX2
+version and a plain one, and pick the best the computer it runs on has:
+
+```c
+@cpu("avx512f,avx512bw,avx512vl,avx512vnni")
+fn matmul_avx512([]u8 a, []i8 b, []i32 out) { ... }      // may use AVX-512 inside
+
+@cpu("x86-64-v3")
+fn matmul_avx2([]u8 a, []i8 b, []i32 out) { ... }        // a level works too
+
+fn matmul([]u8 a, []i8 b, []i32 out) {
+    if cpu_has("avx512vnni") && cpu_has("avx512vl") {
+        matmul_avx512(a, b, out)
+    } else if cpu_has("avx2") {
+        matmul_avx2(a, b, out)
+    } else {
+        matmul_plain(a, b, out)
+    }
+}
+```
+
+- The features are x86's `sse3 ssse3 sse4.1 sse4.2 popcnt avx avx2 fma f16c bmi1 bmi2 avx512f avx512bw avx512dq avx512vl
+  avx512vnni avx512bf16 avx512fp16 avxvnni`, the levels `x86-64-v2`/`v3`/`v4`, and ARM's `neon fp16 dotprod i8mm bf16 sve sve2`.
+- Names of the other architecture are left out when building, so the x86 and ARM versions sit side by side in one file
+  (and `cpu_has` says `false` for them).
+- Call a `@cpu` function only after `cpu_has` said yes: on a processor without the feature it stops with *Illegal instruction*.
+- A vector wider than 16 bytes (`f32x8`, `i32x16` …) can't be passed between functions with different `@cpu`: the two
+  sides would expect it in different registers. Pass the list (or a `ptr`) instead; Finch says so if you try.
 
 Lanes wider than the processor's registers are split by LLVM, and the same code builds for x86-64 and ARM64 alike.
 `v.sum()` on float lanes adds them in whatever order is fastest, so the last digits can differ from adding them one by one.
+
+On NixOS, Nix's clang wrapper removes `-march=native` inside `nix-shell`. Finch tells the wrapper to keep it, so
+`--cpu native` works there too; if it ever has no effect, Finch warns.

docs/pl/dla-inzynierow.md

@@ -310,11 +310,18 @@ wszędzie tam, gdzie może stać typ; wywołanie `f32x8(…)` trafia do `vecMake
 | `f32x8(i32x8 v)` | `sitofp`, `fptosi`, `sext`, `trunc`… na całym wektorze |
 | `v + w`, `v * 2.0` | `fadd`/`add`… na wektorach; argument skalarny jest dopasowywany do typu elementu i rozpowielany |
 | `v[i]` | miejsce w pamięci: `getelementptr <element>, ptr %v, i64 i` po sprawdzeniu zakresu; albo `extractelement` na wartości tymczasowej |
-| `v == w` | `fcmp oeq`/`icmp eq` → `<N x i1>` → `llvm.vector.reduce.and` |
+| `v < w`, `v == w` … | `fcmp`/`icmp` → `<N x i1>`, maska (`boolxN`: `vec` z elementami `bool`) |
+| `select(m, a, b)` | `select <N x i1>` |
+| `m.any()`, `.all()`, `.count()` | `llvm.vector.reduce.or`/`.and`; `zext` + `llvm.vector.reduce.add` |
+| `f32x8(lista, i, wypelnienie)`, `v.store_part(lista, i)` | `llvm.masked.load`/`llvm.masked.store` z maską `<0, 1, …> < len - i` |
+| `v.shuffle(…)`, `.lo()`, `.hi()`, `concat(a, b)` | `shufflevector` ze stałą maską |
+| `tabela.lookup(idx)` | `and` z `len - 1`, potem `extractelement`/`insertelement` dla każdego elementu: wzorzec, który zaplecza x86 i ARM zamieniają na `pshufb`/`tbl` |
+| `acc.dot_add(a, b)` | `llvm.x86.avx512.vpdpbusd.{128,256,512}` albo `llvm.aarch64.neon.{usdot,sdot,udot}`, gdy funkcja ma te rozszerzenia; w przeciwnym razie `zext`/`sext`, `mul`, cztery `shufflevector` co czwarty element i `add` |
 | `min`, `max`, `abs` | `llvm.minnum`/`maxnum`, `llvm.smin`/`umin`…, `llvm.fabs`/`llvm.abs` |
 | `v.sum()`, `.min()`, `.max()` | `llvm.vector.reduce.fadd` (z `reassoc`), `.add`, `.fmin`, `.smin`… |
 | `v.sqrt()`, `.floor()`, `.ceil()`, `.round()` | funkcja wewnętrzna `llvm.*` o tej nazwie |
-| `v.fma(b, c)` | `llvm.fmuladd` (jedna instrukcja tam, gdzie procesor ma FMA; `llvm.fma` bez FMA wołałoby funkcję z libm dla każdego elementu) |
+| `v.fma(b, c)`, `fma(a, b, c)` | `llvm.fma` (jedno zaokrąglenie; bez sprzętowego FMA wywołanie libm dla każdego elementu) |
+| `v.muladd(b, c)`, `muladd(a, b, c)` | `llvm.fmuladd` (jedna instrukcja tam, gdzie procesor ma FMA, a gdzie nie, `fmul` + `fadd`) |
 | `v.store(lista, i)` | sprawdzenie zakresu, `store <8 x float> …, align 4` |
 
 Dzielenie całkowite `/` i `%` jest odrzucane: ani x86, ani ARM nie mają instrukcji dzielącej liczby całkowite element
@@ -330,6 +337,20 @@ użyć `vmovaps` na adresie wyrównanym do 16 i program by się wysypał.
 do kompilacji IR, więc to samo IR staje się kodem SSE2, AVX2 + FMA, AVX-512 albo NEON. Wektory szersze niż rejestry
 procesora są dzielone na części.
 
+Niektóre działania muszą znać rozszerzenia procesora już przy generowaniu kodu (`dot_add` wybiera `vpdpbusd` tylko tam,
+gdzie ono istnieje). `linker.cpuFeatures()` pyta clanga o makra, które sam definiuje dla tego samego systemu i `--cpu`
+(`clang -march=X -dM -E pusty.c`: `__AVXVNNI__`, `__AVX512VNNI__`, `__ARM_FEATURE_MATMUL_INT8` …), zamienia je na nazwy
+rozszerzeń Fincha i zapamiętuje odpowiedź dla danego clanga i flag; do `gen` trafiają jako `Options.features`. `@cpu("…")`
+przy funkcji (`Node.cpu`) dodaje własne: `define … "target-features"="+avx2,+fma"` (z `+evex512` przy każdym rozszerzeniu
+AVX-512, którego LLVM 18+ wymaga dla rejestrów 512-bitowych), a `g.curFeatures` to suma obu na czas generowania tej
+funkcji. Rozszerzenia drugiej architektury są pomijane (`cpuOf`). `checkCpuCall` nie pozwala przekazać wektora szerszego
+niż 128 bitów jako parametru albo wyniku między funkcjami o różnym `@cpu`, bo wywołujący i wywoływany umieściliby go
+w innych rejestrach. `cpu_has(nazwa)` woła `finch_cpu_has` z runtime'u: na x86 CPUID (liście 1 i 7) z XGETBV dla stanu
+AVX/AVX-512, na Linuksie ARM64 `getauxval(AT_HWCAP/AT_HWCAP2)`, na macOS `sysctlbyname("hw.optional.arm.FEAT_…")`.
+
+Na Uniksach clang jest zawsze uruchamiany jako `NIX_ENFORCE_NO_NATIVE=0 clang …`: inaczej nakładka cc-wrapper z Niksa
+usuwa `-march=native` w `nix-shell`. Jeśli `--cpu native` mimo to nie daje na x86 makra SSE4.2, sterownik ostrzega.
+
 ---
 
 ## 8. Własność i pamięć
@@ -515,6 +536,21 @@ Testem akceptacyjnym jest `tests/lsp_test.py`.
 | Brak `return` | UB, jeśli wynik jest użyty | błąd kompilacji |
 | Kolejność obliczania | w dużej mierze nieokreślona | od lewej do prawej; w przypisaniu najpierw prawa strona |
 | Przesunięcie o ≥ szerokość, liczba zmiennoprzecinkowa poza zakresem przy zamianie na całkowitą | UB | poison, jak w LLVM (niesprawdzane) |
+| `a * b + c` na liczbach zmiennoprzecinkowych | może zostać złożone w FMA (`-ffp-contract`) | nigdy nie jest łączone: dwa zaokrąglenia |
+
+### Liczby zmiennoprzecinkowe
+
+Działania na liczbach zmiennoprzecinkowych to zwykłe `fadd`/`fsub`/`fmul`/`fdiv`/`fneg` **bez flag fast-math**, więc LLVM
+nie może zmieniać kolejności działań, łączyć ich ani zakładać braku NaN; `linker.fch` kompiluje też IR z
+`-ffp-contract=on`, które łączy tylko wywołania `llvm.fmuladd`, o które prosi Finch. Jedyna instrukcja z flagą to
+`llvm.vector.reduce.fadd` w `v.sum()` (`reassoc`). `fma` → `llvm.fma`, `muladd` → `llvm.fmuladd`, na liczbach i wektorach
+(`fmaCall`, `fusedOp`).
+
+`f32_bits`/`f32_from_bits`/`f64_…` to `bitcast` (także między wektorami). `f16_to_f32` to `bitcast i16 → half` + `fpext`,
+`f32_to_f16` to `fptrunc` + `bitcast`: jedna instrukcja z F16C albo na ARM64, a gdzie indziej `__extendhfsf2`/`__truncsfhf2`
+z biblioteki uruchomieniowej kompilatora C. `bf16_to_f32` to `zext` + `shl 16` + `bitcast`; `f32_to_bf16` zaokrągla do
+najbliższej parzystej na liczbach całkowitych (`u + 0x7fff + ((u >> 16) & 1)`, potem `>> 16`) i zostawia NaN cichym
+(`| 0x40`), więc nie potrzebuje żadnej biblioteki.
 | Wiszący wskaźnik z `addr()` / `new` + `free` | UB | nadal UB |
 
 ### Ścieżka wywołań przy błędach w trakcie działania
@@ -569,4 +605,4 @@ nie jest obliczane dwa razy, a wskaźnik nigdy nie wskazuje na wartość tymczas
 - Typy generyczne są monomorfizowane (kod rośnie z każdą instancją); nie ma ograniczeń na parametry typu, więc złe
   użycie daje błąd wewnątrz ciała instancji.
 - Pamięć, którą miał nieudany test, nie jest zwalniana (uruchamianie testów wychodzi z niego przez `longjmp`).
-- Wektory nie mają masek (`v < w`), przetasowań ani dzielenia całkowitego.
+- Wektory nie mają dzielenia całkowitego, a `lookup` na podstawowym x86-64 (bez SSSE3) działa element po elemencie.

docs/pl/dla-technikow.md

@@ -348,6 +348,41 @@ str(p)           // wskaźnik na znaki z C jako tekst
 **Liczby o stałym rozmiarze się zawijają**, jak w prawdziwym sprzęcie: `u8` równe 255 plus 1 daje 0.
 Obliczenia odbywają się w typie użytych wartości, więc `u8(250) + 10` to `4`, a nie `260`.
 
+### Liczby zmiennoprzecinkowe: co jest gwarantowane
+
+`f32` i `f64` to liczby w standardzie IEEE 754, a Finch liczy je **dokładnie tak, jak są zapisane**, tak samo na każdym
+procesorze i z każdym `--cpu`:
+
+- Każde działanie zaokrągla raz, w zapisanej kolejności. `a * b - c * d` to dwa mnożenia i odejmowanie, nigdy nie
+  złożone w jedno „mnożenie z dodawaniem” (FMA); kompilatory C często robią to domyślnie, co zmienia ostatnie cyfry.
+- Nic nie jest przestawiane ani upraszczane: `(a + b) + c` nie zamienia się w `a + (b + c)`, a `x * 0.0` nie jest
+  z góry uznawane za 0 (gdy `x` to NaN, wynik też jest NaN). Jedyny wyjątek to `v.sum()` na wektorze, które dodaje
+  elementy w najszybszej kolejności.
+
+Gdy szybkość jest ważniejsza niż ostatni bit, poproś o nią wprost:
+
+| | |
+|---|---|
+| `fma(a, b, c)` | `a * b + c` z jednym zaokrągleniem, zawsze. Jedna instrukcja tam, gdzie procesor ma FMA (z `--cpu`, na ARM64); gdzie indziej wolniejsza funkcja biblioteczna. |
+| `muladd(a, b, c)` | `a * b + c` najszybciej, jak się da na tym procesorze: jedną instrukcją, gdzie można, a gdzie nie, mnożeniem i dodawaniem. |
+
+Obie działają na liczbach i na wektorach (`v.fma(b, c)`, `v.muladd(b, c)`).
+
+**Liczby jako bity.** Formaty plików i wagi modeli zapisują liczby jako surowe bity. Te funkcje patrzą na bity bez
+przeliczania wartości (wszystkie działają też na wektorach, element po elemencie):
+
+| | |
+|---|---|
+| `f32_bits(x)` / `f32_from_bits(u)` | `f32` ↔ `u32` z tymi samymi bitami (`f32_bits(1.0)` to `0x3f800000`) |
+| `f64_bits(x)` / `f64_from_bits(u)` | `f64` ↔ `u64` |
+| `f16_to_f32(h)` / `f32_to_f16(x)` | liczba połówkowej precyzji, trzymana w `u16`, ↔ `f32` (zaokrąglenie do najbliższej) |
+| `bf16_to_f32(b)` / `f32_to_bf16(x)` | bfloat16, trzymany w `u16`, ↔ `f32` (zaokrąglenie do najbliższej, remisy do parzystej) |
+
+```c
+[]u16 wagi = wczytaj_wagi()                  // liczby f16 z pliku
+v := f16_to_f32(u16x8(wagi, 0))              // osiem z nich jako f32, naraz
+```
+
 ---
 
 ## 7. Operatory
@@ -700,6 +735,8 @@ Użycie `.value`, `.pole` albo `[i]` na wskaźniku `null` **zatrzymuje program**
 | `exit(kod)` | Natychmiast kończy program. |
 | `addr(x)` | Wskaźnik na `x`; `addr(funkcja)` daje C wskaźnik na funkcję (funkcję zwrotną). |
 | `new(wartosc)`, `free(p)` | Ręczna pamięć na stercie (zob. [Pamięć](#13-pamięć-kto-co-zwalnia)). |
+| `fma(a, b, c)`, `muladd(a, b, c)` | `a * b + c` z jednym zaokrągleniem / najszybciej (zob. [Liczby zmiennoprzecinkowe](#liczby-zmiennoprzecinkowe-co-jest-gwarantowane)). |
+| `f32_bits(x)`, `f32_from_bits(u)`, `f16_to_f32(h)`, … | Bity liczby i liczby połówkowej precyzji (zob. [Liczby zmiennoprzecinkowe](#liczby-zmiennoprzecinkowe-co-jest-gwarantowane)). |
 | `int(x)`, `str(x)`, `u8(x)`, … | Zamiany typów (zob. [Zamiana typów](#zamiana-typów)). `str(x)` działa na każdym typie i daje ten sam tekst, który pokazuje `print`. `int(tekst) or 0` obsługuje tekst, który nie jest liczbą. |
 
 ---
@@ -1371,17 +1408,15 @@ struktury, elementem tablicy, wartością w mapie i parametrem funkcji, a `print
 | `+ - * /` | element po elemencie (`/` tylko na liczbach zmiennoprzecinkowych: x86 i ARM nie dzielą liczb całkowitych element po elemencie) |
 | `& \| ^ << >> ~` | element po elemencie, na liczbach całkowitych |
 | `-v` | każdy element ze zmienionym znakiem |
-| `v == w`, `v != w` | `true`, jeśli wszystkie elementy są równe / jeśli któryś się różni |
 | `v[i]`, `v[i] = x`, `v.len` | jeden element; liczba elementów |
 | `min(v, w)`, `max(v, w)`, `abs(v)`, `clamp(v, lo, hi)` | element po elemencie; `lo`, `hi` i `w` mogą być zwykłymi liczbami |
 | `v.sum()`, `v.min()`, `v.max()` | wszystkie elementy złożone w jedną liczbę |
 | `v.dot(w)` | `(v * w).sum()` |
 | `v.sqrt()`, `v.floor()`, `v.ceil()`, `v.round()` | każdy element (liczby zmiennoprzecinkowe) |
-| `v.fma(b, c)` | `v * b + c`, jedną instrukcją tam, gdzie procesor ją ma |
+| `v.fma(b, c)` | `v * b + c` z jednym zaokrągleniem, zawsze (tak samo na każdym procesorze) |
+| `v.muladd(b, c)` | `v * b + c` najszybciej, jak się da: jedną instrukcją, gdzie procesor to potrafi (ostatni bit może się różnić między procesorami) |
 | `v.store(lista, i)` | zapisuje elementy do `lista[i]` … (albo przez `ptr`) |
 
-`<` i inne porównania dałyby osobną odpowiedź dla każdego elementu, więc ich nie ma; użyj `min`, `max` albo `v[i] < w[i]`.
-
 **Prawdziwa pętla.** Zwykły schemat: tyle całych wektorów, ile się zmieści, a resztę po jednej liczbie.
 
 ```c
@@ -1389,7 +1424,7 @@ fn iloczyn([]f32 a, []f32 b) -> f32 {
     suma := f32x8(0.0)
     i := 0
     while i + 8 <= a.len {
-        suma = f32x8(a, i).fma(f32x8(b, i), suma)
+        suma = f32x8(a, i).muladd(f32x8(b, i), suma)
         i += 8
     }
     wynik := suma.sum()
@@ -1403,12 +1438,129 @@ fn iloczyn([]f32 a, []f32 b) -> f32 {
 
 `examples/simd.fch` mierzy czas tej pętli i porównuje go ze zwykłą.
 
-**Jakie instrukcje.** Domyślnie program działa na każdym procesorze swojego systemu (na x86-64 oznacza to SSE2, więc
+### Maski: porównywanie, wybieranie, koniec listy
-`f32x8` liczy się w dwóch połówkach). `--cpu native` (albo `cpu = "native"` w `finch.toml`) pozwala LLVM użyć wszystkiego,
+
-co ma procesor, na którym budujesz: AVX2 i FMA, AVX-512 albo nowszych części NEON na ARM. Taki program potrzebuje potem
+Porównanie wektorów porównuje każdy element, więc odpowiedzią jest **maska**: jedno `true`/`false` na element,
+typu `boolx8` (dla 8 elementów).
+
+```c
+v := f32x8(-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, -4.0, 5.0)
+m := v > 0.0                      // boolx8(false, false, false, true, true, true, false, true)
+relu := select(m, v, 0.0)         // v tam, gdzie maska jest prawdą, a gdzie indziej 0
+print(m.count(), m.any(), m.all())    // 4 true false
+if (v == w).all() { ... }         // warunek potrzebuje jednego bool: all() albo any()
+```
+
+| | |
+|---|---|
+| `v < w`, `<=`, `>`, `>=`, `==`, `!=` | maska (zwykła liczba po jednej stronie działa na każdy element) |
+| `select(m, a, b)` | element po elemencie: `a` tam, gdzie `m` jest prawdą, `b` tam, gdzie fałszem (każde z nich może być zwykłą liczbą) |
+| `m.any()`, `m.all()`, `m.count()` | czy któryś element jest prawdą, czy wszystkie, ile ich jest |
+| `m & n`, `m \| n`, `m ^ n`, `!m` | łączenie masek |
+| `m[i]` | jeden element jako `bool` |
+| `i32x8(m)`, `boolx8(v)` | prawda to 1; element różny od 0 to prawda |
+
+**Koniec listy.** Lista rzadko ma długość będącą wielokrotnością wektora. Dwie formy dotykają tylko elementów,
+które są w liście:
+
+| | |
+|---|---|
+| `f32x8(lista, i, wypelnienie)` | `lista[i]` … tak daleko, jak sięga lista; elementy za jej końcem dostają `wypelnienie` |
+| `v.store_part(lista, i)` | zapisuje tylko elementy, które mieszczą się w liście |
+
+```c
+fn dodaj_jeden([]f32 xs) {
+    i := 0
+    while i < xs.len {                          // bez osobnej pętli na resztę
+        (f32x8(xs, i, 0.0) + 1.0).store_part(xs, i)
+        i += 8
+    }
+}
+```
+
+### Przestawianie elementów
+
+| | |
+|---|---|
+| `v.shuffle(3, 2, 1, 0)` | elementy o tych numerach (zapisanych w kodzie), w tej kolejności; tyle elementów, ile numerów |
+| `v.lo()`, `v.hi()` | dolna i górna połowa |
+| `concat(a, b)` | elementy `a`, a po nich elementy `b` |
+| `tabela.lookup(idx)` | dla każdego elementu `idx`: `tabela[idx % tabela.len]`. Tabela bajtów (`u8x16`, `i8x16`) to jedno `pshufb` na x86 i `tbl` na ARM |
+
+Rozpakowanie liczb 4-bitowych (po dwie w bajcie, tak jak trzymają je skwantowane modele) to przestawienie, maska i tabela:
+
+```c
+fn rozpakuj4(u8x8 spakowane, i8x16 znaczenie) -> i8x16 {
+    pary := spakowane.shuffle(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7)  // każdy bajt dwa razy
+    nieparzyste := u8x16(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1) == 1
+    polowki := select(nieparzyste, pary >> 4, pary & 15)                      // najpierw dolne 4 bity, potem górne
+    return znaczenie.lookup(polowki)                                          // 0..15 -> to, co oznacza
+}
+```
+
+### Iloczyn skalarny na bajtach: dot_add
+
+`acc.dot_add(a, b)` to krok mnożenia macierzy w skwantowanym modelu. `acc` to `i32x8` z sumami; `a` i `b` mają cztery
+razy więcej bajtów (`u8x32` i `i8x32`; działają też `i8` × `i8` i `u8` × `u8`). Każdy element `acc` dostaje sumę czterech
+iloczynów, `a[4k]*b[4k] + … + a[4k+3]*b[4k+3]`, dokładnie (bez nasycania), a sumy zostają w wektorze:
+
+```c
+acc := i32x8(0)
+i := 0
+while i + 32 <= wagi.len {
+    acc = acc.dot_add(u8x32(aktywacje, i), i8x32(wagi, i))
+    i += 32
+}
+suma := acc.sum()                 // raz, na końcu
+```
+
+To jedna instrukcja tam, gdzie procesor ją ma: `vpdpbusd` z AVX-VNNI (Intel od Alder Lake, AMD Zen 5) albo
+AVX-512 VNNI, `usdot`/`sdot`/`udot` na ARM z rozszerzeniami do iloczynu skalarnego. Gdzie indziej to kilka instrukcji
+(`pmaddwd` i dodawania), z tym samym wynikiem. Wybór zależy od `--cpu` albo `@cpu` funkcji (niżej).
+
+### Wybór instrukcji: --cpu, @cpu i cpu_has
+
+Domyślnie program działa na każdym procesorze swojego systemu (na x86-64 oznacza to SSE2, więc `f32x8` liczy się
+w dwóch połówkach). Są trzy sposoby, żeby użyć więcej:
+
+**`--cpu`, dla całego programu.** `--cpu native` (albo `cpu = "native"` w `finch.toml`) pozwala LLVM użyć wszystkiego,
+co ma procesor, na którym budujesz: AVX2 i FMA, AVX-512, VNNI albo nowszych części NEON na ARM. Program potrzebuje wtedy
 podobnego procesora. Można też wybrać poziom: `--cpu x86-64-v3` (AVX2, większość komputerów od 2015 roku),
-`--cpu x86-64-v4` (AVX-512), `--cpu apple-m1`.
+`--cpu x86-64-v4` (AVX-512), `--cpu alderlake`, `--cpu apple-m1`, `--cpu neoverse-v1`.
+
+**`@cpu` dla jednej funkcji i `cpu_has` do wyboru w trakcie działania.** Jeden program może mieć wersję z AVX-512,
+wersję z AVX2 i zwykłą, i wybrać najlepszą, jaką ma komputer, na którym działa:
+
+```c
+@cpu("avx512f,avx512bw,avx512vl,avx512vnni")
+fn mnoz_avx512([]u8 a, []i8 b, []i32 wynik) { ... }      // może w środku używać AVX-512
+
+@cpu("x86-64-v3")
+fn mnoz_avx2([]u8 a, []i8 b, []i32 wynik) { ... }        // poziom też działa
+
+fn mnoz([]u8 a, []i8 b, []i32 wynik) {
+    if cpu_has("avx512vnni") && cpu_has("avx512vl") {
+        mnoz_avx512(a, b, wynik)
+    } else if cpu_has("avx2") {
+        mnoz_avx2(a, b, wynik)
+    } else {
+        mnoz_zwykle(a, b, wynik)
+    }
+}
+```
+
+- Rozszerzenia to dla x86 `sse3 ssse3 sse4.1 sse4.2 popcnt avx avx2 fma f16c bmi1 bmi2 avx512f avx512bw avx512dq avx512vl
+  avx512vnni avx512bf16 avx512fp16 avxvnni`, poziomy `x86-64-v2`/`v3`/`v4`, a dla ARM `neon fp16 dotprod i8mm bf16 sve sve2`.
+- Nazwy z drugiej architektury są przy budowaniu pomijane, więc wersje dla x86 i ARM mogą stać obok siebie w jednym pliku
+  (a `cpu_has` zwraca dla nich `false`).
+- Funkcję z `@cpu` wołaj dopiero wtedy, gdy `cpu_has` odpowie „tak”: na procesorze bez tego rozszerzenia program
+  zatrzyma się z *Illegal instruction*.
+- Wektora szerszego niż 16 bajtów (`f32x8`, `i32x16` …) nie da się przekazać między funkcjami o różnym `@cpu`: obie strony
+  oczekiwałyby go w innych rejestrach. Przekaż zamiast tego listę (albo `ptr`); Finch o tym powie, jeśli spróbujesz.
 
 Wektory szersze niż rejestry procesora LLVM dzieli na części, a ten sam kod buduje się tak samo na x86-64 i na ARM64.
 `v.sum()` na liczbach zmiennoprzecinkowych dodaje elementy w kolejności, która jest najszybsza, więc ostatnie cyfry mogą
 się różnić od dodawania ich po kolei.
+
+Na NixOS nakładka Niksa na clanga usuwa `-march=native` w `nix-shell`. Finch mówi jej, żeby to zostawiła, więc
+`--cpu native` działa i tam; gdyby kiedyś nie miało żadnego skutku, Finch ostrzeże.

docs/pl/sciaga.md

@@ -255,12 +255,19 @@ Typy: `f32x4/8/16`, `f64x2/4/8`, `i8x16/32/64`, `i16x8/16/32`, `i32x4/8/16`, `i6
 | `f32x8(v)` | 8-elementowy wektor przeliczony element po elemencie (`f32x8(calkowite)`, `i32x8(ulamki)`) |
 | `v + w`, `v * 2.0`, `-v` | element po elemencie; zwykła liczba działa na każdy element (`/` tylko dla liczb zmiennoprzecinkowych) |
 | `& \| ^ << >> ~` | element po elemencie, dla liczb całkowitych |
-| `v == w`, `v != w` | bool: czy wszystkie elementy są równe / czy któryś się różni |
+| `v < w`, `v == w`, … | maska `boolx8`: `m.all()`, `m.any()`, `m.count()`, `m & n`, `!m`, `m[i]` |
+| `select(m, a, b)` | element po elemencie: `a` tam, gdzie `m` jest prawdą, a gdzie nie, `b` |
 | `v[i]`, `v[i] = x`, `v.len` | jeden element; liczba elementów |
 | `min(v, w)`, `max`, `abs(v)`, `clamp(v, lo, hi)` | element po elemencie |
 | `v.sum()`, `v.min()`, `v.max()`, `v.dot(w)` | jedna liczba ze wszystkich elementów |
-| `v.sqrt()`, `v.floor()`, `v.ceil()`, `v.round()`, `v.fma(b, c)` | każdy element (liczby zmiennoprzecinkowe; `fma` = `v * b + c`) |
+| `v.sqrt()`, `v.floor()`, `v.ceil()`, `v.round()` | każdy element (liczby zmiennoprzecinkowe) |
+| `v.fma(b, c)`, `v.muladd(b, c)` | `v * b + c`: zawsze z jednym zaokrągleniem / najszybciej na tym procesorze |
 | `v.store(lista, i)` | zapisuje elementy do `lista[i]` … |
+| `f32x8(lista, i, wypelnienie)`, `v.store_part(lista, i)` | koniec listy: tylko elementy, które są w niej (pozostałe to `wypelnienie`) |
+| `v.shuffle(3, 2, 1, 0)`, `v.lo()`, `v.hi()`, `concat(a, b)` | przestawione elementy (shuffle przyjmuje numery zapisane w kodzie) |
+| `tabela.lookup(idx)` | `tabela[idx % tabela.len]` dla każdego elementu (`pshufb`, `tbl`) |
+| `acc.dot_add(a, b)` | sumy `i32x8` dostają po 4 iloczyny bajtów (`u8x32` × `i8x32`): `vpdpbusd`, `usdot` tam, gdzie są |
+| `@cpu("avx2,fma") fn …`, `cpu_has("avx2")` | funkcja, która może używać tych rozszerzeń; przed jej wywołaniem sprawdź procesor |
 
 ---
 
@@ -324,6 +331,9 @@ Automatycznie tylko wtedy, gdy nic nie ginie: mniejsza liczba → większa, licz
 | `shell(polecenie)` | uruchamia polecenie, zwraca jego kod wyjścia |
 | `exit(kod)` | od razu kończy program (kończy też blok `or { }`) |
 | `min(a, b, …)`, `max(a, b, …)`, `abs(x)`, `clamp(x, lo, hi)` | dla dowolnych liczb (twoja funkcja o tej samej nazwie ma pierwszeństwo) |
+| `fma(a, b, c)`, `muladd(a, b, c)` | `a * b + c` z jednym zaokrągleniem / jedną instrukcją, gdzie procesor potrafi (zapisane `a * b + c` nigdy nie jest łączone) |
+| `f32_bits(x)`, `f32_from_bits(u)`, `f64_bits`, `f64_from_bits` | bity liczby zmiennoprzecinkowej jako `u32`/`u64` i z powrotem, bez zmian |
+| `f16_to_f32(h)`, `f32_to_f16(x)`, `bf16_to_f32(b)`, `f32_to_bf16(x)` | liczby połówkowej precyzji i bfloat16 (w `u16`) ↔ `f32`; także na wektorach |
 | `error(komunikat)` | `return error("…")` w funkcji `-> T!` |
 | `addr(x)` | wskaźnik na x; `addr(fn)` = funkcja zwrotna dla C |
 | `assert(warunek)`, `assert(warunek, "dlaczego")` | zatrzymuje program, gdy warunek jest fałszywy (w bloku `test`: ten test nie przechodzi) |

editors/kate/finch.xml

@@ -82,6 +82,19 @@
       <item>atomic_store</item>
       <item>atomic_swap</item>
       <item>atomic_cas</item>
+      <item>select</item>
+      <item>concat</item>
+      <item>fma</item>
+      <item>muladd</item>
+      <item>cpu_has</item>
+      <item>f32_bits</item>
+      <item>f32_from_bits</item>
+      <item>f64_bits</item>
+      <item>f64_from_bits</item>
+      <item>f16_to_f32</item>
+      <item>f32_to_f16</item>
+      <item>bf16_to_f32</item>
+      <item>f32_to_bf16</item>
     </list>
 
     <contexts>
@@ -104,7 +117,8 @@
         <keyword String="constants" attribute="Constant"/>
         <keyword String="types" attribute="Data Type"/>
         <!-- vectors: f32x8, i8x32, u16x16, f64x4 ... -->
-        <RegExpr String="\b[iuf](8|16|32|64)x(2|4|8|16|32|64)\b" attribute="Data Type"/>
+        <RegExpr String="\b([iuf](8|16|32|64)|bool)x(2|4|8|16|32|64)\b" attribute="Data Type"/>
+        <RegExpr String="@cpu\b" attribute="Keyword"/>
         <keyword String="builtins" attribute="Built-in"/>
 
         <RegExpr String="\b0[xX][0-9a-fA-F_]+\b" attribute="Hex"/>

editors/vscode/CHANGELOG.md

@@ -2,7 +2,8 @@
 
 ## 2.6.0
 
-- Vector types (`f32x8`, `i8x32`, `f64x4`, …) highlighted as types; completion of their methods (`sum`, `dot`, `fma`, `store`, …).
+- Vector types (`f32x8`, `i8x32`, `f64x4`, …) and masks (`boolx8`) highlighted as types; completion of their methods (`sum`, `dot_add`, `shuffle`, `lookup`, `store`, …).
+- `@cpu(...)` and the new built-ins (`select`, `concat`, `fma`, `muladd`, `cpu_has`, `f32_bits`, `f16_to_f32`, …) highlighted.
 
 ## 2.5.0
 

editors/vscode/syntaxes/finch.tmLanguage.json

@@ -147,11 +147,15 @@
           "match": "\\bself\\b"
         },
         {
-          "name": "storage.type.primitive.finch",
+          "name": "storage.modifier.attribute.finch",
-          "match": "\\b(int|float|bool|char|str|ptr|map|i8|i16|i32|i64|u8|u16|u32|u64|f32|f64|[iuf](?:8|16|32|64)x(?:2|4|8|16|32|64))\\b"
+          "match": "@cpu\\b"
         },
         {
-          "match": "\\b(print|input|read_file|write_file|file_exists|delete_file|shell|exit|addr|new|free|error|min|max|abs|clamp|assert|atomic_add|atomic_sub|atomic_load|atomic_store|atomic_swap|atomic_cas)\\s*(?=\\()",
+          "name": "storage.type.primitive.finch",
+          "match": "\\b(int|float|bool|char|str|ptr|map|i8|i16|i32|i64|u8|u16|u32|u64|f32|f64|(?:[iuf](?:8|16|32|64)|bool)x(?:2|4|8|16|32|64))\\b"
+        },
+        {
+          "match": "\\b(print|input|read_file|write_file|file_exists|delete_file|shell|exit|addr|new|free|error|min|max|abs|clamp|assert|atomic_add|atomic_sub|atomic_load|atomic_store|atomic_swap|atomic_cas|select|concat|fma|muladd|cpu_has|f32_bits|f32_from_bits|f64_bits|f64_from_bits|f16_to_f32|f32_to_f16|bf16_to_f32|f32_to_bf16)\\s*(?=\\()",
           "captures": {
             "1": {
               "name": "support.function.builtin.finch"

editors/vscode/test/grammar.js

@@ -30,6 +30,7 @@ const code = [
   '    assert(clamp(x, 0, 9) == 3)',
   '}',
   'fn dot(f32x8 a, i8x32 b) {',
+  '@cpu("avx2") fn k(boolx8 m) { select(m, a, b) }',
 ];
 const expect = [  // [line, token text, scope that must be present]
   [0, 'import', 'keyword.control.import'], [0, '"stdio.h"', 'string.quoted.double'],
@@ -51,6 +52,7 @@ const expect = [  // [line, token text, scope that must be present]
   [17, 'f', 'meta.embedded.line'], [17, 'left', 'string.quoted.double'],
   [18, 'assert', 'support.function.builtin'], [18, 'clamp', 'support.function.builtin'],
   [20, 'f32x8', 'storage.type.primitive'], [20, 'i8x32', 'storage.type.primitive'],
+  [21, '@cpu', 'storage.modifier.attribute'], [21, 'boolx8', 'storage.type.primitive'], [21, 'select', 'support.function.builtin'],
 ];
 registry.loadGrammar('source.finch').then((g) => {
   let state = vsctm.INITIAL, toks = [];

examples/simd.fch

@@ -13,12 +13,12 @@ fn dot_plain([]f32 a, []f32 b) -> f32 {
     return total
 }
 
-// eight at a time: f32x8(list, i) takes list[i] ... list[i+7], fma multiplies and adds in one step
+// eight at a time: f32x8(list, i) takes list[i] ... list[i+7]; muladd multiplies and adds, in one step where it can
 fn dot_simd([]f32 a, []f32 b) -> f32 {
     acc := f32x8(0.0)
     i := 0
     while i + 8 <= a.len {
-        acc = f32x8(a, i).fma(f32x8(b, i), acc)
+        acc = f32x8(a, i).muladd(f32x8(b, i), acc)
         i += 8
     }
     total := acc.sum()

runtime/finch_rt.c

@@ -1147,3 +1147,96 @@ __attribute__((noreturn, cold)) void finch_map_missing(const char *file, int64_t
              (int)(key->len > 100 ? 100 : key->len), key->ptr ? key->ptr : "");
     finch_panic(file, line, msg);
 }
+
+// ---------- cpu_has("avx2"): what the processor running the program can do ----------
+//
+// x86: CPUID, and for the AVX and AVX-512 families also XGETBV (the system must save those registers, or the
+// instructions fault). ARM64: the kernel's hwcaps on Linux and Android, sysctl on macOS. A name of the other
+// architecture is false.
+
+#if defined(__x86_64__) || defined(__i386__)
+#include <cpuid.h>
+static int x86_leaf(unsigned leaf, unsigned sub, unsigned r[4]) {
+    unsigned max = __get_cpuid_max(0, 0);
+    if (leaf > max) return 0;
+    __cpuid_count(leaf, sub, r[0], r[1], r[2], r[3]);
+    return 1;
+}
+static unsigned long long x86_xcr0(void) {
+    unsigned lo, hi;
+    __asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0));
+    return ((unsigned long long)hi << 32) | lo;
+}
+static int finch_cpu_has_c(const char *n) {
+    unsigned l1[4] = {0}, l7[4] = {0}, l71[4] = {0};
+    x86_leaf(1, 0, l1);
+    x86_leaf(7, 0, l7);
+    x86_leaf(7, 1, l71);
+    int osx = (l1[2] >> 27) & 1;                                    // OSXSAVE
+    unsigned long long xcr0 = osx ? x86_xcr0() : 0;
+    int ymm = osx && (xcr0 & 6) == 6;                               // the system saves xmm and ymm
+    int zmm = ymm && (xcr0 & 0xe0) == 0xe0;                         // and the AVX-512 registers
+    if (!strcmp(n, "sse3")) return l1[2] & 1;
+    if (!strcmp(n, "ssse3")) return (l1[2] >> 9) & 1;
+    if (!strcmp(n, "sse4.1")) return (l1[2] >> 19) & 1;
+    if (!strcmp(n, "sse4.2")) return (l1[2] >> 20) & 1;
+    if (!strcmp(n, "popcnt")) return (l1[2] >> 23) & 1;
+    if (!strcmp(n, "avx")) return ymm && ((l1[2] >> 28) & 1);
+    if (!strcmp(n, "fma")) return ymm && ((l1[2] >> 12) & 1);
+    if (!strcmp(n, "f16c")) return ymm && ((l1[2] >> 29) & 1);
+    if (!strcmp(n, "avx2")) return ymm && ((l7[1] >> 5) & 1);
+    if (!strcmp(n, "bmi1")) return (l7[1] >> 3) & 1;
+    if (!strcmp(n, "bmi2")) return (l7[1] >> 8) & 1;
+    if (!strcmp(n, "avxvnni")) return ymm && ((l71[0] >> 4) & 1);
+    if (!strcmp(n, "avx512f")) return zmm && ((l7[1] >> 16) & 1);
+    if (!strcmp(n, "avx512dq")) return zmm && ((l7[1] >> 17) & 1);
+    if (!strcmp(n, "avx512bw")) return zmm && ((l7[1] >> 30) & 1);
+    if (!strcmp(n, "avx512vl")) return zmm && ((l7[1] >> 31) & 1);
+    if (!strcmp(n, "avx512vnni")) return zmm && ((l7[2] >> 11) & 1);
+    if (!strcmp(n, "avx512fp16")) return zmm && ((l7[3] >> 23) & 1);
+    if (!strcmp(n, "avx512bf16")) return zmm && ((l71[0] >> 5) & 1);
+    return 0;
+}
+#elif defined(__aarch64__) && defined(__APPLE__)
+#include <sys/sysctl.h>
+static int sysctl_flag(const char *name) {
+    int v = 0;
+    size_t len = sizeof v;
+    if (sysctlbyname(name, &v, &len, NULL, 0) != 0) return 0;
+    return v != 0;
+}
+static int finch_cpu_has_c(const char *n) {
+    if (!strcmp(n, "neon")) return 1;
+    if (!strcmp(n, "fp16")) return sysctl_flag("hw.optional.arm.FEAT_FP16");
+    if (!strcmp(n, "dotprod")) return sysctl_flag("hw.optional.arm.FEAT_DotProd");
+    if (!strcmp(n, "i8mm")) return sysctl_flag("hw.optional.arm.FEAT_I8MM");
+    if (!strcmp(n, "bf16")) return sysctl_flag("hw.optional.arm.FEAT_BF16");
+    return 0;                                                       // no SVE on Apple chips
+}
+#elif defined(__aarch64__) && defined(__linux__)
+#include <sys/auxv.h>
+static int finch_cpu_has_c(const char *n) {
+    unsigned long h = getauxval(AT_HWCAP), h2 = getauxval(AT_HWCAP2);
+    if (!strcmp(n, "neon")) return 1;
+    if (!strcmp(n, "fp16")) return (h >> 10) & 1;                    // HWCAP_ASIMDHP
+    if (!strcmp(n, "dotprod")) return (h >> 20) & 1;                 // HWCAP_ASIMDDP
+    if (!strcmp(n, "sve")) return (h >> 22) & 1;                     // HWCAP_SVE
+    if (!strcmp(n, "sve2")) return (h2 >> 1) & 1;                    // HWCAP2_SVE2
+    if (!strcmp(n, "i8mm")) return (h2 >> 13) & 1;                   // HWCAP2_I8MM
+    if (!strcmp(n, "bf16")) return (h2 >> 14) & 1;                   // HWCAP2_BF16
+    return 0;
+}
+#else
+static int finch_cpu_has_c(const char *n) {
+    (void)n;
+    return 0;
+}
+#endif
+
+int finch_cpu_has(const FStr *name) {
+    char n[32];
+    int64_t len = name->len < 31 ? name->len : 31;
+    memcpy(n, name->ptr ? name->ptr : "", (size_t)len);
+    n[len] = 0;
+    return finch_cpu_has_c(n) ? 1 : 0;
+}

shell.nix

@@ -15,4 +15,7 @@ pkgs.mkShell {
     vulkan-headers
     vulkan-loader
   ];
+
+  # Nix's clang wrapper drops -march=native unless this is 0 (finch --cpu native also tells it so itself)
+  NIX_ENFORCE_NO_NATIVE = 0;
 }

src/ast.fch

@@ -44,7 +44,7 @@ fn vecType(str name) -> Type {
         return Type()
     }
     lane := name.sub(0, x)
-    if !["i8", "i16", "i32", "i64", "u8", "u16", "u32", "u64", "f32", "f64"].contains(lane) {
+    if !["i8", "i16", "i32", "i64", "u8", "u16", "u32", "u64", "f32", "f64", "bool"].contains(lane) {
         return Type()
     }
     n := 0
@@ -57,7 +57,7 @@ fn vecType(str name) -> Type {
     if n != 2 && n != 4 && n != 8 && n != 16 && n != 32 && n != 64 {
         return Type()
     }
-    bits := int(lane.sub(1, lane.len)) or 0
+    bits := int(lane.sub(1, lane.len)) or 1        // bool: a mask, one bit a lane
     if n * bits > 512 {
         return Type()           // the widest registers (AVX-512) hold 512 bits
     }
@@ -147,6 +147,7 @@ struct Node {
     bool flag     // decl: has a type; or: the right side is a { block }
     bool fallible // fn: -> T!  or  -> !
     bool exported // export fn: a C function of a library (finch build --shared)
+    str cpu       // @cpu("avx2,fma") fn: the processor features the function may use
     []Node kids
     []str names   // named arguments: Point(x: 1)
     int line

src/docs.fch

@@ -26,6 +26,19 @@ fn builtins() -> []Doc {
         doc("max", "max(a, b, ...) -> number", "The biggest of the numbers, in the type they all fit."),
         doc("abs", "abs(x) -> number", "x without its minus sign (any number type)."),
         doc("clamp", "clamp(x, lo, hi) -> number", "x, but at least lo and at most hi."),
+        doc("cpu_has", "cpu_has(str feature) -> bool", "Does the processor running the program have this feature (\"avx2\", \"avx512f\", \"avxvnni\", \"dotprod\", \"i8mm\" ...)? Check before calling a @cpu(...) function."),
+        doc("concat", "concat(V a, V b) -> V2", "One vector with a's lanes, then b's (twice as many)."),
+        doc("select", "select(mask, a, b) -> V", "Lane by lane: a where the mask is true, b where it is false. The mask comes from comparing vectors (v > 0.0)."),
+        doc("fma", "fma(a, b, c) -> float", "a * b + c with one rounding, always (numbers or vectors). a * b + c written out is never fused: two roundings."),
+        doc("muladd", "muladd(a, b, c) -> float", "a * b + c the fastest way here: one fused instruction where the processor has one, else a multiply and an add."),
+        doc("f32_bits", "f32_bits(f32 x) -> u32", "The bits of x, unchanged (also f32xN -> u32xN). f64_bits gives a u64."),
+        doc("f32_from_bits", "f32_from_bits(u32 bits) -> f32", "The f32 with these bits (also u32xN -> f32xN). f64_from_bits takes a u64."),
+        doc("f64_bits", "f64_bits(float x) -> u64", "The bits of x, unchanged (also f64xN -> u64xN)."),
+        doc("f64_from_bits", "f64_from_bits(u64 bits) -> float", "The float with these bits (also u64xN -> f64xN)."),
+        doc("f16_to_f32", "f16_to_f32(u16 half) -> f32", "A half-precision number, kept in a u16, as an f32 (also u16xN -> f32xN)."),
+        doc("f32_to_f16", "f32_to_f16(f32 x) -> u16", "x as a half-precision number in a u16, rounded to the nearest (also f32xN -> u16xN)."),
+        doc("bf16_to_f32", "bf16_to_f32(u16 b) -> f32", "A bfloat16, kept in a u16, as an f32 (also u16xN -> f32xN)."),
+        doc("f32_to_bf16", "f32_to_bf16(f32 x) -> u16", "x as a bfloat16 in a u16, rounded to the nearest, ties to even (also f32xN -> u16xN)."),
         doc("atomic_add", "atomic_add(ptr[int] p, n) -> int", "Adds n to the number at p in one step other threads can't split; gives back what was there before."),
         doc("atomic_sub", "atomic_sub(ptr[int] p, n) -> int", "Takes n from the number at p in one step; gives back what was there before."),
         doc("atomic_load", "atomic_load(ptr[int] p) -> int", "Reads the number at p, as another thread last wrote it."),
@@ -107,8 +120,18 @@ fn vecMethods() -> []Doc {
         doc("ceil", "ceil() -> V", "Every lane rounded up (f32/f64 lanes)."),
         doc("round", "round() -> V", "Every lane rounded to the nearest whole number, halves away from 0 (f32/f64 lanes)."),
         doc("abs", "abs() -> V", "Every lane without its minus sign."),
-        doc("fma", "fma(V b, V c) -> V", "v * b + c, as one instruction where the processor has one (f32/f64 lanes)."),
+        doc("fma", "fma(V b, V c) -> V", "v * b + c with one rounding, always (f32/f64 lanes). One instruction with FMA hardware, slower without."),
-        doc("store", "store([]T list, int i)", "Writes the lanes into list[i], list[i+1], ... (or through a ptr[T]).")
+        doc("muladd", "muladd(V b, V c) -> V", "v * b + c the fastest way here: fused into one instruction where the processor can (f32/f64 lanes)."),
+        doc("store", "store([]T list, int i)", "Writes the lanes into list[i], list[i+1], ... (or through a ptr[T])."),
+        doc("store_part", "store_part([]T list, int i)", "Writes the lanes that fit in the list (for its last, shorter piece)."),
+        doc("shuffle", "shuffle(int a, int b, ...) -> V", "A vector of the lanes with these numbers (written in the code): v.shuffle(3, 2, 1, 0) reverses an f32x4."),
+        doc("lo", "lo() -> V", "The lower half of the lanes."),
+        doc("hi", "hi() -> V", "The upper half of the lanes."),
+        doc("lookup", "lookup(I indices) -> V", "For every lane of indices: this vector's lane (index % len). A byte-table lookup (pshufb, tbl)."),
+        doc("dot_add", "dot_add(a, b) -> i32xN", "On an i32xN of sums: each lane gains the sum of 4 byte products of a and b (u8/i8, 4N lanes). One instruction with AVX-VNNI, AVX-512 VNNI or ARM's dot product."),
+        doc("any", "any() -> bool", "A mask: is any lane true?"),
+        doc("all", "all() -> bool", "A mask: are all lanes true?"),
+        doc("count", "count() -> int", "A mask: how many lanes are true.")
     ]
 }
 

src/gen.fch

@@ -33,6 +33,7 @@ struct Options {
     bool trace          // runtime errors show the chain of calls (off for --shared and --no-trace)
     bool test           // finch test: main runs the test "..." blocks instead
     []str testModules   // the modules whose tests run ("" is the main file); not the standard library's
+    []str features      // what the processor it is built for has (from --cpu, asked of clang): avx2, i8mm ...
 }
 
 // A generic struct or function, kept as written until something asks for a version of it.
@@ -218,6 +219,8 @@ struct Gen {
     bool deferOn    // running a defer: variables declared after it are out of sight
     int deferLimit
     int inDefer
+    []str features      // the processor features of the whole program (--cpu)
+    []str curFeatures   // and of the function being built: those and its @cpu(...)
 }
 
 // ---------- basics ----------
@@ -1460,6 +1463,9 @@ fn coerce(ptr[Gen] g, Val v, ast.Type want, ast.Node at, str what) -> Val {
     if want.kind == "str" && (isNumber(v.ty) || v.ty.kind == "bool" || v.ty.kind == "char") {
         msg = msg + " (turn it into text with str(...))"
     }
+    if want.kind == "bool" && v.ty.kind == "vec" && v.ty.elem[0].kind == "bool" {
+        msg = msg + " (a comparison of vectors has an answer for every lane: use .all() or .any())"
+    }
     if want.kind == "vec" && isNumber(v.ty) {
         msg = msg + " (" + ast.show(want) + "(x) puts x in every lane)"
     }
@@ -2357,6 +2363,13 @@ fn define(ptr[Gen] g, int si) {
         g.instNote = " (while building " + g.cur.shown + ")"
     }
     f := g.progs[g.cur.prog].fns[g.cur.index]
+    ownCpu := cpuOf(g, f)
+    g.curFeatures = g.features
+    for x in ownCpu {
+        if !g.curFeatures.contains(x) {
+            g.curFeatures.push(x)
+        }
+    }
     []str empty
     g.body = empty
     g.allocas = empty
@@ -2441,8 +2454,20 @@ fn define(ptr[Gen] g, int si) {
         }
     }
     attach := ""
+    if ownCpu.len > 0 {
+        []str plus
+        wide := false
+        for x in ownCpu {
+            plus.push("+" + llvmFeature(x))
+            wide = wide || x.starts_with("avx512")
+        }
+        if wide {
+            plus.push("+evex512")       // LLVM 18 and later: the 512-bit registers themselves (clang adds it for C too)
+        }
+        attach = " \"target-features\"=\"" + plus.join(",") + "\""
+    }
     if dbgsp >= 0 {
-        attach = " !dbg !" + str(dbgsp)
+        attach = attach + " !dbg !" + str(dbgsp)
     }
     g.dbgScope = -1
     g.out.push("define internal " + retTy(g, g.cur) + " " + g.cur.ll + "(" + params.join(", ") + ")" + attach + " {\nentry:\n" +
@@ -3680,7 +3705,7 @@ fn expr(ptr[Gen] g, ast.Node e) -> Val {
 
 fn unary(ptr[Gen] g, ast.Node e) -> Val {
     v := expr(g, e.kids[0])
-    if v.ty.kind == "vec" && e.name != "!" {
+    if v.ty.kind == "vec" && (e.name != "!" || v.ty.elem[0].kind == "bool") {
         return vecUnary(g, e, v)
     }
     if e.name == "!" {
@@ -4453,6 +4478,18 @@ fn indexOf(ptr[Gen] g, ast.Node e, Ref obj, bool write) -> Ref {
         return mapIndex(g, e, obj, t, write)
     }
     i := coerce(g, expr(g, e.kids[1]), T("int"), e.kids[1], "an index").v
+    if t.kind == "vec" && t.elem[0].kind == "bool" {
+        // a mask keeps a lane in a bit, so a lane is read from the whole value
+        if write {
+            fail(g, e, "a lane of a mask can't be set by itself; make a new mask (with a comparison or select)")
+        }
+        boundsCheck(g, i, str(t.count), e)
+        mv := obj.val.v
+        if obj.isPlace {
+            mv = load(g, llty(g, t), obj.pl.addr)
+        }
+        return Ref(isPlace: false, val: Val(v: op(g, "extractelement " + llty(g, t) + " " + mv + ", i64 " + i), ty: T("bool")))
+    }
     if t.kind == "vec" {
         // a lane: v[2], v[i] = x
         boundsCheck(g, i, str(t.count), e)
@@ -5572,6 +5609,21 @@ fn call(ptr[Gen] g, ast.Node c) -> Val {
     if ei >= 0 {
         return enumFrom(g, c, ei)
     }
+    if n == "select" && lookup(g, n) < 0 && findFn(g, g.module, n) < 0 && !g.ci.fns.has(n) {
+        return selectCall(g, c)
+    }
+    if n == "cpu_has" && lookup(g, n) < 0 && findFn(g, g.module, n) < 0 {
+        return cpuHasCall(g, c)
+    }
+    if n == "concat" && lookup(g, n) < 0 && findFn(g, g.module, n) < 0 && !g.ci.fns.has(n) {
+        return concatCall(g, c)
+    }
+    if ["fma", "muladd"].contains(n) {
+        return fmaCall(g, c)        // Finch's, also for f32 and vectors (C's fma is the same for one double)
+    }
+    if ["f32_bits", "f32_from_bits", "f64_bits", "f64_from_bits", "f16_to_f32", "f32_to_f16", "bf16_to_f32", "f32_to_bf16"].contains(n) && !g.ci.fns.has(n) {
+        return bitsCall(g, c)
+    }
     if g.ci.fns.has(n) && !["min", "max", "abs", "clamp"].contains(n) {
         return callC(g, c, g.ci.fns[n])        // (C's abs only takes an int; Finch's takes any number)
     }
@@ -5810,6 +5862,7 @@ fn callFn(ptr[Gen] g, int fi, ast.Node c, int first, str shown, str selfAddr, st
 
 // The call itself, once the arguments are values ("i64 5"); the receiver's address, if any, is already first.
 fn emitCall(ptr[Gen] g, FnSig sig, ast.Node c, []str args, str shown) -> Val {
+    checkCpuCall(g, sig, c, shown)
     if sig.fallible && g.ftarget != nodeKey(c) {
         fail(g, c, "'" + shown + "' can fail, so say what happens then:  " + shown + "(...) or <value>,  " + shown + "(...) or { ... },  or  try " + shown + "(...)")
     }
@@ -6498,6 +6551,7 @@ fn runtimeDecls() -> str {
         "declare i32 @finch_test_summary()",
         "declare void @finch_input(ptr, ptr)",
         "declare i32 @finch_file_exists(ptr)",
+        "declare i32 @finch_cpu_has(ptr)",
         "declare i32 @finch_str_try_int(ptr, ptr, ptr)",
         "declare i32 @finch_str_try_float(ptr, ptr, ptr)",
         "declare i32 @finch_try_read_file(ptr, ptr, ptr)",
@@ -6546,7 +6600,7 @@ fn assemble(ptr[Gen] g) -> str {
 
 fn generate([]ast.Program progs, []str files, cimport.CImports ci, Options opt) -> str {
     shared := opt.shared
-    gen := Gen(progs: progs, files: files, ci: ci, shared: opt.shared, debug: opt.debug, index: opt.index, cwd: opt.cwd, tg: opt.target, dbgScope: -1)
+    gen := Gen(progs: progs, files: files, ci: ci, shared: opt.shared, debug: opt.debug, index: opt.index, cwd: opt.cwd, tg: opt.target, dbgScope: -1, features: opt.features)
     g := addr(gen)
     g.trace = opt.trace && !opt.shared
     if g.debug {
@@ -6647,8 +6701,14 @@ fn laneBytes(ast.Type vt) -> str {
     return str(bitsOf(vt.elem[0]) / 8)
 }
 
-// v8f32, v32i8: how LLVM's vector intrinsics are named for this type
+// v8f32, v32i8 (or f32, i64 for a plain number): how LLVM's intrinsics are named for this type
 fn vecSuffix(ast.Type vt) -> str {
+    if vt.kind != "vec" {
+        if isFloat(vt) {
+            return "f" + str(bitsOf(vt))
+        }
+        return "i" + str(bitsOf(vt))
+    }
     if isFloat(vt.elem[0]) {
         return "v" + str(vt.count) + "f" + str(bitsOf(vt.elem[0]))
     }
@@ -6731,9 +6791,15 @@ fn vecMake(ptr[Gen] g, ast.Node c, ast.Type vt) -> Val {
         ft = first.pl.ty
     }
     if ft.kind == "array" || ft.kind == "ptr" {
-        if c.kids.len != 2 {
+        if lane.kind == "bool" {
+            fail(g, c, "a mask isn't loaded from a list; compare instead, like f32x8(list, i) > 0.0")
+        }
+        if c.kids.len != 2 && c.kids.len != 3 {
             fail(g, c, shown + "(list, i): which index to start at? (" + shown + "(list, 0) for the first " + str(n) + ")")
         }
+        if c.kids.len == 3 && ft.kind == "ptr" {
+            fail(g, c, shown + "(list, i, fill) needs a list: a ptr doesn't know where its numbers end")
+        }
         if ft.elem.len == 0 || !ast.same(ft.elem[0], lane) {
             fail(g, c.kids[0], shown + "(list, i) loads " + ast.show(lane) + " numbers, but this is " + ast.show(ft))
         }
@@ -6742,6 +6808,10 @@ fn vecMake(ptr[Gen] g, ast.Node c, ast.Type vt) -> Val {
             hold(g, first.val)
         }
         i := coerce(g, expr(g, c.kids[1]), T("int"), c.kids[1], "an index").v
+        fill := ""
+        if c.kids.len == 3 {
+            fill = coerce(g, expr(g, c.kids[2]), lane, c.kids[2], "the number for the lanes past the end").v
+        }
         heldEnd(g, mark)
         data := ""
         if ft.kind == "ptr" {
@@ -6754,8 +6824,22 @@ fn vecMake(ptr[Gen] g, ast.Node c, ast.Type vt) -> Val {
             if first.isPlace {
                 av = load(g, "{ ptr, i64, i64 }", first.pl.addr)
             }
-            vecBounds(g, i, n, op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1"), c)
+            len := op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1")
             data = op(g, "extractvalue { ptr, i64, i64 } " + av + ", 0")
+            if fill.len > 0 {
+                // the end of a list: the lanes that are in it are loaded, the others get `fill`
+                m := tailMask(g, i, n, len, c)
+                ep := op(g, "getelementptr " + llty(g, lane) + ", ptr " + data + ", i64 " + i)
+                ml := llty(g, shaped(vt, "bool"))
+                fname := "@llvm.masked.load." + vecSuffix(vt) + ".p0"
+                declareOnce(g, "declare " + lt + " " + fname + "(ptr, i32, " + ml + ", " + lt + ")")
+                out := op(g, "call " + lt + " " + fname + "(ptr " + ep + ", i32 " + laneBytes(vt) + ", " + ml + " " + m + ", " + lt + " " + splat(g, fill, vt) + ")")
+                if !first.isPlace {
+                    release(g, first.val)
+                }
+                return Val(v: out, ty: vt)
+            }
+            vecBounds(g, i, n, len, c)
         }
         ep := op(g, "getelementptr inbounds " + llty(g, lane) + ", ptr " + data + ", i64 " + i)
         out := op(g, "load " + lt + ", ptr " + ep + ", align " + laneBytes(vt))
@@ -6804,6 +6888,20 @@ fn vecConvert(ptr[Gen] g, Val v, ast.Type to, ast.Node at) -> Val {
     }
     fl := from.elem[0]
     tl := to.elem[0]
+    if tl.kind == "bool" {
+        // a lane that isn't 0 is true
+        if isFloat(fl) {
+            return Val(v: op(g, "fcmp une " + llty(g, from) + " " + v.v + ", zeroinitializer"), ty: to)
+        }
+        return Val(v: op(g, "icmp ne " + llty(g, from) + " " + v.v + ", zeroinitializer"), ty: to)
+    }
+    if fl.kind == "bool" {
+        // true is 1
+        if isFloat(tl) {
+            return Val(v: op(g, "uitofp " + llty(g, from) + " " + v.v + " to " + llty(g, to)), ty: to)
+        }
+        return Val(v: op(g, "zext " + llty(g, from) + " " + v.v + " to " + llty(g, to)), ty: to)
+    }
     inst := ""
     if isInt(fl) && isInt(tl) {
         if bitsOf(fl) == bitsOf(tl) {
@@ -6837,6 +6935,12 @@ fn vecConvert(ptr[Gen] g, Val v, ast.Type to, ast.Node at) -> Val {
 fn vecUnary(ptr[Gen] g, ast.Node e, Val v) -> Val {
     lane := v.ty.elem[0]
     lt := llty(g, v.ty)
+    if lane.kind == "bool" {
+        if e.name != "!" && e.name != "~" {
+            fail(g, e, "can't put '" + e.name + "' in front of a mask (" + ast.show(v.ty) + "); '!' flips it")
+        }
+        return Val(v: op(g, "xor " + lt + " " + v.v + ", " + splat(g, "true", v.ty)), ty: v.ty)
+    }
     if e.name == "~" {
         if !isInt(lane) {
             fail(g, e, "'~' flips the bits of whole numbers, not of " + ast.show(v.ty))
@@ -6857,6 +6961,9 @@ fn vecArith(ptr[Gen] g, str o, Val l, Val r, ast.Node at) -> Val {
     a := asVec(g, l, vt, o, at)
     b := asVec(g, r, vt, o, at)
     lane := vt.elem[0]
+    if lane.kind == "bool" && o != "&" && o != "|" && o != "^" {
+        fail(g, at, "can't use '" + o + "' on a mask (" + ast.show(vt) + "); masks take & | ^ and !")
+    }
     names := ["+", "-", "*", "/", "&", "|", "^", "<<", ">>"]
     insts := ["add", "sub", "mul", "", "and", "or", "xor", "shl", "ashr"]
     if isUnsigned(lane) {
@@ -6879,27 +6986,22 @@ fn vecArith(ptr[Gen] g, str o, Val l, Val r, ast.Node at) -> Val {
     return Val(v: op(g, insts[k] + " " + llty(g, vt) + " " + a.v + ", " + b.v), ty: vt)
 }
 
-// v == w: every lane is equal. (Lane by lane: min, max, or v[i] < w[i].)
+// v < w, v == w ...: a mask (boolxN), the answer for every lane. m.all(), m.any(), select(m, a, b) use it.
 fn vecCompare(ptr[Gen] g, str o, Val l, Val r, ast.Node at) -> Val {
     vt := vecOf(l, r)
-    if o != "==" && o != "!=" {
-        fail(g, at, "'" + o + "' on " + ast.show(vt) + " would have an answer for every lane; use min(v, w), max(v, w), or compare lanes: v[i] " + o + " w[i]")
-    }
     a := asVec(g, l, vt, o, at)
     b := asVec(g, r, vt, o, at)
-    cmp := ""
+    lane := vt.elem[0]
-    if isFloat(vt.elem[0]) {
+    mask := shaped(vt, "bool")
-        cmp = op(g, "fcmp oeq " + llty(g, vt) + " " + a.v + ", " + b.v)
+    if lane.kind == "bool" && o != "==" && o != "!=" {
-    } else {
+        fail(g, at, "masks can only be compared with == and !=")
-        cmp = op(g, "icmp eq " + llty(g, vt) + " " + a.v + ", " + b.v)
     }
-    bits := ast.Type(kind: "vec", count: vt.count)
+    if isFloat(lane) {
-    bits.elem.push(T("bool"))
+        ops := ["==", "!=", "<", "<=", ">", ">="]
-    all := vecIntrinsic(g, "i1", "vector.reduce.and", bits, ["<" + str(vt.count) + " x i1>"], [cmp], "")
+        preds := ["oeq", "une", "olt", "ole", "ogt", "oge"]
-    if o == "!=" {
+        return Val(v: op(g, "fcmp " + preds[ops.find(o)] + " " + llty(g, vt) + " " + a.v + ", " + b.v), ty: mask)
-        return Val(v: op(g, "xor i1 " + all + ", true"), ty: T("bool"))
     }
-    return Val(v: all, ty: T("bool"))
+    return Val(v: op(g, "icmp " + intPred(o, isSigned(lane)) + " " + llty(g, vt) + " " + a.v + ", " + b.v), ty: mask)
 }
 
 // min(v, w) and max(v, w), lane by lane (clamp is the two)
@@ -6967,6 +7069,9 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
     lane := vt.elem[0]
     lt := llty(g, vt)
     n := m.name
+    if lane.kind == "bool" && n != "shuffle" && n != "lo" && n != "hi" {
+        return maskMethod(g, m, v)
+    }
     if n == "sum" || n == "min" || n == "max" {
         need(g, m, 0)
         how := n
@@ -6984,22 +7089,20 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
         need(g, m, 0)
         return vecAbs(g, v)
     }
-    if n == "sqrt" || n == "floor" || n == "ceil" || n == "round" || n == "fma" {
+    if n == "sqrt" || n == "floor" || n == "ceil" || n == "round" || n == "fma" || n == "muladd" {
         if !isFloat(lane) {
             fail(g, m, n + "() is for f32 and f64 lanes, but this is " + ast.show(vt))
         }
-        if n == "fma" {
+        if n == "fma" || n == "muladd" {
             need(g, m, 2)
-            b := asVec(g, expr(g, m.kids[1]), vt, "fma", m.kids[1])
+            b := asVec(g, expr(g, m.kids[1]), vt, n, m.kids[1])
-            c := asVec(g, expr(g, m.kids[2]), vt, "fma", m.kids[2])
+            c := asVec(g, expr(g, m.kids[2]), vt, n, m.kids[2])
-            // fmuladd: one fused instruction where the processor has one (FMA, NEON), a multiply and an add where
+            return Val(v: fusedOp(g, n, vt, v.v, b.v, c.v), ty: vt)
-            // it doesn't (llvm.fma would call a slow library function there, to round only once)
-            return Val(v: vecIntrinsic(g, lt, "fmuladd", vt, [lt, lt, lt], [v.v, b.v, c.v], ""), ty: vt)
         }
         need(g, m, 0)
         return Val(v: vecIntrinsic(g, lt, n, vt, [lt], [v.v], ""), ty: vt)
     }
-    if n == "store" {
+    if n == "store" || n == "store_part" {
         need(g, m, 2)
         dst := ref(g, m.kids[1], true)
         dt := dst.val.ty
@@ -7010,7 +7113,10 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
             fail(g, m.kids[1], "store(list, i) writes " + ast.show(lane) + " numbers into a []" + ast.show(lane) + " or a ptr[" + ast.show(lane) + "], but this is " + ast.show(dt))
         }
         if dt.kind == "array" && !dst.isPlace {
-            fail(g, m.kids[1], "store(list, i) needs a list in a variable (this one would be thrown away)")
+            fail(g, m.kids[1], n + "(list, i) needs a list in a variable (this one would be thrown away)")
+        }
+        if n == "store_part" && dt.kind != "array" {
+            fail(g, m.kids[1], "store_part(list, i) needs a list: a ptr doesn't know where its numbers end")
         }
         i := coerce(g, expr(g, m.kids[2]), T("int"), m.kids[2], "an index").v
         data := ""
@@ -7021,13 +7127,615 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
             }
         } else {
             av := load(g, "{ ptr, i64, i64 }", dst.pl.addr)
-            vecBounds(g, i, vt.count, op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1"), m)
+            len := op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1")
             data = op(g, "extractvalue { ptr, i64, i64 } " + av + ", 0")
+            if n == "store_part" {
+                // only the lanes that are in the list are written
+                mk := tailMask(g, i, vt.count, len, m)
+                ep := op(g, "getelementptr " + llty(g, lane) + ", ptr " + data + ", i64 " + i)
+                ml := llty(g, shaped(vt, "bool"))
+                fname := "@llvm.masked.store." + vecSuffix(vt) + ".p0"
+                declareOnce(g, "declare void " + fname + "(" + lt + ", ptr, i32, " + ml + ")")
+                emit(g, "call void " + fname + "(" + lt + " " + v.v + ", ptr " + ep + ", i32 " + laneBytes(vt) + ", " + ml + " " + mk + ")")
+                return Val(ty: T("void"))
+            }
+            vecBounds(g, i, vt.count, len, m)
         }
         ep := op(g, "getelementptr inbounds " + llty(g, lane) + ", ptr " + data + ", i64 " + i)
         emit(g, "store " + lt + " " + v.v + ", ptr " + ep + ", align " + laneBytes(vt))
         return Val(ty: T("void"))
     }
-    fail(g, m, "a " + ast.show(vt) + " has no method '" + n + "' (it has: sum, min, max, dot, abs, sqrt, floor, ceil, round, fma, store)")
+    if n == "shuffle" || n == "lo" || n == "hi" || n == "lookup" {
+        return shuffleMethod(g, m, v)
+    }
+    if n == "dot_add" {
+        return dotAdd(g, m, v)
+    }
+    fail(g, m, "a " + ast.show(vt) + " has no method '" + n + "' (it has: sum, min, max, dot, abs, sqrt, floor, ceil, round, fma, muladd, store, store_part, shuffle, lo, hi, lookup, dot_add)")
     return v
 }
+
+// ---------- fused multiply-add, and numbers as bits ----------
+//
+// Finch never fuses on its own: a * b + c is a multiply and an add, two roundings, on every processor and with every
+// --cpu (the IR has no fast-math flags, and clang compiles it with -ffp-contract=on, which fuses only what asks to be).
+// Fusing is asked for by name:
+//   fma(a, b, c)      one rounding, always (one instruction with FMA hardware, a library call without it)
+//   muladd(a, b, c)   whichever is faster here: fused where the processor can, a multiply and an add where it can't
+
+// the llvm.fma / llvm.fmuladd call for a number or a vector type
+fn fusedOp(ptr[Gen] g, str name, ast.Type t, str a, str b, str c) -> str {
+    lt := llty(g, t)
+    which := "fma"
+    if name == "muladd" {
+        which = "fmuladd"
+    }
+    return vecIntrinsic(g, lt, which, t, [lt, lt, lt], [a, b, c], "")
+}
+
+fn fmaCall(ptr[Gen] g, ast.Node c) -> Val {
+    n := c.name
+    if g.index {
+        noteRef(g, c, c.col, n.len, docs.hover(docs.find(docs.builtins(), n)), "", 0, 0)
+    }
+    checkArgs(g, c, 0, 3, n)
+    []Val vals
+    for a in c.kids {
+        v := expr(g, a)
+        if !isNumber(v.ty) && v.ty.kind != "vec" {
+            fail(g, a, n + "(...) is for numbers, but this is " + ast.show(v.ty))
+        }
+        vals.push(v)
+    }
+    // the type all three take: a vector if one is, else the float type they fit
+    t := T("void")
+    for v in vals {
+        if v.ty.kind == "vec" {
+            t = v.ty
+        }
+    }
+    if t.kind == "vec" {
+        if !isFloat(t.elem[0]) {
+            fail(g, c, n + "(...) is for f32 and f64 lanes, but this is " + ast.show(t))
+        }
+        []str vs
+        for i in 0..3 {
+            vs.push(asVec(g, vals[i], t, n, c.kids[i]).v)
+        }
+        return Val(v: fusedOp(g, n, t, vs[0], vs[1], vs[2]), ty: t)
+    }
+    // f32 when an f32 takes part and nothing wider does (numbers written in the code adapt), else float
+    narrow := false
+    wide := false
+    for v in vals {
+        if v.ty.kind == "f32" {
+            narrow = true
+        } else if !v.lit {
+            wide = true
+        }
+    }
+    t = T("float")
+    if narrow && !wide {
+        t = T("f32")
+    }
+    []str vs
+    for i in 0..3 {
+        vs.push(coerce(g, vals[i], t, c.kids[i], "each number of " + n + "(...)").v)
+    }
+    return Val(v: fusedOp(g, n, t, vs[0], vs[1], vs[2]), ty: t)
+}
+
+// the same shape as t (a number, or a vector of as many lanes) with lanes of another kind
+fn shaped(ast.Type t, str lane) -> ast.Type {
+    if t.kind != "vec" {
+        return T(lane)
+    }
+    out := ast.Type(kind: "vec", count: t.count)
+    out.elem.push(T(lane))
+    return out
+}
+
+// a constant in every lane of t (or just the number)
+fn konst(ptr[Gen] g, ast.Type t, str n) -> str {
+    if t.kind != "vec" {
+        return n
+    }
+    return splat(g, n, t)
+}
+
+// f32_bits(x) -> u32 and back, f64 the same; f16 and bf16 (as the u16 that holds them) to f32 and back.
+// Each works on a number or on a vector, lane by lane: f16_to_f32(u16x8) is an f32x8.
+fn bitsCall(ptr[Gen] g, ast.Node c) -> Val {
+    n := c.name
+    if g.index {
+        noteRef(g, c, c.col, n.len, docs.hover(docs.find(docs.builtins(), n)), "", 0, 0)
+    }
+    checkArgs(g, c, 0, 1, n)
+    v := expr(g, c.kids[0])
+    from := ""          // the lane type it takes
+    to := ""
+    if n == "f32_bits" || n == "f32_to_f16" || n == "f32_to_bf16" {
+        from = "f32"
+        to = "u32"
+        if n != "f32_bits" {
+            to = "u16"
+        }
+    } else if n == "f64_bits" {
+        from = "float"
+        to = "u64"
+    } else if n == "f32_from_bits" {
+        from = "u32"
+        to = "f32"
+    } else if n == "f64_from_bits" {
+        from = "u64"
+        to = "float"
+    } else {
+        from = "u16"
+        to = "f32"
+    }
+    lane := v.ty
+    if v.ty.kind == "vec" {
+        lane = v.ty.elem[0]
+    }
+    // the bits may come as a signed number of the same size (i32 for u32), and a number written in the code fits
+    if isInt(T(from)) && isInt(lane) && bitsOf(lane) == bitsOf(T(from)) {
+        v = Val(v: v.v, ty: shaped(v.ty, from))
+    } else if v.ty.kind != "vec" && v.lit {
+        v = coerce(g, v, T(from), c.kids[0], "the value of " + n + "(...)")
+    } else if lane.kind != from {
+        fail(g, c.kids[0], n + "(...) takes " + ast.show(T(from)) + " (or a vector of them), but this is " + ast.show(v.ty))
+    }
+    src := shaped(v.ty, from)
+    out := shaped(v.ty, to)
+    il := llty(g, src)
+    ol := llty(g, out)
+    if n == "f32_bits" || n == "f64_bits" || n == "f32_from_bits" || n == "f64_from_bits" {
+        return Val(v: op(g, "bitcast " + il + " " + v.v + " to " + ol), ty: out)
+    }
+    if n == "bf16_to_f32" {
+        // bf16 is the top half of an f32
+        wide := shaped(v.ty, "u32")
+        wl := llty(g, wide)
+        z := op(g, "zext " + il + " " + v.v + " to " + wl)
+        sh := op(g, "shl " + wl + " " + z + ", " + konst(g, wide, "16"))
+        return Val(v: op(g, "bitcast " + wl + " " + sh + " to " + ol), ty: out)
+    }
+    if n == "f32_to_bf16" {
+        // the top half, rounded to the nearest (ties to even); a NaN stays a NaN
+        wide := shaped(v.ty, "u32")
+        wl := llty(g, wide)
+        u := op(g, "bitcast " + il + " " + v.v + " to " + wl)
+        hi := op(g, "lshr " + wl + " " + u + ", " + konst(g, wide, "16"))
+        odd := op(g, "and " + wl + " " + hi + ", " + konst(g, wide, "1"))
+        bias := op(g, "add " + wl + " " + odd + ", " + konst(g, wide, "32767"))
+        rounded := op(g, "lshr " + wl + " " + op(g, "add " + wl + " " + u + ", " + bias) + ", " + konst(g, wide, "16"))
+        quiet := op(g, "or " + wl + " " + hi + ", " + konst(g, wide, "64"))
+        isNan := op(g, "fcmp uno " + il + " " + v.v + ", " + v.v)
+        pickd := op(g, "select " + llty(g, shaped(v.ty, "bool")) + " " + isNan + ", " + wl + " " + quiet + ", " + wl + " " + rounded)
+        return Val(v: op(g, "trunc " + wl + " " + pickd + " to " + ol), ty: out)
+    }
+    // f16: LLVM's half type (one instruction with F16C or on ARM64, a small library function elsewhere)
+    hl := "half"
+    if v.ty.kind == "vec" {
+        hl = "<" + str(v.ty.count) + " x half>"
+    }
+    if n == "f16_to_f32" {
+        h := op(g, "bitcast " + il + " " + v.v + " to " + hl)
+        return Val(v: op(g, "fpext " + hl + " " + h + " to " + ol), ty: out)
+    }
+    h := op(g, "fptrunc " + il + " " + v.v + " to " + hl)
+    return Val(v: op(g, "bitcast " + hl + " " + h + " to " + ol), ty: out)
+}
+
+// ---------- masks: boolxN ----------
+//
+// A comparison of vectors gives a mask, one true/false a lane (<N x i1>). It picks lanes with select, says
+// whether any or all lanes are true, and loads and stores the end of a list (f32x8(list, i, fill), store_part).
+
+// lanes 0, 1, ... n-1 that are inside a list of `len` from index i (i may be past the end: then none)
+fn tailMask(ptr[Gen] g, str i, int n, str len, ast.Node at) -> str {
+    panicIf(g, op(g, "icmp slt i64 " + i + ", 0"), "a vector's index is below 0", at)
+    rest := op(g, "sub i64 " + len + ", " + i)
+    []str lanes
+    for k in 0..n {
+        lanes.push("i64 " + str(k))
+    }
+    wide := ast.Type(kind: "vec", count: n)
+    wide.elem.push(T("int"))
+    return op(g, "icmp slt <" + str(n) + " x i64> <" + lanes.join(", ") + ">, " + splat(g, rest, wide))
+}
+
+// select(mask, a, b): a's lane where the mask is true, b's where it is false
+fn selectCall(ptr[Gen] g, ast.Node c) -> Val {
+    if g.index {
+        noteRef(g, c, c.col, c.name.len, docs.hover(docs.find(docs.builtins(), "select")), "", 0, 0)
+    }
+    checkArgs(g, c, 0, 3, "select")
+    m := expr(g, c.kids[0])
+    if m.ty.kind != "vec" || m.ty.elem[0].kind != "bool" {
+        fail(g, c.kids[0], "select(mask, a, b) takes a mask first, from a comparison of vectors like v > 0.0; this is " + ast.show(m.ty))
+    }
+    a := expr(g, c.kids[1])
+    b := expr(g, c.kids[2])
+    vt := a.ty
+    if vt.kind != "vec" {
+        vt = b.ty
+    }
+    if vt.kind != "vec" {
+        fail(g, c, "select(mask, a, b): a or b has to be a vector (the other may be a number for every lane)")
+    }
+    if vt.count != m.ty.count {
+        fail(g, c, "the mask has " + str(m.ty.count) + " lanes, but " + ast.show(vt) + " has " + str(vt.count))
+    }
+    av := asVec(g, a, vt, "select", c.kids[1])
+    bv := asVec(g, b, vt, "select", c.kids[2])
+    lt := llty(g, vt)
+    return Val(v: op(g, "select " + llty(g, m.ty) + " " + m.v + ", " + lt + " " + av.v + ", " + lt + " " + bv.v), ty: vt)
+}
+
+fn maskMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
+    n := m.name
+    ml := llty(g, v.ty)
+    if n == "any" || n == "all" {
+        need(g, m, 0)
+        how := "or"
+        if n == "all" {
+            how = "and"
+        }
+        return Val(v: vecIntrinsic(g, "i1", "vector.reduce." + how, v.ty, [ml], [v.v], ""), ty: T("bool"))
+    }
+    if n == "count" {
+        need(g, m, 0)
+        wide := shaped(v.ty, "i32")
+        wl := llty(g, wide)
+        z := op(g, "zext " + ml + " " + v.v + " to " + wl)
+        sum := vecIntrinsic(g, "i32", "vector.reduce.add", wide, [wl], [z], "")
+        return Val(v: op(g, "zext i32 " + sum + " to i64"), ty: T("int"))
+    }
+    fail(g, m, "a mask (" + ast.show(v.ty) + ") has .len, [i] and the methods any, all, count; pick lanes with select(mask, a, b)")
+    return v
+}
+
+// ---------- moving lanes around ----------
+
+fn indexList([]int idx) -> str {
+    []str parts
+    for k in idx {
+        parts.push("i32 " + str(k))
+    }
+    return "<" + parts.join(", ") + ">"
+}
+
+fn shuffleOf(ptr[Gen] g, Val v, []int idx, str v2) -> Val {
+    vt := v.ty
+    out := ast.Type(kind: "vec", count: idx.len)
+    out.elem.push(vt.elem[0])
+    second := v2
+    if second.len == 0 {
+        second = "poison"
+    }
+    return Val(v: op(g, "shufflevector " + llty(g, vt) + " " + v.v + ", " + llty(g, vt) + " " + second + ", <" + str(idx.len) + " x i32> " + indexList(idx)), ty: out)
+}
+
+fn fitsVector(ptr[Gen] g, ast.Node at, ast.Type lane, int n) {
+    if n != 2 && n != 4 && n != 8 && n != 16 && n != 32 && n != 64 {
+        fail(g, at, "a vector has 2, 4, 8, 16, 32 or 64 lanes, not " + str(n))
+    }
+    bits := bitsOf(lane)
+    if n * bits > 512 {
+        fail(g, at, "that would be " + str(n) + " lanes of " + ast.show(lane) + ", more than 512 bits")
+    }
+}
+
+// v.shuffle(3, 2, 1, 0): lanes picked by number (numbers written in the code); v.lo(), v.hi(): the halves;
+// table.lookup(idx): table[idx % table.len] for every lane of idx (pshufb on x86, tbl on ARM)
+fn shuffleMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
+    vt := v.ty
+    n := m.name
+    if n == "lo" || n == "hi" {
+        need(g, m, 0)
+        if vt.count < 4 {
+            fail(g, m, "a " + ast.show(vt) + " has only " + str(vt.count) + " lanes; take them with v[0] and v[1]")
+        }
+        []int idx
+        start := 0
+        if n == "hi" {
+            start = vt.count / 2
+        }
+        for k in 0..vt.count / 2 {
+            idx.push(start + k)
+        }
+        return shuffleOf(g, v, idx, "")
+    }
+    if n == "shuffle" {
+        []int idx
+        for q in 1..m.kids.len {
+            x := expr(g, m.kids[q])
+            if !isInt(x.ty) || !x.lit {
+                fail(g, m.kids[q], "shuffle(...) takes lane numbers written in the code (or constants), like v.shuffle(3, 2, 1, 0)")
+            }
+            k := int(x.v) or -1
+            if k < 0 || k >= vt.count {
+                fail(g, m.kids[q], "lane " + x.v + " isn't in a " + ast.show(vt) + " (its lanes are 0 to " + str(vt.count - 1) + ")")
+            }
+            idx.push(k)
+        }
+        fitsVector(g, m, vt.elem[0], idx.len)
+        return shuffleOf(g, v, idx, "")
+    }
+    // lookup
+    need(g, m, 1)
+    ix := expr(g, m.kids[1])
+    if ix.ty.kind != "vec" || !isInt(ix.ty.elem[0]) {
+        fail(g, m.kids[1], "lookup(indices) takes a vector of whole numbers (like u8x16), but this is " + ast.show(ix.ty))
+    }
+    if vt.elem[0].kind == "bool" {
+        fail(g, m, "a mask can't be a lookup table")
+    }
+    fitsVector(g, m.kids[1], vt.elem[0], ix.ty.count)
+    il := llty(g, ix.ty)
+    masked := op(g, "and " + il + " " + ix.v + ", " + splat(g, str(vt.count - 1), ix.ty))
+    out := shaped(ix.ty, vt.elem[0].kind)
+    ol := llty(g, out)
+    el := llty(g, vt.elem[0])
+    xl := llty(g, ix.ty.elem[0])
+    cur := "poison"
+    for k in 0..ix.ty.count {
+        at := op(g, "extractelement " + il + " " + masked + ", i64 " + str(k))
+        lane := op(g, "extractelement " + llty(g, vt) + " " + v.v + ", " + xl + " " + at)
+        cur = op(g, "insertelement " + ol + " " + cur + ", " + el + " " + lane + ", i64 " + str(k))
+    }
+    return Val(v: cur, ty: out)
+}
+
+// concat(a, b): a's lanes, then b's
+fn concatCall(ptr[Gen] g, ast.Node c) -> Val {
+    if g.index {
+        noteRef(g, c, c.col, c.name.len, docs.hover(docs.find(docs.builtins(), "concat")), "", 0, 0)
+    }
+    checkArgs(g, c, 0, 2, "concat")
+    a := expr(g, c.kids[0])
+    b := expr(g, c.kids[1])
+    if a.ty.kind != "vec" || !ast.same(a.ty, b.ty) {
+        fail(g, c, "concat(a, b) joins two vectors of the same type, but these are " + ast.show(a.ty) + " and " + ast.show(b.ty))
+    }
+    fitsVector(g, c, a.ty.elem[0], a.ty.count * 2)
+    []int idx
+    for k in 0..a.ty.count * 2 {
+        idx.push(k)
+    }
+    return shuffleOf(g, a, idx, b.v)
+}
+
+// ---------- processor features: @cpu(...), cpu_has(...) ----------
+//
+// --cpu gives the whole program a processor (main.fch asks clang which features that means); @cpu("avx2,fma") gives one
+// function more, as LLVM's "target-features". Such a function is only called after cpu_has("avx2") said yes. Features of
+// the other architecture are dropped, so one source holds the x86 and the ARM versions side by side.
+
+fn x86Features() -> []str {
+    return ["sse3", "ssse3", "sse4.1", "sse4.2", "popcnt", "avx", "avx2", "fma", "f16c", "bmi1", "bmi2", "avx512f", "avx512bw",
+            "avx512dq", "avx512vl", "avx512vnni", "avx512bf16", "avx512fp16", "avxvnni"]
+}
+
+fn armFeatures() -> []str {
+    return ["neon", "fp16", "dotprod", "i8mm", "bf16", "sve", "sve2"]
+}
+
+// the name LLVM uses for a feature
+fn llvmFeature(str x) -> str {
+    if x == "bmi1" {
+        return "bmi"
+    }
+    if x == "fp16" {
+        return "fullfp16"
+    }
+    return x
+}
+
+// the features of a level: x86-64-v3 is the AVX2 generation, v4 the AVX-512 one
+fn levelFeatures(str name) -> []str {
+    []str out
+    if name == "x86-64-v2" || name == "x86-64-v3" || name == "x86-64-v4" {
+        for x in ["sse3", "ssse3", "sse4.1", "sse4.2", "popcnt"] {
+            out.push(x)
+        }
+    }
+    if name == "x86-64-v3" || name == "x86-64-v4" {
+        for x in ["avx", "avx2", "bmi1", "bmi2", "f16c", "fma"] {
+            out.push(x)
+        }
+    }
+    if name == "x86-64-v4" {
+        for x in ["avx512f", "avx512bw", "avx512dq", "avx512vl"] {
+            out.push(x)
+        }
+    }
+    return out
+}
+
+// a function's own @cpu features, for the architecture it is built for
+fn cpuOf(ptr[Gen] g, ast.Node f) -> []str {
+    []str out
+    if f.cpu.len == 0 {
+        return out
+    }
+    for part in f.cpu.split(",") {
+        x := part.trim()
+        names := levelFeatures(x)
+        if names.len == 0 {
+            if !x86Features().contains(x) && !armFeatures().contains(x) {
+                fail(g, f, "@cpu: '" + x + "' isn't a feature Finch knows (x86: " + x86Features().join(", ") + ", x86-64-v2/v3/v4; ARM: " + armFeatures().join(", ") + ")")
+            }
+            names.push(x)
+        }
+        for y in names {
+            if armFeatures().contains(y) == g.tg.arm64 && !out.contains(y) {
+                out.push(y)
+            }
+        }
+    }
+    return out
+}
+
+// a vector wider than 16 bytes travels in different registers depending on the features (two xmm, or one ymm), so it
+// can't cross from a function with one set of features to a function with another
+fn checkCpuCall(ptr[Gen] g, FnSig sig, ast.Node c, str shown) {
+    if sig.prog < 0 || sig.prog >= g.progs.len || sig.index >= g.progs[sig.prog].fns.len {
+        return
+    }
+    callee := g.progs[sig.prog].fns[sig.index].cpu
+    mine := g.progs[g.cur.prog].fns[g.cur.index].cpu
+    if callee == mine {
+        return
+    }
+    []ast.Type ts = sig.params
+    ts.push(sig.ret)
+    for t in ts {
+        if t.kind == "vec" && t.count * bitsOf(t.elem[0]) > 128 {
+            fail(g, c, "'" + shown + "' has other @cpu features than this function, so a " + ast.show(t) + " can't be passed between them (its registers differ); pass a list or a ptr, or give both the same @cpu")
+        }
+    }
+}
+
+// cpu_has("avx2"): does the processor running the program have it? (false for the other architecture's names)
+fn cpuHasCall(ptr[Gen] g, ast.Node c) -> Val {
+    if g.index {
+        noteRef(g, c, c.col, c.name.len, docs.hover(docs.find(docs.builtins(), "cpu_has")), "", 0, 0)
+    }
+    checkArgs(g, c, 0, 1, "cpu_has")
+    if c.kids[0].kind == "str" {
+        x := c.kids[0].text
+        if !x86Features().contains(x) && !armFeatures().contains(x) {
+            fail(g, c.kids[0], "cpu_has: '" + x + "' isn't a feature Finch knows (x86: " + x86Features().join(", ") + "; ARM: " + armFeatures().join(", ") + ")")
+        }
+    }
+    s := strArg(g, c.kids[0], "the feature's name")
+    r := op(g, "call i32 @finch_cpu_has(ptr " + spill(g, s) + ")")
+    release(g, s)
+    return Val(v: op(g, "icmp ne i32 " + r + ", 0"), ty: T("bool"))
+}
+
+// ---------- acc.dot_add(a, b): the step of an int8 dot product ----------
+//
+// acc is i32xN; a and b have 4N bytes (a u8 or i8, b i8 or u8). Lane k of acc gains a[4k]*b[4k] + ... + a[4k+3]*b[4k+3],
+// exactly (no saturation). That is vpdpbusd (AVX-VNNI, AVX-512 VNNI) and usdot / sdot / udot (ARM), used when the
+// function's features have them; elsewhere the same sum is written out, which LLVM makes into pmaddwd and adds.
+fn dotAdd(ptr[Gen] g, ast.Node m, Val acc) -> Val {
+    need(g, m, 2)
+    at := acc.ty
+    if at.elem[0].kind != "i32" && at.elem[0].kind != "u32" {
+        fail(g, m, "dot_add is called on an i32 vector (the sums), like acc.dot_add(a, b) with acc an i32x8; this is " + ast.show(at))
+    }
+    a := expr(g, m.kids[1])
+    b := expr(g, m.kids[2])
+    for x in [a, b] {
+        if x.ty.kind != "vec" || (x.ty.elem[0].kind != "u8" && x.ty.elem[0].kind != "i8") || x.ty.count != at.count * 4 {
+            fail(g, m, "dot_add(a, b) on an " + ast.show(at) + " takes two vectors of " + str(at.count * 4) + " bytes (u8 or i8), but these are " + ast.show(a.ty) + " and " + ast.show(b.ty))
+        }
+    }
+    au := a.ty.elem[0].kind == "u8"
+    bu := b.ty.elem[0].kind == "u8"
+    if !au && bu {
+        // u8 x i8 is the same sum as i8 x u8: put the unsigned one first
+        t := a
+        a = b
+        b = t
+        au = true
+        bu = false
+    }
+    al := llty(g, at)
+    bits := at.count * 32
+    f := g.curFeatures
+    if !g.tg.arm64 && au && !bu {
+        vnni := (bits == 512 && f.contains("avx512vnni")) || (bits <= 256 && (f.contains("avxvnni") || (f.contains("avx512vnni") && f.contains("avx512vl"))))
+        if vnni {
+            a32 := op(g, "bitcast " + llty(g, a.ty) + " " + a.v + " to " + al)
+            b32 := op(g, "bitcast " + llty(g, b.ty) + " " + b.v + " to " + al)
+            name := "@llvm.x86.avx512.vpdpbusd." + str(bits)
+            declareOnce(g, "declare " + al + " " + name + "(" + al + ", " + al + ", " + al + ")")
+            return Val(v: op(g, "call " + al + " " + name + "(" + al + " " + acc.v + ", " + al + " " + a32 + ", " + al + " " + b32 + ")"), ty: at)
+        }
+    }
+    if g.tg.arm64 && bits % 128 == 0 {
+        which := ""
+        if au && !bu && f.contains("i8mm") {
+            which = "usdot"
+        } else if au && bu && f.contains("dotprod") {
+            which = "udot"
+        } else if !au && !bu && f.contains("dotprod") {
+            which = "sdot"
+        }
+        if which.len > 0 {
+            // one instruction per 16 bytes
+            name := "@llvm.aarch64.neon." + which + ".v4i32.v16i8"
+            declareOnce(g, "declare <4 x i32> " + name + "(<4 x i32>, <16 x i8>, <16 x i8>)")
+            parts := at.count / 4
+            []str outs
+            for q in 0..parts {
+                []int ai
+                for k in 0..4 {
+                    ai.push(q * 4 + k)
+                }
+                []int bi
+                for k in 0..16 {
+                    bi.push(q * 16 + k)
+                }
+                ac := shuffleOf(g, acc, ai, "").v
+                ap := shuffleOf(g, a, bi, "").v
+                bp := shuffleOf(g, b, bi, "").v
+                outs.push(op(g, "call <4 x i32> " + name + "(<4 x i32> " + ac + ", <16 x i8> " + ap + ", <16 x i8> " + bp + ")"))
+            }
+            return Val(v: joinParts(g, outs, 4), ty: at)
+        }
+    }
+    // written out: widen, multiply, add each group of four
+    w := shaped(a.ty, "i32")
+    wl := llty(g, w)
+    ext := "sext"
+    if au {
+        ext = "zext"
+    }
+    aw := op(g, ext + " " + llty(g, a.ty) + " " + a.v + " to " + wl)
+    ext = "sext"
+    if bu {
+        ext = "zext"
+    }
+    bw := op(g, ext + " " + llty(g, b.ty) + " " + b.v + " to " + wl)
+    prod := Val(v: op(g, "mul " + wl + " " + aw + ", " + bw), ty: w)
+    sum := acc.v
+    for k in 0..4 {
+        []int idx
+        for j in 0..at.count {
+            idx.push(4 * j + k)
+        }
+        sum = op(g, "add " + al + " " + sum + ", " + shuffleOf(g, prod, idx, "").v)
+    }
+    return Val(v: sum, ty: at)
+}
+
+// <4 x i32> pieces back into one vector of n * 4 lanes
+fn joinParts(ptr[Gen] g, []str parts, int each) -> str {
+    if parts.len == 1 {
+        return parts[0]
+    }
+    []str cur = parts
+    size := each
+    while cur.len > 1 {
+        []str next
+        t := ast.Type(kind: "vec", count: size)
+        t.elem.push(T("i32"))
+        k := 0
+        while k < cur.len {
+            []int idx
+            for j in 0..size * 2 {
+                idx.push(j)
+            }
+            next.push(shuffleOf(g, Val(v: cur[k], ty: t), idx, cur[k + 1]).v)
+            k += 2
+        }
+        cur = next
+        size = size * 2
+    }
+    return cur[0]
+}

src/lexer.fch

@@ -70,7 +70,7 @@ fn operator(ptr[Lexer] lx) -> str {
             return two
         }
     }
-    ones := "(){}[],.:+-*/%&|^~<>=!"
+    ones := "(){}[],.:+-*/%&|^~<>=!@"
     c := lx.src.sub(lx.i, lx.i + 1)
     if ones.contains(c) {
         return c

src/linker.fch

@@ -371,10 +371,7 @@ struct Build {
 
 // IR -> program. 0 if it worked.
 fn build(Build b, platform.Target tg, cimport.CImports ci) -> int {
-    clang := getenv("FINCH_CLANG")
+    clang := clangCmd()
-    if clang.len == 0 {
-        clang = "clang"
-    }
     // the C compiler of the runtime and the linker: the target's own when building for another system
     cc := clang
     if tg.cross {
@@ -391,7 +388,8 @@ fn build(Build b, platform.Target tg, cimport.CImports ci) -> int {
         target = " --target=" + tg.triple
     }
     log := b.out + ".log"
-    obj := irObject(clang, target + " " + b.opt + b.extra, b.ir, b.name + "\n" + tg.triple, log) or {
+    // -ffp-contract=on: only what asks to be fused is (muladd); a * b + c stays two roundings, whatever clang's default
+    obj := irObject(clang, target + " " + b.opt + " -ffp-contract=on" + b.extra, b.ir, b.name + "\n" + tg.triple, log) or {
         print(err)
         host.remove(log)
         return 1
@@ -477,3 +475,74 @@ fn build(Build b, platform.Target tg, cimport.CImports ci) -> int {
     }
     return 0
 }
+
+// clang, as the program is compiled with it. Nix's clang wrapper removes -march=native when NIX_ENFORCE_NO_NATIVE
+// is 1, as it is in nix-shell; --cpu native asks for it on purpose, so the wrapper is told to keep it.
+fn clangCmd() -> str {
+    clang := getenv("FINCH_CLANG")
+    if clang.len == 0 {
+        clang = "clang"
+    }
+    if host.pathSeparator() != ";" {
+        return "NIX_ENFORCE_NO_NATIVE=0 " + clang
+    }
+    return clang
+}
+
+// -march=x86-64-v3 or -mcpu=apple-m1: how clang is told the processor
+fn cpuFlag(platform.Target tg, str cpu) -> str {
+    if cpu.len == 0 {
+        return ""
+    }
+    if tg.arm64 {
+        return " -mcpu=" + quote(cpu)
+    }
+    return " -march=" + quote(cpu)
+}
+
+// What the processor the program is for can do, as Finch's feature names (avx2, avxvnni, i8mm ...): clang says it in
+// the macros it predefines for that target and --cpu. Kept in the cache, one file per clang, target and cpu.
+fn cpuFeatures(platform.Target tg, str cpu) -> []str {
+    clang := clangCmd()
+    flags := cpuFlag(tg, cpu)
+    if tg.cross || tg.windows {
+        flags = " --target=" + tg.triple + flags
+    }
+    dir := host.cacheDir() + "/cpu"
+    host.makeDirs(dir)
+    slot := dir + "/" + fingerprint2(clang + flags)
+    text := read_file(slot + ".txt") or ""
+    if text.len == 0 {
+        empty := slot + ".part" + str(host.pid()) + ".c"
+        out := slot + ".part" + str(host.pid()) + ".txt"
+        write_file(empty, "")
+        host.run(clang + flags + " -dM -E " + quote(empty) + " > " + quote(out) + " 2> " + host.nullDevice())
+        host.remove(empty)
+        text = read_file(out) or ""
+        if text.len == 0 {
+            host.remove(out)
+            []str none
+            return none
+        }
+        host.move(out, slot + ".txt")
+    }
+    macros := ["__SSE3__", "__SSSE3__", "__SSE4_1__", "__SSE4_2__", "__POPCNT__", "__AVX__", "__AVX2__", "__FMA__", "__F16C__",
+               "__BMI__", "__BMI2__", "__AVX512F__", "__AVX512BW__", "__AVX512DQ__", "__AVX512VL__", "__AVX512VNNI__",
+               "__AVX512BF16__", "__AVX512FP16__", "__AVXVNNI__", "__ARM_NEON", "__ARM_FEATURE_FP16_VECTOR_ARITHMETIC",
+               "__ARM_FEATURE_DOTPROD", "__ARM_FEATURE_MATMUL_INT8", "__ARM_FEATURE_BF16_VECTOR_ARITHMETIC", "__ARM_FEATURE_SVE",
+               "__ARM_FEATURE_SVE2"]
+    names := ["sse3", "ssse3", "sse4.1", "sse4.2", "popcnt", "avx", "avx2", "fma", "f16c", "bmi1", "bmi2", "avx512f", "avx512bw",
+              "avx512dq", "avx512vl", "avx512vnni", "avx512bf16", "avx512fp16", "avxvnni", "neon", "fp16", "dotprod", "i8mm", "bf16",
+              "sve", "sve2"]
+    []str have
+    for line in text.split("\n") {
+        parts := line.split(" ")
+        if parts.len >= 2 && parts[0] == "#define" {
+            k := macros.find(parts[1])
+            if k >= 0 && !have.contains(names[k]) {
+                have.push(names[k])
+            }
+        }
+    }
+    return have
+}

src/main.fch

@@ -471,7 +471,15 @@ fn main([]str args) -> int {
             testModules.push(prog.module)       // your own files' tests, not the standard library's
         }
     }
-    ir := gen.generate(loader.progs, loader.files, ci, gen.Options(shared: shared, debug: debug, index: cmd == "index", cwd: host.cwd(), target: tg, test: cmd == "test", testModules: testModules, trace: trace))
+    // what the processor can do (dot_add and the like pick their instructions by it); not needed for the editor's index
+    []str features
+    if cmd != "index" {
+        features = linker.cpuFeatures(tg, cpu)
+        if cpu == "native" && !tg.arm64 && !features.contains("sse4.2") {
+            print("warning: --cpu native made no difference to clang (a clang wrapper may remove -march=native; try --cpu with a name, like x86-64-v3)")
+        }
+    }
+    ir := gen.generate(loader.progs, loader.files, ci, gen.Options(shared: shared, debug: debug, index: cmd == "index", cwd: host.cwd(), target: tg, test: cmd == "test", testModules: testModules, trace: trace, features: features))
     if cmd == "ir" || cmd == "index" {
         print(ir)
         return 0
@@ -501,14 +509,7 @@ fn main([]str args) -> int {
     if debug {
         extra = extra + " -g"      // or clang drops the debug info that is in the IR
     }
-    if cpu.len > 0 {
+    extra = extra + linker.cpuFlag(tg, cpu)          // x86-64 names a processor with -march, ARM with -mcpu
-        // x86-64 names a processor with -march, ARM with -mcpu
-        if tg.arm64 {
-            extra = extra + " -mcpu=" + linker.quote(cpu)
-        } else {
-            extra = extra + " -march=" + linker.quote(cpu)
-        }
-    }
     []str cflagList
     for d in dirs {
         cflagList.push(" -I" + linker.quote(d))      // also the program's own folders (its headers, for inline C functions)

src/parser.fch

@@ -289,6 +289,22 @@ fn parseProgram(ptr[Parser] p) -> ast.Program {
             if e.flag {
                 prog.structs.push(sumStruct(e))       // an enum with values inside is stored as a struct (see sumStruct)
             }
+        } else if at(p, "@") {
+            // @cpu("avx2,fma") fn ...: a function that may use these processor features (check with cpu_has first)
+            a := next(p)
+            name := expect(p, "ident", "an attribute name, like @cpu(\"avx2\")").text
+            if name != "cpu" {
+                fail(p, a, "the attribute here is @cpu(\"...\"): the processor features a function may use")
+            }
+            expect(p, "(", "'('")
+            feats := expect(p, "string", "the features in quotes, like @cpu(\"avx2,fma\")").text
+            expect(p, ")", "')'")
+            if !at(p, "fn") && !at(p, "export") {
+                fail(p, cur(p), "@cpu(...) goes right before a function")
+            }
+            f := parseFn(p)
+            f.cpu = feats
+            prog.fns.push(f)
         } else {
             prog.fns.push(parseFn(p))
         }

tests/fail/cpu_attr_name.fch

@@ -0,0 +1,7 @@
+// expect: @cpu: 'avx3' isn't a feature Finch knows
+@cpu("avx2,avx3")
+fn f() {
+}
+fn main() {
+    f()
+}

tests/fail/cpu_has_name.fch

@@ -0,0 +1,4 @@
+// expect: cpu_has: 'avx-512' isn't a feature Finch knows
+fn main() {
+    print(cpu_has("avx-512"))
+}

tests/fail/cpu_wide_vector.fch

@@ -0,0 +1,8 @@
+// expect: can't be passed between them (its registers differ)
+@cpu("avx2")
+fn double(f32x8 v) -> f32x8 {
+    return v * 2.0
+}
+fn main() {
+    print(double(f32x8(1.0)))
+}

tests/fail/vec_less.fch

@@ -1,5 +1,7 @@
-// expect: use min(v, w), max(v, w), or compare lanes: v[i] < w[i]
+// expect: use .all() or .any()
 fn main() {
     a := f32x4(1.0)
-    print(a < a)
+    if a < a {
+        print("no")
+    }
 }

tests/fail/vec_shuffle_range.fch

@@ -0,0 +1,5 @@
+// expect: lane 4 isn't in a f32x4 (its lanes are 0 to 3)
+fn main() {
+    v := f32x4(1.0)
+    print(v.shuffle(0, 1, 2, 4))
+}

tests/fail/vec_shuffle_var.fch

@@ -0,0 +1,6 @@
+// expect: shuffle(...) takes lane numbers written in the code
+fn main() {
+    v := f32x4(1.0)
+    k := 2
+    print(v.shuffle(k, 0, 1, 3))
+}

tests/run/dot_int8.fch

@@ -0,0 +1,69 @@
+// The int8 dot product of quantized models: plain, with dot_add, and in @cpu functions picked by cpu_has
+
+fn plain([]u8 a, []i8 b) -> int {
+    total := 0
+    for i in 0..a.len {
+        total += int(a[i]) * int(b[i])
+    }
+    return total
+}
+
+// 32 bytes a step, 8 sums kept in a vector until the end
+fn with_vectors([]u8 a, []i8 b) -> int {
+    acc := i32x8(0)
+    i := 0
+    while i + 32 <= a.len {
+        acc = acc.dot_add(u8x32(a, i), i8x32(b, i))
+        i += 32
+    }
+    return int(acc.sum())
+}
+
+@cpu("avxvnni")
+fn with_vnni([]u8 a, []i8 b) -> int {
+    acc := i32x8(0)
+    i := 0
+    while i + 32 <= a.len {
+        acc = acc.dot_add(u8x32(a, i), i8x32(b, i))
+        i += 32
+    }
+    return int(acc.sum())
+}
+
+@cpu("avx512f,avx512bw,avx512vl,avx512vnni,i8mm")
+fn with_avx512([]u8 a, []i8 b) -> int {
+    acc := i32x16(0)
+    i := 0
+    while i + 64 <= a.len {
+        acc = acc.dot_add(u8x64(a, i), i8x64(b, i))
+        i += 64
+    }
+    return int(acc.sum())
+}
+
+fn best([]u8 a, []i8 b) -> int {
+    if cpu_has("avx512vnni") && cpu_has("avx512vl") && cpu_has("avx512bw") {
+        return with_avx512(a, b)
+    }
+    if cpu_has("avxvnni") {
+        return with_vnni(a, b)
+    }
+    return with_vectors(a, b)
+}
+
+fn main() {
+    []u8 a
+    []i8 b
+    for k in 0..1024 {
+        a.push(u8((k * 37 + 11) % 256))
+        b.push(i8((k * 53 + 7) % 256 - 128))
+    }
+    want := plain(a, b)
+    print(want, with_vectors(a, b) == want, best(a, b) == want)
+
+    // the extremes: 255 * -128 four times in a lane, no saturation
+    acc := i32x4(1, 2, 3, 4).dot_add(u8x16(255), i8x16(-128))
+    print(acc, i32x4(0).dot_add(i8x16(-128), i8x16(-128)), i32x4(0).dot_add(u8x16(255), u8x16(255)))
+    print(i32x4(0).dot_add(i8x16(-1), u8x16(200)))
+    print(cpu_has("avx2") || cpu_has("neon") || true)
+}

tests/run/dot_int8.out

@@ -0,0 +1,4 @@
+-68096 true true
+i32x4(-130559, -130558, -130557, -130556) i32x4(65536, 65536, 65536, 65536) i32x4(260100, 260100, 260100, 260100)
+i32x4(-800, -800, -800, -800)
+true

tests/run/floatbits.fch

@@ -0,0 +1,31 @@
+// a * b + c is never fused; fma rounds once; numbers as their bits; f16 and bf16
+
+fn plain(float a, float b, float c) -> float {
+    return a * b + c
+}
+
+fn main() {
+    // 0.1 * 10 is 1.0000000000000000555..., which rounds to 1 before the - 1 (two roundings), but not inside fma
+    print(plain(0.1, 10.0, -1.0), fma(0.1, 10.0, -1.0))
+    print(fma(f32(0.1), 10.0, -1.0), fma(2, 3, 4))
+    m := muladd(1.5, 2.0, 0.25)          // fused or not, exact here
+    print(m, f32x4(1.5).muladd(2.0, 0.25), fma(f32x4(0.1), 10.0, f32x4(-1.0))[0])
+    det := f32x4(0.1) * f32x4(10.0) - f32x4(1.0)
+    print(det[0])
+
+    // bits
+    print(f32_bits(1.0), f64_bits(1.0), f32_bits(-0.0))
+    print(f32_from_bits(0x40490fdb), f64_from_bits(f64_bits(2.5)), f32_from_bits(i32(-1082130432)))
+    u32 inf = 0x7f800000
+    print(f32_from_bits(inf), f32_bits(f32x4(1.0, 2.0, -2.0, 0.5)))
+
+    // f16
+    print(f16_to_f32(0x3c00), f16_to_f32(0xc000), f16_to_f32(0x7bff), f16_to_f32(0x0001))
+    print(f32_to_f16(0.1), f32_to_f16(65504.0), f32_to_f16(1.0e6), f16_to_f32(f32_to_f16(0.333)))
+    print(f16_to_f32(u16x8(0x3c00, 0x4000, 0x4200, 0x4400, 0, 0x8000, 0x3800, 0x3400)))
+
+    // bf16
+    print(bf16_to_f32(0x3f80), bf16_to_f32(0x4049), f32_to_bf16(3.14159))
+    print(f32_to_bf16(f32_from_bits(0x3f808000)), f32_to_bf16(f32_from_bits(0x3f818000)), f32_to_bf16(f32_from_bits(0x7fc00001)))
+    print(f32_to_bf16(f32x4(1.0, -2.0, 0.5, 3.14159)), bf16_to_f32(u16x4(0x3f80, 0xc000, 0x3f00, 0x4049)))
+}

tests/run/floatbits.out

@@ -0,0 +1,13 @@
+0 5.551115123125783e-17
+1.4901161e-08 10
+3.25 f32x4(3.25, 3.25, 3.25, 3.25) 1.4901161e-08
+0
+1065353216 4607182418800017408 2147483648
+3.1415927 2.5 -1
+inf u32x4(1065353216, 1073741824, 3221225472, 1056964608)
+1 -2 65504 5.9604645e-08
+11878 31743 31744 0.3330078
+f32x8(1, 2, 3, 4, 0, -0, 0.5, 0.25)
+1 3.140625 16457
+16256 16258 32704
+u16x4(16256, 49152, 16128, 16457) f32x4(1, -2, 0.5, 3.140625)

tests/run/masks.fch

@@ -0,0 +1,50 @@
+// Masks: comparisons of vectors, select, any/all/count, and the end of a list
+
+fn relu(f32x8 v) -> f32x8 {
+    return select(v > 0.0, v, 0.0)
+}
+
+// the sum of a list of any length: whole vectors, then the last piece with its missing lanes as 0
+fn total([]f32 xs) -> f32 {
+    acc := f32x8(0.0)
+    i := 0
+    while i < xs.len {
+        acc = acc + f32x8(xs, i, 0.0)
+        i += 8
+    }
+    return acc.sum()
+}
+
+// add 1 to every number of a list, in place, the last piece too
+fn bump([]f32 xs) {
+    i := 0
+    while i < xs.len {
+        (f32x8(xs, i, 0.0) + 1.0).store_part(xs, i)
+        i += 8
+    }
+}
+
+fn main() {
+    v := f32x8(-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, -4.0, 5.0)
+    m := v > 0.0
+    print(m, m.count(), m.any(), m.all(), m[3])
+    print(relu(v), select(!m, v, -1.0))
+    print((v >= -1.0) & (v < 3.0), (v == 0.0) | (v == 5.0), m ^ (v > 1.0))
+
+    a := i32x4(1, 5, 3, 7)
+    b := i32x4(4, 4, 4, 4)
+    print(select(a > b, a, b), (a == a).all(), (a != b).all())
+    u := u8x16(200)
+    print((u > u8x16(100)).all(), i32x4(a > b), f32x4(a < b))
+    k := i32x4(0, 2, 0, -1)
+    print(boolx4(k), boolx4(true, false, true, false))
+
+    []f32 xs
+    for n in 0..13 {
+        xs.push(f32(n))
+    }
+    print(total(xs))
+    bump(xs)
+    print(xs)
+    print(f32x8(xs, 8, -1.0), f32x8(xs, 13, 9.0), f32x8(xs, 100, 7.0))
+}

tests/run/masks.out

@@ -0,0 +1,9 @@
+boolx8(false, false, false, true, true, true, false, true) 4 true false true
+f32x8(0, 0, 0, 1, 2, 3, 0, 5) f32x8(-2, -1, 0, -1, -1, -1, -4, -1)
+boolx8(false, true, true, true, true, false, false, false) boolx8(false, false, true, false, false, false, false, true) boolx8(false, false, false, true, false, false, false, false)
+i32x4(4, 5, 4, 7) true true
+true i32x4(0, 1, 0, 1) f32x4(1, 0, 1, 0)
+boolx4(false, true, false, true) boolx4(true, false, true, false)
+78
+[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
+f32x8(9, 10, 11, 12, 13, -1, -1, -1) f32x8(9, 9, 9, 9, 9, 9, 9, 9) f32x8(7, 7, 7, 7, 7, 7, 7, 7)

tests/run/shuffles.fch

@@ -0,0 +1,27 @@
+// Moving lanes: shuffle, lo/hi, concat, and a byte table lookup (how 4-bit weights are unpacked)
+
+// sixteen 4-bit numbers, two to a byte, made into sixteen bytes through a table of what each nibble means
+fn unpack4(u8x8 packed, i8x16 table) -> i8x16 {
+    pairs := packed.shuffle(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7)   // u8x16: each byte twice
+    odd := u8x16(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1) == 1
+    nibbles := select(odd, pairs >> 4, pairs & 15)   // low nibble first, then the high one
+    return table.lookup(nibbles)
+}
+
+fn main() {
+    v := f32x4(1.0, 2.0, 3.0, 4.0)
+    print(v.shuffle(3, 2, 1, 0), v.shuffle(0, 0, 1, 1, 2, 2, 3, 3), v.shuffle(1, 2))
+    w := i32x8(0, 1, 2, 3, 4, 5, 6, 7)
+    print(w.lo(), w.hi(), concat(w.hi(), w.lo()))
+    print(concat(w, w).len, concat(v, v * 10.0).shuffle(0, 4, 1, 5, 2, 6, 3, 7))
+
+    table := u8x16(10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25)
+    print(table.lookup(u8x16(15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14)))
+    print(table.lookup(u8x16(16, 17, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)).lo())
+    print(table.lookup(u8x32(3)).hi().lo().lo())
+
+    q4 := i8x16(-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7)       // what each 4-bit number means
+    print(unpack4(u8x8(0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe), q4))
+    m := f32x4(1.0, -1.0, 2.0, -2.0) > 0.0
+    print(m.shuffle(1, 0, 3, 2), m.lo())
+}

tests/run/shuffles.out

@@ -0,0 +1,8 @@
+f32x4(4, 3, 2, 1) f32x8(1, 1, 2, 2, 3, 3, 4, 4) f32x2(2, 3)
+i32x4(0, 1, 2, 3) i32x4(4, 5, 6, 7) i32x8(4, 5, 6, 7, 0, 1, 2, 3)
+16 f32x8(1, 10, 2, 20, 3, 30, 4, 40)
+u8x16(25, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24)
+u8x8(10, 11, 25, 10, 10, 10, 10, 10)
+u8x4(13, 13, 13, 13)
+i8x16(-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7)
+boolx4(false, true, false, true) boolx2(true, false)

tests/run/vectors.fch

@@ -14,7 +14,7 @@ fn dot([]f32 a, []f32 b) -> f32 {
     acc := f32x8(0.0)
     i := 0
     while i + 8 <= a.len {
-        acc = f32x8(a, i).fma(f32x8(b, i), acc)
+        acc = f32x8(a, i).muladd(f32x8(b, i), acc)
         i += 8
     }
     total := acc.sum()
@@ -36,7 +36,7 @@ fn main() {
     // lanes
     a[0] = 9.0
     print(a[0], a[3], a)
-    print(a == f32x4(9.0, 2.0, 3.0, 4.0), a != a)
+    print((a == f32x4(9.0, 2.0, 3.0, 4.0)).all(), (a != a).any())
 
     // whole numbers, bits, conversions
     n := i32x8(1, 2, 3, 4, 5, 6, 7, 8)