Vectors for kernels: masks and select, shuffles and byte lookup, dot_add, @cpu and cpu_has; --cpu native on Nix

- comparisons of vectors give masks (boolxN): select(m, a, b), m.any/all/count, & | ^ !;
  f32x8(list, i, fill) and v.store_part(list, i) for the end of a list (masked load/store)
- v.shuffle(constant lanes), v.lo(), v.hi(), concat(a, b), table.lookup(indices):
  pshufb / vpshufb on x86 (SSSE3 and up), tbl on ARM64 (checked in the assembly, LLVM 18 and 21)
- acc.dot_add(a, b): i32 lanes gain 4 byte products each, exactly; vpdpbusd with AVX-VNNI or AVX-512 VNNI,
  usdot/sdot/udot on ARM64, the sum written out elsewhere (pmaddwd); run here with all three paths agreeing
- @cpu("avx2,fma") fn: LLVM target-features for one function (+evex512 for AVX-512, which LLVM 18+ needs);
  cpu_has("avx2"): CPUID + XGETBV on x86, hwcaps on Linux ARM64, sysctl on macOS; a wide vector can't be passed
  between functions with different @cpu (its registers differ), and the compiler says so
- the compiler asks clang which features --cpu means (its predefined macros, cached), so dot_add picks the instruction
- Nix's clang wrapper dropped -march=native in nix-shell (reproduced): finch tells it NIX_ENFORCE_NO_NATIVE=0, shell.nix too,
  and warns if --cpu native still made no difference

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Psc42moPLwP43Cmb9oZqSR

runtime/finch_rt.c

@@ -1147,3 +1147,96 @@ __attribute__((noreturn, cold)) void finch_map_missing(const char *file, int64_t
              (int)(key->len > 100 ? 100 : key->len), key->ptr ? key->ptr : "");
     finch_panic(file, line, msg);
 }
+
+// ---------- cpu_has("avx2"): what the processor running the program can do ----------
+//
+// x86: CPUID, and for the AVX and AVX-512 families also XGETBV (the system must save those registers, or the
+// instructions fault). ARM64: the kernel's hwcaps on Linux and Android, sysctl on macOS. A name of the other
+// architecture is false.
+
+#if defined(__x86_64__) || defined(__i386__)
+#include <cpuid.h>
+static int x86_leaf(unsigned leaf, unsigned sub, unsigned r[4]) {
+    unsigned max = __get_cpuid_max(0, 0);
+    if (leaf > max) return 0;
+    __cpuid_count(leaf, sub, r[0], r[1], r[2], r[3]);
+    return 1;
+}
+static unsigned long long x86_xcr0(void) {
+    unsigned lo, hi;
+    __asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0));
+    return ((unsigned long long)hi << 32) | lo;
+}
+static int finch_cpu_has_c(const char *n) {
+    unsigned l1[4] = {0}, l7[4] = {0}, l71[4] = {0};
+    x86_leaf(1, 0, l1);
+    x86_leaf(7, 0, l7);
+    x86_leaf(7, 1, l71);
+    int osx = (l1[2] >> 27) & 1;                                    // OSXSAVE
+    unsigned long long xcr0 = osx ? x86_xcr0() : 0;
+    int ymm = osx && (xcr0 & 6) == 6;                               // the system saves xmm and ymm
+    int zmm = ymm && (xcr0 & 0xe0) == 0xe0;                         // and the AVX-512 registers
+    if (!strcmp(n, "sse3")) return l1[2] & 1;
+    if (!strcmp(n, "ssse3")) return (l1[2] >> 9) & 1;
+    if (!strcmp(n, "sse4.1")) return (l1[2] >> 19) & 1;
+    if (!strcmp(n, "sse4.2")) return (l1[2] >> 20) & 1;
+    if (!strcmp(n, "popcnt")) return (l1[2] >> 23) & 1;
+    if (!strcmp(n, "avx")) return ymm && ((l1[2] >> 28) & 1);
+    if (!strcmp(n, "fma")) return ymm && ((l1[2] >> 12) & 1);
+    if (!strcmp(n, "f16c")) return ymm && ((l1[2] >> 29) & 1);
+    if (!strcmp(n, "avx2")) return ymm && ((l7[1] >> 5) & 1);
+    if (!strcmp(n, "bmi1")) return (l7[1] >> 3) & 1;
+    if (!strcmp(n, "bmi2")) return (l7[1] >> 8) & 1;
+    if (!strcmp(n, "avxvnni")) return ymm && ((l71[0] >> 4) & 1);
+    if (!strcmp(n, "avx512f")) return zmm && ((l7[1] >> 16) & 1);
+    if (!strcmp(n, "avx512dq")) return zmm && ((l7[1] >> 17) & 1);
+    if (!strcmp(n, "avx512bw")) return zmm && ((l7[1] >> 30) & 1);
+    if (!strcmp(n, "avx512vl")) return zmm && ((l7[1] >> 31) & 1);
+    if (!strcmp(n, "avx512vnni")) return zmm && ((l7[2] >> 11) & 1);
+    if (!strcmp(n, "avx512fp16")) return zmm && ((l7[3] >> 23) & 1);
+    if (!strcmp(n, "avx512bf16")) return zmm && ((l71[0] >> 5) & 1);
+    return 0;
+}
+#elif defined(__aarch64__) && defined(__APPLE__)
+#include <sys/sysctl.h>
+static int sysctl_flag(const char *name) {
+    int v = 0;
+    size_t len = sizeof v;
+    if (sysctlbyname(name, &v, &len, NULL, 0) != 0) return 0;
+    return v != 0;
+}
+static int finch_cpu_has_c(const char *n) {
+    if (!strcmp(n, "neon")) return 1;
+    if (!strcmp(n, "fp16")) return sysctl_flag("hw.optional.arm.FEAT_FP16");
+    if (!strcmp(n, "dotprod")) return sysctl_flag("hw.optional.arm.FEAT_DotProd");
+    if (!strcmp(n, "i8mm")) return sysctl_flag("hw.optional.arm.FEAT_I8MM");
+    if (!strcmp(n, "bf16")) return sysctl_flag("hw.optional.arm.FEAT_BF16");
+    return 0;                                                       // no SVE on Apple chips
+}
+#elif defined(__aarch64__) && defined(__linux__)
+#include <sys/auxv.h>
+static int finch_cpu_has_c(const char *n) {
+    unsigned long h = getauxval(AT_HWCAP), h2 = getauxval(AT_HWCAP2);
+    if (!strcmp(n, "neon")) return 1;
+    if (!strcmp(n, "fp16")) return (h >> 10) & 1;                    // HWCAP_ASIMDHP
+    if (!strcmp(n, "dotprod")) return (h >> 20) & 1;                 // HWCAP_ASIMDDP
+    if (!strcmp(n, "sve")) return (h >> 22) & 1;                     // HWCAP_SVE
+    if (!strcmp(n, "sve2")) return (h2 >> 1) & 1;                    // HWCAP2_SVE2
+    if (!strcmp(n, "i8mm")) return (h2 >> 13) & 1;                   // HWCAP2_I8MM
+    if (!strcmp(n, "bf16")) return (h2 >> 14) & 1;                   // HWCAP2_BF16
+    return 0;
+}
+#else
+static int finch_cpu_has_c(const char *n) {
+    (void)n;
+    return 0;
+}
+#endif
+
+int finch_cpu_has(const FStr *name) {
+    char n[32];
+    int64_t len = name->len < 31 ? name->len : 31;
+    memcpy(n, name->ptr ? name->ptr : "", (size_t)len);
+    n[len] = 0;
+    return finch_cpu_has_c(n) ? 1 : 0;
+}

shell.nix

@@ -15,4 +15,7 @@ pkgs.mkShell {
     vulkan-headers
     vulkan-loader
   ];
+
+  # Nix's clang wrapper drops -march=native unless this is 0 (finch --cpu native also tells it so itself)
+  NIX_ENFORCE_NO_NATIVE = 0;
 }

src/ast.fch

@@ -44,7 +44,7 @@ fn vecType(str name) -> Type {
         return Type()
     }
     lane := name.sub(0, x)
-    if !["i8", "i16", "i32", "i64", "u8", "u16", "u32", "u64", "f32", "f64"].contains(lane) {
+    if !["i8", "i16", "i32", "i64", "u8", "u16", "u32", "u64", "f32", "f64", "bool"].contains(lane) {
         return Type()
     }
     n := 0
@@ -57,7 +57,7 @@ fn vecType(str name) -> Type {
     if n != 2 && n != 4 && n != 8 && n != 16 && n != 32 && n != 64 {
         return Type()
     }
-    bits := int(lane.sub(1, lane.len)) or 0
+    bits := int(lane.sub(1, lane.len)) or 1        // bool: a mask, one bit a lane
     if n * bits > 512 {
         return Type()           // the widest registers (AVX-512) hold 512 bits
     }
@@ -147,6 +147,7 @@ struct Node {
     bool flag     // decl: has a type; or: the right side is a { block }
     bool fallible // fn: -> T!  or  -> !
     bool exported // export fn: a C function of a library (finch build --shared)
+    str cpu       // @cpu("avx2,fma") fn: the processor features the function may use
     []Node kids
     []str names   // named arguments: Point(x: 1)
     int line

src/docs.fch

@@ -26,6 +26,9 @@ fn builtins() -> []Doc {
         doc("max", "max(a, b, ...) -> number", "The biggest of the numbers, in the type they all fit."),
         doc("abs", "abs(x) -> number", "x without its minus sign (any number type)."),
         doc("clamp", "clamp(x, lo, hi) -> number", "x, but at least lo and at most hi."),
+        doc("cpu_has", "cpu_has(str feature) -> bool", "Does the processor running the program have this feature (\"avx2\", \"avx512f\", \"avxvnni\", \"dotprod\", \"i8mm\" ...)? Check before calling a @cpu(...) function."),
+        doc("concat", "concat(V a, V b) -> V2", "One vector with a's lanes, then b's (twice as many)."),
+        doc("select", "select(mask, a, b) -> V", "Lane by lane: a where the mask is true, b where it is false. The mask comes from comparing vectors (v > 0.0)."),
         doc("fma", "fma(a, b, c) -> float", "a * b + c with one rounding, always (numbers or vectors). a * b + c written out is never fused: two roundings."),
         doc("muladd", "muladd(a, b, c) -> float", "a * b + c the fastest way here: one fused instruction where the processor has one, else a multiply and an add."),
         doc("f32_bits", "f32_bits(f32 x) -> u32", "The bits of x, unchanged (also f32xN -> u32xN). f64_bits gives a u64."),
@@ -119,7 +122,16 @@ fn vecMethods() -> []Doc {
         doc("abs", "abs() -> V", "Every lane without its minus sign."),
         doc("fma", "fma(V b, V c) -> V", "v * b + c with one rounding, always (f32/f64 lanes). One instruction with FMA hardware, slower without."),
         doc("muladd", "muladd(V b, V c) -> V", "v * b + c the fastest way here: fused into one instruction where the processor can (f32/f64 lanes)."),
-        doc("store", "store([]T list, int i)", "Writes the lanes into list[i], list[i+1], ... (or through a ptr[T]).")
+        doc("store", "store([]T list, int i)", "Writes the lanes into list[i], list[i+1], ... (or through a ptr[T])."),
+        doc("store_part", "store_part([]T list, int i)", "Writes the lanes that fit in the list (for its last, shorter piece)."),
+        doc("shuffle", "shuffle(int a, int b, ...) -> V", "A vector of the lanes with these numbers (written in the code): v.shuffle(3, 2, 1, 0) reverses an f32x4."),
+        doc("lo", "lo() -> V", "The lower half of the lanes."),
+        doc("hi", "hi() -> V", "The upper half of the lanes."),
+        doc("lookup", "lookup(I indices) -> V", "For every lane of indices: this vector's lane (index % len). A byte-table lookup (pshufb, tbl)."),
+        doc("dot_add", "dot_add(a, b) -> i32xN", "On an i32xN of sums: each lane gains the sum of 4 byte products of a and b (u8/i8, 4N lanes). One instruction with AVX-VNNI, AVX-512 VNNI or ARM's dot product."),
+        doc("any", "any() -> bool", "A mask: is any lane true?"),
+        doc("all", "all() -> bool", "A mask: are all lanes true?"),
+        doc("count", "count() -> int", "A mask: how many lanes are true.")
     ]
 }
 

src/gen.fch

@@ -33,6 +33,7 @@ struct Options {
     bool trace          // runtime errors show the chain of calls (off for --shared and --no-trace)
     bool test           // finch test: main runs the test "..." blocks instead
     []str testModules   // the modules whose tests run ("" is the main file); not the standard library's
+    []str features      // what the processor it is built for has (from --cpu, asked of clang): avx2, i8mm ...
 }
 
 // A generic struct or function, kept as written until something asks for a version of it.
@@ -218,6 +219,8 @@ struct Gen {
     bool deferOn    // running a defer: variables declared after it are out of sight
     int deferLimit
     int inDefer
+    []str features      // the processor features of the whole program (--cpu)
+    []str curFeatures   // and of the function being built: those and its @cpu(...)
 }
 
 // ---------- basics ----------
@@ -1460,6 +1463,9 @@ fn coerce(ptr[Gen] g, Val v, ast.Type want, ast.Node at, str what) -> Val {
     if want.kind == "str" && (isNumber(v.ty) || v.ty.kind == "bool" || v.ty.kind == "char") {
         msg = msg + " (turn it into text with str(...))"
     }
+    if want.kind == "bool" && v.ty.kind == "vec" && v.ty.elem[0].kind == "bool" {
+        msg = msg + " (a comparison of vectors has an answer for every lane: use .all() or .any())"
+    }
     if want.kind == "vec" && isNumber(v.ty) {
         msg = msg + " (" + ast.show(want) + "(x) puts x in every lane)"
     }
@@ -2357,6 +2363,13 @@ fn define(ptr[Gen] g, int si) {
         g.instNote = " (while building " + g.cur.shown + ")"
     }
     f := g.progs[g.cur.prog].fns[g.cur.index]
+    ownCpu := cpuOf(g, f)
+    g.curFeatures = g.features
+    for x in ownCpu {
+        if !g.curFeatures.contains(x) {
+            g.curFeatures.push(x)
+        }
+    }
     []str empty
     g.body = empty
     g.allocas = empty
@@ -2441,8 +2454,20 @@ fn define(ptr[Gen] g, int si) {
         }
     }
     attach := ""
+    if ownCpu.len > 0 {
+        []str plus
+        wide := false
+        for x in ownCpu {
+            plus.push("+" + llvmFeature(x))
+            wide = wide || x.starts_with("avx512")
+        }
+        if wide {
+            plus.push("+evex512")       // LLVM 18 and later: the 512-bit registers themselves (clang adds it for C too)
+        }
+        attach = " \"target-features\"=\"" + plus.join(",") + "\""
+    }
     if dbgsp >= 0 {
-        attach = " !dbg !" + str(dbgsp)
+        attach = attach + " !dbg !" + str(dbgsp)
     }
     g.dbgScope = -1
     g.out.push("define internal " + retTy(g, g.cur) + " " + g.cur.ll + "(" + params.join(", ") + ")" + attach + " {\nentry:\n" +
@@ -3680,7 +3705,7 @@ fn expr(ptr[Gen] g, ast.Node e) -> Val {
 
 fn unary(ptr[Gen] g, ast.Node e) -> Val {
     v := expr(g, e.kids[0])
-    if v.ty.kind == "vec" && e.name != "!" {
+    if v.ty.kind == "vec" && (e.name != "!" || v.ty.elem[0].kind == "bool") {
         return vecUnary(g, e, v)
     }
     if e.name == "!" {
@@ -4453,6 +4478,18 @@ fn indexOf(ptr[Gen] g, ast.Node e, Ref obj, bool write) -> Ref {
         return mapIndex(g, e, obj, t, write)
     }
     i := coerce(g, expr(g, e.kids[1]), T("int"), e.kids[1], "an index").v
+    if t.kind == "vec" && t.elem[0].kind == "bool" {
+        // a mask keeps a lane in a bit, so a lane is read from the whole value
+        if write {
+            fail(g, e, "a lane of a mask can't be set by itself; make a new mask (with a comparison or select)")
+        }
+        boundsCheck(g, i, str(t.count), e)
+        mv := obj.val.v
+        if obj.isPlace {
+            mv = load(g, llty(g, t), obj.pl.addr)
+        }
+        return Ref(isPlace: false, val: Val(v: op(g, "extractelement " + llty(g, t) + " " + mv + ", i64 " + i), ty: T("bool")))
+    }
     if t.kind == "vec" {
         // a lane: v[2], v[i] = x
         boundsCheck(g, i, str(t.count), e)
@@ -5572,6 +5609,15 @@ fn call(ptr[Gen] g, ast.Node c) -> Val {
     if ei >= 0 {
         return enumFrom(g, c, ei)
     }
+    if n == "select" && lookup(g, n) < 0 && findFn(g, g.module, n) < 0 && !g.ci.fns.has(n) {
+        return selectCall(g, c)
+    }
+    if n == "cpu_has" && lookup(g, n) < 0 && findFn(g, g.module, n) < 0 {
+        return cpuHasCall(g, c)
+    }
+    if n == "concat" && lookup(g, n) < 0 && findFn(g, g.module, n) < 0 && !g.ci.fns.has(n) {
+        return concatCall(g, c)
+    }
     if ["fma", "muladd"].contains(n) {
         return fmaCall(g, c)        // Finch's, also for f32 and vectors (C's fma is the same for one double)
     }
@@ -5816,6 +5862,7 @@ fn callFn(ptr[Gen] g, int fi, ast.Node c, int first, str shown, str selfAddr, st
 
 // The call itself, once the arguments are values ("i64 5"); the receiver's address, if any, is already first.
 fn emitCall(ptr[Gen] g, FnSig sig, ast.Node c, []str args, str shown) -> Val {
+    checkCpuCall(g, sig, c, shown)
     if sig.fallible && g.ftarget != nodeKey(c) {
         fail(g, c, "'" + shown + "' can fail, so say what happens then:  " + shown + "(...) or <value>,  " + shown + "(...) or { ... },  or  try " + shown + "(...)")
     }
@@ -6504,6 +6551,7 @@ fn runtimeDecls() -> str {
         "declare i32 @finch_test_summary()",
         "declare void @finch_input(ptr, ptr)",
         "declare i32 @finch_file_exists(ptr)",
+        "declare i32 @finch_cpu_has(ptr)",
         "declare i32 @finch_str_try_int(ptr, ptr, ptr)",
         "declare i32 @finch_str_try_float(ptr, ptr, ptr)",
         "declare i32 @finch_try_read_file(ptr, ptr, ptr)",
@@ -6552,7 +6600,7 @@ fn assemble(ptr[Gen] g) -> str {
 
 fn generate([]ast.Program progs, []str files, cimport.CImports ci, Options opt) -> str {
     shared := opt.shared
-    gen := Gen(progs: progs, files: files, ci: ci, shared: opt.shared, debug: opt.debug, index: opt.index, cwd: opt.cwd, tg: opt.target, dbgScope: -1)
+    gen := Gen(progs: progs, files: files, ci: ci, shared: opt.shared, debug: opt.debug, index: opt.index, cwd: opt.cwd, tg: opt.target, dbgScope: -1, features: opt.features)
     g := addr(gen)
     g.trace = opt.trace && !opt.shared
     if g.debug {
@@ -6743,9 +6791,15 @@ fn vecMake(ptr[Gen] g, ast.Node c, ast.Type vt) -> Val {
         ft = first.pl.ty
     }
     if ft.kind == "array" || ft.kind == "ptr" {
-        if c.kids.len != 2 {
+        if lane.kind == "bool" {
+            fail(g, c, "a mask isn't loaded from a list; compare instead, like f32x8(list, i) > 0.0")
+        }
+        if c.kids.len != 2 && c.kids.len != 3 {
             fail(g, c, shown + "(list, i): which index to start at? (" + shown + "(list, 0) for the first " + str(n) + ")")
         }
+        if c.kids.len == 3 && ft.kind == "ptr" {
+            fail(g, c, shown + "(list, i, fill) needs a list: a ptr doesn't know where its numbers end")
+        }
         if ft.elem.len == 0 || !ast.same(ft.elem[0], lane) {
             fail(g, c.kids[0], shown + "(list, i) loads " + ast.show(lane) + " numbers, but this is " + ast.show(ft))
         }
@@ -6754,6 +6808,10 @@ fn vecMake(ptr[Gen] g, ast.Node c, ast.Type vt) -> Val {
             hold(g, first.val)
         }
         i := coerce(g, expr(g, c.kids[1]), T("int"), c.kids[1], "an index").v
+        fill := ""
+        if c.kids.len == 3 {
+            fill = coerce(g, expr(g, c.kids[2]), lane, c.kids[2], "the number for the lanes past the end").v
+        }
         heldEnd(g, mark)
         data := ""
         if ft.kind == "ptr" {
@@ -6766,8 +6824,22 @@ fn vecMake(ptr[Gen] g, ast.Node c, ast.Type vt) -> Val {
             if first.isPlace {
                 av = load(g, "{ ptr, i64, i64 }", first.pl.addr)
             }
-            vecBounds(g, i, n, op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1"), c)
+            len := op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1")
             data = op(g, "extractvalue { ptr, i64, i64 } " + av + ", 0")
+            if fill.len > 0 {
+                // the end of a list: the lanes that are in it are loaded, the others get `fill`
+                m := tailMask(g, i, n, len, c)
+                ep := op(g, "getelementptr " + llty(g, lane) + ", ptr " + data + ", i64 " + i)
+                ml := llty(g, shaped(vt, "bool"))
+                fname := "@llvm.masked.load." + vecSuffix(vt) + ".p0"
+                declareOnce(g, "declare " + lt + " " + fname + "(ptr, i32, " + ml + ", " + lt + ")")
+                out := op(g, "call " + lt + " " + fname + "(ptr " + ep + ", i32 " + laneBytes(vt) + ", " + ml + " " + m + ", " + lt + " " + splat(g, fill, vt) + ")")
+                if !first.isPlace {
+                    release(g, first.val)
+                }
+                return Val(v: out, ty: vt)
+            }
+            vecBounds(g, i, n, len, c)
         }
         ep := op(g, "getelementptr inbounds " + llty(g, lane) + ", ptr " + data + ", i64 " + i)
         out := op(g, "load " + lt + ", ptr " + ep + ", align " + laneBytes(vt))
@@ -6816,6 +6888,20 @@ fn vecConvert(ptr[Gen] g, Val v, ast.Type to, ast.Node at) -> Val {
     }
     fl := from.elem[0]
     tl := to.elem[0]
+    if tl.kind == "bool" {
+        // a lane that isn't 0 is true
+        if isFloat(fl) {
+            return Val(v: op(g, "fcmp une " + llty(g, from) + " " + v.v + ", zeroinitializer"), ty: to)
+        }
+        return Val(v: op(g, "icmp ne " + llty(g, from) + " " + v.v + ", zeroinitializer"), ty: to)
+    }
+    if fl.kind == "bool" {
+        // true is 1
+        if isFloat(tl) {
+            return Val(v: op(g, "uitofp " + llty(g, from) + " " + v.v + " to " + llty(g, to)), ty: to)
+        }
+        return Val(v: op(g, "zext " + llty(g, from) + " " + v.v + " to " + llty(g, to)), ty: to)
+    }
     inst := ""
     if isInt(fl) && isInt(tl) {
         if bitsOf(fl) == bitsOf(tl) {
@@ -6849,6 +6935,12 @@ fn vecConvert(ptr[Gen] g, Val v, ast.Type to, ast.Node at) -> Val {
 fn vecUnary(ptr[Gen] g, ast.Node e, Val v) -> Val {
     lane := v.ty.elem[0]
     lt := llty(g, v.ty)
+    if lane.kind == "bool" {
+        if e.name != "!" && e.name != "~" {
+            fail(g, e, "can't put '" + e.name + "' in front of a mask (" + ast.show(v.ty) + "); '!' flips it")
+        }
+        return Val(v: op(g, "xor " + lt + " " + v.v + ", " + splat(g, "true", v.ty)), ty: v.ty)
+    }
     if e.name == "~" {
         if !isInt(lane) {
             fail(g, e, "'~' flips the bits of whole numbers, not of " + ast.show(v.ty))
@@ -6869,6 +6961,9 @@ fn vecArith(ptr[Gen] g, str o, Val l, Val r, ast.Node at) -> Val {
     a := asVec(g, l, vt, o, at)
     b := asVec(g, r, vt, o, at)
     lane := vt.elem[0]
+    if lane.kind == "bool" && o != "&" && o != "|" && o != "^" {
+        fail(g, at, "can't use '" + o + "' on a mask (" + ast.show(vt) + "); masks take & | ^ and !")
+    }
     names := ["+", "-", "*", "/", "&", "|", "^", "<<", ">>"]
     insts := ["add", "sub", "mul", "", "and", "or", "xor", "shl", "ashr"]
     if isUnsigned(lane) {
@@ -6891,27 +6986,22 @@ fn vecArith(ptr[Gen] g, str o, Val l, Val r, ast.Node at) -> Val {
     return Val(v: op(g, insts[k] + " " + llty(g, vt) + " " + a.v + ", " + b.v), ty: vt)
 }
 
-// v == w: every lane is equal. (Lane by lane: min, max, or v[i] < w[i].)
+// v < w, v == w ...: a mask (boolxN), the answer for every lane. m.all(), m.any(), select(m, a, b) use it.
 fn vecCompare(ptr[Gen] g, str o, Val l, Val r, ast.Node at) -> Val {
     vt := vecOf(l, r)
-    if o != "==" && o != "!=" {
-        fail(g, at, "'" + o + "' on " + ast.show(vt) + " would have an answer for every lane; use min(v, w), max(v, w), or compare lanes: v[i] " + o + " w[i]")
-    }
     a := asVec(g, l, vt, o, at)
     b := asVec(g, r, vt, o, at)
-    cmp := ""
+    lane := vt.elem[0]
-    if isFloat(vt.elem[0]) {
+    mask := shaped(vt, "bool")
-        cmp = op(g, "fcmp oeq " + llty(g, vt) + " " + a.v + ", " + b.v)
+    if lane.kind == "bool" && o != "==" && o != "!=" {
-    } else {
+        fail(g, at, "masks can only be compared with == and !=")
-        cmp = op(g, "icmp eq " + llty(g, vt) + " " + a.v + ", " + b.v)
     }
-    bits := ast.Type(kind: "vec", count: vt.count)
+    if isFloat(lane) {
-    bits.elem.push(T("bool"))
+        ops := ["==", "!=", "<", "<=", ">", ">="]
-    all := vecIntrinsic(g, "i1", "vector.reduce.and", bits, ["<" + str(vt.count) + " x i1>"], [cmp], "")
+        preds := ["oeq", "une", "olt", "ole", "ogt", "oge"]
-    if o == "!=" {
+        return Val(v: op(g, "fcmp " + preds[ops.find(o)] + " " + llty(g, vt) + " " + a.v + ", " + b.v), ty: mask)
-        return Val(v: op(g, "xor i1 " + all + ", true"), ty: T("bool"))
     }
-    return Val(v: all, ty: T("bool"))
+    return Val(v: op(g, "icmp " + intPred(o, isSigned(lane)) + " " + llty(g, vt) + " " + a.v + ", " + b.v), ty: mask)
 }
 
 // min(v, w) and max(v, w), lane by lane (clamp is the two)
@@ -6979,6 +7069,9 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
     lane := vt.elem[0]
     lt := llty(g, vt)
     n := m.name
+    if lane.kind == "bool" && n != "shuffle" && n != "lo" && n != "hi" {
+        return maskMethod(g, m, v)
+    }
     if n == "sum" || n == "min" || n == "max" {
         need(g, m, 0)
         how := n
@@ -7009,7 +7102,7 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
         need(g, m, 0)
         return Val(v: vecIntrinsic(g, lt, n, vt, [lt], [v.v], ""), ty: vt)
     }
-    if n == "store" {
+    if n == "store" || n == "store_part" {
         need(g, m, 2)
         dst := ref(g, m.kids[1], true)
         dt := dst.val.ty
@@ -7020,7 +7113,10 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
             fail(g, m.kids[1], "store(list, i) writes " + ast.show(lane) + " numbers into a []" + ast.show(lane) + " or a ptr[" + ast.show(lane) + "], but this is " + ast.show(dt))
         }
         if dt.kind == "array" && !dst.isPlace {
-            fail(g, m.kids[1], "store(list, i) needs a list in a variable (this one would be thrown away)")
+            fail(g, m.kids[1], n + "(list, i) needs a list in a variable (this one would be thrown away)")
+        }
+        if n == "store_part" && dt.kind != "array" {
+            fail(g, m.kids[1], "store_part(list, i) needs a list: a ptr doesn't know where its numbers end")
         }
         i := coerce(g, expr(g, m.kids[2]), T("int"), m.kids[2], "an index").v
         data := ""
@@ -7031,14 +7127,31 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
             }
         } else {
             av := load(g, "{ ptr, i64, i64 }", dst.pl.addr)
-            vecBounds(g, i, vt.count, op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1"), m)
+            len := op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1")
             data = op(g, "extractvalue { ptr, i64, i64 } " + av + ", 0")
+            if n == "store_part" {
+                // only the lanes that are in the list are written
+                mk := tailMask(g, i, vt.count, len, m)
+                ep := op(g, "getelementptr " + llty(g, lane) + ", ptr " + data + ", i64 " + i)
+                ml := llty(g, shaped(vt, "bool"))
+                fname := "@llvm.masked.store." + vecSuffix(vt) + ".p0"
+                declareOnce(g, "declare void " + fname + "(" + lt + ", ptr, i32, " + ml + ")")
+                emit(g, "call void " + fname + "(" + lt + " " + v.v + ", ptr " + ep + ", i32 " + laneBytes(vt) + ", " + ml + " " + mk + ")")
+                return Val(ty: T("void"))
+            }
+            vecBounds(g, i, vt.count, len, m)
         }
         ep := op(g, "getelementptr inbounds " + llty(g, lane) + ", ptr " + data + ", i64 " + i)
         emit(g, "store " + lt + " " + v.v + ", ptr " + ep + ", align " + laneBytes(vt))
         return Val(ty: T("void"))
     }
-    fail(g, m, "a " + ast.show(vt) + " has no method '" + n + "' (it has: sum, min, max, dot, abs, sqrt, floor, ceil, round, fma, muladd, store)")
+    if n == "shuffle" || n == "lo" || n == "hi" || n == "lookup" {
+        return shuffleMethod(g, m, v)
+    }
+    if n == "dot_add" {
+        return dotAdd(g, m, v)
+    }
+    fail(g, m, "a " + ast.show(vt) + " has no method '" + n + "' (it has: sum, min, max, dot, abs, sqrt, floor, ceil, round, fma, muladd, store, store_part, shuffle, lo, hi, lookup, dot_add)")
     return v
 }
 
@@ -7213,3 +7326,416 @@ fn bitsCall(ptr[Gen] g, ast.Node c) -> Val {
     h := op(g, "fptrunc " + il + " " + v.v + " to " + hl)
     return Val(v: op(g, "bitcast " + hl + " " + h + " to " + ol), ty: out)
 }
+
+// ---------- masks: boolxN ----------
+//
+// A comparison of vectors gives a mask, one true/false a lane (<N x i1>). It picks lanes with select, says
+// whether any or all lanes are true, and loads and stores the end of a list (f32x8(list, i, fill), store_part).
+
+// lanes 0, 1, ... n-1 that are inside a list of `len` from index i (i may be past the end: then none)
+fn tailMask(ptr[Gen] g, str i, int n, str len, ast.Node at) -> str {
+    panicIf(g, op(g, "icmp slt i64 " + i + ", 0"), "a vector's index is below 0", at)
+    rest := op(g, "sub i64 " + len + ", " + i)
+    []str lanes
+    for k in 0..n {
+        lanes.push("i64 " + str(k))
+    }
+    wide := ast.Type(kind: "vec", count: n)
+    wide.elem.push(T("int"))
+    return op(g, "icmp slt <" + str(n) + " x i64> <" + lanes.join(", ") + ">, " + splat(g, rest, wide))
+}
+
+// select(mask, a, b): a's lane where the mask is true, b's where it is false
+fn selectCall(ptr[Gen] g, ast.Node c) -> Val {
+    if g.index {
+        noteRef(g, c, c.col, c.name.len, docs.hover(docs.find(docs.builtins(), "select")), "", 0, 0)
+    }
+    checkArgs(g, c, 0, 3, "select")
+    m := expr(g, c.kids[0])
+    if m.ty.kind != "vec" || m.ty.elem[0].kind != "bool" {
+        fail(g, c.kids[0], "select(mask, a, b) takes a mask first, from a comparison of vectors like v > 0.0; this is " + ast.show(m.ty))
+    }
+    a := expr(g, c.kids[1])
+    b := expr(g, c.kids[2])
+    vt := a.ty
+    if vt.kind != "vec" {
+        vt = b.ty
+    }
+    if vt.kind != "vec" {
+        fail(g, c, "select(mask, a, b): a or b has to be a vector (the other may be a number for every lane)")
+    }
+    if vt.count != m.ty.count {
+        fail(g, c, "the mask has " + str(m.ty.count) + " lanes, but " + ast.show(vt) + " has " + str(vt.count))
+    }
+    av := asVec(g, a, vt, "select", c.kids[1])
+    bv := asVec(g, b, vt, "select", c.kids[2])
+    lt := llty(g, vt)
+    return Val(v: op(g, "select " + llty(g, m.ty) + " " + m.v + ", " + lt + " " + av.v + ", " + lt + " " + bv.v), ty: vt)
+}
+
+fn maskMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
+    n := m.name
+    ml := llty(g, v.ty)
+    if n == "any" || n == "all" {
+        need(g, m, 0)
+        how := "or"
+        if n == "all" {
+            how = "and"
+        }
+        return Val(v: vecIntrinsic(g, "i1", "vector.reduce." + how, v.ty, [ml], [v.v], ""), ty: T("bool"))
+    }
+    if n == "count" {
+        need(g, m, 0)
+        wide := shaped(v.ty, "i32")
+        wl := llty(g, wide)
+        z := op(g, "zext " + ml + " " + v.v + " to " + wl)
+        sum := vecIntrinsic(g, "i32", "vector.reduce.add", wide, [wl], [z], "")
+        return Val(v: op(g, "zext i32 " + sum + " to i64"), ty: T("int"))
+    }
+    fail(g, m, "a mask (" + ast.show(v.ty) + ") has .len, [i] and the methods any, all, count; pick lanes with select(mask, a, b)")
+    return v
+}
+
+// ---------- moving lanes around ----------
+
+fn indexList([]int idx) -> str {
+    []str parts
+    for k in idx {
+        parts.push("i32 " + str(k))
+    }
+    return "<" + parts.join(", ") + ">"
+}
+
+fn shuffleOf(ptr[Gen] g, Val v, []int idx, str v2) -> Val {
+    vt := v.ty
+    out := ast.Type(kind: "vec", count: idx.len)
+    out.elem.push(vt.elem[0])
+    second := v2
+    if second.len == 0 {
+        second = "poison"
+    }
+    return Val(v: op(g, "shufflevector " + llty(g, vt) + " " + v.v + ", " + llty(g, vt) + " " + second + ", <" + str(idx.len) + " x i32> " + indexList(idx)), ty: out)
+}
+
+fn fitsVector(ptr[Gen] g, ast.Node at, ast.Type lane, int n) {
+    if n != 2 && n != 4 && n != 8 && n != 16 && n != 32 && n != 64 {
+        fail(g, at, "a vector has 2, 4, 8, 16, 32 or 64 lanes, not " + str(n))
+    }
+    bits := bitsOf(lane)
+    if n * bits > 512 {
+        fail(g, at, "that would be " + str(n) + " lanes of " + ast.show(lane) + ", more than 512 bits")
+    }
+}
+
+// v.shuffle(3, 2, 1, 0): lanes picked by number (numbers written in the code); v.lo(), v.hi(): the halves;
+// table.lookup(idx): table[idx % table.len] for every lane of idx (pshufb on x86, tbl on ARM)
+fn shuffleMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
+    vt := v.ty
+    n := m.name
+    if n == "lo" || n == "hi" {
+        need(g, m, 0)
+        if vt.count < 4 {
+            fail(g, m, "a " + ast.show(vt) + " has only " + str(vt.count) + " lanes; take them with v[0] and v[1]")
+        }
+        []int idx
+        start := 0
+        if n == "hi" {
+            start = vt.count / 2
+        }
+        for k in 0..vt.count / 2 {
+            idx.push(start + k)
+        }
+        return shuffleOf(g, v, idx, "")
+    }
+    if n == "shuffle" {
+        []int idx
+        for q in 1..m.kids.len {
+            x := expr(g, m.kids[q])
+            if !isInt(x.ty) || !x.lit {
+                fail(g, m.kids[q], "shuffle(...) takes lane numbers written in the code (or constants), like v.shuffle(3, 2, 1, 0)")
+            }
+            k := int(x.v) or -1
+            if k < 0 || k >= vt.count {
+                fail(g, m.kids[q], "lane " + x.v + " isn't in a " + ast.show(vt) + " (its lanes are 0 to " + str(vt.count - 1) + ")")
+            }
+            idx.push(k)
+        }
+        fitsVector(g, m, vt.elem[0], idx.len)
+        return shuffleOf(g, v, idx, "")
+    }
+    // lookup
+    need(g, m, 1)
+    ix := expr(g, m.kids[1])
+    if ix.ty.kind != "vec" || !isInt(ix.ty.elem[0]) {
+        fail(g, m.kids[1], "lookup(indices) takes a vector of whole numbers (like u8x16), but this is " + ast.show(ix.ty))
+    }
+    if vt.elem[0].kind == "bool" {
+        fail(g, m, "a mask can't be a lookup table")
+    }
+    fitsVector(g, m.kids[1], vt.elem[0], ix.ty.count)
+    il := llty(g, ix.ty)
+    masked := op(g, "and " + il + " " + ix.v + ", " + splat(g, str(vt.count - 1), ix.ty))
+    out := shaped(ix.ty, vt.elem[0].kind)
+    ol := llty(g, out)
+    el := llty(g, vt.elem[0])
+    xl := llty(g, ix.ty.elem[0])
+    cur := "poison"
+    for k in 0..ix.ty.count {
+        at := op(g, "extractelement " + il + " " + masked + ", i64 " + str(k))
+        lane := op(g, "extractelement " + llty(g, vt) + " " + v.v + ", " + xl + " " + at)
+        cur = op(g, "insertelement " + ol + " " + cur + ", " + el + " " + lane + ", i64 " + str(k))
+    }
+    return Val(v: cur, ty: out)
+}
+
+// concat(a, b): a's lanes, then b's
+fn concatCall(ptr[Gen] g, ast.Node c) -> Val {
+    if g.index {
+        noteRef(g, c, c.col, c.name.len, docs.hover(docs.find(docs.builtins(), "concat")), "", 0, 0)
+    }
+    checkArgs(g, c, 0, 2, "concat")
+    a := expr(g, c.kids[0])
+    b := expr(g, c.kids[1])
+    if a.ty.kind != "vec" || !ast.same(a.ty, b.ty) {
+        fail(g, c, "concat(a, b) joins two vectors of the same type, but these are " + ast.show(a.ty) + " and " + ast.show(b.ty))
+    }
+    fitsVector(g, c, a.ty.elem[0], a.ty.count * 2)
+    []int idx
+    for k in 0..a.ty.count * 2 {
+        idx.push(k)
+    }
+    return shuffleOf(g, a, idx, b.v)
+}
+
+// ---------- processor features: @cpu(...), cpu_has(...) ----------
+//
+// --cpu gives the whole program a processor (main.fch asks clang which features that means); @cpu("avx2,fma") gives one
+// function more, as LLVM's "target-features". Such a function is only called after cpu_has("avx2") said yes. Features of
+// the other architecture are dropped, so one source holds the x86 and the ARM versions side by side.
+
+fn x86Features() -> []str {
+    return ["sse3", "ssse3", "sse4.1", "sse4.2", "popcnt", "avx", "avx2", "fma", "f16c", "bmi1", "bmi2", "avx512f", "avx512bw",
+            "avx512dq", "avx512vl", "avx512vnni", "avx512bf16", "avx512fp16", "avxvnni"]
+}
+
+fn armFeatures() -> []str {
+    return ["neon", "fp16", "dotprod", "i8mm", "bf16", "sve", "sve2"]
+}
+
+// the name LLVM uses for a feature
+fn llvmFeature(str x) -> str {
+    if x == "bmi1" {
+        return "bmi"
+    }
+    if x == "fp16" {
+        return "fullfp16"
+    }
+    return x
+}
+
+// the features of a level: x86-64-v3 is the AVX2 generation, v4 the AVX-512 one
+fn levelFeatures(str name) -> []str {
+    []str out
+    if name == "x86-64-v2" || name == "x86-64-v3" || name == "x86-64-v4" {
+        for x in ["sse3", "ssse3", "sse4.1", "sse4.2", "popcnt"] {
+            out.push(x)
+        }
+    }
+    if name == "x86-64-v3" || name == "x86-64-v4" {
+        for x in ["avx", "avx2", "bmi1", "bmi2", "f16c", "fma"] {
+            out.push(x)
+        }
+    }
+    if name == "x86-64-v4" {
+        for x in ["avx512f", "avx512bw", "avx512dq", "avx512vl"] {
+            out.push(x)
+        }
+    }
+    return out
+}
+
+// a function's own @cpu features, for the architecture it is built for
+fn cpuOf(ptr[Gen] g, ast.Node f) -> []str {
+    []str out
+    if f.cpu.len == 0 {
+        return out
+    }
+    for part in f.cpu.split(",") {
+        x := part.trim()
+        names := levelFeatures(x)
+        if names.len == 0 {
+            if !x86Features().contains(x) && !armFeatures().contains(x) {
+                fail(g, f, "@cpu: '" + x + "' isn't a feature Finch knows (x86: " + x86Features().join(", ") + ", x86-64-v2/v3/v4; ARM: " + armFeatures().join(", ") + ")")
+            }
+            names.push(x)
+        }
+        for y in names {
+            if armFeatures().contains(y) == g.tg.arm64 && !out.contains(y) {
+                out.push(y)
+            }
+        }
+    }
+    return out
+}
+
+// a vector wider than 16 bytes travels in different registers depending on the features (two xmm, or one ymm), so it
+// can't cross from a function with one set of features to a function with another
+fn checkCpuCall(ptr[Gen] g, FnSig sig, ast.Node c, str shown) {
+    if sig.prog < 0 || sig.prog >= g.progs.len || sig.index >= g.progs[sig.prog].fns.len {
+        return
+    }
+    callee := g.progs[sig.prog].fns[sig.index].cpu
+    mine := g.progs[g.cur.prog].fns[g.cur.index].cpu
+    if callee == mine {
+        return
+    }
+    []ast.Type ts = sig.params
+    ts.push(sig.ret)
+    for t in ts {
+        if t.kind == "vec" && t.count * bitsOf(t.elem[0]) > 128 {
+            fail(g, c, "'" + shown + "' has other @cpu features than this function, so a " + ast.show(t) + " can't be passed between them (its registers differ); pass a list or a ptr, or give both the same @cpu")
+        }
+    }
+}
+
+// cpu_has("avx2"): does the processor running the program have it? (false for the other architecture's names)
+fn cpuHasCall(ptr[Gen] g, ast.Node c) -> Val {
+    if g.index {
+        noteRef(g, c, c.col, c.name.len, docs.hover(docs.find(docs.builtins(), "cpu_has")), "", 0, 0)
+    }
+    checkArgs(g, c, 0, 1, "cpu_has")
+    if c.kids[0].kind == "str" {
+        x := c.kids[0].text
+        if !x86Features().contains(x) && !armFeatures().contains(x) {
+            fail(g, c.kids[0], "cpu_has: '" + x + "' isn't a feature Finch knows (x86: " + x86Features().join(", ") + "; ARM: " + armFeatures().join(", ") + ")")
+        }
+    }
+    s := strArg(g, c.kids[0], "the feature's name")
+    r := op(g, "call i32 @finch_cpu_has(ptr " + spill(g, s) + ")")
+    release(g, s)
+    return Val(v: op(g, "icmp ne i32 " + r + ", 0"), ty: T("bool"))
+}
+
+// ---------- acc.dot_add(a, b): the step of an int8 dot product ----------
+//
+// acc is i32xN; a and b have 4N bytes (a u8 or i8, b i8 or u8). Lane k of acc gains a[4k]*b[4k] + ... + a[4k+3]*b[4k+3],
+// exactly (no saturation). That is vpdpbusd (AVX-VNNI, AVX-512 VNNI) and usdot / sdot / udot (ARM), used when the
+// function's features have them; elsewhere the same sum is written out, which LLVM makes into pmaddwd and adds.
+fn dotAdd(ptr[Gen] g, ast.Node m, Val acc) -> Val {
+    need(g, m, 2)
+    at := acc.ty
+    if at.elem[0].kind != "i32" && at.elem[0].kind != "u32" {
+        fail(g, m, "dot_add is called on an i32 vector (the sums), like acc.dot_add(a, b) with acc an i32x8; this is " + ast.show(at))
+    }
+    a := expr(g, m.kids[1])
+    b := expr(g, m.kids[2])
+    for x in [a, b] {
+        if x.ty.kind != "vec" || (x.ty.elem[0].kind != "u8" && x.ty.elem[0].kind != "i8") || x.ty.count != at.count * 4 {
+            fail(g, m, "dot_add(a, b) on an " + ast.show(at) + " takes two vectors of " + str(at.count * 4) + " bytes (u8 or i8), but these are " + ast.show(a.ty) + " and " + ast.show(b.ty))
+        }
+    }
+    au := a.ty.elem[0].kind == "u8"
+    bu := b.ty.elem[0].kind == "u8"
+    if !au && bu {
+        // u8 x i8 is the same sum as i8 x u8: put the unsigned one first
+        t := a
+        a = b
+        b = t
+        au = true
+        bu = false
+    }
+    al := llty(g, at)
+    bits := at.count * 32
+    f := g.curFeatures
+    if !g.tg.arm64 && au && !bu {
+        vnni := (bits == 512 && f.contains("avx512vnni")) || (bits <= 256 && (f.contains("avxvnni") || (f.contains("avx512vnni") && f.contains("avx512vl"))))
+        if vnni {
+            a32 := op(g, "bitcast " + llty(g, a.ty) + " " + a.v + " to " + al)
+            b32 := op(g, "bitcast " + llty(g, b.ty) + " " + b.v + " to " + al)
+            name := "@llvm.x86.avx512.vpdpbusd." + str(bits)
+            declareOnce(g, "declare " + al + " " + name + "(" + al + ", " + al + ", " + al + ")")
+            return Val(v: op(g, "call " + al + " " + name + "(" + al + " " + acc.v + ", " + al + " " + a32 + ", " + al + " " + b32 + ")"), ty: at)
+        }
+    }
+    if g.tg.arm64 && bits % 128 == 0 {
+        which := ""
+        if au && !bu && f.contains("i8mm") {
+            which = "usdot"
+        } else if au && bu && f.contains("dotprod") {
+            which = "udot"
+        } else if !au && !bu && f.contains("dotprod") {
+            which = "sdot"
+        }
+        if which.len > 0 {
+            // one instruction per 16 bytes
+            name := "@llvm.aarch64.neon." + which + ".v4i32.v16i8"
+            declareOnce(g, "declare <4 x i32> " + name + "(<4 x i32>, <16 x i8>, <16 x i8>)")
+            parts := at.count / 4
+            []str outs
+            for q in 0..parts {
+                []int ai
+                for k in 0..4 {
+                    ai.push(q * 4 + k)
+                }
+                []int bi
+                for k in 0..16 {
+                    bi.push(q * 16 + k)
+                }
+                ac := shuffleOf(g, acc, ai, "").v
+                ap := shuffleOf(g, a, bi, "").v
+                bp := shuffleOf(g, b, bi, "").v
+                outs.push(op(g, "call <4 x i32> " + name + "(<4 x i32> " + ac + ", <16 x i8> " + ap + ", <16 x i8> " + bp + ")"))
+            }
+            return Val(v: joinParts(g, outs, 4), ty: at)
+        }
+    }
+    // written out: widen, multiply, add each group of four
+    w := shaped(a.ty, "i32")
+    wl := llty(g, w)
+    ext := "sext"
+    if au {
+        ext = "zext"
+    }
+    aw := op(g, ext + " " + llty(g, a.ty) + " " + a.v + " to " + wl)
+    ext = "sext"
+    if bu {
+        ext = "zext"
+    }
+    bw := op(g, ext + " " + llty(g, b.ty) + " " + b.v + " to " + wl)
+    prod := Val(v: op(g, "mul " + wl + " " + aw + ", " + bw), ty: w)
+    sum := acc.v
+    for k in 0..4 {
+        []int idx
+        for j in 0..at.count {
+            idx.push(4 * j + k)
+        }
+        sum = op(g, "add " + al + " " + sum + ", " + shuffleOf(g, prod, idx, "").v)
+    }
+    return Val(v: sum, ty: at)
+}
+
+// <4 x i32> pieces back into one vector of n * 4 lanes
+fn joinParts(ptr[Gen] g, []str parts, int each) -> str {
+    if parts.len == 1 {
+        return parts[0]
+    }
+    []str cur = parts
+    size := each
+    while cur.len > 1 {
+        []str next
+        t := ast.Type(kind: "vec", count: size)
+        t.elem.push(T("i32"))
+        k := 0
+        while k < cur.len {
+            []int idx
+            for j in 0..size * 2 {
+                idx.push(j)
+            }
+            next.push(shuffleOf(g, Val(v: cur[k], ty: t), idx, cur[k + 1]).v)
+            k += 2
+        }
+        cur = next
+        size = size * 2
+    }
+    return cur[0]
+}

src/lexer.fch

@@ -70,7 +70,7 @@ fn operator(ptr[Lexer] lx) -> str {
             return two
         }
     }
-    ones := "(){}[],.:+-*/%&|^~<>=!"
+    ones := "(){}[],.:+-*/%&|^~<>=!@"
     c := lx.src.sub(lx.i, lx.i + 1)
     if ones.contains(c) {
         return c

src/linker.fch

@@ -371,10 +371,7 @@ struct Build {
 
 // IR -> program. 0 if it worked.
 fn build(Build b, platform.Target tg, cimport.CImports ci) -> int {
-    clang := getenv("FINCH_CLANG")
+    clang := clangCmd()
-    if clang.len == 0 {
-        clang = "clang"
-    }
     // the C compiler of the runtime and the linker: the target's own when building for another system
     cc := clang
     if tg.cross {
@@ -478,3 +475,74 @@ fn build(Build b, platform.Target tg, cimport.CImports ci) -> int {
     }
     return 0
 }
+
+// clang, as the program is compiled with it. Nix's clang wrapper removes -march=native when NIX_ENFORCE_NO_NATIVE
+// is 1, as it is in nix-shell; --cpu native asks for it on purpose, so the wrapper is told to keep it.
+fn clangCmd() -> str {
+    clang := getenv("FINCH_CLANG")
+    if clang.len == 0 {
+        clang = "clang"
+    }
+    if host.pathSeparator() != ";" {
+        return "NIX_ENFORCE_NO_NATIVE=0 " + clang
+    }
+    return clang
+}
+
+// -march=x86-64-v3 or -mcpu=apple-m1: how clang is told the processor
+fn cpuFlag(platform.Target tg, str cpu) -> str {
+    if cpu.len == 0 {
+        return ""
+    }
+    if tg.arm64 {
+        return " -mcpu=" + quote(cpu)
+    }
+    return " -march=" + quote(cpu)
+}
+
+// What the processor the program is for can do, as Finch's feature names (avx2, avxvnni, i8mm ...): clang says it in
+// the macros it predefines for that target and --cpu. Kept in the cache, one file per clang, target and cpu.
+fn cpuFeatures(platform.Target tg, str cpu) -> []str {
+    clang := clangCmd()
+    flags := cpuFlag(tg, cpu)
+    if tg.cross || tg.windows {
+        flags = " --target=" + tg.triple + flags
+    }
+    dir := host.cacheDir() + "/cpu"
+    host.makeDirs(dir)
+    slot := dir + "/" + fingerprint2(clang + flags)
+    text := read_file(slot + ".txt") or ""
+    if text.len == 0 {
+        empty := slot + ".part" + str(host.pid()) + ".c"
+        out := slot + ".part" + str(host.pid()) + ".txt"
+        write_file(empty, "")
+        host.run(clang + flags + " -dM -E " + quote(empty) + " > " + quote(out) + " 2> " + host.nullDevice())
+        host.remove(empty)
+        text = read_file(out) or ""
+        if text.len == 0 {
+            host.remove(out)
+            []str none
+            return none
+        }
+        host.move(out, slot + ".txt")
+    }
+    macros := ["__SSE3__", "__SSSE3__", "__SSE4_1__", "__SSE4_2__", "__POPCNT__", "__AVX__", "__AVX2__", "__FMA__", "__F16C__",
+               "__BMI__", "__BMI2__", "__AVX512F__", "__AVX512BW__", "__AVX512DQ__", "__AVX512VL__", "__AVX512VNNI__",
+               "__AVX512BF16__", "__AVX512FP16__", "__AVXVNNI__", "__ARM_NEON", "__ARM_FEATURE_FP16_VECTOR_ARITHMETIC",
+               "__ARM_FEATURE_DOTPROD", "__ARM_FEATURE_MATMUL_INT8", "__ARM_FEATURE_BF16_VECTOR_ARITHMETIC", "__ARM_FEATURE_SVE",
+               "__ARM_FEATURE_SVE2"]
+    names := ["sse3", "ssse3", "sse4.1", "sse4.2", "popcnt", "avx", "avx2", "fma", "f16c", "bmi1", "bmi2", "avx512f", "avx512bw",
+              "avx512dq", "avx512vl", "avx512vnni", "avx512bf16", "avx512fp16", "avxvnni", "neon", "fp16", "dotprod", "i8mm", "bf16",
+              "sve", "sve2"]
+    []str have
+    for line in text.split("\n") {
+        parts := line.split(" ")
+        if parts.len >= 2 && parts[0] == "#define" {
+            k := macros.find(parts[1])
+            if k >= 0 && !have.contains(names[k]) {
+                have.push(names[k])
+            }
+        }
+    }
+    return have
+}

src/main.fch

@@ -471,7 +471,15 @@ fn main([]str args) -> int {
             testModules.push(prog.module)       // your own files' tests, not the standard library's
         }
     }
-    ir := gen.generate(loader.progs, loader.files, ci, gen.Options(shared: shared, debug: debug, index: cmd == "index", cwd: host.cwd(), target: tg, test: cmd == "test", testModules: testModules, trace: trace))
+    // what the processor can do (dot_add and the like pick their instructions by it); not needed for the editor's index
+    []str features
+    if cmd != "index" {
+        features = linker.cpuFeatures(tg, cpu)
+        if cpu == "native" && !tg.arm64 && !features.contains("sse4.2") {
+            print("warning: --cpu native made no difference to clang (a clang wrapper may remove -march=native; try --cpu with a name, like x86-64-v3)")
+        }
+    }
+    ir := gen.generate(loader.progs, loader.files, ci, gen.Options(shared: shared, debug: debug, index: cmd == "index", cwd: host.cwd(), target: tg, test: cmd == "test", testModules: testModules, trace: trace, features: features))
     if cmd == "ir" || cmd == "index" {
         print(ir)
         return 0
@@ -501,14 +509,7 @@ fn main([]str args) -> int {
     if debug {
         extra = extra + " -g"      // or clang drops the debug info that is in the IR
     }
-    if cpu.len > 0 {
+    extra = extra + linker.cpuFlag(tg, cpu)          // x86-64 names a processor with -march, ARM with -mcpu
-        // x86-64 names a processor with -march, ARM with -mcpu
-        if tg.arm64 {
-            extra = extra + " -mcpu=" + linker.quote(cpu)
-        } else {
-            extra = extra + " -march=" + linker.quote(cpu)
-        }
-    }
     []str cflagList
     for d in dirs {
         cflagList.push(" -I" + linker.quote(d))      // also the program's own folders (its headers, for inline C functions)

src/parser.fch

@@ -289,6 +289,22 @@ fn parseProgram(ptr[Parser] p) -> ast.Program {
             if e.flag {
                 prog.structs.push(sumStruct(e))       // an enum with values inside is stored as a struct (see sumStruct)
             }
+        } else if at(p, "@") {
+            // @cpu("avx2,fma") fn ...: a function that may use these processor features (check with cpu_has first)
+            a := next(p)
+            name := expect(p, "ident", "an attribute name, like @cpu(\"avx2\")").text
+            if name != "cpu" {
+                fail(p, a, "the attribute here is @cpu(\"...\"): the processor features a function may use")
+            }
+            expect(p, "(", "'('")
+            feats := expect(p, "string", "the features in quotes, like @cpu(\"avx2,fma\")").text
+            expect(p, ")", "')'")
+            if !at(p, "fn") && !at(p, "export") {
+                fail(p, cur(p), "@cpu(...) goes right before a function")
+            }
+            f := parseFn(p)
+            f.cpu = feats
+            prog.fns.push(f)
         } else {
             prog.fns.push(parseFn(p))
         }

tests/fail/cpu_attr_name.fch

@@ -0,0 +1,7 @@
+// expect: @cpu: 'avx3' isn't a feature Finch knows
+@cpu("avx2,avx3")
+fn f() {
+}
+fn main() {
+    f()
+}

tests/fail/cpu_has_name.fch

@@ -0,0 +1,4 @@
+// expect: cpu_has: 'avx-512' isn't a feature Finch knows
+fn main() {
+    print(cpu_has("avx-512"))
+}

tests/fail/cpu_wide_vector.fch

@@ -0,0 +1,8 @@
+// expect: can't be passed between them (its registers differ)
+@cpu("avx2")
+fn double(f32x8 v) -> f32x8 {
+    return v * 2.0
+}
+fn main() {
+    print(double(f32x8(1.0)))
+}

tests/fail/vec_less.fch

@@ -1,5 +1,7 @@
-// expect: use min(v, w), max(v, w), or compare lanes: v[i] < w[i]
+// expect: use .all() or .any()
 fn main() {
     a := f32x4(1.0)
-    print(a < a)
+    if a < a {
+        print("no")
+    }
 }

tests/fail/vec_shuffle_range.fch

@@ -0,0 +1,5 @@
+// expect: lane 4 isn't in a f32x4 (its lanes are 0 to 3)
+fn main() {
+    v := f32x4(1.0)
+    print(v.shuffle(0, 1, 2, 4))
+}

tests/fail/vec_shuffle_var.fch

@@ -0,0 +1,6 @@
+// expect: shuffle(...) takes lane numbers written in the code
+fn main() {
+    v := f32x4(1.0)
+    k := 2
+    print(v.shuffle(k, 0, 1, 3))
+}

tests/run/dot_int8.fch

@@ -0,0 +1,69 @@
+// The int8 dot product of quantized models: plain, with dot_add, and in @cpu functions picked by cpu_has
+
+fn plain([]u8 a, []i8 b) -> int {
+    total := 0
+    for i in 0..a.len {
+        total += int(a[i]) * int(b[i])
+    }
+    return total
+}
+
+// 32 bytes a step, 8 sums kept in a vector until the end
+fn with_vectors([]u8 a, []i8 b) -> int {
+    acc := i32x8(0)
+    i := 0
+    while i + 32 <= a.len {
+        acc = acc.dot_add(u8x32(a, i), i8x32(b, i))
+        i += 32
+    }
+    return int(acc.sum())
+}
+
+@cpu("avxvnni")
+fn with_vnni([]u8 a, []i8 b) -> int {
+    acc := i32x8(0)
+    i := 0
+    while i + 32 <= a.len {
+        acc = acc.dot_add(u8x32(a, i), i8x32(b, i))
+        i += 32
+    }
+    return int(acc.sum())
+}
+
+@cpu("avx512f,avx512bw,avx512vl,avx512vnni,i8mm")
+fn with_avx512([]u8 a, []i8 b) -> int {
+    acc := i32x16(0)
+    i := 0
+    while i + 64 <= a.len {
+        acc = acc.dot_add(u8x64(a, i), i8x64(b, i))
+        i += 64
+    }
+    return int(acc.sum())
+}
+
+fn best([]u8 a, []i8 b) -> int {
+    if cpu_has("avx512vnni") && cpu_has("avx512vl") && cpu_has("avx512bw") {
+        return with_avx512(a, b)
+    }
+    if cpu_has("avxvnni") {
+        return with_vnni(a, b)
+    }
+    return with_vectors(a, b)
+}
+
+fn main() {
+    []u8 a
+    []i8 b
+    for k in 0..1024 {
+        a.push(u8((k * 37 + 11) % 256))
+        b.push(i8((k * 53 + 7) % 256 - 128))
+    }
+    want := plain(a, b)
+    print(want, with_vectors(a, b) == want, best(a, b) == want)
+
+    // the extremes: 255 * -128 four times in a lane, no saturation
+    acc := i32x4(1, 2, 3, 4).dot_add(u8x16(255), i8x16(-128))
+    print(acc, i32x4(0).dot_add(i8x16(-128), i8x16(-128)), i32x4(0).dot_add(u8x16(255), u8x16(255)))
+    print(i32x4(0).dot_add(i8x16(-1), u8x16(200)))
+    print(cpu_has("avx2") || cpu_has("neon") || true)
+}

tests/run/dot_int8.out

@@ -0,0 +1,4 @@
+-68096 true true
+i32x4(-130559, -130558, -130557, -130556) i32x4(65536, 65536, 65536, 65536) i32x4(260100, 260100, 260100, 260100)
+i32x4(-800, -800, -800, -800)
+true

tests/run/masks.fch

@@ -0,0 +1,50 @@
+// Masks: comparisons of vectors, select, any/all/count, and the end of a list
+
+fn relu(f32x8 v) -> f32x8 {
+    return select(v > 0.0, v, 0.0)
+}
+
+// the sum of a list of any length: whole vectors, then the last piece with its missing lanes as 0
+fn total([]f32 xs) -> f32 {
+    acc := f32x8(0.0)
+    i := 0
+    while i < xs.len {
+        acc = acc + f32x8(xs, i, 0.0)
+        i += 8
+    }
+    return acc.sum()
+}
+
+// add 1 to every number of a list, in place, the last piece too
+fn bump([]f32 xs) {
+    i := 0
+    while i < xs.len {
+        (f32x8(xs, i, 0.0) + 1.0).store_part(xs, i)
+        i += 8
+    }
+}
+
+fn main() {
+    v := f32x8(-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, -4.0, 5.0)
+    m := v > 0.0
+    print(m, m.count(), m.any(), m.all(), m[3])
+    print(relu(v), select(!m, v, -1.0))
+    print((v >= -1.0) & (v < 3.0), (v == 0.0) | (v == 5.0), m ^ (v > 1.0))
+
+    a := i32x4(1, 5, 3, 7)
+    b := i32x4(4, 4, 4, 4)
+    print(select(a > b, a, b), (a == a).all(), (a != b).all())
+    u := u8x16(200)
+    print((u > u8x16(100)).all(), i32x4(a > b), f32x4(a < b))
+    k := i32x4(0, 2, 0, -1)
+    print(boolx4(k), boolx4(true, false, true, false))
+
+    []f32 xs
+    for n in 0..13 {
+        xs.push(f32(n))
+    }
+    print(total(xs))
+    bump(xs)
+    print(xs)
+    print(f32x8(xs, 8, -1.0), f32x8(xs, 13, 9.0), f32x8(xs, 100, 7.0))
+}

tests/run/masks.out

@@ -0,0 +1,9 @@
+boolx8(false, false, false, true, true, true, false, true) 4 true false true
+f32x8(0, 0, 0, 1, 2, 3, 0, 5) f32x8(-2, -1, 0, -1, -1, -1, -4, -1)
+boolx8(false, true, true, true, true, false, false, false) boolx8(false, false, true, false, false, false, false, true) boolx8(false, false, false, true, false, false, false, false)
+i32x4(4, 5, 4, 7) true true
+true i32x4(0, 1, 0, 1) f32x4(1, 0, 1, 0)
+boolx4(false, true, false, true) boolx4(true, false, true, false)
+78
+[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
+f32x8(9, 10, 11, 12, 13, -1, -1, -1) f32x8(9, 9, 9, 9, 9, 9, 9, 9) f32x8(7, 7, 7, 7, 7, 7, 7, 7)

tests/run/shuffles.fch

@@ -0,0 +1,27 @@
+// Moving lanes: shuffle, lo/hi, concat, and a byte table lookup (how 4-bit weights are unpacked)
+
+// sixteen 4-bit numbers, two to a byte, made into sixteen bytes through a table of what each nibble means
+fn unpack4(u8x8 packed, i8x16 table) -> i8x16 {
+    pairs := packed.shuffle(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7)   // u8x16: each byte twice
+    odd := u8x16(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1) == 1
+    nibbles := select(odd, pairs >> 4, pairs & 15)   // low nibble first, then the high one
+    return table.lookup(nibbles)
+}
+
+fn main() {
+    v := f32x4(1.0, 2.0, 3.0, 4.0)
+    print(v.shuffle(3, 2, 1, 0), v.shuffle(0, 0, 1, 1, 2, 2, 3, 3), v.shuffle(1, 2))
+    w := i32x8(0, 1, 2, 3, 4, 5, 6, 7)
+    print(w.lo(), w.hi(), concat(w.hi(), w.lo()))
+    print(concat(w, w).len, concat(v, v * 10.0).shuffle(0, 4, 1, 5, 2, 6, 3, 7))
+
+    table := u8x16(10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25)
+    print(table.lookup(u8x16(15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14)))
+    print(table.lookup(u8x16(16, 17, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)).lo())
+    print(table.lookup(u8x32(3)).hi().lo().lo())
+
+    q4 := i8x16(-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7)       // what each 4-bit number means
+    print(unpack4(u8x8(0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe), q4))
+    m := f32x4(1.0, -1.0, 2.0, -2.0) > 0.0
+    print(m.shuffle(1, 0, 3, 2), m.lo())
+}

tests/run/shuffles.out

@@ -0,0 +1,8 @@
+f32x4(4, 3, 2, 1) f32x8(1, 1, 2, 2, 3, 3, 4, 4) f32x2(2, 3)
+i32x4(0, 1, 2, 3) i32x4(4, 5, 6, 7) i32x8(4, 5, 6, 7, 0, 1, 2, 3)
+16 f32x8(1, 10, 2, 20, 3, 30, 4, 40)
+u8x16(25, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24)
+u8x8(10, 11, 25, 10, 10, 10, 10, 10)
+u8x4(13, 13, 13, 13)
+i8x16(-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7)
+boolx4(false, true, false, true) boolx2(true, false)

tests/run/vectors.fch

@@ -36,7 +36,7 @@ fn main() {
     // lanes
     a[0] = 9.0
     print(a[0], a[3], a)
-    print(a == f32x4(9.0, 2.0, 3.0, 4.0), a != a)
+    print((a == f32x4(9.0, 2.0, 3.0, 4.0)).all(), (a != a).any())
 
     // whole numbers, bits, conversions
     n := i32x8(1, 2, 3, 4, 5, 6, 7, 8)