nexoniarz/Finch
Watch
1
0
Fork
You've already forked Finch
 0
mirror of https://github.com/Nexoniarz/Finch.git synced 2026-11-03 00:44:27 +00:00
Code Issues Projects Releases Packages Wiki Activity

Vectors for kernels: masks and select, shuffles and byte lookup, dot_add, @cpu and cpu_has; --cpu native on Nix

Browse source 
- comparisons of vectors give masks (boolxN): select(m, a, b), m.any/all/count, & | ^ !;
  f32x8(list, i, fill) and v.store_part(list, i) for the end of a list (masked load/store)
- v.shuffle(constant lanes), v.lo(), v.hi(), concat(a, b), table.lookup(indices):
  pshufb / vpshufb on x86 (SSSE3 and up), tbl on ARM64 (checked in the assembly, LLVM 18 and 21)
- acc.dot_add(a, b): i32 lanes gain 4 byte products each, exactly; vpdpbusd with AVX-VNNI or AVX-512 VNNI,
  usdot/sdot/udot on ARM64, the sum written out elsewhere (pmaddwd); run here with all three paths agreeing
- @cpu("avx2,fma") fn: LLVM target-features for one function (+evex512 for AVX-512, which LLVM 18+ needs);
  cpu_has("avx2"): CPUID + XGETBV on x86, hwcaps on Linux ARM64, sysctl on macOS; a wide vector can't be passed
  between functions with different @cpu (its registers differ), and the compiler says so
- the compiler asks clang which features --cpu means (its predefined macros, cached), so dot_add picks the instruction
- Nix's clang wrapper dropped -march=native in nix-shell (reproduced): finch tells it NIX_ENFORCE_NO_NATIVE=0, shell.nix too,
  and warns if --cpu native still made no difference

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Psc42moPLwP43Cmb9oZqSR
This commit is contained in:
Claude 2026-11-01 15:41:47 +00:00
commit 04f1f94a21
No known key found for this signature in database
23 changed files with 964 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

93
runtime/finch_rt.c
View file

@ -1147,3 +1147,96 @@ __attribute__((noreturn, cold)) void finch_map_missing(const char *file, int64_t
(int)(key->len > 100 ? 100 : key->len), key->ptr ? key->ptr : "");
finch_panic(file, line, msg);
}
// ---------- cpu_has("avx2"): what the processor running the program can do ----------
//
// x86: CPUID, and for the AVX and AVX-512 families also XGETBV (the system must save those registers, or the
// instructions fault). ARM64: the kernel's hwcaps on Linux and Android, sysctl on macOS. A name of the other
// architecture is false.
#if defined(__x86_64__) || defined(__i386__)
#include <cpuid.h>
static int x86_leaf(unsigned leaf, unsigned sub, unsigned r[4]) {
unsigned max = __get_cpuid_max(0, 0);
if (leaf > max) return 0;
__cpuid_count(leaf, sub, r[0], r[1], r[2], r[3]);
return 1;
}
static unsigned long long x86_xcr0(void) {
unsigned lo, hi;
__asm__ volatile("xgetbv" : "=a"(lo), "=d"(hi) : "c"(0));
return ((unsigned long long)hi << 32) | lo;
}
static int finch_cpu_has_c(const char *n) {
unsigned l1[4] = {0}, l7[4] = {0}, l71[4] = {0};
x86_leaf(1, 0, l1);
x86_leaf(7, 0, l7);
x86_leaf(7, 1, l71);
int osx = (l1[2] >> 27) & 1; // OSXSAVE
unsigned long long xcr0 = osx ? x86_xcr0() : 0;
int ymm = osx && (xcr0 & 6) == 6; // the system saves xmm and ymm
int zmm = ymm && (xcr0 & 0xe0) == 0xe0; // and the AVX-512 registers
if (!strcmp(n, "sse3")) return l1[2] & 1;
if (!strcmp(n, "ssse3")) return (l1[2] >> 9) & 1;
if (!strcmp(n, "sse4.1")) return (l1[2] >> 19) & 1;
if (!strcmp(n, "sse4.2")) return (l1[2] >> 20) & 1;
if (!strcmp(n, "popcnt")) return (l1[2] >> 23) & 1;
if (!strcmp(n, "avx")) return ymm && ((l1[2] >> 28) & 1);
if (!strcmp(n, "fma")) return ymm && ((l1[2] >> 12) & 1);
if (!strcmp(n, "f16c")) return ymm && ((l1[2] >> 29) & 1);
if (!strcmp(n, "avx2")) return ymm && ((l7[1] >> 5) & 1);
if (!strcmp(n, "bmi1")) return (l7[1] >> 3) & 1;
if (!strcmp(n, "bmi2")) return (l7[1] >> 8) & 1;
if (!strcmp(n, "avxvnni")) return ymm && ((l71[0] >> 4) & 1);
if (!strcmp(n, "avx512f")) return zmm && ((l7[1] >> 16) & 1);
if (!strcmp(n, "avx512dq")) return zmm && ((l7[1] >> 17) & 1);
if (!strcmp(n, "avx512bw")) return zmm && ((l7[1] >> 30) & 1);
if (!strcmp(n, "avx512vl")) return zmm && ((l7[1] >> 31) & 1);
if (!strcmp(n, "avx512vnni")) return zmm && ((l7[2] >> 11) & 1);
if (!strcmp(n, "avx512fp16")) return zmm && ((l7[3] >> 23) & 1);
if (!strcmp(n, "avx512bf16")) return zmm && ((l71[0] >> 5) & 1);
return 0;
}
#elif defined(__aarch64__) && defined(__APPLE__)
#include <sys/sysctl.h>
static int sysctl_flag(const char *name) {
int v = 0;
size_t len = sizeof v;
if (sysctlbyname(name, &v, &len, NULL, 0) != 0) return 0;
return v != 0;
}
static int finch_cpu_has_c(const char *n) {
if (!strcmp(n, "neon")) return 1;
if (!strcmp(n, "fp16")) return sysctl_flag("hw.optional.arm.FEAT_FP16");
if (!strcmp(n, "dotprod")) return sysctl_flag("hw.optional.arm.FEAT_DotProd");
if (!strcmp(n, "i8mm")) return sysctl_flag("hw.optional.arm.FEAT_I8MM");
if (!strcmp(n, "bf16")) return sysctl_flag("hw.optional.arm.FEAT_BF16");
return 0; // no SVE on Apple chips
}
#elif defined(__aarch64__) && defined(__linux__)
#include <sys/auxv.h>
static int finch_cpu_has_c(const char *n) {
unsigned long h = getauxval(AT_HWCAP), h2 = getauxval(AT_HWCAP2);
if (!strcmp(n, "neon")) return 1;
if (!strcmp(n, "fp16")) return (h >> 10) & 1; // HWCAP_ASIMDHP
if (!strcmp(n, "dotprod")) return (h >> 20) & 1; // HWCAP_ASIMDDP
if (!strcmp(n, "sve")) return (h >> 22) & 1; // HWCAP_SVE
if (!strcmp(n, "sve2")) return (h2 >> 1) & 1; // HWCAP2_SVE2
if (!strcmp(n, "i8mm")) return (h2 >> 13) & 1; // HWCAP2_I8MM
if (!strcmp(n, "bf16")) return (h2 >> 14) & 1; // HWCAP2_BF16
return 0;
}
#else
static int finch_cpu_has_c(const char *n) {
(void)n;
return 0;
}
#endif
int finch_cpu_has(const FStr *name) {
char n[32];
int64_t len = name->len < 31 ? name->len : 31;
memcpy(n, name->ptr ? name->ptr : "", (size_t)len);
n[len] = 0;
return finch_cpu_has_c(n) ? 1 : 0;
}

3
shell.nix
View file

@ -15,4 +15,7 @@ pkgs.mkShell {
vulkan-headers
vulkan-loader
];
# Nix's clang wrapper drops -march=native unless this is 0 (finch --cpu native also tells it so itself)
NIX_ENFORCE_NO_NATIVE = 0;
}

5
src/ast.fch
View file

@ -44,7 +44,7 @@ fn vecType(str name) -> Type {
return Type()
}
lane := name.sub(0, x)
if !["i8", "i16", "i32", "i64", "u8", "u16", "u32", "u64", "f32", "f64"].contains(lane) {
if !["i8", "i16", "i32", "i64", "u8", "u16", "u32", "u64", "f32", "f64", "bool"].contains(lane) {
return Type()
}
n := 0
@ -57,7 +57,7 @@ fn vecType(str name) -> Type {
if n != 2 && n != 4 && n != 8 && n != 16 && n != 32 && n != 64 {
return Type()
}
bits := int(lane.sub(1, lane.len)) or 0
bits := int(lane.sub(1, lane.len)) or 1 // bool: a mask, one bit a lane
if n * bits > 512 {
return Type() // the widest registers (AVX-512) hold 512 bits
}
@ -147,6 +147,7 @@ struct Node {
bool flag // decl: has a type; or: the right side is a { block }
bool fallible // fn: -> T! or -> !
bool exported // export fn: a C function of a library (finch build --shared)
str cpu // @cpu("avx2,fma") fn: the processor features the function may use
[]Node kids
[]str names // named arguments: Point(x: 1)
int line

14
src/docs.fch
View file

@ -26,6 +26,9 @@ fn builtins() -> []Doc {
doc("max", "max(a, b, ...) -> number", "The biggest of the numbers, in the type they all fit."),
doc("abs", "abs(x) -> number", "x without its minus sign (any number type)."),
doc("clamp", "clamp(x, lo, hi) -> number", "x, but at least lo and at most hi."),
doc("cpu_has", "cpu_has(str feature) -> bool", "Does the processor running the program have this feature (\"avx2\", \"avx512f\", \"avxvnni\", \"dotprod\", \"i8mm\" ...)? Check before calling a @cpu(...) function."),
doc("concat", "concat(V a, V b) -> V2", "One vector with a's lanes, then b's (twice as many)."),
doc("select", "select(mask, a, b) -> V", "Lane by lane: a where the mask is true, b where it is false. The mask comes from comparing vectors (v > 0.0)."),
doc("fma", "fma(a, b, c) -> float", "a * b + c with one rounding, always (numbers or vectors). a * b + c written out is never fused: two roundings."),
doc("muladd", "muladd(a, b, c) -> float", "a * b + c the fastest way here: one fused instruction where the processor has one, else a multiply and an add."),
doc("f32_bits", "f32_bits(f32 x) -> u32", "The bits of x, unchanged (also f32xN -> u32xN). f64_bits gives a u64."),
@ -119,7 +122,16 @@ fn vecMethods() -> []Doc {
doc("abs", "abs() -> V", "Every lane without its minus sign."),
doc("fma", "fma(V b, V c) -> V", "v * b + c with one rounding, always (f32/f64 lanes). One instruction with FMA hardware, slower without."),
doc("muladd", "muladd(V b, V c) -> V", "v * b + c the fastest way here: fused into one instruction where the processor can (f32/f64 lanes)."),
doc("store", "store([]T list, int i)", "Writes the lanes into list[i], list[i+1], ... (or through a ptr[T]).")
doc("store", "store([]T list, int i)", "Writes the lanes into list[i], list[i+1], ... (or through a ptr[T])."),
doc("store_part", "store_part([]T list, int i)", "Writes the lanes that fit in the list (for its last, shorter piece)."),
doc("shuffle", "shuffle(int a, int b, ...) -> V", "A vector of the lanes with these numbers (written in the code): v.shuffle(3, 2, 1, 0) reverses an f32x4."),
doc("lo", "lo() -> V", "The lower half of the lanes."),
doc("hi", "hi() -> V", "The upper half of the lanes."),
doc("lookup", "lookup(I indices) -> V", "For every lane of indices: this vector's lane (index % len). A byte-table lookup (pshufb, tbl)."),
doc("dot_add", "dot_add(a, b) -> i32xN", "On an i32xN of sums: each lane gains the sum of 4 byte products of a and b (u8/i8, 4N lanes). One instruction with AVX-VNNI, AVX-512 VNNI or ARM's dot product."),
doc("any", "any() -> bool", "A mask: is any lane true?"),
doc("all", "all() -> bool", "A mask: are all lanes true?"),
doc("count", "count() -> int", "A mask: how many lanes are true.")
]
}

2
src/embedded.fch
View file

File diff suppressed because one or more lines are too long

574
src/gen.fch
View file

@ -33,6 +33,7 @@ struct Options {
bool trace // runtime errors show the chain of calls (off for --shared and --no-trace)
bool test // finch test: main runs the test "..." blocks instead
[]str testModules // the modules whose tests run ("" is the main file); not the standard library's
[]str features // what the processor it is built for has (from --cpu, asked of clang): avx2, i8mm ...
}
// A generic struct or function, kept as written until something asks for a version of it.
@ -218,6 +219,8 @@ struct Gen {
bool deferOn // running a defer: variables declared after it are out of sight
int deferLimit
int inDefer
[]str features // the processor features of the whole program (--cpu)
[]str curFeatures // and of the function being built: those and its @cpu(...)
}
// ---------- basics ----------
@ -1460,6 +1463,9 @@ fn coerce(ptr[Gen] g, Val v, ast.Type want, ast.Node at, str what) -> Val {
if want.kind == "str" && (isNumber(v.ty) || v.ty.kind == "bool" || v.ty.kind == "char") {
msg = msg + " (turn it into text with str(...))"
}
if want.kind == "bool" && v.ty.kind == "vec" && v.ty.elem[0].kind == "bool" {
msg = msg + " (a comparison of vectors has an answer for every lane: use .all() or .any())"
}
if want.kind == "vec" && isNumber(v.ty) {
msg = msg + " (" + ast.show(want) + "(x) puts x in every lane)"
}
@ -2357,6 +2363,13 @@ fn define(ptr[Gen] g, int si) {
g.instNote = " (while building " + g.cur.shown + ")"
}
f := g.progs[g.cur.prog].fns[g.cur.index]
ownCpu := cpuOf(g, f)
g.curFeatures = g.features
for x in ownCpu {
if !g.curFeatures.contains(x) {
g.curFeatures.push(x)
}
}
[]str empty
g.body = empty
g.allocas = empty
@ -2441,8 +2454,20 @@ fn define(ptr[Gen] g, int si) {
}
}
attach := ""
if ownCpu.len > 0 {
[]str plus
wide := false
for x in ownCpu {
plus.push("+" + llvmFeature(x))
wide = wide || x.starts_with("avx512")
}
if wide {
plus.push("+evex512") // LLVM 18 and later: the 512-bit registers themselves (clang adds it for C too)
}
attach = " \"target-features\"=\"" + plus.join(",") + "\""
}
if dbgsp >= 0 {
attach = " !dbg !" + str(dbgsp)
attach = attach + " !dbg !" + str(dbgsp)
}
g.dbgScope = -1
g.out.push("define internal " + retTy(g, g.cur) + " " + g.cur.ll + "(" + params.join(", ") + ")" + attach + " {\nentry:\n" +
@ -3680,7 +3705,7 @@ fn expr(ptr[Gen] g, ast.Node e) -> Val {
fn unary(ptr[Gen] g, ast.Node e) -> Val {
v := expr(g, e.kids[0])
if v.ty.kind == "vec" && e.name != "!" {
if v.ty.kind == "vec" && (e.name != "!" || v.ty.elem[0].kind == "bool") {
return vecUnary(g, e, v)
}
if e.name == "!" {
@ -4453,6 +4478,18 @@ fn indexOf(ptr[Gen] g, ast.Node e, Ref obj, bool write) -> Ref {
return mapIndex(g, e, obj, t, write)
}
i := coerce(g, expr(g, e.kids[1]), T("int"), e.kids[1], "an index").v
if t.kind == "vec" && t.elem[0].kind == "bool" {
// a mask keeps a lane in a bit, so a lane is read from the whole value
if write {
fail(g, e, "a lane of a mask can't be set by itself; make a new mask (with a comparison or select)")
}
boundsCheck(g, i, str(t.count), e)
mv := obj.val.v
if obj.isPlace {
mv = load(g, llty(g, t), obj.pl.addr)
}
return Ref(isPlace: false, val: Val(v: op(g, "extractelement " + llty(g, t) + " " + mv + ", i64 " + i), ty: T("bool")))
}
if t.kind == "vec" {
// a lane: v[2], v[i] = x
boundsCheck(g, i, str(t.count), e)
@ -5572,6 +5609,15 @@ fn call(ptr[Gen] g, ast.Node c) -> Val {
if ei >= 0 {
return enumFrom(g, c, ei)
}
if n == "select" && lookup(g, n) < 0 && findFn(g, g.module, n) < 0 && !g.ci.fns.has(n) {
return selectCall(g, c)
}
if n == "cpu_has" && lookup(g, n) < 0 && findFn(g, g.module, n) < 0 {
return cpuHasCall(g, c)
}
if n == "concat" && lookup(g, n) < 0 && findFn(g, g.module, n) < 0 && !g.ci.fns.has(n) {
return concatCall(g, c)
}
if ["fma", "muladd"].contains(n) {
return fmaCall(g, c) // Finch's, also for f32 and vectors (C's fma is the same for one double)
}
@ -5816,6 +5862,7 @@ fn callFn(ptr[Gen] g, int fi, ast.Node c, int first, str shown, str selfAddr, st
// The call itself, once the arguments are values ("i64 5"); the receiver's address, if any, is already first.
fn emitCall(ptr[Gen] g, FnSig sig, ast.Node c, []str args, str shown) -> Val {
checkCpuCall(g, sig, c, shown)
if sig.fallible && g.ftarget != nodeKey(c) {
fail(g, c, "'" + shown + "' can fail, so say what happens then: " + shown + "(...) or <value>, " + shown + "(...) or { ... }, or try " + shown + "(...)")
}
@ -6504,6 +6551,7 @@ fn runtimeDecls() -> str {
"declare i32 @finch_test_summary()",
"declare void @finch_input(ptr, ptr)",
"declare i32 @finch_file_exists(ptr)",
"declare i32 @finch_cpu_has(ptr)",
"declare i32 @finch_str_try_int(ptr, ptr, ptr)",
"declare i32 @finch_str_try_float(ptr, ptr, ptr)",
"declare i32 @finch_try_read_file(ptr, ptr, ptr)",
@ -6552,7 +6600,7 @@ fn assemble(ptr[Gen] g) -> str {
fn generate([]ast.Program progs, []str files, cimport.CImports ci, Options opt) -> str {
shared := opt.shared
gen := Gen(progs: progs, files: files, ci: ci, shared: opt.shared, debug: opt.debug, index: opt.index, cwd: opt.cwd, tg: opt.target, dbgScope: -1)
gen := Gen(progs: progs, files: files, ci: ci, shared: opt.shared, debug: opt.debug, index: opt.index, cwd: opt.cwd, tg: opt.target, dbgScope: -1, features: opt.features)
g := addr(gen)
g.trace = opt.trace && !opt.shared
if g.debug {
@ -6743,9 +6791,15 @@ fn vecMake(ptr[Gen] g, ast.Node c, ast.Type vt) -> Val {
ft = first.pl.ty
}
if ft.kind == "array" || ft.kind == "ptr" {
if c.kids.len != 2 {
if lane.kind == "bool" {
fail(g, c, "a mask isn't loaded from a list; compare instead, like f32x8(list, i) > 0.0")
}
if c.kids.len != 2 && c.kids.len != 3 {
fail(g, c, shown + "(list, i): which index to start at? (" + shown + "(list, 0) for the first " + str(n) + ")")
}
if c.kids.len == 3 && ft.kind == "ptr" {
fail(g, c, shown + "(list, i, fill) needs a list: a ptr doesn't know where its numbers end")
}
if ft.elem.len == 0 || !ast.same(ft.elem[0], lane) {
fail(g, c.kids[0], shown + "(list, i) loads " + ast.show(lane) + " numbers, but this is " + ast.show(ft))
}
@ -6754,6 +6808,10 @@ fn vecMake(ptr[Gen] g, ast.Node c, ast.Type vt) -> Val {
hold(g, first.val)
}
i := coerce(g, expr(g, c.kids[1]), T("int"), c.kids[1], "an index").v
fill := ""
if c.kids.len == 3 {
fill = coerce(g, expr(g, c.kids[2]), lane, c.kids[2], "the number for the lanes past the end").v
}
heldEnd(g, mark)
data := ""
if ft.kind == "ptr" {
@ -6766,8 +6824,22 @@ fn vecMake(ptr[Gen] g, ast.Node c, ast.Type vt) -> Val {
if first.isPlace {
av = load(g, "{ ptr, i64, i64 }", first.pl.addr)
}
vecBounds(g, i, n, op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1"), c)
len := op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1")
data = op(g, "extractvalue { ptr, i64, i64 } " + av + ", 0")
if fill.len > 0 {
// the end of a list: the lanes that are in it are loaded, the others get `fill`
m := tailMask(g, i, n, len, c)
ep := op(g, "getelementptr " + llty(g, lane) + ", ptr " + data + ", i64 " + i)
ml := llty(g, shaped(vt, "bool"))
fname := "@llvm.masked.load." + vecSuffix(vt) + ".p0"
declareOnce(g, "declare " + lt + " " + fname + "(ptr, i32, " + ml + ", " + lt + ")")
out := op(g, "call " + lt + " " + fname + "(ptr " + ep + ", i32 " + laneBytes(vt) + ", " + ml + " " + m + ", " + lt + " " + splat(g, fill, vt) + ")")
if !first.isPlace {
release(g, first.val)
}
return Val(v: out, ty: vt)
}
vecBounds(g, i, n, len, c)
}
ep := op(g, "getelementptr inbounds " + llty(g, lane) + ", ptr " + data + ", i64 " + i)
out := op(g, "load " + lt + ", ptr " + ep + ", align " + laneBytes(vt))
@ -6816,6 +6888,20 @@ fn vecConvert(ptr[Gen] g, Val v, ast.Type to, ast.Node at) -> Val {
}
fl := from.elem[0]
tl := to.elem[0]
if tl.kind == "bool" {
// a lane that isn't 0 is true
if isFloat(fl) {
return Val(v: op(g, "fcmp une " + llty(g, from) + " " + v.v + ", zeroinitializer"), ty: to)
}
return Val(v: op(g, "icmp ne " + llty(g, from) + " " + v.v + ", zeroinitializer"), ty: to)
}
if fl.kind == "bool" {
// true is 1
if isFloat(tl) {
return Val(v: op(g, "uitofp " + llty(g, from) + " " + v.v + " to " + llty(g, to)), ty: to)
}
return Val(v: op(g, "zext " + llty(g, from) + " " + v.v + " to " + llty(g, to)), ty: to)
}
inst := ""
if isInt(fl) && isInt(tl) {
if bitsOf(fl) == bitsOf(tl) {
@ -6849,6 +6935,12 @@ fn vecConvert(ptr[Gen] g, Val v, ast.Type to, ast.Node at) -> Val {
fn vecUnary(ptr[Gen] g, ast.Node e, Val v) -> Val {
lane := v.ty.elem[0]
lt := llty(g, v.ty)
if lane.kind == "bool" {
if e.name != "!" && e.name != "~" {
fail(g, e, "can't put '" + e.name + "' in front of a mask (" + ast.show(v.ty) + "); '!' flips it")
}
return Val(v: op(g, "xor " + lt + " " + v.v + ", " + splat(g, "true", v.ty)), ty: v.ty)
}
if e.name == "~" {
if !isInt(lane) {
fail(g, e, "'~' flips the bits of whole numbers, not of " + ast.show(v.ty))
@ -6869,6 +6961,9 @@ fn vecArith(ptr[Gen] g, str o, Val l, Val r, ast.Node at) -> Val {
a := asVec(g, l, vt, o, at)
b := asVec(g, r, vt, o, at)
lane := vt.elem[0]
if lane.kind == "bool" && o != "&" && o != "|" && o != "^" {
fail(g, at, "can't use '" + o + "' on a mask (" + ast.show(vt) + "); masks take & | ^ and !")
}
names := ["+", "-", "*", "/", "&", "|", "^", "<<", ">>"]
insts := ["add", "sub", "mul", "", "and", "or", "xor", "shl", "ashr"]
if isUnsigned(lane) {
@ -6891,27 +6986,22 @@ fn vecArith(ptr[Gen] g, str o, Val l, Val r, ast.Node at) -> Val {
return Val(v: op(g, insts[k] + " " + llty(g, vt) + " " + a.v + ", " + b.v), ty: vt)
}
// v == w: every lane is equal. (Lane by lane: min, max, or v[i] < w[i].)
// v < w, v == w ...: a mask (boolxN), the answer for every lane. m.all(), m.any(), select(m, a, b) use it.
fn vecCompare(ptr[Gen] g, str o, Val l, Val r, ast.Node at) -> Val {
vt := vecOf(l, r)
if o != "==" && o != "!=" {
fail(g, at, "'" + o + "' on " + ast.show(vt) + " would have an answer for every lane; use min(v, w), max(v, w), or compare lanes: v[i] " + o + " w[i]")
}
a := asVec(g, l, vt, o, at)
b := asVec(g, r, vt, o, at)
cmp := ""
if isFloat(vt.elem[0]) {
cmp = op(g, "fcmp oeq " + llty(g, vt) + " " + a.v + ", " + b.v)
} else {
cmp = op(g, "icmp eq " + llty(g, vt) + " " + a.v + ", " + b.v)
lane := vt.elem[0]
mask := shaped(vt, "bool")
if lane.kind == "bool" && o != "==" && o != "!=" {
fail(g, at, "masks can only be compared with == and !=")
}
bits := ast.Type(kind: "vec", count: vt.count)
bits.elem.push(T("bool"))
all := vecIntrinsic(g, "i1", "vector.reduce.and", bits, ["<" + str(vt.count) + " x i1>"], [cmp], "")
if o == "!=" {
return Val(v: op(g, "xor i1 " + all + ", true"), ty: T("bool"))
if isFloat(lane) {
ops := ["==", "!=", "<", "<=", ">", ">="]
preds := ["oeq", "une", "olt", "ole", "ogt", "oge"]
return Val(v: op(g, "fcmp " + preds[ops.find(o)] + " " + llty(g, vt) + " " + a.v + ", " + b.v), ty: mask)
}
return Val(v: all, ty: T("bool"))
return Val(v: op(g, "icmp " + intPred(o, isSigned(lane)) + " " + llty(g, vt) + " " + a.v + ", " + b.v), ty: mask)
}
// min(v, w) and max(v, w), lane by lane (clamp is the two)
@ -6979,6 +7069,9 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
lane := vt.elem[0]
lt := llty(g, vt)
n := m.name
if lane.kind == "bool" && n != "shuffle" && n != "lo" && n != "hi" {
return maskMethod(g, m, v)
}
if n == "sum" || n == "min" || n == "max" {
need(g, m, 0)
how := n
@ -7009,7 +7102,7 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
need(g, m, 0)
return Val(v: vecIntrinsic(g, lt, n, vt, [lt], [v.v], ""), ty: vt)
}
if n == "store" {
if n == "store" || n == "store_part" {
need(g, m, 2)
dst := ref(g, m.kids[1], true)
dt := dst.val.ty
@ -7020,7 +7113,10 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
fail(g, m.kids[1], "store(list, i) writes " + ast.show(lane) + " numbers into a []" + ast.show(lane) + " or a ptr[" + ast.show(lane) + "], but this is " + ast.show(dt))
}
if dt.kind == "array" && !dst.isPlace {
fail(g, m.kids[1], "store(list, i) needs a list in a variable (this one would be thrown away)")
fail(g, m.kids[1], n + "(list, i) needs a list in a variable (this one would be thrown away)")
}
if n == "store_part" && dt.kind != "array" {
fail(g, m.kids[1], "store_part(list, i) needs a list: a ptr doesn't know where its numbers end")
}
i := coerce(g, expr(g, m.kids[2]), T("int"), m.kids[2], "an index").v
data := ""
@ -7031,14 +7127,31 @@ fn vecMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
}
} else {
av := load(g, "{ ptr, i64, i64 }", dst.pl.addr)
vecBounds(g, i, vt.count, op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1"), m)
len := op(g, "extractvalue { ptr, i64, i64 } " + av + ", 1")
data = op(g, "extractvalue { ptr, i64, i64 } " + av + ", 0")
if n == "store_part" {
// only the lanes that are in the list are written
mk := tailMask(g, i, vt.count, len, m)
ep := op(g, "getelementptr " + llty(g, lane) + ", ptr " + data + ", i64 " + i)
ml := llty(g, shaped(vt, "bool"))
fname := "@llvm.masked.store." + vecSuffix(vt) + ".p0"
declareOnce(g, "declare void " + fname + "(" + lt + ", ptr, i32, " + ml + ")")
emit(g, "call void " + fname + "(" + lt + " " + v.v + ", ptr " + ep + ", i32 " + laneBytes(vt) + ", " + ml + " " + mk + ")")
return Val(ty: T("void"))
}
vecBounds(g, i, vt.count, len, m)
}
ep := op(g, "getelementptr inbounds " + llty(g, lane) + ", ptr " + data + ", i64 " + i)
emit(g, "store " + lt + " " + v.v + ", ptr " + ep + ", align " + laneBytes(vt))
return Val(ty: T("void"))
}
fail(g, m, "a " + ast.show(vt) + " has no method '" + n + "' (it has: sum, min, max, dot, abs, sqrt, floor, ceil, round, fma, muladd, store)")
if n == "shuffle" || n == "lo" || n == "hi" || n == "lookup" {
return shuffleMethod(g, m, v)
}
if n == "dot_add" {
return dotAdd(g, m, v)
}
fail(g, m, "a " + ast.show(vt) + " has no method '" + n + "' (it has: sum, min, max, dot, abs, sqrt, floor, ceil, round, fma, muladd, store, store_part, shuffle, lo, hi, lookup, dot_add)")
return v
}
@ -7213,3 +7326,416 @@ fn bitsCall(ptr[Gen] g, ast.Node c) -> Val {
h := op(g, "fptrunc " + il + " " + v.v + " to " + hl)
return Val(v: op(g, "bitcast " + hl + " " + h + " to " + ol), ty: out)
}
// ---------- masks: boolxN ----------
//
// A comparison of vectors gives a mask, one true/false a lane (<N x i1>). It picks lanes with select, says
// whether any or all lanes are true, and loads and stores the end of a list (f32x8(list, i, fill), store_part).
// lanes 0, 1, ... n-1 that are inside a list of `len` from index i (i may be past the end: then none)
fn tailMask(ptr[Gen] g, str i, int n, str len, ast.Node at) -> str {
panicIf(g, op(g, "icmp slt i64 " + i + ", 0"), "a vector's index is below 0", at)
rest := op(g, "sub i64 " + len + ", " + i)
[]str lanes
for k in 0..n {
lanes.push("i64 " + str(k))
}
wide := ast.Type(kind: "vec", count: n)
wide.elem.push(T("int"))
return op(g, "icmp slt <" + str(n) + " x i64> <" + lanes.join(", ") + ">, " + splat(g, rest, wide))
}
// select(mask, a, b): a's lane where the mask is true, b's where it is false
fn selectCall(ptr[Gen] g, ast.Node c) -> Val {
if g.index {
noteRef(g, c, c.col, c.name.len, docs.hover(docs.find(docs.builtins(), "select")), "", 0, 0)
}
checkArgs(g, c, 0, 3, "select")
m := expr(g, c.kids[0])
if m.ty.kind != "vec" || m.ty.elem[0].kind != "bool" {
fail(g, c.kids[0], "select(mask, a, b) takes a mask first, from a comparison of vectors like v > 0.0; this is " + ast.show(m.ty))
}
a := expr(g, c.kids[1])
b := expr(g, c.kids[2])
vt := a.ty
if vt.kind != "vec" {
vt = b.ty
}
if vt.kind != "vec" {
fail(g, c, "select(mask, a, b): a or b has to be a vector (the other may be a number for every lane)")
}
if vt.count != m.ty.count {
fail(g, c, "the mask has " + str(m.ty.count) + " lanes, but " + ast.show(vt) + " has " + str(vt.count))
}
av := asVec(g, a, vt, "select", c.kids[1])
bv := asVec(g, b, vt, "select", c.kids[2])
lt := llty(g, vt)
return Val(v: op(g, "select " + llty(g, m.ty) + " " + m.v + ", " + lt + " " + av.v + ", " + lt + " " + bv.v), ty: vt)
}
fn maskMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
n := m.name
ml := llty(g, v.ty)
if n == "any" || n == "all" {
need(g, m, 0)
how := "or"
if n == "all" {
how = "and"
}
return Val(v: vecIntrinsic(g, "i1", "vector.reduce." + how, v.ty, [ml], [v.v], ""), ty: T("bool"))
}
if n == "count" {
need(g, m, 0)
wide := shaped(v.ty, "i32")
wl := llty(g, wide)
z := op(g, "zext " + ml + " " + v.v + " to " + wl)
sum := vecIntrinsic(g, "i32", "vector.reduce.add", wide, [wl], [z], "")
return Val(v: op(g, "zext i32 " + sum + " to i64"), ty: T("int"))
}
fail(g, m, "a mask (" + ast.show(v.ty) + ") has .len, [i] and the methods any, all, count; pick lanes with select(mask, a, b)")
return v
}
// ---------- moving lanes around ----------
fn indexList([]int idx) -> str {
[]str parts
for k in idx {
parts.push("i32 " + str(k))
}
return "<" + parts.join(", ") + ">"
}
fn shuffleOf(ptr[Gen] g, Val v, []int idx, str v2) -> Val {
vt := v.ty
out := ast.Type(kind: "vec", count: idx.len)
out.elem.push(vt.elem[0])
second := v2
if second.len == 0 {
second = "poison"
}
return Val(v: op(g, "shufflevector " + llty(g, vt) + " " + v.v + ", " + llty(g, vt) + " " + second + ", <" + str(idx.len) + " x i32> " + indexList(idx)), ty: out)
}
fn fitsVector(ptr[Gen] g, ast.Node at, ast.Type lane, int n) {
if n != 2 && n != 4 && n != 8 && n != 16 && n != 32 && n != 64 {
fail(g, at, "a vector has 2, 4, 8, 16, 32 or 64 lanes, not " + str(n))
}
bits := bitsOf(lane)
if n * bits > 512 {
fail(g, at, "that would be " + str(n) + " lanes of " + ast.show(lane) + ", more than 512 bits")
}
}
// v.shuffle(3, 2, 1, 0): lanes picked by number (numbers written in the code); v.lo(), v.hi(): the halves;
// table.lookup(idx): table[idx % table.len] for every lane of idx (pshufb on x86, tbl on ARM)
fn shuffleMethod(ptr[Gen] g, ast.Node m, Val v) -> Val {
vt := v.ty
n := m.name
if n == "lo" || n == "hi" {
need(g, m, 0)
if vt.count < 4 {
fail(g, m, "a " + ast.show(vt) + " has only " + str(vt.count) + " lanes; take them with v[0] and v[1]")
}
[]int idx
start := 0
if n == "hi" {
start = vt.count / 2
}
for k in 0..vt.count / 2 {
idx.push(start + k)
}
return shuffleOf(g, v, idx, "")
}
if n == "shuffle" {
[]int idx
for q in 1..m.kids.len {
x := expr(g, m.kids[q])
if !isInt(x.ty) || !x.lit {
fail(g, m.kids[q], "shuffle(...) takes lane numbers written in the code (or constants), like v.shuffle(3, 2, 1, 0)")
}
k := int(x.v) or -1
if k < 0 || k >= vt.count {
fail(g, m.kids[q], "lane " + x.v + " isn't in a " + ast.show(vt) + " (its lanes are 0 to " + str(vt.count - 1) + ")")
}
idx.push(k)
}
fitsVector(g, m, vt.elem[0], idx.len)
return shuffleOf(g, v, idx, "")
}
// lookup
need(g, m, 1)
ix := expr(g, m.kids[1])
if ix.ty.kind != "vec" || !isInt(ix.ty.elem[0]) {
fail(g, m.kids[1], "lookup(indices) takes a vector of whole numbers (like u8x16), but this is " + ast.show(ix.ty))
}
if vt.elem[0].kind == "bool" {
fail(g, m, "a mask can't be a lookup table")
}
fitsVector(g, m.kids[1], vt.elem[0], ix.ty.count)
il := llty(g, ix.ty)
masked := op(g, "and " + il + " " + ix.v + ", " + splat(g, str(vt.count - 1), ix.ty))
out := shaped(ix.ty, vt.elem[0].kind)
ol := llty(g, out)
el := llty(g, vt.elem[0])
xl := llty(g, ix.ty.elem[0])
cur := "poison"
for k in 0..ix.ty.count {
at := op(g, "extractelement " + il + " " + masked + ", i64 " + str(k))
lane := op(g, "extractelement " + llty(g, vt) + " " + v.v + ", " + xl + " " + at)
cur = op(g, "insertelement " + ol + " " + cur + ", " + el + " " + lane + ", i64 " + str(k))
}
return Val(v: cur, ty: out)
}
// concat(a, b): a's lanes, then b's
fn concatCall(ptr[Gen] g, ast.Node c) -> Val {
if g.index {
noteRef(g, c, c.col, c.name.len, docs.hover(docs.find(docs.builtins(), "concat")), "", 0, 0)
}
checkArgs(g, c, 0, 2, "concat")
a := expr(g, c.kids[0])
b := expr(g, c.kids[1])
if a.ty.kind != "vec" || !ast.same(a.ty, b.ty) {
fail(g, c, "concat(a, b) joins two vectors of the same type, but these are " + ast.show(a.ty) + " and " + ast.show(b.ty))
}
fitsVector(g, c, a.ty.elem[0], a.ty.count * 2)
[]int idx
for k in 0..a.ty.count * 2 {
idx.push(k)
}
return shuffleOf(g, a, idx, b.v)
}
// ---------- processor features: @cpu(...), cpu_has(...) ----------
//
// --cpu gives the whole program a processor (main.fch asks clang which features that means); @cpu("avx2,fma") gives one
// function more, as LLVM's "target-features". Such a function is only called after cpu_has("avx2") said yes. Features of
// the other architecture are dropped, so one source holds the x86 and the ARM versions side by side.
fn x86Features() -> []str {
return ["sse3", "ssse3", "sse4.1", "sse4.2", "popcnt", "avx", "avx2", "fma", "f16c", "bmi1", "bmi2", "avx512f", "avx512bw",
"avx512dq", "avx512vl", "avx512vnni", "avx512bf16", "avx512fp16", "avxvnni"]
}
fn armFeatures() -> []str {
return ["neon", "fp16", "dotprod", "i8mm", "bf16", "sve", "sve2"]
}
// the name LLVM uses for a feature
fn llvmFeature(str x) -> str {
if x == "bmi1" {
return "bmi"
}
if x == "fp16" {
return "fullfp16"
}
return x
}
// the features of a level: x86-64-v3 is the AVX2 generation, v4 the AVX-512 one
fn levelFeatures(str name) -> []str {
[]str out
if name == "x86-64-v2" || name == "x86-64-v3" || name == "x86-64-v4" {
for x in ["sse3", "ssse3", "sse4.1", "sse4.2", "popcnt"] {
out.push(x)
}
}
if name == "x86-64-v3" || name == "x86-64-v4" {
for x in ["avx", "avx2", "bmi1", "bmi2", "f16c", "fma"] {
out.push(x)
}
}
if name == "x86-64-v4" {
for x in ["avx512f", "avx512bw", "avx512dq", "avx512vl"] {
out.push(x)
}
}
return out
}
// a function's own @cpu features, for the architecture it is built for
fn cpuOf(ptr[Gen] g, ast.Node f) -> []str {
[]str out
if f.cpu.len == 0 {
return out
}
for part in f.cpu.split(",") {
x := part.trim()
names := levelFeatures(x)
if names.len == 0 {
if !x86Features().contains(x) && !armFeatures().contains(x) {
fail(g, f, "@cpu: '" + x + "' isn't a feature Finch knows (x86: " + x86Features().join(", ") + ", x86-64-v2/v3/v4; ARM: " + armFeatures().join(", ") + ")")
}
names.push(x)
}
for y in names {
if armFeatures().contains(y) == g.tg.arm64 && !out.contains(y) {
out.push(y)
}
}
}
return out
}
// a vector wider than 16 bytes travels in different registers depending on the features (two xmm, or one ymm), so it
// can't cross from a function with one set of features to a function with another
fn checkCpuCall(ptr[Gen] g, FnSig sig, ast.Node c, str shown) {
if sig.prog < 0 || sig.prog >= g.progs.len || sig.index >= g.progs[sig.prog].fns.len {
return
}
callee := g.progs[sig.prog].fns[sig.index].cpu
mine := g.progs[g.cur.prog].fns[g.cur.index].cpu
if callee == mine {
return
}
[]ast.Type ts = sig.params
ts.push(sig.ret)
for t in ts {
if t.kind == "vec" && t.count * bitsOf(t.elem[0]) > 128 {
fail(g, c, "'" + shown + "' has other @cpu features than this function, so a " + ast.show(t) + " can't be passed between them (its registers differ); pass a list or a ptr, or give both the same @cpu")
}
}
}
// cpu_has("avx2"): does the processor running the program have it? (false for the other architecture's names)
fn cpuHasCall(ptr[Gen] g, ast.Node c) -> Val {
if g.index {
noteRef(g, c, c.col, c.name.len, docs.hover(docs.find(docs.builtins(), "cpu_has")), "", 0, 0)
}
checkArgs(g, c, 0, 1, "cpu_has")
if c.kids[0].kind == "str" {
x := c.kids[0].text
if !x86Features().contains(x) && !armFeatures().contains(x) {
fail(g, c.kids[0], "cpu_has: '" + x + "' isn't a feature Finch knows (x86: " + x86Features().join(", ") + "; ARM: " + armFeatures().join(", ") + ")")
}
}
s := strArg(g, c.kids[0], "the feature's name")
r := op(g, "call i32 @finch_cpu_has(ptr " + spill(g, s) + ")")
release(g, s)
return Val(v: op(g, "icmp ne i32 " + r + ", 0"), ty: T("bool"))
}
// ---------- acc.dot_add(a, b): the step of an int8 dot product ----------
//
// acc is i32xN; a and b have 4N bytes (a u8 or i8, b i8 or u8). Lane k of acc gains a[4k]*b[4k] + ... + a[4k+3]*b[4k+3],
// exactly (no saturation). That is vpdpbusd (AVX-VNNI, AVX-512 VNNI) and usdot / sdot / udot (ARM), used when the
// function's features have them; elsewhere the same sum is written out, which LLVM makes into pmaddwd and adds.
fn dotAdd(ptr[Gen] g, ast.Node m, Val acc) -> Val {
need(g, m, 2)
at := acc.ty
if at.elem[0].kind != "i32" && at.elem[0].kind != "u32" {
fail(g, m, "dot_add is called on an i32 vector (the sums), like acc.dot_add(a, b) with acc an i32x8; this is " + ast.show(at))
}
a := expr(g, m.kids[1])
b := expr(g, m.kids[2])
for x in [a, b] {
if x.ty.kind != "vec" || (x.ty.elem[0].kind != "u8" && x.ty.elem[0].kind != "i8") || x.ty.count != at.count * 4 {
fail(g, m, "dot_add(a, b) on an " + ast.show(at) + " takes two vectors of " + str(at.count * 4) + " bytes (u8 or i8), but these are " + ast.show(a.ty) + " and " + ast.show(b.ty))
}
}
au := a.ty.elem[0].kind == "u8"
bu := b.ty.elem[0].kind == "u8"
if !au && bu {
// u8 x i8 is the same sum as i8 x u8: put the unsigned one first
t := a
a = b
b = t
au = true
bu = false
}
al := llty(g, at)
bits := at.count * 32
f := g.curFeatures
if !g.tg.arm64 && au && !bu {
vnni := (bits == 512 && f.contains("avx512vnni")) || (bits <= 256 && (f.contains("avxvnni") || (f.contains("avx512vnni") && f.contains("avx512vl"))))
if vnni {
a32 := op(g, "bitcast " + llty(g, a.ty) + " " + a.v + " to " + al)
b32 := op(g, "bitcast " + llty(g, b.ty) + " " + b.v + " to " + al)
name := "@llvm.x86.avx512.vpdpbusd." + str(bits)
declareOnce(g, "declare " + al + " " + name + "(" + al + ", " + al + ", " + al + ")")
return Val(v: op(g, "call " + al + " " + name + "(" + al + " " + acc.v + ", " + al + " " + a32 + ", " + al + " " + b32 + ")"), ty: at)
}
}
if g.tg.arm64 && bits % 128 == 0 {
which := ""
if au && !bu && f.contains("i8mm") {
which = "usdot"
} else if au && bu && f.contains("dotprod") {
which = "udot"
} else if !au && !bu && f.contains("dotprod") {
which = "sdot"
}
if which.len > 0 {
// one instruction per 16 bytes
name := "@llvm.aarch64.neon." + which + ".v4i32.v16i8"
declareOnce(g, "declare <4 x i32> " + name + "(<4 x i32>, <16 x i8>, <16 x i8>)")
parts := at.count / 4
[]str outs
for q in 0..parts {
[]int ai
for k in 0..4 {
ai.push(q * 4 + k)
}
[]int bi
for k in 0..16 {
bi.push(q * 16 + k)
}
ac := shuffleOf(g, acc, ai, "").v
ap := shuffleOf(g, a, bi, "").v
bp := shuffleOf(g, b, bi, "").v
outs.push(op(g, "call <4 x i32> " + name + "(<4 x i32> " + ac + ", <16 x i8> " + ap + ", <16 x i8> " + bp + ")"))
}
return Val(v: joinParts(g, outs, 4), ty: at)
}
}
// written out: widen, multiply, add each group of four
w := shaped(a.ty, "i32")
wl := llty(g, w)
ext := "sext"
if au {
ext = "zext"
}
aw := op(g, ext + " " + llty(g, a.ty) + " " + a.v + " to " + wl)
ext = "sext"
if bu {
ext = "zext"
}
bw := op(g, ext + " " + llty(g, b.ty) + " " + b.v + " to " + wl)
prod := Val(v: op(g, "mul " + wl + " " + aw + ", " + bw), ty: w)
sum := acc.v
for k in 0..4 {
[]int idx
for j in 0..at.count {
idx.push(4 * j + k)
}
sum = op(g, "add " + al + " " + sum + ", " + shuffleOf(g, prod, idx, "").v)
}
return Val(v: sum, ty: at)
}
// <4 x i32> pieces back into one vector of n * 4 lanes
fn joinParts(ptr[Gen] g, []str parts, int each) -> str {
if parts.len == 1 {
return parts[0]
}
[]str cur = parts
size := each
while cur.len > 1 {
[]str next
t := ast.Type(kind: "vec", count: size)
t.elem.push(T("i32"))
k := 0
while k < cur.len {
[]int idx
for j in 0..size * 2 {
idx.push(j)
}
next.push(shuffleOf(g, Val(v: cur[k], ty: t), idx, cur[k + 1]).v)
k += 2
}
cur = next
size = size * 2
}
return cur[0]
}

2
src/lexer.fch
View file

@ -70,7 +70,7 @@ fn operator(ptr[Lexer] lx) -> str {
return two
}
}
ones := "(){}[],.:+-*/%&|^~<>=!"
ones := "(){}[],.:+-*/%&|^~<>=!@"
c := lx.src.sub(lx.i, lx.i + 1)
if ones.contains(c) {
return c

76
src/linker.fch
View file

@ -371,10 +371,7 @@ struct Build {
// IR -> program. 0 if it worked.
fn build(Build b, platform.Target tg, cimport.CImports ci) -> int {
clang := getenv("FINCH_CLANG")
if clang.len == 0 {
clang = "clang"
}
clang := clangCmd()
// the C compiler of the runtime and the linker: the target's own when building for another system
cc := clang
if tg.cross {
@ -478,3 +475,74 @@ fn build(Build b, platform.Target tg, cimport.CImports ci) -> int {
}
return 0
}
// clang, as the program is compiled with it. Nix's clang wrapper removes -march=native when NIX_ENFORCE_NO_NATIVE
// is 1, as it is in nix-shell; --cpu native asks for it on purpose, so the wrapper is told to keep it.
fn clangCmd() -> str {
clang := getenv("FINCH_CLANG")
if clang.len == 0 {
clang = "clang"
}
if host.pathSeparator() != ";" {
return "NIX_ENFORCE_NO_NATIVE=0 " + clang
}
return clang
}
// -march=x86-64-v3 or -mcpu=apple-m1: how clang is told the processor
fn cpuFlag(platform.Target tg, str cpu) -> str {
if cpu.len == 0 {
return ""
}
if tg.arm64 {
return " -mcpu=" + quote(cpu)
}
return " -march=" + quote(cpu)
}
// What the processor the program is for can do, as Finch's feature names (avx2, avxvnni, i8mm ...): clang says it in
// the macros it predefines for that target and --cpu. Kept in the cache, one file per clang, target and cpu.
fn cpuFeatures(platform.Target tg, str cpu) -> []str {
clang := clangCmd()
flags := cpuFlag(tg, cpu)
if tg.cross || tg.windows {
flags = " --target=" + tg.triple + flags
}
dir := host.cacheDir() + "/cpu"
host.makeDirs(dir)
slot := dir + "/" + fingerprint2(clang + flags)
text := read_file(slot + ".txt") or ""
if text.len == 0 {
empty := slot + ".part" + str(host.pid()) + ".c"
out := slot + ".part" + str(host.pid()) + ".txt"
write_file(empty, "")
host.run(clang + flags + " -dM -E " + quote(empty) + " > " + quote(out) + " 2> " + host.nullDevice())
host.remove(empty)
text = read_file(out) or ""
if text.len == 0 {
host.remove(out)
[]str none
return none
}
host.move(out, slot + ".txt")
}
macros := ["__SSE3__", "__SSSE3__", "__SSE4_1__", "__SSE4_2__", "__POPCNT__", "__AVX__", "__AVX2__", "__FMA__", "__F16C__",
"__BMI__", "__BMI2__", "__AVX512F__", "__AVX512BW__", "__AVX512DQ__", "__AVX512VL__", "__AVX512VNNI__",
"__AVX512BF16__", "__AVX512FP16__", "__AVXVNNI__", "__ARM_NEON", "__ARM_FEATURE_FP16_VECTOR_ARITHMETIC",
"__ARM_FEATURE_DOTPROD", "__ARM_FEATURE_MATMUL_INT8", "__ARM_FEATURE_BF16_VECTOR_ARITHMETIC", "__ARM_FEATURE_SVE",
"__ARM_FEATURE_SVE2"]
names := ["sse3", "ssse3", "sse4.1", "sse4.2", "popcnt", "avx", "avx2", "fma", "f16c", "bmi1", "bmi2", "avx512f", "avx512bw",
"avx512dq", "avx512vl", "avx512vnni", "avx512bf16", "avx512fp16", "avxvnni", "neon", "fp16", "dotprod", "i8mm", "bf16",
"sve", "sve2"]
[]str have
for line in text.split("\n") {
parts := line.split(" ")
if parts.len >= 2 && parts[0] == "#define" {
k := macros.find(parts[1])
if k >= 0 && !have.contains(names[k]) {
have.push(names[k])
}
}
}
return have
}

19
src/main.fch
View file

@ -471,7 +471,15 @@ fn main([]str args) -> int {
testModules.push(prog.module) // your own files' tests, not the standard library's
}
}
ir := gen.generate(loader.progs, loader.files, ci, gen.Options(shared: shared, debug: debug, index: cmd == "index", cwd: host.cwd(), target: tg, test: cmd == "test", testModules: testModules, trace: trace))
// what the processor can do (dot_add and the like pick their instructions by it); not needed for the editor's index
[]str features
if cmd != "index" {
features = linker.cpuFeatures(tg, cpu)
if cpu == "native" && !tg.arm64 && !features.contains("sse4.2") {
print("warning: --cpu native made no difference to clang (a clang wrapper may remove -march=native; try --cpu with a name, like x86-64-v3)")
}
}
ir := gen.generate(loader.progs, loader.files, ci, gen.Options(shared: shared, debug: debug, index: cmd == "index", cwd: host.cwd(), target: tg, test: cmd == "test", testModules: testModules, trace: trace, features: features))
if cmd == "ir" || cmd == "index" {
print(ir)
return 0
@ -501,14 +509,7 @@ fn main([]str args) -> int {
if debug {
extra = extra + " -g" // or clang drops the debug info that is in the IR
}
if cpu.len > 0 {
// x86-64 names a processor with -march, ARM with -mcpu
if tg.arm64 {
extra = extra + " -mcpu=" + linker.quote(cpu)
} else {
extra = extra + " -march=" + linker.quote(cpu)
}
}
extra = extra + linker.cpuFlag(tg, cpu) // x86-64 names a processor with -march, ARM with -mcpu
[]str cflagList
for d in dirs {
cflagList.push(" -I" + linker.quote(d)) // also the program's own folders (its headers, for inline C functions)

16
src/parser.fch
View file

@ -289,6 +289,22 @@ fn parseProgram(ptr[Parser] p) -> ast.Program {
if e.flag {
prog.structs.push(sumStruct(e)) // an enum with values inside is stored as a struct (see sumStruct)
}
} else if at(p, "@") {
// @cpu("avx2,fma") fn ...: a function that may use these processor features (check with cpu_has first)
a := next(p)
name := expect(p, "ident", "an attribute name, like @cpu(\"avx2\")").text
if name != "cpu" {
fail(p, a, "the attribute here is @cpu(\"...\"): the processor features a function may use")
}
expect(p, "(", "'('")
feats := expect(p, "string", "the features in quotes, like @cpu(\"avx2,fma\")").text
expect(p, ")", "')'")
if !at(p, "fn") && !at(p, "export") {
fail(p, cur(p), "@cpu(...) goes right before a function")
}
f := parseFn(p)
f.cpu = feats
prog.fns.push(f)
} else {
prog.fns.push(parseFn(p))
}

7
tests/fail/cpu_attr_name.fch Normal file
View file

@ -0,0 +1,7 @@
// expect: @cpu: 'avx3' isn't a feature Finch knows
@cpu("avx2,avx3")
fn f() {
}
fn main() {
f()
}

4
tests/fail/cpu_has_name.fch Normal file
View file

@ -0,0 +1,4 @@
// expect: cpu_has: 'avx-512' isn't a feature Finch knows
fn main() {
print(cpu_has("avx-512"))
}

8
tests/fail/cpu_wide_vector.fch Normal file
View file

@ -0,0 +1,8 @@
// expect: can't be passed between them (its registers differ)
@cpu("avx2")
fn double(f32x8 v) -> f32x8 {
return v * 2.0
}
fn main() {
print(double(f32x8(1.0)))
}

6
tests/fail/vec_less.fch
View file

@ -1,5 +1,7 @@
// expect: use min(v, w), max(v, w), or compare lanes: v[i] < w[i]
// expect: use .all() or .any()
fn main() {
a := f32x4(1.0)
print(a < a)
if a < a {
print("no")
}
}

5
tests/fail/vec_shuffle_range.fch Normal file
View file

@ -0,0 +1,5 @@
// expect: lane 4 isn't in a f32x4 (its lanes are 0 to 3)
fn main() {
v := f32x4(1.0)
print(v.shuffle(0, 1, 2, 4))
}

6
tests/fail/vec_shuffle_var.fch Normal file
View file

@ -0,0 +1,6 @@
// expect: shuffle(...) takes lane numbers written in the code
fn main() {
v := f32x4(1.0)
k := 2
print(v.shuffle(k, 0, 1, 3))
}

69
tests/run/dot_int8.fch Normal file
View file

@ -0,0 +1,69 @@
// The int8 dot product of quantized models: plain, with dot_add, and in @cpu functions picked by cpu_has
fn plain([]u8 a, []i8 b) -> int {
total := 0
for i in 0..a.len {
total += int(a[i]) * int(b[i])
}
return total
}
// 32 bytes a step, 8 sums kept in a vector until the end
fn with_vectors([]u8 a, []i8 b) -> int {
acc := i32x8(0)
i := 0
while i + 32 <= a.len {
acc = acc.dot_add(u8x32(a, i), i8x32(b, i))
i += 32
}
return int(acc.sum())
}
@cpu("avxvnni")
fn with_vnni([]u8 a, []i8 b) -> int {
acc := i32x8(0)
i := 0
while i + 32 <= a.len {
acc = acc.dot_add(u8x32(a, i), i8x32(b, i))
i += 32
}
return int(acc.sum())
}
@cpu("avx512f,avx512bw,avx512vl,avx512vnni,i8mm")
fn with_avx512([]u8 a, []i8 b) -> int {
acc := i32x16(0)
i := 0
while i + 64 <= a.len {
acc = acc.dot_add(u8x64(a, i), i8x64(b, i))
i += 64
}
return int(acc.sum())
}
fn best([]u8 a, []i8 b) -> int {
if cpu_has("avx512vnni") && cpu_has("avx512vl") && cpu_has("avx512bw") {
return with_avx512(a, b)
}
if cpu_has("avxvnni") {
return with_vnni(a, b)
}
return with_vectors(a, b)
}
fn main() {
[]u8 a
[]i8 b
for k in 0..1024 {
a.push(u8((k * 37 + 11) % 256))
b.push(i8((k * 53 + 7) % 256 - 128))
}
want := plain(a, b)
print(want, with_vectors(a, b) == want, best(a, b) == want)
// the extremes: 255 * -128 four times in a lane, no saturation
acc := i32x4(1, 2, 3, 4).dot_add(u8x16(255), i8x16(-128))
print(acc, i32x4(0).dot_add(i8x16(-128), i8x16(-128)), i32x4(0).dot_add(u8x16(255), u8x16(255)))
print(i32x4(0).dot_add(i8x16(-1), u8x16(200)))
print(cpu_has("avx2") || cpu_has("neon") || true)
}

4
tests/run/dot_int8.out Normal file
View file

@ -0,0 +1,4 @@
-68096 true true
i32x4(-130559, -130558, -130557, -130556) i32x4(65536, 65536, 65536, 65536) i32x4(260100, 260100, 260100, 260100)
i32x4(-800, -800, -800, -800)
true

50
tests/run/masks.fch Normal file
View file

@ -0,0 +1,50 @@
// Masks: comparisons of vectors, select, any/all/count, and the end of a list
fn relu(f32x8 v) -> f32x8 {
return select(v > 0.0, v, 0.0)
}
// the sum of a list of any length: whole vectors, then the last piece with its missing lanes as 0
fn total([]f32 xs) -> f32 {
acc := f32x8(0.0)
i := 0
while i < xs.len {
acc = acc + f32x8(xs, i, 0.0)
i += 8
}
return acc.sum()
}
// add 1 to every number of a list, in place, the last piece too
fn bump([]f32 xs) {
i := 0
while i < xs.len {
(f32x8(xs, i, 0.0) + 1.0).store_part(xs, i)
i += 8
}
}
fn main() {
v := f32x8(-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, -4.0, 5.0)
m := v > 0.0
print(m, m.count(), m.any(), m.all(), m[3])
print(relu(v), select(!m, v, -1.0))
print((v >= -1.0) & (v < 3.0), (v == 0.0) | (v == 5.0), m ^ (v > 1.0))
a := i32x4(1, 5, 3, 7)
b := i32x4(4, 4, 4, 4)
print(select(a > b, a, b), (a == a).all(), (a != b).all())
u := u8x16(200)
print((u > u8x16(100)).all(), i32x4(a > b), f32x4(a < b))
k := i32x4(0, 2, 0, -1)
print(boolx4(k), boolx4(true, false, true, false))
[]f32 xs
for n in 0..13 {
xs.push(f32(n))
}
print(total(xs))
bump(xs)
print(xs)
print(f32x8(xs, 8, -1.0), f32x8(xs, 13, 9.0), f32x8(xs, 100, 7.0))
}

9
tests/run/masks.out Normal file
View file

@ -0,0 +1,9 @@
boolx8(false, false, false, true, true, true, false, true) 4 true false true
f32x8(0, 0, 0, 1, 2, 3, 0, 5) f32x8(-2, -1, 0, -1, -1, -1, -4, -1)
boolx8(false, true, true, true, true, false, false, false) boolx8(false, false, true, false, false, false, false, true) boolx8(false, false, false, true, false, false, false, false)
i32x4(4, 5, 4, 7) true true
true i32x4(0, 1, 0, 1) f32x4(1, 0, 1, 0)
boolx4(false, true, false, true) boolx4(true, false, true, false)
78
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
f32x8(9, 10, 11, 12, 13, -1, -1, -1) f32x8(9, 9, 9, 9, 9, 9, 9, 9) f32x8(7, 7, 7, 7, 7, 7, 7, 7)

27
tests/run/shuffles.fch Normal file
View file

@ -0,0 +1,27 @@
// Moving lanes: shuffle, lo/hi, concat, and a byte table lookup (how 4-bit weights are unpacked)
// sixteen 4-bit numbers, two to a byte, made into sixteen bytes through a table of what each nibble means
fn unpack4(u8x8 packed, i8x16 table) -> i8x16 {
pairs := packed.shuffle(0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7) // u8x16: each byte twice
odd := u8x16(0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1) == 1
nibbles := select(odd, pairs >> 4, pairs & 15) // low nibble first, then the high one
return table.lookup(nibbles)
}
fn main() {
v := f32x4(1.0, 2.0, 3.0, 4.0)
print(v.shuffle(3, 2, 1, 0), v.shuffle(0, 0, 1, 1, 2, 2, 3, 3), v.shuffle(1, 2))
w := i32x8(0, 1, 2, 3, 4, 5, 6, 7)
print(w.lo(), w.hi(), concat(w.hi(), w.lo()))
print(concat(w, w).len, concat(v, v * 10.0).shuffle(0, 4, 1, 5, 2, 6, 3, 7))
table := u8x16(10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25)
print(table.lookup(u8x16(15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14)))
print(table.lookup(u8x16(16, 17, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)).lo())
print(table.lookup(u8x32(3)).hi().lo().lo())
q4 := i8x16(-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7) // what each 4-bit number means
print(unpack4(u8x8(0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe), q4))
m := f32x4(1.0, -1.0, 2.0, -2.0) > 0.0
print(m.shuffle(1, 0, 3, 2), m.lo())
}

8
tests/run/shuffles.out Normal file
View file

@ -0,0 +1,8 @@
f32x4(4, 3, 2, 1) f32x8(1, 1, 2, 2, 3, 3, 4, 4) f32x2(2, 3)
i32x4(0, 1, 2, 3) i32x4(4, 5, 6, 7) i32x8(4, 5, 6, 7, 0, 1, 2, 3)
16 f32x8(1, 10, 2, 20, 3, 30, 4, 40)
u8x16(25, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24)
u8x8(10, 11, 25, 10, 10, 10, 10, 10)
u8x4(13, 13, 13, 13)
i8x16(-8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7)
boolx4(false, true, false, true) boolx2(true, false)

2
tests/run/vectors.fch
View file

@ -36,7 +36,7 @@ fn main() {
// lanes
a[0] = 9.0
print(a[0], a[3], a)
print(a == f32x4(9.0, 2.0, 3.0, 4.0), a != a)
print((a == f32x4(9.0, 2.0, 3.0, 4.0)).all(), (a != a).any())
// whole numbers, bits, conversions
n := i32x8(1, 2, 3, 4, 5, 6, 7, 8)