Finch/src/lexer.fch

// The Finch lexer, written in Finch.
// A token's kind is a word: "ident", "int", "float", "string", "char", "eof",
// or the keyword / operator itself ("fn", "{", "+=", ...).

struct Token {
    str kind
    str text
    int line
    int col
    bool nl        // a newline came before this token
}

fn isAlpha(char c) -> bool {
    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
}

fn isDigit(char c) -> bool {
    return c >= '0' && c <= '9'
}

fn isHex(char c) -> bool {
    return isDigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
}

fn isKeyword(str w) -> bool {
    keywords := ["fn", "return", "if", "else", "while", "for", "in", "break", "continue", "true", "false",
                 "null", "import", "link", "struct", "defer", "or", "try", "enum", "match", "export", "const"]
    return keywords.contains(w)
}

struct Lexer {
    str src
    str path
    int i
    int line
    int col
    bool nl
}

fn fail(ptr[Lexer] lx, int line, int col, str msg) {
    print(lx.path + ":" + str(line) + ":" + str(col) + ": error: " + msg)
    exit(1)
}

fn peek(ptr[Lexer] lx, int off) -> char {
    if lx.i + off < lx.src.len {
        return lx.src[lx.i + off]
    }
    return char(0)
}

fn advance(ptr[Lexer] lx) {
    c := lx.src[lx.i]
    if c == '\n' {
        lx.line += 1
        lx.col = 1
        lx.nl = true
    } else if int(c) & 0xC0 != 0x80 {
        lx.col += 1
    }
    lx.i += 1
}

// operators, longest first
fn operator(ptr[Lexer] lx) -> str {
    twos := ["->", "<<", ">>", "<=", ">=", "==", "!=", "&&", "||", "..", ":=", "+=", "-=", "*=", "/=", "%="]
    if lx.i + 1 < lx.src.len {
        two := lx.src.sub(lx.i, lx.i + 2)
        if twos.contains(two) {
            return two
        }
    }
    ones := "(){}[],.:+-*/%&|^~<>=!"
    c := lx.src.sub(lx.i, lx.i + 1)
    if ones.contains(c) {
        return c
    }
    return ""
}

// """ text over several lines """: the line break after the opening quotes is dropped, and so is
// the indentation of the closing quotes, from every line (the text can be indented with the code).
// Gives the text as written (escapes and \(...) still in it); cook() finishes it.
fn longString(ptr[Lexer] lx, Token t) -> str {
    for k in 0..3 {
        advance(lx)
    }
    []str parts
    while !(peek(lx, 0) == '"' && peek(lx, 1) == '"' && peek(lx, 2) == '"') {
        if lx.i >= lx.src.len {
            fail(lx, t.line, t.col, "text is never closed (missing \"\"\")")
        }
        ch := peek(lx, 0)
        if ch == '\\' && peek(lx, 1) == '(' {
            start := lx.i
            advance(lx)
            advance(lx)
            skipEmbedded(lx, t)
            parts.push(lx.src.sub(start, lx.i))
            continue
        }
        advance(lx)
        if ch == '\r' {
            continue
        }
        parts.push(str(ch))
        if ch == '\\' && lx.i < lx.src.len {
            parts.push(str(peek(lx, 0)))       // \" doesn't end the text
            advance(lx)
        }
    }
    for k in 0..3 {
        advance(lx)
    }
    lx.nl = false                       // the line breaks inside were text, not the end of a statement
    body := parts.join("")
    if body.starts_with("\n") {
        body = body.sub(1, body.len)
    }
    last := body.split("\n")
    indent := last[last.len - 1]
    if last.len > 1 && indent.trim().len == 0 {
        // the closing quotes are on their own line: their indentation goes from every line
        []str lines
        for i in 0..last.len - 1 {
            line := last[i]
            if line.starts_with(indent) {
                line = line.sub(indent.len, line.len)
            } else if line.trim().len > 0 {
                fail(lx, t.line, t.col, "a line of this text is indented less than the closing \"\"\"")
            } else {
                line = ""
            }
            lines.push(line)
        }
        return lines.join("\n")
    }
    return body
}

// "text" as written, up to the closing quote (which is skipped); escapes and \(...) are still in it
fn rawString(ptr[Lexer] lx, Token t) -> str {
    advance(lx)
    start := lx.i
    while peek(lx, 0) != '"' {
        if lx.i >= lx.src.len || peek(lx, 0) == '\n' {
            fail(lx, t.line, t.col, "string is never closed (missing \")")
        }
        if peek(lx, 0) == '\\' && peek(lx, 1) == '(' {
            advance(lx)
            advance(lx)
            skipEmbedded(lx, t)
            continue
        }
        if peek(lx, 0) == '\\' {
            advance(lx)                 // the escaped character can't end the string
        }
        advance(lx)
    }
    raw := lx.src.sub(start, lx.i)
    advance(lx)
    return raw
}

// past the code of a \( ... ), up to and including its )
fn skipEmbedded(ptr[Lexer] lx, Token t) {
    depth := 1
    while depth > 0 {
        c := peek(lx, 0)
        if lx.i >= lx.src.len || c == '\n' {
            fail(lx, t.line, t.col, "a \\( in this text is never closed (missing ')')")
        }
        if c == '(' {
            depth += 1
        } else if c == ')' {
            depth -= 1
        } else if c == '"' || c == '\'' {
            advance(lx)
            while peek(lx, 0) != c {
                if lx.i >= lx.src.len || peek(lx, 0) == '\n' {
                    fail(lx, t.line, t.col, "a text inside \\( ... ) is never closed")
                }
                if peek(lx, 0) == '\\' {
                    advance(lx)
                }
                advance(lx)
            }
        }
        advance(lx)
    }
}

fn punct(str k, Token at) -> Token {
    return Token(kind: k, text: k, line: at.line, col: at.col, nl: false)
}

// The tokens of a text: one string, or for "x is \(x)" the tokens of ("x is " + str(x) + "").
// \(v, 2) is str(v, 2): a number with two digits after the point.
fn cook(ptr[Lexer] lx, str raw, Token t) -> []Token {
    []Token toks
    []str parts
    pieces := 0
    i := 0
    while i < raw.len {
        ch := raw[i]
        if ch == '\\' && i + 1 < raw.len && raw[i + 1] == '(' {
            j := i + 2
            depth := 1
            while depth > 0 {
                c := raw[j]
                if c == '(' {
                    depth += 1
                } else if c == ')' {
                    depth -= 1
                } else if c == '"' || c == '\'' {
                    j += 1
                    while raw[j] != c {
                        if raw[j] == '\\' {
                            j += 1
                        }
                        j += 1
                    }
                }
                j += 1
            }
            code := raw.sub(i + 2, j - 1)
            if code.trim().len == 0 {
                fail(lx, t.line, t.col, "\\( ) needs a value inside, like \\(x)")
            }
            if pieces == 0 {
                open := punct("(", t)
                open.nl = t.nl
                toks.push(open)
            }
            toks.push(Token(kind: "string", text: parts.join(""), line: t.line, col: t.col, nl: false))
            []str none
            parts = none
            toks.push(punct("+", t))
            toks.push(Token(kind: "ident", text: "str", line: t.line, col: t.col, nl: false))
            toks.push(punct("(", t))
            inner := lexFrom(code, lx.path, t.line, t.col + 3 + i)
            for k in 0..inner.len - 1 {
                tk := inner[k]
                tk.nl = false
                toks.push(tk)
            }
            toks.push(punct(")", t))
            toks.push(punct("+", t))
            pieces += 1
            i = j
            continue
        }
        if ch == '\\' {
            i += 1
            ch = escapeAt(lx, raw[i], t.line, t.col)
        }
        parts.push(str(ch))
        i += 1
    }
    last := Token(kind: "string", text: parts.join(""), line: t.line, col: t.col, nl: false)
    if pieces == 0 {
        last.nl = t.nl
        toks.push(last)
        return toks
    }
    toks.push(last)
    toks.push(punct(")", t))
    return toks
}

fn escapeAt(ptr[Lexer] lx, char e, int line, int col) -> char {
    if e == 'n' {
        return '\n'
    }
    if e == 't' {
        return '\t'
    }
    if e == 'r' {
        return '\r'
    }
    if e == '0' {
        return char(0)
    }
    if e == '\\' || e == '"' || e == '\'' {
        return e
    }
    fail(lx, line, col, "unknown escape \\" + str(e) + " in this text (\\n \\t \\r \\0 \\\\ \\\" \\' and \\(value))")
    return e
}

fn escape(ptr[Lexer] lx, char e) -> char {
    if e == 'n' {
        return '\n'
    }
    if e == 't' {
        return '\t'
    }
    if e == 'r' {
        return '\r'
    }
    if e == '0' {
        return char(0)
    }
    if e == '\\' || e == '"' || e == '\'' {
        return e
    }
    fail(lx, lx.line, lx.col - 1, "unknown escape \\" + str(e))
    return e
}

fn lex(str src, str path) -> []Token {
    return lexFrom(src, path, 1, 1)
}

// `src` starts at this line and column of the file (the code inside a "\(...)")
fn lexFrom(str src, str path, int firstLine, int firstCol) -> []Token {
    lexer := Lexer(src: src, path: path, i: 0, line: firstLine, col: firstCol, nl: true)
    lx := addr(lexer)
    []Token out
    while lx.i < lx.src.len {
        c := lx.src[lx.i]
        if c == ' ' || c == '\t' || c == '\n' || c == '\r' {
            advance(lx)
            continue
        }
        if c == '/' && peek(lx, 1) == '/' {
            while lx.i < lx.src.len && lx.src[lx.i] != '\n' {
                advance(lx)
            }
            continue
        }
        if c == '/' && peek(lx, 1) == '*' {
            line := lx.line
            col := lx.col
            advance(lx)
            advance(lx)
            while lx.i < lx.src.len && !(lx.src[lx.i] == '*' && peek(lx, 1) == '/') {
                advance(lx)
            }
            if lx.i >= lx.src.len {
                fail(lx, line, col, "comment is never closed (missing */)")
            }
            advance(lx)
            advance(lx)
            continue
        }

        t := Token(kind: "", text: "", line: lx.line, col: lx.col, nl: lx.nl)
        lx.nl = false
        start := lx.i
        if isAlpha(c) {
            while lx.i < lx.src.len && (isAlpha(lx.src[lx.i]) || isDigit(lx.src[lx.i])) {
                advance(lx)
            }
            t.text = lx.src.sub(start, lx.i)
            t.kind = "ident"
            if isKeyword(t.text) {
                t.kind = t.text
            }
        } else if isDigit(c) {
            t.kind = "int"
            mant := ""
            expStart := 0
            if c == '0' && (peek(lx, 1) == 'x' || peek(lx, 1) == 'X') {
                advance(lx)
                advance(lx)
                while lx.i < lx.src.len && (isHex(lx.src[lx.i]) || lx.src[lx.i] == '_') {
                    advance(lx)
                }
            } else {
                while lx.i < lx.src.len && (isDigit(lx.src[lx.i]) || lx.src[lx.i] == '_') {
                    advance(lx)
                }
                if peek(lx, 0) == '.' && isDigit(peek(lx, 1)) {
                    t.kind = "float"
                    advance(lx)
                    while lx.i < lx.src.len && (isDigit(lx.src[lx.i]) || lx.src[lx.i] == '_') {
                        advance(lx)
                    }
                }
                // an exponent: 1e9, 2.5e-3
                e := peek(lx, 0)
                sign := peek(lx, 1) == '+' || peek(lx, 1) == '-'
                if (e == 'e' || e == 'E') && (isDigit(peek(lx, 1)) || (sign && isDigit(peek(lx, 2)))) {
                    if t.kind == "int" {
                        t.kind = "float"
                        mant = lx.src.sub(start, lx.i) + ".0"
                    }
                    expStart = lx.i
                    advance(lx)
                    if sign {
                        advance(lx)
                    }
                    while lx.i < lx.src.len && isDigit(lx.src[lx.i]) {
                        advance(lx)
                    }
                }
            }
            t.text = lx.src.sub(start, lx.i).replace("_", "")
            if mant.len > 0 {
                t.text = (mant + lx.src.sub(expStart, lx.i)).replace("_", "")   // LLVM wants the '.'
            }
        } else if c == '"' && peek(lx, 1) == '"' && peek(lx, 2) == '"' {
            for tk in cook(lx, longString(lx, t), t) {
                out.push(tk)
            }
            continue
        } else if c == '"' {
            for tk in cook(lx, rawString(lx, t), t) {
                out.push(tk)
            }
            continue
        } else if c == '"' || c == '\'' {
            quote := c
            t.kind = "string"
            if quote == '\'' {
                t.kind = "char"
            }
            advance(lx)
            []str parts
            while peek(lx, 0) != quote {
                if peek(lx, 0) == char(0) || peek(lx, 0) == '\n' {
                    if quote == '"' {
                        fail(lx, t.line, t.col, "string is never closed (missing \")")
                    }
                    fail(lx, t.line, t.col, "char is never closed (missing ')")
                }
                ch := peek(lx, 0)
                advance(lx)
                if ch == '\\' {
                    e := peek(lx, 0)
                    advance(lx)
                    ch = escape(lx, e)
                }
                parts.push(str(ch))
            }
            advance(lx)
            t.text = parts.join("")
            if quote == '\'' && t.text.len != 1 {
                fail(lx, t.line, t.col, "a char holds exactly one character, like 'a'")
            }
        } else {
            op := operator(lx)
            if op.len == 0 {
                // show the whole UTF-8 character, not just its first byte
                n := 1
                if (int(c) & 0xE0) == 0xC0 {
                    n = 2
                } else if (int(c) & 0xF0) == 0xE0 {
                    n = 3
                } else if (int(c) & 0xF8) == 0xF0 {
                    n = 4
                }
                hint := ""
                if int(c) >= 0x80 {
                    hint = " (names use only a-z, A-Z, 0-9 and _; other letters only go inside \"text\")"
                }
                fail(lx, t.line, t.col, "unexpected character '" + lx.src.sub(lx.i, lx.i + n) + "'" + hint)
            }
            for k in 0..op.len {
                advance(lx)
            }
            t.kind = op
            t.text = op
        }
        out.push(t)
    }
    out.push(Token(kind: "eof", text: "", line: lx.line, col: lx.col, nl: true))
    return out
}